averello/linux-stable-mirror
1
0
Fork 0
mirror of https://git.kernel.org/pub/scm/linux/kernel/git/stable/linux.git synced 2026-09-22 09:34:56 +02:00
Code Issues Packages Projects Releases Wiki Activity

bpf: Skip redundant IBPB in pack allocator

Browse Source 
commit a23c1c5396 upstream.

bpf_prog_pack_alloc() issues IBPB on all CPUs on every cBPF allocation,
even when reusing chunks from an existing pack where no new memory was
touched since the last IBPB.

Since IBPB on all CPUs is heavy, Dave Hansen suggested to track allocation
since last IBPB, and only issue IBPB at reuse for the chunks that have not
seen an IBPB since they were last freed.

Track per-pack whether an IBPB is needed via arch_flush_needed. Set it when
allocating a chunk, reset on IBPB flush. On reuse, conditionally issue the
flush. Since IBPB invalidates all BTB entries, clear the flag on all packs
after flushing.

Signed-off-by: Pawan Gupta <pawan.kumar.gupta@linux.intel.com>
Acked-by: Daniel Borkmann <daniel@iogearbox.net>
Signed-off-by: Daniel Borkmann <daniel@iogearbox.net>
Signed-off-by: Sasha Levin <sashal@kernel.org>
This commit is contained in:
Pawan Gupta
2026-07-18 16:53:26 +02:00
committed by Greg Kroah-Hartman
parent 666fc2e6e4
commit f1f36bf9bb
1 changed files with 14 additions and 1 deletions
Download Patch File Download Diff File Expand all files Collapse all files
kernel/bpf/core.c
+14 -1
View File
@@ -857,6 +857,7 @@ int bpf_jit_add_poke_descriptor(struct bpf_prog *prog,
struct bpf_prog_pack {
struct list_head list;
void *ptr;
bool arch_flush_needed;
unsigned long bitmap[];
};
@@ -910,6 +911,8 @@ static struct bpf_prog_pack *alloc_new_pack(bpf_jit_fill_hole_t bpf_fill_ill_ins
bpf_fill_ill_insns(pack->ptr, BPF_PROG_PACK_SIZE);
bitmap_zero(pack->bitmap, BPF_PROG_PACK_SIZE / BPF_PROG_CHUNK_SIZE);
if (static_branch_unlikely(&bpf_pred_flush_enabled))
pack->arch_flush_needed = true;
set_vm_flush_reset_perms(pack->ptr);
err = set_memory_rox((unsigned long)pack->ptr,
BPF_PROG_PACK_SIZE / PAGE_SIZE);
@@ -972,8 +975,15 @@ void *bpf_prog_pack_alloc(u32 size, bpf_jit_fill_hole_t bpf_fill_ill_insns, bool
found_free_area:
/* Flush only for cBPF as it may contain a crafted gadget */
if (static_branch_unlikely(&bpf_pred_flush_enabled) && was_classic)
if (static_branch_unlikely(&bpf_pred_flush_enabled) &&
pack->arch_flush_needed &&
was_classic) {
struct bpf_prog_pack *p;
static_call_cond(bpf_arch_pred_flush)();
list_for_each_entry(p, &pack_list, list)
p->arch_flush_needed = false;
}
bitmap_set(pack->bitmap, pos, nbits);
ptr = (void *)(pack->ptr) + (pos << BPF_PROG_CHUNK_SHIFT);
@@ -1011,6 +1021,9 @@ void bpf_prog_pack_free(void *ptr, u32 size)
"bpf_prog_pack bug: missing bpf_arch_text_invalidate?\n");
bitmap_clear(pack->bitmap, pos, nbits);
if (static_branch_unlikely(&bpf_pred_flush_enabled))
pack->arch_flush_needed = true;
if (bitmap_find_next_zero_area(pack->bitmap, BPF_PROG_CHUNK_COUNT, 0,
BPF_PROG_CHUNK_COUNT, 0) == 0) {
list_del(&pack->list);