mirror of
https://git.kernel.org/pub/scm/linux/kernel/git/stable/linux.git
synced 2026-08-09 06:14:34 +02:00
Mykyta YatsenkoandAlexei Starovoitov
89edbdfc5d
NMI and tracepoint BPF programs can re-enter the per-CPU or global
LRU lock that bpf_lru_pop_free()/push_free() already hold on the
same CPU, AA-deadlocking. Lockdep reports "inconsistent
{INITIAL USE} -> {IN-NMI}" on &l->lock (syzbot c69a0a2c816716f1e0d5)
and "possible recursive locking detected" on &loc_l->lock (syzbot
18b26edb69b2e19f3b33).
Prior trylock and rqspinlock based fixes (see links) were nacked
because compromised on reliability.
This patch converts every LRU lock site to rqspinlock_t and adds a
recovery path for some failure windows to avoid node leaks.
Failure recovery:
- *_pop_free top-level: return NULL; prealloc_lru_pop() already
treats that as no-free-element (-ENOMEM).
- Cross-CPU steal: skip the victim's locked loc_l, try next CPU.
- Post-steal local lock fail: publish stolen node to lockless
per-CPU free_llist; next pop on this CPU picks it up.
- push_free fail: mark node pending_free=1. __local_list_flush(),
__local_list_pop_pending() reclaim the node from pending_list.
__bpf_lru_list_shrink_inactive() reclaims the node from inactive
list. Nodes from active list are reclaimed by __bpf_lru_list_shrink()
or after __bpf_lru_list_rotate_active() demotes it to the inactive.
Fixes: 3a08c2fd76 ("bpf: LRU List")
Reported-by: syzbot+c69a0a2c816716f1e0d5@syzkaller.appspotmail.com
Closes: https://syzkaller.appspot.com/bug?extid=c69a0a2c816716f1e0d5
Reported-by: syzbot+18b26edb69b2e19f3b33@syzkaller.appspotmail.com
Closes: https://syzkaller.appspot.com/bug?extid=18b26edb69b2e19f3b33
Link: https://lore.kernel.org/bpf/CAPPBnEYO4R+m+SpVc2gNj_x31R6fo1uJvj2bK2YS1P09GWT6kQ@mail.gmail.com/
Link: https://lore.kernel.org/bpf/CAPPBnEZmFA3ab8Uc=PEm0bdojZy=7T_F5_+eyZSHyZR3MBG4Vw@mail.gmail.com/
Link: https://lore.kernel.org/bpf/20251030030010.95352-1-dongml2@chinatelecom.cn/
Link: https://lore.kernel.org/bpf/20260119142120.28170-1-leon.hwang@linux.dev/
Signed-off-by: Mykyta Yatsenko <yatsenko@meta.com>
Link: https://lore.kernel.org/r/20260607-lru_map_spin-v3-1-bcd9332e911b@meta.com
Signed-off-by: Alexei Starovoitov <ast@kernel.org>
94 lines
2.3 KiB
C
94 lines
2.3 KiB
C
/* SPDX-License-Identifier: GPL-2.0-only */
|
|
/* Copyright (c) 2016 Facebook
|
|
*/
|
|
#ifndef __BPF_LRU_LIST_H_
|
|
#define __BPF_LRU_LIST_H_
|
|
|
|
#include <linux/cache.h>
|
|
#include <linux/list.h>
|
|
#include <linux/llist.h>
|
|
#include <asm/rqspinlock.h>
|
|
|
|
#define NR_BPF_LRU_LIST_T (3)
|
|
#define NR_BPF_LRU_LIST_COUNT (2)
|
|
#define BPF_LOCAL_LIST_T_OFFSET NR_BPF_LRU_LIST_T
|
|
|
|
enum bpf_lru_list_type {
|
|
BPF_LRU_LIST_T_ACTIVE,
|
|
BPF_LRU_LIST_T_INACTIVE,
|
|
BPF_LRU_LIST_T_FREE,
|
|
BPF_LRU_LOCAL_LIST_T_FREE,
|
|
BPF_LRU_LOCAL_LIST_T_PENDING,
|
|
};
|
|
|
|
struct bpf_lru_node {
|
|
/*
|
|
* A node is in at most one list at a time. The free path on the
|
|
* per-CPU locallist uses an llist, so share storage via a union.
|
|
*/
|
|
union {
|
|
struct list_head list;
|
|
struct llist_node llist;
|
|
};
|
|
u16 cpu;
|
|
u8 type;
|
|
u8 ref;
|
|
/*
|
|
* Marks nodes whose *_push_free() lock acquire failed; reclaimed
|
|
* by flush/shrink which honor the flag instead of del_from_htab().
|
|
*/
|
|
u8 pending_free;
|
|
};
|
|
|
|
struct bpf_lru_list {
|
|
struct list_head lists[NR_BPF_LRU_LIST_T];
|
|
unsigned int counts[NR_BPF_LRU_LIST_COUNT];
|
|
/* The next inactive list rotation starts from here */
|
|
struct list_head *next_inactive_rotation;
|
|
|
|
rqspinlock_t lock ____cacheline_aligned_in_smp;
|
|
};
|
|
|
|
struct bpf_lru_locallist {
|
|
struct list_head pending_list;
|
|
struct llist_head free_llist;
|
|
u16 next_steal;
|
|
rqspinlock_t lock;
|
|
};
|
|
|
|
struct bpf_common_lru {
|
|
struct bpf_lru_list lru_list;
|
|
struct bpf_lru_locallist __percpu *local_list;
|
|
};
|
|
|
|
typedef bool (*del_from_htab_func)(void *arg, struct bpf_lru_node *node);
|
|
|
|
struct bpf_lru {
|
|
union {
|
|
struct bpf_common_lru common_lru;
|
|
struct bpf_lru_list __percpu *percpu_lru;
|
|
};
|
|
del_from_htab_func del_from_htab;
|
|
void *del_arg;
|
|
unsigned int hash_offset;
|
|
unsigned int target_free;
|
|
unsigned int nr_scans;
|
|
bool percpu;
|
|
};
|
|
|
|
static inline void bpf_lru_node_set_ref(struct bpf_lru_node *node)
|
|
{
|
|
if (!READ_ONCE(node->ref))
|
|
WRITE_ONCE(node->ref, 1);
|
|
}
|
|
|
|
int bpf_lru_init(struct bpf_lru *lru, bool percpu, u32 hash_offset,
|
|
del_from_htab_func del_from_htab, void *delete_arg);
|
|
void bpf_lru_populate(struct bpf_lru *lru, void *buf, u32 node_offset,
|
|
u32 elem_size, u32 nr_elems);
|
|
void bpf_lru_destroy(struct bpf_lru *lru);
|
|
struct bpf_lru_node *bpf_lru_pop_free(struct bpf_lru *lru, u32 hash);
|
|
void bpf_lru_push_free(struct bpf_lru *lru, struct bpf_lru_node *node);
|
|
|
|
#endif
|