mirror of
https://git.kernel.org/pub/scm/linux/kernel/git/stable/linux.git
synced 2026-08-09 06:14:34 +02:00
JP Kobryn (Meta)andAndrew Morton
f2a950170f
When reclaim is triggered by high order allocations on a fragmented system, vmpressure() can report poor reclaim efficiency even though the system has plenty of free memory. This is because many pages are scanned, but few are found to actually reclaim - the pages are actively in use and don't need to be freed. The resulting scan:reclaim ratio causes vmpressure() to assert socket pressure, throttling TCP throughput unnecessarily. Costly order allocations (above PAGE_ALLOC_COSTLY_ORDER) rely heavily on compaction to succeed, so poor reclaim efficiency at these orders does not necessarily indicate memory pressure. The kernel already treats this order as the boundary where reclaim is no longer expected to succeed and compaction may take over. Make vmpressure() order-aware through an additional parameter sourced from scan_control at existing call sites. Socket pressure is now only asserted when order <= PAGE_ALLOC_COSTLY_ORDER. Memcg reclaim is unaffected since try_to_free_mem_cgroup_pages() always uses order 0, which passes the filter unconditionally. Similarly, vmpressure_prio() now passes order 0 internally when calling vmpressure(), ensuring critical pressure from low reclaim priority is not suppressed by the order filter. The patch was motivated by a case of impacted net throughput in production. On one affected host, the memory state at the time showed ~15GB available, zero cgroup pressure, and the following buddyinfo state: Order FreePages 0: 133,970 1: 29,230 2: 17,351 3: 18,984 7+: 0 Using bpf, it was found that 94% of vmpressure calls on this host were from order-7 kswapd reclaim. TCP minimum recv window is rcv_ssthresh:19712. Before patch: 723 out of 3,843 (19%) TCP connections stuck at minimum recv window After live-patching and ~30min elapsed: 0 out of 3,470 TCP connections stuck at minimum recv window Link: https://lore.kernel.org/20260406195014.112521-1-jp.kobryn@linux.dev Signed-off-by: JP Kobryn (Meta) <jp.kobryn@linux.dev> Reviewed-by: Rik van Riel <riel@surriel.com> Acked-by: Johannes Weiner <hannes@cmpxchg.org> Acked-by: Shakeel Butt <shakeel.butt@linux.dev> Acked-by: Jakub Kicinski <kuba@kernel.org> Reviewed-by: Barry Song <baohua@kernel.org> Acked-by: Vlastimil Babka (SUSE) <vbabka@kernel.org> Cc: Axel Rasmussen <axelrasmussen@google.com> Cc: David Hildenbrand <david@kernel.org> Cc: Eric Dumazet <edumazet@google.com> Cc: Kairui Song <kasong@tencent.com> Cc: Liam Howlett <liam@infradead.org> Cc: Lorenzo Stoakes <ljs@kernel.org> Cc: Matthew Wilcox (Oracle) <willy@infradead.org> Cc: Michal Hocko <mhocko@suse.com> Cc: Mike Rapoport <rppt@kernel.org> Cc: Qi Zheng <qi.zheng@linux.dev> Cc: Suren Baghdasaryan <surenb@google.com> Cc: Wei Xu <weixugc@google.com> Cc: Yuanchu Xie <yuanchu@google.com> Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
54 lines
1.7 KiB
C
54 lines
1.7 KiB
C
/* SPDX-License-Identifier: GPL-2.0 */
|
|
#ifndef __LINUX_VMPRESSURE_H
|
|
#define __LINUX_VMPRESSURE_H
|
|
|
|
#include <linux/mutex.h>
|
|
#include <linux/list.h>
|
|
#include <linux/workqueue.h>
|
|
#include <linux/gfp.h>
|
|
#include <linux/types.h>
|
|
#include <linux/cgroup.h>
|
|
#include <linux/eventfd.h>
|
|
|
|
struct vmpressure {
|
|
unsigned long scanned;
|
|
unsigned long reclaimed;
|
|
|
|
unsigned long tree_scanned;
|
|
unsigned long tree_reclaimed;
|
|
/* The lock is used to keep the scanned/reclaimed above in sync. */
|
|
spinlock_t sr_lock;
|
|
|
|
/* The list of vmpressure_event structs. */
|
|
struct list_head events;
|
|
/* Have to grab the lock on events traversal or modifications. */
|
|
struct mutex events_lock;
|
|
|
|
struct work_struct work;
|
|
};
|
|
|
|
struct mem_cgroup;
|
|
|
|
#ifdef CONFIG_MEMCG
|
|
void vmpressure(gfp_t gfp, int order, struct mem_cgroup *memcg, bool tree,
|
|
unsigned long scanned, unsigned long reclaimed);
|
|
extern void vmpressure_prio(gfp_t gfp, struct mem_cgroup *memcg, int prio);
|
|
|
|
extern void vmpressure_init(struct vmpressure *vmpr);
|
|
extern void vmpressure_cleanup(struct vmpressure *vmpr);
|
|
extern struct vmpressure *memcg_to_vmpressure(struct mem_cgroup *memcg);
|
|
extern struct mem_cgroup *vmpressure_to_memcg(struct vmpressure *vmpr);
|
|
extern int vmpressure_register_event(struct mem_cgroup *memcg,
|
|
struct eventfd_ctx *eventfd,
|
|
const char *args);
|
|
extern void vmpressure_unregister_event(struct mem_cgroup *memcg,
|
|
struct eventfd_ctx *eventfd);
|
|
#else
|
|
static inline void vmpressure(gfp_t gfp, int order, struct mem_cgroup *memcg,
|
|
bool tree, unsigned long scanned,
|
|
unsigned long reclaimed) {}
|
|
static inline void vmpressure_prio(gfp_t gfp, struct mem_cgroup *memcg,
|
|
int prio) {}
|
|
#endif /* CONFIG_MEMCG */
|
|
#endif /* __LINUX_VMPRESSURE_H */
|