mirror of
https://git.kernel.org/pub/scm/linux/kernel/git/stable/linux.git
synced 2026-07-25 16:19:48 +02:00
Mel GormanandGreg Kroah-Hartman
e3a2d3f6c4
[ Upstream commitf169c62ff7] VMAs are skipped if there is no recent fault activity but this represents a chicken-and-egg problem as there may be no fault activity if the PTEs are never updated to trap NUMA hints. There is an indirect reliance on scanning to be forced early in the lifetime of a task but this may fail to detect changes in phase behaviour. Force inactive VMAs to be scanned when all other eligible VMAs have been updated within the same scan sequence. Test results in general look good with some changes in performance, both negative and positive, depending on whether the additional scanning and faulting was beneficial or not to the workload. The autonuma benchmark workload NUMA01_THREADLOCAL was picked for closer examination. The workload creates two processes with numerous threads and thread-local storage that is zero-filled in a loop. It exercises the corner case where unrelated threads may skip VMAs that are thread-local to another thread and still has some VMAs that inactive while the workload executes. The VMA skipping activity frequency with and without the patch: 6.6.0-rc2-sched-numabtrace-v1 ============================= 649 reason=scan_delay 9,094 reason=unsuitable 48,915 reason=shared_ro 143,919 reason=inaccessible 193,050 reason=pid_inactive 6.6.0-rc2-sched-numabselective-v1 ============================= 146 reason=seq_completed 622 reason=ignore_pid_inactive 624 reason=scan_delay 6,570 reason=unsuitable 16,101 reason=shared_ro 27,608 reason=inaccessible 41,939 reason=pid_inactive Note that with the patch applied, the PID activity is ignored (ignore_pid_inactive) to ensure a VMA with some activity is completely scanned. In addition, a small number of VMAs are scanned when no other eligible VMA is available during a single scan window (seq_completed). The number of times a VMA is skipped due to no PID activity from the scanning task (pid_inactive) drops dramatically. It is expected that this will increase the number of PTEs updated for NUMA hinting faults as well as hinting faults but these represent PTEs that would otherwise have been missed. The tradeoff is scan+fault overhead versus improving locality due to migration. On a 2-socket Cascade Lake test machine, the time to complete the workload is as follows; 6.6.0-rc2 6.6.0-rc2 sched-numabtrace-v1 sched-numabselective-v1 Min elsp-NUMA01_THREADLOCAL 174.22 ( 0.00%) 117.64 ( 32.48%) Amean elsp-NUMA01_THREADLOCAL 175.68 ( 0.00%) 123.34 * 29.79%* Stddev elsp-NUMA01_THREADLOCAL 1.20 ( 0.00%) 4.06 (-238.20%) CoeffVar elsp-NUMA01_THREADLOCAL 0.68 ( 0.00%) 3.29 (-381.70%) Max elsp-NUMA01_THREADLOCAL 177.18 ( 0.00%) 128.03 ( 27.74%) The time to complete the workload is reduced by almost 30%: 6.6.0-rc2 6.6.0-rc2 sched-numabtrace-v1 sched-numabselective-v1 / Duration User 91201.80 63506.64 Duration System 2015.53 1819.78 Duration Elapsed 1234.77 868.37 In this specific case, system CPU time was not increased but it's not universally true. From vmstat, the NUMA scanning and fault activity is as follows; 6.6.0-rc2 6.6.0-rc2 sched-numabtrace-v1 sched-numabselective-v1 Ops NUMA base-page range updates 64272.00 26374386.00 Ops NUMA PTE updates 36624.00 55538.00 Ops NUMA PMD updates 54.00 51404.00 Ops NUMA hint faults 15504.00 75786.00 Ops NUMA hint local faults % 14860.00 56763.00 Ops NUMA hint local percent 95.85 74.90 Ops NUMA pages migrated 1629.00 6469222.00 Both the number of PTE updates and hint faults is dramatically increased. While this is superficially unfortunate, it represents ranges that were simply skipped without the patch. As a result of the scanning and hinting faults, many more pages were also migrated but as the time to completion is reduced, the overhead is offset by the gain. Signed-off-by: Mel Gorman <mgorman@techsingularity.net> Signed-off-by: Ingo Molnar <mingo@kernel.org> Tested-by: Raghavendra K T <raghavendra.kt@amd.com> Link: https://lore.kernel.org/r/20231010083143.19593-7-mgorman@techsingularity.net Stable-dep-of:f22cde4371("sched/numa: Fix the vma scan starving issue") Signed-off-by: Sasha Levin <sashal@kernel.org>
58 lines
1.5 KiB
C
58 lines
1.5 KiB
C
/* SPDX-License-Identifier: GPL-2.0 */
|
|
#ifndef _LINUX_SCHED_NUMA_BALANCING_H
|
|
#define _LINUX_SCHED_NUMA_BALANCING_H
|
|
|
|
/*
|
|
* This is the interface between the scheduler and the MM that
|
|
* implements memory access pattern based NUMA-balancing:
|
|
*/
|
|
|
|
#include <linux/sched.h>
|
|
|
|
#define TNF_MIGRATED 0x01
|
|
#define TNF_NO_GROUP 0x02
|
|
#define TNF_SHARED 0x04
|
|
#define TNF_FAULT_LOCAL 0x08
|
|
#define TNF_MIGRATE_FAIL 0x10
|
|
|
|
enum numa_vmaskip_reason {
|
|
NUMAB_SKIP_UNSUITABLE,
|
|
NUMAB_SKIP_SHARED_RO,
|
|
NUMAB_SKIP_INACCESSIBLE,
|
|
NUMAB_SKIP_SCAN_DELAY,
|
|
NUMAB_SKIP_PID_INACTIVE,
|
|
NUMAB_SKIP_IGNORE_PID,
|
|
NUMAB_SKIP_SEQ_COMPLETED,
|
|
};
|
|
|
|
#ifdef CONFIG_NUMA_BALANCING
|
|
extern void task_numa_fault(int last_node, int node, int pages, int flags);
|
|
extern pid_t task_numa_group_id(struct task_struct *p);
|
|
extern void set_numabalancing_state(bool enabled);
|
|
extern void task_numa_free(struct task_struct *p, bool final);
|
|
extern bool should_numa_migrate_memory(struct task_struct *p, struct page *page,
|
|
int src_nid, int dst_cpu);
|
|
#else
|
|
static inline void task_numa_fault(int last_node, int node, int pages,
|
|
int flags)
|
|
{
|
|
}
|
|
static inline pid_t task_numa_group_id(struct task_struct *p)
|
|
{
|
|
return 0;
|
|
}
|
|
static inline void set_numabalancing_state(bool enabled)
|
|
{
|
|
}
|
|
static inline void task_numa_free(struct task_struct *p, bool final)
|
|
{
|
|
}
|
|
static inline bool should_numa_migrate_memory(struct task_struct *p,
|
|
struct page *page, int src_nid, int dst_cpu)
|
|
{
|
|
return true;
|
|
}
|
|
#endif
|
|
|
|
#endif /* _LINUX_SCHED_NUMA_BALANCING_H */
|