mirror of
https://git.kernel.org/pub/scm/linux/kernel/git/stable/linux.git
synced 2026-08-09 06:14:34 +02:00
Tiezhu YangandHuacai Chen
7b5944d6ed
Like other architectures such as x86, arm64, riscv, powerpc and s390, select THREAD_INFO_IN_TASK for LoongArch to move thread_info off the stack into task_struct. This follows modern kernel standards and also makes the system more secure. With this patch, thread_info is included in task_struct at an offset of 0 instead of being placed at the bottom of the kernel stack. Thus, the $tp register points to both thread_info and task_struct. To support this, introduce a per-CPU variable cpu_tasks to store the pointer to the current task_struct. This decouples the recovery of the $tp register from the stack pointer during exception entry. Then initialize cpu_tasks for the primary and secondary CPUs during arch-specific setup and SMP boot paths. To eliminate the dangerous windows during the early initialization where the cpu_tasks remains uninitialized, set_current() is invoked as early as possible in both setup_arch() and start_secondary(). This ensures the $tp recovery barrier is armed in case any early boot exceptions or kernel panics occur. Modify SAVE_SOME and handle_syscall to restore the $tp register from cpu_tasks, and also use the la_abs absolute addressing for cpu_tasks access in assembly to bypass the relocation limits within exception handling sections. By advancing the preservation of u0 in SAVE_SOME, we reuse the PERCPU_BASE_KS value in u0 for the cpu_tasks calculation, effectively eliminating a duplicate csrrd instruction execution on SMP platforms. Update <asm/switch_to.h> and <kernel/switch.S> to fully support the CONFIG_THREAD_INFO_IN_TASK feature. Remove the obsolete next_ti argument from __switch_to(), which shifts the remaining arguments ahead in the calling convention (sched_ra from a3 to a2, and sched_cfa from a4 to a3). Under the new configuration, __switch_to() now directly derives the thread pointer ($tp) from the next task_struct pointer in a1. To preserve the optimal and clean "move tp, a1" path for 64-bit kernels, the thread pointer ($tp) is assigned directly from a1 in the core path. For 32-bit kernels, where a1 carries a 2000-byte structural pointer bias at entry, an explicit adjustment "PTR_ADDI tp, tp, -TASK_STRUCT_OFFSET" is introduced at the function exit. In the context of __switch_to(), local interrupts are disabled, and the kernel is in a critical switching phase where handling any synchronous exception is practically impossible and prohibited. If any synchronous exception or watchpoint does trigger in this narrow window, it constitutes a fatal double fault and the kernel is expected to die/panic immediately anyway. Therefore, the temporary biased value in $tp is safe and acceptable here. Additionally, evaluate the stack lookup as a single load instruction "LONG_LPTR t0, a1, (TASK_STACK - TASK_STRUCT_OFFSET)", this perfectly satisfies both 32-bit and 64-bit kernels. Using the "next" pointer in a1 as the base register, rather than $tp, effectively unchains the data dependency (RAW hazard) from the preceding move instruction, maximizing the instruction-level parallelism and superscalar execution efficiency while naturally adapting the structural shift. With CONFIG_THREAD_INFO_IN_TASK enabled, the kernel stack life cycle is decoupled from task_struct and can be freed concurrently. Currently, show_stacktrace() reads raw stack data via __get_addr() and subsequently calls show_backtrace() to unwind the frame, without holding any reference to the target task's stack. If show_stacktrace() is called on a concurrently exiting task, it could attempt to read from a freed or reallocated kernel stack. This introduces a severe use-after-free (UAF) read risk or kernel panics. Wrap the entire stack inspection process inside show_stacktrace() with a try_get_task_stack() and put_task_stack() pair. This ensures the task stack remains pinned safely during both the raw stack data dump loop and the subsequent stack unwinding phase. Also, ensure that the task pointer is initialized to "current" early if it is NULL, so that try_get_task_stack() always operates on a valid task reference. Signed-off-by: Tiezhu Yang <yangtiezhu@loongson.cn> Signed-off-by: Huacai Chen <chenhuacai@loongson.cn>
184 lines
5.0 KiB
C
184 lines
5.0 KiB
C
/* SPDX-License-Identifier: GPL-2.0 */
|
|
/*
|
|
* Copyright (C) 2020-2022 Loongson Technology Corporation Limited
|
|
*/
|
|
#ifndef __ASM_PERCPU_H
|
|
#define __ASM_PERCPU_H
|
|
|
|
#include <asm/loongarch.h>
|
|
|
|
/*
|
|
* The "address" (in fact, offset from $r21) of a per-CPU variable is close to
|
|
* the loading address of main kernel image, but far from where the modules are
|
|
* loaded. Tell the compiler this fact when using explicit relocs.
|
|
*/
|
|
#if defined(MODULE) && defined(CONFIG_AS_HAS_EXPLICIT_RELOCS) && defined(CONFIG_64BIT)
|
|
# if __has_attribute(model)
|
|
# define PER_CPU_ATTRIBUTES __attribute__((model("extreme")))
|
|
# else
|
|
# error compiler support for the model attribute is necessary when a recent assembler is used
|
|
# endif
|
|
#endif
|
|
|
|
/* Use r21 for fast access */
|
|
register unsigned long __my_cpu_offset __asm__("$r21");
|
|
|
|
static inline void set_my_cpu_offset(unsigned long off)
|
|
{
|
|
__my_cpu_offset = off;
|
|
csr_write(off, PERCPU_BASE_KS);
|
|
}
|
|
|
|
#define __my_cpu_offset \
|
|
({ \
|
|
__asm__ __volatile__("":"+r"(__my_cpu_offset)); \
|
|
__my_cpu_offset; \
|
|
})
|
|
|
|
#ifdef CONFIG_CPU_HAS_AMO
|
|
|
|
#define PERCPU_OP(op, asm_op, c_op) \
|
|
static __always_inline unsigned long __percpu_##op(void *ptr, \
|
|
unsigned long val, int size) \
|
|
{ \
|
|
unsigned long ret; \
|
|
\
|
|
switch (size) { \
|
|
case 4: \
|
|
__asm__ __volatile__( \
|
|
"am"#asm_op".w" " %[ret], %[val], %[ptr] \n" \
|
|
: [ret] "=&r" (ret), [ptr] "+ZB"(*(u32 *)ptr) \
|
|
: [val] "r" (val)); \
|
|
break; \
|
|
case 8: \
|
|
__asm__ __volatile__( \
|
|
"am"#asm_op".d" " %[ret], %[val], %[ptr] \n" \
|
|
: [ret] "=&r" (ret), [ptr] "+ZB"(*(u64 *)ptr) \
|
|
: [val] "r" (val)); \
|
|
break; \
|
|
default: \
|
|
ret = 0; \
|
|
BUILD_BUG(); \
|
|
} \
|
|
\
|
|
return ret c_op val; \
|
|
}
|
|
|
|
PERCPU_OP(add, add, +)
|
|
PERCPU_OP(and, and, &)
|
|
PERCPU_OP(or, or, |)
|
|
#undef PERCPU_OP
|
|
|
|
#endif
|
|
|
|
#ifdef CONFIG_64BIT
|
|
|
|
#define __pcpu_op_1(op) op ".b "
|
|
#define __pcpu_op_2(op) op ".h "
|
|
#define __pcpu_op_4(op) op ".w "
|
|
#define __pcpu_op_8(op) op ".d "
|
|
|
|
#define _percpu_read(size, _pcp) \
|
|
({ \
|
|
typeof(_pcp) __pcp_ret; \
|
|
\
|
|
__asm__ __volatile__( \
|
|
__pcpu_op_##size("ldx") "%[ret], $r21, %[ptr] \n" \
|
|
: [ret] "=&r"(__pcp_ret) \
|
|
: [ptr] "r"(&(_pcp)) \
|
|
: "memory"); \
|
|
\
|
|
__pcp_ret; \
|
|
})
|
|
|
|
#define _percpu_write(size, _pcp, _val) \
|
|
do { \
|
|
__asm__ __volatile__( \
|
|
__pcpu_op_##size("stx") "%[val], $r21, %[ptr] \n" \
|
|
: \
|
|
: [val] "r"(_val), [ptr] "r"(&(_pcp)) \
|
|
: "memory"); \
|
|
} while (0)
|
|
|
|
#endif
|
|
|
|
#define __percpu_xchg __arch_xchg
|
|
|
|
/* this_cpu_cmpxchg */
|
|
#define _protect_cmpxchg_local(pcp, o, n) \
|
|
({ \
|
|
typeof(*raw_cpu_ptr(&(pcp))) __ret; \
|
|
preempt_disable_notrace(); \
|
|
__ret = cmpxchg_local(raw_cpu_ptr(&(pcp)), o, n); \
|
|
preempt_enable_notrace(); \
|
|
__ret; \
|
|
})
|
|
|
|
#define _pcp_protect(operation, pcp, val) \
|
|
({ \
|
|
typeof(pcp) __retval; \
|
|
preempt_disable_notrace(); \
|
|
__retval = (typeof(pcp))operation(raw_cpu_ptr(&(pcp)), \
|
|
(val), sizeof(pcp)); \
|
|
preempt_enable_notrace(); \
|
|
__retval; \
|
|
})
|
|
|
|
#ifdef CONFIG_CPU_HAS_AMO
|
|
|
|
#define _percpu_add(pcp, val) \
|
|
_pcp_protect(__percpu_add, pcp, val)
|
|
|
|
#define _percpu_add_return(pcp, val) _percpu_add(pcp, val)
|
|
|
|
#define _percpu_and(pcp, val) \
|
|
_pcp_protect(__percpu_and, pcp, val)
|
|
|
|
#define _percpu_or(pcp, val) \
|
|
_pcp_protect(__percpu_or, pcp, val)
|
|
|
|
#define this_cpu_add_4(pcp, val) _percpu_add(pcp, val)
|
|
#define this_cpu_add_8(pcp, val) _percpu_add(pcp, val)
|
|
|
|
#define this_cpu_add_return_4(pcp, val) _percpu_add_return(pcp, val)
|
|
#define this_cpu_add_return_8(pcp, val) _percpu_add_return(pcp, val)
|
|
|
|
#define this_cpu_and_4(pcp, val) _percpu_and(pcp, val)
|
|
#define this_cpu_and_8(pcp, val) _percpu_and(pcp, val)
|
|
|
|
#define this_cpu_or_4(pcp, val) _percpu_or(pcp, val)
|
|
#define this_cpu_or_8(pcp, val) _percpu_or(pcp, val)
|
|
|
|
#endif
|
|
|
|
#ifdef CONFIG_64BIT
|
|
|
|
#define this_cpu_read_1(pcp) _percpu_read(1, pcp)
|
|
#define this_cpu_read_2(pcp) _percpu_read(2, pcp)
|
|
#define this_cpu_read_4(pcp) _percpu_read(4, pcp)
|
|
#define this_cpu_read_8(pcp) _percpu_read(8, pcp)
|
|
|
|
#define this_cpu_write_1(pcp, val) _percpu_write(1, pcp, val)
|
|
#define this_cpu_write_2(pcp, val) _percpu_write(2, pcp, val)
|
|
#define this_cpu_write_4(pcp, val) _percpu_write(4, pcp, val)
|
|
#define this_cpu_write_8(pcp, val) _percpu_write(8, pcp, val)
|
|
|
|
#endif
|
|
|
|
#define _percpu_xchg(pcp, val) ((typeof(pcp)) \
|
|
_pcp_protect(__percpu_xchg, pcp, (unsigned long)(val)))
|
|
|
|
#define this_cpu_xchg_1(pcp, val) _percpu_xchg(pcp, val)
|
|
#define this_cpu_xchg_2(pcp, val) _percpu_xchg(pcp, val)
|
|
#define this_cpu_xchg_4(pcp, val) _percpu_xchg(pcp, val)
|
|
#define this_cpu_xchg_8(pcp, val) _percpu_xchg(pcp, val)
|
|
|
|
#define this_cpu_cmpxchg_1(ptr, o, n) _protect_cmpxchg_local(ptr, o, n)
|
|
#define this_cpu_cmpxchg_2(ptr, o, n) _protect_cmpxchg_local(ptr, o, n)
|
|
#define this_cpu_cmpxchg_4(ptr, o, n) _protect_cmpxchg_local(ptr, o, n)
|
|
#define this_cpu_cmpxchg_8(ptr, o, n) _protect_cmpxchg_local(ptr, o, n)
|
|
|
|
#include <asm-generic/percpu.h>
|
|
|
|
#endif /* __ASM_PERCPU_H */
|