mirror of
https://git.kernel.org/pub/scm/linux/kernel/git/stable/linux.git
synced 2026-08-09 06:14:34 +02:00
With the intended removal of PREEMPT_NONE this_cpu operations based on
atomic instructions, guarded with preempt_disable()/preempt_enable() pairs
become more expensive: the preempt_disable() / preempt_enable() pairs are
not optimized away anymore during compile time.
In particular the conditional call to preempt_schedule_notrace() after
preempt_enable() adds additional code and register pressure.
E.g. this simple C code sequence
DEFINE_PER_CPU(long, foo);
long bar(long a) { return this_cpu_add_return(foo, a); }
generates this code:
11a976: eb af f0 68 00 24 stmg %r10,%r15,104(%r15)
11a97c: b9 04 00 ef lgr %r14,%r15
11a980: b9 04 00 b2 lgr %r11,%r2
11a984: e3 f0 ff c8 ff 71 lay %r15,-56(%r15)
11a98a: e3 e0 f0 98 00 24 stg %r14,152(%r15)
11a990: eb 01 03 a8 00 6a asi 936,1 <- __preempt_count_add(1)
11a996: c0 10 00 d2 ac b5 larl %r1,1b70300 <- address of percpu var
11a9a0: e3 10 23 b8 00 08 ag %r1,952 <- add percpu offset
11a9a6: eb ab 10 00 00 e8 laag %r10,%r11,0(%r1) <- atomic op
11a9ac: eb ff 03 a8 00 6e alsi 936,-1 <- __preempt_count_dec_and_test()
11a9b2: a7 54 00 05 jnhe 11a9bc <bar+0x4c>
11a9b6: c0 e5 00 76 d1 bd brasl %r14,ff4d30 <preempt_schedule_notrace>
11a9bc: b9 e8 b0 2a agrk %r2,%r10,%r11
11a9c0: eb af f0 a0 00 04 lmg %r10,%r15,160(%r15)
11a9c6 07 fe br %r14
Even though the above example is more or less the worst case, since the
branch to preempt_schedule_notrace() requires a stackframe, which
otherwise wouldn't be necessary, there is also the conditional jnhe branch
instruction.
Get rid of the conditional branch with the following code sequence:
11a8e6: c0 30 00 d0 c5 0d larl %r3,1b33300
11a8ec: b9 04 00 43 lgr %r4,%r3
11a8f0: eb 00 43 c0 00 52 mviy 960,4
11a8f6: e3 40 03 b8 00 08 ag %r4,952
11a8fc: eb 52 40 00 00 e8 laag %r5,%r2,0(%r4)
11a902: eb 00 03 c0 00 52 mviy 960,0
11a908: b9 08 00 25 agr %r2,%r5
11a90c 07 fe br %r14
The general idea is that this_cpu operations based on atomic instructions
are guarded with mviy instructions:
- The first mviy instruction writes the register number, which contains
the percpu address variable to lowcore. This also indicates that a
percpu code section is executed.
- The first instruction following the mviy instruction must be the ag
instruction which adds the percpu offset to the percpu address register.
- Afterwards the atomic percpu operation follows.
- Then a second mviy instruction writes a zero to lowcore, which indicates
the end of the percpu code section.
- In case of an interrupt/exception/nmi the register number which was
written to lowcore is copied to the exception frame (pt_regs), and a zero
is written to lowcore.
- On return to the previous context it is checked if a percpu code section
was executed (saved register number not zero), and if the process was
migrated to a different cpu. If the percpu offset was already added to
the percpu address register (instruction address does _not_ point to the
ag instruction) the content of the percpu address register is adjusted so
it points to percpu variable of the new cpu.
Reviewed-by: Alexander Gordeev <agordeev@linux.ibm.com>
Signed-off-by: Heiko Carstens <hca@linux.ibm.com>
Signed-off-by: Alexander Gordeev <agordeev@linux.ibm.com>
320 lines
8.8 KiB
C
320 lines
8.8 KiB
C
/* SPDX-License-Identifier: GPL-2.0 */
|
|
/*
|
|
* S390 version
|
|
* Copyright IBM Corp. 1999, 2000
|
|
* Author(s): Denis Joseph Barrow (djbarrow@de.ibm.com,barrow_dj@yahoo.com)
|
|
*/
|
|
#ifndef _S390_PTRACE_H
|
|
#define _S390_PTRACE_H
|
|
|
|
#include <linux/bits.h>
|
|
#include <linux/typecheck.h>
|
|
#include <uapi/asm/ptrace.h>
|
|
#include <asm/thread_info.h>
|
|
#include <asm/tpi.h>
|
|
|
|
#define PIF_SYSCALL 0 /* inside a system call */
|
|
#define PIF_PSW_ADDR_ADJUSTED 1 /* psw address has been adjusted */
|
|
#define PIF_SYSCALL_RET_SET 2 /* return value was set via ptrace */
|
|
#define PIF_GUEST_FAULT 3 /* indicates program check in sie64a */
|
|
#define PIF_FTRACE_FULL_REGS 4 /* all register contents valid (ftrace) */
|
|
|
|
#define _PIF_SYSCALL BIT(PIF_SYSCALL)
|
|
#define _PIF_ADDR_PSW_ADJUSTED BIT(PIF_PSW_ADDR_ADJUSTED)
|
|
#define _PIF_SYSCALL_RET_SET BIT(PIF_SYSCALL_RET_SET)
|
|
#define _PIF_GUEST_FAULT BIT(PIF_GUEST_FAULT)
|
|
#define _PIF_FTRACE_FULL_REGS BIT(PIF_FTRACE_FULL_REGS)
|
|
|
|
#define PSW32_MASK_PER _AC(0x40000000, UL)
|
|
#define PSW32_MASK_DAT _AC(0x04000000, UL)
|
|
#define PSW32_MASK_IO _AC(0x02000000, UL)
|
|
#define PSW32_MASK_EXT _AC(0x01000000, UL)
|
|
#define PSW32_MASK_KEY _AC(0x00F00000, UL)
|
|
#define PSW32_MASK_BASE _AC(0x00080000, UL) /* Always one */
|
|
#define PSW32_MASK_MCHECK _AC(0x00040000, UL)
|
|
#define PSW32_MASK_WAIT _AC(0x00020000, UL)
|
|
#define PSW32_MASK_PSTATE _AC(0x00010000, UL)
|
|
#define PSW32_MASK_ASC _AC(0x0000C000, UL)
|
|
#define PSW32_MASK_CC _AC(0x00003000, UL)
|
|
#define PSW32_MASK_PM _AC(0x00000f00, UL)
|
|
#define PSW32_MASK_RI _AC(0x00000080, UL)
|
|
|
|
#define PSW32_ADDR_AMODE _AC(0x80000000, UL)
|
|
#define PSW32_ADDR_INSN _AC(0x7FFFFFFF, UL)
|
|
|
|
#define PSW32_DEFAULT_KEY ((PAGE_DEFAULT_ACC) << 20)
|
|
|
|
#define PSW32_ASC_PRIMARY _AC(0x00000000, UL)
|
|
#define PSW32_ASC_ACCREG _AC(0x00004000, UL)
|
|
#define PSW32_ASC_SECONDARY _AC(0x00008000, UL)
|
|
#define PSW32_ASC_HOME _AC(0x0000C000, UL)
|
|
|
|
#define PSW_DEFAULT_KEY ((PAGE_DEFAULT_ACC) << 52)
|
|
|
|
#define PSW_KERNEL_BITS (PSW_DEFAULT_KEY | PSW_MASK_BASE | PSW_ASC_HOME | \
|
|
PSW_MASK_EA | PSW_MASK_BA | PSW_MASK_DAT)
|
|
#define PSW_USER_BITS (PSW_MASK_DAT | PSW_MASK_IO | PSW_MASK_EXT | \
|
|
PSW_DEFAULT_KEY | PSW_MASK_BASE | PSW_MASK_MCHECK | \
|
|
PSW_MASK_PSTATE | PSW_ASC_PRIMARY)
|
|
|
|
#ifndef __ASSEMBLER__
|
|
|
|
struct psw_bits {
|
|
unsigned long : 1;
|
|
unsigned long per : 1; /* PER-Mask */
|
|
unsigned long : 3;
|
|
unsigned long dat : 1; /* DAT Mode */
|
|
unsigned long io : 1; /* Input/Output Mask */
|
|
unsigned long ext : 1; /* External Mask */
|
|
unsigned long key : 4; /* PSW Key */
|
|
unsigned long : 1;
|
|
unsigned long mcheck : 1; /* Machine-Check Mask */
|
|
unsigned long wait : 1; /* Wait State */
|
|
unsigned long pstate : 1; /* Problem State */
|
|
unsigned long as : 2; /* Address Space Control */
|
|
unsigned long cc : 2; /* Condition Code */
|
|
unsigned long pm : 4; /* Program Mask */
|
|
unsigned long ri : 1; /* Runtime Instrumentation */
|
|
unsigned long : 6;
|
|
unsigned long eaba : 2; /* Addressing Mode */
|
|
unsigned long : 31;
|
|
unsigned long ia : 64; /* Instruction Address */
|
|
};
|
|
|
|
enum {
|
|
PSW_BITS_AMODE_24BIT = 0,
|
|
PSW_BITS_AMODE_31BIT = 1,
|
|
PSW_BITS_AMODE_64BIT = 3
|
|
};
|
|
|
|
enum {
|
|
PSW_BITS_AS_PRIMARY = 0,
|
|
PSW_BITS_AS_ACCREG = 1,
|
|
PSW_BITS_AS_SECONDARY = 2,
|
|
PSW_BITS_AS_HOME = 3
|
|
};
|
|
|
|
#define psw_bits(__psw) (*({ \
|
|
typecheck(psw_t, __psw); \
|
|
&(*(struct psw_bits *)(&(__psw))); \
|
|
}))
|
|
|
|
typedef struct {
|
|
unsigned int mask;
|
|
unsigned int addr;
|
|
} psw32_t __aligned(8);
|
|
|
|
#define PGM_INT_CODE_MASK 0x7f
|
|
#define PGM_INT_CODE_PER 0x80
|
|
|
|
/*
|
|
* The pt_regs struct defines the way the registers are stored on
|
|
* the stack during a system call.
|
|
*/
|
|
struct pt_regs {
|
|
union {
|
|
user_pt_regs user_regs;
|
|
struct {
|
|
unsigned long args[1];
|
|
psw_t psw;
|
|
unsigned long gprs[NUM_GPRS];
|
|
};
|
|
};
|
|
union {
|
|
unsigned long orig_gpr2;
|
|
unsigned long monitor_code;
|
|
};
|
|
union {
|
|
struct {
|
|
unsigned int int_code;
|
|
unsigned int int_parm;
|
|
unsigned long int_parm_long;
|
|
};
|
|
struct tpi_info tpi_info;
|
|
};
|
|
unsigned long flags;
|
|
unsigned long last_break;
|
|
unsigned int cpu;
|
|
unsigned char percpu_register;
|
|
};
|
|
|
|
/*
|
|
* Program event recording (PER) register set.
|
|
*/
|
|
struct per_regs {
|
|
unsigned long control; /* PER control bits */
|
|
unsigned long start; /* PER starting address */
|
|
unsigned long end; /* PER ending address */
|
|
};
|
|
|
|
/*
|
|
* PER event contains information about the cause of the last PER exception.
|
|
*/
|
|
struct per_event {
|
|
unsigned short cause; /* PER code, ATMID and AI */
|
|
unsigned long address; /* PER address */
|
|
unsigned char paid; /* PER access identification */
|
|
};
|
|
|
|
/*
|
|
* Simplified per_info structure used to decode the ptrace user space ABI.
|
|
*/
|
|
struct per_struct_kernel {
|
|
unsigned long cr9; /* PER control bits */
|
|
unsigned long cr10; /* PER starting address */
|
|
unsigned long cr11; /* PER ending address */
|
|
unsigned long bits; /* Obsolete software bits */
|
|
unsigned long starting_addr; /* User specified start address */
|
|
unsigned long ending_addr; /* User specified end address */
|
|
unsigned short perc_atmid; /* PER trap ATMID */
|
|
unsigned long address; /* PER trap instruction address */
|
|
unsigned char access_id; /* PER trap access identification */
|
|
};
|
|
|
|
#define PER_EVENT_MASK 0xEB000000UL
|
|
|
|
#define PER_EVENT_BRANCH 0x80000000UL
|
|
#define PER_EVENT_IFETCH 0x40000000UL
|
|
#define PER_EVENT_STORE 0x20000000UL
|
|
#define PER_EVENT_STORE_REAL 0x08000000UL
|
|
#define PER_EVENT_TRANSACTION_END 0x02000000UL
|
|
#define PER_EVENT_NULLIFICATION 0x01000000UL
|
|
|
|
#define PER_CONTROL_MASK 0x00e00000UL
|
|
|
|
#define PER_CONTROL_BRANCH_ADDRESS 0x00800000UL
|
|
#define PER_CONTROL_SUSPENSION 0x00400000UL
|
|
#define PER_CONTROL_ALTERATION 0x00200000UL
|
|
|
|
static inline void set_pt_regs_flag(struct pt_regs *regs, int flag)
|
|
{
|
|
regs->flags |= (1UL << flag);
|
|
}
|
|
|
|
static inline void clear_pt_regs_flag(struct pt_regs *regs, int flag)
|
|
{
|
|
regs->flags &= ~(1UL << flag);
|
|
}
|
|
|
|
static inline int test_pt_regs_flag(struct pt_regs *regs, int flag)
|
|
{
|
|
return !!(regs->flags & (1UL << flag));
|
|
}
|
|
|
|
static inline int test_and_clear_pt_regs_flag(struct pt_regs *regs, int flag)
|
|
{
|
|
int ret = test_pt_regs_flag(regs, flag);
|
|
|
|
clear_pt_regs_flag(regs, flag);
|
|
return ret;
|
|
}
|
|
|
|
struct task_struct;
|
|
|
|
void update_cr_regs(struct task_struct *task);
|
|
|
|
/*
|
|
* These are defined as per linux/ptrace.h, which see.
|
|
*/
|
|
#define arch_has_single_step() (1)
|
|
#define arch_has_block_step() (1)
|
|
|
|
#define profile_pc(regs) instruction_pointer(regs)
|
|
|
|
static __always_inline bool user_mode(const struct pt_regs *regs)
|
|
{
|
|
return psw_bits(regs->psw).pstate;
|
|
}
|
|
|
|
static inline long regs_return_value(const struct pt_regs *regs)
|
|
{
|
|
return regs->gprs[2];
|
|
}
|
|
|
|
static __always_inline unsigned long instruction_pointer(const struct pt_regs *regs)
|
|
{
|
|
return regs->psw.addr;
|
|
}
|
|
|
|
static inline void instruction_pointer_set(struct pt_regs *regs,
|
|
unsigned long val)
|
|
{
|
|
regs->psw.addr = val;
|
|
}
|
|
|
|
int regs_query_register_offset(const char *name);
|
|
const char *regs_query_register_name(unsigned int offset);
|
|
|
|
static __always_inline unsigned long kernel_stack_pointer(const struct pt_regs *regs)
|
|
{
|
|
return regs->gprs[15];
|
|
}
|
|
|
|
static __always_inline unsigned long user_stack_pointer(const struct pt_regs *regs)
|
|
{
|
|
return regs->gprs[15];
|
|
}
|
|
|
|
static __always_inline unsigned long regs_get_register(const struct pt_regs *regs,
|
|
unsigned int offset)
|
|
{
|
|
if (offset >= NUM_GPRS)
|
|
return 0;
|
|
return regs->gprs[offset];
|
|
}
|
|
|
|
static __always_inline int regs_within_kernel_stack(const struct pt_regs *regs,
|
|
unsigned long addr)
|
|
{
|
|
unsigned long ksp = kernel_stack_pointer(regs);
|
|
|
|
return (addr & ~(THREAD_SIZE - 1)) == (ksp & ~(THREAD_SIZE - 1));
|
|
}
|
|
|
|
/**
|
|
* regs_get_kernel_stack_nth() - get Nth entry of the stack
|
|
* @regs:pt_regs which contains kernel stack pointer.
|
|
* @n:stack entry number.
|
|
*
|
|
* regs_get_kernel_stack_nth() returns @n th entry of the kernel stack which
|
|
* is specifined by @regs. If the @n th entry is NOT in the kernel stack,
|
|
* this returns 0.
|
|
*/
|
|
static __always_inline unsigned long regs_get_kernel_stack_nth(const struct pt_regs *regs,
|
|
unsigned int n)
|
|
{
|
|
unsigned long addr;
|
|
|
|
addr = kernel_stack_pointer(regs) + n * sizeof(long);
|
|
if (!regs_within_kernel_stack(regs, addr))
|
|
return 0;
|
|
return READ_ONCE_NOCHECK(*(unsigned long *)addr);
|
|
}
|
|
|
|
/**
|
|
* regs_get_kernel_argument() - get Nth function argument in kernel
|
|
* @regs: pt_regs of that context
|
|
* @n: function argument number (start from 0)
|
|
*
|
|
* regs_get_kernel_argument() returns @n th argument of the function call.
|
|
*/
|
|
static __always_inline unsigned long regs_get_kernel_argument(const struct pt_regs *regs,
|
|
unsigned int n)
|
|
{
|
|
unsigned int argoffset = STACK_FRAME_OVERHEAD / sizeof(long);
|
|
|
|
#define NR_REG_ARGUMENTS 5
|
|
if (n < NR_REG_ARGUMENTS)
|
|
return regs_get_register(regs, 2 + n);
|
|
n -= NR_REG_ARGUMENTS;
|
|
return regs_get_kernel_stack_nth(regs, argoffset + n);
|
|
}
|
|
|
|
static __always_inline void regs_set_return_value(struct pt_regs *regs, unsigned long rc)
|
|
{
|
|
regs->gprs[2] = rc;
|
|
}
|
|
|
|
#endif /* __ASSEMBLER__ */
|
|
#endif /* _S390_PTRACE_H */
|