Files
linux-stable-mirror/arch/s390/include/asm/ptrace.h
T
Heiko CarstensandAlexander Gordeev a737737cdb s390/percpu: Infrastructure for more efficient this_cpu operations
With the intended removal of PREEMPT_NONE this_cpu operations based on
atomic instructions, guarded with preempt_disable()/preempt_enable() pairs
become more expensive: the preempt_disable() / preempt_enable() pairs are
not optimized away anymore during compile time.

In particular the conditional call to preempt_schedule_notrace() after
preempt_enable() adds additional code and register pressure.

E.g. this simple C code sequence

DEFINE_PER_CPU(long, foo);
long bar(long a) { return this_cpu_add_return(foo, a); }

generates this code:

  11a976:       eb af f0 68 00 24       stmg    %r10,%r15,104(%r15)
  11a97c:       b9 04 00 ef             lgr     %r14,%r15
  11a980:       b9 04 00 b2             lgr     %r11,%r2
  11a984:       e3 f0 ff c8 ff 71       lay     %r15,-56(%r15)
  11a98a:       e3 e0 f0 98 00 24       stg     %r14,152(%r15)
  11a990:       eb 01 03 a8 00 6a       asi     936,1            <- __preempt_count_add(1)
  11a996:       c0 10 00 d2 ac b5       larl    %r1,1b70300      <- address of percpu var
  11a9a0:       e3 10 23 b8 00 08       ag      %r1,952          <- add percpu offset
  11a9a6:       eb ab 10 00 00 e8       laag    %r10,%r11,0(%r1) <- atomic op
  11a9ac:       eb ff 03 a8 00 6e       alsi    936,-1           <- __preempt_count_dec_and_test()
  11a9b2:       a7 54 00 05             jnhe    11a9bc <bar+0x4c>
  11a9b6:       c0 e5 00 76 d1 bd       brasl   %r14,ff4d30 <preempt_schedule_notrace>
  11a9bc:       b9 e8 b0 2a             agrk    %r2,%r10,%r11
  11a9c0:       eb af f0 a0 00 04       lmg     %r10,%r15,160(%r15)
  11a9c6        07 fe                   br      %r14

Even though the above example is more or less the worst case, since the
branch to preempt_schedule_notrace() requires a stackframe, which
otherwise wouldn't be necessary, there is also the conditional jnhe branch
instruction.

Get rid of the conditional branch with the following code sequence:

  11a8e6:       c0 30 00 d0 c5 0d       larl    %r3,1b33300
  11a8ec:       b9 04 00 43             lgr     %r4,%r3
  11a8f0:       eb 00 43 c0 00 52       mviy    960,4
  11a8f6:       e3 40 03 b8 00 08       ag      %r4,952
  11a8fc:       eb 52 40 00 00 e8       laag    %r5,%r2,0(%r4)
  11a902:       eb 00 03 c0 00 52       mviy    960,0
  11a908:       b9 08 00 25             agr     %r2,%r5
  11a90c        07 fe                   br      %r14

The general idea is that this_cpu operations based on atomic instructions
are guarded with mviy instructions:

- The first mviy instruction writes the register number, which contains
  the percpu address variable to lowcore. This also indicates that a
  percpu code section is executed.

- The first instruction following the mviy instruction must be the ag
  instruction which adds the percpu offset to the percpu address register.

- Afterwards the atomic percpu operation follows.

- Then a second mviy instruction writes a zero to lowcore, which indicates
  the end of the percpu code section.

- In case of an interrupt/exception/nmi the register number which was
  written to lowcore is copied to the exception frame (pt_regs), and a zero
  is written to lowcore.

- On return to the previous context it is checked if a percpu code section
  was executed (saved register number not zero), and if the process was
  migrated to a different cpu. If the percpu offset was already added to
  the percpu address register (instruction address does _not_ point to the
  ag instruction) the content of the percpu address register is adjusted so
  it points to percpu variable of the new cpu.

Reviewed-by: Alexander Gordeev <agordeev@linux.ibm.com>
Signed-off-by: Heiko Carstens <hca@linux.ibm.com>
Signed-off-by: Alexander Gordeev <agordeev@linux.ibm.com>
2026-06-03 15:32:46 +02:00

320 lines
8.8 KiB
C

/* SPDX-License-Identifier: GPL-2.0 */
/*
* S390 version
* Copyright IBM Corp. 1999, 2000
* Author(s): Denis Joseph Barrow (djbarrow@de.ibm.com,barrow_dj@yahoo.com)
*/
#ifndef _S390_PTRACE_H
#define _S390_PTRACE_H
#include <linux/bits.h>
#include <linux/typecheck.h>
#include <uapi/asm/ptrace.h>
#include <asm/thread_info.h>
#include <asm/tpi.h>
#define PIF_SYSCALL 0 /* inside a system call */
#define PIF_PSW_ADDR_ADJUSTED 1 /* psw address has been adjusted */
#define PIF_SYSCALL_RET_SET 2 /* return value was set via ptrace */
#define PIF_GUEST_FAULT 3 /* indicates program check in sie64a */
#define PIF_FTRACE_FULL_REGS 4 /* all register contents valid (ftrace) */
#define _PIF_SYSCALL BIT(PIF_SYSCALL)
#define _PIF_ADDR_PSW_ADJUSTED BIT(PIF_PSW_ADDR_ADJUSTED)
#define _PIF_SYSCALL_RET_SET BIT(PIF_SYSCALL_RET_SET)
#define _PIF_GUEST_FAULT BIT(PIF_GUEST_FAULT)
#define _PIF_FTRACE_FULL_REGS BIT(PIF_FTRACE_FULL_REGS)
#define PSW32_MASK_PER _AC(0x40000000, UL)
#define PSW32_MASK_DAT _AC(0x04000000, UL)
#define PSW32_MASK_IO _AC(0x02000000, UL)
#define PSW32_MASK_EXT _AC(0x01000000, UL)
#define PSW32_MASK_KEY _AC(0x00F00000, UL)
#define PSW32_MASK_BASE _AC(0x00080000, UL) /* Always one */
#define PSW32_MASK_MCHECK _AC(0x00040000, UL)
#define PSW32_MASK_WAIT _AC(0x00020000, UL)
#define PSW32_MASK_PSTATE _AC(0x00010000, UL)
#define PSW32_MASK_ASC _AC(0x0000C000, UL)
#define PSW32_MASK_CC _AC(0x00003000, UL)
#define PSW32_MASK_PM _AC(0x00000f00, UL)
#define PSW32_MASK_RI _AC(0x00000080, UL)
#define PSW32_ADDR_AMODE _AC(0x80000000, UL)
#define PSW32_ADDR_INSN _AC(0x7FFFFFFF, UL)
#define PSW32_DEFAULT_KEY ((PAGE_DEFAULT_ACC) << 20)
#define PSW32_ASC_PRIMARY _AC(0x00000000, UL)
#define PSW32_ASC_ACCREG _AC(0x00004000, UL)
#define PSW32_ASC_SECONDARY _AC(0x00008000, UL)
#define PSW32_ASC_HOME _AC(0x0000C000, UL)
#define PSW_DEFAULT_KEY ((PAGE_DEFAULT_ACC) << 52)
#define PSW_KERNEL_BITS (PSW_DEFAULT_KEY | PSW_MASK_BASE | PSW_ASC_HOME | \
PSW_MASK_EA | PSW_MASK_BA | PSW_MASK_DAT)
#define PSW_USER_BITS (PSW_MASK_DAT | PSW_MASK_IO | PSW_MASK_EXT | \
PSW_DEFAULT_KEY | PSW_MASK_BASE | PSW_MASK_MCHECK | \
PSW_MASK_PSTATE | PSW_ASC_PRIMARY)
#ifndef __ASSEMBLER__
struct psw_bits {
unsigned long : 1;
unsigned long per : 1; /* PER-Mask */
unsigned long : 3;
unsigned long dat : 1; /* DAT Mode */
unsigned long io : 1; /* Input/Output Mask */
unsigned long ext : 1; /* External Mask */
unsigned long key : 4; /* PSW Key */
unsigned long : 1;
unsigned long mcheck : 1; /* Machine-Check Mask */
unsigned long wait : 1; /* Wait State */
unsigned long pstate : 1; /* Problem State */
unsigned long as : 2; /* Address Space Control */
unsigned long cc : 2; /* Condition Code */
unsigned long pm : 4; /* Program Mask */
unsigned long ri : 1; /* Runtime Instrumentation */
unsigned long : 6;
unsigned long eaba : 2; /* Addressing Mode */
unsigned long : 31;
unsigned long ia : 64; /* Instruction Address */
};
enum {
PSW_BITS_AMODE_24BIT = 0,
PSW_BITS_AMODE_31BIT = 1,
PSW_BITS_AMODE_64BIT = 3
};
enum {
PSW_BITS_AS_PRIMARY = 0,
PSW_BITS_AS_ACCREG = 1,
PSW_BITS_AS_SECONDARY = 2,
PSW_BITS_AS_HOME = 3
};
#define psw_bits(__psw) (*({ \
typecheck(psw_t, __psw); \
&(*(struct psw_bits *)(&(__psw))); \
}))
typedef struct {
unsigned int mask;
unsigned int addr;
} psw32_t __aligned(8);
#define PGM_INT_CODE_MASK 0x7f
#define PGM_INT_CODE_PER 0x80
/*
* The pt_regs struct defines the way the registers are stored on
* the stack during a system call.
*/
struct pt_regs {
union {
user_pt_regs user_regs;
struct {
unsigned long args[1];
psw_t psw;
unsigned long gprs[NUM_GPRS];
};
};
union {
unsigned long orig_gpr2;
unsigned long monitor_code;
};
union {
struct {
unsigned int int_code;
unsigned int int_parm;
unsigned long int_parm_long;
};
struct tpi_info tpi_info;
};
unsigned long flags;
unsigned long last_break;
unsigned int cpu;
unsigned char percpu_register;
};
/*
* Program event recording (PER) register set.
*/
struct per_regs {
unsigned long control; /* PER control bits */
unsigned long start; /* PER starting address */
unsigned long end; /* PER ending address */
};
/*
* PER event contains information about the cause of the last PER exception.
*/
struct per_event {
unsigned short cause; /* PER code, ATMID and AI */
unsigned long address; /* PER address */
unsigned char paid; /* PER access identification */
};
/*
* Simplified per_info structure used to decode the ptrace user space ABI.
*/
struct per_struct_kernel {
unsigned long cr9; /* PER control bits */
unsigned long cr10; /* PER starting address */
unsigned long cr11; /* PER ending address */
unsigned long bits; /* Obsolete software bits */
unsigned long starting_addr; /* User specified start address */
unsigned long ending_addr; /* User specified end address */
unsigned short perc_atmid; /* PER trap ATMID */
unsigned long address; /* PER trap instruction address */
unsigned char access_id; /* PER trap access identification */
};
#define PER_EVENT_MASK 0xEB000000UL
#define PER_EVENT_BRANCH 0x80000000UL
#define PER_EVENT_IFETCH 0x40000000UL
#define PER_EVENT_STORE 0x20000000UL
#define PER_EVENT_STORE_REAL 0x08000000UL
#define PER_EVENT_TRANSACTION_END 0x02000000UL
#define PER_EVENT_NULLIFICATION 0x01000000UL
#define PER_CONTROL_MASK 0x00e00000UL
#define PER_CONTROL_BRANCH_ADDRESS 0x00800000UL
#define PER_CONTROL_SUSPENSION 0x00400000UL
#define PER_CONTROL_ALTERATION 0x00200000UL
static inline void set_pt_regs_flag(struct pt_regs *regs, int flag)
{
regs->flags |= (1UL << flag);
}
static inline void clear_pt_regs_flag(struct pt_regs *regs, int flag)
{
regs->flags &= ~(1UL << flag);
}
static inline int test_pt_regs_flag(struct pt_regs *regs, int flag)
{
return !!(regs->flags & (1UL << flag));
}
static inline int test_and_clear_pt_regs_flag(struct pt_regs *regs, int flag)
{
int ret = test_pt_regs_flag(regs, flag);
clear_pt_regs_flag(regs, flag);
return ret;
}
struct task_struct;
void update_cr_regs(struct task_struct *task);
/*
* These are defined as per linux/ptrace.h, which see.
*/
#define arch_has_single_step() (1)
#define arch_has_block_step() (1)
#define profile_pc(regs) instruction_pointer(regs)
static __always_inline bool user_mode(const struct pt_regs *regs)
{
return psw_bits(regs->psw).pstate;
}
static inline long regs_return_value(const struct pt_regs *regs)
{
return regs->gprs[2];
}
static __always_inline unsigned long instruction_pointer(const struct pt_regs *regs)
{
return regs->psw.addr;
}
static inline void instruction_pointer_set(struct pt_regs *regs,
unsigned long val)
{
regs->psw.addr = val;
}
int regs_query_register_offset(const char *name);
const char *regs_query_register_name(unsigned int offset);
static __always_inline unsigned long kernel_stack_pointer(const struct pt_regs *regs)
{
return regs->gprs[15];
}
static __always_inline unsigned long user_stack_pointer(const struct pt_regs *regs)
{
return regs->gprs[15];
}
static __always_inline unsigned long regs_get_register(const struct pt_regs *regs,
unsigned int offset)
{
if (offset >= NUM_GPRS)
return 0;
return regs->gprs[offset];
}
static __always_inline int regs_within_kernel_stack(const struct pt_regs *regs,
unsigned long addr)
{
unsigned long ksp = kernel_stack_pointer(regs);
return (addr & ~(THREAD_SIZE - 1)) == (ksp & ~(THREAD_SIZE - 1));
}
/**
* regs_get_kernel_stack_nth() - get Nth entry of the stack
* @regs:pt_regs which contains kernel stack pointer.
* @n:stack entry number.
*
* regs_get_kernel_stack_nth() returns @n th entry of the kernel stack which
* is specifined by @regs. If the @n th entry is NOT in the kernel stack,
* this returns 0.
*/
static __always_inline unsigned long regs_get_kernel_stack_nth(const struct pt_regs *regs,
unsigned int n)
{
unsigned long addr;
addr = kernel_stack_pointer(regs) + n * sizeof(long);
if (!regs_within_kernel_stack(regs, addr))
return 0;
return READ_ONCE_NOCHECK(*(unsigned long *)addr);
}
/**
* regs_get_kernel_argument() - get Nth function argument in kernel
* @regs: pt_regs of that context
* @n: function argument number (start from 0)
*
* regs_get_kernel_argument() returns @n th argument of the function call.
*/
static __always_inline unsigned long regs_get_kernel_argument(const struct pt_regs *regs,
unsigned int n)
{
unsigned int argoffset = STACK_FRAME_OVERHEAD / sizeof(long);
#define NR_REG_ARGUMENTS 5
if (n < NR_REG_ARGUMENTS)
return regs_get_register(regs, 2 + n);
n -= NR_REG_ARGUMENTS;
return regs_get_kernel_stack_nth(regs, argoffset + n);
}
static __always_inline void regs_set_return_value(struct pt_regs *regs, unsigned long rc)
{
regs->gprs[2] = rc;
}
#endif /* __ASSEMBLER__ */
#endif /* _S390_PTRACE_H */