averello/linux-stable-mirror
1
0
Fork 0
mirror of https://git.kernel.org/pub/scm/linux/kernel/git/stable/linux.git synced 2026-05-05 09:57:21 +02:00
Code Issues Packages Projects Releases Wiki Activity

Merge tag 'riscv-for-linus-7.1-mw1' of git://git.kernel.org/pub/scm/linux/kernel/git/riscv/linux

Browse Source 
Pull RISC-V updates from Paul Walmsley:
 "There is one significant change outside arch/riscv in this pull
  request: the addition of a set of KUnit tests for strlen(), strnlen(),
  and strrchr().

  Otherwise, the most notable changes are to add some RISC-V-specific
  string function implementations, to remove XIP kernel support, to add
  hardware error exception handling, and to optimize our runtime
  unaligned access speed testing.

  A few comments on the motivation for removing XIP support. It's been
  broken in the RISC-V kernel for months. The code is not easy to
  maintain. Furthermore, for XIP support to truly be useful for RISC-V,
  we think that compile-time feature switches would need to be added for
  many of the RISC-V ISA features and microarchitectural properties that
  are currently implemented with runtime patching. No one has stepped
  forward to take responsibility for that work, so many of us think it's
  best to remove it until clear use cases and champions emerge.

  Summary:

   - Add Kunit correctness testing and microbenchmarks for strlen(),
     strnlen(), and strrchr()

   - Add RISC-V-specific strnlen(), strchr(), strrchr() implementations

   - Add hardware error exception handling

   - Clean up and optimize our unaligned access probe code

   - Enable HAVE_IOREMAP_PROT to be able to use generic_access_phys()

   - Remove XIP kernel support

   - Warn when addresses outside the vmemmap range are passed to
     vmemmap_populate()

   - Update the ACPI FADT revision check to warn if it's not at least
     ACPI v6.6, which is when key RISC-V-specific tables were added to
     the specification

   - Increase COMMAND_LINE_SIZE to 2048 to match ARM64, x86, PowerPC,
     etc.

   - Make kaslr_offset() a static inline function, since there's no need
     for it to show up in the symbol table

   - Add KASLR offset and SATP to the VMCOREINFO ELF notes to improve
     kdump support

   - Add Makefile cleanup rule for vdso_cfi copied source files, and add
     a .gitignore for the build artifacts in that directory

   - Remove some redundant ifdefs that check Kconfig macros

   - Add missing SPDX license tag to the CFI selftest

   - Simplify UTS_MACHINE assignment in the RISC-V Makefile

   - Clarify some unclear comments and remove some superfluous comments

   - Fix various English typos across the RISC-V codebase"

* tag 'riscv-for-linus-7.1-mw1' of git://git.kernel.org/pub/scm/linux/kernel/git/riscv/linux: (31 commits)
  riscv: Remove support for XIP kernel
  riscv: Reuse compare_unaligned_access() in check_vector_unaligned_access()
  riscv: Split out compare_unaligned_access()
  riscv: Reuse measure_cycles() in check_vector_unaligned_access()
  riscv: Split out measure_cycles() for reuse
  riscv: Clean up & optimize unaligned scalar access probe
  riscv: lib: add strrchr() implementation
  riscv: lib: add strchr() implementation
  riscv: lib: add strnlen() implementation
  lib/string_kunit: extend benchmarks to strnlen() and chr searches
  lib/string_kunit: add performance benchmark for strlen()
  lib/string_kunit: add correctness test for strrchr()
  lib/string_kunit: add correctness test for strnlen()
  lib/string_kunit: add correctness test for strlen()
  riscv: vdso_cfi: Add .gitignore for build artifacts
  riscv: vdso_cfi: Add clean rule for copied sources
  riscv: enable HAVE_IOREMAP_PROT
  riscv: mm: WARN_ON() for bad addresses in vmemmap_populate()
  riscv: acpi: update FADT revision check to 6.6
  riscv: add hardware error trap handler support
  ...
This commit is contained in:
Linus Torvalds
2026-04-24 10:00:37 -07:00
parent ff57d59200 9b3a2be848
commit feff82eb5f
56 changed files with 735 additions and 539 deletions
Download Patch File Download Diff File Expand all files Collapse all files
Documentation/features/vm/ioremap_prot/arch-support.txt
+1 -1
View File
@@ -20,7 +20,7 @@
| openrisc: | TODO |
| parisc: | TODO |
| powerpc: | ok |
| riscv: | TODO |
| riscv: | ok |
| s390: | ok |
| sh: | ok |
| sparc: | TODO |
arch/riscv/Kconfig
+24 -63
View File
@@ -47,8 +47,8 @@ config RISCV
select ARCH_HAS_PTE_SPECIAL
select ARCH_HAS_SET_DIRECT_MAP if MMU
select ARCH_HAS_SET_MEMORY if MMU
select ARCH_HAS_STRICT_KERNEL_RWX if MMU && !XIP_KERNEL
select ARCH_HAS_STRICT_MODULE_RWX if MMU && !XIP_KERNEL
select ARCH_HAS_STRICT_KERNEL_RWX if MMU
select ARCH_HAS_STRICT_MODULE_RWX if MMU
select ARCH_HAS_SYNC_CORE_BEFORE_USERMODE
select ARCH_HAS_SYSCALL_WRAPPER
select ARCH_HAS_TICK_BROADCAST if GENERIC_CLOCKEVENTS_BROADCAST
@@ -84,7 +84,7 @@ config RISCV
select ARCH_WANT_FRAME_POINTERS
select ARCH_WANT_GENERAL_HUGETLB if !RISCV_ISA_SVNAPOT
select ARCH_WANT_HUGE_PMD_SHARE if 64BIT
select ARCH_WANT_LD_ORPHAN_WARN if !XIP_KERNEL
select ARCH_WANT_LD_ORPHAN_WARN
select ARCH_WANT_OPTIMIZE_DAX_VMEMMAP
select ARCH_WANT_OPTIMIZE_HUGETLB_VMEMMAP
select ARCH_WANTS_NO_INSTR
@@ -112,6 +112,7 @@ config RISCV
select GENERIC_GETTIMEOFDAY if HAVE_GENERIC_VDSO && 64BIT
select GENERIC_IDLE_POLL_SETUP
select GENERIC_IOREMAP if MMU
select HAVE_IOREMAP_PROT if MMU
select GENERIC_IRQ_IPI if SMP
select GENERIC_IRQ_IPI_MUX if SMP
select GENERIC_IRQ_MULTI_HANDLER
@@ -129,13 +130,13 @@ config RISCV
select HAVE_ARCH_AUDITSYSCALL
select HAVE_ARCH_HUGE_VMALLOC if HAVE_ARCH_HUGE_VMAP
select HAVE_ARCH_HUGE_VMAP if MMU && 64BIT
select HAVE_ARCH_JUMP_LABEL if !XIP_KERNEL
select HAVE_ARCH_JUMP_LABEL_RELATIVE if !XIP_KERNEL
select HAVE_ARCH_JUMP_LABEL
select HAVE_ARCH_JUMP_LABEL_RELATIVE
select HAVE_ARCH_KASAN if MMU && 64BIT
select HAVE_ARCH_KASAN_VMALLOC if MMU && 64BIT
select HAVE_ARCH_KFENCE if MMU && 64BIT
select HAVE_ARCH_KSTACK_ERASE
select HAVE_ARCH_KGDB if !XIP_KERNEL
select HAVE_ARCH_KGDB
select HAVE_ARCH_KGDB_QXFER_PKT
select HAVE_ARCH_MMAP_RND_BITS if MMU
select HAVE_ARCH_MMAP_RND_COMPAT_BITS if COMPAT
@@ -153,7 +154,7 @@ config RISCV
select HAVE_CONTEXT_TRACKING_USER
select HAVE_DEBUG_KMEMLEAK
select HAVE_DMA_CONTIGUOUS if MMU
select HAVE_DYNAMIC_FTRACE if !XIP_KERNEL && MMU && (CLANG_SUPPORTS_DYNAMIC_FTRACE || GCC_SUPPORTS_DYNAMIC_FTRACE)
select HAVE_DYNAMIC_FTRACE if MMU && (CLANG_SUPPORTS_DYNAMIC_FTRACE || GCC_SUPPORTS_DYNAMIC_FTRACE)
select FUNCTION_ALIGNMENT_4B if HAVE_DYNAMIC_FTRACE && RISCV_ISA_C
select HAVE_DYNAMIC_FTRACE_WITH_DIRECT_CALLS if HAVE_DYNAMIC_FTRACE_WITH_CALL_OPS
select HAVE_DYNAMIC_FTRACE_WITH_CALL_OPS if (DYNAMIC_FTRACE_WITH_ARGS && !CFI)
@@ -161,7 +162,7 @@ config RISCV
select HAVE_FTRACE_GRAPH_FUNC
select HAVE_FUNCTION_GRAPH_TRACER if HAVE_DYNAMIC_FTRACE_WITH_ARGS
select HAVE_FUNCTION_GRAPH_FREGS
select HAVE_FUNCTION_TRACER if !XIP_KERNEL && HAVE_DYNAMIC_FTRACE
select HAVE_FUNCTION_TRACER if HAVE_DYNAMIC_FTRACE
select HAVE_EBPF_JIT if MMU
select HAVE_GENERIC_TIF_BITS
select HAVE_GUP_FAST if MMU
@@ -170,16 +171,16 @@ config RISCV
select HAVE_GCC_PLUGINS
select HAVE_GENERIC_VDSO if MMU
select HAVE_IRQ_TIME_ACCOUNTING
select HAVE_KERNEL_BZIP2 if !XIP_KERNEL && !EFI_ZBOOT
select HAVE_KERNEL_GZIP if !XIP_KERNEL && !EFI_ZBOOT
select HAVE_KERNEL_LZ4 if !XIP_KERNEL && !EFI_ZBOOT
select HAVE_KERNEL_LZMA if !XIP_KERNEL && !EFI_ZBOOT
select HAVE_KERNEL_LZO if !XIP_KERNEL && !EFI_ZBOOT
select HAVE_KERNEL_UNCOMPRESSED if !XIP_KERNEL && !EFI_ZBOOT
select HAVE_KERNEL_ZSTD if !XIP_KERNEL && !EFI_ZBOOT
select HAVE_KERNEL_XZ if !XIP_KERNEL && !EFI_ZBOOT
select HAVE_KPROBES if !XIP_KERNEL
select HAVE_KRETPROBES if !XIP_KERNEL
select HAVE_KERNEL_BZIP2 if !EFI_ZBOOT
select HAVE_KERNEL_GZIP if !EFI_ZBOOT
select HAVE_KERNEL_LZ4 if !EFI_ZBOOT
select HAVE_KERNEL_LZMA if !EFI_ZBOOT
select HAVE_KERNEL_LZO if !EFI_ZBOOT
select HAVE_KERNEL_UNCOMPRESSED if !EFI_ZBOOT
select HAVE_KERNEL_ZSTD if !EFI_ZBOOT
select HAVE_KERNEL_XZ if !EFI_ZBOOT
select HAVE_KPROBES
select HAVE_KRETPROBES
# https://github.com/ClangBuiltLinux/linux/issues/1881
select HAVE_LD_DEAD_CODE_DATA_ELIMINATION if !LD_IS_LLD
select HAVE_MOVE_PMD
@@ -190,9 +191,9 @@ config RISCV
select HAVE_PERF_REGS
select HAVE_PERF_USER_STACK_DUMP
select HAVE_POSIX_CPU_TIMERS_TASK_WORK
select HAVE_PREEMPT_DYNAMIC_KEY if !XIP_KERNEL
select HAVE_PREEMPT_DYNAMIC_KEY
select HAVE_REGS_AND_STACK_ACCESS_API
select HAVE_RETHOOK if !XIP_KERNEL
select HAVE_RETHOOK
select HAVE_RSEQ
select HAVE_RUST if RUSTC_SUPPORTS_RISCV && CC_IS_CLANG
select HAVE_SAMPLE_FTRACE_DIRECT
@@ -213,7 +214,7 @@ config RISCV
select PCI_ECAM if (ACPI && PCI)
select PCI_MSI if PCI
select RELOCATABLE if !MMU && !PHYS_RAM_BASE_FIXED
select RISCV_ALTERNATIVE if !XIP_KERNEL
select RISCV_ALTERNATIVE
select RISCV_APLIC
select RISCV_IMSIC
select RISCV_INTC
@@ -537,7 +538,6 @@ endchoice
config RISCV_ALTERNATIVE
bool
depends on !XIP_KERNEL
help
This Kconfig allows the kernel to automatically patch the
erratum or cpufeature required by the execution platform at run
@@ -1126,7 +1126,6 @@ config PARAVIRT_TIME_ACCOUNTING
config RELOCATABLE
bool "Build a relocatable kernel"
depends on !XIP_KERNEL
select MODULE_SECTIONS if MODULES
select ARCH_VMLINUX_NEEDS_RELOCS
help
@@ -1143,7 +1142,7 @@ config RELOCATABLE
config RANDOMIZE_BASE
bool "Randomize the address of the kernel image"
select RELOCATABLE
depends on MMU && 64BIT && !XIP_KERNEL
depends on MMU && 64BIT
help
Randomizes the virtual address at which the kernel image is
loaded, as a security feature that deters exploit attempts
@@ -1233,7 +1232,7 @@ config EFI_STUB
config EFI
bool "UEFI runtime support"
depends on OF && !XIP_KERNEL
depends on OF
depends on MMU
default y
select ARCH_SUPPORTS_ACPI if 64BIT
@@ -1284,44 +1283,6 @@ config PHYS_RAM_BASE
explicitly specified to run early relocations of read-write data
from flash to RAM.
config XIP_KERNEL
bool "Kernel Execute-In-Place from ROM"
depends on MMU && SPARSEMEM && NONPORTABLE
# This prevents XIP from being enabled by all{yes,mod}config, which
# fail to build since XIP doesn't support large kernels.
depends on !COMPILE_TEST
select PHYS_RAM_BASE_FIXED
help
Execute-In-Place allows the kernel to run from non-volatile storage
directly addressable by the CPU, such as NOR flash. This saves RAM
space since the text section of the kernel is not loaded from flash
to RAM. Read-write sections, such as the data section and stack,
are still copied to RAM. The XIP kernel is not compressed since
it has to run directly from flash, so it will take more space to
store it. The flash address used to link the kernel object files,
and for storing it, is configuration dependent. Therefore, if you
say Y here, you must know the proper physical address where to
store the kernel image depending on your own flash memory usage.
Also note that the make target becomes "make xipImage" rather than
"make zImage" or "make Image". The final kernel binary to put in
ROM memory will be arch/riscv/boot/xipImage.
SPARSEMEM is required because the kernel text and rodata that are
flash resident are not backed by memmap, then any attempt to get
a struct page on those regions will trigger a fault.
If unsure, say N.
config XIP_PHYS_ADDR
hex "XIP Kernel Physical Location"
depends on XIP_KERNEL
default "0x21000000"
help
This is the physical address in your flash memory the kernel will
be linked for and stored to. This address is dependent on your
own flash usage.
config RISCV_ISA_FALLBACK
bool "Permit falling back to parsing riscv,isa for extension support by default"
default y
arch/riscv/Kconfig.socs
+4 -4
View File
@@ -2,7 +2,7 @@ menu "SoC selection"
config ARCH_ANDES
bool "Andes SoCs"
depends on MMU && !XIP_KERNEL
depends on MMU
select ERRATA_ANDES
help
This enables support for Andes SoC platform hardware.
@@ -33,7 +33,7 @@ config ARCH_RENESAS
config ARCH_SIFIVE
bool "SiFive SoCs"
select ERRATA_SIFIVE if !XIP_KERNEL
select ERRATA_SIFIVE
help
This enables support for SiFive SoC platform hardware.
@@ -61,7 +61,7 @@ config SOC_STARFIVE
config ARCH_SUNXI
bool "Allwinner sun20i SoCs"
depends on MMU && !XIP_KERNEL
depends on MMU
select ERRATA_THEAD
select SUN4I_TIMER
help
@@ -78,7 +78,7 @@ config ARCH_TENSTORRENT
config ARCH_THEAD
bool "T-HEAD RISC-V SoCs"
depends on MMU && !XIP_KERNEL
depends on MMU
select ERRATA_THEAD
select PM_GENERIC_DOMAINS if PM
help
arch/riscv/Makefile
+2 -5
View File
@@ -28,7 +28,6 @@ endif
export BITS
ifeq ($(CONFIG_ARCH_RV64I),y)
BITS := 64
UTS_MACHINE := riscv64
KBUILD_CFLAGS += -mabi=lp64
KBUILD_AFLAGS += -mabi=lp64
@@ -39,13 +38,14 @@ ifeq ($(CONFIG_ARCH_RV64I),y)
-Cno-redzone
else
BITS := 32
UTS_MACHINE := riscv32
KBUILD_CFLAGS += -mabi=ilp32
KBUILD_AFLAGS += -mabi=ilp32
KBUILD_LDFLAGS += -melf32lriscv
endif
UTS_MACHINE := riscv$(BITS)
# LLVM has an issue with target-features and LTO: https://github.com/llvm/llvm-project/issues/59350
# Ensure it is aware of linker relaxation with LTO, otherwise relocations may
# be incorrect: https://github.com/llvm/llvm-project/issues/65090
@@ -150,7 +150,6 @@ ifdef CONFIG_RISCV_M_MODE
boot-image-$(CONFIG_SOC_CANAAN_K210) := loader.bin
endif
boot-image-$(CONFIG_EFI_ZBOOT) := vmlinuz.efi
boot-image-$(CONFIG_XIP_KERNEL) := xipImage
KBUILD_IMAGE := $(boot)/$(boot-image-y)
libs-y += arch/riscv/lib/
@@ -218,8 +217,6 @@ define archhelp
echo ' Image.xz - Compressed kernel image (arch/riscv/boot/Image.xz)'
echo ' vmlinuz.efi - Compressed EFI kernel image (arch/riscv/boot/vmlinuz.efi)'
echo ' Default when CONFIG_EFI_ZBOOT=y'
echo ' xipImage - Execute-in-place kernel image (arch/riscv/boot/xipImage)'
echo ' Default when CONFIG_XIP_KERNEL=y'
echo ' install - Install kernel using (your) ~/bin/$(INSTALLKERNEL) or'
echo ' (distribution) /sbin/$(INSTALLKERNEL) or install to '
echo ' $$(INSTALL_PATH)'
arch/riscv/boot/Makefile
-11
View File
@@ -20,17 +20,6 @@ OBJCOPYFLAGS_xipImage :=-O binary -R .note -R .note.gnu.build-id -R .comment -S
targets := Image Image.* loader loader.o loader.lds loader.bin xipImage
ifeq ($(CONFIG_XIP_KERNEL),y)
quiet_cmd_mkxip = $(quiet_cmd_objcopy)
cmd_mkxip = $(cmd_objcopy)
$(obj)/xipImage: vmlinux FORCE
$(call if_changed,mkxip)
@$(kecho) ' Physical Address of xipImage: $(CONFIG_XIP_PHYS_ADDR)'
endif
$(obj)/Image: vmlinux FORCE
$(call if_changed,objcopy)
arch/riscv/errata/thead/errata.c
+1 -1
View File
@@ -153,7 +153,7 @@ static bool errata_probe_ghostwrite(unsigned int stage,
* target-c9xx cores report arch_id and impid as 0
*
* While ghostwrite may not affect all c9xx cores that implement
* xtheadvector, there is no futher granularity than c9xx. Assume
* xtheadvector, there is no further granularity than c9xx. Assume
* vulnerable for this entire class of processors when xtheadvector is
* enabled.
*/
arch/riscv/include/asm/asm-prototypes.h
+1
View File
@@ -40,6 +40,7 @@ asmlinkage void riscv_v_context_nesting_end(struct pt_regs *regs);
#define DECLARE_DO_ERROR_INFO(name) asmlinkage void name(struct pt_regs *regs)
DECLARE_DO_ERROR_INFO(do_trap_unknown);
DECLARE_DO_ERROR_INFO(do_trap_hardware_error);
DECLARE_DO_ERROR_INFO(do_trap_insn_misaligned);
DECLARE_DO_ERROR_INFO(do_trap_insn_fault);
DECLARE_DO_ERROR_INFO(do_trap_insn_illegal);
arch/riscv/include/asm/atomic.h
+2 -2
View File
@@ -46,7 +46,7 @@ static __always_inline void arch_atomic64_set(atomic64_t *v, s64 i)
#endif
/*
* First, the atomic ops that have no ordering constraints and therefor don't
* First, the atomic ops that have no ordering constraints and therefore don't
* have the AQ or RL bits set. These don't return anything, so there's only
* one version to worry about.
*/
@@ -81,7 +81,7 @@ ATOMIC_OPS(xor, xor, i)
/*
* Atomic ops that have ordered, relaxed, acquire, and release variants.
* There's two flavors of these: the arithmatic ops have both fetch and return
* There's two flavors of these: the arithmetic ops have both fetch and return
* versions, while the logical ops only have fetch versions.
*/
#define ATOMIC_FETCH_OP(op, asm_op, I, asm_type, c_type, prefix) \
arch/riscv/include/asm/elf.h
+2 -2
View File
@@ -59,8 +59,8 @@ extern bool compat_elf_check_arch(Elf32_Ehdr *hdr);
#endif
/*
* Provides information on the availiable set of ISA extensions to userspace,
* via a bitmap that coorespends to each single-letter ISA extension. This is
* Provides information on the available set of ISA extensions to userspace,
* via a bitmap that corresponds to each single-letter ISA extension. This is
* essentially defunct, but will remain for compatibility with userspace.
*/
#define ELF_HWCAP riscv_get_elf_hwcap()
arch/riscv/include/asm/page.h
+4 -30
View File
@@ -29,11 +29,7 @@
#define PAGE_OFFSET_L5 _AC(0xff60000000000000, UL)
#define PAGE_OFFSET_L4 _AC(0xffffaf8000000000, UL)
#define PAGE_OFFSET_L3 _AC(0xffffffd600000000, UL)
#ifdef CONFIG_XIP_KERNEL
#define PAGE_OFFSET PAGE_OFFSET_L3
#else
#define PAGE_OFFSET kernel_map.page_offset
#endif /* CONFIG_XIP_KERNEL */
#else
#define PAGE_OFFSET _AC(0xc0000000, UL)
#endif /* CONFIG_64BIT */
@@ -104,15 +100,8 @@ struct kernel_mapping {
/* Offset between linear mapping virtual address and kernel load address */
unsigned long va_pa_offset;
/* Offset between kernel mapping virtual address and kernel load address */
#ifdef CONFIG_XIP_KERNEL
unsigned long va_kernel_xip_text_pa_offset;
unsigned long va_kernel_xip_data_pa_offset;
uintptr_t xiprom;
uintptr_t xiprom_sz;
#else
unsigned long page_offset;
unsigned long va_kernel_pa_offset;
#endif
};
extern struct kernel_mapping kernel_map;
@@ -131,16 +120,7 @@ extern unsigned long vmemmap_start_pfn;
void *linear_mapping_pa_to_va(unsigned long x);
#endif
#ifdef CONFIG_XIP_KERNEL
#define kernel_mapping_pa_to_va(y) ({ \
unsigned long _y = (unsigned long)(y); \
(_y < phys_ram_base) ? \
(void *)(_y + kernel_map.va_kernel_xip_text_pa_offset) : \
(void *)(_y + kernel_map.va_kernel_xip_data_pa_offset); \
})
#else
#define kernel_mapping_pa_to_va(y) ((void *)((unsigned long)(y) + kernel_map.va_kernel_pa_offset))
#endif
#define __pa_to_va_nodebug(x) linear_mapping_pa_to_va(x)
@@ -150,16 +130,7 @@ void *linear_mapping_pa_to_va(unsigned long x);
phys_addr_t linear_mapping_va_to_pa(unsigned long x);
#endif
#ifdef CONFIG_XIP_KERNEL
#define kernel_mapping_va_to_pa(y) ({ \
unsigned long _y = (unsigned long)(y); \
(_y < kernel_map.virt_addr + kernel_map.xiprom_sz) ? \
(_y - kernel_map.va_kernel_xip_text_pa_offset) : \
(_y - kernel_map.va_kernel_xip_data_pa_offset); \
})
#else
#define kernel_mapping_va_to_pa(y) ((unsigned long)(y) - kernel_map.va_kernel_pa_offset)
#endif
#define __va_to_pa_nodebug(x) ({ \
unsigned long _x = x; \
@@ -190,7 +161,10 @@ extern phys_addr_t __phys_addr_symbol(unsigned long x);
#define sym_to_pfn(x) __phys_to_pfn(__pa_symbol(x))
unsigned long kaslr_offset(void);
static inline unsigned long kaslr_offset(void)
{
return kernel_map.virt_offset;
}
static __always_inline void *pfn_to_kaddr(unsigned long pfn)
{
arch/riscv/include/asm/pgtable.h
-20
View File
@@ -134,21 +134,6 @@
#include <linux/page_table_check.h>
#ifdef CONFIG_XIP_KERNEL
#define XIP_FIXUP(addr) ({ \
extern char _sdata[], _start[], _end[]; \
uintptr_t __rom_start_data = CONFIG_XIP_PHYS_ADDR \
+ (uintptr_t)&_sdata - (uintptr_t)&_start; \
uintptr_t __rom_end_data = CONFIG_XIP_PHYS_ADDR \
+ (uintptr_t)&_end - (uintptr_t)&_start; \
uintptr_t __a = (uintptr_t)(addr); \
(__a >= __rom_start_data && __a < __rom_end_data) ? \
__a - __rom_start_data + CONFIG_PHYS_RAM_BASE : __a; \
})
#else
#define XIP_FIXUP(addr) (addr)
#endif /* CONFIG_XIP_KERNEL */
struct pt_alloc_ops {
pte_t *(*get_pte_virt)(phys_addr_t pa);
phys_addr_t (*alloc_pte)(uintptr_t va);
@@ -1272,13 +1257,8 @@ static inline pte_t pte_swp_clear_exclusive(pte_t pte)
extern char _start[];
extern void *_dtb_early_va;
extern uintptr_t _dtb_early_pa;
#if defined(CONFIG_XIP_KERNEL) && defined(CONFIG_MMU)
#define dtb_early_va (*(void **)XIP_FIXUP(&_dtb_early_va))
#define dtb_early_pa (*(uintptr_t *)XIP_FIXUP(&_dtb_early_pa))
#else
#define dtb_early_va _dtb_early_va
#define dtb_early_pa _dtb_early_pa
#endif /* CONFIG_XIP_KERNEL */
extern u64 satp_mode;
void paging_init(void);
arch/riscv/include/asm/processor.h
+1 -1
View File
@@ -86,7 +86,7 @@ struct pt_regs;
* preempt_v. All preempt_v context should be dropped in such case because
* V-regs are caller-saved. Only sstatus.VS=ON is persisted across a
* schedule() call.
* - bit 30: The in-kernel preempt_v context is saved, and requries to be
* - bit 30: The in-kernel preempt_v context is saved, and is required to be
* restored when returning to the context that owns the preempt_v.
* - bit 31: The in-kernel preempt_v context is dirty, as signaled by the
* trap entry code. Any context switches out-of current task need to save
arch/riscv/include/asm/scs.h
-1
View File
@@ -10,7 +10,6 @@
/* Load init_shadow_call_stack to gp. */
.macro scs_load_init_stack
la gp, init_shadow_call_stack
XIP_FIXUP_OFFSET gp
.endm
/* Load the per-CPU IRQ shadow call stack to gp. */
arch/riscv/include/asm/set_memory.h
+1 -1
View File
@@ -47,7 +47,7 @@ bool kernel_page_present(struct page *page);
#endif /* __ASSEMBLER__ */
#if defined(CONFIG_STRICT_KERNEL_RWX) || defined(CONFIG_XIP_KERNEL)
#if defined(CONFIG_STRICT_KERNEL_RWX)
#ifdef CONFIG_64BIT
#define SECTION_ALIGN (1 << 21)
#else
arch/riscv/include/asm/smp.h
+1 -1
View File
@@ -105,7 +105,7 @@ static inline void riscv_ipi_set_virq_range(int virq, int nr)
#endif /* CONFIG_SMP */
#if defined(CONFIG_HOTPLUG_CPU) && (CONFIG_SMP)
#if defined(CONFIG_HOTPLUG_CPU)
bool cpu_has_hotplug(unsigned int cpu);
#else
static inline bool cpu_has_hotplug(unsigned int cpu)
arch/riscv/include/asm/string.h
+9
View File
@@ -28,6 +28,15 @@ extern asmlinkage __kernel_size_t strlen(const char *);
#define __HAVE_ARCH_STRNCMP
extern asmlinkage int strncmp(const char *cs, const char *ct, size_t count);
#define __HAVE_ARCH_STRNLEN
extern asmlinkage __kernel_size_t strnlen(const char *, size_t);
#define __HAVE_ARCH_STRCHR
extern asmlinkage char *strchr(const char *, int);
#define __HAVE_ARCH_STRRCHR
extern asmlinkage char *strrchr(const char *, int);
#endif
/* For those files which don't want to check by kasan. */
arch/riscv/include/asm/thread_info.h
+1 -1
View File
@@ -120,7 +120,7 @@ int arch_dup_task_struct(struct task_struct *dst, struct task_struct *src);
#include <asm-generic/thread_info_tif.h>
#define TIF_32BIT 16 /* compat-mode 32bit process */
#define TIF_RISCV_V_DEFER_RESTORE 17 /* restore Vector before returing to user */
#define TIF_RISCV_V_DEFER_RESTORE 17 /* restore Vector before returning to user */
#define _TIF_RISCV_V_DEFER_RESTORE BIT(TIF_RISCV_V_DEFER_RESTORE)
arch/riscv/include/asm/xip_fixup.h
-49
View File
@@ -1,49 +0,0 @@
/* SPDX-License-Identifier: GPL-2.0-only */
/*
* XIP fixup macros, only useful in assembly.
*/
#ifndef _ASM_RISCV_XIP_FIXUP_H
#define _ASM_RISCV_XIP_FIXUP_H
#include <linux/pgtable.h>
#ifdef CONFIG_XIP_KERNEL
.macro XIP_FIXUP_OFFSET reg
/* Fix-up address in Flash into address in RAM early during boot before
* MMU is up. Because generated code "thinks" data is in Flash, but it
* is actually in RAM (actually data is also in Flash, but Flash is
* read-only, thus we need to use the data residing in RAM).
*
* The start of data in Flash is _sdata and the start of data in RAM is
* CONFIG_PHYS_RAM_BASE. So this fix-up essentially does this:
* reg += CONFIG_PHYS_RAM_BASE - _start
*/
li t0, CONFIG_PHYS_RAM_BASE
add \reg, \reg, t0
la t0, _sdata
sub \reg, \reg, t0
.endm
.macro XIP_FIXUP_FLASH_OFFSET reg
/* In linker script, at the transition from read-only section to
* writable section, the VMA is increased while LMA remains the same.
* (See in linker script how _sdata, __data_loc and LOAD_OFFSET is
* changed)
*
* Consequently, early during boot before MMU is up, the generated code
* reads the "writable" section at wrong addresses, because VMA is used
* by compiler to generate code, but the data is located in Flash using
* LMA.
*/
la t0, _sdata
sub \reg, \reg, t0
la t0, __data_loc
add \reg, \reg, t0
.endm
#else
.macro XIP_FIXUP_OFFSET reg
.endm
.macro XIP_FIXUP_FLASH_OFFSET reg
.endm
#endif /* CONFIG_XIP_KERNEL */
#endif
arch/riscv/include/uapi/asm/setup.h
+1 -1
View File
@@ -3,6 +3,6 @@
#ifndef _UAPI_ASM_RISCV_SETUP_H
#define _UAPI_ASM_RISCV_SETUP_H
#define COMMAND_LINE_SIZE 1024
#define COMMAND_LINE_SIZE 2048
#endif /* _UAPI_ASM_RISCV_SETUP_H */
arch/riscv/kernel/acpi.c
+6 -6
View File
@@ -69,7 +69,7 @@ static int __init acpi_fadt_sanity_check(void)
/*
* FADT is required on riscv; retrieve it to check its presence
* and carry out revision and ACPI HW reduced compliancy tests
* and carry out revision and ACPI HW reduced compliance tests
*/
status = acpi_get_table(ACPI_SIG_FADT, 0, &table);
if (ACPI_FAILURE(status)) {
@@ -85,12 +85,12 @@ static int __init acpi_fadt_sanity_check(void)
* The revision in the table header is the FADT's Major revision. The
* FADT also has a minor revision, which is stored in the FADT itself.
*
* TODO: Currently, we check for 6.5 as the minimum version to check
* for HW_REDUCED flag. However, once RISC-V updates are released in
* the ACPI spec, we need to update this check for exact minor revision
* ACPI 6.6 is required for RISC-V as it introduces RISC-V specific
* tables such as RHCT (RISC-V Hart Capabilities Table) and RIMT
* (RISC-V I/O Mapping Table).
*/
if (table->revision < 6 || (table->revision == 6 && fadt->minor_revision < 5))
pr_err(FW_BUG "Unsupported FADT revision %d.%d, should be 6.5+\n",
if (table->revision < 6 || (table->revision == 6 && fadt->minor_revision < 6))
pr_err(FW_BUG "Unsupported FADT revision %d.%d, should be 6.6+\n",
table->revision, fadt->minor_revision);
if (!(fadt->flags & ACPI_FADT_HW_REDUCED)) {
arch/riscv/kernel/cpu-hotplug.c
-2
View File
@@ -43,7 +43,6 @@ int __cpu_disable(void)
return 0;
}
#ifdef CONFIG_HOTPLUG_CPU
/*
* Called on the thread which is asking for a CPU to be shutdown, if the
* CPU reported dead to the hotplug core.
@@ -75,4 +74,3 @@ void __noreturn arch_cpu_idle_dead(void)
/* It should never reach here */
BUG();
}
#endif
arch/riscv/kernel/entry.S
+1
View File
@@ -498,6 +498,7 @@ SYM_DATA_START_LOCAL(excp_vect_table)
RISCV_PTR do_trap_unknown /* cause=16 */
RISCV_PTR do_trap_unknown /* cause=17 */
RISCV_PTR do_trap_software_check /* cause=18 is sw check exception */
RISCV_PTR do_trap_hardware_error /* hardware error (19) */
SYM_DATA_END_LABEL(excp_vect_table, SYM_L_LOCAL, excp_vect_table_end)
#ifndef CONFIG_MMU
arch/riscv/kernel/ftrace.c
+1 -1
View File
@@ -148,7 +148,7 @@ int ftrace_make_nop(struct module *mod, struct dyn_ftrace *rec, unsigned long ad
/*
* This is called early on, and isn't wrapped by
* ftrace_arch_code_modify_{prepare,post_process}() and therefor doesn't hold
* ftrace_arch_code_modify_{prepare,post_process}() and therefore doesn't hold
* text_mutex, which triggers a lockdep failure. SMP isn't running so we could
* just directly poke the text, but it's simpler to just take the lock
* ourselves.
arch/riscv/kernel/head.S
+1 -42
View File
@@ -14,7 +14,6 @@
#include <asm/hwcap.h>
#include <asm/image.h>
#include <asm/scs.h>
#include <asm/xip_fixup.h>
#include <asm/usercfi.h>
#include "efi-header.S"
@@ -75,13 +74,12 @@ pe_head_start:
relocate_enable_mmu:
/* Relocate return address */
la a1, kernel_map
XIP_FIXUP_OFFSET a1
REG_L a1, KERNEL_MAP_VIRT_ADDR(a1)
la a2, _start
sub a1, a1, a2
add ra, ra, a1
/* Point stvec to virtual address of intruction after satp write */
/* Point stvec to virtual address of instruction after satp write */
la a2, 1f
add a2, a2, a1
csrw CSR_TVEC, a2
@@ -89,7 +87,6 @@ relocate_enable_mmu:
/* Compute satp for kernel page tables, but don't load it yet */
srl a2, a0, PAGE_SHIFT
la a1, satp_mode
XIP_FIXUP_OFFSET a1
REG_L a1, 0(a1)
or a2, a2, a1
@@ -100,7 +97,6 @@ relocate_enable_mmu:
* to ensure the new translations are in use.
*/
la a0, trampoline_pg_dir
XIP_FIXUP_OFFSET a0
srl a0, a0, PAGE_SHIFT
or a0, a0, a1
sfence.vma
@@ -154,11 +150,9 @@ secondary_start_sbi:
/* a0 contains the hartid & a1 contains boot data */
li a2, SBI_HART_BOOT_TASK_PTR_OFFSET
XIP_FIXUP_OFFSET a2
add a2, a2, a1
REG_L tp, (a2)
li a3, SBI_HART_BOOT_STACK_PTR_OFFSET
XIP_FIXUP_OFFSET a3
add a3, a3, a1
REG_L sp, (a3)
@@ -167,7 +161,6 @@ secondary_start_sbi:
#ifdef CONFIG_MMU
/* Enable virtual memory and relocate to virtual address */
la a0, swapper_pg_dir
XIP_FIXUP_OFFSET a0
call relocate_enable_mmu
#endif
call .Lsetup_trap_vector
@@ -269,40 +262,13 @@ SYM_CODE_START(_start_kernel)
.Lgood_cores:
/* The lottery system is only required for spinwait booting method */
#ifndef CONFIG_XIP_KERNEL
/* Pick one hart to run the main boot sequence */
la a3, hart_lottery
li a2, 1
amoadd.w a3, a2, (a3)
bnez a3, .Lsecondary_start
#else
/* hart_lottery in flash contains a magic number */
la a3, hart_lottery
mv a2, a3
XIP_FIXUP_OFFSET a2
XIP_FIXUP_FLASH_OFFSET a3
lw t1, (a3)
amoswap.w t0, t1, (a2)
/* first time here if hart_lottery in RAM is not set */
beq t0, t1, .Lsecondary_start
#endif /* CONFIG_XIP */
#endif /* CONFIG_RISCV_BOOT_SPINWAIT */
#ifdef CONFIG_XIP_KERNEL
la sp, _end + THREAD_SIZE
XIP_FIXUP_OFFSET sp
mv s0, a0
mv s1, a1
call __copy_data
/* Restore a0 & a1 copy */
mv a0, s0
mv a1, s1
#endif
#ifndef CONFIG_XIP_KERNEL
/* Clear BSS for flat non-ELF images */
la a3, __bss_start
la a4, __bss_stop
@@ -312,20 +278,16 @@ SYM_CODE_START(_start_kernel)
add a3, a3, RISCV_SZPTR
blt a3, a4, .Lclear_bss
.Lclear_bss_done:
#endif
la a2, boot_cpu_hartid
XIP_FIXUP_OFFSET a2
REG_S a0, (a2)
/* Initialize page tables and relocate to virtual addresses */
la tp, init_task
la sp, init_thread_union + THREAD_SIZE
XIP_FIXUP_OFFSET sp
addi sp, sp, -PT_SIZE_ON_STACK
scs_load_init_stack
#ifdef CONFIG_BUILTIN_DTB
la a0, __dtb_start
XIP_FIXUP_OFFSET a0
#else
mv a0, a1
#endif /* CONFIG_BUILTIN_DTB */
@@ -335,7 +297,6 @@ SYM_CODE_START(_start_kernel)
call setup_vm
#ifdef CONFIG_MMU
la a0, early_pg_dir
XIP_FIXUP_OFFSET a0
call relocate_enable_mmu
#endif /* CONFIG_MMU */
@@ -374,9 +335,7 @@ SYM_CODE_START(_start_kernel)
slli a3, a0, LGREG
la a1, __cpu_spinwait_stack_pointer
XIP_FIXUP_OFFSET a1
la a2, __cpu_spinwait_task_pointer
XIP_FIXUP_OFFSET a2
add a1, a3, a1
add a2, a3, a2
arch/riscv/kernel/head.h
-3
View File
@@ -11,9 +11,6 @@
extern atomic_t hart_lottery;
asmlinkage void __init setup_vm(uintptr_t dtb_pa);
#ifdef CONFIG_XIP_KERNEL
asmlinkage void __init __copy_data(void);
#endif
#ifdef CONFIG_RISCV_BOOT_SPINWAIT
extern void *__cpu_spinwait_stack_pointer[];
arch/riscv/kernel/mcount-dyn.S
+1 -1
View File
@@ -66,7 +66,7 @@
* 8(sp) stores the function return address (i.e. parent IP) that
* can be accessed by &(fregs)->ra in tracing function.
*
* The other regs are saved at the respective localtion and accessed
* The other regs are saved at the respective location and accessed
* by the respective ftrace_regs member.
*
* Here is the layout of stack for your reference.
arch/riscv/kernel/module-sections.c
+1 -1
View File
@@ -148,7 +148,7 @@ int module_frob_arch_sections(Elf_Ehdr *ehdr, Elf_Shdr *sechdrs,
return -ENOEXEC;
}
/* Calculate the maxinum number of entries */
/* Calculate the maximum number of entries */
for (i = 0; i < ehdr->e_shnum; i++) {
size_t num_relas = sechdrs[i].sh_size / sizeof(Elf_Rela);
Elf_Rela *relas = (void *)ehdr + sechdrs[i].sh_offset;
arch/riscv/kernel/probes/kprobes.c
+1 -1
View File
@@ -149,7 +149,7 @@ static void __kprobes set_current_kprobe(struct kprobe *p)
/*
* Interrupts need to be disabled before single-step mode is set, and not
* reenabled until after single-step mode ends.
* re-enabled until after single-step mode ends.
* Without disabling interrupt on local CPU, there is a chance of
* interrupt occurrence in the period of exception return and start of
* out-of-line single-step, that result in wrongly single stepping
arch/riscv/kernel/probes/uprobes.c
+1 -1
View File
@@ -111,7 +111,7 @@ void arch_uprobe_abort_xol(struct arch_uprobe *auprobe, struct pt_regs *regs)
current->thread.bad_cause = utask->autask.saved_cause;
/*
* Task has received a fatal signal, so reset back to probbed
* Task has received a fatal signal, so reset back to probed
* address.
*/
instruction_pointer_set(regs, utask->vaddr);
arch/riscv/kernel/setup.c
+1 -5
View File
@@ -46,11 +46,7 @@
* This is used before the kernel initializes the BSS so it can't be in the
* BSS.
*/
atomic_t hart_lottery __section(".sdata")
#ifdef CONFIG_XIP_KERNEL
= ATOMIC_INIT(0xC001BEEF)
#endif
;
atomic_t hart_lottery __section(".sdata");
unsigned long boot_cpu_hartid;
EXPORT_SYMBOL_GPL(boot_cpu_hartid);
arch/riscv/kernel/smpboot.c
+2 -6
View File
@@ -251,18 +251,14 @@ asmlinkage __visible void smp_callin(void)
set_cpu_online(curr_cpuid, true);
/*
* Remote cache and TLB flushes are ignored while the CPU is offline,
* so flush them both right now just in case.
* Remote instruction cache and TLB flushes are ignored while the CPU
* is offline, so flush them both right now just in case.
*/
local_flush_icache_all();
local_flush_tlb_all();
#ifndef CONFIG_HOTPLUG_PARALLEL
complete(&cpu_running);
#endif
/*
* Disable preemption before enabling interrupts, so we don't try to
* schedule a CPU that hasn't actually started yet.
*/
local_irq_enable();
cpu_startup_entry(CPUHP_AP_ONLINE_IDLE);
}
arch/riscv/kernel/soc.c
+1 -1
View File
@@ -8,7 +8,7 @@
#include <asm/soc.h>
/*
* This is called extremly early, before parse_dtb(), to allow initializing
* This is called extremely early, before parse_dtb(), to allow initializing
* SoC hardware before memory or any device driver initialization.
*/
void __init soc_early_init(void)
arch/riscv/kernel/suspend.c
+1 -1
View File
@@ -78,7 +78,7 @@ int cpu_suspend(unsigned long arg,
suspend_save_csrs(&context);
/*
* Function graph tracer state gets incosistent when the kernel
* Function graph tracer state gets inconsistent when the kernel
* calls functions that never return (aka finishers) hence disable
* graph tracing during their execution.
*/
arch/riscv/kernel/suspend_entry.S
-2
View File
@@ -10,7 +10,6 @@
#include <asm/asm-offsets.h>
#include <asm/assembler.h>
#include <asm/csr.h>
#include <asm/xip_fixup.h>
.text
.altmacro
@@ -70,7 +69,6 @@ SYM_TYPED_FUNC_START(__cpu_resume_enter)
/* Enable MMU */
la a0, swapper_pg_dir
XIP_FIXUP_OFFSET a0
call relocate_enable_mmu
/* Restore A0 and A1 */
arch/riscv/kernel/traps.c
+2 -4
View File
@@ -142,11 +142,7 @@ static void do_trap_error(struct pt_regs *regs, int signo, int code,
}
}
#if defined(CONFIG_XIP_KERNEL) && defined(CONFIG_RISCV_ALTERNATIVE)
#define __trap_section __noinstr_section(".xip.traps")
#else
#define __trap_section noinstr
#endif
#define DO_ERROR_INFO(name, signo, code, str) \
asmlinkage __visible __trap_section void name(struct pt_regs *regs) \
{ \
@@ -165,6 +161,8 @@ asmlinkage __visible __trap_section void name(struct pt_regs *regs) \
DO_ERROR_INFO(do_trap_unknown,
SIGILL, ILL_ILLTRP, "unknown exception");
DO_ERROR_INFO(do_trap_hardware_error,
SIGBUS, BUS_MCEERR_AR, "hardware error");
DO_ERROR_INFO(do_trap_insn_misaligned,
SIGBUS, BUS_ADRALN, "instruction address misaligned");
DO_ERROR_INFO(do_trap_insn_fault,
arch/riscv/kernel/unaligned_access_speed.c
+86 -137
View File
@@ -16,7 +16,7 @@
#include "copy-unaligned.h"
#define MISALIGNED_ACCESS_JIFFIES_LG2 1
#define MISALIGNED_ACCESS_NS 8000000
#define MISALIGNED_BUFFER_SIZE 0x4000
#define MISALIGNED_BUFFER_ORDER get_order(MISALIGNED_BUFFER_SIZE)
#define MISALIGNED_COPY_SIZE ((MISALIGNED_BUFFER_SIZE / 2) - 0x80)
@@ -29,109 +29,125 @@ static long unaligned_vector_speed_param = RISCV_HWPROBE_MISALIGNED_VECTOR_UNKNO
static cpumask_t fast_misaligned_access;
#ifdef CONFIG_RISCV_PROBE_UNALIGNED_ACCESS
static int check_unaligned_access(void *param)
static u64 __maybe_unused
measure_cycles(void (*func)(void *dst, const void *src, size_t len),
void *dst, void *src, size_t len)
{
int cpu = smp_processor_id();
u64 start_cycles, end_cycles;
u64 word_cycles;
u64 byte_cycles;
int ratio;
unsigned long start_jiffies, now;
struct page *page = param;
void *dst;
void *src;
long speed = RISCV_HWPROBE_MISALIGNED_SCALAR_SLOW;
u64 start_cycles, end_cycles, cycles = -1ULL;
u64 start_ns;
if (per_cpu(misaligned_access_speed, cpu) != RISCV_HWPROBE_MISALIGNED_SCALAR_UNKNOWN)
return 0;
/* Make an unaligned destination buffer. */
dst = (void *)((unsigned long)page_address(page) | 0x1);
/* Unalign src as well, but differently (off by 1 + 2 = 3). */
src = dst + (MISALIGNED_BUFFER_SIZE / 2);
src += 2;
word_cycles = -1ULL;
/* Do a warmup. */
__riscv_copy_words_unaligned(dst, src, MISALIGNED_COPY_SIZE);
func(dst, src, len);
preempt_disable();
start_jiffies = jiffies;
while ((now = jiffies) == start_jiffies)
cpu_relax();
/*
* For a fixed amount of time, repeatedly try the function, and take
* the best time in cycles as the measurement.
*/
while (time_before(jiffies, now + (1 << MISALIGNED_ACCESS_JIFFIES_LG2))) {
start_ns = ktime_get_mono_fast_ns();
while (ktime_get_mono_fast_ns() < start_ns + MISALIGNED_ACCESS_NS) {
start_cycles = get_cycles64();
/* Ensure the CSR read can't reorder WRT to the copy. */
mb();
__riscv_copy_words_unaligned(dst, src, MISALIGNED_COPY_SIZE);
func(dst, src, len);
/* Ensure the copy ends before the end time is snapped. */
mb();
end_cycles = get_cycles64();
if ((end_cycles - start_cycles) < word_cycles)
word_cycles = end_cycles - start_cycles;
}
byte_cycles = -1ULL;
__riscv_copy_bytes_unaligned(dst, src, MISALIGNED_COPY_SIZE);
start_jiffies = jiffies;
while ((now = jiffies) == start_jiffies)
cpu_relax();
while (time_before(jiffies, now + (1 << MISALIGNED_ACCESS_JIFFIES_LG2))) {
start_cycles = get_cycles64();
mb();
__riscv_copy_bytes_unaligned(dst, src, MISALIGNED_COPY_SIZE);
mb();
end_cycles = get_cycles64();
if ((end_cycles - start_cycles) < byte_cycles)
byte_cycles = end_cycles - start_cycles;
if ((end_cycles - start_cycles) < cycles)
cycles = end_cycles - start_cycles;
}
preempt_enable();
return cycles;
}
/*
* Return:
* 1 if unaligned accesses are fast
* 0 if unaligned accesses are slow
* -1 if check cannot be done
*/
static int __maybe_unused
compare_unaligned_access(void (*word_copy)(void *dst, const void *src, size_t len),
void (*byte_copy)(void *dst, const void *src, size_t len),
void *buf, const char *type)
{
int cpu = smp_processor_id();
u64 word_cycles;
u64 byte_cycles;
void *dst, *src;
bool fast;
int ratio;
/* Make an unaligned destination buffer. */
dst = (void *)((unsigned long)buf | 0x1);
/* Unalign src as well, but differently (off by 1 + 2 = 3). */
src = dst + (MISALIGNED_BUFFER_SIZE / 2);
src += 2;
word_cycles = measure_cycles(word_copy, dst, src, MISALIGNED_COPY_SIZE);
byte_cycles = measure_cycles(byte_copy, dst, src, MISALIGNED_COPY_SIZE);
/* Don't divide by zero. */
if (!word_cycles || !byte_cycles) {
pr_warn("cpu%d: rdtime lacks granularity needed to measure unaligned access speed\n",
cpu);
pr_warn("cpu%d: rdtime lacks granularity needed to measure %s unaligned access speed\n",
cpu, type);
return 0;
return -1;
}
if (word_cycles < byte_cycles)
speed = RISCV_HWPROBE_MISALIGNED_SCALAR_FAST;
fast = word_cycles < byte_cycles;
ratio = div_u64((byte_cycles * 100), word_cycles);
pr_info("cpu%d: Ratio of byte access time to unaligned word access is %d.%02d, unaligned accesses are %s\n",
pr_info("cpu%d: %s unaligned word access speed is %d.%02dx byte access speed (%s)\n",
cpu,
type,
ratio / 100,
ratio % 100,
(speed == RISCV_HWPROBE_MISALIGNED_SCALAR_FAST) ? "fast" : "slow");
fast ? "fast" : "slow");
per_cpu(misaligned_access_speed, cpu) = speed;
return fast;
}
#ifdef CONFIG_RISCV_PROBE_UNALIGNED_ACCESS
static int check_unaligned_access(struct page *page)
{
void *buf = page_address(page);
int cpu = smp_processor_id();
int ret;
if (per_cpu(misaligned_access_speed, cpu) != RISCV_HWPROBE_MISALIGNED_SCALAR_UNKNOWN)
return 0;
ret = compare_unaligned_access(__riscv_copy_words_unaligned,
__riscv_copy_bytes_unaligned,
buf, "scalar");
if (ret < 0)
return 0;
/*
* Set the value of fast_misaligned_access of a CPU. These operations
* are atomic to avoid race conditions.
*/
if (speed == RISCV_HWPROBE_MISALIGNED_SCALAR_FAST)
if (ret) {
per_cpu(misaligned_access_speed, cpu) = RISCV_HWPROBE_MISALIGNED_SCALAR_FAST;
cpumask_set_cpu(cpu, &fast_misaligned_access);
else
} else {
per_cpu(misaligned_access_speed, cpu) = RISCV_HWPROBE_MISALIGNED_SCALAR_SLOW;
cpumask_clear_cpu(cpu, &fast_misaligned_access);
}
return 0;
}
static void __init check_unaligned_access_nonboot_cpu(void *param)
static void __init _check_unaligned_access(void *param)
{
unsigned int cpu = smp_processor_id();
struct page **pages = param;
if (smp_processor_id() != 0)
check_unaligned_access(pages[cpu]);
check_unaligned_access(pages[cpu]);
}
/* Measure unaligned access speed on all CPUs present at boot in parallel. */
@@ -158,11 +174,7 @@ static void __init check_unaligned_access_speed_all_cpus(void)
}
}
/* Check everybody except 0, who stays behind to tend jiffies. */
on_each_cpu(check_unaligned_access_nonboot_cpu, bufs, 1);
/* Check core 0. */
smp_call_on_cpu(0, check_unaligned_access, bufs[0], true);
on_each_cpu(_check_unaligned_access, bufs, 1);
out:
for_each_cpu(cpu, cpu_online_mask) {
@@ -281,15 +293,8 @@ static int riscv_offline_cpu(unsigned int cpu)
static void check_vector_unaligned_access(struct work_struct *work __always_unused)
{
int cpu = smp_processor_id();
u64 start_cycles, end_cycles;
u64 word_cycles;
u64 byte_cycles;
int ratio;
unsigned long start_jiffies, now;
struct page *page;
void *dst;
void *src;
long speed = RISCV_HWPROBE_MISALIGNED_VECTOR_SLOW;
int ret;
if (per_cpu(vector_misaligned_access, cpu) != RISCV_HWPROBE_MISALIGNED_VECTOR_UNKNOWN)
return;
@@ -300,76 +305,20 @@ static void check_vector_unaligned_access(struct work_struct *work __always_unus
return;
}
/* Make an unaligned destination buffer. */
dst = (void *)((unsigned long)page_address(page) | 0x1);
/* Unalign src as well, but differently (off by 1 + 2 = 3). */
src = dst + (MISALIGNED_BUFFER_SIZE / 2);
src += 2;
word_cycles = -1ULL;
/* Do a warmup. */
kernel_vector_begin();
__riscv_copy_vec_words_unaligned(dst, src, MISALIGNED_COPY_SIZE);
start_jiffies = jiffies;
while ((now = jiffies) == start_jiffies)
cpu_relax();
/*
* For a fixed amount of time, repeatedly try the function, and take
* the best time in cycles as the measurement.
*/
while (time_before(jiffies, now + (1 << MISALIGNED_ACCESS_JIFFIES_LG2))) {
start_cycles = get_cycles64();
/* Ensure the CSR read can't reorder WRT to the copy. */
mb();
__riscv_copy_vec_words_unaligned(dst, src, MISALIGNED_COPY_SIZE);
/* Ensure the copy ends before the end time is snapped. */
mb();
end_cycles = get_cycles64();
if ((end_cycles - start_cycles) < word_cycles)
word_cycles = end_cycles - start_cycles;
}
byte_cycles = -1ULL;
__riscv_copy_vec_bytes_unaligned(dst, src, MISALIGNED_COPY_SIZE);
start_jiffies = jiffies;
while ((now = jiffies) == start_jiffies)
cpu_relax();
while (time_before(jiffies, now + (1 << MISALIGNED_ACCESS_JIFFIES_LG2))) {
start_cycles = get_cycles64();
/* Ensure the CSR read can't reorder WRT to the copy. */
mb();
__riscv_copy_vec_bytes_unaligned(dst, src, MISALIGNED_COPY_SIZE);
/* Ensure the copy ends before the end time is snapped. */
mb();
end_cycles = get_cycles64();
if ((end_cycles - start_cycles) < byte_cycles)
byte_cycles = end_cycles - start_cycles;
}
ret = compare_unaligned_access(__riscv_copy_vec_words_unaligned,
__riscv_copy_vec_bytes_unaligned,
page_address(page), "vector");
kernel_vector_end();
/* Don't divide by zero. */
if (!word_cycles || !byte_cycles) {
pr_warn("cpu%d: rdtime lacks granularity needed to measure unaligned vector access speed\n",
cpu);
if (ret < 0)
goto free;
}
if (word_cycles < byte_cycles)
speed = RISCV_HWPROBE_MISALIGNED_VECTOR_FAST;
ratio = div_u64((byte_cycles * 100), word_cycles);
pr_info("cpu%d: Ratio of vector byte access time to vector unaligned word access is %d.%02d, unaligned accesses are %s\n",
cpu,
ratio / 100,
ratio % 100,
(speed == RISCV_HWPROBE_MISALIGNED_VECTOR_FAST) ? "fast" : "slow");
per_cpu(vector_misaligned_access, cpu) = speed;
if (ret)
per_cpu(vector_misaligned_access, cpu) = RISCV_HWPROBE_MISALIGNED_VECTOR_FAST;
else
per_cpu(vector_misaligned_access, cpu) = RISCV_HWPROBE_MISALIGNED_VECTOR_SLOW;
free:
__free_pages(page, MISALIGNED_BUFFER_ORDER);
@@ -494,4 +443,4 @@ static int __init check_unaligned_access_all_cpus(void)
return 0;
}
arch_initcall(check_unaligned_access_all_cpus);
late_initcall(check_unaligned_access_all_cpus);
arch/riscv/kernel/vdso_cfi/.gitignore
+8
View File
@@ -0,0 +1,8 @@
# SPDX-License-Identifier: GPL-2.0-only
# Copied source files from the main vdso directory
*.c
*.S
!vdso-cfi.S
vdso.lds
*.tmp
vdso-syms.S
arch/riscv/kernel/vdso_cfi/Makefile
+3
View File
@@ -23,3 +23,6 @@ $(vdso_c_objects): $(obj)/%.c: $(src)/%.c
# Include the main VDSO Makefile which contains all the build rules and sources
# The VDSO_CFI_BUILD variable will be passed to it to enable CFI compilation
include $(src)/Makefile
# Clean rules - remove the copied source files
clean-files += $(notdir $(vdso_c_sources)) $(notdir $(vdso_S_sources))
arch/riscv/kernel/vmcore_info.c
+7 -7
View File
@@ -3,6 +3,11 @@
#include <linux/vmcore_info.h>
#include <linux/pagemap.h>
static inline u64 get_satp_value(void)
{
return csr_read(CSR_SATP);
}
void arch_crash_save_vmcoreinfo(void)
{
VMCOREINFO_NUMBER(phys_ram_base);
@@ -19,13 +24,8 @@ void arch_crash_save_vmcoreinfo(void)
#endif
#endif
vmcoreinfo_append_str("NUMBER(KERNEL_LINK_ADDR)=0x%lx\n", KERNEL_LINK_ADDR);
#ifdef CONFIG_XIP_KERNEL
/* TODO: Communicate with crash-utility developers on the information to
* export. The XIP case is more complicated, because the virtual-physical
* address offset depends on whether the address is in ROM or in RAM.
*/
#else
vmcoreinfo_append_str("NUMBER(va_kernel_pa_offset)=0x%lx\n",
kernel_map.va_kernel_pa_offset);
#endif
vmcoreinfo_append_str("KERNELOFFSET=%lx\n", kaslr_offset());
vmcoreinfo_append_str("NUMBER(satp)=0x%llx\n", get_satp_value());
}
arch/riscv/kernel/vmlinux.lds.S
-5
View File
@@ -7,10 +7,6 @@
#define RO_EXCEPTION_TABLE_ALIGN 4
#define RUNTIME_DISCARD_EXIT
#ifdef CONFIG_XIP_KERNEL
#include "vmlinux-xip.lds.S"
#else
#include <asm/pgtable.h>
#define LOAD_OFFSET KERNEL_LINK_ADDR
@@ -176,4 +172,3 @@ SECTIONS
DISCARDS
}
#endif /* CONFIG_XIP_KERNEL */
arch/riscv/kvm/tlb.c
+1 -1
View File
@@ -182,7 +182,7 @@ void kvm_riscv_local_tlb_sanitize(struct kvm_vcpu *vcpu)
/*
* Flush VS-stage TLB entries for implementation where VS-stage
* TLB does not cahce guest physical address and VMID.
* TLB does not cache guest physical address and VMID.
*/
if (static_branch_unlikely(&kvm_riscv_vsstage_tlb_no_gpa))
kvm_riscv_local_hfence_vvma_all(vmid);
arch/riscv/kvm/vcpu_pmu.c
+1 -1
View File
@@ -694,7 +694,7 @@ int kvm_riscv_vcpu_pmu_ctr_stop(struct kvm_vcpu *vcpu, unsigned long ctr_base,
pmc->counter_val += perf_event_read_value(pmc->perf_event,
&enabled, &running);
/*
* The counter and overflow indicies in the snapshot region are w.r.to
* The counter and overflow indices in the snapshot region are w.r.to
* cbase. Modify the set bit in the counter mask instead of the pmc_index
* which indicates the absolute counter index.
*/
arch/riscv/lib/Makefile
+3
View File
@@ -7,6 +7,9 @@ ifeq ($(CONFIG_KASAN_GENERIC)$(CONFIG_KASAN_SW_TAGS),)
lib-y += strcmp.o
lib-y += strlen.o
lib-y += strncmp.o
lib-y += strnlen.o
lib-y += strchr.o
lib-y += strrchr.o
endif
lib-y += csum.o
ifeq ($(CONFIG_MMU), y)
arch/riscv/lib/csum.c
+1 -1
View File
@@ -269,7 +269,7 @@ unsigned int do_csum(const unsigned char *buff, int len)
* on machines with fast misaligned accesses.
*
* There is some duplicate code between the "with_alignment" and
* "no_alignment" implmentations, but the overlap is too awkward to be
* "no_alignment" implementations, but the overlap is too awkward to be
* able to fit in one function without introducing multiple static
* branches. The largest chunk of overlap was delegated into the
* do_csum_common function.
arch/riscv/lib/memmove.S
+2 -2
View File
@@ -40,8 +40,8 @@ SYM_FUNC_START(__memmove)
* Both Copy Modes: t1 - Temporary for load-store
* Both Copy Modes: t2 - Temporary for load-store
* Both Copy Modes: a5 - dest to src alignment offset
* Both Copy Modes: a6 - Shift ammount
* Both Copy Modes: a7 - Inverse Shift ammount
* Both Copy Modes: a6 - Shift amount
* Both Copy Modes: a7 - Inverse Shift amount
* Both Copy Modes: a2 - Alternate breakpoint for unrolled loops
*/
arch/riscv/lib/strchr.S
+35
View File
@@ -0,0 +1,35 @@
/* SPDX-License-Identifier: GPL-2.0-only */
/*
* Copyright (C) 2025 Feng Jiang <jiangfeng@kylinos.cn>
*/
#include <linux/linkage.h>
#include <asm/asm.h>
/* char *strchr(const char *s, int c) */
SYM_FUNC_START(strchr)
/*
* Parameters
* a0 - The string to be searched
* a1 - The character to search for
*
* Returns
* a0 - Address of first occurrence of 'c' or 0
*
* Clobbers
* t0
*/
andi a1, a1, 0xff
1:
lbu t0, 0(a0)
beq t0, a1, 2f
addi a0, a0, 1
bnez t0, 1b
li a0, 0
2:
ret
SYM_FUNC_END(strchr)
SYM_FUNC_ALIAS_WEAK(__pi_strchr, strchr)
EXPORT_SYMBOL(strchr)
arch/riscv/lib/strnlen.S
+164
View File
@@ -0,0 +1,164 @@
/* SPDX-License-Identifier: GPL-2.0-only */
/*
* Base on arch/riscv/lib/strlen.S
*
* Copyright (C) Feng Jiang <jiangfeng@kylinos.cn>
*/
#include <linux/linkage.h>
#include <asm/asm.h>
#include <asm/alternative-macros.h>
#include <asm/hwcap.h>
/* size_t strnlen(const char *s, size_t count) */
SYM_FUNC_START(strnlen)
__ALTERNATIVE_CFG("nop", "j strnlen_zbb", 0, RISCV_ISA_EXT_ZBB,
IS_ENABLED(CONFIG_RISCV_ISA_ZBB) && IS_ENABLED(CONFIG_TOOLCHAIN_HAS_ZBB))
/*
* Returns
* a0 - String length
*
* Parameters
* a0 - String to measure
* a1 - Max length of string
*
* Clobbers
* t0, t1, t2
*/
addi t1, a0, -1
add t2, a0, a1
1:
addi t1, t1, 1
beq t1, t2, 2f
lbu t0, 0(t1)
bnez t0, 1b
2:
sub a0, t1, a0
ret
/*
* Variant of strnlen using the ZBB extension if available
*/
#if defined(CONFIG_RISCV_ISA_ZBB) && defined(CONFIG_TOOLCHAIN_HAS_ZBB)
strnlen_zbb:
#ifdef CONFIG_CPU_BIG_ENDIAN
# define CZ clz
# define SHIFT sll
#else
# define CZ ctz
# define SHIFT srl
#endif
.option push
.option arch,+zbb
/*
* Returns
* a0 - String length
*
* Parameters
* a0 - String to measure
* a1 - Max length of string
*
* Clobbers
* t0, t1, t2, t3, t4
*/
/* If maxlen is 0, return 0. */
beqz a1, 3f
/* Number of irrelevant bytes in the first word. */
andi t2, a0, SZREG-1
/* Align pointer. */
andi t0, a0, -SZREG
li t3, SZREG
sub t3, t3, t2
slli t2, t2, 3
/* Aligned boundary. */
add t4, a0, a1
andi t4, t4, -SZREG
/* Get the first word. */
REG_L t1, 0(t0)
/*
* Shift away the partial data we loaded to remove the irrelevant bytes
* preceding the string with the effect of adding NUL bytes at the
* end of the string's first word.
*/
SHIFT t1, t1, t2
/* Convert non-NUL into 0xff and NUL into 0x00. */
orc.b t1, t1
/* Convert non-NUL into 0x00 and NUL into 0xff. */
not t1, t1
/*
* Search for the first set bit (corresponding to a NUL byte in the
* original chunk).
*/
CZ t1, t1
/*
* The first chunk is special: compare against the number
* of valid bytes in this chunk.
*/
srli a0, t1, 3
/* Limit the result by maxlen. */
minu a0, a0, a1
bgtu t3, a0, 2f
/* Prepare for the word comparison loop. */
addi t2, t0, SZREG
li t3, -1
/*
* Our critical loop is 4 instructions and processes data in
* 4 byte or 8 byte chunks.
*/
.p2align 3
1:
REG_L t1, SZREG(t0)
addi t0, t0, SZREG
orc.b t1, t1
bgeu t0, t4, 4f
beq t1, t3, 1b
4:
not t1, t1
CZ t1, t1
srli t1, t1, 3
/* Get number of processed bytes. */
sub t2, t0, t2
/* Add number of characters in the first word. */
add a0, a0, t2
/* Add number of characters in the last word. */
add a0, a0, t1
/* Ensure the final result does not exceed maxlen. */
minu a0, a0, a1
2:
ret
3:
mv a0, a1
ret
.option pop
#endif
SYM_FUNC_END(strnlen)
SYM_FUNC_ALIAS(__pi_strnlen, strnlen)
EXPORT_SYMBOL(strnlen)
arch/riscv/lib/strrchr.S
+37
View File
@@ -0,0 +1,37 @@
/* SPDX-License-Identifier: GPL-2.0-only */
/*
* Copyright (C) 2025 Feng Jiang <jiangfeng@kylinos.cn>
*/
#include <linux/linkage.h>
#include <asm/asm.h>
/* char *strrchr(const char *s, int c) */
SYM_FUNC_START(strrchr)
/*
* Parameters
* a0 - The string to be searched
* a1 - The character to seaerch for
*
* Returns
* a0 - Address of last occurrence of 'c' or 0
*
* Clobbers
* t0, t1
*/
andi a1, a1, 0xff
mv t1, a0
li a0, 0
1:
lbu t0, 0(t1)
bne t0, a1, 2f
mv a0, t1
2:
addi t1, t1, 1
bnez t0, 1b
ret
SYM_FUNC_END(strrchr)
SYM_FUNC_ALIAS_WEAK(__pi_strrchr, strrchr)
EXPORT_SYMBOL(strrchr)
arch/riscv/mm/cacheflush.c
+1 -1
View File
@@ -29,7 +29,7 @@ void flush_icache_all(void)
* Make sure all previous writes to the D$ are ordered before making
* the IPI. The RISC-V spec states that a hart must execute a data fence
* before triggering a remote fence.i in order to make the modification
* visable for remote harts.
* visible for remote harts.
*
* IPIs on RISC-V are triggered by MMIO writes to either CLINT or
* S-IMSIC, so the fence ensures previous data writes "happen before"
arch/riscv/mm/init.c
+10 -108
View File
@@ -41,20 +41,17 @@ u64 new_vmalloc[NR_CPUS / sizeof(u64) + 1];
struct kernel_mapping kernel_map __ro_after_init;
EXPORT_SYMBOL(kernel_map);
#ifdef CONFIG_XIP_KERNEL
#define kernel_map (*(struct kernel_mapping *)XIP_FIXUP(&kernel_map))
#endif
#ifdef CONFIG_64BIT
u64 satp_mode __ro_after_init = !IS_ENABLED(CONFIG_XIP_KERNEL) ? SATP_MODE_57 : SATP_MODE_39;
u64 satp_mode __ro_after_init = SATP_MODE_57;
#else
u64 satp_mode __ro_after_init = SATP_MODE_32;
#endif
EXPORT_SYMBOL(satp_mode);
#ifdef CONFIG_64BIT
bool pgtable_l4_enabled __ro_after_init = !IS_ENABLED(CONFIG_XIP_KERNEL);
bool pgtable_l5_enabled __ro_after_init = !IS_ENABLED(CONFIG_XIP_KERNEL);
bool pgtable_l4_enabled __ro_after_init = true;
bool pgtable_l5_enabled __ro_after_init = true;
EXPORT_SYMBOL(pgtable_l4_enabled);
EXPORT_SYMBOL(pgtable_l5_enabled);
#endif
@@ -190,9 +187,6 @@ void __init arch_mm_preinit(void)
/* Limit the memory size via mem. */
static phys_addr_t memory_limit;
#ifdef CONFIG_XIP_KERNEL
#define memory_limit (*(phys_addr_t *)XIP_FIXUP(&memory_limit))
#endif /* CONFIG_XIP_KERNEL */
static int __init early_mem(char *p)
{
@@ -216,10 +210,7 @@ static void __init setup_bootmem(void)
phys_addr_t max_mapped_addr;
phys_addr_t phys_ram_end, vmlinux_start;
if (IS_ENABLED(CONFIG_XIP_KERNEL))
vmlinux_start = __pa_symbol(&_sdata);
else
vmlinux_start = __pa_symbol(&_start);
vmlinux_start = __pa_symbol(&_start);
memblock_enforce_memory_limit(memory_limit);
@@ -239,12 +230,10 @@ static void __init setup_bootmem(void)
* Make sure we align the start of the memory on a PMD boundary so that
* at worst, we map the linear mapping with PMD mappings.
*/
if (!IS_ENABLED(CONFIG_XIP_KERNEL)) {
phys_ram_base = memblock_start_of_DRAM() & PMD_MASK;
phys_ram_base = memblock_start_of_DRAM() & PMD_MASK;
#ifdef CONFIG_SPARSEMEM_VMEMMAP
vmemmap_start_pfn = round_down(phys_ram_base, VMEMMAP_ADDR_ALIGN) >> PAGE_SHIFT;
vmemmap_start_pfn = round_down(phys_ram_base, VMEMMAP_ADDR_ALIGN) >> PAGE_SHIFT;
#endif
}
/*
* In 64-bit, any use of __va/__pa before this point is wrong as we
@@ -357,13 +346,6 @@ static pte_t fixmap_pte[PTRS_PER_PTE] __page_aligned_bss;
pgd_t early_pg_dir[PTRS_PER_PGD] __initdata __aligned(PAGE_SIZE);
#ifdef CONFIG_XIP_KERNEL
#define pt_ops (*(struct pt_alloc_ops *)XIP_FIXUP(&pt_ops))
#define trampoline_pg_dir ((pgd_t *)XIP_FIXUP(trampoline_pg_dir))
#define fixmap_pte ((pte_t *)XIP_FIXUP(fixmap_pte))
#define early_pg_dir ((pgd_t *)XIP_FIXUP(early_pg_dir))
#endif /* CONFIG_XIP_KERNEL */
static const pgprot_t protection_map[16] = {
[VM_NONE] = PAGE_NONE,
[VM_READ] = PAGE_READ,
@@ -460,32 +442,14 @@ static pmd_t trampoline_pmd[PTRS_PER_PMD] __page_aligned_bss;
static pmd_t fixmap_pmd[PTRS_PER_PMD] __page_aligned_bss;
static pmd_t early_pmd[PTRS_PER_PMD] __initdata __aligned(PAGE_SIZE);
#ifdef CONFIG_XIP_KERNEL
#define trampoline_pmd ((pmd_t *)XIP_FIXUP(trampoline_pmd))
#define fixmap_pmd ((pmd_t *)XIP_FIXUP(fixmap_pmd))
#define early_pmd ((pmd_t *)XIP_FIXUP(early_pmd))
#endif /* CONFIG_XIP_KERNEL */
static p4d_t trampoline_p4d[PTRS_PER_P4D] __page_aligned_bss;
static p4d_t fixmap_p4d[PTRS_PER_P4D] __page_aligned_bss;
static p4d_t early_p4d[PTRS_PER_P4D] __initdata __aligned(PAGE_SIZE);
#ifdef CONFIG_XIP_KERNEL
#define trampoline_p4d ((p4d_t *)XIP_FIXUP(trampoline_p4d))
#define fixmap_p4d ((p4d_t *)XIP_FIXUP(fixmap_p4d))
#define early_p4d ((p4d_t *)XIP_FIXUP(early_p4d))
#endif /* CONFIG_XIP_KERNEL */
static pud_t trampoline_pud[PTRS_PER_PUD] __page_aligned_bss;
static pud_t fixmap_pud[PTRS_PER_PUD] __page_aligned_bss;
static pud_t early_pud[PTRS_PER_PUD] __initdata __aligned(PAGE_SIZE);
#ifdef CONFIG_XIP_KERNEL
#define trampoline_pud ((pud_t *)XIP_FIXUP(trampoline_pud))
#define fixmap_pud ((pud_t *)XIP_FIXUP(fixmap_pud))
#define early_pud ((pud_t *)XIP_FIXUP(early_pud))
#endif /* CONFIG_XIP_KERNEL */
static pmd_t *__init get_pmd_virt_early(phys_addr_t pa)
{
/* Before MMU is enabled */
@@ -756,21 +720,6 @@ static uintptr_t __meminit best_map_size(phys_addr_t pa, uintptr_t va, phys_addr
return PAGE_SIZE;
}
#ifdef CONFIG_XIP_KERNEL
#define phys_ram_base (*(phys_addr_t *)XIP_FIXUP(&phys_ram_base))
extern char _xiprom[], _exiprom[], __data_loc;
/* called from head.S with MMU off */
asmlinkage void __init __copy_data(void)
{
void *from = (void *)(&__data_loc);
void *to = (void *)CONFIG_PHYS_RAM_BASE;
size_t sz = (size_t)((uintptr_t)(&_end) - (uintptr_t)(&_sdata));
memcpy(to, from, sz);
}
#endif
#ifdef CONFIG_STRICT_KERNEL_RWX
static __meminit pgprot_t pgprot_from_va(uintptr_t va)
{
@@ -806,7 +755,7 @@ static __meminit pgprot_t pgprot_from_va(uintptr_t va)
}
#endif /* CONFIG_STRICT_KERNEL_RWX */
#if defined(CONFIG_64BIT) && !defined(CONFIG_XIP_KERNEL)
#if defined(CONFIG_64BIT)
u64 __pi_set_satp_mode_from_cmdline(uintptr_t dtb_pa);
u64 __pi_set_satp_mode_from_fdt(uintptr_t dtb_pa);
@@ -931,28 +880,6 @@ retry:
#error "setup_vm() is called from head.S before relocate so it should not use absolute addressing."
#endif
#ifdef CONFIG_XIP_KERNEL
static void __init create_kernel_page_table(pgd_t *pgdir,
__always_unused bool early)
{
uintptr_t va, start_va, end_va;
/* Map the flash resident part */
end_va = kernel_map.virt_addr + kernel_map.xiprom_sz;
for (va = kernel_map.virt_addr; va < end_va; va += PMD_SIZE)
create_pgd_mapping(pgdir, va,
kernel_map.xiprom + (va - kernel_map.virt_addr),
PMD_SIZE, PAGE_KERNEL_EXEC);
/* Map the data in RAM */
start_va = kernel_map.virt_addr + (uintptr_t)&_sdata - (uintptr_t)&_start;
end_va = kernel_map.virt_addr + kernel_map.size;
for (va = start_va; va < end_va; va += PMD_SIZE)
create_pgd_mapping(pgdir, va,
kernel_map.phys_addr + (va - start_va),
PMD_SIZE, PAGE_KERNEL);
}
#else
static void __init create_kernel_page_table(pgd_t *pgdir, bool early)
{
uintptr_t va, end_va;
@@ -965,7 +892,6 @@ static void __init create_kernel_page_table(pgd_t *pgdir, bool early)
early ?
PAGE_KERNEL_EXEC : pgprot_from_va(va));
}
#endif
/*
* Setup a 4MB mapping that encompasses the device tree: for 64-bit kernel,
@@ -1075,11 +1001,6 @@ static int __init print_nokaslr(char *p)
return 0;
}
early_param("nokaslr", print_nokaslr);
unsigned long kaslr_offset(void)
{
return kernel_map.virt_offset;
}
#endif
asmlinkage void __init setup_vm(uintptr_t dtb_pa)
@@ -1107,27 +1028,11 @@ asmlinkage void __init setup_vm(uintptr_t dtb_pa)
kernel_map.virt_addr = KERNEL_LINK_ADDR + kernel_map.virt_offset;
#ifdef CONFIG_XIP_KERNEL
kernel_map.xiprom = (uintptr_t)CONFIG_XIP_PHYS_ADDR;
kernel_map.xiprom_sz = (uintptr_t)(&_exiprom) - (uintptr_t)(&_xiprom);
phys_ram_base = CONFIG_PHYS_RAM_BASE;
#ifdef CONFIG_SPARSEMEM_VMEMMAP
vmemmap_start_pfn = round_down(phys_ram_base, VMEMMAP_ADDR_ALIGN) >> PAGE_SHIFT;
#endif
kernel_map.phys_addr = (uintptr_t)CONFIG_PHYS_RAM_BASE;
kernel_map.size = (uintptr_t)(&_end) - (uintptr_t)(&_start);
kernel_map.va_kernel_xip_text_pa_offset = kernel_map.virt_addr - kernel_map.xiprom;
kernel_map.va_kernel_xip_data_pa_offset = kernel_map.virt_addr - kernel_map.phys_addr
+ (uintptr_t)&_sdata - (uintptr_t)&_start;
#else
kernel_map.phys_addr = (uintptr_t)(&_start);
kernel_map.size = (uintptr_t)(&_end) - kernel_map.phys_addr;
kernel_map.va_kernel_pa_offset = kernel_map.virt_addr - kernel_map.phys_addr;
#endif
#if defined(CONFIG_64BIT) && !defined(CONFIG_XIP_KERNEL)
#if defined(CONFIG_64BIT)
set_satp_mode(dtb_pa);
set_mmap_rnd_bits_max();
#endif
@@ -1200,13 +1105,8 @@ asmlinkage void __init setup_vm(uintptr_t dtb_pa)
if (pgtable_l4_enabled)
create_pud_mapping(trampoline_pud, kernel_map.virt_addr,
(uintptr_t)trampoline_pmd, PUD_SIZE, PAGE_TABLE);
#ifdef CONFIG_XIP_KERNEL
create_pmd_mapping(trampoline_pmd, kernel_map.virt_addr,
kernel_map.xiprom, PMD_SIZE, PAGE_KERNEL_EXEC);
#else
create_pmd_mapping(trampoline_pmd, kernel_map.virt_addr,
kernel_map.phys_addr, PMD_SIZE, PAGE_KERNEL_EXEC);
#endif
#else
/* Setup trampoline PGD */
create_pgd_mapping(trampoline_pg_dir, kernel_map.virt_addr,
@@ -1450,6 +1350,8 @@ int __meminit vmemmap_check_pmd(pmd_t *pmdp, int node,
int __meminit vmemmap_populate(unsigned long start, unsigned long end, int node,
struct vmem_altmap *altmap)
{
WARN_ON((start < VMEMMAP_START) || (end > VMEMMAP_END));
/*
* Note that SPARSEMEM_VMEMMAP is only selected for rv64 and that we
* can't use hugepage mappings for 2-level page table because in case of
arch/riscv/mm/physaddr.c
+2 -2
View File
@@ -9,7 +9,7 @@
phys_addr_t __virt_to_phys(unsigned long x)
{
/*
* Boundary checking aginst the kernel linear mapping space.
* Boundary checking against the kernel linear mapping space.
*/
WARN(!is_linear_mapping(x) && !is_kernel_mapping(x),
"virt_to_phys used for non-linear address: %p (%pS)\n",
@@ -25,7 +25,7 @@ phys_addr_t __phys_addr_symbol(unsigned long x)
unsigned long kernel_end = kernel_start + kernel_map.size;
/*
* Boundary checking aginst the kernel image mapping.
* Boundary checking against the kernel image mapping.
* __pa_symbol should only be used on kernel symbol addresses.
*/
VIRTUAL_BUG_ON(x < kernel_start || x > kernel_end);
arch/riscv/purgatory/Makefile
+10 -1
View File
@@ -2,7 +2,7 @@
purgatory-y := purgatory.o sha256.o entry.o string.o ctype.o memcpy.o memset.o
ifeq ($(CONFIG_KASAN_GENERIC)$(CONFIG_KASAN_SW_TAGS),)
purgatory-y += strcmp.o strlen.o strncmp.o
purgatory-y += strcmp.o strlen.o strncmp.o strnlen.o strchr.o strrchr.o
endif
targets += $(purgatory-y)
@@ -32,6 +32,15 @@ $(obj)/strncmp.o: $(srctree)/arch/riscv/lib/strncmp.S FORCE
$(obj)/sha256.o: $(srctree)/lib/crypto/sha256.c FORCE
$(call if_changed_rule,cc_o_c)
$(obj)/strnlen.o: $(srctree)/arch/riscv/lib/strnlen.S FORCE
$(call if_changed_rule,as_o_S)
$(obj)/strchr.o: $(srctree)/arch/riscv/lib/strchr.S FORCE
$(call if_changed_rule,as_o_S)
$(obj)/strrchr.o: $(srctree)/arch/riscv/lib/strrchr.S FORCE
$(call if_changed_rule,as_o_S)
CFLAGS_sha256.o := -D__DISABLE_EXPORTS -D__NO_FORTIFY
CFLAGS_string.o := -D__DISABLE_EXPORTS
CFLAGS_ctype.o := -D__DISABLE_EXPORTS
arch/riscv/purgatory/kexec-purgatory.S
+2 -2
View File
@@ -6,9 +6,9 @@
kexec_purgatory:
.globl kexec_purgatory
.incbin "arch/riscv/purgatory/purgatory.ro"
.Lkexec_purgatroy_end:
.Lkexec_purgatory_end:
.align 8
kexec_purgatory_size:
.globl kexec_purgatory_size
.quad .Lkexec_purgatroy_end - kexec_purgatory
.quad .Lkexec_purgatory_end - kexec_purgatory
lib/Kconfig.debug
+11
View File
@@ -2526,6 +2526,17 @@ config STRING_HELPERS_KUNIT_TEST
depends on KUNIT
default KUNIT_ALL_TESTS
config STRING_KUNIT_BENCH
bool "Benchmark string functions at runtime"
depends on STRING_KUNIT_TEST
help
Enable performance measurement for string functions.
This measures the execution efficiency of string functions
during the KUnit test run.
If unsure, say N.
config FFS_KUNIT_TEST
tristate "KUnit test ffs-family functions at runtime" if !KUNIT_ALL_TESTS
depends on KUNIT
lib/tests/string_kunit.c
+274
View File
@@ -6,10 +6,18 @@
#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
#include <kunit/test.h>
#include <linux/ktime.h>
#include <linux/math64.h>
#include <linux/minmax.h>
#include <linux/mm.h>
#include <linux/module.h>
#include <linux/prandom.h>
#include <linux/printk.h>
#include <linux/slab.h>
#include <linux/string.h>
#include <linux/time64.h>
#include <linux/units.h>
#include <linux/vmalloc.h>
#define STRCMP_LARGE_BUF_LEN 2048
#define STRCMP_CHANGE_POINT 1337
@@ -17,6 +25,12 @@
#define STRCMP_TEST_EXPECT_LOWER(test, fn, ...) KUNIT_EXPECT_LT(test, fn(__VA_ARGS__), 0)
#define STRCMP_TEST_EXPECT_GREATER(test, fn, ...) KUNIT_EXPECT_GT(test, fn(__VA_ARGS__), 0)
#define STRING_TEST_MAX_LEN 128
#define STRING_TEST_MAX_OFFSET 16
#define STRING_BENCH_SEED 888
#define STRING_BENCH_WORKLOAD (1 * MEGA)
static void string_test_memset16(struct kunit *test)
{
unsigned i, j, k;
@@ -104,6 +118,64 @@ static void string_test_memset64(struct kunit *test)
}
}
static void string_test_strlen(struct kunit *test)
{
size_t buf_size;
char *buf, *s;
buf_size = PAGE_ALIGN(STRING_TEST_MAX_LEN + STRING_TEST_MAX_OFFSET + 1);
buf = vmalloc(buf_size);
KUNIT_ASSERT_NOT_ERR_OR_NULL(test, buf);
memset(buf, 'A', buf_size);
for (size_t offset = 0; offset < STRING_TEST_MAX_OFFSET; offset++) {
for (size_t len = 0; len <= STRING_TEST_MAX_LEN; len++) {
s = buf + buf_size - 1 - offset - len;
s[len] = '\0';
KUNIT_EXPECT_EQ_MSG(test, strlen(s), len,
"offset:%zu len:%zu", offset, len);
s[len] = 'A';
}
}
vfree(buf);
}
static void string_test_strnlen(struct kunit *test)
{
size_t buf_size;
char *buf, *s;
buf_size = PAGE_ALIGN(STRING_TEST_MAX_LEN + STRING_TEST_MAX_OFFSET + 1);
buf = vmalloc(buf_size);
KUNIT_ASSERT_NOT_ERR_OR_NULL(test, buf);
memset(buf, 'A', buf_size);
for (size_t offset = 0; offset < STRING_TEST_MAX_OFFSET; offset++) {
for (size_t len = 0; len <= STRING_TEST_MAX_LEN; len++) {
s = buf + buf_size - 1 - offset - len;
s[len] = '\0';
if (len > 0)
KUNIT_EXPECT_EQ(test, strnlen(s, len - 1), len - 1);
if (len > 1)
KUNIT_EXPECT_EQ(test, strnlen(s, len - 2), len - 2);
KUNIT_EXPECT_EQ(test, strnlen(s, len), len);
KUNIT_EXPECT_EQ(test, strnlen(s, len + 1), len);
KUNIT_EXPECT_EQ(test, strnlen(s, len + 2), len);
KUNIT_EXPECT_EQ(test, strnlen(s, len + 10), len);
s[len] = 'A';
}
}
vfree(buf);
}
static void string_test_strchr(struct kunit *test)
{
const char *test_string = "abcdefghijkl";
@@ -127,6 +199,36 @@ static void string_test_strchr(struct kunit *test)
KUNIT_ASSERT_NULL(test, result);
}
static void string_test_strrchr(struct kunit *test)
{
size_t buf_size;
char *buf, *s;
buf_size = PAGE_ALIGN(STRING_TEST_MAX_LEN + STRING_TEST_MAX_OFFSET + 1);
buf = vmalloc(buf_size);
KUNIT_ASSERT_NOT_ERR_OR_NULL(test, buf);
memset(buf, 'A', buf_size);
for (size_t offset = 0; offset < STRING_TEST_MAX_OFFSET; offset++) {
for (size_t len = 0; len <= STRING_TEST_MAX_LEN; len++) {
s = buf + buf_size - 1 - offset - len;
s[len] = '\0';
KUNIT_EXPECT_PTR_EQ(test, strrchr(s, 'Z'), NULL);
if (len > 0)
KUNIT_EXPECT_PTR_EQ(test, strrchr(s, 'A'), s + len - 1);
else
KUNIT_EXPECT_PTR_EQ(test, strrchr(s, 'A'), NULL);
s[len] = 'A';
}
}
vfree(buf);
}
static void string_test_strnchr(struct kunit *test)
{
const char *test_string = "abcdefghijkl";
@@ -614,12 +716,180 @@ static void string_test_strends(struct kunit *test)
KUNIT_EXPECT_TRUE(test, strends("", ""));
}
#if IS_ENABLED(CONFIG_STRING_KUNIT_BENCH)
/* Target string lengths for benchmarking */
static const size_t bench_lens[] = {
0, 1, 7, 8, 16, 31, 64, 127, 512, 1024, 3173, 4096,
};
/**
* alloc_max_bench_buffer() - Allocate buffer for the max test case.
* @test: KUnit context for managed allocation.
* @lens: Array of lengths used in the benchmark cases.
* @count: Number of elements in the @lens array.
* @buf_len: [out] Pointer to store the actually allocated buffer
* size (including NUL character).
*
* Return: Pointer to the allocated memory, or NULL on failure.
*/
static void *alloc_max_bench_buffer(struct kunit *test, const size_t *lens,
size_t count, size_t *buf_len)
{
size_t max_len = 0;
void *buf;
for (size_t i = 0; i < count; i++)
max_len = max(lens[i], max_len);
/* Add space for NUL character */
max_len += 1;
buf = kunit_kzalloc(test, max_len, GFP_KERNEL);
if (!buf)
return NULL;
if (buf_len)
*buf_len = max_len;
return buf;
}
/**
* fill_random_string() - Populate a buffer with a random NUL-terminated string.
* @buf: Buffer to fill.
* @len: Length of the buffer in bytes.
*
* Fills the buffer with random non-NUL bytes and ensures the string is
* properly NUL-terminated.
*/
static void fill_random_string(char *buf, size_t len)
{
struct rnd_state state;
if (!buf || !len)
return;
/* Use a fixed seed to ensure deterministic benchmark results */
prandom_seed_state(&state, STRING_BENCH_SEED);
prandom_bytes_state(&state, buf, len);
/* Replace NUL characters to avoid early string termination */
for (size_t i = 0; i < len; i++) {
if (buf[i] == '\0')
buf[i] = 0x01;
}
buf[len - 1] = '\0';
}
/**
* STRING_BENCH() - Benchmark string functions.
* @iters: Number of iterations to run.
* @func: Function to benchmark.
* @...: Variable arguments passed to @func.
*
* Disables preemption and measures the total time in nanoseconds to execute
* @func(@__VA_ARGS__) for @iters times, including a small warm-up phase.
*
* Context: Disables preemption during measurement.
* Return: Total execution time in nanoseconds (u64).
*/
#define STRING_BENCH(iters, func, ...) \
({ \
/* Volatile function pointer prevents dead code elimination */ \
typeof(func) (* volatile __func) = (func); \
size_t __bn_iters = (iters); \
size_t __bn_warm_iters; \
u64 __bn_t; \
\
/* Use 10% of the given iterations (maximum 50) to warm up */ \
__bn_warm_iters = max(__bn_iters / 10, 50U); \
\
for (size_t __bn_i = 0; __bn_i < __bn_warm_iters; __bn_i++) \
(void)__func(__VA_ARGS__); \
\
preempt_disable(); \
__bn_t = ktime_get_ns(); \
for (size_t __bn_i = 0; __bn_i < __bn_iters; __bn_i++) \
(void)__func(__VA_ARGS__); \
__bn_t = ktime_get_ns() - __bn_t; \
preempt_enable(); \
__bn_t; \
})
/**
* STRING_BENCH_BUF() - Benchmark harness for single-buffer functions.
* @test: KUnit context.
* @buf_name: Local char * variable name to be defined.
* @buf_size: Local size_t variable name to be defined.
* @func: Function to benchmark.
* @...: Extra arguments for @func.
*
* Prepares a randomized, NUL-terminated buffer and iterates through lengths
* in bench_lens, defining @buf_name and @buf_size in each loop.
*/
#define STRING_BENCH_BUF(test, buf_name, buf_size, func, ...) \
do { \
size_t _bn_i, _bn_iters, _bn_size = 0; \
u64 _bn_t, _bn_mbps = 0, _bn_lat = 0; \
char *_bn_buf; \
\
_bn_buf = alloc_max_bench_buffer(test, bench_lens, \
ARRAY_SIZE(bench_lens), &_bn_size); \
KUNIT_ASSERT_NOT_ERR_OR_NULL(test, _bn_buf); \
\
fill_random_string(_bn_buf, _bn_size); \
\
for (_bn_i = 0; _bn_i < ARRAY_SIZE(bench_lens); _bn_i++) { \
size_t buf_size = bench_lens[_bn_i]; \
char *buf_name = _bn_buf + _bn_size - buf_size - 1; \
_bn_iters = STRING_BENCH_WORKLOAD / max(buf_size, 1U); \
\
_bn_t = STRING_BENCH(_bn_iters, func, ##__VA_ARGS__); \
if (_bn_t > 0) { \
_bn_mbps = (u64)(buf_size) * _bn_iters * \
(NSEC_PER_SEC / MEGA); \
_bn_mbps = div64_u64(_bn_mbps, _bn_t); \
_bn_lat = div64_u64(_bn_t, _bn_iters); \
} \
kunit_info(test, "len=%zu: %llu MB/s (%llu ns/call)\n", \
buf_size, _bn_mbps, _bn_lat); \
} \
} while (0)
#else
#define STRING_BENCH_BUF(test, buf_name, buf_size, func, ...) \
kunit_skip(test, "not enabled")
#endif /* IS_ENABLED(CONFIG_STRING_KUNIT_BENCH) */
static void string_bench_strlen(struct kunit *test)
{
STRING_BENCH_BUF(test, buf, len, strlen, buf);
}
static void string_bench_strnlen(struct kunit *test)
{
STRING_BENCH_BUF(test, buf, len, strnlen, buf, len);
}
static void string_bench_strchr(struct kunit *test)
{
STRING_BENCH_BUF(test, buf, len, strchr, buf, '\0');
}
static void string_bench_strrchr(struct kunit *test)
{
STRING_BENCH_BUF(test, buf, len, strrchr, buf, '\0');
}
static struct kunit_case string_test_cases[] = {
KUNIT_CASE(string_test_memset16),
KUNIT_CASE(string_test_memset32),
KUNIT_CASE(string_test_memset64),
KUNIT_CASE(string_test_strlen),
KUNIT_CASE(string_test_strnlen),
KUNIT_CASE(string_test_strchr),
KUNIT_CASE(string_test_strnchr),
KUNIT_CASE(string_test_strrchr),
KUNIT_CASE(string_test_strspn),
KUNIT_CASE(string_test_strcmp),
KUNIT_CASE(string_test_strcmp_long_strings),
@@ -636,6 +906,10 @@ static struct kunit_case string_test_cases[] = {
KUNIT_CASE(string_test_strtomem),
KUNIT_CASE(string_test_memtostr),
KUNIT_CASE(string_test_strends),
KUNIT_CASE(string_bench_strlen),
KUNIT_CASE(string_bench_strnlen),
KUNIT_CASE(string_bench_strchr),
KUNIT_CASE(string_bench_strrchr),
{}
};
tools/testing/selftests/riscv/cfi/Makefile
+2
View File
@@ -1,3 +1,5 @@
# SPDX-License-Identifier: GPL-2.0
CFLAGS += $(KHDR_INCLUDES)
CFLAGS += -I$(top_srcdir)/tools/include