diff --git a/anolis/configs/L0-MANDATORY/riscv/CONFIG_MEMORY_FAILURE b/anolis/configs/L0-MANDATORY/riscv/CONFIG_MEMORY_FAILURE deleted file mode 100644 index 50d51d72166b2f83c573d5df6ca5647d95d2f57d..0000000000000000000000000000000000000000 --- a/anolis/configs/L0-MANDATORY/riscv/CONFIG_MEMORY_FAILURE +++ /dev/null @@ -1 +0,0 @@ -# CONFIG_MEMORY_FAILURE is not set diff --git a/anolis/configs/L1-RECOMMEND/riscv/CONFIG_ACPI_APEI_EINJ b/anolis/configs/L1-RECOMMEND/riscv/CONFIG_ACPI_APEI_EINJ deleted file mode 100644 index 2d03b69a13589b925f1ce8554d43d7bef9a659ba..0000000000000000000000000000000000000000 --- a/anolis/configs/L1-RECOMMEND/riscv/CONFIG_ACPI_APEI_EINJ +++ /dev/null @@ -1 +0,0 @@ -# CONFIG_ACPI_APEI_EINJ is not set diff --git a/anolis/configs/L1-RECOMMEND/riscv/CONFIG_ACPI_APEI_EINJ_CXL b/anolis/configs/L1-RECOMMEND/riscv/CONFIG_ACPI_APEI_EINJ_CXL deleted file mode 100644 index 6e449b8c0a920a081091a9a41c01fd8a5483eb62..0000000000000000000000000000000000000000 --- a/anolis/configs/L1-RECOMMEND/riscv/CONFIG_ACPI_APEI_EINJ_CXL +++ /dev/null @@ -1 +0,0 @@ -# CONFIG_ACPI_APEI_EINJ_CXL is not set diff --git a/anolis/configs/L1-RECOMMEND/riscv/CONFIG_ACPI_APEI_MEMORY_FAILURE b/anolis/configs/L1-RECOMMEND/riscv/CONFIG_ACPI_APEI_MEMORY_FAILURE deleted file mode 100644 index 5dc4cc094f84c3f419988ca5d53cf1060b970c2c..0000000000000000000000000000000000000000 --- a/anolis/configs/L1-RECOMMEND/riscv/CONFIG_ACPI_APEI_MEMORY_FAILURE +++ /dev/null @@ -1 +0,0 @@ -# CONFIG_ACPI_APEI_MEMORY_FAILURE is not set diff --git a/anolis/configs/L1-RECOMMEND/riscv/CONFIG_ACPI_APEI_PCIEAER b/anolis/configs/L1-RECOMMEND/riscv/CONFIG_ACPI_APEI_PCIEAER deleted file mode 100644 index fde74a41c47eefeaffcf3deedcb187656663c0c4..0000000000000000000000000000000000000000 --- a/anolis/configs/L1-RECOMMEND/riscv/CONFIG_ACPI_APEI_PCIEAER +++ /dev/null @@ -1 +0,0 @@ -# CONFIG_ACPI_APEI_PCIEAER is not set diff --git a/anolis/configs/L1-RECOMMEND/riscv/CONFIG_RISCV_SBI_SSE b/anolis/configs/L1-RECOMMEND/riscv/CONFIG_RISCV_SBI_SSE new file mode 100644 index 0000000000000000000000000000000000000000..f485dd8d3051646702871d360d83adc4acade27c --- /dev/null +++ b/anolis/configs/L1-RECOMMEND/riscv/CONFIG_RISCV_SBI_SSE @@ -0,0 +1 @@ +CONFIG_RISCV_SBI_SSE=y diff --git a/anolis/configs/L1-RECOMMEND/riscv/CONFIG_RISCV_SSE b/anolis/configs/L1-RECOMMEND/riscv/CONFIG_RISCV_SSE deleted file mode 100644 index 8bbcd54f202555c8b6cd5be6f286fe7e73cb816d..0000000000000000000000000000000000000000 --- a/anolis/configs/L1-RECOMMEND/riscv/CONFIG_RISCV_SSE +++ /dev/null @@ -1 +0,0 @@ -CONFIG_RISCV_SSE=y diff --git a/anolis/configs/L2-OPTIONAL/riscv/CONFIG_ARCH_SUPPORTS_MEMORY_FAILURE b/anolis/configs/L2-OPTIONAL/riscv/CONFIG_ARCH_SUPPORTS_MEMORY_FAILURE deleted file mode 100644 index 77d6d6ad95fdf7ba267991810bb7892ca65b78c8..0000000000000000000000000000000000000000 --- a/anolis/configs/L2-OPTIONAL/riscv/CONFIG_ARCH_SUPPORTS_MEMORY_FAILURE +++ /dev/null @@ -1 +0,0 @@ -# CONFIG_ARCH_SUPPORTS_MEMORY_FAILURE is not set diff --git a/anolis/configs/L2-OPTIONAL/riscv/CONFIG_ARCH_SUPPORTS_SHADOW_CALL_STACK b/anolis/configs/L2-OPTIONAL/riscv/CONFIG_ARCH_SUPPORTS_SHADOW_CALL_STACK new file mode 100644 index 0000000000000000000000000000000000000000..289fd1ea24df0b34f7b375d08b22f8619fe73eff --- /dev/null +++ b/anolis/configs/L2-OPTIONAL/riscv/CONFIG_ARCH_SUPPORTS_SHADOW_CALL_STACK @@ -0,0 +1 @@ +CONFIG_ARCH_SUPPORTS_SHADOW_CALL_STACK=y diff --git a/anolis/configs/L2-OPTIONAL/riscv/CONFIG_HAVE_SHADOW_CALL_STACK b/anolis/configs/L2-OPTIONAL/riscv/CONFIG_HAVE_SHADOW_CALL_STACK new file mode 100644 index 0000000000000000000000000000000000000000..46dedd06a608da6468f04913060e0c36cc71ec9f --- /dev/null +++ b/anolis/configs/L2-OPTIONAL/riscv/CONFIG_HAVE_SHADOW_CALL_STACK @@ -0,0 +1 @@ +CONFIG_HAVE_SHADOW_CALL_STACK=y diff --git a/anolis/configs/L2-OPTIONAL/riscv/CONFIG_RISCV_PMU_SBI_SSE b/anolis/configs/L2-OPTIONAL/riscv/CONFIG_RISCV_PMU_SBI_SSE new file mode 100644 index 0000000000000000000000000000000000000000..f525451986cc73f050f069d3dcd88f3b99ce3e17 --- /dev/null +++ b/anolis/configs/L2-OPTIONAL/riscv/CONFIG_RISCV_PMU_SBI_SSE @@ -0,0 +1 @@ +# CONFIG_RISCV_PMU_SBI_SSE is not set diff --git a/anolis/configs/L2-OPTIONAL/riscv/CONFIG_SHADOW_CALL_STACK b/anolis/configs/L2-OPTIONAL/riscv/CONFIG_SHADOW_CALL_STACK new file mode 100644 index 0000000000000000000000000000000000000000..08f4cf8044ea422b17eaacab5824229f0492febf --- /dev/null +++ b/anolis/configs/L2-OPTIONAL/riscv/CONFIG_SHADOW_CALL_STACK @@ -0,0 +1 @@ +# CONFIG_SHADOW_CALL_STACK is not set diff --git a/arch/riscv/Kconfig b/arch/riscv/Kconfig index fc786ab33f4a62e2b5c93697930375c2dc52e9c7..39ff1650aef774f9000631e1a902b88808c8fd86 100644 --- a/arch/riscv/Kconfig +++ b/arch/riscv/Kconfig @@ -59,8 +59,10 @@ config RISCV select ARCH_SUPPORTS_CFI_CLANG if CLANG_VERSION >= 170000 select ARCH_SUPPORTS_DEBUG_PAGEALLOC if MMU select ARCH_SUPPORTS_HUGETLBFS if MMU + select ARCH_SUPPORTS_MEMORY_FAILURE select ARCH_SUPPORTS_PAGE_TABLE_CHECK if MMU select ARCH_SUPPORTS_PER_VMA_LOCK if MMU + select ARCH_SUPPORTS_SHADOW_CALL_STACK if HAVE_SHADOW_CALL_STACK select ARCH_USE_CMPXCHG_LOCKREF if 64BIT select ARCH_USE_MEMTEST select ARCH_USE_QUEUED_RWLOCKS @@ -202,6 +204,11 @@ config GCC_SUPPORTS_DYNAMIC_FTRACE depends on $(cc-option,-fpatchable-function-entry=8) depends on CC_HAS_MIN_FUNCTION_ALIGNMENT || !RISCV_ISA_C +config HAVE_SHADOW_CALL_STACK + def_bool $(cc-option,-fsanitize=shadow-call-stack) + # https://github.com/riscv-non-isa/riscv-elf-psabi-doc/commit/a484e843e6eeb51f0cb7b8819e50da6d2444d769 + depends on $(ld-option,--no-relax-gp) + config ARCH_MMAP_RND_BITS_MIN default 18 if 64BIT default 8 diff --git a/arch/riscv/Makefile b/arch/riscv/Makefile index 788615ec9b3a0416ff93a55d6bc5bfcb87bae5c8..b2a1f0bdce70491fb5ad0856a5b3fb981401187e 100644 --- a/arch/riscv/Makefile +++ b/arch/riscv/Makefile @@ -55,6 +55,10 @@ endif endif endif +ifeq ($(CONFIG_SHADOW_CALL_STACK),y) + KBUILD_LDFLAGS += --no-relax-gp +endif + include $(srctree)/${RISCV_ARCH_PATH}/Makefile.isa # Check if the toolchain supports Zacas diff --git a/arch/riscv/configs/defconfig b/arch/riscv/configs/defconfig index 37217cffa76be2ca50d447b92a636aa0623e49fb..90e5ff1aac19be014f22b5677ebbb3cf60c99252 100644 --- a/arch/riscv/configs/defconfig +++ b/arch/riscv/configs/defconfig @@ -48,6 +48,9 @@ CONFIG_ACPI_CPPC_CPUFREQ=m CONFIG_VIRTUALIZATION=y CONFIG_KVM=m CONFIG_ACPI=y +CONFIG_ACPI_APEI=y +CONFIG_ACPI_APEI_GHES=y +CONFIG_ACPI_APEI_ERST_DEBUG=y CONFIG_JUMP_LABEL=y CONFIG_MODULES=y CONFIG_MODULE_UNLOAD=y diff --git a/arch/riscv/include/asm/asm-prototypes.h b/arch/riscv/include/asm/asm-prototypes.h index cd627ec289f163a630b73dd03dd52a6b28692997..959a6588be5cb7f5a6460d2b6862d9e8133068fe 100644 --- a/arch/riscv/include/asm/asm-prototypes.h +++ b/arch/riscv/include/asm/asm-prototypes.h @@ -40,6 +40,7 @@ asmlinkage void riscv_v_context_nesting_end(struct pt_regs *regs); #define DECLARE_DO_ERROR_INFO(name) asmlinkage void name(struct pt_regs *regs) DECLARE_DO_ERROR_INFO(do_trap_unknown); +DECLARE_DO_ERROR_INFO(do_trap_hardware_error); DECLARE_DO_ERROR_INFO(do_trap_insn_misaligned); DECLARE_DO_ERROR_INFO(do_trap_insn_fault); DECLARE_DO_ERROR_INFO(do_trap_insn_illegal); diff --git a/arch/riscv/include/asm/asm.h b/arch/riscv/include/asm/asm.h index c0fd904e0a8fbe0ea20c10582dc1d63292cf4cdd..dbd986efe3449c87f3c8303b096c870341b18226 100644 --- a/arch/riscv/include/asm/asm.h +++ b/arch/riscv/include/asm/asm.h @@ -83,20 +83,15 @@ .endm #ifdef CONFIG_SMP -#ifdef CONFIG_32BIT -#define PER_CPU_OFFSET_SHIFT 2 -#else -#define PER_CPU_OFFSET_SHIFT 3 -#endif - .macro asm_per_cpu_with_cpu dst sym tmp cpu - slli \tmp, \cpu, PER_CPU_OFFSET_SHIFT + slli \tmp, \cpu, RISCV_LGPTR la \dst, __per_cpu_offset add \dst, \dst, \tmp REG_L \tmp, 0(\dst) la \dst, \sym add \dst, \dst, \tmp .endm + .macro asm_per_cpu dst sym tmp lw \tmp, TASK_TI_CPU_NUM(tp) asm_per_cpu_with_cpu \dst \sym \tmp \tmp @@ -111,6 +106,25 @@ .endm #endif /* CONFIG_SMP */ +.macro load_per_cpu dst ptr tmp + asm_per_cpu \dst \ptr \tmp + REG_L \dst, 0(\dst) +.endm + +#ifdef CONFIG_SHADOW_CALL_STACK +/* gp is used as the shadow call stack pointer instead */ +.macro load_global_pointer +.endm +#else +/* load __global_pointer to gp */ +.macro load_global_pointer +.option push +.option norelax + la gp, __global_pointer$ +.option pop +.endm +#endif /* CONFIG_SHADOW_CALL_STACK */ + /* save all GPs except x1 ~ x5 */ .macro save_from_x6_to_x31 REG_S x6, PT_T1(sp) diff --git a/arch/riscv/include/asm/fixmap.h b/arch/riscv/include/asm/fixmap.h index fa3a0ec0c55c11f6e813ba66b127987f1fb62b94..e874fd9522865002c0346dc7a894a08e217828c4 100644 --- a/arch/riscv/include/asm/fixmap.h +++ b/arch/riscv/include/asm/fixmap.h @@ -41,10 +41,10 @@ enum fixed_addresses { #ifdef CONFIG_ACPI_APEI_GHES /* Used for GHES mapping from assorted contexts */ FIX_APEI_GHES_IRQ, -#ifdef CONFIG_RISCV_SSE +#ifdef CONFIG_RISCV_SBI_SSE FIX_APEI_GHES_SSE_LOW_PRIORITY, FIX_APEI_GHES_SSE_HIGH_PRIORITY, -#endif /* CONFIG_RISCV_SSE */ +#endif /* CONFIG_RISCV_SBI_SSE */ #endif /* CONFIG_ACPI_APEI_GHES */ __end_of_permanent_fixed_addresses, /* diff --git a/arch/riscv/include/asm/io.h b/arch/riscv/include/asm/io.h index 662f3dd15ac5eb48ef71b9547d695ca453e2ce5c..e35a01ef9a6107b0b6eb5726fa121a54a1252d83 100644 --- a/arch/riscv/include/asm/io.h +++ b/arch/riscv/include/asm/io.h @@ -30,9 +30,6 @@ #define PCI_IOBASE ((void __iomem *)PCI_IO_START) #endif /* CONFIG_MMU */ -#define ioremap_cache(addr, size) \ - ((__force void *)ioremap_prot((addr), (size), _PAGE_KERNEL)) - /* * Emulation routines for the port-mapped IO space used by some PCI drivers. * These are defined as being "fully synchronous", but also "not guaranteed to @@ -141,6 +138,8 @@ __io_writes_outs(outs, u64, q, __io_pbr(), __io_paw()) #ifdef CONFIG_MMU #define arch_memremap_wb(addr, size) \ ((__force void *)ioremap_prot((addr), (size), _PAGE_KERNEL)) -#endif +#define ioremap_cache(addr, size) \ + ((__force void *)ioremap_prot((addr), (size), _PAGE_KERNEL)) +#endif #endif /* _ASM_RISCV_IO_H */ diff --git a/arch/riscv/include/asm/irq_stack.h b/arch/riscv/include/asm/irq_stack.h index e4042d297580080c0c3dfffed0dd711f5f371ca1..6441ded3b0cf2cf894312be28aa9e731e151f2a8 100644 --- a/arch/riscv/include/asm/irq_stack.h +++ b/arch/riscv/include/asm/irq_stack.h @@ -12,6 +12,9 @@ DECLARE_PER_CPU(ulong *, irq_stack_ptr); +asmlinkage void call_on_irq_stack(struct pt_regs *regs, + void (*func)(struct pt_regs *)); + #ifdef CONFIG_VMAP_STACK /* * To ensure that VMAP'd stack overflow detection works correctly, all VMAP'd diff --git a/arch/riscv/include/asm/sbi.h b/arch/riscv/include/asm/sbi.h index e32d060e74af646da1df180cad75e94c22c9263a..b773a9b89993d6565b79119cfe2d082ed4fc67be 100644 --- a/arch/riscv/include/asm/sbi.h +++ b/arch/riscv/include/asm/sbi.h @@ -273,66 +273,6 @@ enum sbi_ext_sta_fid { SBI_EXT_STA_STEAL_TIME_SET_SHMEM = 0, }; -enum sbi_ext_sse_fid { - SBI_SSE_EVENT_ATTR_READ = 0, - SBI_SSE_EVENT_ATTR_WRITE, - SBI_SSE_EVENT_REGISTER, - SBI_SSE_EVENT_UNREGISTER, - SBI_SSE_EVENT_ENABLE, - SBI_SSE_EVENT_DISABLE, - SBI_SSE_EVENT_COMPLETE, - SBI_SSE_EVENT_SIGNAL, - SBI_SSE_EVENT_HART_UNMASK, - SBI_SSE_EVENT_HART_MASK, -}; - -enum sbi_sse_state { - SBI_SSE_STATE_UNUSED = 0, - SBI_SSE_STATE_REGISTERED = 1, - SBI_SSE_STATE_ENABLED = 2, - SBI_SSE_STATE_RUNNING = 3, -}; - -/* SBI SSE Event Attributes. */ -enum sbi_sse_attr_id { - SBI_SSE_ATTR_STATUS = 0x00000000, - SBI_SSE_ATTR_PRIO = 0x00000001, - SBI_SSE_ATTR_CONFIG = 0x00000002, - SBI_SSE_ATTR_PREFERRED_HART = 0x00000003, - SBI_SSE_ATTR_ENTRY_PC = 0x00000004, - SBI_SSE_ATTR_ENTRY_ARG = 0x00000005, - SBI_SSE_ATTR_INTERRUPTED_SEPC = 0x00000006, - SBI_SSE_ATTR_INTERRUPTED_FLAGS = 0x00000007, - SBI_SSE_ATTR_INTERRUPTED_A6 = 0x00000008, - SBI_SSE_ATTR_INTERRUPTED_A7 = 0x00000009, - - SBI_SSE_ATTR_MAX = 0x0000000A -}; - -#define SBI_SSE_ATTR_STATUS_STATE_OFFSET 0 -#define SBI_SSE_ATTR_STATUS_STATE_MASK 0x3 -#define SBI_SSE_ATTR_STATUS_PENDING_OFFSET 2 -#define SBI_SSE_ATTR_STATUS_INJECT_OFFSET 3 - -#define SBI_SSE_ATTR_CONFIG_ONESHOT BIT(0) - -#define SBI_SSE_ATTR_INTERRUPTED_FLAGS_SSTATUS_SPP BIT(0) -#define SBI_SSE_ATTR_INTERRUPTED_FLAGS_SSTATUS_SPIE BIT(1) -#define SBI_SSE_ATTR_INTERRUPTED_FLAGS_HSTATUS_SPV BIT(2) -#define SBI_SSE_ATTR_INTERRUPTED_FLAGS_HSTATUS_SPVP BIT(3) - -#define SBI_SSE_EVENT_LOCAL_HIGH_PRIO_RAS 0x00000000 -#define SBI_SSE_EVENT_LOCAL_DOUBLE_TRAP 0x00000001 -#define SBI_SSE_EVENT_GLOBAL_HIGH_PRIO_RAS 0x00008000 -#define SBI_SSE_EVENT_LOCAL_PMU_OVERFLOW 0x00010000 -#define SBI_SSE_EVENT_LOCAL_LOW_PRIO_RAS 0x00100000 -#define SBI_SSE_EVENT_GLOBAL_LOW_PRIO_RAS 0x00108000 -#define SBI_SSE_EVENT_LOCAL_SOFTWARE_INJECTED 0xffff0000 -#define SBI_SSE_EVENT_GLOBAL_SOFTWARE_INJECTED 0xffff8000 - -#define SBI_SSE_EVENT_PLATFORM BIT(14) -#define SBI_SSE_EVENT_GLOBAL BIT(15) - struct sbi_sta_struct { __le32 sequence; __le32 flags; @@ -489,6 +429,66 @@ enum sbi_fwft_feature_t { #define SBI_FWFT_SET_FLAG_LOCK BIT(0) +enum sbi_ext_sse_fid { + SBI_SSE_EVENT_ATTR_READ = 0, + SBI_SSE_EVENT_ATTR_WRITE, + SBI_SSE_EVENT_REGISTER, + SBI_SSE_EVENT_UNREGISTER, + SBI_SSE_EVENT_ENABLE, + SBI_SSE_EVENT_DISABLE, + SBI_SSE_EVENT_COMPLETE, + SBI_SSE_EVENT_INJECT, + SBI_SSE_HART_UNMASK, + SBI_SSE_HART_MASK, +}; + +enum sbi_sse_state { + SBI_SSE_STATE_UNUSED = 0, + SBI_SSE_STATE_REGISTERED = 1, + SBI_SSE_STATE_ENABLED = 2, + SBI_SSE_STATE_RUNNING = 3, +}; + +/* SBI SSE Event Attributes. */ +enum sbi_sse_attr_id { + SBI_SSE_ATTR_STATUS = 0x00000000, + SBI_SSE_ATTR_PRIO = 0x00000001, + SBI_SSE_ATTR_CONFIG = 0x00000002, + SBI_SSE_ATTR_PREFERRED_HART = 0x00000003, + SBI_SSE_ATTR_ENTRY_PC = 0x00000004, + SBI_SSE_ATTR_ENTRY_ARG = 0x00000005, + SBI_SSE_ATTR_INTERRUPTED_SEPC = 0x00000006, + SBI_SSE_ATTR_INTERRUPTED_FLAGS = 0x00000007, + SBI_SSE_ATTR_INTERRUPTED_A6 = 0x00000008, + SBI_SSE_ATTR_INTERRUPTED_A7 = 0x00000009, + + SBI_SSE_ATTR_MAX = 0x0000000A +}; + +#define SBI_SSE_ATTR_STATUS_STATE_OFFSET 0 +#define SBI_SSE_ATTR_STATUS_STATE_MASK 0x3 +#define SBI_SSE_ATTR_STATUS_PENDING_OFFSET 2 +#define SBI_SSE_ATTR_STATUS_INJECT_OFFSET 3 + +#define SBI_SSE_ATTR_CONFIG_ONESHOT BIT(0) + +#define SBI_SSE_ATTR_INTERRUPTED_FLAGS_SSTATUS_SPP BIT(0) +#define SBI_SSE_ATTR_INTERRUPTED_FLAGS_SSTATUS_SPIE BIT(1) +#define SBI_SSE_ATTR_INTERRUPTED_FLAGS_HSTATUS_SPV BIT(2) +#define SBI_SSE_ATTR_INTERRUPTED_FLAGS_HSTATUS_SPVP BIT(3) + +#define SBI_SSE_EVENT_LOCAL_HIGH_PRIO_RAS 0x00000000 +#define SBI_SSE_EVENT_LOCAL_DOUBLE_TRAP 0x00000001 +#define SBI_SSE_EVENT_GLOBAL_HIGH_PRIO_RAS 0x00008000 +#define SBI_SSE_EVENT_LOCAL_PMU_OVERFLOW 0x00010000 +#define SBI_SSE_EVENT_LOCAL_LOW_PRIO_RAS 0x00100000 +#define SBI_SSE_EVENT_GLOBAL_LOW_PRIO_RAS 0x00108000 +#define SBI_SSE_EVENT_LOCAL_SOFTWARE_INJECTED 0xffff0000 +#define SBI_SSE_EVENT_GLOBAL_SOFTWARE_INJECTED 0xffff8000 + +#define SBI_SSE_EVENT_PLATFORM BIT(14) +#define SBI_SSE_EVENT_GLOBAL BIT(15) + /* SBI spec version fields */ #define SBI_SPEC_VERSION_DEFAULT 0x1 #define SBI_SPEC_VERSION_MAJOR_SHIFT 24 diff --git a/arch/riscv/include/asm/scs.h b/arch/riscv/include/asm/scs.h new file mode 100644 index 0000000000000000000000000000000000000000..62344daad73dc63a4d344ba145bf9684445c63b5 --- /dev/null +++ b/arch/riscv/include/asm/scs.h @@ -0,0 +1,61 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef _ASM_SCS_H +#define _ASM_SCS_H + +#ifdef __ASSEMBLY__ +#include + +#ifdef CONFIG_SHADOW_CALL_STACK + +/* Load init_shadow_call_stack to gp. */ +.macro scs_load_init_stack + la gp, init_shadow_call_stack + XIP_FIXUP_OFFSET gp +.endm + +/* Load the per-CPU IRQ shadow call stack to gp. */ +.macro scs_load_irq_stack tmp + load_per_cpu gp, irq_shadow_call_stack_ptr, \tmp +.endm + +/* Load the per-CPU IRQ shadow call stack to gp. */ +.macro scs_load_sse_stack reg_evt + REG_L gp, SSE_REG_EVT_SHADOW_STACK(\reg_evt) +.endm + +/* Load task_scs_sp(current) to gp. */ +.macro scs_load_current + REG_L gp, TASK_TI_SCS_SP(tp) +.endm + +/* Load task_scs_sp(current) to gp, but only if tp has changed. */ +.macro scs_load_current_if_task_changed prev + beq \prev, tp, _skip_scs + scs_load_current +_skip_scs: +.endm + +/* Save gp to task_scs_sp(current). */ +.macro scs_save_current + REG_S gp, TASK_TI_SCS_SP(tp) +.endm + +#else /* CONFIG_SHADOW_CALL_STACK */ + +.macro scs_load_init_stack +.endm +.macro scs_load_irq_stack tmp +.endm +.macro scs_load_sse_stack reg_evt +.endm +.macro scs_load_current +.endm +.macro scs_load_current_if_task_changed prev +.endm +.macro scs_save_current +.endm + +#endif /* CONFIG_SHADOW_CALL_STACK */ +#endif /* __ASSEMBLY__ */ + +#endif /* _ASM_SCS_H */ diff --git a/arch/riscv/include/asm/set_memory.h b/arch/riscv/include/asm/set_memory.h index ec11001c3fe04223e3417161ca5468d29590d092..9d2a09ceed89dd1487271814daf336adea25e304 100644 --- a/arch/riscv/include/asm/set_memory.h +++ b/arch/riscv/include/asm/set_memory.h @@ -39,6 +39,8 @@ static inline int set_kernel_memory(char *startp, char *endp, return 0; } #endif +/* No support for setting memory_np */ +static inline int set_memory_np(unsigned long addr, int numpages) { return 0; } int set_direct_map_invalid_noflush(struct page *page); int set_direct_map_default_noflush(struct page *page); diff --git a/arch/riscv/include/asm/sse.h b/arch/riscv/include/asm/sse.h index 8929a268462cf446d36a188f574ca490c1a47159..d3ce8c2b52212307e2e2f689b1402471788a00eb 100644 --- a/arch/riscv/include/asm/sse.h +++ b/arch/riscv/include/asm/sse.h @@ -7,7 +7,7 @@ #include -#ifdef CONFIG_RISCV_SSE +#ifdef CONFIG_RISCV_SBI_SSE struct sse_event_interrupted_state { unsigned long a6; diff --git a/arch/riscv/include/asm/switch_to.h b/arch/riscv/include/asm/switch_to.h index 1d24b68ca350a010c63dee93ed4ce3e51d512cd6..61a526a601d5095b0530a21b2c8cf7dc1c1cb469 100644 --- a/arch/riscv/include/asm/switch_to.h +++ b/arch/riscv/include/asm/switch_to.h @@ -103,15 +103,15 @@ static inline void __switch_to_envcfg(struct task_struct *next) :: "r" (next->thread.envcfg) : "memory"); } -#ifdef CONFIG_RISCV_SSE -DECLARE_PER_CPU(struct task_struct *, __sse_entry_task); +#ifdef CONFIG_RISCV_SBI_SSE +DECLARE_PER_CPU(struct task_struct *, __sbi_sse_entry_task); -static inline void __switch_sse_entry_task(struct task_struct *next) +static inline void __switch_sbi_sse_entry_task(struct task_struct *next) { - __this_cpu_write(__sse_entry_task, next); + __this_cpu_write(__sbi_sse_entry_task, next); } #else -static inline void __switch_sse_entry_task(struct task_struct *next) +static inline void __switch_sbi_sse_entry_task(struct task_struct *next) { } #endif @@ -150,7 +150,7 @@ do { \ if (switch_to_should_flush_icache(__next)) \ local_flush_icache_all(); \ __switch_to_envcfg(__next); \ - __switch_sse_entry_task(__next); \ + __switch_sbi_sse_entry_task(__next); \ ((last) = __switch_to(__prev, __next)); \ } while (0) diff --git a/arch/riscv/include/asm/thread_info.h b/arch/riscv/include/asm/thread_info.h index 59357bb8ffb3421d5212e054348e8aa13c8ce2ca..3d9b56935e836e048e9c4274f7e326a7d46e20c5 100644 --- a/arch/riscv/include/asm/thread_info.h +++ b/arch/riscv/include/asm/thread_info.h @@ -62,6 +62,10 @@ struct thread_info { long user_sp; /* User stack pointer */ int cpu; unsigned long syscall_work; /* SYSCALL_WORK_ flags */ +#ifdef CONFIG_SHADOW_CALL_STACK + void *scs_base; + void *scs_sp; +#endif #ifdef CONFIG_64BIT /* * Used in handle_exception() to save a0, a1 and a2 before knowing if we @@ -71,6 +75,14 @@ struct thread_info { #endif }; +#ifdef CONFIG_SHADOW_CALL_STACK +#define INIT_SCS \ + .scs_base = init_shadow_call_stack, \ + .scs_sp = init_shadow_call_stack, +#else +#define INIT_SCS +#endif + /* * macros/functions for gaining access to the thread information structure * @@ -80,6 +92,7 @@ struct thread_info { { \ .flags = 0, \ .preempt_count = INIT_PREEMPT_COUNT, \ + INIT_SCS \ } void arch_release_task_struct(struct task_struct *tsk); diff --git a/arch/riscv/kernel/Makefile b/arch/riscv/kernel/Makefile index a92768501411921013c6ae61a3a8075507b1b184..ba99b5ddad75e1e39a67bd11b769309de76cb76a 100644 --- a/arch/riscv/kernel/Makefile +++ b/arch/riscv/kernel/Makefile @@ -93,7 +93,7 @@ obj-$(CONFIG_DYNAMIC_FTRACE) += mcount-dyn.o obj-$(CONFIG_PERF_EVENTS) += perf_callchain.o obj-$(CONFIG_HAVE_PERF_REGS) += perf_regs.o obj-$(CONFIG_RISCV_SBI) += sbi.o sbi_ecall.o -obj-$(CONFIG_RISCV_SSE) += sse.o sse_entry.o +obj-$(CONFIG_RISCV_SBI_SSE) += sbi_sse.o sbi_sse_entry.o ifeq ($(CONFIG_RISCV_SBI), y) obj-$(CONFIG_SMP) += sbi-ipi.o obj-$(CONFIG_SMP) += cpu_ops_sbi.o diff --git a/arch/riscv/kernel/asm-offsets.c b/arch/riscv/kernel/asm-offsets.c index cdf21d2053f3f225a1bc6792174cf8847b998c6b..7c76974ba0e2bc42b20b9e7f3c5920fda617395c 100644 --- a/arch/riscv/kernel/asm-offsets.c +++ b/arch/riscv/kernel/asm-offsets.c @@ -13,6 +13,7 @@ #include #include #include +#include #include #include #include @@ -41,6 +42,9 @@ void asm_offsets(void) OFFSET(TASK_TI_PREEMPT_COUNT, task_struct, thread_info.preempt_count); OFFSET(TASK_TI_KERNEL_SP, task_struct, thread_info.kernel_sp); OFFSET(TASK_TI_USER_SP, task_struct, thread_info.user_sp); +#ifdef CONFIG_SHADOW_CALL_STACK + OFFSET(TASK_TI_SCS_SP, task_struct, thread_info.scs_sp); +#endif #ifdef CONFIG_64BIT OFFSET(TASK_TI_A0, task_struct, thread_info.a0); OFFSET(TASK_TI_A1, task_struct, thread_info.a1); @@ -489,6 +493,9 @@ void asm_offsets(void) OFFSET(SBI_HART_BOOT_TASK_PTR_OFFSET, sbi_hart_boot_data, task_ptr); OFFSET(SBI_HART_BOOT_STACK_PTR_OFFSET, sbi_hart_boot_data, stack_ptr); + DEFINE(STACKFRAME_SIZE_ON_STACK, ALIGN(sizeof(struct stackframe), STACK_ALIGN)); + OFFSET(STACKFRAME_FP, stackframe, fp); + OFFSET(STACKFRAME_RA, stackframe, ra); #ifdef CONFIG_FUNCTION_TRACER DEFINE(FTRACE_OPS_FUNC, offsetof(struct ftrace_ops, func)); #ifdef CONFIG_DYNAMIC_FTRACE_WITH_DIRECT_CALLS @@ -520,7 +527,7 @@ void asm_offsets(void) DEFINE(FREGS_A7, offsetof(struct __arch_ftrace_regs, a7)); #endif -#ifdef CONFIG_RISCV_SSE +#ifdef CONFIG_RISCV_SBI_SSE OFFSET(SSE_REG_EVT_STACK, sse_event_arch_data, stack); OFFSET(SSE_REG_EVT_SHADOW_STACK, sse_event_arch_data, shadow_stack); OFFSET(SSE_REG_EVT_TMP, sse_event_arch_data, tmp); @@ -529,7 +536,6 @@ void asm_offsets(void) DEFINE(SBI_EXT_SSE, SBI_EXT_SSE); DEFINE(SBI_SSE_EVENT_COMPLETE, SBI_SSE_EVENT_COMPLETE); - #define ASM_MAX_CPUS NR_CPUS - DEFINE(ASM_NR_CPUS, ASM_MAX_CPUS); + DEFINE(ASM_NR_CPUS, NR_CPUS); #endif } diff --git a/arch/riscv/kernel/entry.S b/arch/riscv/kernel/entry.S index 7167ee40af4cc2a6f7ac9254482b9f4bff4b46b8..0e36d7218331fcf3ccf104457f43637c12010807 100644 --- a/arch/riscv/kernel/entry.S +++ b/arch/riscv/kernel/entry.S @@ -9,6 +9,7 @@ #include #include +#include #include #include #include @@ -164,10 +165,11 @@ SYM_CODE_START(handle_exception) csrw CSR_SCRATCH, x0 /* Load the global pointer */ -.option push -.option norelax - la gp, __global_pointer$ -.option pop + load_global_pointer + + /* Load the kernel shadow call stack pointer if coming from userspace */ + scs_load_current_if_task_changed s5 + #ifdef CONFIG_RISCV_ISA_V_PREEMPTIVE move a0, sp call riscv_v_context_nesting_start @@ -219,6 +221,9 @@ SYM_CODE_START_NOALIGN(ret_from_exception) addi s0, sp, PT_SIZE_ON_STACK REG_S s0, TASK_TI_KERNEL_SP(tp) + /* Save the kernel shadow call stack pointer */ + scs_save_current + /* * Save TP into the scratch register , so we can find the kernel data * structures again. @@ -318,6 +323,43 @@ SYM_CODE_START(ret_from_fork) j ret_from_exception SYM_CODE_END(ret_from_fork) +#ifdef CONFIG_IRQ_STACKS +/* + * void call_on_irq_stack(struct pt_regs *regs, + * void (*func)(struct pt_regs *)); + * + * Calls func(regs) using the per-CPU IRQ stack. + */ +SYM_FUNC_START(call_on_irq_stack) + /* Create a frame record to save ra and s0 (fp) */ + addi sp, sp, -STACKFRAME_SIZE_ON_STACK + REG_S ra, STACKFRAME_RA(sp) + REG_S s0, STACKFRAME_FP(sp) + addi s0, sp, STACKFRAME_SIZE_ON_STACK + + /* Switch to the per-CPU shadow call stack */ + scs_save_current + scs_load_irq_stack t0 + + /* Switch to the per-CPU IRQ stack and call the handler */ + load_per_cpu t0, irq_stack_ptr, t1 + li t1, IRQ_STACK_SIZE + add sp, t0, t1 + jalr a1 + + /* Switch back to the thread shadow call stack */ + scs_load_current + + /* Switch back to the thread stack and restore ra and s0 */ + addi sp, s0, -STACKFRAME_SIZE_ON_STACK + REG_L ra, STACKFRAME_RA(sp) + REG_L s0, STACKFRAME_FP(sp) + addi sp, sp, STACKFRAME_SIZE_ON_STACK + + ret +SYM_FUNC_END(call_on_irq_stack) +#endif /* CONFIG_IRQ_STACKS */ + /* * Integer register context switch * The callee-saved registers must be saved and restored. @@ -347,6 +389,8 @@ SYM_FUNC_START(__switch_to) REG_S s9, TASK_THREAD_S9_RA(a3) REG_S s10, TASK_THREAD_S10_RA(a3) REG_S s11, TASK_THREAD_S11_RA(a3) + /* Save the kernel shadow call stack pointer */ + scs_save_current /* Restore context from next->thread */ REG_L ra, TASK_THREAD_RA_RA(a4) REG_L sp, TASK_THREAD_SP_RA(a4) @@ -364,6 +408,8 @@ SYM_FUNC_START(__switch_to) REG_L s11, TASK_THREAD_S11_RA(a4) /* The offset of thread_info in task_struct is zero. */ move tp, a1 + /* Switch to the next shadow call stack */ + scs_load_current ret SYM_FUNC_END(__switch_to) @@ -392,6 +438,10 @@ SYM_DATA_START_LOCAL(excp_vect_table) RISCV_PTR do_page_fault /* load page fault */ RISCV_PTR do_trap_unknown RISCV_PTR do_page_fault /* store page fault */ + RISCV_PTR do_trap_unknown /* cause=16 */ + RISCV_PTR do_trap_unknown /* cause=17 */ + RISCV_PTR do_trap_unknown /* cause=18 */ + RISCV_PTR do_trap_hardware_error /* hardware error (19) */ SYM_DATA_END_LABEL(excp_vect_table, SYM_L_LOCAL, excp_vect_table_end) #ifndef CONFIG_MMU diff --git a/arch/riscv/kernel/head.S b/arch/riscv/kernel/head.S index 756bf0b68c3c1f49ffe72841c98db8df25240913..db814fdade7205e3970d1fe4469ffc03be909a7d 100644 --- a/arch/riscv/kernel/head.S +++ b/arch/riscv/kernel/head.S @@ -14,6 +14,7 @@ #include #include #include +#include #include #include "efi-header.S" @@ -111,10 +112,7 @@ relocate_enable_mmu: csrw CSR_TVEC, a0 /* Reload the global pointer */ -.option push -.option norelax - la gp, __global_pointer$ -.option pop + load_global_pointer /* * Switch to kernel page tables. A full fence is necessary in order to @@ -141,10 +139,7 @@ secondary_start_sbi: #endif /* Load the global pointer */ - .option push - .option norelax - la gp, __global_pointer$ - .option pop + load_global_pointer /* * Disable FPU & VECTOR to detect illegal usage of @@ -166,6 +161,7 @@ secondary_start_sbi: XIP_FIXUP_OFFSET a3 add a3, a3, a1 REG_L sp, (a3) + scs_load_current .Lsecondary_start_common: @@ -244,10 +240,7 @@ SYM_CODE_START(_start_kernel) #endif /* CONFIG_RISCV_M_MODE */ /* Load the global pointer */ -.option push -.option norelax - la gp, __global_pointer$ -.option pop + load_global_pointer /* * Disable FPU & VECTOR to detect illegal usage of @@ -314,6 +307,7 @@ SYM_CODE_START(_start_kernel) la sp, init_thread_union + THREAD_SIZE XIP_FIXUP_OFFSET sp addi sp, sp, -PT_SIZE_ON_STACK + scs_load_init_stack #ifdef CONFIG_BUILTIN_DTB la a0, __dtb_start XIP_FIXUP_OFFSET a0 @@ -335,6 +329,7 @@ SYM_CODE_START(_start_kernel) la tp, init_task la sp, init_thread_union + THREAD_SIZE addi sp, sp, -PT_SIZE_ON_STACK + scs_load_current #ifdef CONFIG_KASAN call kasan_early_init diff --git a/arch/riscv/kernel/irq.c b/arch/riscv/kernel/irq.c index 9cc0a76692715ea6ff2ec56630f7b60ed0a37f92..9ceda02507cae9c73efac8c0589695b38d03e499 100644 --- a/arch/riscv/kernel/irq.c +++ b/arch/riscv/kernel/irq.c @@ -9,6 +9,7 @@ #include #include #include +#include #include #include #include @@ -34,6 +35,24 @@ EXPORT_SYMBOL_GPL(riscv_get_intc_hwnode); #ifdef CONFIG_IRQ_STACKS #include +DECLARE_PER_CPU(ulong *, irq_shadow_call_stack_ptr); + +#ifdef CONFIG_SHADOW_CALL_STACK +DEFINE_PER_CPU(ulong *, irq_shadow_call_stack_ptr); +#endif + +static void init_irq_scs(void) +{ + int cpu; + + if (!scs_is_enabled()) + return; + + for_each_possible_cpu(cpu) + per_cpu(irq_shadow_call_stack_ptr, cpu) = + scs_alloc(cpu_to_node(cpu)); +} + DEFINE_PER_CPU(ulong *, irq_stack_ptr); #ifdef CONFIG_VMAP_STACK @@ -61,40 +80,22 @@ static void init_irq_stacks(void) #endif /* CONFIG_VMAP_STACK */ #ifdef CONFIG_SOFTIRQ_ON_OWN_STACK +static void ___do_softirq(struct pt_regs *regs) +{ + __do_softirq(); +} + void do_softirq_own_stack(void) { -#ifdef CONFIG_IRQ_STACKS - if (on_thread_stack()) { - ulong *sp = per_cpu(irq_stack_ptr, smp_processor_id()) - + IRQ_STACK_SIZE/sizeof(ulong); - __asm__ __volatile( - "addi sp, sp, -"RISCV_SZPTR "\n" - REG_S" ra, (sp) \n" - "addi sp, sp, -"RISCV_SZPTR "\n" - REG_S" s0, (sp) \n" - "addi s0, sp, 2*"RISCV_SZPTR "\n" - "move sp, %[sp] \n" - "call __do_softirq \n" - "addi sp, s0, -2*"RISCV_SZPTR"\n" - REG_L" s0, (sp) \n" - "addi sp, sp, "RISCV_SZPTR "\n" - REG_L" ra, (sp) \n" - "addi sp, sp, "RISCV_SZPTR "\n" - : - : [sp] "r" (sp) - : "a0", "a1", "a2", "a3", "a4", "a5", "a6", "a7", - "t0", "t1", "t2", "t3", "t4", "t5", "t6", -#ifndef CONFIG_FRAME_POINTER - "s0", -#endif - "memory"); - } else -#endif + if (on_thread_stack()) + call_on_irq_stack(NULL, ___do_softirq); + else __do_softirq(); } #endif /* CONFIG_SOFTIRQ_ON_OWN_STACK */ #else +static void init_irq_scs(void) {} static void init_irq_stacks(void) {} #endif /* CONFIG_IRQ_STACKS */ @@ -106,6 +107,7 @@ int arch_show_interrupts(struct seq_file *p, int prec) void __init init_IRQ(void) { + init_irq_scs(); init_irq_stacks(); irqchip_init(); if (!handle_arch_irq) diff --git a/arch/riscv/kernel/sse.c b/arch/riscv/kernel/sbi_sse.c similarity index 80% rename from arch/riscv/kernel/sse.c rename to arch/riscv/kernel/sbi_sse.c index d2da7e23a74ac8dc231e3dad07145416503be572..f30ae9ff77c97519f6317cf55718a9814b50bba0 100644 --- a/arch/riscv/kernel/sse.c +++ b/arch/riscv/kernel/sbi_sse.c @@ -1,11 +1,10 @@ // SPDX-License-Identifier: GPL-2.0-or-later /* - * Copyright (C) 2024 Rivos Inc. + * Copyright (C) 2025 Rivos Inc. */ #include #include #include -#include #include #include @@ -14,7 +13,7 @@ #include #include -DEFINE_PER_CPU(struct task_struct *, __sse_entry_task); +DEFINE_PER_CPU(struct task_struct *, __sbi_sse_entry_task); void __weak sse_handle_event(struct sse_event_arch_data *arch_evt, struct pt_regs *regs) { @@ -34,18 +33,6 @@ void do_sse(struct sse_event_arch_data *arch_evt, struct pt_regs *regs) sse_handle_event(arch_evt, regs); - /* - * The SSE delivery path does not uses the "standard" exception path - * (see sse_entry.S) and does not process any pending signal/softirqs - * due to being similar to a NMI. - * Some drivers (PMU, RAS) enqueue pending work that needs to be handled - * as soon as possible by bottom halves. For that purpose, set the SIP - * software interrupt pending bit which will force a software interrupt - * to be serviced once interrupts are reenabled in the interrupted - * context if they were masked or directly if unmasked. - */ - csr_set(CSR_IP, IE_SIE); - nmi_exit(); } @@ -71,6 +58,22 @@ static void sse_stack_free(void *stack) { vfree(stack_pointer_to_alloc(stack)); } + +static void arch_sse_stack_cpu_sync(struct sse_event_arch_data *arch_evt) +{ + void *p_stack = arch_evt->stack; + unsigned long stack = (unsigned long)stack_pointer_to_alloc(p_stack); + unsigned long stack_end = stack + SSE_STACK_SIZE; + + /* + * Flush the tlb to avoid taking any exception when accessing the + * vmapped stack inside the SSE handler + */ + if (sse_event_is_global(arch_evt->evt_id)) + flush_tlb_kernel_range(stack, stack_end); + else + local_flush_tlb_kernel_range(stack, stack_end); +} #else /* CONFIG_VMAP_STACK */ static void *sse_stack_alloc(unsigned int cpu) { @@ -83,6 +86,8 @@ static void sse_stack_free(void *stack) { kfree(stack_pointer_to_alloc(stack)); } + +static void arch_sse_stack_cpu_sync(struct sse_event_arch_data *arch_evt) {} #endif /* CONFIG_VMAP_STACK */ static int sse_init_scs(int cpu, struct sse_event_arch_data *arch_evt) @@ -107,7 +112,8 @@ void arch_sse_event_update_cpu(struct sse_event_arch_data *arch_evt, int cpu) arch_evt->hart_id = cpuid_to_hartid_map(cpu); } -int arch_sse_init_event(struct sse_event_arch_data *arch_evt, u32 evt_id, int cpu) +int arch_sse_init_event(struct sse_event_arch_data *arch_evt, u32 evt_id, + int cpu) { void *stack; @@ -146,6 +152,8 @@ int arch_sse_register_event(struct sse_event_arch_data *arch_evt) { struct sbiret sret; + arch_sse_stack_cpu_sync(arch_evt); + sret = sbi_ecall(SBI_EXT_SSE, SBI_SSE_EVENT_REGISTER, arch_evt->evt_id, (unsigned long)handle_sse, (unsigned long)arch_evt, 0, 0, 0); diff --git a/arch/riscv/kernel/sse_entry.S b/arch/riscv/kernel/sbi_sse_entry.S similarity index 77% rename from arch/riscv/kernel/sse_entry.S rename to arch/riscv/kernel/sbi_sse_entry.S index 3837a22b6e4f7558f0048315070e44bffe72d8ce..b3f308a159127f97d14ed8349d201e284f1f2eec 100644 --- a/arch/riscv/kernel/sse_entry.S +++ b/arch/riscv/kernel/sbi_sse_entry.S @@ -1,6 +1,6 @@ -/* SPDX-License-Identifier: GPL-2.0-only */ +/* SPDX-License-Identifier: GPL-2.0-or-later */ /* - * Copyright (C) 2024 Rivos Inc. + * Copyright (C) 2025 Rivos Inc. */ #include @@ -8,36 +8,12 @@ #include #include -#include +#include /* When entering handle_sse, the following registers are set: * a6: contains the hartid * a7: contains a sse_event_arch_data struct pointer */ - -#ifdef CONFIG_SHADOW_CALL_STACK -/* gp is used as the shadow call stack pointer instead */ -.macro load_global_pointer -.endm - -/* Load the per-CPU SSE shadow call stack to gp. */ -.macro scs_load_sse_stack reg_evt - REG_L gp, SSE_REG_EVT_SHADOW_STACK(\reg_evt) -.endm -#else -/* load __global_pointer to gp */ -.macro load_global_pointer -.option push -.option norelax - la gp, __global_pointer$ -.option pop -.endm - -/* Don't load shadow stack if SCS is disabled */ -.macro scs_load_sse_stack reg_evt -.endm -#endif /* CONFIG_SHADOW_CALL_STACK */ - SYM_CODE_START(handle_sse) /* Save stack temporarily */ REG_S sp, SSE_REG_EVT_TMP(a7) @@ -95,19 +71,18 @@ SYM_CODE_START(handle_sse) load_global_pointer scs_load_sse_stack a7 - /* Restore current task struct from __sse_entry_task */ - li t1, ASM_NR_CPUS - mv t3, zero - #ifdef CONFIG_SMP - REG_L t4, SSE_REG_HART_ID(a7) - REG_L t3, SSE_REG_CPU_ID(a7) + lw t4, SSE_REG_HART_ID(a7) + lw t3, SSE_REG_CPU_ID(a7) bne t4, a6, .Lfind_hart_id_slowpath .Lcpu_id_found: +#else + mv t3, zero #endif - asm_per_cpu_with_cpu t2 __sse_entry_task t1 t3 + + asm_per_cpu_with_cpu t2 __sbi_sse_entry_task t1 t3 REG_L tp, 0(t2) mv a1, sp /* pt_regs on stack */ @@ -117,9 +92,7 @@ SYM_CODE_START(handle_sse) * kernel in early exception path and thus, we don't know the content of * sscratch. */ - csrr s4, CSR_SSCRATCH - /* In-kernel scratch is 0 */ - csrw CSR_SCRATCH, x0 + csrrw s4, CSR_SSCRATCH, x0 mv a0, a7 @@ -174,8 +147,10 @@ SYM_CODE_START(handle_sse) #ifdef CONFIG_SMP .Lfind_hart_id_slowpath: -/* Slowpath to find the CPU id associated to the hart id */ -la t0, __cpuid_to_hartid_map + /* Restore current task struct from __sbi_sse_entry_task */ + li t1, ASM_NR_CPUS + /* Slowpath to find the CPU id associated to the hart id */ + la t0, __cpuid_to_hartid_map .Lhart_id_loop: REG_L t2, 0(t0) @@ -194,7 +169,6 @@ la t0, __cpuid_to_hartid_map la a0, sse_hart_id_panic_string la t0, panic jalr t0 - #endif SYM_CODE_END(handle_sse) diff --git a/arch/riscv/kernel/suspend_entry.S b/arch/riscv/kernel/suspend_entry.S index a59c4c9036963c453f8842937af7c91124cea54e..2d54f309c14059ad901f80448df2ff257f047388 100644 --- a/arch/riscv/kernel/suspend_entry.S +++ b/arch/riscv/kernel/suspend_entry.S @@ -61,10 +61,7 @@ SYM_FUNC_END(__cpu_suspend_enter) SYM_TYPED_FUNC_START(__cpu_resume_enter) /* Load the global pointer */ - .option push - .option norelax - la gp, __global_pointer$ - .option pop + load_global_pointer #ifdef CONFIG_MMU /* Save A0 and A1 */ diff --git a/arch/riscv/kernel/traps.c b/arch/riscv/kernel/traps.c index dbb20ce4534c82c712b02da1ec7de22d1ff248a6..d3b0a4299a2e2b45d30ad5b8201c849f01c158f0 100644 --- a/arch/riscv/kernel/traps.c +++ b/arch/riscv/kernel/traps.c @@ -146,6 +146,8 @@ asmlinkage __visible __trap_section void name(struct pt_regs *regs) \ DO_ERROR_INFO(do_trap_unknown, SIGILL, ILL_ILLTRP, "unknown exception"); +DO_ERROR_INFO(do_trap_hardware_error, + SIGBUS, BUS_MCEERR_AR, "hardware error"); DO_ERROR_INFO(do_trap_insn_misaligned, SIGBUS, BUS_ADRALN, "instruction address misaligned"); DO_ERROR_INFO(do_trap_insn_fault, @@ -360,34 +362,10 @@ static void noinstr handle_riscv_irq(struct pt_regs *regs) asmlinkage void noinstr do_irq(struct pt_regs *regs) { irqentry_state_t state = irqentry_enter(regs); -#ifdef CONFIG_IRQ_STACKS - if (on_thread_stack()) { - ulong *sp = per_cpu(irq_stack_ptr, smp_processor_id()) - + IRQ_STACK_SIZE/sizeof(ulong); - __asm__ __volatile( - "addi sp, sp, -"RISCV_SZPTR "\n" - REG_S" ra, (sp) \n" - "addi sp, sp, -"RISCV_SZPTR "\n" - REG_S" s0, (sp) \n" - "addi s0, sp, 2*"RISCV_SZPTR "\n" - "move sp, %[sp] \n" - "move a0, %[regs] \n" - "call handle_riscv_irq \n" - "addi sp, s0, -2*"RISCV_SZPTR"\n" - REG_L" s0, (sp) \n" - "addi sp, sp, "RISCV_SZPTR "\n" - REG_L" ra, (sp) \n" - "addi sp, sp, "RISCV_SZPTR "\n" - : - : [sp] "r" (sp), [regs] "r" (regs) - : "a0", "a1", "a2", "a3", "a4", "a5", "a6", "a7", - "t0", "t1", "t2", "t3", "t4", "t5", "t6", -#ifndef CONFIG_FRAME_POINTER - "s0", -#endif - "memory"); - } else -#endif + + if (IS_ENABLED(CONFIG_IRQ_STACKS) && on_thread_stack()) + call_on_irq_stack(regs, handle_riscv_irq); + else handle_riscv_irq(regs); irqentry_exit(regs, state); diff --git a/arch/riscv/kernel/vdso/Makefile b/arch/riscv/kernel/vdso/Makefile index d58f32a2035b1fee68119f0c92478a49211524e9..7de304acef6d5f179de1de38927eb94aa9d69fa5 100644 --- a/arch/riscv/kernel/vdso/Makefile +++ b/arch/riscv/kernel/vdso/Makefile @@ -37,7 +37,7 @@ CPPFLAGS_vdso.lds += -DHAS_VGETTIMEOFDAY endif # Disable -pg to prevent insert call site -CFLAGS_REMOVE_vgettimeofday.o = $(CC_FLAGS_FTRACE) +CFLAGS_REMOVE_vgettimeofday.o = $(CC_FLAGS_FTRACE) $(CC_FLAGS_SCS) # Disable profiling and instrumentation for VDSO code GCOV_PROFILE := n diff --git a/arch/riscv/purgatory/Makefile b/arch/riscv/purgatory/Makefile index 9e6476719abbbb4e9206cd89292be7f25261a5b6..6a3c16bd5ca3bb7fbd357e48f6b19cd5e7a360b0 100644 --- a/arch/riscv/purgatory/Makefile +++ b/arch/riscv/purgatory/Makefile @@ -81,6 +81,10 @@ ifdef CONFIG_CFI_CLANG PURGATORY_CFLAGS_REMOVE += $(CC_FLAGS_CFI) endif +ifdef CONFIG_SHADOW_CALL_STACK +PURGATORY_CFLAGS_REMOVE += $(CC_FLAGS_SCS) +endif + CFLAGS_REMOVE_purgatory.o += $(PURGATORY_CFLAGS_REMOVE) CFLAGS_purgatory.o += $(PURGATORY_CFLAGS) diff --git a/drivers/acpi/apei/Kconfig b/drivers/acpi/apei/Kconfig index 897ff8ce76370d60af378cdba61413485c922f18..ada95a50805fbf9bbb3e06cfb6e9c4366b859278 100644 --- a/drivers/acpi/apei/Kconfig +++ b/drivers/acpi/apei/Kconfig @@ -48,7 +48,7 @@ config ACPI_APEI_SEA config ACPI_APEI_SSE bool - depends on RISCV && RISCV_SSE && ACPI_APEI_GHES + depends on RISCV && RISCV_SBI_SSE && ACPI_APEI_GHES default y config ACPI_APEI_MEMORY_FAILURE diff --git a/drivers/acpi/apei/ghes.c b/drivers/acpi/apei/ghes.c index 648e86e3000f05ac1417be796810ef346166b07d..59ffb6e9b0c24a0a9f57e7090e2f2b3733c2316f 100644 --- a/drivers/acpi/apei/ghes.c +++ b/drivers/acpi/apei/ghes.c @@ -17,7 +17,8 @@ * Author: Huang Ying */ -#include +#include +#include #include #include #include @@ -98,7 +99,7 @@ #define FIX_APEI_GHES_SDEI_CRITICAL __end_of_fixed_addresses #endif -#ifndef CONFIG_RISCV_SSE +#ifndef CONFIG_RISCV_SBI_SSE #define FIX_APEI_GHES_SSE_LOW_PRIORITY __end_of_fixed_addresses #define FIX_APEI_GHES_SSE_HIGH_PRIORITY __end_of_fixed_addresses #endif @@ -1675,7 +1676,8 @@ static int ghes_probe(struct platform_device *ghes_dev) case ACPI_HEST_NOTIFY_SSE: rc = apei_sse_register_ghes(ghes); if (rc) { - pr_err(GHES_PFX "Failed to register for SSE notification on vector %d\n", + pr_err(GHES_PFX "Failed to register for SSE notification" + " on vector %d\n", generic->notify.vector); goto err; } @@ -1746,6 +1748,7 @@ static int ghes_remove(struct platform_device *ghes_dev) case ACPI_HEST_NOTIFY_SOFTWARE_DELEGATED: apei_sdei_unregister_ghes(ghes); break; + case ACPI_HEST_NOTIFY_SSE: apei_sse_unregister_ghes(ghes); break; diff --git a/drivers/firmware/riscv/Kconfig b/drivers/firmware/riscv/Kconfig index 8056ed3262d912ec4bb155f917aa8b8295f60a01..ed5b663ac5f91badc4b4d298a9e4ba129542a41c 100644 --- a/drivers/firmware/riscv/Kconfig +++ b/drivers/firmware/riscv/Kconfig @@ -2,13 +2,13 @@ menu "Risc-V Specific firmware drivers" depends on RISCV -config RISCV_SSE +config RISCV_SBI_SSE bool "Enable SBI Supervisor Software Events support" depends on RISCV_SBI default y help - The Supervisor Software Events support allow the SBI to deliver - NMI-like notifications to the supervisor mode software. When enable, + The Supervisor Software Events support allows the SBI to deliver + NMI-like notifications to the supervisor mode software. When enabled, this option provides support to register callbacks on specific SSE events. diff --git a/drivers/firmware/riscv/Makefile b/drivers/firmware/riscv/Makefile index 4ccfcbbc28ea3eff92eaee954ebfc55fafda06f4..c8795d4bbb2eab410d2310701728c210cf628002 100644 --- a/drivers/firmware/riscv/Makefile +++ b/drivers/firmware/riscv/Makefile @@ -1,3 +1,3 @@ # SPDX-License-Identifier: GPL-2.0 -obj-$(CONFIG_RISCV_SSE) += riscv_sse.o +obj-$(CONFIG_RISCV_SBI_SSE) += riscv_sbi_sse.o diff --git a/drivers/firmware/riscv/riscv_sse.c b/drivers/firmware/riscv/riscv_sbi_sse.c similarity index 93% rename from drivers/firmware/riscv/riscv_sse.c rename to drivers/firmware/riscv/riscv_sbi_sse.c index 672b9970ad5df87df36a050b9bf45c2535d34877..ee288a0c1680778b632e4457188049ae858c6299 100644 --- a/drivers/firmware/riscv/riscv_sse.c +++ b/drivers/firmware/riscv/riscv_sbi_sse.c @@ -1,6 +1,6 @@ -// SPDX-License-Identifier: GPL-2.0-only +// SPDX-License-Identifier: GPL-2.0-or-later /* - * Copyright (C) 2024 Rivos Inc. + * Copyright (C) 2025 Rivos Inc. */ #define pr_fmt(fmt) "sse: " fmt @@ -14,7 +14,7 @@ #include #include #include -#include +#include #include #include @@ -24,7 +24,7 @@ struct sse_event { struct list_head list; u32 evt_id; u32 priority; - sse_event_handler *handler; + sse_event_handler_fn *handler; void *handler_arg; /* Only valid for global events */ unsigned int cpu; @@ -182,7 +182,7 @@ static void sse_global_event_update_cpu(struct sse_event *event, static int sse_event_set_target_cpu_nolock(struct sse_event *event, unsigned int cpu) { - unsigned int hart_id = cpuid_to_hartid_map(cpu); + unsigned long hart_id = cpuid_to_hartid_map(cpu); struct sse_registered_event *reg_evt = event->global; u32 evt = event->evt_id; bool was_enabled; @@ -292,7 +292,8 @@ static int sse_event_alloc_local(struct sse_event *event) } static struct sse_event *sse_event_alloc(u32 evt, u32 priority, - sse_event_handler *handler, void *arg) + sse_event_handler_fn *handler, + void *arg) { int err; struct sse_event *event; @@ -390,8 +391,8 @@ static void sse_event_free(struct sse_event *event) kfree(event); } -static void sse_on_each_cpu(struct sse_event *event, unsigned long func, - unsigned long revert_func) +static int sse_on_each_cpu(struct sse_event *event, unsigned long func, + unsigned long revert_func) { struct sse_per_cpu_evt cpu_evt; @@ -409,7 +410,11 @@ static void sse_on_each_cpu(struct sse_event *event, unsigned long func, cpumask_andnot(&revert, cpu_online_mask, &cpu_evt.error); cpu_evt.func = revert_func; on_each_cpu_mask(&revert, sse_event_per_cpu_func, &cpu_evt, 1); + + return -EIO; } + + return 0; } int sse_event_enable(struct sse_event *event) @@ -421,8 +426,9 @@ int sse_event_enable(struct sse_event *event) if (sse_event_is_global(event->evt_id)) { ret = sse_event_enable_local(event); } else { - sse_on_each_cpu(event, SBI_SSE_EVENT_ENABLE, - SBI_SSE_EVENT_DISABLE); + ret = sse_on_each_cpu(event, + SBI_SSE_EVENT_ENABLE, + SBI_SSE_EVENT_DISABLE); } } } @@ -434,7 +440,7 @@ static int sse_events_mask(void) { struct sbiret ret; - ret = sbi_ecall(SBI_EXT_SSE, SBI_SSE_EVENT_HART_MASK, 0, 0, 0, 0, 0, 0); + ret = sbi_ecall(SBI_EXT_SSE, SBI_SSE_HART_MASK, 0, 0, 0, 0, 0, 0); return sbi_err_map_linux_errno(ret.error); } @@ -443,7 +449,7 @@ static int sse_events_unmask(void) { struct sbiret ret; - ret = sbi_ecall(SBI_EXT_SSE, SBI_SSE_EVENT_HART_UNMASK, 0, 0, 0, 0, 0, 0); + ret = sbi_ecall(SBI_EXT_SSE, SBI_SSE_HART_UNMASK, 0, 0, 0, 0, 0, 0); return sbi_err_map_linux_errno(ret.error); } @@ -472,7 +478,7 @@ void sse_event_disable(struct sse_event *event) EXPORT_SYMBOL_GPL(sse_event_disable); struct sse_event *sse_event_register(u32 evt, u32 priority, - sse_event_handler *handler, void *arg) + sse_event_handler_fn *handler, void *arg) { struct sse_event *event; int cpu; @@ -494,8 +500,8 @@ struct sse_event *sse_event_register(u32 evt, u32 priority, unsigned long preferred_hart; ret = sse_event_attr_get_no_lock(event->global, - SBI_SSE_ATTR_PREFERRED_HART, - &preferred_hart); + SBI_SSE_ATTR_PREFERRED_HART, + &preferred_hart); if (ret) goto err_event_free; @@ -507,8 +513,10 @@ struct sse_event *sse_event_register(u32 evt, u32 priority, goto err_event_free; } else { - sse_on_each_cpu(event, SBI_SSE_EVENT_REGISTER, - SBI_SSE_EVENT_DISABLE); + ret = sse_on_each_cpu(event, SBI_SSE_EVENT_REGISTER, + SBI_SSE_EVENT_DISABLE); + if (ret) + goto err_event_free; } } @@ -575,6 +583,7 @@ static int sse_cpu_teardown(unsigned int cpu) int ret = 0; unsigned int next_cpu; struct sse_event *event; + struct sse_registered_event *reg_evt; /* Mask the sse events */ ret = sse_events_mask(); @@ -583,8 +592,10 @@ static int sse_cpu_teardown(unsigned int cpu) scoped_guard(spinlock, &events_list_lock) { list_for_each_entry(event, &events, list) { + /* Disable local event on current cpu */ if (!sse_event_is_global(event->evt_id)) { - if (event->global->is_enabled) + reg_evt = sse_get_reg_evt(event); + if (reg_evt->is_enabled) sse_event_disable_local(event); sse_sbi_unregister_event(event); @@ -603,16 +614,6 @@ static int sse_cpu_teardown(unsigned int cpu) return ret; } -static void sse_reset(void) -{ - struct sse_event *event; - - list_for_each_entry(event, &events, list) { - sse_event_disable_nolock(event); - sse_event_unregister_nolock(event); - } -} - static int sse_pm_notifier(struct notifier_block *nb, unsigned long action, void *data) { @@ -644,7 +645,6 @@ static int sse_reboot_notifier(struct notifier_block *nb, unsigned long action, void *data) { cpuhp_remove_state(sse_hp_state); - sse_reset(); return NOTIFY_OK; } @@ -658,7 +658,7 @@ static int __init sse_init(void) int ret; if (sbi_probe_extension(SBI_EXT_SSE) <= 0) { - pr_err("Missing SBI SSE extension\n"); + pr_info("Missing SBI SSE extension\n"); return -EOPNOTSUPP; } pr_info("SBI SSE extension detected\n"); @@ -698,7 +698,7 @@ arch_initcall(sse_init); struct sse_ghes_callback { struct list_head head; struct ghes *ghes; - sse_event_handler *callback; + sse_event_handler_fn *callback; }; struct sse_ghes_event_data { @@ -724,8 +724,8 @@ static int sse_ghes_handler(u32 event_num, void *arg, struct pt_regs *regs) return 0; } -int sse_register_ghes(struct ghes *ghes, sse_event_handler *lo_cb, - sse_event_handler *hi_cb) +int sse_register_ghes(struct ghes *ghes, sse_event_handler_fn *lo_cb, + sse_event_handler_fn *hi_cb) { struct sse_ghes_event_data *ev_data, *evd; struct sse_ghes_callback *cb; diff --git a/drivers/misc/lkdtm/cfi.c b/drivers/misc/lkdtm/cfi.c index fc28714ae3a610e44c361e8c608a70148fe02f91..6a33889d0902af7430d0d51451148787e8cbacf0 100644 --- a/drivers/misc/lkdtm/cfi.c +++ b/drivers/misc/lkdtm/cfi.c @@ -68,12 +68,20 @@ static void lkdtm_CFI_FORWARD_PROTO(void) #define no_pac_addr(addr) \ ((__force __typeof__(addr))((uintptr_t)(addr) | PAGE_OFFSET)) +#ifdef CONFIG_RISCV +/* https://github.com/riscv-non-isa/riscv-elf-psabi-doc/blob/master/riscv-cc.adoc#frame-pointer-convention */ +#define FRAME_RA_OFFSET (-1) +#else +#define FRAME_RA_OFFSET 1 +#endif + /* The ultimate ROP gadget. */ static noinline __no_ret_protection void set_return_addr_unchecked(unsigned long *expected, unsigned long *addr) { /* Use of volatile is to make sure final write isn't seen as a dead store. */ - unsigned long * volatile *ret_addr = (unsigned long **)__builtin_frame_address(0) + 1; + unsigned long * volatile *ret_addr = + (unsigned long **)__builtin_frame_address(0) + FRAME_RA_OFFSET; /* Make sure we've found the right place on the stack before writing it. */ if (no_pac_addr(*ret_addr) == expected) @@ -88,7 +96,8 @@ static noinline void set_return_addr(unsigned long *expected, unsigned long *addr) { /* Use of volatile is to make sure final write isn't seen as a dead store. */ - unsigned long * volatile *ret_addr = (unsigned long **)__builtin_frame_address(0) + 1; + unsigned long * volatile *ret_addr = + (unsigned long **)__builtin_frame_address(0) + FRAME_RA_OFFSET; /* Make sure we've found the right place on the stack before writing it. */ if (no_pac_addr(*ret_addr) == expected) diff --git a/drivers/perf/Kconfig b/drivers/perf/Kconfig index a433c5693c6c74dbd49a010fe6eed10b25a3d0b9..451916a7b1dee97a0ada5ecbd611ebb424e8b096 100644 --- a/drivers/perf/Kconfig +++ b/drivers/perf/Kconfig @@ -98,6 +98,16 @@ config RISCV_PMU_SBI full perf feature support i.e. counter overflow, privilege mode filtering, counter configuration. +config RISCV_PMU_SBI_SSE + depends on RISCV_PMU && RISCV_SBI_SSE + bool "RISC-V PMU SSE events" + default n + help + Say y if you want to use SSE events to deliver PMU interrupts. This + provides a way to profile the kernel at any level by using NMI-like + SSE events. SSE events being really intrusive, this option allows + to select it only if needed. + config ANDES_CUSTOM_PMU bool "Andes custom PMU support" depends on ARCH_RENESAS && RISCV_ALTERNATIVE && RISCV_PMU_SBI diff --git a/drivers/perf/riscv_pmu.c b/drivers/perf/riscv_pmu.c index 0a02e85a895138e4e976bc66ad80fa269dad5828..1e98352591691ab6822e7d1942c3ff323ee27653 100644 --- a/drivers/perf/riscv_pmu.c +++ b/drivers/perf/riscv_pmu.c @@ -13,6 +13,7 @@ #include #include #include +#include #include #include @@ -256,6 +257,24 @@ void riscv_pmu_start(struct perf_event *event, int flags) perf_event_update_userpage(event); } +#ifdef CONFIG_RISCV_PMU_SBI_SSE +static void riscv_pmu_disable(struct pmu *pmu) +{ + struct riscv_pmu *rvpmu = to_riscv_pmu(pmu); + + if (rvpmu->sse_evt) + sse_event_disable_local(rvpmu->sse_evt); +} + +static void riscv_pmu_enable(struct pmu *pmu) +{ + struct riscv_pmu *rvpmu = to_riscv_pmu(pmu); + + if (rvpmu->sse_evt) + sse_event_enable_local(rvpmu->sse_evt); +} +#endif + static int riscv_pmu_add(struct perf_event *event, int flags) { struct riscv_pmu *rvpmu = to_riscv_pmu(event->pmu); @@ -413,6 +432,10 @@ struct riscv_pmu *riscv_pmu_alloc(void) .event_mapped = riscv_pmu_event_mapped, .event_unmapped = riscv_pmu_event_unmapped, .event_idx = riscv_pmu_event_idx, +#ifdef CONFIG_RISCV_PMU_SBI_SSE + .pmu_enable = riscv_pmu_enable, + .pmu_disable = riscv_pmu_disable, +#endif .add = riscv_pmu_add, .del = riscv_pmu_del, .start = riscv_pmu_start, diff --git a/drivers/perf/riscv_pmu_sbi.c b/drivers/perf/riscv_pmu_sbi.c index f0e0e32c8b716d132af91a886ec4c1ecef4262b0..fc5c80d6a61abd6e62387066b9894e924adde516 100644 --- a/drivers/perf/riscv_pmu_sbi.c +++ b/drivers/perf/riscv_pmu_sbi.c @@ -17,6 +17,7 @@ #include #include #include +#include #include #include #include @@ -898,10 +899,10 @@ static void pmu_sbi_start_overflow_mask(struct riscv_pmu *pmu, pmu_sbi_start_ovf_ctrs_sbi(cpu_hw_evt, ctr_ovf_mask); } -static irqreturn_t pmu_sbi_ovf_handler(int irq, void *dev) +static irqreturn_t pmu_sbi_ovf_handler(struct cpu_hw_events *cpu_hw_evt, + struct pt_regs *regs, bool from_sse) { struct perf_sample_data data; - struct pt_regs *regs; struct hw_perf_event *hw_evt; union sbi_pmu_ctr_info *info; int lidx, hidx, fidx; @@ -909,7 +910,6 @@ static irqreturn_t pmu_sbi_ovf_handler(int irq, void *dev) struct perf_event *event; u64 overflow; u64 overflowed_ctrs = 0; - struct cpu_hw_events *cpu_hw_evt = dev; u64 start_clock = sched_clock(); struct riscv_pmu_snapshot_data *sdata = cpu_hw_evt->snapshot_addr; @@ -919,13 +919,15 @@ static irqreturn_t pmu_sbi_ovf_handler(int irq, void *dev) /* Firmware counter don't support overflow yet */ fidx = find_first_bit(cpu_hw_evt->used_hw_ctrs, RISCV_MAX_COUNTERS); if (fidx == RISCV_MAX_COUNTERS) { - csr_clear(CSR_SIP, BIT(riscv_pmu_irq_num)); + if (!from_sse) + csr_clear(CSR_SIP, BIT(riscv_pmu_irq_num)); return IRQ_NONE; } event = cpu_hw_evt->events[fidx]; if (!event) { - ALT_SBI_PMU_OVF_CLEAR_PENDING(riscv_pmu_irq_mask); + if (!from_sse) + ALT_SBI_PMU_OVF_CLEAR_PENDING(riscv_pmu_irq_mask); return IRQ_NONE; } @@ -940,16 +942,16 @@ static irqreturn_t pmu_sbi_ovf_handler(int irq, void *dev) /* * Overflow interrupt pending bit should only be cleared after stopping - * all the counters to avoid any race condition. + * all the counters to avoid any race condition. When using SSE, + * interrupt is cleared when stopping counters. */ - ALT_SBI_PMU_OVF_CLEAR_PENDING(riscv_pmu_irq_mask); + if (!from_sse) + ALT_SBI_PMU_OVF_CLEAR_PENDING(riscv_pmu_irq_mask); /* No overflow bit is set */ if (!overflow) return IRQ_NONE; - regs = get_irq_regs(); - for_each_set_bit(lidx, cpu_hw_evt->used_hw_ctrs, RISCV_MAX_COUNTERS) { struct perf_event *event = cpu_hw_evt->events[lidx]; @@ -1005,6 +1007,51 @@ static irqreturn_t pmu_sbi_ovf_handler(int irq, void *dev) return IRQ_HANDLED; } +static irqreturn_t pmu_sbi_ovf_irq_handler(int irq, void *dev) +{ + return pmu_sbi_ovf_handler(dev, get_irq_regs(), false); +} + +#ifdef CONFIG_RISCV_PMU_SBI_SSE +static int pmu_sbi_ovf_sse_handler(u32 evt, void *arg, struct pt_regs *regs) +{ + struct cpu_hw_events __percpu *hw_events = arg; + struct cpu_hw_events *hw_event = raw_cpu_ptr(hw_events); + + pmu_sbi_ovf_handler(hw_event, regs, true); + + return 0; +} + +static int pmu_sbi_setup_sse(struct riscv_pmu *pmu) +{ + int ret; + struct sse_event *evt; + struct cpu_hw_events __percpu *hw_events = pmu->hw_events; + + evt = sse_event_register(SBI_SSE_EVENT_LOCAL_PMU_OVERFLOW, 0, + pmu_sbi_ovf_sse_handler, hw_events); + if (IS_ERR(evt)) + return PTR_ERR(evt); + + ret = sse_event_enable(evt); + if (ret) { + sse_event_unregister(evt); + return ret; + } + + pr_info("using SSE for PMU event delivery\n"); + pmu->sse_evt = evt; + + return ret; +} +#else +static int pmu_sbi_setup_sse(struct riscv_pmu *pmu) +{ + return -EOPNOTSUPP; +} +#endif + static int pmu_sbi_starting_cpu(unsigned int cpu, struct hlist_node *node) { struct riscv_pmu *pmu = hlist_entry_safe(node, struct riscv_pmu, node); @@ -1056,6 +1103,10 @@ static int pmu_sbi_setup_irqs(struct riscv_pmu *pmu, struct platform_device *pde struct cpu_hw_events __percpu *hw_events = pmu->hw_events; struct irq_domain *domain = NULL; + ret = pmu_sbi_setup_sse(pmu); + if (!ret) + return 0; + if (riscv_isa_extension_available(NULL, SSCOFPMF)) { riscv_pmu_irq_num = RV_IRQ_PMU; riscv_pmu_use_irq = true; @@ -1090,7 +1141,7 @@ static int pmu_sbi_setup_irqs(struct riscv_pmu *pmu, struct platform_device *pde return -ENODEV; } - ret = request_percpu_irq(riscv_pmu_irq, pmu_sbi_ovf_handler, "riscv-pmu", hw_events); + ret = request_percpu_irq(riscv_pmu_irq, pmu_sbi_ovf_irq_handler, "riscv-pmu", hw_events); if (ret) { pr_err("registering percpu irq failed [%d]\n", ret); return ret; diff --git a/include/linux/perf/riscv_pmu.h b/include/linux/perf/riscv_pmu.h index 701974639ff2ca12c00d93fa2c4989d7520833ae..cd493fcab9b3286fb1404b476edc1143e68852d3 100644 --- a/include/linux/perf/riscv_pmu.h +++ b/include/linux/perf/riscv_pmu.h @@ -28,6 +28,8 @@ #define RISCV_PMU_CONFIG1_GUEST_EVENTS 0x1 +struct sse_event; + struct cpu_hw_events { /* currently enabled events */ int n_events; @@ -54,6 +56,9 @@ struct riscv_pmu { char *name; irqreturn_t (*handle_irq)(int irq_num, void *dev); +#ifdef CONFIG_RISCV_PMU_SBI_SSE + struct sse_event *sse_evt; +#endif unsigned long cmask; u64 (*ctr_read)(struct perf_event *event); diff --git a/include/linux/riscv_sse.h b/include/linux/riscv_sbi_sse.h similarity index 68% rename from include/linux/riscv_sse.h rename to include/linux/riscv_sbi_sse.h index f4a0a7add79062e6491cc7e3cbdfe82371d5ad58..be0c9ad0a00c1fc276fda9faa1d1f1dc42d94a5d 100644 --- a/include/linux/riscv_sse.h +++ b/include/linux/riscv_sbi_sse.h @@ -1,32 +1,34 @@ -/* SPDX-License-Identifier: GPL-2.0 */ +/* SPDX-License-Identifier: GPL-2.0-or-later */ /* - * Copyright (C) 2024 Rivos Inc. + * Copyright (C) 2025 Rivos Inc. */ -#ifndef __LINUX_RISCV_SSE_H -#define __LINUX_RISCV_SSE_H +#ifndef __LINUX_RISCV_SBI_SSE_H +#define __LINUX_RISCV_SBI_SSE_H #include #include -#include -#include struct sse_event; struct pt_regs; - struct ghes; -typedef int (sse_event_handler)(u32 event_num, void *arg, struct pt_regs *regs); +typedef int (sse_event_handler_fn)(u32 event_num, void *arg, + struct pt_regs *regs); -#ifdef CONFIG_RISCV_SSE +#ifdef CONFIG_RISCV_SBI_SSE struct sse_event *sse_event_register(u32 event_num, u32 priority, - sse_event_handler *handler, void *arg); + sse_event_handler_fn *handler, void *arg); void sse_event_unregister(struct sse_event *evt); int sse_event_set_target_cpu(struct sse_event *sse_evt, unsigned int cpu); +int sse_register_ghes(struct ghes *ghes, sse_event_handler_fn *lo_cb, + sse_event_handler_fn *hi_cb); +int sse_unregister_ghes(struct ghes *ghes); + int sse_event_enable(struct sse_event *sse_evt); void sse_event_disable(struct sse_event *sse_evt); @@ -34,12 +36,9 @@ void sse_event_disable(struct sse_event *sse_evt); int sse_event_enable_local(struct sse_event *sse_evt); int sse_event_disable_local(struct sse_event *sse_evt); -int sse_register_ghes(struct ghes *ghes, sse_event_handler *lo_cb, - sse_event_handler *hi_cb); -int sse_unregister_ghes(struct ghes *ghes); #else static inline struct sse_event *sse_event_register(u32 event_num, u32 priority, - sse_event_handler *handler, + sse_event_handler_fn *handler, void *arg) { return ERR_PTR(-EOPNOTSUPP); @@ -53,23 +52,22 @@ static inline int sse_event_set_target_cpu(struct sse_event *sse_evt, return -EOPNOTSUPP; } -static inline int sse_event_enable(struct sse_event *sse_evt) +static inline int sse_register_ghes(struct ghes *ghes, sse_event_handler_fn *lo_cb, + sse_event_handler_fn *hi_cb) { return -EOPNOTSUPP; } -static inline void sse_event_disable(struct sse_event *sse_evt) {} - -static inline int sse_register_ghes(struct ghes *ghes, sse_event_handler *lo_cb, - sse_event_handler *hi_cb) +static inline int sse_unregister_ghes(struct ghes *ghes) { return -EOPNOTSUPP; } -static inline int sse_unregister_ghes(struct ghes *ghes) +static inline int sse_event_enable(struct sse_event *sse_evt) { return -EOPNOTSUPP; } +static inline void sse_event_disable(struct sse_event *sse_evt) {} #endif -#endif /* __LINUX_RISCV_SSE_H */ +#endif /* __LINUX_RISCV_SBI_SSE_H */ diff --git a/tools/testing/selftests/riscv/sse/module/riscv_sse_test.c b/tools/testing/selftests/riscv/sse/module/riscv_sse_test.c index 65df41a2d40ae989247d41f91204262278fa2aa6..fcff1e346fee33066bfeb78918aed2d00b4ff793 100644 --- a/tools/testing/selftests/riscv/sse/module/riscv_sse_test.c +++ b/tools/testing/selftests/riscv/sse/module/riscv_sse_test.c @@ -1,6 +1,6 @@ -// SPDX-License-Identifier: GPL-2.0-only +// SPDX-License-Identifier: GPL-2.0-or-later /* - * Copyright (C) 2024 Rivos Inc. + * Copyright (C) 2025 Rivos Inc. */ #define pr_fmt(fmt) "riscv_sse_test: " fmt @@ -11,7 +11,7 @@ #include #include #include -#include +#include #include #include @@ -79,14 +79,14 @@ static const char *sse_evt_name(u32 evt) { struct sse_event_desc *desc = sse_get_evt_desc(evt); - return desc != NULL ? desc->name : NULL; + return desc ? desc->name : NULL; } static bool sse_test_can_inject_event(u32 evt) { struct sse_event_desc *desc = sse_get_evt_desc(evt); - return desc != NULL ? desc->can_inject : false; + return desc ? desc->can_inject : false; } static struct sbiret sbi_sse_ecall(int fid, unsigned long arg0, unsigned long arg1) @@ -121,7 +121,7 @@ static int sse_test_signal(u32 evt, unsigned int cpu) unsigned int hart_id = cpuid_to_hartid_map(cpu); struct sbiret ret; - ret = sbi_sse_ecall(SBI_SSE_EVENT_SIGNAL, evt, hart_id); + ret = sbi_sse_ecall(SBI_SSE_EVENT_INJECT, evt, hart_id); if (ret.error) { sse_err("Failed to signal event %x, error %ld\n", evt, ret.error); return sbi_err_map_linux_errno(ret.error); @@ -210,7 +210,7 @@ static void sse_run_fast_test(struct fast_test_arg *test_arg, struct sse_event * timeout = jiffies + HZ / 100; /* We can not use since they are not NMI safe */ while (!READ_ONCE(test_arg->completion) && - time_before(jiffies, timeout)) { + time_before(jiffies, timeout)) { cpu_relax(); } if (!time_before(jiffies, timeout)) { @@ -287,7 +287,7 @@ static int sse_hi_priority_test_handler(u32 evt, void *arg, sse_test_signal(next->evt, smp_processor_id()); if (!READ_ONCE(next->called)) { sse_err("Higher priority event %s was not handled %s\n", - sse_evt_name(next->evt), sse_evt_name(evt)); + sse_evt_name(next->evt), sse_evt_name(evt)); } } @@ -313,7 +313,7 @@ static int sse_low_priority_test_handler(u32 evt, void *arg, struct pt_regs *reg } static void sse_test_injection_priority_arg(struct priority_test_arg *args, unsigned int args_size, - sse_event_handler handler, const char *test_name) + sse_event_handler_fn handler, const char *test_name) { unsigned int i; int ret; @@ -382,11 +382,10 @@ static void sse_test_injection_priority_arg(struct priority_test_arg *args, unsi while (arg) { if (!READ_ONCE(arg->called)) { sse_err("Event %s handler was not called\n", - sse_evt_name(arg->evt)); + sse_evt_name(arg->evt)); ret = -EINVAL; } - event = arg->event; arg = READ_ONCE(arg->next_evt_arg); } @@ -463,7 +462,6 @@ static void sse_test_injection_priority(void) sse_low_priority_test_handler, "same_prio_args"); } - static bool sse_get_inject_status(u32 evt) { int ret;