diff options
Diffstat (limited to 'arch/riscv')
36 files changed, 976 insertions, 247 deletions
diff --git a/arch/riscv/Kconfig b/arch/riscv/Kconfig index 6c4bce7cad8a..7b5905529146 100644 --- a/arch/riscv/Kconfig +++ b/arch/riscv/Kconfig @@ -13,11 +13,14 @@ config 32BIT config RISCV def_bool y select ARCH_CLOCKSOURCE_INIT + select ARCH_SUPPORTS_ATOMIC_RMW select ARCH_HAS_BINFMT_FLAT + select ARCH_HAS_DEBUG_VM_PGTABLE select ARCH_HAS_DEBUG_VIRTUAL if MMU select ARCH_HAS_DEBUG_WX select ARCH_HAS_GCOV_PROFILE_ALL select ARCH_HAS_GIGANTIC_PAGE + select ARCH_HAS_KCOV select ARCH_HAS_MMIOWB select ARCH_HAS_PTE_SPECIAL select ARCH_HAS_SET_DIRECT_MAP @@ -47,6 +50,8 @@ config RISCV select GENERIC_TIME_VSYSCALL if MMU && 64BIT select HANDLE_DOMAIN_IRQ select HAVE_ARCH_AUDITSYSCALL + select HAVE_ARCH_JUMP_LABEL + select HAVE_ARCH_JUMP_LABEL_RELATIVE select HAVE_ARCH_KASAN if MMU && 64BIT select HAVE_ARCH_KGDB select HAVE_ARCH_KGDB_QXFER_PKT @@ -54,14 +59,18 @@ config RISCV select HAVE_ARCH_SECCOMP_FILTER select HAVE_ARCH_TRACEHOOK select HAVE_ASM_MODVERSIONS + select HAVE_CONTEXT_TRACKING + select HAVE_DEBUG_KMEMLEAK select HAVE_DMA_CONTIGUOUS if MMU select HAVE_EBPF_JIT if MMU select HAVE_FUTEX_CMPXCHG if FUTEX + select HAVE_GCC_PLUGINS select HAVE_GENERIC_VDSO if MMU && 64BIT select HAVE_PCI select HAVE_PERF_EVENTS select HAVE_PERF_REGS select HAVE_PERF_USER_STACK_DUMP + select HAVE_STACKPROTECTOR select HAVE_SYSCALL_TRACEPOINTS select IRQ_DOMAIN select MODULES_USE_ELF_RELA if MODULES @@ -179,6 +188,9 @@ config PGTABLE_LEVELS default 3 if 64BIT default 2 +config LOCKDEP_SUPPORT + def_bool y + source "arch/riscv/Kconfig.socs" menu "Platform type" diff --git a/arch/riscv/boot/Makefile b/arch/riscv/boot/Makefile index 3530c59b3ea7..c59fca695f9d 100644 --- a/arch/riscv/boot/Makefile +++ b/arch/riscv/boot/Makefile @@ -14,6 +14,8 @@ # Based on the ia64 and arm64 boot/Makefile. # +KCOV_INSTRUMENT := n + OBJCOPYFLAGS_Image :=-O binary -R .note -R .note.gnu.build-id -R .comment -S targets := Image loader diff --git a/arch/riscv/configs/defconfig b/arch/riscv/configs/defconfig index 4da4886246a4..d58c93efb603 100644 --- a/arch/riscv/configs/defconfig +++ b/arch/riscv/configs/defconfig @@ -17,6 +17,7 @@ CONFIG_BPF_SYSCALL=y CONFIG_SOC_SIFIVE=y CONFIG_SOC_VIRT=y CONFIG_SMP=y +CONFIG_JUMP_LABEL=y CONFIG_MODULES=y CONFIG_MODULE_UNLOAD=y CONFIG_NET=y diff --git a/arch/riscv/configs/nommu_k210_defconfig b/arch/riscv/configs/nommu_k210_defconfig index b48138e329ea..cd1df62b13c7 100644 --- a/arch/riscv/configs/nommu_k210_defconfig +++ b/arch/riscv/configs/nommu_k210_defconfig @@ -33,6 +33,7 @@ CONFIG_SMP=y CONFIG_NR_CPUS=2 CONFIG_CMDLINE="earlycon console=ttySIF0" CONFIG_CMDLINE_FORCE=y +CONFIG_JUMP_LABEL=y # CONFIG_BLOCK is not set CONFIG_BINFMT_FLAT=y # CONFIG_COREDUMP is not set diff --git a/arch/riscv/configs/nommu_virt_defconfig b/arch/riscv/configs/nommu_virt_defconfig index cf74e179bf90..f27596e9663e 100644 --- a/arch/riscv/configs/nommu_virt_defconfig +++ b/arch/riscv/configs/nommu_virt_defconfig @@ -30,6 +30,7 @@ CONFIG_MAXPHYSMEM_2GB=y CONFIG_SMP=y CONFIG_CMDLINE="root=/dev/vda rw earlycon=uart8250,mmio,0x10000000,115200n8 console=ttyS0" CONFIG_CMDLINE_FORCE=y +CONFIG_JUMP_LABEL=y # CONFIG_BLK_DEV_BSG is not set CONFIG_PARTITION_ADVANCED=y # CONFIG_MSDOS_PARTITION is not set diff --git a/arch/riscv/configs/rv32_defconfig b/arch/riscv/configs/rv32_defconfig index 05bbf5240569..3a55f0e00d6c 100644 --- a/arch/riscv/configs/rv32_defconfig +++ b/arch/riscv/configs/rv32_defconfig @@ -17,6 +17,7 @@ CONFIG_BPF_SYSCALL=y CONFIG_SOC_VIRT=y CONFIG_ARCH_RV32I=y CONFIG_SMP=y +CONFIG_JUMP_LABEL=y CONFIG_MODULES=y CONFIG_MODULE_UNLOAD=y CONFIG_NET=y diff --git a/arch/riscv/include/asm/irq_work.h b/arch/riscv/include/asm/irq_work.h new file mode 100644 index 000000000000..d6c277992f76 --- /dev/null +++ b/arch/riscv/include/asm/irq_work.h @@ -0,0 +1,10 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef _ASM_RISCV_IRQ_WORK_H +#define _ASM_RISCV_IRQ_WORK_H + +static inline bool arch_irq_work_has_interrupt(void) +{ + return true; +} +extern void arch_irq_work_raise(void); +#endif /* _ASM_RISCV_IRQ_WORK_H */ diff --git a/arch/riscv/include/asm/jump_label.h b/arch/riscv/include/asm/jump_label.h new file mode 100644 index 000000000000..38af2ec7b9bf --- /dev/null +++ b/arch/riscv/include/asm/jump_label.h @@ -0,0 +1,60 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * Copyright (C) 2020 Emil Renner Berthing + * + * Based on arch/arm64/include/asm/jump_label.h + */ +#ifndef __ASM_JUMP_LABEL_H +#define __ASM_JUMP_LABEL_H + +#ifndef __ASSEMBLY__ + +#include <linux/types.h> +#include <asm/asm.h> + +#define JUMP_LABEL_NOP_SIZE 4 + +static __always_inline bool arch_static_branch(struct static_key *key, + bool branch) +{ + asm_volatile_goto( + " .option push \n\t" + " .option norelax \n\t" + " .option norvc \n\t" + "1: nop \n\t" + " .option pop \n\t" + " .pushsection __jump_table, \"aw\" \n\t" + " .align " RISCV_LGPTR " \n\t" + " .long 1b - ., %l[label] - . \n\t" + " " RISCV_PTR " %0 - . \n\t" + " .popsection \n\t" + : : "i"(&((char *)key)[branch]) : : label); + + return false; +label: + return true; +} + +static __always_inline bool arch_static_branch_jump(struct static_key *key, + bool branch) +{ + asm_volatile_goto( + " .option push \n\t" + " .option norelax \n\t" + " .option norvc \n\t" + "1: jal zero, %l[label] \n\t" + " .option pop \n\t" + " .pushsection __jump_table, \"aw\" \n\t" + " .align " RISCV_LGPTR " \n\t" + " .long 1b - ., %l[label] - . \n\t" + " " RISCV_PTR " %0 - . \n\t" + " .popsection \n\t" + : : "i"(&((char *)key)[branch]) : : label); + + return false; +label: + return true; +} + +#endif /* __ASSEMBLY__ */ +#endif /* __ASM_JUMP_LABEL_H */ diff --git a/arch/riscv/include/asm/mmio.h b/arch/riscv/include/asm/mmio.h index 56053c9838b2..aff6c33ab0c0 100644 --- a/arch/riscv/include/asm/mmio.h +++ b/arch/riscv/include/asm/mmio.h @@ -14,12 +14,6 @@ #include <linux/types.h> #include <asm/mmiowb.h> -#ifndef CONFIG_MMU -#define pgprot_noncached(x) (x) -#define pgprot_writecombine(x) (x) -#define pgprot_device(x) (x) -#endif /* CONFIG_MMU */ - /* Generic IO read/write. These perform native-endian accesses. */ #define __raw_writeb __raw_writeb static inline void __raw_writeb(u8 val, volatile void __iomem *addr) diff --git a/arch/riscv/include/asm/pgalloc.h b/arch/riscv/include/asm/pgalloc.h index 3f601ee8233f..23b1544e0ca5 100644 --- a/arch/riscv/include/asm/pgalloc.h +++ b/arch/riscv/include/asm/pgalloc.h @@ -11,7 +11,7 @@ #include <asm/tlb.h> #ifdef CONFIG_MMU -#include <asm-generic/pgalloc.h> /* for pte_{alloc,free}_one */ +#include <asm-generic/pgalloc.h> static inline void pmd_populate_kernel(struct mm_struct *mm, pmd_t *pmd, pte_t *pte) @@ -55,24 +55,8 @@ static inline pgd_t *pgd_alloc(struct mm_struct *mm) return pgd; } -static inline void pgd_free(struct mm_struct *mm, pgd_t *pgd) -{ - free_page((unsigned long)pgd); -} - #ifndef __PAGETABLE_PMD_FOLDED -static inline pmd_t *pmd_alloc_one(struct mm_struct *mm, unsigned long addr) -{ - return (pmd_t *)__get_free_page( - GFP_KERNEL | __GFP_RETRY_MAYFAIL | __GFP_ZERO); -} - -static inline void pmd_free(struct mm_struct *mm, pmd_t *pmd) -{ - free_page((unsigned long)pmd); -} - #define __pmd_free_tlb(tlb, pmd, addr) pmd_free((tlb)->mm, pmd) #endif /* __PAGETABLE_PMD_FOLDED */ diff --git a/arch/riscv/include/asm/smp.h b/arch/riscv/include/asm/smp.h index 40bb1c15a731..6dfd2a1446d5 100644 --- a/arch/riscv/include/asm/smp.h +++ b/arch/riscv/include/asm/smp.h @@ -40,6 +40,9 @@ void arch_send_call_function_single_ipi(int cpu); int riscv_hartid_to_cpuid(int hartid); void riscv_cpuid_to_hartid_mask(const struct cpumask *in, struct cpumask *out); +/* Secondary hart entry */ +asmlinkage void smp_callin(void); + /* * Obtains the hart ID of the currently executing task. This relies on * THREAD_INFO_IN_TASK, but we define that unconditionally. diff --git a/arch/riscv/include/asm/stackprotector.h b/arch/riscv/include/asm/stackprotector.h new file mode 100644 index 000000000000..d95f7b2a7f37 --- /dev/null +++ b/arch/riscv/include/asm/stackprotector.h @@ -0,0 +1,33 @@ +/* SPDX-License-Identifier: GPL-2.0 */ + +#ifndef _ASM_RISCV_STACKPROTECTOR_H +#define _ASM_RISCV_STACKPROTECTOR_H + +#include <linux/random.h> +#include <linux/version.h> +#include <asm/timex.h> + +extern unsigned long __stack_chk_guard; + +/* + * Initialize the stackprotector canary value. + * + * NOTE: this must only be called from functions that never return, + * and it must always be inlined. + */ +static __always_inline void boot_init_stack_canary(void) +{ + unsigned long canary; + unsigned long tsc; + + /* Try to get a semi random initial value. */ + get_random_bytes(&canary, sizeof(canary)); + tsc = get_cycles(); + canary += tsc + (tsc << BITS_PER_LONG/2); + canary ^= LINUX_VERSION_CODE; + canary &= CANARY_MASK; + + current->stack_canary = canary; + __stack_chk_guard = current->stack_canary; +} +#endif /* _ASM_RISCV_STACKPROTECTOR_H */ diff --git a/arch/riscv/include/asm/uaccess.h b/arch/riscv/include/asm/uaccess.h index 8ce9d607b53d..f56c66b3f5fe 100644 --- a/arch/riscv/include/asm/uaccess.h +++ b/arch/riscv/include/asm/uaccess.h @@ -8,6 +8,8 @@ #ifndef _ASM_RISCV_UACCESS_H #define _ASM_RISCV_UACCESS_H +#include <asm/pgtable.h> /* for TASK_SIZE */ + /* * User space memory access functions */ @@ -62,11 +64,9 @@ static inline void set_fs(mm_segment_t fs) current_thread_info()->addr_limit = fs; } -#define segment_eq(a, b) ((a).seg == (b).seg) - +#define uaccess_kernel() (get_fs().seg == KERNEL_DS.seg) #define user_addr_max() (get_fs().seg) - /** * access_ok: - Checks if a user space pointer is valid * @addr: User space pointer to start of block to check diff --git a/arch/riscv/include/uapi/asm/hwcap.h b/arch/riscv/include/uapi/asm/hwcap.h index dee98ee28318..46dc3f5ee99f 100644 --- a/arch/riscv/include/uapi/asm/hwcap.h +++ b/arch/riscv/include/uapi/asm/hwcap.h @@ -11,7 +11,7 @@ /* * Linux saves the floating-point registers according to the ISA Linux is * executing on, as opposed to the ISA the user program is compiled for. This - * is necessary for a handful of esoteric use cases: for example, userpsace + * is necessary for a handful of esoteric use cases: for example, userspace * threading libraries must be able to examine the actual machine state in * order to fully reconstruct the state of a thread. */ diff --git a/arch/riscv/include/uapi/asm/unistd.h b/arch/riscv/include/uapi/asm/unistd.h index 13ce76cc5aff..4b989ae15d59 100644 --- a/arch/riscv/include/uapi/asm/unistd.h +++ b/arch/riscv/include/uapi/asm/unistd.h @@ -12,7 +12,7 @@ * GNU General Public License for more details. * * You should have received a copy of the GNU General Public License - * along with this program. If not, see <http://www.gnu.org/licenses/>. + * along with this program. If not, see <https://www.gnu.org/licenses/>. */ #ifdef __LP64__ diff --git a/arch/riscv/kernel/Makefile b/arch/riscv/kernel/Makefile index b355cf485671..a5287ab9f7f2 100644 --- a/arch/riscv/kernel/Makefile +++ b/arch/riscv/kernel/Makefile @@ -53,4 +53,6 @@ endif obj-$(CONFIG_HOTPLUG_CPU) += cpu-hotplug.o obj-$(CONFIG_KGDB) += kgdb.o +obj-$(CONFIG_JUMP_LABEL) += jump_label.o + clean: diff --git a/arch/riscv/kernel/asm-offsets.c b/arch/riscv/kernel/asm-offsets.c index 07cb9c10de4e..db203442c08f 100644 --- a/arch/riscv/kernel/asm-offsets.c +++ b/arch/riscv/kernel/asm-offsets.c @@ -27,9 +27,6 @@ void asm_offsets(void) OFFSET(TASK_THREAD_S9, task_struct, thread.s[9]); OFFSET(TASK_THREAD_S10, task_struct, thread.s[10]); OFFSET(TASK_THREAD_S11, task_struct, thread.s[11]); - OFFSET(TASK_THREAD_SP, task_struct, thread.sp); - OFFSET(TASK_STACK, task_struct, stack); - OFFSET(TASK_TI, task_struct, thread_info); OFFSET(TASK_TI_FLAGS, task_struct, thread_info.flags); OFFSET(TASK_TI_PREEMPT_COUNT, task_struct, thread_info.preempt_count); OFFSET(TASK_TI_KERNEL_SP, task_struct, thread_info.kernel_sp); diff --git a/arch/riscv/kernel/entry.S b/arch/riscv/kernel/entry.S index cae7e6d4c7ef..524d918f3601 100644 --- a/arch/riscv/kernel/entry.S +++ b/arch/riscv/kernel/entry.S @@ -97,19 +97,36 @@ _save_context: la gp, __global_pointer$ .option pop - la ra, ret_from_exception +#ifdef CONFIG_TRACE_IRQFLAGS + call trace_hardirqs_off +#endif + +#ifdef CONFIG_CONTEXT_TRACKING + /* If previous state is in user mode, call context_tracking_user_exit. */ + li a0, SR_PP + and a0, s1, a0 + bnez a0, skip_context_tracking + call context_tracking_user_exit +skip_context_tracking: +#endif + /* * MSB of cause differentiates between * interrupts and exceptions */ bge s4, zero, 1f + la ra, ret_from_exception + /* Handle interrupts */ move a0, sp /* pt_regs */ la a1, handle_arch_irq REG_L a1, (a1) jr a1 1: +#ifdef CONFIG_TRACE_IRQFLAGS + call trace_hardirqs_on +#endif /* * Exceptions run with interrupts enabled or disabled depending on the * state of SR_PIE in m/sstatus. @@ -119,6 +136,7 @@ _save_context: csrs CSR_STATUS, SR_IE 1: + la ra, ret_from_exception /* Handle syscalls */ li t0, EXC_SYSCALL beq s4, t0, handle_syscall @@ -137,6 +155,17 @@ _save_context: tail do_trap_unknown handle_syscall: +#if defined(CONFIG_TRACE_IRQFLAGS) || defined(CONFIG_CONTEXT_TRACKING) + /* Recover a0 - a7 for system calls */ + REG_L a0, PT_A0(sp) + REG_L a1, PT_A1(sp) + REG_L a2, PT_A2(sp) + REG_L a3, PT_A3(sp) + REG_L a4, PT_A4(sp) + REG_L a5, PT_A5(sp) + REG_L a6, PT_A6(sp) + REG_L a7, PT_A7(sp) +#endif /* save the initial A0 value (needed in signal handlers) */ REG_S a0, PT_ORIG_A0(sp) /* @@ -190,6 +219,9 @@ ret_from_syscall_rejected: ret_from_exception: REG_L s0, PT_STATUS(sp) csrc CSR_STATUS, SR_IE +#ifdef CONFIG_TRACE_IRQFLAGS + call trace_hardirqs_off +#endif #ifdef CONFIG_RISCV_M_MODE /* the MPP value is too large to be used as an immediate arg for addi */ li t0, SR_MPP @@ -205,6 +237,10 @@ resume_userspace: andi s1, s0, _TIF_WORK_MASK bnez s1, work_pending +#ifdef CONFIG_CONTEXT_TRACKING + call context_tracking_user_enter +#endif + /* Save unwound kernel stack pointer in thread_info */ addi s0, sp, PT_SIZE_ON_STACK REG_S s0, TASK_TI_KERNEL_SP(tp) @@ -216,6 +252,16 @@ resume_userspace: csrw CSR_SCRATCH, tp restore_all: +#ifdef CONFIG_TRACE_IRQFLAGS + REG_L s1, PT_STATUS(sp) + andi t0, s1, SR_PIE + beqz t0, 1f + call trace_hardirqs_on + j 2f +1: + call trace_hardirqs_off +2: +#endif REG_L a0, PT_STATUS(sp) /* * The current load reservation is effectively part of the processor's @@ -389,12 +435,8 @@ ENTRY(__switch_to) lw a4, TASK_TI_CPU(a1) sw a3, TASK_TI_CPU(a1) sw a4, TASK_TI_CPU(a0) -#if TASK_TI != 0 -#error "TASK_TI != 0: tp will contain a 'struct thread_info', not a 'struct task_struct' so get_current() won't work." - addi tp, a1, TASK_TI -#else + /* The offset of thread_info in task_struct is zero. */ move tp, a1 -#endif ret ENDPROC(__switch_to) diff --git a/arch/riscv/kernel/head.S b/arch/riscv/kernel/head.S index 7ed1b22950fd..d0c5c316e9bb 100644 --- a/arch/riscv/kernel/head.S +++ b/arch/riscv/kernel/head.S @@ -77,10 +77,16 @@ relocate: csrw CSR_SATP, a0 .align 2 1: - /* Set trap vector to spin forever to help debug */ - la a0, .Lsecondary_park + /* Set trap vector to exception handler */ + la a0, handle_exception csrw CSR_TVEC, a0 + /* + * Set sup0 scratch register to 0, indicating to exception vector that + * we are presently executing in kernel. + */ + csrw CSR_SCRATCH, zero + /* Reload the global pointer */ .option push .option norelax diff --git a/arch/riscv/kernel/jump_label.c b/arch/riscv/kernel/jump_label.c new file mode 100644 index 000000000000..20e09056d141 --- /dev/null +++ b/arch/riscv/kernel/jump_label.c @@ -0,0 +1,53 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Copyright (C) 2020 Emil Renner Berthing + * + * Based on arch/arm64/kernel/jump_label.c + */ +#include <linux/jump_label.h> +#include <linux/kernel.h> +#include <linux/memory.h> +#include <linux/mutex.h> +#include <asm/bug.h> +#include <asm/patch.h> + +#define RISCV_INSN_NOP 0x00000013U +#define RISCV_INSN_JAL 0x0000006fU + +void arch_jump_label_transform(struct jump_entry *entry, + enum jump_label_type type) +{ + void *addr = (void *)jump_entry_code(entry); + u32 insn; + + if (type == JUMP_LABEL_JMP) { + long offset = jump_entry_target(entry) - jump_entry_code(entry); + + if (WARN_ON(offset & 1 || offset < -524288 || offset >= 524288)) + return; + + insn = RISCV_INSN_JAL | + (((u32)offset & GENMASK(19, 12)) << (12 - 12)) | + (((u32)offset & GENMASK(11, 11)) << (20 - 11)) | + (((u32)offset & GENMASK(10, 1)) << (21 - 1)) | + (((u32)offset & GENMASK(20, 20)) << (31 - 20)); + } else { + insn = RISCV_INSN_NOP; + } + + mutex_lock(&text_mutex); + patch_text_nosync(addr, &insn, sizeof(insn)); + mutex_unlock(&text_mutex); +} + +void arch_jump_label_transform_static(struct jump_entry *entry, + enum jump_label_type type) +{ + /* + * We use the same instructions in the arch_static_branch and + * arch_static_branch_jump inline functions, so there's no + * need to patch them up here. + * The core will call arch_jump_label_transform when those + * instructions need to be replaced. + */ +} diff --git a/arch/riscv/kernel/module.c b/arch/riscv/kernel/module.c index 7191342c54da..104fba889cf7 100644 --- a/arch/riscv/kernel/module.c +++ b/arch/riscv/kernel/module.c @@ -263,6 +263,13 @@ static int apply_r_riscv_add32_rela(struct module *me, u32 *location, return 0; } +static int apply_r_riscv_add64_rela(struct module *me, u32 *location, + Elf_Addr v) +{ + *(u64 *)location += (u64)v; + return 0; +} + static int apply_r_riscv_sub32_rela(struct module *me, u32 *location, Elf_Addr v) { @@ -270,6 +277,13 @@ static int apply_r_riscv_sub32_rela(struct module *me, u32 *location, return 0; } +static int apply_r_riscv_sub64_rela(struct module *me, u32 *location, + Elf_Addr v) +{ + *(u64 *)location -= (u64)v; + return 0; +} + static int (*reloc_handlers_rela[]) (struct module *me, u32 *location, Elf_Addr v) = { [R_RISCV_32] = apply_r_riscv_32_rela, @@ -290,7 +304,9 @@ static int (*reloc_handlers_rela[]) (struct module *me, u32 *location, [R_RISCV_RELAX] = apply_r_riscv_relax_rela, [R_RISCV_ALIGN] = apply_r_riscv_align_rela, [R_RISCV_ADD32] = apply_r_riscv_add32_rela, + [R_RISCV_ADD64] = apply_r_riscv_add64_rela, [R_RISCV_SUB32] = apply_r_riscv_sub32_rela, + [R_RISCV_SUB64] = apply_r_riscv_sub64_rela, }; int apply_relocate_add(Elf_Shdr *sechdrs, const char *strtab, diff --git a/arch/riscv/kernel/process.c b/arch/riscv/kernel/process.c index 31f39442df72..2b97c493427c 100644 --- a/arch/riscv/kernel/process.c +++ b/arch/riscv/kernel/process.c @@ -24,6 +24,12 @@ register unsigned long gp_in_global __asm__("gp"); +#ifdef CONFIG_STACKPROTECTOR +#include <linux/stackprotector.h> +unsigned long __stack_chk_guard __read_mostly; +EXPORT_SYMBOL(__stack_chk_guard); +#endif + extern asmlinkage void ret_from_fork(void); extern asmlinkage void ret_from_kernel_thread(void); diff --git a/arch/riscv/kernel/ptrace.c b/arch/riscv/kernel/ptrace.c index 444dc7b0fd78..2d6395f5ad54 100644 --- a/arch/riscv/kernel/ptrace.c +++ b/arch/riscv/kernel/ptrace.c @@ -30,13 +30,10 @@ enum riscv_regset { static int riscv_gpr_get(struct task_struct *target, const struct user_regset *regset, - unsigned int pos, unsigned int count, - void *kbuf, void __user *ubuf) + struct membuf to) { - struct pt_regs *regs; - - regs = task_pt_regs(target); - return user_regset_copyout(&pos, &count, &kbuf, &ubuf, regs, 0, -1); + return membuf_write(&to, task_pt_regs(target), + sizeof(struct user_regs_struct)); } static int riscv_gpr_set(struct task_struct *target, @@ -55,21 +52,13 @@ static int riscv_gpr_set(struct task_struct *target, #ifdef CONFIG_FPU static int riscv_fpr_get(struct task_struct *target, const struct user_regset *regset, - unsigned int pos, unsigned int count, - void *kbuf, void __user *ubuf) + struct membuf to) { - int ret; struct __riscv_d_ext_state *fstate = &target->thread.fstate; - ret = user_regset_copyout(&pos, &count, &kbuf, &ubuf, fstate, 0, - offsetof(struct __riscv_d_ext_state, fcsr)); - if (!ret) { - ret = user_regset_copyout(&pos, &count, &kbuf, &ubuf, fstate, 0, - offsetof(struct __riscv_d_ext_state, fcsr) + - sizeof(fstate->fcsr)); - } - - return ret; + membuf_write(&to, fstate, offsetof(struct __riscv_d_ext_state, fcsr)); + membuf_store(&to, fstate->fcsr); + return membuf_zero(&to, 4); // explicitly pad } static int riscv_fpr_set(struct task_struct *target, @@ -98,8 +87,8 @@ static const struct user_regset riscv_user_regset[] = { .n = ELF_NGREG, .size = sizeof(elf_greg_t), .align = sizeof(elf_greg_t), - .get = &riscv_gpr_get, - .set = &riscv_gpr_set, + .regset_get = riscv_gpr_get, + .set = riscv_gpr_set, }, #ifdef CONFIG_FPU [REGSET_F] = { @@ -107,8 +96,8 @@ static const struct user_regset riscv_user_regset[] = { .n = ELF_NFPREG, .size = sizeof(elf_fpreg_t), .align = sizeof(elf_fpreg_t), - .get = &riscv_fpr_get, - .set = &riscv_fpr_set, + .regset_get = riscv_fpr_get, + .set = riscv_fpr_set, }, #endif }; diff --git a/arch/riscv/kernel/smp.c b/arch/riscv/kernel/smp.c index b1d4f452f843..554b0fb47060 100644 --- a/arch/riscv/kernel/smp.c +++ b/arch/riscv/kernel/smp.c @@ -16,6 +16,7 @@ #include <linux/sched.h> #include <linux/seq_file.h> #include <linux/delay.h> +#include <linux/irq_work.h> #include <asm/clint.h> #include <asm/sbi.h> @@ -26,6 +27,7 @@ enum ipi_message_type { IPI_RESCHEDULE, IPI_CALL_FUNC, IPI_CPU_STOP, + IPI_IRQ_WORK, IPI_MAX }; @@ -123,6 +125,13 @@ static inline void clear_ipi(void) clint_clear_ipi(cpuid_to_hartid_map(smp_processor_id())); } +#ifdef CONFIG_IRQ_WORK +void arch_irq_work_raise(void) +{ + send_ipi_single(smp_processor_id(), IPI_IRQ_WORK); +} +#endif + void handle_IPI(struct pt_regs *regs) { struct pt_regs *old_regs = set_irq_regs(regs); @@ -158,6 +167,11 @@ void handle_IPI(struct pt_regs *regs) ipi_stop(); } + if (ops & (1 << IPI_IRQ_WORK)) { + stats[IPI_IRQ_WORK]++; + irq_work_run(); + } + BUG_ON((ops >> IPI_MAX) != 0); /* Order data access and bit testing. */ @@ -173,6 +187,7 @@ static const char * const ipi_names[] = { [IPI_RESCHEDULE] = "Rescheduling interrupts", [IPI_CALL_FUNC] = "Function call interrupts", [IPI_CPU_STOP] = "CPU stop interrupts", + [IPI_IRQ_WORK] = "IRQ work interrupts", }; void show_ipi_stats(struct seq_file *p, int prec) diff --git a/arch/riscv/kernel/smpboot.c b/arch/riscv/kernel/smpboot.c index 4e9922790f6e..356825a57551 100644 --- a/arch/riscv/kernel/smpboot.c +++ b/arch/riscv/kernel/smpboot.c @@ -106,7 +106,7 @@ void __init setup_smp(void) } } -int start_secondary_cpu(int cpu, struct task_struct *tidle) +static int start_secondary_cpu(int cpu, struct task_struct *tidle) { if (cpu_ops[cpu]->cpu_start) return cpu_ops[cpu]->cpu_start(cpu, tidle); @@ -121,7 +121,6 @@ int __cpu_up(unsigned int cpu, struct task_struct *tidle) ret = start_secondary_cpu(cpu, tidle); if (!ret) { - lockdep_assert_held(&cpu_running); wait_for_completion_timeout(&cpu_running, msecs_to_jiffies(1000)); @@ -146,6 +145,7 @@ void __init smp_cpus_done(unsigned int max_cpus) asmlinkage __visible void smp_callin(void) { struct mm_struct *mm = &init_mm; + unsigned int curr_cpuid = smp_processor_id(); if (!IS_ENABLED(CONFIG_RISCV_SBI)) clint_clear_ipi(cpuid_to_hartid_map(smp_processor_id())); @@ -154,10 +154,10 @@ asmlinkage __visible void smp_callin(void) mmgrab(mm); current->active_mm = mm; - trap_init(); - notify_cpu_starting(smp_processor_id()); - update_siblings_masks(smp_processor_id()); - set_cpu_online(smp_processor_id(), 1); + notify_cpu_starting(curr_cpuid); + update_siblings_masks(curr_cpuid); + set_cpu_online(curr_cpuid, 1); + /* * Remote TLB flushes are ignored while the CPU is offline, so emit * a local TLB flush right now just in case. diff --git a/arch/riscv/kernel/traps.c b/arch/riscv/kernel/traps.c index 7d95cce5e47c..ad14f4466d92 100644 --- a/arch/riscv/kernel/traps.c +++ b/arch/riscv/kernel/traps.c @@ -174,13 +174,7 @@ int is_valid_bugaddr(unsigned long pc) } #endif /* CONFIG_GENERIC_BUG */ +/* stvec & scratch is already set from head.S */ void trap_init(void) { - /* - * Set sup0 scratch register to 0, indicating to exception vector - * that we are presently executing in the kernel - */ - csr_write(CSR_SCRATCH, 0); - /* Set the exception vector address */ - csr_write(CSR_TVEC, &handle_exception); } diff --git a/arch/riscv/kernel/vdso/Makefile b/arch/riscv/kernel/vdso/Makefile index e4c7c2c8a02f..478e7338ddc1 100644 --- a/arch/riscv/kernel/vdso/Makefile +++ b/arch/riscv/kernel/vdso/Makefile @@ -16,6 +16,8 @@ vdso-syms += flush_icache # Files to link into the vdso obj-vdso = $(patsubst %, %.o, $(vdso-syms)) note.o +ccflags-y := -fno-stack-protector + ifneq ($(c-gettimeofday-y),) CFLAGS_vgettimeofday.o += -fPIC -include $(c-gettimeofday-y) endif @@ -32,6 +34,7 @@ CFLAGS_REMOVE_vgettimeofday.o = $(CC_FLAGS_FTRACE) -Os # Disable gcov profiling for VDSO code GCOV_PROFILE := n +KCOV_INSTRUMENT := n # Force dependency $(obj)/vdso.o: $(obj)/vdso.so diff --git a/arch/riscv/kernel/vmlinux.lds.S b/arch/riscv/kernel/vmlinux.lds.S index e6f8016b366a..f3586e31ed1e 100644 --- a/arch/riscv/kernel/vmlinux.lds.S +++ b/arch/riscv/kernel/vmlinux.lds.S @@ -22,6 +22,7 @@ SECTIONS /* Beginning of code and text segment */ . = LOAD_OFFSET; _start = .; + _stext = .; HEAD_TEXT_SECTION . = ALIGN(PAGE_SIZE); @@ -54,7 +55,6 @@ SECTIONS . = ALIGN(SECTION_ALIGN); .text : { _text = .; - _stext = .; TEXT_TEXT SCHED_TEXT CPUIDLE_TEXT diff --git a/arch/riscv/mm/Makefile b/arch/riscv/mm/Makefile index 363ef01c30b1..c0185e556ca5 100644 --- a/arch/riscv/mm/Makefile +++ b/arch/riscv/mm/Makefile @@ -5,6 +5,8 @@ ifdef CONFIG_FTRACE CFLAGS_REMOVE_init.o = -pg endif +KCOV_INSTRUMENT_init.o := n + obj-y += init.o obj-y += extable.o obj-$(CONFIG_MMU) += fault.o pageattr.o diff --git a/arch/riscv/mm/fault.c b/arch/riscv/mm/fault.c index ae7b7fe24658..716d64e36f83 100644 --- a/arch/riscv/mm/fault.c +++ b/arch/riscv/mm/fault.c @@ -14,7 +14,6 @@ #include <linux/signal.h> #include <linux/uaccess.h> -#include <asm/pgalloc.h> #include <asm/ptrace.h> #include <asm/tlbflush.h> @@ -110,7 +109,7 @@ good_area: * make sure we exit gracefully rather than endlessly redo * the fault. */ - fault = handle_mm_fault(vma, addr, flags); + fault = handle_mm_fault(vma, addr, flags, regs); /* * If we need to retry but a fatal signal is pending, handle the @@ -128,21 +127,7 @@ good_area: BUG(); } - /* - * Major/minor page fault accounting is only done on the - * initial attempt. If we go through a retry, it is extremely - * likely that the page will be found in page cache at that point. - */ if (flags & FAULT_FLAG_ALLOW_RETRY) { - if (fault & VM_FAULT_MAJOR) { - tsk->maj_flt++; - perf_sw_event(PERF_COUNT_SW_PAGE_FAULTS_MAJ, - 1, regs, addr); - } else { - tsk->min_flt++; - perf_sw_event(PERF_COUNT_SW_PAGE_FAULTS_MIN, - 1, regs, addr); - } if (fault & VM_FAULT_RETRY) { flags |= FAULT_FLAG_TRIED; diff --git a/arch/riscv/mm/init.c b/arch/riscv/mm/init.c index 79e9d55bdf1a..787c75f751a5 100644 --- a/arch/riscv/mm/init.c +++ b/arch/riscv/mm/init.c @@ -541,19 +541,45 @@ void mark_rodata_ro(void) } #endif +static void __init resource_init(void) +{ + struct memblock_region *region; + + for_each_memblock(memory, region) { + struct resource *res; + + res = memblock_alloc(sizeof(struct resource), SMP_CACHE_BYTES); + if (!res) + panic("%s: Failed to allocate %zu bytes\n", __func__, + sizeof(struct resource)); + + if (memblock_is_nomap(region)) { + res->name = "reserved"; + res->flags = IORESOURCE_MEM; + } else { + res->name = "System RAM"; + res->flags = IORESOURCE_SYSTEM_RAM | IORESOURCE_BUSY; + } + res->start = __pfn_to_phys(memblock_region_memory_base_pfn(region)); + res->end = __pfn_to_phys(memblock_region_memory_end_pfn(region)) - 1; + + request_resource(&iomem_resource, res); + } +} + void __init paging_init(void) { setup_vm_final(); - memblocks_present(); sparse_init(); setup_zero_page(); zone_sizes_init(); + resource_init(); } #ifdef CONFIG_SPARSEMEM_VMEMMAP int __meminit vmemmap_populate(unsigned long start, unsigned long end, int node, struct vmem_altmap *altmap) { - return vmemmap_populate_basepages(start, end, node); + return vmemmap_populate_basepages(start, end, node, NULL); } #endif diff --git a/arch/riscv/mm/pageattr.c b/arch/riscv/mm/pageattr.c index 289a9a5ea5b5..19fecb362d81 100644 --- a/arch/riscv/mm/pageattr.c +++ b/arch/riscv/mm/pageattr.c @@ -7,6 +7,7 @@ #include <linux/pgtable.h> #include <asm/tlbflush.h> #include <asm/bitops.h> +#include <asm/set_memory.h> struct pageattr_masks { pgprot_t set_mask; @@ -94,7 +95,7 @@ static int pageattr_pte_hole(unsigned long addr, unsigned long next, return 0; } -const static struct mm_walk_ops pageattr_ops = { +static const struct mm_walk_ops pageattr_ops = { .pgd_entry = pageattr_pgd_entry, .p4d_entry = pageattr_p4d_entry, .pud_entry = pageattr_pud_entry, diff --git a/arch/riscv/net/bpf_jit.h b/arch/riscv/net/bpf_jit.h index 20e235d06f66..75c1e9996867 100644 --- a/arch/riscv/net/bpf_jit.h +++ b/arch/riscv/net/bpf_jit.h @@ -13,6 +13,11 @@ #include <linux/filter.h> #include <asm/cacheflush.h> +static inline bool rvc_enabled(void) +{ + return IS_ENABLED(CONFIG_RISCV_ISA_C); +} + enum { RV_REG_ZERO = 0, /* The constant value 0 */ RV_REG_RA = 1, /* Return address */ @@ -48,9 +53,21 @@ enum { RV_REG_T6 = 31, }; +static inline bool is_creg(u8 reg) +{ + return (1 << reg) & (BIT(RV_REG_FP) | + BIT(RV_REG_S1) | + BIT(RV_REG_A0) | + BIT(RV_REG_A1) | + BIT(RV_REG_A2) | + BIT(RV_REG_A3) | + BIT(RV_REG_A4) | + BIT(RV_REG_A5)); +} + struct rv_jit_context { struct bpf_prog *prog; - u32 *insns; /* RV insns */ + u16 *insns; /* RV insns */ int ninsns; int epilogue_offset; int *offset; /* BPF to RV */ @@ -58,6 +75,12 @@ struct rv_jit_context { int stack_size; }; +/* Convert from ninsns to bytes. */ +static inline int ninsns_rvoff(int ninsns) +{ + return ninsns << 1; +} + struct rv_jit_data { struct bpf_binary_header *header; u8 *image; @@ -74,8 +97,22 @@ static inline void bpf_flush_icache(void *start, void *end) flush_icache_range((unsigned long)start, (unsigned long)end); } +/* Emit a 4-byte riscv instruction. */ static inline void emit(const u32 insn, struct rv_jit_context *ctx) { + if (ctx->insns) { + ctx->insns[ctx->ninsns] = insn; + ctx->insns[ctx->ninsns + 1] = (insn >> 16); + } + + ctx->ninsns += 2; +} + +/* Emit a 2-byte riscv compressed instruction. */ +static inline void emitc(const u16 insn, struct rv_jit_context *ctx) +{ + BUILD_BUG_ON(!rvc_enabled()); + if (ctx->insns) ctx->insns[ctx->ninsns] = insn; @@ -86,7 +123,7 @@ static inline int epilogue_offset(struct rv_jit_context *ctx) { int to = ctx->epilogue_offset, from = ctx->ninsns; - return (to - from) << 2; + return ninsns_rvoff(to - from); } /* Return -1 or inverted cond. */ @@ -117,6 +154,36 @@ static inline int invert_bpf_cond(u8 cond) return -1; } +static inline bool is_6b_int(long val) +{ + return -(1L << 5) <= val && val < (1L << 5); +} + +static inline bool is_7b_uint(unsigned long val) +{ + return val < (1UL << 7); +} + +static inline bool is_8b_uint(unsigned long val) +{ + return val < (1UL << 8); +} + +static inline bool is_9b_uint(unsigned long val) +{ + return val < (1UL << 9); +} + +static inline bool is_10b_int(long val) +{ + return -(1L << 9) <= val && val < (1L << 9); +} + +static inline bool is_10b_uint(unsigned long val) +{ + return val < (1UL << 10); +} + static inline bool is_12b_int(long val) { return -(1L << 11) <= val && val < (1L << 11); @@ -149,7 +216,7 @@ static inline int rv_offset(int insn, int off, struct rv_jit_context *ctx) off++; /* BPF branch is from PC+1, RV is from PC */ from = (insn > 0) ? ctx->offset[insn - 1] : 0; to = (insn + off > 0) ? ctx->offset[insn + off - 1] : 0; - return (to - from) << 2; + return ninsns_rvoff(to - from); } /* Instruction formats. */ @@ -207,6 +274,59 @@ static inline u32 rv_amo_insn(u8 funct5, u8 aq, u8 rl, u8 rs2, u8 rs1, return rv_r_insn(funct7, rs2, rs1, funct3, rd, opcode); } +/* RISC-V compressed instruction formats. */ + +static inline u16 rv_cr_insn(u8 funct4, u8 rd, u8 rs2, u8 op) +{ + return (funct4 << 12) | (rd << 7) | (rs2 << 2) | op; +} + +static inline u16 rv_ci_insn(u8 funct3, u32 imm6, u8 rd, u8 op) +{ + u32 imm; + + imm = ((imm6 & 0x20) << 7) | ((imm6 & 0x1f) << 2); + return (funct3 << 13) | (rd << 7) | op | imm; +} + +static inline u16 rv_css_insn(u8 funct3, u32 uimm, u8 rs2, u8 op) +{ + return (funct3 << 13) | (uimm << 7) | (rs2 << 2) | op; +} + +static inline u16 rv_ciw_insn(u8 funct3, u32 uimm, u8 rd, u8 op) +{ + return (funct3 << 13) | (uimm << 5) | ((rd & 0x7) << 2) | op; +} + +static inline u16 rv_cl_insn(u8 funct3, u32 imm_hi, u8 rs1, u32 imm_lo, u8 rd, + u8 op) +{ + return (funct3 << 13) | (imm_hi << 10) | ((rs1 & 0x7) << 7) | + (imm_lo << 5) | ((rd & 0x7) << 2) | op; +} + +static inline u16 rv_cs_insn(u8 funct3, u32 imm_hi, u8 rs1, u32 imm_lo, u8 rs2, + u8 op) +{ + return (funct3 << 13) | (imm_hi << 10) | ((rs1 & 0x7) << 7) | + (imm_lo << 5) | ((rs2 & 0x7) << 2) | op; +} + +static inline u16 rv_ca_insn(u8 funct6, u8 rd, u8 funct2, u8 rs2, u8 op) +{ + return (funct6 << 10) | ((rd & 0x7) << 7) | (funct2 << 5) | + ((rs2 & 0x7) << 2) | op; +} + +static inline u16 rv_cb_insn(u8 funct3, u32 imm6, u8 funct2, u8 rd, u8 op) +{ + u32 imm; + + imm = ((imm6 & 0x20) << 7) | ((imm6 & 0x1f) << 2); + return (funct3 << 13) | (funct2 << 10) | ((rd & 0x7) << 7) | op | imm; +} + /* Instructions shared by both RV32 and RV64. */ static inline u32 rv_addi(u8 rd, u8 rs1, u16 imm11_0) @@ -414,6 +534,135 @@ static inline u32 rv_amoadd_w(u8 rd, u8 rs2, u8 rs1, u8 aq, u8 rl) return rv_amo_insn(0, aq, rl, rs2, rs1, 2, rd, 0x2f); } +/* RVC instrutions. */ + +static inline u16 rvc_addi4spn(u8 rd, u32 imm10) +{ + u32 imm; + + imm = ((imm10 & 0x30) << 2) | ((imm10 & 0x3c0) >> 4) | + ((imm10 & 0x4) >> 1) | ((imm10 & 0x8) >> 3); + return rv_ciw_insn(0x0, imm, rd, 0x0); +} + +static inline u16 rvc_lw(u8 rd, u32 imm7, u8 rs1) +{ + u32 imm_hi, imm_lo; + + imm_hi = (imm7 & 0x38) >> 3; + imm_lo = ((imm7 & 0x4) >> 1) | ((imm7 & 0x40) >> 6); + return rv_cl_insn(0x2, imm_hi, rs1, imm_lo, rd, 0x0); +} + +static inline u16 rvc_sw(u8 rs1, u32 imm7, u8 rs2) +{ + u32 imm_hi, imm_lo; + + imm_hi = (imm7 & 0x38) >> 3; + imm_lo = ((imm7 & 0x4) >> 1) | ((imm7 & 0x40) >> 6); + return rv_cs_insn(0x6, imm_hi, rs1, imm_lo, rs2, 0x0); +} + +static inline u16 rvc_addi(u8 rd, u32 imm6) +{ + return rv_ci_insn(0, imm6, rd, 0x1); +} + +static inline u16 rvc_li(u8 rd, u32 imm6) +{ + return rv_ci_insn(0x2, imm6, rd, 0x1); +} + +static inline u16 rvc_addi16sp(u32 imm10) +{ + u32 imm; + + imm = ((imm10 & 0x200) >> 4) | (imm10 & 0x10) | ((imm10 & 0x40) >> 3) | + ((imm10 & 0x180) >> 6) | ((imm10 & 0x20) >> 5); + return rv_ci_insn(0x3, imm, RV_REG_SP, 0x1); +} + +static inline u16 rvc_lui(u8 rd, u32 imm6) +{ + return rv_ci_insn(0x3, imm6, rd, 0x1); +} + +static inline u16 rvc_srli(u8 rd, u32 imm6) +{ + return rv_cb_insn(0x4, imm6, 0, rd, 0x1); +} + +static inline u16 rvc_srai(u8 rd, u32 imm6) +{ + return rv_cb_insn(0x4, imm6, 0x1, rd, 0x1); +} + +static inline u16 rvc_andi(u8 rd, u32 imm6) +{ + return rv_cb_insn(0x4, imm6, 0x2, rd, 0x1); +} + +static inline u16 rvc_sub(u8 rd, u8 rs) +{ + return rv_ca_insn(0x23, rd, 0, rs, 0x1); +} + +static inline u16 rvc_xor(u8 rd, u8 rs) +{ + return rv_ca_insn(0x23, rd, 0x1, rs, 0x1); +} + +static inline u16 rvc_or(u8 rd, u8 rs) +{ + return rv_ca_insn(0x23, rd, 0x2, rs, 0x1); +} + +static inline u16 rvc_and(u8 rd, u8 rs) +{ + return rv_ca_insn(0x23, rd, 0x3, rs, 0x1); +} + +static inline u16 rvc_slli(u8 rd, u32 imm6) +{ + return rv_ci_insn(0, imm6, rd, 0x2); +} + +static inline u16 rvc_lwsp(u8 rd, u32 imm8) +{ + u32 imm; + + imm = ((imm8 & 0xc0) >> 6) | (imm8 & 0x3c); + return rv_ci_insn(0x2, imm, rd, 0x2); +} + +static inline u16 rvc_jr(u8 rs1) +{ + return rv_cr_insn(0x8, rs1, RV_REG_ZERO, 0x2); +} + +static inline u16 rvc_mv(u8 rd, u8 rs) +{ + return rv_cr_insn(0x8, rd, rs, 0x2); +} + +static inline u16 rvc_jalr(u8 rs1) +{ + return rv_cr_insn(0x9, rs1, RV_REG_ZERO, 0x2); +} + +static inline u16 rvc_add(u8 rd, u8 rs) +{ + return rv_cr_insn(0x9, rd, rs, 0x2); +} + +static inline u16 rvc_swsp(u32 imm8, u8 rs2) +{ + u32 imm; + + imm = (imm8 & 0x3c) | ((imm8 & 0xc0) >> 6); + return rv_css_insn(0x6, imm, rs2, 0x2); +} + /* * RV64-only instructions. * @@ -503,6 +752,234 @@ static inline u32 rv_amoadd_d(u8 rd, u8 rs2, u8 rs1, u8 aq, u8 rl) return rv_amo_insn(0, aq, rl, rs2, rs1, 3, rd, 0x2f); } +/* RV64-only RVC instructions. */ + +static inline u16 rvc_ld(u8 rd, u32 imm8, u8 rs1) +{ + u32 imm_hi, imm_lo; + + imm_hi = (imm8 & 0x38) >> 3; + imm_lo = (imm8 & 0xc0) >> 6; + return rv_cl_insn(0x3, imm_hi, rs1, imm_lo, rd, 0x0); +} + +static inline u16 rvc_sd(u8 rs1, u32 imm8, u8 rs2) +{ + u32 imm_hi, imm_lo; + + imm_hi = (imm8 & 0x38) >> 3; + imm_lo = (imm8 & 0xc0) >> 6; + return rv_cs_insn(0x7, imm_hi, rs1, imm_lo, rs2, 0x0); +} + +static inline u16 rvc_subw(u8 rd, u8 rs) +{ + return rv_ca_insn(0x27, rd, 0, rs, 0x1); +} + +static inline u16 rvc_addiw(u8 rd, u32 imm6) +{ + return rv_ci_insn(0x1, imm6, rd, 0x1); +} + +static inline u16 rvc_ldsp(u8 rd, u32 imm9) +{ + u32 imm; + + imm = ((imm9 & 0x1c0) >> 6) | (imm9 & 0x38); + return rv_ci_insn(0x3, imm, rd, 0x2); +} + +static inline u16 rvc_sdsp(u32 imm9, u8 rs2) +{ + u32 imm; + + imm = (imm9 & 0x38) | ((imm9 & 0x1c0) >> 6); + return rv_css_insn(0x7, imm, rs2, 0x2); +} + +#endif /* __riscv_xlen == 64 */ + +/* Helper functions that emit RVC instructions when possible. */ + +static inline void emit_jalr(u8 rd, u8 rs, s32 imm, struct rv_jit_context *ctx) +{ + if (rvc_enabled() && rd == RV_REG_RA && rs && !imm) + emitc(rvc_jalr(rs), ctx); + else if (rvc_enabled() && !rd && rs && !imm) + emitc(rvc_jr(rs), ctx); + else + emit(rv_jalr(rd, rs, imm), ctx); +} + +static inline void emit_mv(u8 rd, u8 rs, struct rv_jit_context *ctx) +{ + if (rvc_enabled() && rd && rs) + emitc(rvc_mv(rd, rs), ctx); + else + emit(rv_addi(rd, rs, 0), ctx); +} + +static inline void emit_add(u8 rd, u8 rs1, u8 rs2, struct rv_jit_context *ctx) +{ + if (rvc_enabled() && rd && rd == rs1 && rs2) + emitc(rvc_add(rd, rs2), ctx); + else + emit(rv_add(rd, rs1, rs2), ctx); +} + +static inline void emit_addi(u8 rd, u8 rs, s32 imm, struct rv_jit_context *ctx) +{ + if (rvc_enabled() && rd == RV_REG_SP && rd == rs && is_10b_int(imm) && imm && !(imm & 0xf)) + emitc(rvc_addi16sp(imm), ctx); + else if (rvc_enabled() && is_creg(rd) && rs == RV_REG_SP && is_10b_uint(imm) && + !(imm & 0x3) && imm) + emitc(rvc_addi4spn(rd, imm), ctx); + else if (rvc_enabled() && rd && rd == rs && imm && is_6b_int(imm)) + emitc(rvc_addi(rd, imm), ctx); + else + emit(rv_addi(rd, rs, imm), ctx); +} + +static inline void emit_li(u8 rd, s32 imm, struct rv_jit_context *ctx) +{ + if (rvc_enabled() && rd && is_6b_int(imm)) + emitc(rvc_li(rd, imm), ctx); + else + emit(rv_addi(rd, RV_REG_ZERO, imm), ctx); +} + +static inline void emit_lui(u8 rd, s32 imm, struct rv_jit_context *ctx) +{ + if (rvc_enabled() && rd && rd != RV_REG_SP && is_6b_int(imm) && imm) + emitc(rvc_lui(rd, imm), ctx); + else + emit(rv_lui(rd, imm), ctx); +} + +static inline void emit_slli(u8 rd, u8 rs, s32 imm, struct rv_jit_context *ctx) +{ + if (rvc_enabled() && rd && rd == rs && imm && (u32)imm < __riscv_xlen) + emitc(rvc_slli(rd, imm), ctx); + else + emit(rv_slli(rd, rs, imm), ctx); +} + +static inline void emit_andi(u8 rd, u8 rs, s32 imm, struct rv_jit_context *ctx) +{ + if (rvc_enabled() && is_creg(rd) && rd == rs && is_6b_int(imm)) + emitc(rvc_andi(rd, imm), ctx); + else + emit(rv_andi(rd, rs, imm), ctx); +} + +static inline void emit_srli(u8 rd, u8 rs, s32 imm, struct rv_jit_context *ctx) +{ + if (rvc_enabled() && is_creg(rd) && rd == rs && imm && (u32)imm < __riscv_xlen) + emitc(rvc_srli(rd, imm), ctx); + else + emit(rv_srli(rd, rs, imm), ctx); +} + +static inline void emit_srai(u8 rd, u8 rs, s32 imm, struct rv_jit_context *ctx) +{ + if (rvc_enabled() && is_creg(rd) && rd == rs && imm && (u32)imm < __riscv_xlen) + emitc(rvc_srai(rd, imm), ctx); + else + emit(rv_srai(rd, rs, imm), ctx); +} + +static inline void emit_sub(u8 rd, u8 rs1, u8 rs2, struct rv_jit_context *ctx) +{ + if (rvc_enabled() && is_creg(rd) && rd == rs1 && is_creg(rs2)) + emitc(rvc_sub(rd, rs2), ctx); + else + emit(rv_sub(rd, rs1, rs2), ctx); +} + +static inline void emit_or(u8 rd, u8 rs1, u8 rs2, struct rv_jit_context *ctx) +{ + if (rvc_enabled() && is_creg(rd) && rd == rs1 && is_creg(rs2)) + emitc(rvc_or(rd, rs2), ctx); + else + emit(rv_or(rd, rs1, rs2), ctx); +} + +static inline void emit_and(u8 rd, u8 rs1, u8 rs2, struct rv_jit_context *ctx) +{ + if (rvc_enabled() && is_creg(rd) && rd == rs1 && is_creg(rs2)) + emitc(rvc_and(rd, rs2), ctx); + else + emit(rv_and(rd, rs1, rs2), ctx); +} + +static inline void emit_xor(u8 rd, u8 rs1, u8 rs2, struct rv_jit_context *ctx) +{ + if (rvc_enabled() && is_creg(rd) && rd == rs1 && is_creg(rs2)) + emitc(rvc_xor(rd, rs2), ctx); + else + emit(rv_xor(rd, rs1, rs2), ctx); +} + +static inline void emit_lw(u8 rd, s32 off, u8 rs1, struct rv_jit_context *ctx) +{ + if (rvc_enabled() && rs1 == RV_REG_SP && rd && is_8b_uint(off) && !(off & 0x3)) + emitc(rvc_lwsp(rd, off), ctx); + else if (rvc_enabled() && is_creg(rd) && is_creg(rs1) && is_7b_uint(off) && !(off & 0x3)) + emitc(rvc_lw(rd, off, rs1), ctx); + else + emit(rv_lw(rd, off, rs1), ctx); +} + +static inline void emit_sw(u8 rs1, s32 off, u8 rs2, struct rv_jit_context *ctx) +{ + if (rvc_enabled() && rs1 == RV_REG_SP && is_8b_uint(off) && !(off & 0x3)) + emitc(rvc_swsp(off, rs2), ctx); + else if (rvc_enabled() && is_creg(rs1) && is_creg(rs2) && is_7b_uint(off) && !(off & 0x3)) + emitc(rvc_sw(rs1, off, rs2), ctx); + else + emit(rv_sw(rs1, off, rs2), ctx); +} + +/* RV64-only helper functions. */ +#if __riscv_xlen == 64 + +static inline void emit_addiw(u8 rd, u8 rs, s32 imm, struct rv_jit_context *ctx) +{ + if (rvc_enabled() && rd && rd == rs && is_6b_int(imm)) + emitc(rvc_addiw(rd, imm), ctx); + else + emit(rv_addiw(rd, rs, imm), ctx); +} + +static inline void emit_ld(u8 rd, s32 off, u8 rs1, struct rv_jit_context *ctx) +{ + if (rvc_enabled() && rs1 == RV_REG_SP && rd && is_9b_uint(off) && !(off & 0x7)) + emitc(rvc_ldsp(rd, off), ctx); + else if (rvc_enabled() && is_creg(rd) && is_creg(rs1) && is_8b_uint(off) && !(off & 0x7)) + emitc(rvc_ld(rd, off, rs1), ctx); + else + emit(rv_ld(rd, off, rs1), ctx); +} + +static inline void emit_sd(u8 rs1, s32 off, u8 rs2, struct rv_jit_context *ctx) +{ + if (rvc_enabled() && rs1 == RV_REG_SP && is_9b_uint(off) && !(off & 0x7)) + emitc(rvc_sdsp(off, rs2), ctx); + else if (rvc_enabled() && is_creg(rs1) && is_creg(rs2) && is_8b_uint(off) && !(off & 0x7)) + emitc(rvc_sd(rs1, off, rs2), ctx); + else + emit(rv_sd(rs1, off, rs2), ctx); +} + +static inline void emit_subw(u8 rd, u8 rs1, u8 rs2, struct rv_jit_context *ctx) +{ + if (rvc_enabled() && is_creg(rd) && rd == rs1 && is_creg(rs2)) + emitc(rvc_subw(rd, rs2), ctx); + else + emit(rv_subw(rd, rs1, rs2), ctx); +} + #endif /* __riscv_xlen == 64 */ void bpf_jit_build_prologue(struct rv_jit_context *ctx); diff --git a/arch/riscv/net/bpf_jit_comp32.c b/arch/riscv/net/bpf_jit_comp32.c index b198eaa74456..bc5f2204693f 100644 --- a/arch/riscv/net/bpf_jit_comp32.c +++ b/arch/riscv/net/bpf_jit_comp32.c @@ -644,7 +644,7 @@ static int emit_branch_r64(const s8 *src1, const s8 *src2, s32 rvoff, e = ctx->ninsns; /* Adjust for extra insns. */ - rvoff -= (e - s) << 2; + rvoff -= ninsns_rvoff(e - s); emit_jump_and_link(RV_REG_ZERO, rvoff, true, ctx); return 0; } @@ -713,7 +713,7 @@ static int emit_bcc(u8 op, u8 rd, u8 rs, int rvoff, struct rv_jit_context *ctx) if (far) { e = ctx->ninsns; /* Adjust for extra insns. */ - rvoff -= (e - s) << 2; + rvoff -= ninsns_rvoff(e - s); emit_jump_and_link(RV_REG_ZERO, rvoff, true, ctx); } return 0; @@ -731,7 +731,7 @@ static int emit_branch_r32(const s8 *src1, const s8 *src2, s32 rvoff, e = ctx->ninsns; /* Adjust for extra insns. */ - rvoff -= (e - s) << 2; + rvoff -= ninsns_rvoff(e - s); if (emit_bcc(op, lo(rs1), lo(rs2), rvoff, ctx)) return -1; @@ -795,7 +795,7 @@ static int emit_bpf_tail_call(int insn, struct rv_jit_context *ctx) * if (index >= max_entries) * goto out; */ - off = (tc_ninsn - (ctx->ninsns - start_insn)) << 2; + off = ninsns_rvoff(tc_ninsn - (ctx->ninsns - start_insn)); emit_bcc(BPF_JGE, lo(idx_reg), RV_REG_T1, off, ctx); /* @@ -804,7 +804,7 @@ static int emit_bpf_tail_call(int insn, struct rv_jit_context *ctx) * goto out; */ emit(rv_addi(RV_REG_T1, RV_REG_TCC, -1), ctx); - off = (tc_ninsn - (ctx->ninsns - start_insn)) << 2; + off = ninsns_rvoff(tc_ninsn - (ctx->ninsns - start_insn)); emit_bcc(BPF_JSLT, RV_REG_TCC, RV_REG_ZERO, off, ctx); /* @@ -818,7 +818,7 @@ static int emit_bpf_tail_call(int insn, struct rv_jit_context *ctx) if (is_12b_check(off, insn)) return -1; emit(rv_lw(RV_REG_T0, off, RV_REG_T0), ctx); - off = (tc_ninsn - (ctx->ninsns - start_insn)) << 2; + off = ninsns_rvoff(tc_ninsn - (ctx->ninsns - start_insn)); emit_bcc(BPF_JEQ, RV_REG_T0, RV_REG_ZERO, off, ctx); /* @@ -1214,7 +1214,7 @@ int bpf_jit_emit_insn(const struct bpf_insn *insn, struct rv_jit_context *ctx, emit_imm32(tmp2, imm, ctx); src = tmp2; e = ctx->ninsns; - rvoff -= (e - s) << 2; + rvoff -= ninsns_rvoff(e - s); } if (is64) diff --git a/arch/riscv/net/bpf_jit_comp64.c b/arch/riscv/net/bpf_jit_comp64.c index 6cfd164cbe88..8a56b5293117 100644 --- a/arch/riscv/net/bpf_jit_comp64.c +++ b/arch/riscv/net/bpf_jit_comp64.c @@ -132,19 +132,23 @@ static void emit_imm(u8 rd, s64 val, struct rv_jit_context *ctx) * * This also means that we need to process LSB to MSB. */ - s64 upper = (val + (1 << 11)) >> 12, lower = val & 0xfff; + s64 upper = (val + (1 << 11)) >> 12; + /* Sign-extend lower 12 bits to 64 bits since immediates for li, addiw, + * and addi are signed and RVC checks will perform signed comparisons. + */ + s64 lower = ((val & 0xfff) << 52) >> 52; int shift; if (is_32b_int(val)) { if (upper) - emit(rv_lui(rd, upper), ctx); + emit_lui(rd, upper, ctx); if (!upper) { - emit(rv_addi(rd, RV_REG_ZERO, lower), ctx); + emit_li(rd, lower, ctx); return; } - emit(rv_addiw(rd, rd, lower), ctx); + emit_addiw(rd, rd, lower, ctx); return; } @@ -154,9 +158,9 @@ static void emit_imm(u8 rd, s64 val, struct rv_jit_context *ctx) emit_imm(rd, upper, ctx); - emit(rv_slli(rd, rd, shift), ctx); + emit_slli(rd, rd, shift, ctx); if (lower) - emit(rv_addi(rd, rd, lower), ctx); + emit_addi(rd, rd, lower, ctx); } static void __build_epilogue(bool is_tail_call, struct rv_jit_context *ctx) @@ -164,43 +168,43 @@ static void __build_epilogue(bool is_tail_call, struct rv_jit_context *ctx) int stack_adjust = ctx->stack_size, store_offset = stack_adjust - 8; if (seen_reg(RV_REG_RA, ctx)) { - emit(rv_ld(RV_REG_RA, store_offset, RV_REG_SP), ctx); + emit_ld(RV_REG_RA, store_offset, RV_REG_SP, ctx); store_offset -= 8; } - emit(rv_ld(RV_REG_FP, store_offset, RV_REG_SP), ctx); + emit_ld(RV_REG_FP, store_offset, RV_REG_SP, ctx); store_offset -= 8; if (seen_reg(RV_REG_S1, ctx)) { - emit(rv_ld(RV_REG_S1, store_offset, RV_REG_SP), ctx); + emit_ld(RV_REG_S1, store_offset, RV_REG_SP, ctx); store_offset -= 8; } if (seen_reg(RV_REG_S2, ctx)) { - emit(rv_ld(RV_REG_S2, store_offset, RV_REG_SP), ctx); + emit_ld(RV_REG_S2, store_offset, RV_REG_SP, ctx); store_offset -= 8; } if (seen_reg(RV_REG_S3, ctx)) { - emit(rv_ld(RV_REG_S3, store_offset, RV_REG_SP), ctx); + emit_ld(RV_REG_S3, store_offset, RV_REG_SP, ctx); store_offset -= 8; } if (seen_reg(RV_REG_S4, ctx)) { - emit(rv_ld(RV_REG_S4, store_offset, RV_REG_SP), ctx); + emit_ld(RV_REG_S4, store_offset, RV_REG_SP, ctx); store_offset -= 8; } if (seen_reg(RV_REG_S5, ctx)) { - emit(rv_ld(RV_REG_S5, store_offset, RV_REG_SP), ctx); + emit_ld(RV_REG_S5, store_offset, RV_REG_SP, ctx); store_offset -= 8; } if (seen_reg(RV_REG_S6, ctx)) { - emit(rv_ld(RV_REG_S6, store_offset, RV_REG_SP), ctx); + emit_ld(RV_REG_S6, store_offset, RV_REG_SP, ctx); store_offset -= 8; } - emit(rv_addi(RV_REG_SP, RV_REG_SP, stack_adjust), ctx); + emit_addi(RV_REG_SP, RV_REG_SP, stack_adjust, ctx); /* Set return value. */ if (!is_tail_call) - emit(rv_addi(RV_REG_A0, RV_REG_A5, 0), ctx); - emit(rv_jalr(RV_REG_ZERO, is_tail_call ? RV_REG_T3 : RV_REG_RA, - is_tail_call ? 4 : 0), /* skip TCC init */ - ctx); + emit_mv(RV_REG_A0, RV_REG_A5, ctx); + emit_jalr(RV_REG_ZERO, is_tail_call ? RV_REG_T3 : RV_REG_RA, + is_tail_call ? 4 : 0, /* skip TCC init */ + ctx); } static void emit_bcc(u8 cond, u8 rd, u8 rs, int rvoff, @@ -280,8 +284,8 @@ static void emit_branch(u8 cond, u8 rd, u8 rs, int rvoff, static void emit_zext_32(u8 reg, struct rv_jit_context *ctx) { - emit(rv_slli(reg, reg, 32), ctx); - emit(rv_srli(reg, reg, 32), ctx); + emit_slli(reg, reg, 32, ctx); + emit_srli(reg, reg, 32, ctx); } static int emit_bpf_tail_call(int insn, struct rv_jit_context *ctx) @@ -304,35 +308,35 @@ static int emit_bpf_tail_call(int insn, struct rv_jit_context *ctx) if (is_12b_check(off, insn)) return -1; emit(rv_lwu(RV_REG_T1, off, RV_REG_A1), ctx); - off = (tc_ninsn - (ctx->ninsns - start_insn)) << 2; + off = ninsns_rvoff(tc_ninsn - (ctx->ninsns - start_insn)); emit_branch(BPF_JGE, RV_REG_A2, RV_REG_T1, off, ctx); /* if (TCC-- < 0) * goto out; */ - emit(rv_addi(RV_REG_T1, tcc, -1), ctx); - off = (tc_ninsn - (ctx->ninsns - start_insn)) << 2; + emit_addi(RV_REG_T1, tcc, -1, ctx); + off = ninsns_rvoff(tc_ninsn - (ctx->ninsns - start_insn)); emit_branch(BPF_JSLT, tcc, RV_REG_ZERO, off, ctx); /* prog = array->ptrs[index]; * if (!prog) * goto out; */ - emit(rv_slli(RV_REG_T2, RV_REG_A2, 3), ctx); - emit(rv_add(RV_REG_T2, RV_REG_T2, RV_REG_A1), ctx); + emit_slli(RV_REG_T2, RV_REG_A2, 3, ctx); + emit_add(RV_REG_T2, RV_REG_T2, RV_REG_A1, ctx); off = offsetof(struct bpf_array, ptrs); if (is_12b_check(off, insn)) return -1; - emit(rv_ld(RV_REG_T2, off, RV_REG_T2), ctx); - off = (tc_ninsn - (ctx->ninsns - start_insn)) << 2; + emit_ld(RV_REG_T2, off, RV_REG_T2, ctx); + off = ninsns_rvoff(tc_ninsn - (ctx->ninsns - start_insn)); emit_branch(BPF_JEQ, RV_REG_T2, RV_REG_ZERO, off, ctx); /* goto *(prog->bpf_func + 4); */ off = offsetof(struct bpf_prog, bpf_func); if (is_12b_check(off, insn)) return -1; - emit(rv_ld(RV_REG_T3, off, RV_REG_T2), ctx); - emit(rv_addi(RV_REG_TCC, RV_REG_T1, 0), ctx); + emit_ld(RV_REG_T3, off, RV_REG_T2, ctx); + emit_mv(RV_REG_TCC, RV_REG_T1, ctx); __build_epilogue(true, ctx); return 0; } @@ -360,9 +364,9 @@ static void init_regs(u8 *rd, u8 *rs, const struct bpf_insn *insn, static void emit_zext_32_rd_rs(u8 *rd, u8 *rs, struct rv_jit_context *ctx) { - emit(rv_addi(RV_REG_T2, *rd, 0), ctx); + emit_mv(RV_REG_T2, *rd, ctx); emit_zext_32(RV_REG_T2, ctx); - emit(rv_addi(RV_REG_T1, *rs, 0), ctx); + emit_mv(RV_REG_T1, *rs, ctx); emit_zext_32(RV_REG_T1, ctx); *rd = RV_REG_T2; *rs = RV_REG_T1; @@ -370,15 +374,15 @@ static void emit_zext_32_rd_rs(u8 *rd, u8 *rs, struct rv_jit_context *ctx) static void emit_sext_32_rd_rs(u8 *rd, u8 *rs, struct rv_jit_context *ctx) { - emit(rv_addiw(RV_REG_T2, *rd, 0), ctx); - emit(rv_addiw(RV_REG_T1, *rs, 0), ctx); + emit_addiw(RV_REG_T2, *rd, 0, ctx); + emit_addiw(RV_REG_T1, *rs, 0, ctx); *rd = RV_REG_T2; *rs = RV_REG_T1; } static void emit_zext_32_rd_t1(u8 *rd, struct rv_jit_context *ctx) { - emit(rv_addi(RV_REG_T2, *rd, 0), ctx); + emit_mv(RV_REG_T2, *rd, ctx); emit_zext_32(RV_REG_T2, ctx); emit_zext_32(RV_REG_T1, ctx); *rd = RV_REG_T2; @@ -386,7 +390,7 @@ static void emit_zext_32_rd_t1(u8 *rd, struct rv_jit_context *ctx) static void emit_sext_32_rd(u8 *rd, struct rv_jit_context *ctx) { - emit(rv_addiw(RV_REG_T2, *rd, 0), ctx); + emit_addiw(RV_REG_T2, *rd, 0, ctx); *rd = RV_REG_T2; } @@ -432,7 +436,7 @@ static int emit_call(bool fixed, u64 addr, struct rv_jit_context *ctx) if (ret) return ret; rd = bpf_to_rv_reg(BPF_REG_0, ctx); - emit(rv_addi(rd, RV_REG_A0, 0), ctx); + emit_mv(rd, RV_REG_A0, ctx); return 0; } @@ -458,7 +462,7 @@ int bpf_jit_emit_insn(const struct bpf_insn *insn, struct rv_jit_context *ctx, emit_zext_32(rd, ctx); break; } - emit(is64 ? rv_addi(rd, rs, 0) : rv_addiw(rd, rs, 0), ctx); + emit_mv(rd, rs, ctx); if (!is64 && !aux->verifier_zext) emit_zext_32(rd, ctx); break; @@ -466,31 +470,35 @@ int bpf_jit_emit_insn(const struct bpf_insn *insn, struct rv_jit_context *ctx, /* dst = dst OP src */ case BPF_ALU | BPF_ADD | BPF_X: case BPF_ALU64 | BPF_ADD | BPF_X: - emit(is64 ? rv_add(rd, rd, rs) : rv_addw(rd, rd, rs), ctx); + emit_add(rd, rd, rs, ctx); if (!is64 && !aux->verifier_zext) emit_zext_32(rd, ctx); break; case BPF_ALU | BPF_SUB | BPF_X: case BPF_ALU64 | BPF_SUB | BPF_X: - emit(is64 ? rv_sub(rd, rd, rs) : rv_subw(rd, rd, rs), ctx); + if (is64) + emit_sub(rd, rd, rs, ctx); + else + emit_subw(rd, rd, rs, ctx); + if (!is64 && !aux->verifier_zext) emit_zext_32(rd, ctx); break; case BPF_ALU | BPF_AND | BPF_X: case BPF_ALU64 | BPF_AND | BPF_X: - emit(rv_and(rd, rd, rs), ctx); + emit_and(rd, rd, rs, ctx); if (!is64 && !aux->verifier_zext) emit_zext_32(rd, ctx); break; case BPF_ALU | BPF_OR | BPF_X: case BPF_ALU64 | BPF_OR | BPF_X: - emit(rv_or(rd, rd, rs), ctx); + emit_or(rd, rd, rs, ctx); if (!is64 && !aux->verifier_zext) emit_zext_32(rd, ctx); break; case BPF_ALU | BPF_XOR | BPF_X: case BPF_ALU64 | BPF_XOR | BPF_X: - emit(rv_xor(rd, rd, rs), ctx); + emit_xor(rd, rd, rs, ctx); if (!is64 && !aux->verifier_zext) emit_zext_32(rd, ctx); break; @@ -534,8 +542,7 @@ int bpf_jit_emit_insn(const struct bpf_insn *insn, struct rv_jit_context *ctx, /* dst = -dst */ case BPF_ALU | BPF_NEG: case BPF_ALU64 | BPF_NEG: - emit(is64 ? rv_sub(rd, RV_REG_ZERO, rd) : - rv_subw(rd, RV_REG_ZERO, rd), ctx); + emit_sub(rd, RV_REG_ZERO, rd, ctx); if (!is64 && !aux->verifier_zext) emit_zext_32(rd, ctx); break; @@ -544,8 +551,8 @@ int bpf_jit_emit_insn(const struct bpf_insn *insn, struct rv_jit_context *ctx, case BPF_ALU | BPF_END | BPF_FROM_LE: switch (imm) { case 16: - emit(rv_slli(rd, rd, 48), ctx); - emit(rv_srli(rd, rd, 48), ctx); + emit_slli(rd, rd, 48, ctx); + emit_srli(rd, rd, 48, ctx); break; case 32: if (!aux->verifier_zext) @@ -558,51 +565,51 @@ int bpf_jit_emit_insn(const struct bpf_insn *insn, struct rv_jit_context *ctx, break; case BPF_ALU | BPF_END | BPF_FROM_BE: - emit(rv_addi(RV_REG_T2, RV_REG_ZERO, 0), ctx); + emit_li(RV_REG_T2, 0, ctx); - emit(rv_andi(RV_REG_T1, rd, 0xff), ctx); - emit(rv_add(RV_REG_T2, RV_REG_T2, RV_REG_T1), ctx); - emit(rv_slli(RV_REG_T2, RV_REG_T2, 8), ctx); - emit(rv_srli(rd, rd, 8), ctx); + emit_andi(RV_REG_T1, rd, 0xff, ctx); + emit_add(RV_REG_T2, RV_REG_T2, RV_REG_T1, ctx); + emit_slli(RV_REG_T2, RV_REG_T2, 8, ctx); + emit_srli(rd, rd, 8, ctx); if (imm == 16) goto out_be; - emit(rv_andi(RV_REG_T1, rd, 0xff), ctx); - emit(rv_add(RV_REG_T2, RV_REG_T2, RV_REG_T1), ctx); - emit(rv_slli(RV_REG_T2, RV_REG_T2, 8), ctx); - emit(rv_srli(rd, rd, 8), ctx); + emit_andi(RV_REG_T1, rd, 0xff, ctx); + emit_add(RV_REG_T2, RV_REG_T2, RV_REG_T1, ctx); + emit_slli(RV_REG_T2, RV_REG_T2, 8, ctx); + emit_srli(rd, rd, 8, ctx); - emit(rv_andi(RV_REG_T1, rd, 0xff), ctx); - emit(rv_add(RV_REG_T2, RV_REG_T2, RV_REG_T1), ctx); - emit(rv_slli(RV_REG_T2, RV_REG_T2, 8), ctx); - emit(rv_srli(rd, rd, 8), ctx); + emit_andi(RV_REG_T1, rd, 0xff, ctx); + emit_add(RV_REG_T2, RV_REG_T2, RV_REG_T1, ctx); + emit_slli(RV_REG_T2, RV_REG_T2, 8, ctx); + emit_srli(rd, rd, 8, ctx); if (imm == 32) goto out_be; - emit(rv_andi(RV_REG_T1, rd, 0xff), ctx); - emit(rv_add(RV_REG_T2, RV_REG_T2, RV_REG_T1), ctx); - emit(rv_slli(RV_REG_T2, RV_REG_T2, 8), ctx); - emit(rv_srli(rd, rd, 8), ctx); - - emit(rv_andi(RV_REG_T1, rd, 0xff), ctx); - emit(rv_add(RV_REG_T2, RV_REG_T2, RV_REG_T1), ctx); - emit(rv_slli(RV_REG_T2, RV_REG_T2, 8), ctx); - emit(rv_srli(rd, rd, 8), ctx); - - emit(rv_andi(RV_REG_T1, rd, 0xff), ctx); - emit(rv_add(RV_REG_T2, RV_REG_T2, RV_REG_T1), ctx); - emit(rv_slli(RV_REG_T2, RV_REG_T2, 8), ctx); - emit(rv_srli(rd, rd, 8), ctx); - - emit(rv_andi(RV_REG_T1, rd, 0xff), ctx); - emit(rv_add(RV_REG_T2, RV_REG_T2, RV_REG_T1), ctx); - emit(rv_slli(RV_REG_T2, RV_REG_T2, 8), ctx); - emit(rv_srli(rd, rd, 8), ctx); + emit_andi(RV_REG_T1, rd, 0xff, ctx); + emit_add(RV_REG_T2, RV_REG_T2, RV_REG_T1, ctx); + emit_slli(RV_REG_T2, RV_REG_T2, 8, ctx); + emit_srli(rd, rd, 8, ctx); + + emit_andi(RV_REG_T1, rd, 0xff, ctx); + emit_add(RV_REG_T2, RV_REG_T2, RV_REG_T1, ctx); + emit_slli(RV_REG_T2, RV_REG_T2, 8, ctx); + emit_srli(rd, rd, 8, ctx); + + emit_andi(RV_REG_T1, rd, 0xff, ctx); + emit_add(RV_REG_T2, RV_REG_T2, RV_REG_T1, ctx); + emit_slli(RV_REG_T2, RV_REG_T2, 8, ctx); + emit_srli(rd, rd, 8, ctx); + + emit_andi(RV_REG_T1, rd, 0xff, ctx); + emit_add(RV_REG_T2, RV_REG_T2, RV_REG_T1, ctx); + emit_slli(RV_REG_T2, RV_REG_T2, 8, ctx); + emit_srli(rd, rd, 8, ctx); out_be: - emit(rv_andi(RV_REG_T1, rd, 0xff), ctx); - emit(rv_add(RV_REG_T2, RV_REG_T2, RV_REG_T1), ctx); + emit_andi(RV_REG_T1, rd, 0xff, ctx); + emit_add(RV_REG_T2, RV_REG_T2, RV_REG_T1, ctx); - emit(rv_addi(rd, RV_REG_T2, 0), ctx); + emit_mv(rd, RV_REG_T2, ctx); break; /* dst = imm */ @@ -617,12 +624,10 @@ out_be: case BPF_ALU | BPF_ADD | BPF_K: case BPF_ALU64 | BPF_ADD | BPF_K: if (is_12b_int(imm)) { - emit(is64 ? rv_addi(rd, rd, imm) : - rv_addiw(rd, rd, imm), ctx); + emit_addi(rd, rd, imm, ctx); } else { emit_imm(RV_REG_T1, imm, ctx); - emit(is64 ? rv_add(rd, rd, RV_REG_T1) : - rv_addw(rd, rd, RV_REG_T1), ctx); + emit_add(rd, rd, RV_REG_T1, ctx); } if (!is64 && !aux->verifier_zext) emit_zext_32(rd, ctx); @@ -630,12 +635,10 @@ out_be: case BPF_ALU | BPF_SUB | BPF_K: case BPF_ALU64 | BPF_SUB | BPF_K: if (is_12b_int(-imm)) { - emit(is64 ? rv_addi(rd, rd, -imm) : - rv_addiw(rd, rd, -imm), ctx); + emit_addi(rd, rd, -imm, ctx); } else { emit_imm(RV_REG_T1, imm, ctx); - emit(is64 ? rv_sub(rd, rd, RV_REG_T1) : - rv_subw(rd, rd, RV_REG_T1), ctx); + emit_sub(rd, rd, RV_REG_T1, ctx); } if (!is64 && !aux->verifier_zext) emit_zext_32(rd, ctx); @@ -643,10 +646,10 @@ out_be: case BPF_ALU | BPF_AND | BPF_K: case BPF_ALU64 | BPF_AND | BPF_K: if (is_12b_int(imm)) { - emit(rv_andi(rd, rd, imm), ctx); + emit_andi(rd, rd, imm, ctx); } else { emit_imm(RV_REG_T1, imm, ctx); - emit(rv_and(rd, rd, RV_REG_T1), ctx); + emit_and(rd, rd, RV_REG_T1, ctx); } if (!is64 && !aux->verifier_zext) emit_zext_32(rd, ctx); @@ -657,7 +660,7 @@ out_be: emit(rv_ori(rd, rd, imm), ctx); } else { emit_imm(RV_REG_T1, imm, ctx); - emit(rv_or(rd, rd, RV_REG_T1), ctx); + emit_or(rd, rd, RV_REG_T1, ctx); } if (!is64 && !aux->verifier_zext) emit_zext_32(rd, ctx); @@ -668,7 +671,7 @@ out_be: emit(rv_xori(rd, rd, imm), ctx); } else { emit_imm(RV_REG_T1, imm, ctx); - emit(rv_xor(rd, rd, RV_REG_T1), ctx); + emit_xor(rd, rd, RV_REG_T1, ctx); } if (!is64 && !aux->verifier_zext) emit_zext_32(rd, ctx); @@ -699,19 +702,28 @@ out_be: break; case BPF_ALU | BPF_LSH | BPF_K: case BPF_ALU64 | BPF_LSH | BPF_K: - emit(is64 ? rv_slli(rd, rd, imm) : rv_slliw(rd, rd, imm), ctx); + emit_slli(rd, rd, imm, ctx); + if (!is64 && !aux->verifier_zext) emit_zext_32(rd, ctx); break; case BPF_ALU | BPF_RSH | BPF_K: case BPF_ALU64 | BPF_RSH | BPF_K: - emit(is64 ? rv_srli(rd, rd, imm) : rv_srliw(rd, rd, imm), ctx); + if (is64) + emit_srli(rd, rd, imm, ctx); + else + emit(rv_srliw(rd, rd, imm), ctx); + if (!is64 && !aux->verifier_zext) emit_zext_32(rd, ctx); break; case BPF_ALU | BPF_ARSH | BPF_K: case BPF_ALU64 | BPF_ARSH | BPF_K: - emit(is64 ? rv_srai(rd, rd, imm) : rv_sraiw(rd, rd, imm), ctx); + if (is64) + emit_srai(rd, rd, imm, ctx); + else + emit(rv_sraiw(rd, rd, imm), ctx); + if (!is64 && !aux->verifier_zext) emit_zext_32(rd, ctx); break; @@ -757,13 +769,13 @@ out_be: e = ctx->ninsns; /* Adjust for extra insns */ - rvoff -= (e - s) << 2; + rvoff -= ninsns_rvoff(e - s); } if (BPF_OP(code) == BPF_JSET) { /* Adjust for and */ rvoff -= 4; - emit(rv_and(RV_REG_T1, rd, rs), ctx); + emit_and(RV_REG_T1, rd, rs, ctx); emit_branch(BPF_JNE, RV_REG_T1, RV_REG_ZERO, rvoff, ctx); } else { @@ -810,7 +822,7 @@ out_be: e = ctx->ninsns; /* Adjust for extra insns */ - rvoff -= (e - s) << 2; + rvoff -= ninsns_rvoff(e - s); emit_branch(BPF_OP(code), rd, rs, rvoff, ctx); break; @@ -819,19 +831,19 @@ out_be: rvoff = rv_offset(i, off, ctx); s = ctx->ninsns; if (is_12b_int(imm)) { - emit(rv_andi(RV_REG_T1, rd, imm), ctx); + emit_andi(RV_REG_T1, rd, imm, ctx); } else { emit_imm(RV_REG_T1, imm, ctx); - emit(rv_and(RV_REG_T1, rd, RV_REG_T1), ctx); + emit_and(RV_REG_T1, rd, RV_REG_T1, ctx); } /* For jset32, we should clear the upper 32 bits of t1, but * sign-extension is sufficient here and saves one instruction, * as t1 is used only in comparison against zero. */ if (!is64 && imm < 0) - emit(rv_addiw(RV_REG_T1, RV_REG_T1, 0), ctx); + emit_addiw(RV_REG_T1, RV_REG_T1, 0, ctx); e = ctx->ninsns; - rvoff -= (e - s) << 2; + rvoff -= ninsns_rvoff(e - s); emit_branch(BPF_JNE, RV_REG_T1, RV_REG_ZERO, rvoff, ctx); break; @@ -887,7 +899,7 @@ out_be: } emit_imm(RV_REG_T1, off, ctx); - emit(rv_add(RV_REG_T1, RV_REG_T1, rs), ctx); + emit_add(RV_REG_T1, RV_REG_T1, rs, ctx); emit(rv_lbu(rd, 0, RV_REG_T1), ctx); if (insn_is_zext(&insn[1])) return 1; @@ -899,7 +911,7 @@ out_be: } emit_imm(RV_REG_T1, off, ctx); - emit(rv_add(RV_REG_T1, RV_REG_T1, rs), ctx); + emit_add(RV_REG_T1, RV_REG_T1, rs, ctx); emit(rv_lhu(rd, 0, RV_REG_T1), ctx); if (insn_is_zext(&insn[1])) return 1; @@ -911,20 +923,20 @@ out_be: } emit_imm(RV_REG_T1, off, ctx); - emit(rv_add(RV_REG_T1, RV_REG_T1, rs), ctx); + emit_add(RV_REG_T1, RV_REG_T1, rs, ctx); emit(rv_lwu(rd, 0, RV_REG_T1), ctx); if (insn_is_zext(&insn[1])) return 1; break; case BPF_LDX | BPF_MEM | BPF_DW: if (is_12b_int(off)) { - emit(rv_ld(rd, off, rs), ctx); + emit_ld(rd, off, rs, ctx); break; } emit_imm(RV_REG_T1, off, ctx); - emit(rv_add(RV_REG_T1, RV_REG_T1, rs), ctx); - emit(rv_ld(rd, 0, RV_REG_T1), ctx); + emit_add(RV_REG_T1, RV_REG_T1, rs, ctx); + emit_ld(rd, 0, RV_REG_T1, ctx); break; /* ST: *(size *)(dst + off) = imm */ @@ -936,7 +948,7 @@ out_be: } emit_imm(RV_REG_T2, off, ctx); - emit(rv_add(RV_REG_T2, RV_REG_T2, rd), ctx); + emit_add(RV_REG_T2, RV_REG_T2, rd, ctx); emit(rv_sb(RV_REG_T2, 0, RV_REG_T1), ctx); break; @@ -948,30 +960,30 @@ out_be: } emit_imm(RV_REG_T2, off, ctx); - emit(rv_add(RV_REG_T2, RV_REG_T2, rd), ctx); + emit_add(RV_REG_T2, RV_REG_T2, rd, ctx); emit(rv_sh(RV_REG_T2, 0, RV_REG_T1), ctx); break; case BPF_ST | BPF_MEM | BPF_W: emit_imm(RV_REG_T1, imm, ctx); if (is_12b_int(off)) { - emit(rv_sw(rd, off, RV_REG_T1), ctx); + emit_sw(rd, off, RV_REG_T1, ctx); break; } emit_imm(RV_REG_T2, off, ctx); - emit(rv_add(RV_REG_T2, RV_REG_T2, rd), ctx); - emit(rv_sw(RV_REG_T2, 0, RV_REG_T1), ctx); + emit_add(RV_REG_T2, RV_REG_T2, rd, ctx); + emit_sw(RV_REG_T2, 0, RV_REG_T1, ctx); break; case BPF_ST | BPF_MEM | BPF_DW: emit_imm(RV_REG_T1, imm, ctx); if (is_12b_int(off)) { - emit(rv_sd(rd, off, RV_REG_T1), ctx); + emit_sd(rd, off, RV_REG_T1, ctx); break; } emit_imm(RV_REG_T2, off, ctx); - emit(rv_add(RV_REG_T2, RV_REG_T2, rd), ctx); - emit(rv_sd(RV_REG_T2, 0, RV_REG_T1), ctx); + emit_add(RV_REG_T2, RV_REG_T2, rd, ctx); + emit_sd(RV_REG_T2, 0, RV_REG_T1, ctx); break; /* STX: *(size *)(dst + off) = src */ @@ -982,7 +994,7 @@ out_be: } emit_imm(RV_REG_T1, off, ctx); - emit(rv_add(RV_REG_T1, RV_REG_T1, rd), ctx); + emit_add(RV_REG_T1, RV_REG_T1, rd, ctx); emit(rv_sb(RV_REG_T1, 0, rs), ctx); break; case BPF_STX | BPF_MEM | BPF_H: @@ -992,28 +1004,28 @@ out_be: } emit_imm(RV_REG_T1, off, ctx); - emit(rv_add(RV_REG_T1, RV_REG_T1, rd), ctx); + emit_add(RV_REG_T1, RV_REG_T1, rd, ctx); emit(rv_sh(RV_REG_T1, 0, rs), ctx); break; case BPF_STX | BPF_MEM | BPF_W: if (is_12b_int(off)) { - emit(rv_sw(rd, off, rs), ctx); + emit_sw(rd, off, rs, ctx); break; } emit_imm(RV_REG_T1, off, ctx); - emit(rv_add(RV_REG_T1, RV_REG_T1, rd), ctx); - emit(rv_sw(RV_REG_T1, 0, rs), ctx); + emit_add(RV_REG_T1, RV_REG_T1, rd, ctx); + emit_sw(RV_REG_T1, 0, rs, ctx); break; case BPF_STX | BPF_MEM | BPF_DW: if (is_12b_int(off)) { - emit(rv_sd(rd, off, rs), ctx); + emit_sd(rd, off, rs, ctx); break; } emit_imm(RV_REG_T1, off, ctx); - emit(rv_add(RV_REG_T1, RV_REG_T1, rd), ctx); - emit(rv_sd(RV_REG_T1, 0, rs), ctx); + emit_add(RV_REG_T1, RV_REG_T1, rd, ctx); + emit_sd(RV_REG_T1, 0, rs, ctx); break; /* STX XADD: lock *(u32 *)(dst + off) += src */ case BPF_STX | BPF_XADD | BPF_W: @@ -1021,10 +1033,10 @@ out_be: case BPF_STX | BPF_XADD | BPF_DW: if (off) { if (is_12b_int(off)) { - emit(rv_addi(RV_REG_T1, rd, off), ctx); + emit_addi(RV_REG_T1, rd, off, ctx); } else { emit_imm(RV_REG_T1, off, ctx); - emit(rv_add(RV_REG_T1, RV_REG_T1, rd), ctx); + emit_add(RV_REG_T1, RV_REG_T1, rd, ctx); } rd = RV_REG_T1; @@ -1073,52 +1085,53 @@ void bpf_jit_build_prologue(struct rv_jit_context *ctx) /* First instruction is always setting the tail-call-counter * (TCC) register. This instruction is skipped for tail calls. + * Force using a 4-byte (non-compressed) instruction. */ emit(rv_addi(RV_REG_TCC, RV_REG_ZERO, MAX_TAIL_CALL_CNT), ctx); - emit(rv_addi(RV_REG_SP, RV_REG_SP, -stack_adjust), ctx); + emit_addi(RV_REG_SP, RV_REG_SP, -stack_adjust, ctx); if (seen_reg(RV_REG_RA, ctx)) { - emit(rv_sd(RV_REG_SP, store_offset, RV_REG_RA), ctx); + emit_sd(RV_REG_SP, store_offset, RV_REG_RA, ctx); store_offset -= 8; } - emit(rv_sd(RV_REG_SP, store_offset, RV_REG_FP), ctx); + emit_sd(RV_REG_SP, store_offset, RV_REG_FP, ctx); store_offset -= 8; if (seen_reg(RV_REG_S1, ctx)) { - emit(rv_sd(RV_REG_SP, store_offset, RV_REG_S1), ctx); + emit_sd(RV_REG_SP, store_offset, RV_REG_S1, ctx); store_offset -= 8; } if (seen_reg(RV_REG_S2, ctx)) { - emit(rv_sd(RV_REG_SP, store_offset, RV_REG_S2), ctx); + emit_sd(RV_REG_SP, store_offset, RV_REG_S2, ctx); store_offset -= 8; } if (seen_reg(RV_REG_S3, ctx)) { - emit(rv_sd(RV_REG_SP, store_offset, RV_REG_S3), ctx); + emit_sd(RV_REG_SP, store_offset, RV_REG_S3, ctx); store_offset -= 8; } if (seen_reg(RV_REG_S4, ctx)) { - emit(rv_sd(RV_REG_SP, store_offset, RV_REG_S4), ctx); + emit_sd(RV_REG_SP, store_offset, RV_REG_S4, ctx); store_offset -= 8; } if (seen_reg(RV_REG_S5, ctx)) { - emit(rv_sd(RV_REG_SP, store_offset, RV_REG_S5), ctx); + emit_sd(RV_REG_SP, store_offset, RV_REG_S5, ctx); store_offset -= 8; } if (seen_reg(RV_REG_S6, ctx)) { - emit(rv_sd(RV_REG_SP, store_offset, RV_REG_S6), ctx); + emit_sd(RV_REG_SP, store_offset, RV_REG_S6, ctx); store_offset -= 8; } - emit(rv_addi(RV_REG_FP, RV_REG_SP, stack_adjust), ctx); + emit_addi(RV_REG_FP, RV_REG_SP, stack_adjust, ctx); if (bpf_stack_adjust) - emit(rv_addi(RV_REG_S5, RV_REG_SP, bpf_stack_adjust), ctx); + emit_addi(RV_REG_S5, RV_REG_SP, bpf_stack_adjust, ctx); /* Program contains calls and tail calls, so RV_REG_TCC need * to be saved across calls. */ if (seen_tail_call(ctx) && seen_call(ctx)) - emit(rv_addi(RV_REG_TCC_SAVED, RV_REG_TCC, 0), ctx); + emit_mv(RV_REG_TCC_SAVED, RV_REG_TCC, ctx); ctx->stack_size = stack_adjust; } diff --git a/arch/riscv/net/bpf_jit_core.c b/arch/riscv/net/bpf_jit_core.c index 709b94ece3ed..3630d447352c 100644 --- a/arch/riscv/net/bpf_jit_core.c +++ b/arch/riscv/net/bpf_jit_core.c @@ -73,7 +73,7 @@ struct bpf_prog *bpf_int_jit_compile(struct bpf_prog *prog) if (ctx->offset) { extra_pass = true; - image_size = sizeof(u32) * ctx->ninsns; + image_size = sizeof(*ctx->insns) * ctx->ninsns; goto skip_init_ctx; } @@ -103,7 +103,7 @@ struct bpf_prog *bpf_int_jit_compile(struct bpf_prog *prog) if (jit_data->header) break; - image_size = sizeof(u32) * ctx->ninsns; + image_size = sizeof(*ctx->insns) * ctx->ninsns; jit_data->header = bpf_jit_binary_alloc(image_size, &jit_data->image, @@ -114,7 +114,7 @@ struct bpf_prog *bpf_int_jit_compile(struct bpf_prog *prog) goto out_offset; } - ctx->insns = (u32 *)jit_data->image; + ctx->insns = (u16 *)jit_data->image; /* * Now, when the image is allocated, the image can * potentially shrink more (auipc/jalr -> jal). |