aboutsummaryrefslogtreecommitdiffstats
path: root/arch/riscv/kernel
diff options
context:
space:
mode:
Diffstat (limited to 'arch/riscv/kernel')
-rw-r--r--arch/riscv/kernel/Makefile2
-rw-r--r--arch/riscv/kernel/asm-offsets.c3
-rw-r--r--arch/riscv/kernel/entry.S54
-rw-r--r--arch/riscv/kernel/head.S10
-rw-r--r--arch/riscv/kernel/jump_label.c53
-rw-r--r--arch/riscv/kernel/module.c16
-rw-r--r--arch/riscv/kernel/process.c6
-rw-r--r--arch/riscv/kernel/ptrace.c33
-rw-r--r--arch/riscv/kernel/smp.c15
-rw-r--r--arch/riscv/kernel/smpboot.c12
-rw-r--r--arch/riscv/kernel/traps.c8
-rw-r--r--arch/riscv/kernel/vdso/Makefile3
-rw-r--r--arch/riscv/kernel/vmlinux.lds.S2
13 files changed, 170 insertions, 47 deletions
diff --git a/arch/riscv/kernel/Makefile b/arch/riscv/kernel/Makefile
index b355cf485671..a5287ab9f7f2 100644
--- a/arch/riscv/kernel/Makefile
+++ b/arch/riscv/kernel/Makefile
@@ -53,4 +53,6 @@ endif
obj-$(CONFIG_HOTPLUG_CPU) += cpu-hotplug.o
obj-$(CONFIG_KGDB) += kgdb.o
+obj-$(CONFIG_JUMP_LABEL) += jump_label.o
+
clean:
diff --git a/arch/riscv/kernel/asm-offsets.c b/arch/riscv/kernel/asm-offsets.c
index 07cb9c10de4e..db203442c08f 100644
--- a/arch/riscv/kernel/asm-offsets.c
+++ b/arch/riscv/kernel/asm-offsets.c
@@ -27,9 +27,6 @@ void asm_offsets(void)
OFFSET(TASK_THREAD_S9, task_struct, thread.s[9]);
OFFSET(TASK_THREAD_S10, task_struct, thread.s[10]);
OFFSET(TASK_THREAD_S11, task_struct, thread.s[11]);
- OFFSET(TASK_THREAD_SP, task_struct, thread.sp);
- OFFSET(TASK_STACK, task_struct, stack);
- OFFSET(TASK_TI, task_struct, thread_info);
OFFSET(TASK_TI_FLAGS, task_struct, thread_info.flags);
OFFSET(TASK_TI_PREEMPT_COUNT, task_struct, thread_info.preempt_count);
OFFSET(TASK_TI_KERNEL_SP, task_struct, thread_info.kernel_sp);
diff --git a/arch/riscv/kernel/entry.S b/arch/riscv/kernel/entry.S
index cae7e6d4c7ef..524d918f3601 100644
--- a/arch/riscv/kernel/entry.S
+++ b/arch/riscv/kernel/entry.S
@@ -97,19 +97,36 @@ _save_context:
la gp, __global_pointer$
.option pop
- la ra, ret_from_exception
+#ifdef CONFIG_TRACE_IRQFLAGS
+ call trace_hardirqs_off
+#endif
+
+#ifdef CONFIG_CONTEXT_TRACKING
+ /* If previous state is in user mode, call context_tracking_user_exit. */
+ li a0, SR_PP
+ and a0, s1, a0
+ bnez a0, skip_context_tracking
+ call context_tracking_user_exit
+skip_context_tracking:
+#endif
+
/*
* MSB of cause differentiates between
* interrupts and exceptions
*/
bge s4, zero, 1f
+ la ra, ret_from_exception
+
/* Handle interrupts */
move a0, sp /* pt_regs */
la a1, handle_arch_irq
REG_L a1, (a1)
jr a1
1:
+#ifdef CONFIG_TRACE_IRQFLAGS
+ call trace_hardirqs_on
+#endif
/*
* Exceptions run with interrupts enabled or disabled depending on the
* state of SR_PIE in m/sstatus.
@@ -119,6 +136,7 @@ _save_context:
csrs CSR_STATUS, SR_IE
1:
+ la ra, ret_from_exception
/* Handle syscalls */
li t0, EXC_SYSCALL
beq s4, t0, handle_syscall
@@ -137,6 +155,17 @@ _save_context:
tail do_trap_unknown
handle_syscall:
+#if defined(CONFIG_TRACE_IRQFLAGS) || defined(CONFIG_CONTEXT_TRACKING)
+ /* Recover a0 - a7 for system calls */
+ REG_L a0, PT_A0(sp)
+ REG_L a1, PT_A1(sp)
+ REG_L a2, PT_A2(sp)
+ REG_L a3, PT_A3(sp)
+ REG_L a4, PT_A4(sp)
+ REG_L a5, PT_A5(sp)
+ REG_L a6, PT_A6(sp)
+ REG_L a7, PT_A7(sp)
+#endif
/* save the initial A0 value (needed in signal handlers) */
REG_S a0, PT_ORIG_A0(sp)
/*
@@ -190,6 +219,9 @@ ret_from_syscall_rejected:
ret_from_exception:
REG_L s0, PT_STATUS(sp)
csrc CSR_STATUS, SR_IE
+#ifdef CONFIG_TRACE_IRQFLAGS
+ call trace_hardirqs_off
+#endif
#ifdef CONFIG_RISCV_M_MODE
/* the MPP value is too large to be used as an immediate arg for addi */
li t0, SR_MPP
@@ -205,6 +237,10 @@ resume_userspace:
andi s1, s0, _TIF_WORK_MASK
bnez s1, work_pending
+#ifdef CONFIG_CONTEXT_TRACKING
+ call context_tracking_user_enter
+#endif
+
/* Save unwound kernel stack pointer in thread_info */
addi s0, sp, PT_SIZE_ON_STACK
REG_S s0, TASK_TI_KERNEL_SP(tp)
@@ -216,6 +252,16 @@ resume_userspace:
csrw CSR_SCRATCH, tp
restore_all:
+#ifdef CONFIG_TRACE_IRQFLAGS
+ REG_L s1, PT_STATUS(sp)
+ andi t0, s1, SR_PIE
+ beqz t0, 1f
+ call trace_hardirqs_on
+ j 2f
+1:
+ call trace_hardirqs_off
+2:
+#endif
REG_L a0, PT_STATUS(sp)
/*
* The current load reservation is effectively part of the processor's
@@ -389,12 +435,8 @@ ENTRY(__switch_to)
lw a4, TASK_TI_CPU(a1)
sw a3, TASK_TI_CPU(a1)
sw a4, TASK_TI_CPU(a0)
-#if TASK_TI != 0
-#error "TASK_TI != 0: tp will contain a 'struct thread_info', not a 'struct task_struct' so get_current() won't work."
- addi tp, a1, TASK_TI
-#else
+ /* The offset of thread_info in task_struct is zero. */
move tp, a1
-#endif
ret
ENDPROC(__switch_to)
diff --git a/arch/riscv/kernel/head.S b/arch/riscv/kernel/head.S
index 7ed1b22950fd..d0c5c316e9bb 100644
--- a/arch/riscv/kernel/head.S
+++ b/arch/riscv/kernel/head.S
@@ -77,10 +77,16 @@ relocate:
csrw CSR_SATP, a0
.align 2
1:
- /* Set trap vector to spin forever to help debug */
- la a0, .Lsecondary_park
+ /* Set trap vector to exception handler */
+ la a0, handle_exception
csrw CSR_TVEC, a0
+ /*
+ * Set sup0 scratch register to 0, indicating to exception vector that
+ * we are presently executing in kernel.
+ */
+ csrw CSR_SCRATCH, zero
+
/* Reload the global pointer */
.option push
.option norelax
diff --git a/arch/riscv/kernel/jump_label.c b/arch/riscv/kernel/jump_label.c
new file mode 100644
index 000000000000..20e09056d141
--- /dev/null
+++ b/arch/riscv/kernel/jump_label.c
@@ -0,0 +1,53 @@
+// SPDX-License-Identifier: GPL-2.0-only
+/*
+ * Copyright (C) 2020 Emil Renner Berthing
+ *
+ * Based on arch/arm64/kernel/jump_label.c
+ */
+#include <linux/jump_label.h>
+#include <linux/kernel.h>
+#include <linux/memory.h>
+#include <linux/mutex.h>
+#include <asm/bug.h>
+#include <asm/patch.h>
+
+#define RISCV_INSN_NOP 0x00000013U
+#define RISCV_INSN_JAL 0x0000006fU
+
+void arch_jump_label_transform(struct jump_entry *entry,
+ enum jump_label_type type)
+{
+ void *addr = (void *)jump_entry_code(entry);
+ u32 insn;
+
+ if (type == JUMP_LABEL_JMP) {
+ long offset = jump_entry_target(entry) - jump_entry_code(entry);
+
+ if (WARN_ON(offset & 1 || offset < -524288 || offset >= 524288))
+ return;
+
+ insn = RISCV_INSN_JAL |
+ (((u32)offset & GENMASK(19, 12)) << (12 - 12)) |
+ (((u32)offset & GENMASK(11, 11)) << (20 - 11)) |
+ (((u32)offset & GENMASK(10, 1)) << (21 - 1)) |
+ (((u32)offset & GENMASK(20, 20)) << (31 - 20));
+ } else {
+ insn = RISCV_INSN_NOP;
+ }
+
+ mutex_lock(&text_mutex);
+ patch_text_nosync(addr, &insn, sizeof(insn));
+ mutex_unlock(&text_mutex);
+}
+
+void arch_jump_label_transform_static(struct jump_entry *entry,
+ enum jump_label_type type)
+{
+ /*
+ * We use the same instructions in the arch_static_branch and
+ * arch_static_branch_jump inline functions, so there's no
+ * need to patch them up here.
+ * The core will call arch_jump_label_transform when those
+ * instructions need to be replaced.
+ */
+}
diff --git a/arch/riscv/kernel/module.c b/arch/riscv/kernel/module.c
index 7191342c54da..104fba889cf7 100644
--- a/arch/riscv/kernel/module.c
+++ b/arch/riscv/kernel/module.c
@@ -263,6 +263,13 @@ static int apply_r_riscv_add32_rela(struct module *me, u32 *location,
return 0;
}
+static int apply_r_riscv_add64_rela(struct module *me, u32 *location,
+ Elf_Addr v)
+{
+ *(u64 *)location += (u64)v;
+ return 0;
+}
+
static int apply_r_riscv_sub32_rela(struct module *me, u32 *location,
Elf_Addr v)
{
@@ -270,6 +277,13 @@ static int apply_r_riscv_sub32_rela(struct module *me, u32 *location,
return 0;
}
+static int apply_r_riscv_sub64_rela(struct module *me, u32 *location,
+ Elf_Addr v)
+{
+ *(u64 *)location -= (u64)v;
+ return 0;
+}
+
static int (*reloc_handlers_rela[]) (struct module *me, u32 *location,
Elf_Addr v) = {
[R_RISCV_32] = apply_r_riscv_32_rela,
@@ -290,7 +304,9 @@ static int (*reloc_handlers_rela[]) (struct module *me, u32 *location,
[R_RISCV_RELAX] = apply_r_riscv_relax_rela,
[R_RISCV_ALIGN] = apply_r_riscv_align_rela,
[R_RISCV_ADD32] = apply_r_riscv_add32_rela,
+ [R_RISCV_ADD64] = apply_r_riscv_add64_rela,
[R_RISCV_SUB32] = apply_r_riscv_sub32_rela,
+ [R_RISCV_SUB64] = apply_r_riscv_sub64_rela,
};
int apply_relocate_add(Elf_Shdr *sechdrs, const char *strtab,
diff --git a/arch/riscv/kernel/process.c b/arch/riscv/kernel/process.c
index 31f39442df72..2b97c493427c 100644
--- a/arch/riscv/kernel/process.c
+++ b/arch/riscv/kernel/process.c
@@ -24,6 +24,12 @@
register unsigned long gp_in_global __asm__("gp");
+#ifdef CONFIG_STACKPROTECTOR
+#include <linux/stackprotector.h>
+unsigned long __stack_chk_guard __read_mostly;
+EXPORT_SYMBOL(__stack_chk_guard);
+#endif
+
extern asmlinkage void ret_from_fork(void);
extern asmlinkage void ret_from_kernel_thread(void);
diff --git a/arch/riscv/kernel/ptrace.c b/arch/riscv/kernel/ptrace.c
index 444dc7b0fd78..2d6395f5ad54 100644
--- a/arch/riscv/kernel/ptrace.c
+++ b/arch/riscv/kernel/ptrace.c
@@ -30,13 +30,10 @@ enum riscv_regset {
static int riscv_gpr_get(struct task_struct *target,
const struct user_regset *regset,
- unsigned int pos, unsigned int count,
- void *kbuf, void __user *ubuf)
+ struct membuf to)
{
- struct pt_regs *regs;
-
- regs = task_pt_regs(target);
- return user_regset_copyout(&pos, &count, &kbuf, &ubuf, regs, 0, -1);
+ return membuf_write(&to, task_pt_regs(target),
+ sizeof(struct user_regs_struct));
}
static int riscv_gpr_set(struct task_struct *target,
@@ -55,21 +52,13 @@ static int riscv_gpr_set(struct task_struct *target,
#ifdef CONFIG_FPU
static int riscv_fpr_get(struct task_struct *target,
const struct user_regset *regset,
- unsigned int pos, unsigned int count,
- void *kbuf, void __user *ubuf)
+ struct membuf to)
{
- int ret;
struct __riscv_d_ext_state *fstate = &target->thread.fstate;
- ret = user_regset_copyout(&pos, &count, &kbuf, &ubuf, fstate, 0,
- offsetof(struct __riscv_d_ext_state, fcsr));
- if (!ret) {
- ret = user_regset_copyout(&pos, &count, &kbuf, &ubuf, fstate, 0,
- offsetof(struct __riscv_d_ext_state, fcsr) +
- sizeof(fstate->fcsr));
- }
-
- return ret;
+ membuf_write(&to, fstate, offsetof(struct __riscv_d_ext_state, fcsr));
+ membuf_store(&to, fstate->fcsr);
+ return membuf_zero(&to, 4); // explicitly pad
}
static int riscv_fpr_set(struct task_struct *target,
@@ -98,8 +87,8 @@ static const struct user_regset riscv_user_regset[] = {
.n = ELF_NGREG,
.size = sizeof(elf_greg_t),
.align = sizeof(elf_greg_t),
- .get = &riscv_gpr_get,
- .set = &riscv_gpr_set,
+ .regset_get = riscv_gpr_get,
+ .set = riscv_gpr_set,
},
#ifdef CONFIG_FPU
[REGSET_F] = {
@@ -107,8 +96,8 @@ static const struct user_regset riscv_user_regset[] = {
.n = ELF_NFPREG,
.size = sizeof(elf_fpreg_t),
.align = sizeof(elf_fpreg_t),
- .get = &riscv_fpr_get,
- .set = &riscv_fpr_set,
+ .regset_get = riscv_fpr_get,
+ .set = riscv_fpr_set,
},
#endif
};
diff --git a/arch/riscv/kernel/smp.c b/arch/riscv/kernel/smp.c
index b1d4f452f843..554b0fb47060 100644
--- a/arch/riscv/kernel/smp.c
+++ b/arch/riscv/kernel/smp.c
@@ -16,6 +16,7 @@
#include <linux/sched.h>
#include <linux/seq_file.h>
#include <linux/delay.h>
+#include <linux/irq_work.h>
#include <asm/clint.h>
#include <asm/sbi.h>
@@ -26,6 +27,7 @@ enum ipi_message_type {
IPI_RESCHEDULE,
IPI_CALL_FUNC,
IPI_CPU_STOP,
+ IPI_IRQ_WORK,
IPI_MAX
};
@@ -123,6 +125,13 @@ static inline void clear_ipi(void)
clint_clear_ipi(cpuid_to_hartid_map(smp_processor_id()));
}
+#ifdef CONFIG_IRQ_WORK
+void arch_irq_work_raise(void)
+{
+ send_ipi_single(smp_processor_id(), IPI_IRQ_WORK);
+}
+#endif
+
void handle_IPI(struct pt_regs *regs)
{
struct pt_regs *old_regs = set_irq_regs(regs);
@@ -158,6 +167,11 @@ void handle_IPI(struct pt_regs *regs)
ipi_stop();
}
+ if (ops & (1 << IPI_IRQ_WORK)) {
+ stats[IPI_IRQ_WORK]++;
+ irq_work_run();
+ }
+
BUG_ON((ops >> IPI_MAX) != 0);
/* Order data access and bit testing. */
@@ -173,6 +187,7 @@ static const char * const ipi_names[] = {
[IPI_RESCHEDULE] = "Rescheduling interrupts",
[IPI_CALL_FUNC] = "Function call interrupts",
[IPI_CPU_STOP] = "CPU stop interrupts",
+ [IPI_IRQ_WORK] = "IRQ work interrupts",
};
void show_ipi_stats(struct seq_file *p, int prec)
diff --git a/arch/riscv/kernel/smpboot.c b/arch/riscv/kernel/smpboot.c
index 4e9922790f6e..356825a57551 100644
--- a/arch/riscv/kernel/smpboot.c
+++ b/arch/riscv/kernel/smpboot.c
@@ -106,7 +106,7 @@ void __init setup_smp(void)
}
}
-int start_secondary_cpu(int cpu, struct task_struct *tidle)
+static int start_secondary_cpu(int cpu, struct task_struct *tidle)
{
if (cpu_ops[cpu]->cpu_start)
return cpu_ops[cpu]->cpu_start(cpu, tidle);
@@ -121,7 +121,6 @@ int __cpu_up(unsigned int cpu, struct task_struct *tidle)
ret = start_secondary_cpu(cpu, tidle);
if (!ret) {
- lockdep_assert_held(&cpu_running);
wait_for_completion_timeout(&cpu_running,
msecs_to_jiffies(1000));
@@ -146,6 +145,7 @@ void __init smp_cpus_done(unsigned int max_cpus)
asmlinkage __visible void smp_callin(void)
{
struct mm_struct *mm = &init_mm;
+ unsigned int curr_cpuid = smp_processor_id();
if (!IS_ENABLED(CONFIG_RISCV_SBI))
clint_clear_ipi(cpuid_to_hartid_map(smp_processor_id()));
@@ -154,10 +154,10 @@ asmlinkage __visible void smp_callin(void)
mmgrab(mm);
current->active_mm = mm;
- trap_init();
- notify_cpu_starting(smp_processor_id());
- update_siblings_masks(smp_processor_id());
- set_cpu_online(smp_processor_id(), 1);
+ notify_cpu_starting(curr_cpuid);
+ update_siblings_masks(curr_cpuid);
+ set_cpu_online(curr_cpuid, 1);
+
/*
* Remote TLB flushes are ignored while the CPU is offline, so emit
* a local TLB flush right now just in case.
diff --git a/arch/riscv/kernel/traps.c b/arch/riscv/kernel/traps.c
index 7d95cce5e47c..ad14f4466d92 100644
--- a/arch/riscv/kernel/traps.c
+++ b/arch/riscv/kernel/traps.c
@@ -174,13 +174,7 @@ int is_valid_bugaddr(unsigned long pc)
}
#endif /* CONFIG_GENERIC_BUG */
+/* stvec & scratch is already set from head.S */
void trap_init(void)
{
- /*
- * Set sup0 scratch register to 0, indicating to exception vector
- * that we are presently executing in the kernel
- */
- csr_write(CSR_SCRATCH, 0);
- /* Set the exception vector address */
- csr_write(CSR_TVEC, &handle_exception);
}
diff --git a/arch/riscv/kernel/vdso/Makefile b/arch/riscv/kernel/vdso/Makefile
index e4c7c2c8a02f..478e7338ddc1 100644
--- a/arch/riscv/kernel/vdso/Makefile
+++ b/arch/riscv/kernel/vdso/Makefile
@@ -16,6 +16,8 @@ vdso-syms += flush_icache
# Files to link into the vdso
obj-vdso = $(patsubst %, %.o, $(vdso-syms)) note.o
+ccflags-y := -fno-stack-protector
+
ifneq ($(c-gettimeofday-y),)
CFLAGS_vgettimeofday.o += -fPIC -include $(c-gettimeofday-y)
endif
@@ -32,6 +34,7 @@ CFLAGS_REMOVE_vgettimeofday.o = $(CC_FLAGS_FTRACE) -Os
# Disable gcov profiling for VDSO code
GCOV_PROFILE := n
+KCOV_INSTRUMENT := n
# Force dependency
$(obj)/vdso.o: $(obj)/vdso.so
diff --git a/arch/riscv/kernel/vmlinux.lds.S b/arch/riscv/kernel/vmlinux.lds.S
index e6f8016b366a..f3586e31ed1e 100644
--- a/arch/riscv/kernel/vmlinux.lds.S
+++ b/arch/riscv/kernel/vmlinux.lds.S
@@ -22,6 +22,7 @@ SECTIONS
/* Beginning of code and text segment */
. = LOAD_OFFSET;
_start = .;
+ _stext = .;
HEAD_TEXT_SECTION
. = ALIGN(PAGE_SIZE);
@@ -54,7 +55,6 @@ SECTIONS
. = ALIGN(SECTION_ALIGN);
.text : {
_text = .;
- _stext = .;
TEXT_TEXT
SCHED_TEXT
CPUIDLE_TEXT
generated by cgit 1.2.3-korg (git 2.39.0) at 2024-05-02 21:00:30 +0000