Skip to content

L
linux
Commit da192676 authored by Mark Rutland's avatar Mark Rutland Committed by Will Deacon
Browse files
Options

arm64: entry: mark entry code as noinstr 

Functions in entry-common.c are marked as notrace and NOKPROBE_SYMBOL(),
but they're still subject to other instrumentation which may rely on
lockdep/rcu/context-tracking being up-to-date, and may cause nested
exceptions (e.g. for WARN/BUG or KASAN's use of BRK) which will corrupt
exceptions registers which have not yet been read.

Prevent this by marking all functions in entry-common.c as noinstr to
prevent compiler instrumentation. This also blacklists the functions for
tracing and kprobes, so we don't need to handle that separately.
Functions elsewhere will be dealt with in subsequent patches.
Signed-off-by: default avatarMark Rutland <mark.rutland@arm.com>
Cc: Catalin Marinas <catalin.marinas@arm.com>
Cc: James Morse <james.morse@arm.com>
Cc: Will Deacon <will@kernel.org>
Link: https://lore.kernel.org/r/20201130115950.22492-4-mark.rutland@arm.comSigned-off-by: default avatarWill Deacon <will@kernel.org>
parent 114e0a68
Show whitespace changes
Inline Side-by-side
Showing with 25 additions and 50 deletions
arch/arm64/kernel/entry-common.c
View file @ da192676
...@@ -17,7 +17,7 @@ ...@@ -17,7 +17,7 @@
#include <asm/mmu.h> #include <asm/mmu.h>
#include <asm/sysreg.h> #include <asm/sysreg.h>
static void notrace el1_abort(struct pt_regs *regs, unsigned long esr) static void noinstr el1_abort(struct pt_regs *regs, unsigned long esr)
{ {
unsigned long far = read_sysreg(far_el1); unsigned long far = read_sysreg(far_el1);
...@@ -25,32 +25,28 @@ static void notrace el1_abort(struct pt_regs *regs, unsigned long esr) ...@@ -25,32 +25,28 @@ static void notrace el1_abort(struct pt_regs *regs, unsigned long esr)
far = untagged_addr(far); far = untagged_addr(far);
do_mem_abort(far, esr, regs); do_mem_abort(far, esr, regs);
} }
NOKPROBE_SYMBOL(el1_abort);
static void notrace el1_pc(struct pt_regs *regs, unsigned long esr) static void noinstr el1_pc(struct pt_regs *regs, unsigned long esr)
{ {
unsigned long far = read_sysreg(far_el1); unsigned long far = read_sysreg(far_el1);
local_daif_inherit(regs); local_daif_inherit(regs);
do_sp_pc_abort(far, esr, regs); do_sp_pc_abort(far, esr, regs);
} }
NOKPROBE_SYMBOL(el1_pc);
static void notrace el1_undef(struct pt_regs *regs) static void noinstr el1_undef(struct pt_regs *regs)
{ {
local_daif_inherit(regs); local_daif_inherit(regs);
do_undefinstr(regs); do_undefinstr(regs);
} }
NOKPROBE_SYMBOL(el1_undef);
static void notrace el1_inv(struct pt_regs *regs, unsigned long esr) static void noinstr el1_inv(struct pt_regs *regs, unsigned long esr)
{ {
local_daif_inherit(regs); local_daif_inherit(regs);
bad_mode(regs, 0, esr); bad_mode(regs, 0, esr);
} }
NOKPROBE_SYMBOL(el1_inv);
static void notrace el1_dbg(struct pt_regs *regs, unsigned long esr) static void noinstr el1_dbg(struct pt_regs *regs, unsigned long esr)
{ {
unsigned long far = read_sysreg(far_el1); unsigned long far = read_sysreg(far_el1);
...@@ -64,16 +60,14 @@ static void notrace el1_dbg(struct pt_regs *regs, unsigned long esr) ...@@ -64,16 +60,14 @@ static void notrace el1_dbg(struct pt_regs *regs, unsigned long esr)
do_debug_exception(far, esr, regs); do_debug_exception(far, esr, regs);
} }
NOKPROBE_SYMBOL(el1_dbg);
static void notrace el1_fpac(struct pt_regs *regs, unsigned long esr) static void noinstr el1_fpac(struct pt_regs *regs, unsigned long esr)
{ {
local_daif_inherit(regs); local_daif_inherit(regs);
do_ptrauth_fault(regs, esr); do_ptrauth_fault(regs, esr);
} }
NOKPROBE_SYMBOL(el1_fpac);
asmlinkage void notrace el1_sync_handler(struct pt_regs *regs) asmlinkage void noinstr el1_sync_handler(struct pt_regs *regs)
{ {
unsigned long esr = read_sysreg(esr_el1); unsigned long esr = read_sysreg(esr_el1);
...@@ -106,9 +100,8 @@ asmlinkage void notrace el1_sync_handler(struct pt_regs *regs) ...@@ -106,9 +100,8 @@ asmlinkage void notrace el1_sync_handler(struct pt_regs *regs)
el1_inv(regs, esr); el1_inv(regs, esr);
} }
} }
NOKPROBE_SYMBOL(el1_sync_handler);
static void notrace el0_da(struct pt_regs *regs, unsigned long esr) static void noinstr el0_da(struct pt_regs *regs, unsigned long esr)
{ {
unsigned long far = read_sysreg(far_el1); unsigned long far = read_sysreg(far_el1);
...@@ -117,9 +110,8 @@ static void notrace el0_da(struct pt_regs *regs, unsigned long esr) ...@@ -117,9 +110,8 @@ static void notrace el0_da(struct pt_regs *regs, unsigned long esr)
far = untagged_addr(far); far = untagged_addr(far);
do_mem_abort(far, esr, regs); do_mem_abort(far, esr, regs);
} }
NOKPROBE_SYMBOL(el0_da);
static void notrace el0_ia(struct pt_regs *regs, unsigned long esr) static void noinstr el0_ia(struct pt_regs *regs, unsigned long esr)
{ {
unsigned long far = read_sysreg(far_el1); unsigned long far = read_sysreg(far_el1);
...@@ -135,41 +127,36 @@ static void notrace el0_ia(struct pt_regs *regs, unsigned long esr) ...@@ -135,41 +127,36 @@ static void notrace el0_ia(struct pt_regs *regs, unsigned long esr)
local_daif_restore(DAIF_PROCCTX); local_daif_restore(DAIF_PROCCTX);
do_mem_abort(far, esr, regs); do_mem_abort(far, esr, regs);
} }
NOKPROBE_SYMBOL(el0_ia);
static void notrace el0_fpsimd_acc(struct pt_regs *regs, unsigned long esr) static void noinstr el0_fpsimd_acc(struct pt_regs *regs, unsigned long esr)
{ {
user_exit_irqoff(); user_exit_irqoff();
local_daif_restore(DAIF_PROCCTX); local_daif_restore(DAIF_PROCCTX);
do_fpsimd_acc(esr, regs); do_fpsimd_acc(esr, regs);
} }
NOKPROBE_SYMBOL(el0_fpsimd_acc);
static void notrace el0_sve_acc(struct pt_regs *regs, unsigned long esr) static void noinstr el0_sve_acc(struct pt_regs *regs, unsigned long esr)
{ {
user_exit_irqoff(); user_exit_irqoff();
local_daif_restore(DAIF_PROCCTX); local_daif_restore(DAIF_PROCCTX);
do_sve_acc(esr, regs); do_sve_acc(esr, regs);
} }
NOKPROBE_SYMBOL(el0_sve_acc);
static void notrace el0_fpsimd_exc(struct pt_regs *regs, unsigned long esr) static void noinstr el0_fpsimd_exc(struct pt_regs *regs, unsigned long esr)
{ {
user_exit_irqoff(); user_exit_irqoff();
local_daif_restore(DAIF_PROCCTX); local_daif_restore(DAIF_PROCCTX);
do_fpsimd_exc(esr, regs); do_fpsimd_exc(esr, regs);
} }
NOKPROBE_SYMBOL(el0_fpsimd_exc);
static void notrace el0_sys(struct pt_regs *regs, unsigned long esr) static void noinstr el0_sys(struct pt_regs *regs, unsigned long esr)
{ {
user_exit_irqoff(); user_exit_irqoff();
local_daif_restore(DAIF_PROCCTX); local_daif_restore(DAIF_PROCCTX);
do_sysinstr(esr, regs); do_sysinstr(esr, regs);
} }
NOKPROBE_SYMBOL(el0_sys);
static void notrace el0_pc(struct pt_regs *regs, unsigned long esr) static void noinstr el0_pc(struct pt_regs *regs, unsigned long esr)
{ {
unsigned long far = read_sysreg(far_el1); unsigned long far = read_sysreg(far_el1);
...@@ -180,41 +167,36 @@ static void notrace el0_pc(struct pt_regs *regs, unsigned long esr) ...@@ -180,41 +167,36 @@ static void notrace el0_pc(struct pt_regs *regs, unsigned long esr)
local_daif_restore(DAIF_PROCCTX); local_daif_restore(DAIF_PROCCTX);
do_sp_pc_abort(far, esr, regs); do_sp_pc_abort(far, esr, regs);
} }
NOKPROBE_SYMBOL(el0_pc);
static void notrace el0_sp(struct pt_regs *regs, unsigned long esr) static void noinstr el0_sp(struct pt_regs *regs, unsigned long esr)
{ {
user_exit_irqoff(); user_exit_irqoff();
local_daif_restore(DAIF_PROCCTX); local_daif_restore(DAIF_PROCCTX);
do_sp_pc_abort(regs->sp, esr, regs); do_sp_pc_abort(regs->sp, esr, regs);
} }
NOKPROBE_SYMBOL(el0_sp);
static void notrace el0_undef(struct pt_regs *regs) static void noinstr el0_undef(struct pt_regs *regs)
{ {
user_exit_irqoff(); user_exit_irqoff();
local_daif_restore(DAIF_PROCCTX); local_daif_restore(DAIF_PROCCTX);
do_undefinstr(regs); do_undefinstr(regs);
} }
NOKPROBE_SYMBOL(el0_undef);
static void notrace el0_bti(struct pt_regs *regs) static void noinstr el0_bti(struct pt_regs *regs)
{ {
user_exit_irqoff(); user_exit_irqoff();
local_daif_restore(DAIF_PROCCTX); local_daif_restore(DAIF_PROCCTX);
do_bti(regs); do_bti(regs);
} }
NOKPROBE_SYMBOL(el0_bti);
static void notrace el0_inv(struct pt_regs *regs, unsigned long esr) static void noinstr el0_inv(struct pt_regs *regs, unsigned long esr)
{ {
user_exit_irqoff(); user_exit_irqoff();
local_daif_restore(DAIF_PROCCTX); local_daif_restore(DAIF_PROCCTX);
bad_el0_sync(regs, 0, esr); bad_el0_sync(regs, 0, esr);
} }
NOKPROBE_SYMBOL(el0_inv);
static void notrace el0_dbg(struct pt_regs *regs, unsigned long esr) static void noinstr el0_dbg(struct pt_regs *regs, unsigned long esr)
{ {
/* Only watchpoints write FAR_EL1, otherwise its UNKNOWN */ /* Only watchpoints write FAR_EL1, otherwise its UNKNOWN */
unsigned long far = read_sysreg(far_el1); unsigned long far = read_sysreg(far_el1);
...@@ -226,26 +208,23 @@ static void notrace el0_dbg(struct pt_regs *regs, unsigned long esr) ...@@ -226,26 +208,23 @@ static void notrace el0_dbg(struct pt_regs *regs, unsigned long esr)
do_debug_exception(far, esr, regs); do_debug_exception(far, esr, regs);
local_daif_restore(DAIF_PROCCTX_NOIRQ); local_daif_restore(DAIF_PROCCTX_NOIRQ);
} }
NOKPROBE_SYMBOL(el0_dbg);
static void notrace el0_svc(struct pt_regs *regs) static void noinstr el0_svc(struct pt_regs *regs)
{ {
if (system_uses_irq_prio_masking()) if (system_uses_irq_prio_masking())
gic_write_pmr(GIC_PRIO_IRQON | GIC_PRIO_PSR_I_SET); gic_write_pmr(GIC_PRIO_IRQON | GIC_PRIO_PSR_I_SET);
do_el0_svc(regs); do_el0_svc(regs);
} }
NOKPROBE_SYMBOL(el0_svc);
static void notrace el0_fpac(struct pt_regs *regs, unsigned long esr) static void noinstr el0_fpac(struct pt_regs *regs, unsigned long esr)
{ {
user_exit_irqoff(); user_exit_irqoff();
local_daif_restore(DAIF_PROCCTX); local_daif_restore(DAIF_PROCCTX);
do_ptrauth_fault(regs, esr); do_ptrauth_fault(regs, esr);
} }
NOKPROBE_SYMBOL(el0_fpac);
asmlinkage void notrace el0_sync_handler(struct pt_regs *regs) asmlinkage void noinstr el0_sync_handler(struct pt_regs *regs)
{ {
unsigned long esr = read_sysreg(esr_el1); unsigned long esr = read_sysreg(esr_el1);
...@@ -297,27 +276,24 @@ asmlinkage void notrace el0_sync_handler(struct pt_regs *regs) ...@@ -297,27 +276,24 @@ asmlinkage void notrace el0_sync_handler(struct pt_regs *regs)
el0_inv(regs, esr); el0_inv(regs, esr);
} }
} }
NOKPROBE_SYMBOL(el0_sync_handler);
#ifdef CONFIG_COMPAT #ifdef CONFIG_COMPAT
static void notrace el0_cp15(struct pt_regs *regs, unsigned long esr) static void noinstr el0_cp15(struct pt_regs *regs, unsigned long esr)
{ {
user_exit_irqoff(); user_exit_irqoff();
local_daif_restore(DAIF_PROCCTX); local_daif_restore(DAIF_PROCCTX);
do_cp15instr(esr, regs); do_cp15instr(esr, regs);
} }
NOKPROBE_SYMBOL(el0_cp15);
static void notrace el0_svc_compat(struct pt_regs *regs) static void noinstr el0_svc_compat(struct pt_regs *regs)
{ {
if (system_uses_irq_prio_masking()) if (system_uses_irq_prio_masking())
gic_write_pmr(GIC_PRIO_IRQON | GIC_PRIO_PSR_I_SET); gic_write_pmr(GIC_PRIO_IRQON | GIC_PRIO_PSR_I_SET);
do_el0_svc_compat(regs); do_el0_svc_compat(regs);
} }
NOKPROBE_SYMBOL(el0_svc_compat);
asmlinkage void notrace el0_sync_compat_handler(struct pt_regs *regs) asmlinkage void noinstr el0_sync_compat_handler(struct pt_regs *regs)
{ {
unsigned long esr = read_sysreg(esr_el1); unsigned long esr = read_sysreg(esr_el1);
...@@ -360,5 +336,4 @@ asmlinkage void notrace el0_sync_compat_handler(struct pt_regs *regs) ...@@ -360,5 +336,4 @@ asmlinkage void notrace el0_sync_compat_handler(struct pt_regs *regs)
el0_inv(regs, esr); el0_inv(regs, esr);
} }
} }
NOKPROBE_SYMBOL(el0_sync_compat_handler);
#endif /* CONFIG_COMPAT */ #endif /* CONFIG_COMPAT */
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment
GitLab Nexedi Edition | About GitLab | About Nexedi | 沪ICP备2021021310号-2 | 沪ICP备2021021310号-7