Skip to content

L
linux
Commit 10b02564 authored by Ingo Molnar's avatar Ingo Molnar
Browse files
Options

Merge branch 'perf/kprobes' into perf/core 

Conflicts:
	arch/x86/kernel/traps.c

The kprobes enhancements are fully cooked, ship them upstream.
Signed-off-by: default avatarIngo Molnar <mingo@kernel.org>
parents c56d3406 69902c71
Expand all Hide whitespace changes
Inline Side-by-side
Showing with 583 additions and 432 deletions
Documentation/kprobes.txt
View file @ 10b02564
...@@ -22,8 +22,9 @@ Appendix B: The kprobes sysctl interface ...@@ -22,8 +22,9 @@ Appendix B: The kprobes sysctl interface
Kprobes enables you to dynamically break into any kernel routine and Kprobes enables you to dynamically break into any kernel routine and
collect debugging and performance information non-disruptively. You collect debugging and performance information non-disruptively. You
can trap at almost any kernel code address, specifying a handler can trap at almost any kernel code address(*), specifying a handler
routine to be invoked when the breakpoint is hit. routine to be invoked when the breakpoint is hit.
(*: some parts of the kernel code can not be trapped, see 1.5 Blacklist)
There are currently three types of probes: kprobes, jprobes, and There are currently three types of probes: kprobes, jprobes, and
kretprobes (also called return probes). A kprobe can be inserted kretprobes (also called return probes). A kprobe can be inserted
...@@ -273,6 +274,19 @@ using one of the following techniques: ...@@ -273,6 +274,19 @@ using one of the following techniques:
or or
- Execute 'sysctl -w debug.kprobes_optimization=n' - Execute 'sysctl -w debug.kprobes_optimization=n'
1.5 Blacklist
Kprobes can probe most of the kernel except itself. This means
that there are some functions where kprobes cannot probe. Probing
(trapping) such functions can cause a recursive trap (e.g. double
fault) or the nested probe handler may never be called.
Kprobes manages such functions as a blacklist.
If you want to add a function into the blacklist, you just need
to (1) include linux/kprobes.h and (2) use NOKPROBE_SYMBOL() macro
to specify a blacklisted function.
Kprobes checks the given probe address against the blacklist and
rejects registering it, if the given address is in the blacklist.
2. Architectures Supported 2. Architectures Supported
Kprobes, jprobes, and return probes are implemented on the following Kprobes, jprobes, and return probes are implemented on the following
......
arch/x86/include/asm/asm.h
View file @ 10b02564
...@@ -57,6 +57,12 @@ ...@@ -57,6 +57,12 @@
.long (from) - . ; \ .long (from) - . ; \
.long (to) - . + 0x7ffffff0 ; \ .long (to) - . + 0x7ffffff0 ; \
.popsection .popsection
# define _ASM_NOKPROBE(entry) \
.pushsection "_kprobe_blacklist","aw" ; \
_ASM_ALIGN ; \
_ASM_PTR (entry); \
.popsection
#else #else
# define _ASM_EXTABLE(from,to) \ # define _ASM_EXTABLE(from,to) \
" .pushsection \"__ex_table\",\"a\"\n" \ " .pushsection \"__ex_table\",\"a\"\n" \
...@@ -71,6 +77,7 @@ ...@@ -71,6 +77,7 @@
" .long (" #from ") - .\n" \ " .long (" #from ") - .\n" \
" .long (" #to ") - . + 0x7ffffff0\n" \ " .long (" #to ") - . + 0x7ffffff0\n" \
" .popsection\n" " .popsection\n"
/* For C file, we already have NOKPROBE_SYMBOL macro */
#endif #endif
#endif /* _ASM_X86_ASM_H */ #endif /* _ASM_X86_ASM_H */
arch/x86/include/asm/kprobes.h
View file @ 10b02564
...@@ -116,4 +116,6 @@ struct kprobe_ctlblk { ...@@ -116,4 +116,6 @@ struct kprobe_ctlblk {
extern int kprobe_fault_handler(struct pt_regs *regs, int trapnr); extern int kprobe_fault_handler(struct pt_regs *regs, int trapnr);
extern int kprobe_exceptions_notify(struct notifier_block *self, extern int kprobe_exceptions_notify(struct notifier_block *self,
unsigned long val, void *data); unsigned long val, void *data);
extern int kprobe_int3_handler(struct pt_regs *regs);
extern int kprobe_debug_handler(struct pt_regs *regs);
#endif /* _ASM_X86_KPROBES_H */ #endif /* _ASM_X86_KPROBES_H */
arch/x86/include/asm/traps.h
View file @ 10b02564
...@@ -68,7 +68,7 @@ dotraplinkage void do_segment_not_present(struct pt_regs *, long); ...@@ -68,7 +68,7 @@ dotraplinkage void do_segment_not_present(struct pt_regs *, long);
dotraplinkage void do_stack_segment(struct pt_regs *, long); dotraplinkage void do_stack_segment(struct pt_regs *, long);
#ifdef CONFIG_X86_64 #ifdef CONFIG_X86_64
dotraplinkage void do_double_fault(struct pt_regs *, long); dotraplinkage void do_double_fault(struct pt_regs *, long);
asmlinkage __kprobes struct pt_regs *sync_regs(struct pt_regs *); asmlinkage struct pt_regs *sync_regs(struct pt_regs *);
#endif #endif
dotraplinkage void do_general_protection(struct pt_regs *, long); dotraplinkage void do_general_protection(struct pt_regs *, long);
dotraplinkage void do_page_fault(struct pt_regs *, unsigned long); dotraplinkage void do_page_fault(struct pt_regs *, unsigned long);
......
arch/x86/kernel/alternative.c
View file @ 10b02564
...@@ -5,7 +5,6 @@ ...@@ -5,7 +5,6 @@
#include <linux/mutex.h> #include <linux/mutex.h>
#include <linux/list.h> #include <linux/list.h>
#include <linux/stringify.h> #include <linux/stringify.h>
#include <linux/kprobes.h>
#include <linux/mm.h> #include <linux/mm.h>
#include <linux/vmalloc.h> #include <linux/vmalloc.h>
#include <linux/memory.h> #include <linux/memory.h>
...@@ -551,7 +550,7 @@ void *__init_or_module text_poke_early(void *addr, const void *opcode, ...@@ -551,7 +550,7 @@ void *__init_or_module text_poke_early(void *addr, const void *opcode,
* *
* Note: Must be called under text_mutex. * Note: Must be called under text_mutex.
*/ */
void *__kprobes text_poke(void *addr, const void *opcode, size_t len) void *text_poke(void *addr, const void *opcode, size_t len)
{ {
unsigned long flags; unsigned long flags;
char *vaddr; char *vaddr;
......
arch/x86/kernel/apic/hw_nmi.c
View file @ 10b02564
...@@ -60,7 +60,7 @@ void arch_trigger_all_cpu_backtrace(void) ...@@ -60,7 +60,7 @@ void arch_trigger_all_cpu_backtrace(void)
smp_mb__after_clear_bit(); smp_mb__after_clear_bit();
} }
static int __kprobes static int
arch_trigger_all_cpu_backtrace_handler(unsigned int cmd, struct pt_regs *regs) arch_trigger_all_cpu_backtrace_handler(unsigned int cmd, struct pt_regs *regs)
{ {
int cpu; int cpu;
...@@ -80,6 +80,7 @@ arch_trigger_all_cpu_backtrace_handler(unsigned int cmd, struct pt_regs *regs) ...@@ -80,6 +80,7 @@ arch_trigger_all_cpu_backtrace_handler(unsigned int cmd, struct pt_regs *regs)
return NMI_DONE; return NMI_DONE;
} }
NOKPROBE_SYMBOL(arch_trigger_all_cpu_backtrace_handler);
static int __init register_trigger_all_cpu_backtrace(void) static int __init register_trigger_all_cpu_backtrace(void)
{ {
......
arch/x86/kernel/cpu/common.c
View file @ 10b02564
...@@ -8,6 +8,7 @@ ...@@ -8,6 +8,7 @@
#include <linux/delay.h> #include <linux/delay.h>
#include <linux/sched.h> #include <linux/sched.h>
#include <linux/init.h> #include <linux/init.h>
#include <linux/kprobes.h>
#include <linux/kgdb.h> #include <linux/kgdb.h>
#include <linux/smp.h> #include <linux/smp.h>
#include <linux/io.h> #include <linux/io.h>
...@@ -1160,6 +1161,7 @@ int is_debug_stack(unsigned long addr) ...@@ -1160,6 +1161,7 @@ int is_debug_stack(unsigned long addr)
(addr <= __get_cpu_var(debug_stack_addr) && (addr <= __get_cpu_var(debug_stack_addr) &&
addr > (__get_cpu_var(debug_stack_addr) - DEBUG_STKSZ)); addr > (__get_cpu_var(debug_stack_addr) - DEBUG_STKSZ));
} }
NOKPROBE_SYMBOL(is_debug_stack);
DEFINE_PER_CPU(u32, debug_idt_ctr); DEFINE_PER_CPU(u32, debug_idt_ctr);
...@@ -1168,6 +1170,7 @@ void debug_stack_set_zero(void) ...@@ -1168,6 +1170,7 @@ void debug_stack_set_zero(void)
this_cpu_inc(debug_idt_ctr); this_cpu_inc(debug_idt_ctr);
load_current_idt(); load_current_idt();
} }
NOKPROBE_SYMBOL(debug_stack_set_zero);
void debug_stack_reset(void) void debug_stack_reset(void)
{ {
...@@ -1176,6 +1179,7 @@ void debug_stack_reset(void) ...@@ -1176,6 +1179,7 @@ void debug_stack_reset(void)
if (this_cpu_dec_return(debug_idt_ctr) == 0) if (this_cpu_dec_return(debug_idt_ctr) == 0)
load_current_idt(); load_current_idt();
} }
NOKPROBE_SYMBOL(debug_stack_reset);
#else /* CONFIG_X86_64 */ #else /* CONFIG_X86_64 */
......
arch/x86/kernel/cpu/perf_event.c
View file @ 10b02564
...@@ -1293,7 +1293,7 @@ void perf_events_lapic_init(void) ...@@ -1293,7 +1293,7 @@ void perf_events_lapic_init(void)
apic_write(APIC_LVTPC, APIC_DM_NMI); apic_write(APIC_LVTPC, APIC_DM_NMI);
} }
static int __kprobes static int
perf_event_nmi_handler(unsigned int cmd, struct pt_regs *regs) perf_event_nmi_handler(unsigned int cmd, struct pt_regs *regs)
{ {
u64 start_clock; u64 start_clock;
...@@ -1311,6 +1311,7 @@ perf_event_nmi_handler(unsigned int cmd, struct pt_regs *regs) ...@@ -1311,6 +1311,7 @@ perf_event_nmi_handler(unsigned int cmd, struct pt_regs *regs)
return ret; return ret;
} }
NOKPROBE_SYMBOL(perf_event_nmi_handler);
struct event_constraint emptyconstraint; struct event_constraint emptyconstraint;
struct event_constraint unconstrained; struct event_constraint unconstrained;
......
arch/x86/kernel/cpu/perf_event_amd_ibs.c
View file @ 10b02564
...@@ -593,7 +593,7 @@ static int perf_ibs_handle_irq(struct perf_ibs *perf_ibs, struct pt_regs *iregs) ...@@ -593,7 +593,7 @@ static int perf_ibs_handle_irq(struct perf_ibs *perf_ibs, struct pt_regs *iregs)
return 1; return 1;
} }
static int __kprobes static int
perf_ibs_nmi_handler(unsigned int cmd, struct pt_regs *regs) perf_ibs_nmi_handler(unsigned int cmd, struct pt_regs *regs)
{ {
int handled = 0; int handled = 0;
...@@ -606,6 +606,7 @@ perf_ibs_nmi_handler(unsigned int cmd, struct pt_regs *regs) ...@@ -606,6 +606,7 @@ perf_ibs_nmi_handler(unsigned int cmd, struct pt_regs *regs)
return handled; return handled;
} }
NOKPROBE_SYMBOL(perf_ibs_nmi_handler);
static __init int perf_ibs_pmu_init(struct perf_ibs *perf_ibs, char *name) static __init int perf_ibs_pmu_init(struct perf_ibs *perf_ibs, char *name)
{ {
......
arch/x86/kernel/dumpstack.c
View file @ 10b02564
...@@ -200,7 +200,7 @@ static arch_spinlock_t die_lock = __ARCH_SPIN_LOCK_UNLOCKED; ...@@ -200,7 +200,7 @@ static arch_spinlock_t die_lock = __ARCH_SPIN_LOCK_UNLOCKED;
static int die_owner = -1; static int die_owner = -1;
static unsigned int die_nest_count; static unsigned int die_nest_count;
unsigned __kprobes long oops_begin(void) unsigned long oops_begin(void)
{ {
int cpu; int cpu;
unsigned long flags; unsigned long flags;
...@@ -223,8 +223,9 @@ unsigned __kprobes long oops_begin(void) ...@@ -223,8 +223,9 @@ unsigned __kprobes long oops_begin(void)
return flags; return flags;
} }
EXPORT_SYMBOL_GPL(oops_begin); EXPORT_SYMBOL_GPL(oops_begin);
NOKPROBE_SYMBOL(oops_begin);
void __kprobes oops_end(unsigned long flags, struct pt_regs *regs, int signr) void oops_end(unsigned long flags, struct pt_regs *regs, int signr)
{ {
if (regs && kexec_should_crash(current)) if (regs && kexec_should_crash(current))
crash_kexec(regs); crash_kexec(regs);
...@@ -247,8 +248,9 @@ void __kprobes oops_end(unsigned long flags, struct pt_regs *regs, int signr) ...@@ -247,8 +248,9 @@ void __kprobes oops_end(unsigned long flags, struct pt_regs *regs, int signr)
panic("Fatal exception"); panic("Fatal exception");
do_exit(signr); do_exit(signr);
} }
NOKPROBE_SYMBOL(oops_end);
int __kprobes __die(const char *str, struct pt_regs *regs, long err) int __die(const char *str, struct pt_regs *regs, long err)
{ {
#ifdef CONFIG_X86_32 #ifdef CONFIG_X86_32
unsigned short ss; unsigned short ss;
...@@ -291,6 +293,7 @@ int __kprobes __die(const char *str, struct pt_regs *regs, long err) ...@@ -291,6 +293,7 @@ int __kprobes __die(const char *str, struct pt_regs *regs, long err)
#endif #endif
return 0; return 0;
} }
NOKPROBE_SYMBOL(__die);
/* /*
* This is gone through when something in the kernel has done something bad * This is gone through when something in the kernel has done something bad
......
arch/x86/kernel/entry_32.S
View file @ 10b02564
...@@ -314,10 +314,6 @@ ENTRY(ret_from_kernel_thread) ...@@ -314,10 +314,6 @@ ENTRY(ret_from_kernel_thread)
CFI_ENDPROC CFI_ENDPROC
ENDPROC(ret_from_kernel_thread) ENDPROC(ret_from_kernel_thread)
/*
* Interrupt exit functions should be protected against kprobes
*/
.pushsection .kprobes.text, "ax"
/* /*
* Return to user mode is not as complex as all this looks, * Return to user mode is not as complex as all this looks,
* but we want the default path for a system call return to * but we want the default path for a system call return to
...@@ -372,10 +368,6 @@ need_resched: ...@@ -372,10 +368,6 @@ need_resched:
END(resume_kernel) END(resume_kernel)
#endif #endif
CFI_ENDPROC CFI_ENDPROC
/*
* End of kprobes section
*/
.popsection
/* SYSENTER_RETURN points to after the "sysenter" instruction in /* SYSENTER_RETURN points to after the "sysenter" instruction in
the vsyscall page. See vsyscall-sysentry.S, which defines the symbol. */ the vsyscall page. See vsyscall-sysentry.S, which defines the symbol. */
...@@ -495,10 +487,6 @@ sysexit_audit: ...@@ -495,10 +487,6 @@ sysexit_audit:
PTGS_TO_GS_EX PTGS_TO_GS_EX
ENDPROC(ia32_sysenter_target) ENDPROC(ia32_sysenter_target)
/*
* syscall stub including irq exit should be protected against kprobes
*/
.pushsection .kprobes.text, "ax"
# system call handler stub # system call handler stub
ENTRY(system_call) ENTRY(system_call)
RING0_INT_FRAME # can't unwind into user space anyway RING0_INT_FRAME # can't unwind into user space anyway
...@@ -691,10 +679,6 @@ syscall_badsys: ...@@ -691,10 +679,6 @@ syscall_badsys:
jmp resume_userspace jmp resume_userspace
END(syscall_badsys) END(syscall_badsys)
CFI_ENDPROC CFI_ENDPROC
/*
* End of kprobes section
*/
.popsection
.macro FIXUP_ESPFIX_STACK .macro FIXUP_ESPFIX_STACK
/* /*
...@@ -781,10 +765,6 @@ common_interrupt: ...@@ -781,10 +765,6 @@ common_interrupt:
ENDPROC(common_interrupt) ENDPROC(common_interrupt)
CFI_ENDPROC CFI_ENDPROC
/*
* Irq entries should be protected against kprobes
*/
.pushsection .kprobes.text, "ax"
#define BUILD_INTERRUPT3(name, nr, fn) \ #define BUILD_INTERRUPT3(name, nr, fn) \
ENTRY(name) \ ENTRY(name) \
RING0_INT_FRAME; \ RING0_INT_FRAME; \
...@@ -961,10 +941,6 @@ ENTRY(spurious_interrupt_bug) ...@@ -961,10 +941,6 @@ ENTRY(spurious_interrupt_bug)
jmp error_code jmp error_code
CFI_ENDPROC CFI_ENDPROC
END(spurious_interrupt_bug) END(spurious_interrupt_bug)
/*
* End of kprobes section
*/
.popsection
#ifdef CONFIG_XEN #ifdef CONFIG_XEN
/* Xen doesn't set %esp to be precisely what the normal sysenter /* Xen doesn't set %esp to be precisely what the normal sysenter
...@@ -1239,11 +1215,6 @@ return_to_handler: ...@@ -1239,11 +1215,6 @@ return_to_handler:
jmp *%ecx jmp *%ecx
#endif #endif
/*
* Some functions should be protected against kprobes
*/
.pushsection .kprobes.text, "ax"
#ifdef CONFIG_TRACING #ifdef CONFIG_TRACING
ENTRY(trace_page_fault) ENTRY(trace_page_fault)
RING0_EC_FRAME RING0_EC_FRAME
...@@ -1453,7 +1424,3 @@ ENTRY(async_page_fault) ...@@ -1453,7 +1424,3 @@ ENTRY(async_page_fault)
END(async_page_fault) END(async_page_fault)
#endif #endif
/*
* End of kprobes section
*/
.popsection
arch/x86/kernel/entry_64.S
View file @ 10b02564
...@@ -487,8 +487,6 @@ ENDPROC(native_usergs_sysret64) ...@@ -487,8 +487,6 @@ ENDPROC(native_usergs_sysret64)
TRACE_IRQS_OFF TRACE_IRQS_OFF
.endm .endm
/* save complete stack frame */
.pushsection .kprobes.text, "ax"
ENTRY(save_paranoid) ENTRY(save_paranoid)
XCPT_FRAME 1 RDI+8 XCPT_FRAME 1 RDI+8
cld cld
...@@ -517,7 +515,6 @@ ENTRY(save_paranoid) ...@@ -517,7 +515,6 @@ ENTRY(save_paranoid)
1: ret 1: ret
CFI_ENDPROC CFI_ENDPROC
END(save_paranoid) END(save_paranoid)
.popsection
/* /*
* A newly forked process directly context switches into this address. * A newly forked process directly context switches into this address.
...@@ -975,10 +972,6 @@ END(interrupt) ...@@ -975,10 +972,6 @@ END(interrupt)
call \func call \func
.endm .endm
/*
* Interrupt entry/exit should be protected against kprobes
*/
.pushsection .kprobes.text, "ax"
/* /*
* The interrupt stubs push (~vector+0x80) onto the stack and * The interrupt stubs push (~vector+0x80) onto the stack and
* then jump to common_interrupt. * then jump to common_interrupt.
...@@ -1113,10 +1106,6 @@ ENTRY(retint_kernel) ...@@ -1113,10 +1106,6 @@ ENTRY(retint_kernel)
CFI_ENDPROC CFI_ENDPROC
END(common_interrupt) END(common_interrupt)
/*
* End of kprobes section
*/
.popsection
/* /*
* APIC interrupts. * APIC interrupts.
...@@ -1477,11 +1466,6 @@ apicinterrupt3 HYPERVISOR_CALLBACK_VECTOR \ ...@@ -1477,11 +1466,6 @@ apicinterrupt3 HYPERVISOR_CALLBACK_VECTOR \
hyperv_callback_vector hyperv_vector_handler hyperv_callback_vector hyperv_vector_handler
#endif /* CONFIG_HYPERV */ #endif /* CONFIG_HYPERV */
/*
* Some functions should be protected against kprobes
*/
.pushsection .kprobes.text, "ax"
paranoidzeroentry_ist debug do_debug DEBUG_STACK paranoidzeroentry_ist debug do_debug DEBUG_STACK
paranoidzeroentry_ist int3 do_int3 DEBUG_STACK paranoidzeroentry_ist int3 do_int3 DEBUG_STACK
paranoiderrorentry stack_segment do_stack_segment paranoiderrorentry stack_segment do_stack_segment
...@@ -1898,7 +1882,3 @@ ENTRY(ignore_sysret) ...@@ -1898,7 +1882,3 @@ ENTRY(ignore_sysret)
CFI_ENDPROC CFI_ENDPROC
END(ignore_sysret) END(ignore_sysret)
/*
* End of kprobes section
*/
.popsection
arch/x86/kernel/hw_breakpoint.c
View file @ 10b02564
...@@ -32,7 +32,6 @@ ...@@ -32,7 +32,6 @@
#include <linux/irqflags.h> #include <linux/irqflags.h>
#include <linux/notifier.h> #include <linux/notifier.h>
#include <linux/kallsyms.h> #include <linux/kallsyms.h>
#include <linux/kprobes.h>
#include <linux/percpu.h> #include <linux/percpu.h>
#include <linux/kdebug.h> #include <linux/kdebug.h>
#include <linux/kernel.h> #include <linux/kernel.h>
...@@ -424,7 +423,7 @@ EXPORT_SYMBOL_GPL(hw_breakpoint_restore); ...@@ -424,7 +423,7 @@ EXPORT_SYMBOL_GPL(hw_breakpoint_restore);
* NOTIFY_STOP returned for all other cases * NOTIFY_STOP returned for all other cases
* *
*/ */
static int __kprobes hw_breakpoint_handler(struct die_args *args) static int hw_breakpoint_handler(struct die_args *args)
{ {
int i, cpu, rc = NOTIFY_STOP; int i, cpu, rc = NOTIFY_STOP;
struct perf_event *bp; struct perf_event *bp;
...@@ -511,7 +510,7 @@ static int __kprobes hw_breakpoint_handler(struct die_args *args) ...@@ -511,7 +510,7 @@ static int __kprobes hw_breakpoint_handler(struct die_args *args)
/* /*
* Handle debug exception notifications. * Handle debug exception notifications.
*/ */
int __kprobes hw_breakpoint_exceptions_notify( int hw_breakpoint_exceptions_notify(
struct notifier_block *unused, unsigned long val, void *data) struct notifier_block *unused, unsigned long val, void *data)
{ {
if (val != DIE_DEBUG) if (val != DIE_DEBUG)
......
arch/x86/kernel/kprobes/core.c
View file @ 10b02564
This diff is collapsed.
arch/x86/kernel/kprobes/ftrace.c
View file @ 10b02564
...@@ -25,8 +25,9 @@ ...@@ -25,8 +25,9 @@
#include "common.h" #include "common.h"
static int __skip_singlestep(struct kprobe *p, struct pt_regs *regs, static nokprobe_inline
struct kprobe_ctlblk *kcb) int __skip_singlestep(struct kprobe *p, struct pt_regs *regs,
struct kprobe_ctlblk *kcb)
{ {
/* /*
* Emulate singlestep (and also recover regs->ip) * Emulate singlestep (and also recover regs->ip)
...@@ -41,18 +42,19 @@ static int __skip_singlestep(struct kprobe *p, struct pt_regs *regs, ...@@ -41,18 +42,19 @@ static int __skip_singlestep(struct kprobe *p, struct pt_regs *regs,
return 1; return 1;
} }
int __kprobes skip_singlestep(struct kprobe *p, struct pt_regs *regs, int skip_singlestep(struct kprobe *p, struct pt_regs *regs,
struct kprobe_ctlblk *kcb) struct kprobe_ctlblk *kcb)
{ {
if (kprobe_ftrace(p)) if (kprobe_ftrace(p))
return __skip_singlestep(p, regs, kcb); return __skip_singlestep(p, regs, kcb);
else else
return 0; return 0;
} }
NOKPROBE_SYMBOL(skip_singlestep);
/* Ftrace callback handler for kprobes */ /* Ftrace callback handler for kprobes */
void __kprobes kprobe_ftrace_handler(unsigned long ip, unsigned long parent_ip, void kprobe_ftrace_handler(unsigned long ip, unsigned long parent_ip,
struct ftrace_ops *ops, struct pt_regs *regs) struct ftrace_ops *ops, struct pt_regs *regs)
{ {
struct kprobe *p; struct kprobe *p;
struct kprobe_ctlblk *kcb; struct kprobe_ctlblk *kcb;
...@@ -84,8 +86,9 @@ void __kprobes kprobe_ftrace_handler(unsigned long ip, unsigned long parent_ip, ...@@ -84,8 +86,9 @@ void __kprobes kprobe_ftrace_handler(unsigned long ip, unsigned long parent_ip,
end: end:
local_irq_restore(flags); local_irq_restore(flags);
} }
NOKPROBE_SYMBOL(kprobe_ftrace_handler);
int __kprobes arch_prepare_kprobe_ftrace(struct kprobe *p) int arch_prepare_kprobe_ftrace(struct kprobe *p)
{ {
p->ainsn.insn = NULL; p->ainsn.insn = NULL;
p->ainsn.boostable = -1; p->ainsn.boostable = -1;
......
arch/x86/kernel/kprobes/opt.c
View file @ 10b02564
...@@ -77,7 +77,7 @@ unsigned long __recover_optprobed_insn(kprobe_opcode_t *buf, unsigned long addr) ...@@ -77,7 +77,7 @@ unsigned long __recover_optprobed_insn(kprobe_opcode_t *buf, unsigned long addr)
} }
/* Insert a move instruction which sets a pointer to eax/rdi (1st arg). */ /* Insert a move instruction which sets a pointer to eax/rdi (1st arg). */
static void __kprobes synthesize_set_arg1(kprobe_opcode_t *addr, unsigned long val) static void synthesize_set_arg1(kprobe_opcode_t *addr, unsigned long val)
{ {
#ifdef CONFIG_X86_64 #ifdef CONFIG_X86_64
*addr++ = 0x48; *addr++ = 0x48;
...@@ -138,7 +138,8 @@ asm ( ...@@ -138,7 +138,8 @@ asm (
#define INT3_SIZE sizeof(kprobe_opcode_t) #define INT3_SIZE sizeof(kprobe_opcode_t)
/* Optimized kprobe call back function: called from optinsn */ /* Optimized kprobe call back function: called from optinsn */
static void __kprobes optimized_callback(struct optimized_kprobe *op, struct pt_regs *regs) static void
optimized_callback(struct optimized_kprobe *op, struct pt_regs *regs)
{ {
struct kprobe_ctlblk *kcb = get_kprobe_ctlblk(); struct kprobe_ctlblk *kcb = get_kprobe_ctlblk();
unsigned long flags; unsigned long flags;
...@@ -168,8 +169,9 @@ static void __kprobes optimized_callback(struct optimized_kprobe *op, struct pt_ ...@@ -168,8 +169,9 @@ static void __kprobes optimized_callback(struct optimized_kprobe *op, struct pt_
} }
local_irq_restore(flags); local_irq_restore(flags);
} }
NOKPROBE_SYMBOL(optimized_callback);
static int __kprobes copy_optimized_instructions(u8 *dest, u8 *src) static int copy_optimized_instructions(u8 *dest, u8 *src)
{ {
int len = 0, ret; int len = 0, ret;
...@@ -189,7 +191,7 @@ static int __kprobes copy_optimized_instructions(u8 *dest, u8 *src) ...@@ -189,7 +191,7 @@ static int __kprobes copy_optimized_instructions(u8 *dest, u8 *src)
} }
/* Check whether insn is indirect jump */ /* Check whether insn is indirect jump */
static int __kprobes insn_is_indirect_jump(struct insn *insn) static int insn_is_indirect_jump(struct insn *insn)
{ {
return ((insn->opcode.bytes[0] == 0xff && return ((insn->opcode.bytes[0] == 0xff &&
(X86_MODRM_REG(insn->modrm.value) & 6) == 4) || /* Jump */ (X86_MODRM_REG(insn->modrm.value) & 6) == 4) || /* Jump */
...@@ -224,7 +226,7 @@ static int insn_jump_into_range(struct insn *insn, unsigned long start, int len) ...@@ -224,7 +226,7 @@ static int insn_jump_into_range(struct insn *insn, unsigned long start, int len)
} }
/* Decode whole function to ensure any instructions don't jump into target */ /* Decode whole function to ensure any instructions don't jump into target */
static int __kprobes can_optimize(unsigned long paddr) static int can_optimize(unsigned long paddr)
{ {
unsigned long addr, size = 0, offset = 0; unsigned long addr, size = 0, offset = 0;
struct insn insn; struct insn insn;
...@@ -275,7 +277,7 @@ static int __kprobes can_optimize(unsigned long paddr) ...@@ -275,7 +277,7 @@ static int __kprobes can_optimize(unsigned long paddr)
} }
/* Check optimized_kprobe can actually be optimized. */ /* Check optimized_kprobe can actually be optimized. */
int __kprobes arch_check_optimized_kprobe(struct optimized_kprobe *op) int arch_check_optimized_kprobe(struct optimized_kprobe *op)
{ {
int i; int i;
struct kprobe *p; struct kprobe *p;
...@@ -290,15 +292,15 @@ int __kprobes arch_check_optimized_kprobe(struct optimized_kprobe *op) ...@@ -290,15 +292,15 @@ int __kprobes arch_check_optimized_kprobe(struct optimized_kprobe *op)
} }
/* Check the addr is within the optimized instructions. */ /* Check the addr is within the optimized instructions. */
int __kprobes int arch_within_optimized_kprobe(struct optimized_kprobe *op,
arch_within_optimized_kprobe(struct optimized_kprobe *op, unsigned long addr) unsigned long addr)
{ {
return ((unsigned long)op->kp.addr <= addr && return ((unsigned long)op->kp.addr <= addr &&
(unsigned long)op->kp.addr + op->optinsn.size > addr); (unsigned long)op->kp.addr + op->optinsn.size > addr);
} }
/* Free optimized instruction slot */ /* Free optimized instruction slot */
static __kprobes static
void __arch_remove_optimized_kprobe(struct optimized_kprobe *op, int dirty) void __arch_remove_optimized_kprobe(struct optimized_kprobe *op, int dirty)
{ {
if (op->optinsn.insn) { if (op->optinsn.insn) {
...@@ -308,7 +310,7 @@ void __arch_remove_optimized_kprobe(struct optimized_kprobe *op, int dirty) ...@@ -308,7 +310,7 @@ void __arch_remove_optimized_kprobe(struct optimized_kprobe *op, int dirty)
} }
} }
void __kprobes arch_remove_optimized_kprobe(struct optimized_kprobe *op) void arch_remove_optimized_kprobe(struct optimized_kprobe *op)
{ {
__arch_remove_optimized_kprobe(op, 1); __arch_remove_optimized_kprobe(op, 1);
} }
...@@ -318,7 +320,7 @@ void __kprobes arch_remove_optimized_kprobe(struct optimized_kprobe *op) ...@@ -318,7 +320,7 @@ void __kprobes arch_remove_optimized_kprobe(struct optimized_kprobe *op)
* Target instructions MUST be relocatable (checked inside) * Target instructions MUST be relocatable (checked inside)
* This is called when new aggr(opt)probe is allocated or reused. * This is called when new aggr(opt)probe is allocated or reused.
*/ */
int __kprobes arch_prepare_optimized_kprobe(struct optimized_kprobe *op) int arch_prepare_optimized_kprobe(struct optimized_kprobe *op)
{ {
u8 *buf; u8 *buf;
int ret; int ret;
...@@ -372,7 +374,7 @@ int __kprobes arch_prepare_optimized_kprobe(struct optimized_kprobe *op) ...@@ -372,7 +374,7 @@ int __kprobes arch_prepare_optimized_kprobe(struct optimized_kprobe *op)
* Replace breakpoints (int3) with relative jumps. * Replace breakpoints (int3) with relative jumps.
* Caller must call with locking kprobe_mutex and text_mutex. * Caller must call with locking kprobe_mutex and text_mutex.
*/ */
void __kprobes arch_optimize_kprobes(struct list_head *oplist) void arch_optimize_kprobes(struct list_head *oplist)
{ {
struct optimized_kprobe *op, *tmp; struct optimized_kprobe *op, *tmp;
u8 insn_buf[RELATIVEJUMP_SIZE]; u8 insn_buf[RELATIVEJUMP_SIZE];
...@@ -398,7 +400,7 @@ void __kprobes arch_optimize_kprobes(struct list_head *oplist) ...@@ -398,7 +400,7 @@ void __kprobes arch_optimize_kprobes(struct list_head *oplist)
} }
/* Replace a relative jump with a breakpoint (int3). */ /* Replace a relative jump with a breakpoint (int3). */
void __kprobes arch_unoptimize_kprobe(struct optimized_kprobe *op) void arch_unoptimize_kprobe(struct optimized_kprobe *op)
{ {
u8 insn_buf[RELATIVEJUMP_SIZE]; u8 insn_buf[RELATIVEJUMP_SIZE];
...@@ -424,8 +426,7 @@ extern void arch_unoptimize_kprobes(struct list_head *oplist, ...@@ -424,8 +426,7 @@ extern void arch_unoptimize_kprobes(struct list_head *oplist,
} }
} }
int __kprobes int setup_detour_execution(struct kprobe *p, struct pt_regs *regs, int reenter)
setup_detour_execution(struct kprobe *p, struct pt_regs *regs, int reenter)
{ {
struct optimized_kprobe *op; struct optimized_kprobe *op;
...@@ -441,3 +442,4 @@ setup_detour_execution(struct kprobe *p, struct pt_regs *regs, int reenter) ...@@ -441,3 +442,4 @@ setup_detour_execution(struct kprobe *p, struct pt_regs *regs, int reenter)
} }
return 0; return 0;
} }
NOKPROBE_SYMBOL(setup_detour_execution);
arch/x86/kernel/kvm.c
View file @ 10b02564
...@@ -251,8 +251,9 @@ u32 kvm_read_and_reset_pf_reason(void) ...@@ -251,8 +251,9 @@ u32 kvm_read_and_reset_pf_reason(void)
return reason; return reason;
} }
EXPORT_SYMBOL_GPL(kvm_read_and_reset_pf_reason); EXPORT_SYMBOL_GPL(kvm_read_and_reset_pf_reason);
NOKPROBE_SYMBOL(kvm_read_and_reset_pf_reason);
dotraplinkage void __kprobes dotraplinkage void
do_async_page_fault(struct pt_regs *regs, unsigned long error_code) do_async_page_fault(struct pt_regs *regs, unsigned long error_code)
{ {
enum ctx_state prev_state; enum ctx_state prev_state;
...@@ -276,6 +277,7 @@ do_async_page_fault(struct pt_regs *regs, unsigned long error_code) ...@@ -276,6 +277,7 @@ do_async_page_fault(struct pt_regs *regs, unsigned long error_code)
break; break;
} }
} }
NOKPROBE_SYMBOL(do_async_page_fault);
static void __init paravirt_ops_setup(void) static void __init paravirt_ops_setup(void)
{ {
......
arch/x86/kernel/nmi.c
View file @ 10b02564
...@@ -110,7 +110,7 @@ static void nmi_max_handler(struct irq_work *w) ...@@ -110,7 +110,7 @@ static void nmi_max_handler(struct irq_work *w)
a->handler, whole_msecs, decimal_msecs); a->handler, whole_msecs, decimal_msecs);
} }
static int __kprobes nmi_handle(unsigned int type, struct pt_regs *regs, bool b2b) static int nmi_handle(unsigned int type, struct pt_regs *regs, bool b2b)
{ {
struct nmi_desc *desc = nmi_to_desc(type); struct nmi_desc *desc = nmi_to_desc(type);
struct nmiaction *a; struct nmiaction *a;
...@@ -146,6 +146,7 @@ static int __kprobes nmi_handle(unsigned int type, struct pt_regs *regs, bool b2 ...@@ -146,6 +146,7 @@ static int __kprobes nmi_handle(unsigned int type, struct pt_regs *regs, bool b2
/* return total number of NMI events handled */ /* return total number of NMI events handled */
return handled; return handled;
} }
NOKPROBE_SYMBOL(nmi_handle);
int __register_nmi_handler(unsigned int type, struct nmiaction *action) int __register_nmi_handler(unsigned int type, struct nmiaction *action)
{ {
...@@ -208,7 +209,7 @@ void unregister_nmi_handler(unsigned int type, const char *name) ...@@ -208,7 +209,7 @@ void unregister_nmi_handler(unsigned int type, const char *name)
} }
EXPORT_SYMBOL_GPL(unregister_nmi_handler); EXPORT_SYMBOL_GPL(unregister_nmi_handler);
static __kprobes void static void
pci_serr_error(unsigned char reason, struct pt_regs *regs) pci_serr_error(unsigned char reason, struct pt_regs *regs)
{ {
/* check to see if anyone registered against these types of errors */ /* check to see if anyone registered against these types of errors */
...@@ -238,8 +239,9 @@ pci_serr_error(unsigned char reason, struct pt_regs *regs) ...@@ -238,8 +239,9 @@ pci_serr_error(unsigned char reason, struct pt_regs *regs)
reason = (reason & NMI_REASON_CLEAR_MASK) | NMI_REASON_CLEAR_SERR; reason = (reason & NMI_REASON_CLEAR_MASK) | NMI_REASON_CLEAR_SERR;
outb(reason, NMI_REASON_PORT); outb(reason, NMI_REASON_PORT);
} }
NOKPROBE_SYMBOL(pci_serr_error);
static __kprobes void static void
io_check_error(unsigned char reason, struct pt_regs *regs) io_check_error(unsigned char reason, struct pt_regs *regs)
{ {
unsigned long i; unsigned long i;
...@@ -269,8 +271,9 @@ io_check_error(unsigned char reason, struct pt_regs *regs) ...@@ -269,8 +271,9 @@ io_check_error(unsigned char reason, struct pt_regs *regs)
reason &= ~NMI_REASON_CLEAR_IOCHK; reason &= ~NMI_REASON_CLEAR_IOCHK;
outb(reason, NMI_REASON_PORT); outb(reason, NMI_REASON_PORT);
} }
NOKPROBE_SYMBOL(io_check_error);
static __kprobes void static void
unknown_nmi_error(unsigned char reason, struct pt_regs *regs) unknown_nmi_error(unsigned char reason, struct pt_regs *regs)
{ {
int handled; int handled;
...@@ -298,11 +301,12 @@ unknown_nmi_error(unsigned char reason, struct pt_regs *regs) ...@@ -298,11 +301,12 @@ unknown_nmi_error(unsigned char reason, struct pt_regs *regs)
pr_emerg("Dazed and confused, but trying to continue\n"); pr_emerg("Dazed and confused, but trying to continue\n");
} }
NOKPROBE_SYMBOL(unknown_nmi_error);
static DEFINE_PER_CPU(bool, swallow_nmi); static DEFINE_PER_CPU(bool, swallow_nmi);
static DEFINE_PER_CPU(unsigned long, last_nmi_rip); static DEFINE_PER_CPU(unsigned long, last_nmi_rip);
static __kprobes void default_do_nmi(struct pt_regs *regs) static void default_do_nmi(struct pt_regs *regs)
{ {
unsigned char reason = 0; unsigned char reason = 0;
int handled; int handled;
...@@ -401,6 +405,7 @@ static __kprobes void default_do_nmi(struct pt_regs *regs) ...@@ -401,6 +405,7 @@ static __kprobes void default_do_nmi(struct pt_regs *regs)
else else
unknown_nmi_error(reason, regs); unknown_nmi_error(reason, regs);
} }
NOKPROBE_SYMBOL(default_do_nmi);
/* /*
* NMIs can hit breakpoints which will cause it to lose its * NMIs can hit breakpoints which will cause it to lose its
...@@ -520,7 +525,7 @@ static inline void nmi_nesting_postprocess(void) ...@@ -520,7 +525,7 @@ static inline void nmi_nesting_postprocess(void)
} }
#endif #endif
dotraplinkage notrace __kprobes void dotraplinkage notrace void
do_nmi(struct pt_regs *regs, long error_code) do_nmi(struct pt_regs *regs, long error_code)
{ {
nmi_nesting_preprocess(regs); nmi_nesting_preprocess(regs);
...@@ -537,6 +542,7 @@ do_nmi(struct pt_regs *regs, long error_code) ...@@ -537,6 +542,7 @@ do_nmi(struct pt_regs *regs, long error_code)
/* On i386, may loop back to preprocess */ /* On i386, may loop back to preprocess */
nmi_nesting_postprocess(); nmi_nesting_postprocess();
} }
NOKPROBE_SYMBOL(do_nmi);
void stop_nmi(void) void stop_nmi(void)
{ {
......
arch/x86/kernel/paravirt.c
View file @ 10b02564
...@@ -23,6 +23,7 @@ ...@@ -23,6 +23,7 @@
#include <linux/efi.h> #include <linux/efi.h>
#include <linux/bcd.h> #include <linux/bcd.h>
#include <linux/highmem.h> #include <linux/highmem.h>
#include <linux/kprobes.h>
#include <asm/bug.h> #include <asm/bug.h>
#include <asm/paravirt.h> #include <asm/paravirt.h>
...@@ -389,6 +390,11 @@ __visible struct pv_cpu_ops pv_cpu_ops = { ...@@ -389,6 +390,11 @@ __visible struct pv_cpu_ops pv_cpu_ops = {
.end_context_switch = paravirt_nop, .end_context_switch = paravirt_nop,
}; };
/* At this point, native_get/set_debugreg has real function entries */
NOKPROBE_SYMBOL(native_get_debugreg);
NOKPROBE_SYMBOL(native_set_debugreg);
NOKPROBE_SYMBOL(native_load_idt);
struct pv_apic_ops pv_apic_ops = { struct pv_apic_ops pv_apic_ops = {
#ifdef CONFIG_X86_LOCAL_APIC #ifdef CONFIG_X86_LOCAL_APIC
.startup_ipi_hook = paravirt_nop, .startup_ipi_hook = paravirt_nop,
......
arch/x86/kernel/traps.c
View file @ 10b02564
...@@ -107,7 +107,7 @@ static inline void preempt_conditional_cli(struct pt_regs *regs) ...@@ -107,7 +107,7 @@ static inline void preempt_conditional_cli(struct pt_regs *regs)
preempt_count_dec(); preempt_count_dec();
} }
static int __kprobes static nokprobe_inline int
do_trap_no_signal(struct task_struct *tsk, int trapnr, char *str, do_trap_no_signal(struct task_struct *tsk, int trapnr, char *str,
struct pt_regs *regs, long error_code) struct pt_regs *regs, long error_code)
{ {
...@@ -168,7 +168,7 @@ static siginfo_t *fill_trap_info(struct pt_regs *regs, int signr, int trapnr, ...@@ -168,7 +168,7 @@ static siginfo_t *fill_trap_info(struct pt_regs *regs, int signr, int trapnr,
return info; return info;
} }
static void __kprobes static void
do_trap(int trapnr, int signr, char *str, struct pt_regs *regs, do_trap(int trapnr, int signr, char *str, struct pt_regs *regs,
long error_code, siginfo_t *info) long error_code, siginfo_t *info)
{ {
...@@ -202,6 +202,7 @@ do_trap(int trapnr, int signr, char *str, struct pt_regs *regs, ...@@ -202,6 +202,7 @@ do_trap(int trapnr, int signr, char *str, struct pt_regs *regs,
force_sig_info(signr, info ?: SEND_SIG_PRIV, tsk); force_sig_info(signr, info ?: SEND_SIG_PRIV, tsk);
} }
NOKPROBE_SYMBOL(do_trap);
static void do_error_trap(struct pt_regs *regs, long error_code, char *str, static void do_error_trap(struct pt_regs *regs, long error_code, char *str,
unsigned long trapnr, int signr) unsigned long trapnr, int signr)
...@@ -277,7 +278,7 @@ dotraplinkage void do_double_fault(struct pt_regs *regs, long error_code) ...@@ -277,7 +278,7 @@ dotraplinkage void do_double_fault(struct pt_regs *regs, long error_code)
} }
#endif #endif
dotraplinkage void __kprobes dotraplinkage void
do_general_protection(struct pt_regs *regs, long error_code) do_general_protection(struct pt_regs *regs, long error_code)
{ {
struct task_struct *tsk; struct task_struct *tsk;
...@@ -323,9 +324,10 @@ do_general_protection(struct pt_regs *regs, long error_code) ...@@ -323,9 +324,10 @@ do_general_protection(struct pt_regs *regs, long error_code)
exit: exit:
exception_exit(prev_state); exception_exit(prev_state);
} }
NOKPROBE_SYMBOL(do_general_protection);
/* May run on IST stack. */ /* May run on IST stack. */
dotraplinkage void __kprobes notrace do_int3(struct pt_regs *regs, long error_code) dotraplinkage void notrace do_int3(struct pt_regs *regs, long error_code)
{ {
enum ctx_state prev_state; enum ctx_state prev_state;
...@@ -341,13 +343,18 @@ dotraplinkage void __kprobes notrace do_int3(struct pt_regs *regs, long error_co ...@@ -341,13 +343,18 @@ dotraplinkage void __kprobes notrace do_int3(struct pt_regs *regs, long error_co
if (poke_int3_handler(regs)) if (poke_int3_handler(regs))
return; return;
prev_state = exception_enter();
#ifdef CONFIG_KGDB_LOW_LEVEL_TRAP #ifdef CONFIG_KGDB_LOW_LEVEL_TRAP
if (kgdb_ll_trap(DIE_INT3, "int3", regs, error_code, X86_TRAP_BP, if (kgdb_ll_trap(DIE_INT3, "int3", regs, error_code, X86_TRAP_BP,
SIGTRAP) == NOTIFY_STOP) SIGTRAP) == NOTIFY_STOP)
goto exit; goto exit;
#endif /* CONFIG_KGDB_LOW_LEVEL_TRAP */ #endif /* CONFIG_KGDB_LOW_LEVEL_TRAP */
#ifdef CONFIG_KPROBES
if (kprobe_int3_handler(regs))
return;
#endif
prev_state = exception_enter();
if (notify_die(DIE_INT3, "int3", regs, error_code, X86_TRAP_BP, if (notify_die(DIE_INT3, "int3", regs, error_code, X86_TRAP_BP,
SIGTRAP) == NOTIFY_STOP) SIGTRAP) == NOTIFY_STOP)
goto exit; goto exit;
...@@ -364,6 +371,7 @@ dotraplinkage void __kprobes notrace do_int3(struct pt_regs *regs, long error_co ...@@ -364,6 +371,7 @@ dotraplinkage void __kprobes notrace do_int3(struct pt_regs *regs, long error_co
exit: exit:
exception_exit(prev_state); exception_exit(prev_state);
} }
NOKPROBE_SYMBOL(do_int3);
#ifdef CONFIG_X86_64 #ifdef CONFIG_X86_64
/* /*
...@@ -371,7 +379,7 @@ dotraplinkage void __kprobes notrace do_int3(struct pt_regs *regs, long error_co ...@@ -371,7 +379,7 @@ dotraplinkage void __kprobes notrace do_int3(struct pt_regs *regs, long error_co
* for scheduling or signal handling. The actual stack switch is done in * for scheduling or signal handling. The actual stack switch is done in
* entry.S * entry.S
*/ */
asmlinkage __kprobes struct pt_regs *sync_regs(struct pt_regs *eregs) asmlinkage struct pt_regs *sync_regs(struct pt_regs *eregs)
{ {
struct pt_regs *regs = eregs; struct pt_regs *regs = eregs;
/* Did already sync */ /* Did already sync */
...@@ -390,6 +398,7 @@ asmlinkage __kprobes struct pt_regs *sync_regs(struct pt_regs *eregs) ...@@ -390,6 +398,7 @@ asmlinkage __kprobes struct pt_regs *sync_regs(struct pt_regs *eregs)
*regs = *eregs; *regs = *eregs;
return regs; return regs;
} }
NOKPROBE_SYMBOL(sync_regs);
#endif #endif
/* /*
...@@ -416,7 +425,7 @@ asmlinkage __kprobes struct pt_regs *sync_regs(struct pt_regs *eregs) ...@@ -416,7 +425,7 @@ asmlinkage __kprobes struct pt_regs *sync_regs(struct pt_regs *eregs)
* *
* May run on IST stack. * May run on IST stack.
*/ */
dotraplinkage void __kprobes do_debug(struct pt_regs *regs, long error_code) dotraplinkage void do_debug(struct pt_regs *regs, long error_code)
{ {
struct task_struct *tsk = current; struct task_struct *tsk = current;
enum ctx_state prev_state; enum ctx_state prev_state;
...@@ -424,8 +433,6 @@ dotraplinkage void __kprobes do_debug(struct pt_regs *regs, long error_code) ...@@ -424,8 +433,6 @@ dotraplinkage void __kprobes do_debug(struct pt_regs *regs, long error_code)
unsigned long dr6; unsigned long dr6;
int si_code; int si_code;
prev_state = exception_enter();
get_debugreg(dr6, 6); get_debugreg(dr6, 6);
/* Filter out all the reserved bits which are preset to 1 */ /* Filter out all the reserved bits which are preset to 1 */
...@@ -454,6 +461,12 @@ dotraplinkage void __kprobes do_debug(struct pt_regs *regs, long error_code) ...@@ -454,6 +461,12 @@ dotraplinkage void __kprobes do_debug(struct pt_regs *regs, long error_code)
/* Store the virtualized DR6 value */ /* Store the virtualized DR6 value */
tsk->thread.debugreg6 = dr6; tsk->thread.debugreg6 = dr6;
#ifdef CONFIG_KPROBES
if (kprobe_debug_handler(regs))
goto exit;
#endif
prev_state = exception_enter();
if (notify_die(DIE_DEBUG, "debug", regs, (long)&dr6, error_code, if (notify_die(DIE_DEBUG, "debug", regs, (long)&dr6, error_code,
SIGTRAP) == NOTIFY_STOP) SIGTRAP) == NOTIFY_STOP)
goto exit; goto exit;
...@@ -496,6 +509,7 @@ dotraplinkage void __kprobes do_debug(struct pt_regs *regs, long error_code) ...@@ -496,6 +509,7 @@ dotraplinkage void __kprobes do_debug(struct pt_regs *regs, long error_code)
exit: exit:
exception_exit(prev_state); exception_exit(prev_state);
} }
NOKPROBE_SYMBOL(do_debug);
/* /*
* Note that we play around with the 'TS' bit in an attempt to get * Note that we play around with the 'TS' bit in an attempt to get
...@@ -667,7 +681,7 @@ void math_state_restore(void) ...@@ -667,7 +681,7 @@ void math_state_restore(void)
} }
EXPORT_SYMBOL_GPL(math_state_restore); EXPORT_SYMBOL_GPL(math_state_restore);
dotraplinkage void __kprobes dotraplinkage void
do_device_not_available(struct pt_regs *regs, long error_code) do_device_not_available(struct pt_regs *regs, long error_code)
{ {
enum ctx_state prev_state; enum ctx_state prev_state;
...@@ -693,6 +707,7 @@ do_device_not_available(struct pt_regs *regs, long error_code) ...@@ -693,6 +707,7 @@ do_device_not_available(struct pt_regs *regs, long error_code)
#endif #endif
exception_exit(prev_state); exception_exit(prev_state);
} }
NOKPROBE_SYMBOL(do_device_not_available);
#ifdef CONFIG_X86_32 #ifdef CONFIG_X86_32
dotraplinkage void do_iret_error(struct pt_regs *regs, long error_code) dotraplinkage void do_iret_error(struct pt_regs *regs, long error_code)
......
arch/x86/lib/thunk_32.S
View file @ 10b02564
...@@ -4,8 +4,8 @@ ...@@ -4,8 +4,8 @@
* (inspired by Andi Kleen's thunk_64.S) * (inspired by Andi Kleen's thunk_64.S)
* Subject to the GNU public license, v.2. No warranty of any kind. * Subject to the GNU public license, v.2. No warranty of any kind.
*/ */
#include <linux/linkage.h> #include <linux/linkage.h>
#include <asm/asm.h>
#ifdef CONFIG_TRACE_IRQFLAGS #ifdef CONFIG_TRACE_IRQFLAGS
/* put return address in eax (arg1) */ /* put return address in eax (arg1) */
...@@ -22,6 +22,7 @@ ...@@ -22,6 +22,7 @@
popl %ecx popl %ecx
popl %eax popl %eax
ret ret
_ASM_NOKPROBE(\name)
.endm .endm
thunk_ra trace_hardirqs_on_thunk,trace_hardirqs_on_caller thunk_ra trace_hardirqs_on_thunk,trace_hardirqs_on_caller
......
arch/x86/lib/thunk_64.S
View file @ 10b02564
...@@ -8,6 +8,7 @@ ...@@ -8,6 +8,7 @@
#include <linux/linkage.h> #include <linux/linkage.h>
#include <asm/dwarf2.h> #include <asm/dwarf2.h>
#include <asm/calling.h> #include <asm/calling.h>
#include <asm/asm.h>
/* rdi: arg1 ... normal C conventions. rax is saved/restored. */ /* rdi: arg1 ... normal C conventions. rax is saved/restored. */
.macro THUNK name, func, put_ret_addr_in_rdi=0 .macro THUNK name, func, put_ret_addr_in_rdi=0
...@@ -25,6 +26,7 @@ ...@@ -25,6 +26,7 @@
call \func call \func
jmp restore jmp restore
CFI_ENDPROC CFI_ENDPROC
_ASM_NOKPROBE(\name)
.endm .endm
#ifdef CONFIG_TRACE_IRQFLAGS #ifdef CONFIG_TRACE_IRQFLAGS
...@@ -43,3 +45,4 @@ restore: ...@@ -43,3 +45,4 @@ restore:
RESTORE_ARGS RESTORE_ARGS
ret ret
CFI_ENDPROC CFI_ENDPROC
_ASM_NOKPROBE(restore)
arch/x86/mm/fault.c
View file @ 10b02564
...@@ -8,7 +8,7 @@ ...@@ -8,7 +8,7 @@
#include <linux/kdebug.h> /* oops_begin/end, ... */ #include <linux/kdebug.h> /* oops_begin/end, ... */
#include <linux/module.h> /* search_exception_table */ #include <linux/module.h> /* search_exception_table */
#include <linux/bootmem.h> /* max_low_pfn */ #include <linux/bootmem.h> /* max_low_pfn */
#include <linux/kprobes.h> /* __kprobes, ... */ #include <linux/kprobes.h> /* NOKPROBE_SYMBOL, ... */
#include <linux/mmiotrace.h> /* kmmio_handler, ... */ #include <linux/mmiotrace.h> /* kmmio_handler, ... */
#include <linux/perf_event.h> /* perf_sw_event */ #include <linux/perf_event.h> /* perf_sw_event */
#include <linux/hugetlb.h> /* hstate_index_to_shift */ #include <linux/hugetlb.h> /* hstate_index_to_shift */
...@@ -45,7 +45,7 @@ enum x86_pf_error_code { ...@@ -45,7 +45,7 @@ enum x86_pf_error_code {
* Returns 0 if mmiotrace is disabled, or if the fault is not * Returns 0 if mmiotrace is disabled, or if the fault is not
* handled by mmiotrace: * handled by mmiotrace:
*/ */
static inline int __kprobes static nokprobe_inline int
kmmio_fault(struct pt_regs *regs, unsigned long addr) kmmio_fault(struct pt_regs *regs, unsigned long addr)
{ {
if (unlikely(is_kmmio_active())) if (unlikely(is_kmmio_active()))
...@@ -54,7 +54,7 @@ kmmio_fault(struct pt_regs *regs, unsigned long addr) ...@@ -54,7 +54,7 @@ kmmio_fault(struct pt_regs *regs, unsigned long addr)
return 0; return 0;
} }
static inline int __kprobes kprobes_fault(struct pt_regs *regs) static nokprobe_inline int kprobes_fault(struct pt_regs *regs)
{ {
int ret = 0; int ret = 0;
...@@ -261,7 +261,7 @@ void vmalloc_sync_all(void) ...@@ -261,7 +261,7 @@ void vmalloc_sync_all(void)
* *
* Handle a fault on the vmalloc or module mapping area * Handle a fault on the vmalloc or module mapping area
*/ */
static noinline __kprobes int vmalloc_fault(unsigned long address) static noinline int vmalloc_fault(unsigned long address)
{ {
unsigned long pgd_paddr; unsigned long pgd_paddr;
pmd_t *pmd_k; pmd_t *pmd_k;
...@@ -291,6 +291,7 @@ static noinline __kprobes int vmalloc_fault(unsigned long address) ...@@ -291,6 +291,7 @@ static noinline __kprobes int vmalloc_fault(unsigned long address)
return 0; return 0;
} }
NOKPROBE_SYMBOL(vmalloc_fault);
/* /*
* Did it hit the DOS screen memory VA from vm86 mode? * Did it hit the DOS screen memory VA from vm86 mode?
...@@ -358,7 +359,7 @@ void vmalloc_sync_all(void) ...@@ -358,7 +359,7 @@ void vmalloc_sync_all(void)
* *
* This assumes no large pages in there. * This assumes no large pages in there.
*/ */
static noinline __kprobes int vmalloc_fault(unsigned long address) static noinline int vmalloc_fault(unsigned long address)
{ {
pgd_t *pgd, *pgd_ref; pgd_t *pgd, *pgd_ref;
pud_t *pud, *pud_ref; pud_t *pud, *pud_ref;
...@@ -425,6 +426,7 @@ static noinline __kprobes int vmalloc_fault(unsigned long address) ...@@ -425,6 +426,7 @@ static noinline __kprobes int vmalloc_fault(unsigned long address)
return 0; return 0;
} }
NOKPROBE_SYMBOL(vmalloc_fault);
#ifdef CONFIG_CPU_SUP_AMD #ifdef CONFIG_CPU_SUP_AMD
static const char errata93_warning[] = static const char errata93_warning[] =
...@@ -927,7 +929,7 @@ static int spurious_fault_check(unsigned long error_code, pte_t *pte) ...@@ -927,7 +929,7 @@ static int spurious_fault_check(unsigned long error_code, pte_t *pte)
* There are no security implications to leaving a stale TLB when * There are no security implications to leaving a stale TLB when
* increasing the permissions on a page. * increasing the permissions on a page.
*/ */
static noinline __kprobes int static noinline int
spurious_fault(unsigned long error_code, unsigned long address) spurious_fault(unsigned long error_code, unsigned long address)
{ {
pgd_t *pgd; pgd_t *pgd;
...@@ -975,6 +977,7 @@ spurious_fault(unsigned long error_code, unsigned long address) ...@@ -975,6 +977,7 @@ spurious_fault(unsigned long error_code, unsigned long address)
return ret; return ret;
} }
NOKPROBE_SYMBOL(spurious_fault);
int show_unhandled_signals = 1; int show_unhandled_signals = 1;
...@@ -1030,7 +1033,7 @@ static inline bool smap_violation(int error_code, struct pt_regs *regs) ...@@ -1030,7 +1033,7 @@ static inline bool smap_violation(int error_code, struct pt_regs *regs)
* {,trace_}do_page_fault() have notrace on. Having this an actual function * {,trace_}do_page_fault() have notrace on. Having this an actual function
* guarantees there's a function trace entry. * guarantees there's a function trace entry.
*/ */
static void __kprobes noinline static noinline void
__do_page_fault(struct pt_regs *regs, unsigned long error_code, __do_page_fault(struct pt_regs *regs, unsigned long error_code,
unsigned long address) unsigned long address)
{ {
...@@ -1253,8 +1256,9 @@ __do_page_fault(struct pt_regs *regs, unsigned long error_code, ...@@ -1253,8 +1256,9 @@ __do_page_fault(struct pt_regs *regs, unsigned long error_code,
up_read(&mm->mmap_sem); up_read(&mm->mmap_sem);
} }
NOKPROBE_SYMBOL(__do_page_fault);
dotraplinkage void __kprobes notrace dotraplinkage void notrace
do_page_fault(struct pt_regs *regs, unsigned long error_code) do_page_fault(struct pt_regs *regs, unsigned long error_code)
{ {
unsigned long address = read_cr2(); /* Get the faulting address */ unsigned long address = read_cr2(); /* Get the faulting address */
...@@ -1272,10 +1276,12 @@ do_page_fault(struct pt_regs *regs, unsigned long error_code) ...@@ -1272,10 +1276,12 @@ do_page_fault(struct pt_regs *regs, unsigned long error_code)
__do_page_fault(regs, error_code, address); __do_page_fault(regs, error_code, address);
exception_exit(prev_state); exception_exit(prev_state);
} }
NOKPROBE_SYMBOL(do_page_fault);
#ifdef CONFIG_TRACING #ifdef CONFIG_TRACING
static void trace_page_fault_entries(unsigned long address, struct pt_regs *regs, static nokprobe_inline void
unsigned long error_code) trace_page_fault_entries(unsigned long address, struct pt_regs *regs,
unsigned long error_code)
{ {
if (user_mode(regs)) if (user_mode(regs))
trace_page_fault_user(address, regs, error_code); trace_page_fault_user(address, regs, error_code);
...@@ -1283,7 +1289,7 @@ static void trace_page_fault_entries(unsigned long address, struct pt_regs *regs ...@@ -1283,7 +1289,7 @@ static void trace_page_fault_entries(unsigned long address, struct pt_regs *regs
trace_page_fault_kernel(address, regs, error_code); trace_page_fault_kernel(address, regs, error_code);
} }
dotraplinkage void __kprobes notrace dotraplinkage void notrace
trace_do_page_fault(struct pt_regs *regs, unsigned long error_code) trace_do_page_fault(struct pt_regs *regs, unsigned long error_code)
{ {
/* /*
...@@ -1300,4 +1306,5 @@ trace_do_page_fault(struct pt_regs *regs, unsigned long error_code) ...@@ -1300,4 +1306,5 @@ trace_do_page_fault(struct pt_regs *regs, unsigned long error_code)
__do_page_fault(regs, error_code, address); __do_page_fault(regs, error_code, address);
exception_exit(prev_state); exception_exit(prev_state);
} }
NOKPROBE_SYMBOL(trace_do_page_fault);
#endif /* CONFIG_TRACING */ #endif /* CONFIG_TRACING */
include/asm-generic/vmlinux.lds.h
View file @ 10b02564
...@@ -109,6 +109,15 @@ ...@@ -109,6 +109,15 @@
#define BRANCH_PROFILE() #define BRANCH_PROFILE()
#endif #endif
#ifdef CONFIG_KPROBES
#define KPROBE_BLACKLIST() . = ALIGN(8); \
VMLINUX_SYMBOL(__start_kprobe_blacklist) = .; \
*(_kprobe_blacklist) \
VMLINUX_SYMBOL(__stop_kprobe_blacklist) = .;
#else
#define KPROBE_BLACKLIST()
#endif
#ifdef CONFIG_EVENT_TRACING #ifdef CONFIG_EVENT_TRACING
#define FTRACE_EVENTS() . = ALIGN(8); \ #define FTRACE_EVENTS() . = ALIGN(8); \
VMLINUX_SYMBOL(__start_ftrace_events) = .; \ VMLINUX_SYMBOL(__start_ftrace_events) = .; \
...@@ -507,6 +516,7 @@ ...@@ -507,6 +516,7 @@
*(.init.rodata) \ *(.init.rodata) \
FTRACE_EVENTS() \ FTRACE_EVENTS() \
TRACE_SYSCALLS() \ TRACE_SYSCALLS() \
KPROBE_BLACKLIST() \
MEM_DISCARD(init.rodata) \ MEM_DISCARD(init.rodata) \
CLK_OF_TABLES() \ CLK_OF_TABLES() \
RESERVEDMEM_OF_TABLES() \ RESERVEDMEM_OF_TABLES() \
......
include/linux/compiler.h
View file @ 10b02564
...@@ -374,7 +374,9 @@ void ftrace_likely_update(struct ftrace_branch_data *f, int val, int expect); ...@@ -374,7 +374,9 @@ void ftrace_likely_update(struct ftrace_branch_data *f, int val, int expect);
/* Ignore/forbid kprobes attach on very low level functions marked by this attribute: */ /* Ignore/forbid kprobes attach on very low level functions marked by this attribute: */
#ifdef CONFIG_KPROBES #ifdef CONFIG_KPROBES
# define __kprobes __attribute__((__section__(".kprobes.text"))) # define __kprobes __attribute__((__section__(".kprobes.text")))
# define nokprobe_inline __always_inline
#else #else
# define __kprobes # define __kprobes
# define nokprobe_inline inline
#endif #endif
#endif /* __LINUX_COMPILER_H */ #endif /* __LINUX_COMPILER_H */
include/linux/kprobes.h
View file @ 10b02564
...@@ -205,10 +205,10 @@ struct kretprobe_blackpoint { ...@@ -205,10 +205,10 @@ struct kretprobe_blackpoint {
void *addr; void *addr;
}; };
struct kprobe_blackpoint { struct kprobe_blacklist_entry {
const char *name; struct list_head list;
unsigned long start_addr; unsigned long start_addr;
unsigned long range; unsigned long end_addr;
}; };
#ifdef CONFIG_KPROBES #ifdef CONFIG_KPROBES
...@@ -265,6 +265,7 @@ extern void arch_disarm_kprobe(struct kprobe *p); ...@@ -265,6 +265,7 @@ extern void arch_disarm_kprobe(struct kprobe *p);
extern int arch_init_kprobes(void); extern int arch_init_kprobes(void);
extern void show_registers(struct pt_regs *regs); extern void show_registers(struct pt_regs *regs);
extern void kprobes_inc_nmissed_count(struct kprobe *p); extern void kprobes_inc_nmissed_count(struct kprobe *p);
extern bool arch_within_kprobe_blacklist(unsigned long addr);
struct kprobe_insn_cache { struct kprobe_insn_cache {
struct mutex mutex; struct mutex mutex;
...@@ -476,4 +477,18 @@ static inline int enable_jprobe(struct jprobe *jp) ...@@ -476,4 +477,18 @@ static inline int enable_jprobe(struct jprobe *jp)
return enable_kprobe(&jp->kp); return enable_kprobe(&jp->kp);
} }
#ifdef CONFIG_KPROBES
/*
* Blacklist ganerating macro. Specify functions which is not probed
* by using this macro.
*/
#define __NOKPROBE_SYMBOL(fname) \
static unsigned long __used \
__attribute__((section("_kprobe_blacklist"))) \
_kbl_addr_##fname = (unsigned long)fname;
#define NOKPROBE_SYMBOL(fname) __NOKPROBE_SYMBOL(fname)
#else
#define NOKPROBE_SYMBOL(fname)
#endif
#endif /* _LINUX_KPROBES_H */ #endif /* _LINUX_KPROBES_H */
kernel/kprobes.c
View file @ 10b02564
This diff is collapsed.
kernel/notifier.c
View file @ 10b02564
...@@ -71,9 +71,9 @@ static int notifier_chain_unregister(struct notifier_block **nl, ...@@ -71,9 +71,9 @@ static int notifier_chain_unregister(struct notifier_block **nl,
* @returns: notifier_call_chain returns the value returned by the * @returns: notifier_call_chain returns the value returned by the
* last notifier function called. * last notifier function called.
*/ */
static int __kprobes notifier_call_chain(struct notifier_block **nl, static int notifier_call_chain(struct notifier_block **nl,
unsigned long val, void *v, unsigned long val, void *v,
int nr_to_call, int *nr_calls) int nr_to_call, int *nr_calls)
{ {
int ret = NOTIFY_DONE; int ret = NOTIFY_DONE;
struct notifier_block *nb, *next_nb; struct notifier_block *nb, *next_nb;
...@@ -102,6 +102,7 @@ static int __kprobes notifier_call_chain(struct notifier_block **nl, ...@@ -102,6 +102,7 @@ static int __kprobes notifier_call_chain(struct notifier_block **nl,
} }
return ret; return ret;
} }
NOKPROBE_SYMBOL(notifier_call_chain);
/* /*
* Atomic notifier chain routines. Registration and unregistration * Atomic notifier chain routines. Registration and unregistration
...@@ -172,9 +173,9 @@ EXPORT_SYMBOL_GPL(atomic_notifier_chain_unregister); ...@@ -172,9 +173,9 @@ EXPORT_SYMBOL_GPL(atomic_notifier_chain_unregister);
* Otherwise the return value is the return value * Otherwise the return value is the return value
* of the last notifier function called. * of the last notifier function called.
*/ */
int __kprobes __atomic_notifier_call_chain(struct atomic_notifier_head *nh, int __atomic_notifier_call_chain(struct atomic_notifier_head *nh,
unsigned long val, void *v, unsigned long val, void *v,
int nr_to_call, int *nr_calls) int nr_to_call, int *nr_calls)
{ {
int ret; int ret;
...@@ -184,13 +185,15 @@ int __kprobes __atomic_notifier_call_chain(struct atomic_notifier_head *nh, ...@@ -184,13 +185,15 @@ int __kprobes __atomic_notifier_call_chain(struct atomic_notifier_head *nh,
return ret; return ret;
} }
EXPORT_SYMBOL_GPL(__atomic_notifier_call_chain); EXPORT_SYMBOL_GPL(__atomic_notifier_call_chain);
NOKPROBE_SYMBOL(__atomic_notifier_call_chain);
int __kprobes atomic_notifier_call_chain(struct atomic_notifier_head *nh, int atomic_notifier_call_chain(struct atomic_notifier_head *nh,
unsigned long val, void *v) unsigned long val, void *v)
{ {
return __atomic_notifier_call_chain(nh, val, v, -1, NULL); return __atomic_notifier_call_chain(nh, val, v, -1, NULL);
} }
EXPORT_SYMBOL_GPL(atomic_notifier_call_chain); EXPORT_SYMBOL_GPL(atomic_notifier_call_chain);
NOKPROBE_SYMBOL(atomic_notifier_call_chain);
/* /*
* Blocking notifier chain routines. All access to the chain is * Blocking notifier chain routines. All access to the chain is
...@@ -527,7 +530,7 @@ EXPORT_SYMBOL_GPL(srcu_init_notifier_head); ...@@ -527,7 +530,7 @@ EXPORT_SYMBOL_GPL(srcu_init_notifier_head);
static ATOMIC_NOTIFIER_HEAD(die_chain); static ATOMIC_NOTIFIER_HEAD(die_chain);
int notrace __kprobes notify_die(enum die_val val, const char *str, int notrace notify_die(enum die_val val, const char *str,
struct pt_regs *regs, long err, int trap, int sig) struct pt_regs *regs, long err, int trap, int sig)
{ {
struct die_args args = { struct die_args args = {
...@@ -540,6 +543,7 @@ int notrace __kprobes notify_die(enum die_val val, const char *str, ...@@ -540,6 +543,7 @@ int notrace __kprobes notify_die(enum die_val val, const char *str,
}; };
return atomic_notifier_call_chain(&die_chain, val, &args); return atomic_notifier_call_chain(&die_chain, val, &args);
} }
NOKPROBE_SYMBOL(notify_die);
int register_die_notifier(struct notifier_block *nb) int register_die_notifier(struct notifier_block *nb)
{ {
......
kernel/sched/core.c
View file @ 10b02564
...@@ -2480,7 +2480,7 @@ notrace unsigned long get_parent_ip(unsigned long addr) ...@@ -2480,7 +2480,7 @@ notrace unsigned long get_parent_ip(unsigned long addr)
#if defined(CONFIG_PREEMPT) && (defined(CONFIG_DEBUG_PREEMPT) || \ #if defined(CONFIG_PREEMPT) && (defined(CONFIG_DEBUG_PREEMPT) || \
defined(CONFIG_PREEMPT_TRACER)) defined(CONFIG_PREEMPT_TRACER))
void __kprobes preempt_count_add(int val) void preempt_count_add(int val)
{ {
#ifdef CONFIG_DEBUG_PREEMPT #ifdef CONFIG_DEBUG_PREEMPT
/* /*
...@@ -2506,8 +2506,9 @@ void __kprobes preempt_count_add(int val) ...@@ -2506,8 +2506,9 @@ void __kprobes preempt_count_add(int val)
} }
} }
EXPORT_SYMBOL(preempt_count_add); EXPORT_SYMBOL(preempt_count_add);
NOKPROBE_SYMBOL(preempt_count_add);
void __kprobes preempt_count_sub(int val) void preempt_count_sub(int val)
{ {
#ifdef CONFIG_DEBUG_PREEMPT #ifdef CONFIG_DEBUG_PREEMPT
/* /*
...@@ -2528,6 +2529,7 @@ void __kprobes preempt_count_sub(int val) ...@@ -2528,6 +2529,7 @@ void __kprobes preempt_count_sub(int val)
__preempt_count_sub(val); __preempt_count_sub(val);
} }
EXPORT_SYMBOL(preempt_count_sub); EXPORT_SYMBOL(preempt_count_sub);
NOKPROBE_SYMBOL(preempt_count_sub);
#endif #endif
...@@ -2804,6 +2806,7 @@ asmlinkage void __sched notrace preempt_schedule(void) ...@@ -2804,6 +2806,7 @@ asmlinkage void __sched notrace preempt_schedule(void)
barrier(); barrier();
} while (need_resched()); } while (need_resched());
} }
NOKPROBE_SYMBOL(preempt_schedule);
EXPORT_SYMBOL(preempt_schedule); EXPORT_SYMBOL(preempt_schedule);
#endif /* CONFIG_PREEMPT */ #endif /* CONFIG_PREEMPT */
......
kernel/trace/trace_event_perf.c
View file @ 10b02564
...@@ -248,8 +248,8 @@ void perf_trace_del(struct perf_event *p_event, int flags) ...@@ -248,8 +248,8 @@ void perf_trace_del(struct perf_event *p_event, int flags)
tp_event->class->reg(tp_event, TRACE_REG_PERF_DEL, p_event); tp_event->class->reg(tp_event, TRACE_REG_PERF_DEL, p_event);
} }
__kprobes void *perf_trace_buf_prepare(int size, unsigned short type, void *perf_trace_buf_prepare(int size, unsigned short type,
struct pt_regs *regs, int *rctxp) struct pt_regs *regs, int *rctxp)
{ {
struct trace_entry *entry; struct trace_entry *entry;
unsigned long flags; unsigned long flags;
...@@ -281,6 +281,7 @@ __kprobes void *perf_trace_buf_prepare(int size, unsigned short type, ...@@ -281,6 +281,7 @@ __kprobes void *perf_trace_buf_prepare(int size, unsigned short type,
return raw_data; return raw_data;
} }
EXPORT_SYMBOL_GPL(perf_trace_buf_prepare); EXPORT_SYMBOL_GPL(perf_trace_buf_prepare);
NOKPROBE_SYMBOL(perf_trace_buf_prepare);
#ifdef CONFIG_FUNCTION_TRACER #ifdef CONFIG_FUNCTION_TRACER
static void static void
......
kernel/trace/trace_kprobe.c
View file @ 10b02564
...@@ -40,27 +40,27 @@ struct trace_kprobe { ...@@ -40,27 +40,27 @@ struct trace_kprobe {
(sizeof(struct probe_arg) * (n))) (sizeof(struct probe_arg) * (n)))
static __kprobes bool trace_kprobe_is_return(struct trace_kprobe *tk) static nokprobe_inline bool trace_kprobe_is_return(struct trace_kprobe *tk)
{ {
return tk->rp.handler != NULL; return tk->rp.handler != NULL;
} }
static __kprobes const char *trace_kprobe_symbol(struct trace_kprobe *tk) static nokprobe_inline const char *trace_kprobe_symbol(struct trace_kprobe *tk)
{ {
return tk->symbol ? tk->symbol : "unknown"; return tk->symbol ? tk->symbol : "unknown";
} }
static __kprobes unsigned long trace_kprobe_offset(struct trace_kprobe *tk) static nokprobe_inline unsigned long trace_kprobe_offset(struct trace_kprobe *tk)
{ {
return tk->rp.kp.offset; return tk->rp.kp.offset;
} }
static __kprobes bool trace_kprobe_has_gone(struct trace_kprobe *tk) static nokprobe_inline bool trace_kprobe_has_gone(struct trace_kprobe *tk)
{ {
return !!(kprobe_gone(&tk->rp.kp)); return !!(kprobe_gone(&tk->rp.kp));
} }
static __kprobes bool trace_kprobe_within_module(struct trace_kprobe *tk, static nokprobe_inline bool trace_kprobe_within_module(struct trace_kprobe *tk,
struct module *mod) struct module *mod)
{ {
int len = strlen(mod->name); int len = strlen(mod->name);
...@@ -68,7 +68,7 @@ static __kprobes bool trace_kprobe_within_module(struct trace_kprobe *tk, ...@@ -68,7 +68,7 @@ static __kprobes bool trace_kprobe_within_module(struct trace_kprobe *tk,
return strncmp(mod->name, name, len) == 0 && name[len] == ':'; return strncmp(mod->name, name, len) == 0 && name[len] == ':';
} }
static __kprobes bool trace_kprobe_is_on_module(struct trace_kprobe *tk) static nokprobe_inline bool trace_kprobe_is_on_module(struct trace_kprobe *tk)
{ {
return !!strchr(trace_kprobe_symbol(tk), ':'); return !!strchr(trace_kprobe_symbol(tk), ':');
} }
...@@ -132,19 +132,21 @@ struct symbol_cache *alloc_symbol_cache(const char *sym, long offset) ...@@ -132,19 +132,21 @@ struct symbol_cache *alloc_symbol_cache(const char *sym, long offset)
* Kprobes-specific fetch functions * Kprobes-specific fetch functions
*/ */
#define DEFINE_FETCH_stack(type) \ #define DEFINE_FETCH_stack(type) \
static __kprobes void FETCH_FUNC_NAME(stack, type)(struct pt_regs *regs,\ static void FETCH_FUNC_NAME(stack, type)(struct pt_regs *regs, \
void *offset, void *dest) \ void *offset, void *dest) \
{ \ { \
*(type *)dest = (type)regs_get_kernel_stack_nth(regs, \ *(type *)dest = (type)regs_get_kernel_stack_nth(regs, \
(unsigned int)((unsigned long)offset)); \ (unsigned int)((unsigned long)offset)); \
} } \
NOKPROBE_SYMBOL(FETCH_FUNC_NAME(stack, type));
DEFINE_BASIC_FETCH_FUNCS(stack) DEFINE_BASIC_FETCH_FUNCS(stack)
/* No string on the stack entry */ /* No string on the stack entry */
#define fetch_stack_string NULL #define fetch_stack_string NULL
#define fetch_stack_string_size NULL #define fetch_stack_string_size NULL
#define DEFINE_FETCH_memory(type) \ #define DEFINE_FETCH_memory(type) \
static __kprobes void FETCH_FUNC_NAME(memory, type)(struct pt_regs *regs,\ static void FETCH_FUNC_NAME(memory, type)(struct pt_regs *regs, \
void *addr, void *dest) \ void *addr, void *dest) \
{ \ { \
type retval; \ type retval; \
...@@ -152,14 +154,16 @@ static __kprobes void FETCH_FUNC_NAME(memory, type)(struct pt_regs *regs,\ ...@@ -152,14 +154,16 @@ static __kprobes void FETCH_FUNC_NAME(memory, type)(struct pt_regs *regs,\
*(type *)dest = 0; \ *(type *)dest = 0; \
else \ else \
*(type *)dest = retval; \ *(type *)dest = retval; \
} } \
NOKPROBE_SYMBOL(FETCH_FUNC_NAME(memory, type));
DEFINE_BASIC_FETCH_FUNCS(memory) DEFINE_BASIC_FETCH_FUNCS(memory)
/* /*
* Fetch a null-terminated string. Caller MUST set *(u32 *)dest with max * Fetch a null-terminated string. Caller MUST set *(u32 *)dest with max
* length and relative data location. * length and relative data location.
*/ */
static __kprobes void FETCH_FUNC_NAME(memory, string)(struct pt_regs *regs, static void FETCH_FUNC_NAME(memory, string)(struct pt_regs *regs,
void *addr, void *dest) void *addr, void *dest)
{ {
long ret; long ret;
int maxlen = get_rloc_len(*(u32 *)dest); int maxlen = get_rloc_len(*(u32 *)dest);
...@@ -193,10 +197,11 @@ static __kprobes void FETCH_FUNC_NAME(memory, string)(struct pt_regs *regs, ...@@ -193,10 +197,11 @@ static __kprobes void FETCH_FUNC_NAME(memory, string)(struct pt_regs *regs,
get_rloc_offs(*(u32 *)dest)); get_rloc_offs(*(u32 *)dest));
} }
} }
NOKPROBE_SYMBOL(FETCH_FUNC_NAME(memory, string));
/* Return the length of string -- including null terminal byte */ /* Return the length of string -- including null terminal byte */
static __kprobes void FETCH_FUNC_NAME(memory, string_size)(struct pt_regs *regs, static void FETCH_FUNC_NAME(memory, string_size)(struct pt_regs *regs,
void *addr, void *dest) void *addr, void *dest)
{ {
mm_segment_t old_fs; mm_segment_t old_fs;
int ret, len = 0; int ret, len = 0;
...@@ -219,17 +224,19 @@ static __kprobes void FETCH_FUNC_NAME(memory, string_size)(struct pt_regs *regs, ...@@ -219,17 +224,19 @@ static __kprobes void FETCH_FUNC_NAME(memory, string_size)(struct pt_regs *regs,
else else
*(u32 *)dest = len; *(u32 *)dest = len;
} }
NOKPROBE_SYMBOL(FETCH_FUNC_NAME(memory, string_size));
#define DEFINE_FETCH_symbol(type) \ #define DEFINE_FETCH_symbol(type) \
__kprobes void FETCH_FUNC_NAME(symbol, type)(struct pt_regs *regs, \ void FETCH_FUNC_NAME(symbol, type)(struct pt_regs *regs, void *data, void *dest)\
void *data, void *dest) \
{ \ { \
struct symbol_cache *sc = data; \ struct symbol_cache *sc = data; \
if (sc->addr) \ if (sc->addr) \
fetch_memory_##type(regs, (void *)sc->addr, dest); \ fetch_memory_##type(regs, (void *)sc->addr, dest); \
else \ else \
*(type *)dest = 0; \ *(type *)dest = 0; \
} } \
NOKPROBE_SYMBOL(FETCH_FUNC_NAME(symbol, type));
DEFINE_BASIC_FETCH_FUNCS(symbol) DEFINE_BASIC_FETCH_FUNCS(symbol)
DEFINE_FETCH_symbol(string) DEFINE_FETCH_symbol(string)
DEFINE_FETCH_symbol(string_size) DEFINE_FETCH_symbol(string_size)
...@@ -907,7 +914,7 @@ static const struct file_operations kprobe_profile_ops = { ...@@ -907,7 +914,7 @@ static const struct file_operations kprobe_profile_ops = {
}; };
/* Kprobe handler */ /* Kprobe handler */
static __kprobes void static nokprobe_inline void
__kprobe_trace_func(struct trace_kprobe *tk, struct pt_regs *regs, __kprobe_trace_func(struct trace_kprobe *tk, struct pt_regs *regs,
struct ftrace_event_file *ftrace_file) struct ftrace_event_file *ftrace_file)
{ {
...@@ -943,7 +950,7 @@ __kprobe_trace_func(struct trace_kprobe *tk, struct pt_regs *regs, ...@@ -943,7 +950,7 @@ __kprobe_trace_func(struct trace_kprobe *tk, struct pt_regs *regs,
entry, irq_flags, pc, regs); entry, irq_flags, pc, regs);
} }
static __kprobes void static void
kprobe_trace_func(struct trace_kprobe *tk, struct pt_regs *regs) kprobe_trace_func(struct trace_kprobe *tk, struct pt_regs *regs)
{ {
struct event_file_link *link; struct event_file_link *link;
...@@ -951,9 +958,10 @@ kprobe_trace_func(struct trace_kprobe *tk, struct pt_regs *regs) ...@@ -951,9 +958,10 @@ kprobe_trace_func(struct trace_kprobe *tk, struct pt_regs *regs)
list_for_each_entry_rcu(link, &tk->tp.files, list) list_for_each_entry_rcu(link, &tk->tp.files, list)
__kprobe_trace_func(tk, regs, link->file); __kprobe_trace_func(tk, regs, link->file);
} }
NOKPROBE_SYMBOL(kprobe_trace_func);
/* Kretprobe handler */ /* Kretprobe handler */
static __kprobes void static nokprobe_inline void
__kretprobe_trace_func(struct trace_kprobe *tk, struct kretprobe_instance *ri, __kretprobe_trace_func(struct trace_kprobe *tk, struct kretprobe_instance *ri,
struct pt_regs *regs, struct pt_regs *regs,
struct ftrace_event_file *ftrace_file) struct ftrace_event_file *ftrace_file)
...@@ -991,7 +999,7 @@ __kretprobe_trace_func(struct trace_kprobe *tk, struct kretprobe_instance *ri, ...@@ -991,7 +999,7 @@ __kretprobe_trace_func(struct trace_kprobe *tk, struct kretprobe_instance *ri,
entry, irq_flags, pc, regs); entry, irq_flags, pc, regs);
} }
static __kprobes void static void
kretprobe_trace_func(struct trace_kprobe *tk, struct kretprobe_instance *ri, kretprobe_trace_func(struct trace_kprobe *tk, struct kretprobe_instance *ri,
struct pt_regs *regs) struct pt_regs *regs)
{ {
...@@ -1000,6 +1008,7 @@ kretprobe_trace_func(struct trace_kprobe *tk, struct kretprobe_instance *ri, ...@@ -1000,6 +1008,7 @@ kretprobe_trace_func(struct trace_kprobe *tk, struct kretprobe_instance *ri,
list_for_each_entry_rcu(link, &tk->tp.files, list) list_for_each_entry_rcu(link, &tk->tp.files, list)
__kretprobe_trace_func(tk, ri, regs, link->file); __kretprobe_trace_func(tk, ri, regs, link->file);
} }
NOKPROBE_SYMBOL(kretprobe_trace_func);
/* Event entry printers */ /* Event entry printers */
static enum print_line_t static enum print_line_t
...@@ -1131,7 +1140,7 @@ static int kretprobe_event_define_fields(struct ftrace_event_call *event_call) ...@@ -1131,7 +1140,7 @@ static int kretprobe_event_define_fields(struct ftrace_event_call *event_call)
#ifdef CONFIG_PERF_EVENTS #ifdef CONFIG_PERF_EVENTS
/* Kprobe profile handler */ /* Kprobe profile handler */
static __kprobes void static void
kprobe_perf_func(struct trace_kprobe *tk, struct pt_regs *regs) kprobe_perf_func(struct trace_kprobe *tk, struct pt_regs *regs)
{ {
struct ftrace_event_call *call = &tk->tp.call; struct ftrace_event_call *call = &tk->tp.call;
...@@ -1158,9 +1167,10 @@ kprobe_perf_func(struct trace_kprobe *tk, struct pt_regs *regs) ...@@ -1158,9 +1167,10 @@ kprobe_perf_func(struct trace_kprobe *tk, struct pt_regs *regs)
store_trace_args(sizeof(*entry), &tk->tp, regs, (u8 *)&entry[1], dsize); store_trace_args(sizeof(*entry), &tk->tp, regs, (u8 *)&entry[1], dsize);
perf_trace_buf_submit(entry, size, rctx, 0, 1, regs, head, NULL); perf_trace_buf_submit(entry, size, rctx, 0, 1, regs, head, NULL);
} }
NOKPROBE_SYMBOL(kprobe_perf_func);
/* Kretprobe profile handler */ /* Kretprobe profile handler */
static __kprobes void static void
kretprobe_perf_func(struct trace_kprobe *tk, struct kretprobe_instance *ri, kretprobe_perf_func(struct trace_kprobe *tk, struct kretprobe_instance *ri,
struct pt_regs *regs) struct pt_regs *regs)
{ {
...@@ -1188,6 +1198,7 @@ kretprobe_perf_func(struct trace_kprobe *tk, struct kretprobe_instance *ri, ...@@ -1188,6 +1198,7 @@ kretprobe_perf_func(struct trace_kprobe *tk, struct kretprobe_instance *ri,
store_trace_args(sizeof(*entry), &tk->tp, regs, (u8 *)&entry[1], dsize); store_trace_args(sizeof(*entry), &tk->tp, regs, (u8 *)&entry[1], dsize);
perf_trace_buf_submit(entry, size, rctx, 0, 1, regs, head, NULL); perf_trace_buf_submit(entry, size, rctx, 0, 1, regs, head, NULL);
} }
NOKPROBE_SYMBOL(kretprobe_perf_func);
#endif /* CONFIG_PERF_EVENTS */ #endif /* CONFIG_PERF_EVENTS */
/* /*
...@@ -1196,9 +1207,8 @@ kretprobe_perf_func(struct trace_kprobe *tk, struct kretprobe_instance *ri, ...@@ -1196,9 +1207,8 @@ kretprobe_perf_func(struct trace_kprobe *tk, struct kretprobe_instance *ri,
* kprobe_trace_self_tests_init() does enable_trace_probe/disable_trace_probe * kprobe_trace_self_tests_init() does enable_trace_probe/disable_trace_probe
* lockless, but we can't race with this __init function. * lockless, but we can't race with this __init function.
*/ */
static __kprobes static int kprobe_register(struct ftrace_event_call *event,
int kprobe_register(struct ftrace_event_call *event, enum trace_reg type, void *data)
enum trace_reg type, void *data)
{ {
struct trace_kprobe *tk = (struct trace_kprobe *)event->data; struct trace_kprobe *tk = (struct trace_kprobe *)event->data;
struct ftrace_event_file *file = data; struct ftrace_event_file *file = data;
...@@ -1224,8 +1234,7 @@ int kprobe_register(struct ftrace_event_call *event, ...@@ -1224,8 +1234,7 @@ int kprobe_register(struct ftrace_event_call *event,
return 0; return 0;
} }
static __kprobes static int kprobe_dispatcher(struct kprobe *kp, struct pt_regs *regs)
int kprobe_dispatcher(struct kprobe *kp, struct pt_regs *regs)
{ {
struct trace_kprobe *tk = container_of(kp, struct trace_kprobe, rp.kp); struct trace_kprobe *tk = container_of(kp, struct trace_kprobe, rp.kp);
...@@ -1239,9 +1248,10 @@ int kprobe_dispatcher(struct kprobe *kp, struct pt_regs *regs) ...@@ -1239,9 +1248,10 @@ int kprobe_dispatcher(struct kprobe *kp, struct pt_regs *regs)
#endif #endif
return 0; /* We don't tweek kernel, so just return 0 */ return 0; /* We don't tweek kernel, so just return 0 */
} }
NOKPROBE_SYMBOL(kprobe_dispatcher);
static __kprobes static int
int kretprobe_dispatcher(struct kretprobe_instance *ri, struct pt_regs *regs) kretprobe_dispatcher(struct kretprobe_instance *ri, struct pt_regs *regs)
{ {
struct trace_kprobe *tk = container_of(ri->rp, struct trace_kprobe, rp); struct trace_kprobe *tk = container_of(ri->rp, struct trace_kprobe, rp);
...@@ -1255,6 +1265,7 @@ int kretprobe_dispatcher(struct kretprobe_instance *ri, struct pt_regs *regs) ...@@ -1255,6 +1265,7 @@ int kretprobe_dispatcher(struct kretprobe_instance *ri, struct pt_regs *regs)
#endif #endif
return 0; /* We don't tweek kernel, so just return 0 */ return 0; /* We don't tweek kernel, so just return 0 */
} }
NOKPROBE_SYMBOL(kretprobe_dispatcher);
static struct trace_event_functions kretprobe_funcs = { static struct trace_event_functions kretprobe_funcs = {
.trace = print_kretprobe_event .trace = print_kretprobe_event
......
kernel/trace/trace_probe.c
View file @ 10b02564
...@@ -37,13 +37,13 @@ const char *reserved_field_names[] = { ...@@ -37,13 +37,13 @@ const char *reserved_field_names[] = {
/* Printing in basic type function template */ /* Printing in basic type function template */
#define DEFINE_BASIC_PRINT_TYPE_FUNC(type, fmt) \ #define DEFINE_BASIC_PRINT_TYPE_FUNC(type, fmt) \
__kprobes int PRINT_TYPE_FUNC_NAME(type)(struct trace_seq *s, \ int PRINT_TYPE_FUNC_NAME(type)(struct trace_seq *s, const char *name, \
const char *name, \ void *data, void *ent) \
void *data, void *ent) \
{ \ { \
return trace_seq_printf(s, " %s=" fmt, name, *(type *)data); \ return trace_seq_printf(s, " %s=" fmt, name, *(type *)data); \
} \ } \
const char PRINT_TYPE_FMT_NAME(type)[] = fmt; const char PRINT_TYPE_FMT_NAME(type)[] = fmt; \
NOKPROBE_SYMBOL(PRINT_TYPE_FUNC_NAME(type));
DEFINE_BASIC_PRINT_TYPE_FUNC(u8 , "0x%x") DEFINE_BASIC_PRINT_TYPE_FUNC(u8 , "0x%x")
DEFINE_BASIC_PRINT_TYPE_FUNC(u16, "0x%x") DEFINE_BASIC_PRINT_TYPE_FUNC(u16, "0x%x")
...@@ -55,9 +55,8 @@ DEFINE_BASIC_PRINT_TYPE_FUNC(s32, "%d") ...@@ -55,9 +55,8 @@ DEFINE_BASIC_PRINT_TYPE_FUNC(s32, "%d")
DEFINE_BASIC_PRINT_TYPE_FUNC(s64, "%Ld") DEFINE_BASIC_PRINT_TYPE_FUNC(s64, "%Ld")
/* Print type function for string type */ /* Print type function for string type */
__kprobes int PRINT_TYPE_FUNC_NAME(string)(struct trace_seq *s, int PRINT_TYPE_FUNC_NAME(string)(struct trace_seq *s, const char *name,
const char *name, void *data, void *ent)
void *data, void *ent)
{ {
int len = *(u32 *)data >> 16; int len = *(u32 *)data >> 16;
...@@ -67,6 +66,7 @@ __kprobes int PRINT_TYPE_FUNC_NAME(string)(struct trace_seq *s, ...@@ -67,6 +66,7 @@ __kprobes int PRINT_TYPE_FUNC_NAME(string)(struct trace_seq *s,
return trace_seq_printf(s, " %s=\"%s\"", name, return trace_seq_printf(s, " %s=\"%s\"", name,
(const char *)get_loc_data(data, ent)); (const char *)get_loc_data(data, ent));
} }
NOKPROBE_SYMBOL(PRINT_TYPE_FUNC_NAME(string));
const char PRINT_TYPE_FMT_NAME(string)[] = "\\\"%s\\\""; const char PRINT_TYPE_FMT_NAME(string)[] = "\\\"%s\\\"";
...@@ -81,23 +81,24 @@ const char PRINT_TYPE_FMT_NAME(string)[] = "\\\"%s\\\""; ...@@ -81,23 +81,24 @@ const char PRINT_TYPE_FMT_NAME(string)[] = "\\\"%s\\\"";
/* Data fetch function templates */ /* Data fetch function templates */
#define DEFINE_FETCH_reg(type) \ #define DEFINE_FETCH_reg(type) \
__kprobes void FETCH_FUNC_NAME(reg, type)(struct pt_regs *regs, \ void FETCH_FUNC_NAME(reg, type)(struct pt_regs *regs, void *offset, void *dest) \
void *offset, void *dest) \
{ \ { \
*(type *)dest = (type)regs_get_register(regs, \ *(type *)dest = (type)regs_get_register(regs, \
(unsigned int)((unsigned long)offset)); \ (unsigned int)((unsigned long)offset)); \
} } \
NOKPROBE_SYMBOL(FETCH_FUNC_NAME(reg, type));
DEFINE_BASIC_FETCH_FUNCS(reg) DEFINE_BASIC_FETCH_FUNCS(reg)
/* No string on the register */ /* No string on the register */
#define fetch_reg_string NULL #define fetch_reg_string NULL
#define fetch_reg_string_size NULL #define fetch_reg_string_size NULL
#define DEFINE_FETCH_retval(type) \ #define DEFINE_FETCH_retval(type) \
__kprobes void FETCH_FUNC_NAME(retval, type)(struct pt_regs *regs, \ void FETCH_FUNC_NAME(retval, type)(struct pt_regs *regs, \
void *dummy, void *dest) \ void *dummy, void *dest) \
{ \ { \
*(type *)dest = (type)regs_return_value(regs); \ *(type *)dest = (type)regs_return_value(regs); \
} } \
NOKPROBE_SYMBOL(FETCH_FUNC_NAME(retval, type));
DEFINE_BASIC_FETCH_FUNCS(retval) DEFINE_BASIC_FETCH_FUNCS(retval)
/* No string on the retval */ /* No string on the retval */
#define fetch_retval_string NULL #define fetch_retval_string NULL
...@@ -112,8 +113,8 @@ struct deref_fetch_param { ...@@ -112,8 +113,8 @@ struct deref_fetch_param {
}; };
#define DEFINE_FETCH_deref(type) \ #define DEFINE_FETCH_deref(type) \
__kprobes void FETCH_FUNC_NAME(deref, type)(struct pt_regs *regs, \ void FETCH_FUNC_NAME(deref, type)(struct pt_regs *regs, \
void *data, void *dest) \ void *data, void *dest) \
{ \ { \
struct deref_fetch_param *dprm = data; \ struct deref_fetch_param *dprm = data; \
unsigned long addr; \ unsigned long addr; \
...@@ -123,12 +124,13 @@ __kprobes void FETCH_FUNC_NAME(deref, type)(struct pt_regs *regs, \ ...@@ -123,12 +124,13 @@ __kprobes void FETCH_FUNC_NAME(deref, type)(struct pt_regs *regs, \
dprm->fetch(regs, (void *)addr, dest); \ dprm->fetch(regs, (void *)addr, dest); \
} else \ } else \
*(type *)dest = 0; \ *(type *)dest = 0; \
} } \
NOKPROBE_SYMBOL(FETCH_FUNC_NAME(deref, type));
DEFINE_BASIC_FETCH_FUNCS(deref) DEFINE_BASIC_FETCH_FUNCS(deref)
DEFINE_FETCH_deref(string) DEFINE_FETCH_deref(string)
__kprobes void FETCH_FUNC_NAME(deref, string_size)(struct pt_regs *regs, void FETCH_FUNC_NAME(deref, string_size)(struct pt_regs *regs,
void *data, void *dest) void *data, void *dest)
{ {
struct deref_fetch_param *dprm = data; struct deref_fetch_param *dprm = data;
unsigned long addr; unsigned long addr;
...@@ -140,16 +142,18 @@ __kprobes void FETCH_FUNC_NAME(deref, string_size)(struct pt_regs *regs, ...@@ -140,16 +142,18 @@ __kprobes void FETCH_FUNC_NAME(deref, string_size)(struct pt_regs *regs,
} else } else
*(string_size *)dest = 0; *(string_size *)dest = 0;
} }
NOKPROBE_SYMBOL(FETCH_FUNC_NAME(deref, string_size));
static __kprobes void update_deref_fetch_param(struct deref_fetch_param *data) static void update_deref_fetch_param(struct deref_fetch_param *data)
{ {
if (CHECK_FETCH_FUNCS(deref, data->orig.fn)) if (CHECK_FETCH_FUNCS(deref, data->orig.fn))
update_deref_fetch_param(data->orig.data); update_deref_fetch_param(data->orig.data);
else if (CHECK_FETCH_FUNCS(symbol, data->orig.fn)) else if (CHECK_FETCH_FUNCS(symbol, data->orig.fn))
update_symbol_cache(data->orig.data); update_symbol_cache(data->orig.data);
} }
NOKPROBE_SYMBOL(update_deref_fetch_param);
static __kprobes void free_deref_fetch_param(struct deref_fetch_param *data) static void free_deref_fetch_param(struct deref_fetch_param *data)
{ {
if (CHECK_FETCH_FUNCS(deref, data->orig.fn)) if (CHECK_FETCH_FUNCS(deref, data->orig.fn))
free_deref_fetch_param(data->orig.data); free_deref_fetch_param(data->orig.data);
...@@ -157,6 +161,7 @@ static __kprobes void free_deref_fetch_param(struct deref_fetch_param *data) ...@@ -157,6 +161,7 @@ static __kprobes void free_deref_fetch_param(struct deref_fetch_param *data)
free_symbol_cache(data->orig.data); free_symbol_cache(data->orig.data);
kfree(data); kfree(data);
} }
NOKPROBE_SYMBOL(free_deref_fetch_param);
/* Bitfield fetch function */ /* Bitfield fetch function */
struct bitfield_fetch_param { struct bitfield_fetch_param {
...@@ -166,8 +171,8 @@ struct bitfield_fetch_param { ...@@ -166,8 +171,8 @@ struct bitfield_fetch_param {
}; };
#define DEFINE_FETCH_bitfield(type) \ #define DEFINE_FETCH_bitfield(type) \
__kprobes void FETCH_FUNC_NAME(bitfield, type)(struct pt_regs *regs, \ void FETCH_FUNC_NAME(bitfield, type)(struct pt_regs *regs, \
void *data, void *dest) \ void *data, void *dest) \
{ \ { \
struct bitfield_fetch_param *bprm = data; \ struct bitfield_fetch_param *bprm = data; \
type buf = 0; \ type buf = 0; \
...@@ -177,13 +182,13 @@ __kprobes void FETCH_FUNC_NAME(bitfield, type)(struct pt_regs *regs, \ ...@@ -177,13 +182,13 @@ __kprobes void FETCH_FUNC_NAME(bitfield, type)(struct pt_regs *regs, \
buf >>= bprm->low_shift; \ buf >>= bprm->low_shift; \
} \ } \
*(type *)dest = buf; \ *(type *)dest = buf; \
} } \
NOKPROBE_SYMBOL(FETCH_FUNC_NAME(bitfield, type));
DEFINE_BASIC_FETCH_FUNCS(bitfield) DEFINE_BASIC_FETCH_FUNCS(bitfield)
#define fetch_bitfield_string NULL #define fetch_bitfield_string NULL
#define fetch_bitfield_string_size NULL #define fetch_bitfield_string_size NULL
static __kprobes void static void
update_bitfield_fetch_param(struct bitfield_fetch_param *data) update_bitfield_fetch_param(struct bitfield_fetch_param *data)
{ {
/* /*
...@@ -196,7 +201,7 @@ update_bitfield_fetch_param(struct bitfield_fetch_param *data) ...@@ -196,7 +201,7 @@ update_bitfield_fetch_param(struct bitfield_fetch_param *data)
update_symbol_cache(data->orig.data); update_symbol_cache(data->orig.data);
} }
static __kprobes void static void
free_bitfield_fetch_param(struct bitfield_fetch_param *data) free_bitfield_fetch_param(struct bitfield_fetch_param *data)
{ {
/* /*
...@@ -255,17 +260,17 @@ static const struct fetch_type *find_fetch_type(const char *type, ...@@ -255,17 +260,17 @@ static const struct fetch_type *find_fetch_type(const char *type,
} }
/* Special function : only accept unsigned long */ /* Special function : only accept unsigned long */
static __kprobes void fetch_kernel_stack_address(struct pt_regs *regs, static void fetch_kernel_stack_address(struct pt_regs *regs, void *dummy, void *dest)
void *dummy, void *dest)
{ {
*(unsigned long *)dest = kernel_stack_pointer(regs); *(unsigned long *)dest = kernel_stack_pointer(regs);
} }
NOKPROBE_SYMBOL(fetch_kernel_stack_address);
static __kprobes void fetch_user_stack_address(struct pt_regs *regs, static void fetch_user_stack_address(struct pt_regs *regs, void *dummy, void *dest)
void *dummy, void *dest)
{ {
*(unsigned long *)dest = user_stack_pointer(regs); *(unsigned long *)dest = user_stack_pointer(regs);
} }
NOKPROBE_SYMBOL(fetch_user_stack_address);
static fetch_func_t get_fetch_size_function(const struct fetch_type *type, static fetch_func_t get_fetch_size_function(const struct fetch_type *type,
fetch_func_t orig_fn, fetch_func_t orig_fn,
......
kernel/trace/trace_probe.h
View file @ 10b02564
...@@ -81,13 +81,13 @@ ...@@ -81,13 +81,13 @@
*/ */
#define convert_rloc_to_loc(dl, offs) ((u32)(dl) + (offs)) #define convert_rloc_to_loc(dl, offs) ((u32)(dl) + (offs))
static inline void *get_rloc_data(u32 *dl) static nokprobe_inline void *get_rloc_data(u32 *dl)
{ {
return (u8 *)dl + get_rloc_offs(*dl); return (u8 *)dl + get_rloc_offs(*dl);
} }
/* For data_loc conversion */ /* For data_loc conversion */
static inline void *get_loc_data(u32 *dl, void *ent) static nokprobe_inline void *get_loc_data(u32 *dl, void *ent)
{ {
return (u8 *)ent + get_rloc_offs(*dl); return (u8 *)ent + get_rloc_offs(*dl);
} }
...@@ -136,9 +136,8 @@ typedef u32 string_size; ...@@ -136,9 +136,8 @@ typedef u32 string_size;
/* Printing in basic type function template */ /* Printing in basic type function template */
#define DECLARE_BASIC_PRINT_TYPE_FUNC(type) \ #define DECLARE_BASIC_PRINT_TYPE_FUNC(type) \
__kprobes int PRINT_TYPE_FUNC_NAME(type)(struct trace_seq *s, \ int PRINT_TYPE_FUNC_NAME(type)(struct trace_seq *s, const char *name, \
const char *name, \ void *data, void *ent); \
void *data, void *ent); \
extern const char PRINT_TYPE_FMT_NAME(type)[] extern const char PRINT_TYPE_FMT_NAME(type)[]
DECLARE_BASIC_PRINT_TYPE_FUNC(u8); DECLARE_BASIC_PRINT_TYPE_FUNC(u8);
...@@ -303,7 +302,7 @@ static inline bool trace_probe_is_registered(struct trace_probe *tp) ...@@ -303,7 +302,7 @@ static inline bool trace_probe_is_registered(struct trace_probe *tp)
return !!(tp->flags & TP_FLAG_REGISTERED); return !!(tp->flags & TP_FLAG_REGISTERED);
} }
static inline __kprobes void call_fetch(struct fetch_param *fprm, static nokprobe_inline void call_fetch(struct fetch_param *fprm,
struct pt_regs *regs, void *dest) struct pt_regs *regs, void *dest)
{ {
return fprm->fn(regs, fprm->data, dest); return fprm->fn(regs, fprm->data, dest);
...@@ -351,7 +350,7 @@ extern ssize_t traceprobe_probes_write(struct file *file, ...@@ -351,7 +350,7 @@ extern ssize_t traceprobe_probes_write(struct file *file,
extern int traceprobe_command(const char *buf, int (*createfn)(int, char**)); extern int traceprobe_command(const char *buf, int (*createfn)(int, char**));
/* Sum up total data length for dynamic arraies (strings) */ /* Sum up total data length for dynamic arraies (strings) */
static inline __kprobes int static nokprobe_inline int
__get_data_size(struct trace_probe *tp, struct pt_regs *regs) __get_data_size(struct trace_probe *tp, struct pt_regs *regs)
{ {
int i, ret = 0; int i, ret = 0;
...@@ -367,7 +366,7 @@ __get_data_size(struct trace_probe *tp, struct pt_regs *regs) ...@@ -367,7 +366,7 @@ __get_data_size(struct trace_probe *tp, struct pt_regs *regs)
} }
/* Store the value of each argument */ /* Store the value of each argument */
static inline __kprobes void static nokprobe_inline void
store_trace_args(int ent_size, struct trace_probe *tp, struct pt_regs *regs, store_trace_args(int ent_size, struct trace_probe *tp, struct pt_regs *regs,
u8 *data, int maxlen) u8 *data, int maxlen)
{ {
......
kernel/trace/trace_uprobe.c
View file @ 10b02564
...@@ -108,8 +108,8 @@ static unsigned long get_user_stack_nth(struct pt_regs *regs, unsigned int n) ...@@ -108,8 +108,8 @@ static unsigned long get_user_stack_nth(struct pt_regs *regs, unsigned int n)
* Uprobes-specific fetch functions * Uprobes-specific fetch functions
*/ */
#define DEFINE_FETCH_stack(type) \ #define DEFINE_FETCH_stack(type) \
static __kprobes void FETCH_FUNC_NAME(stack, type)(struct pt_regs *regs,\ static void FETCH_FUNC_NAME(stack, type)(struct pt_regs *regs, \
void *offset, void *dest) \ void *offset, void *dest) \
{ \ { \
*(type *)dest = (type)get_user_stack_nth(regs, \ *(type *)dest = (type)get_user_stack_nth(regs, \
((unsigned long)offset)); \ ((unsigned long)offset)); \
...@@ -120,8 +120,8 @@ DEFINE_BASIC_FETCH_FUNCS(stack) ...@@ -120,8 +120,8 @@ DEFINE_BASIC_FETCH_FUNCS(stack)
#define fetch_stack_string_size NULL #define fetch_stack_string_size NULL
#define DEFINE_FETCH_memory(type) \ #define DEFINE_FETCH_memory(type) \
static __kprobes void FETCH_FUNC_NAME(memory, type)(struct pt_regs *regs,\ static void FETCH_FUNC_NAME(memory, type)(struct pt_regs *regs, \
void *addr, void *dest) \ void *addr, void *dest) \
{ \ { \
type retval; \ type retval; \
void __user *vaddr = (void __force __user *) addr; \ void __user *vaddr = (void __force __user *) addr; \
...@@ -136,8 +136,8 @@ DEFINE_BASIC_FETCH_FUNCS(memory) ...@@ -136,8 +136,8 @@ DEFINE_BASIC_FETCH_FUNCS(memory)
* Fetch a null-terminated string. Caller MUST set *(u32 *)dest with max * Fetch a null-terminated string. Caller MUST set *(u32 *)dest with max
* length and relative data location. * length and relative data location.
*/ */
static __kprobes void FETCH_FUNC_NAME(memory, string)(struct pt_regs *regs, static void FETCH_FUNC_NAME(memory, string)(struct pt_regs *regs,
void *addr, void *dest) void *addr, void *dest)
{ {
long ret; long ret;
u32 rloc = *(u32 *)dest; u32 rloc = *(u32 *)dest;
...@@ -158,8 +158,8 @@ static __kprobes void FETCH_FUNC_NAME(memory, string)(struct pt_regs *regs, ...@@ -158,8 +158,8 @@ static __kprobes void FETCH_FUNC_NAME(memory, string)(struct pt_regs *regs,
} }
} }
static __kprobes void FETCH_FUNC_NAME(memory, string_size)(struct pt_regs *regs, static void FETCH_FUNC_NAME(memory, string_size)(struct pt_regs *regs,
void *addr, void *dest) void *addr, void *dest)
{ {
int len; int len;
void __user *vaddr = (void __force __user *) addr; void __user *vaddr = (void __force __user *) addr;
...@@ -184,8 +184,8 @@ static unsigned long translate_user_vaddr(void *file_offset) ...@@ -184,8 +184,8 @@ static unsigned long translate_user_vaddr(void *file_offset)
} }
#define DEFINE_FETCH_file_offset(type) \ #define DEFINE_FETCH_file_offset(type) \
static __kprobes void FETCH_FUNC_NAME(file_offset, type)(struct pt_regs *regs,\ static void FETCH_FUNC_NAME(file_offset, type)(struct pt_regs *regs, \
void *offset, void *dest) \ void *offset, void *dest)\
{ \ { \
void *vaddr = (void *)translate_user_vaddr(offset); \ void *vaddr = (void *)translate_user_vaddr(offset); \
\ \
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment
GitLab Nexedi Edition | About GitLab | About Nexedi | 沪ICP备2021021310号-2 | 沪ICP备2021021310号-7