Commit a7636d9e authored by Martin KaFai Lau's avatar Martin KaFai Lau Committed by Ingo Molnar

kprobes: Optimize hot path by using percpu counter to collect 'nhit' statistics

When doing ebpf+kprobe on some hot TCP functions (e.g.
tcp_rcv_established), the kprobe_dispatcher() function
shows up in 'perf report'.

In kprobe_dispatcher(), there is a lot of cache bouncing
on 'tk->nhit++'.  'tk->nhit' and 'tk->tp.flags' also share
the same cacheline.

perf report (cycles:pp):

	8.30%  ipv4_dst_check
	4.74%  copy_user_enhanced_fast_string
	3.93%  dst_release
	2.80%  tcp_v4_rcv
	2.31%  queued_spin_lock_slowpath
	2.30%  _raw_spin_lock
	1.88%  mlx4_en_process_rx_cq
	1.84%  eth_get_headlen
	1.81%  ip_rcv_finish
	~~~~
	1.71%  kprobe_dispatcher
	~~~~
	1.55%  mlx4_en_xmit
	1.09%  __probe_kernel_read

perf report after patch:

	9.15%  ipv4_dst_check
	5.00%  copy_user_enhanced_fast_string
	4.12%  dst_release
	2.96%  tcp_v4_rcv
	2.50%  _raw_spin_lock
	2.39%  queued_spin_lock_slowpath
	2.11%  eth_get_headlen
	2.03%  mlx4_en_process_rx_cq
	1.69%  mlx4_en_xmit
	1.19%  ip_rcv_finish
	1.12%  __probe_kernel_read
	1.02%  ehci_hcd_cleanup
Signed-off-by: default avatarMartin KaFai Lau <kafai@fb.com>
Acked-by: default avatarMasami Hiramatsu <masami.hiramatsu.pt@hitachi.com>
Acked-by: default avatarSteven Rostedt <rostedt@goodmis.org>
Cc: Alexei Starovoitov <alexei.starovoitov@gmail.com>
Cc: Josef Bacik <jbacik@fb.com>
Cc: Kernel Team <kernel-team@fb.com>
Cc: Linus Torvalds <torvalds@linux-foundation.org>
Cc: Peter Zijlstra <peterz@infradead.org>
Cc: Thomas Gleixner <tglx@linutronix.de>
Link: http://lkml.kernel.org/r/1454531308-2441898-1-git-send-email-kafai@fb.comSigned-off-by: default avatarIngo Molnar <mingo@kernel.org>
parent 156d2238
...@@ -30,7 +30,7 @@ ...@@ -30,7 +30,7 @@
struct trace_kprobe { struct trace_kprobe {
struct list_head list; struct list_head list;
struct kretprobe rp; /* Use rp.kp for kprobe use */ struct kretprobe rp; /* Use rp.kp for kprobe use */
unsigned long nhit; unsigned long __percpu *nhit;
const char *symbol; /* symbol name */ const char *symbol; /* symbol name */
struct trace_probe tp; struct trace_probe tp;
}; };
...@@ -274,6 +274,10 @@ static struct trace_kprobe *alloc_trace_kprobe(const char *group, ...@@ -274,6 +274,10 @@ static struct trace_kprobe *alloc_trace_kprobe(const char *group,
if (!tk) if (!tk)
return ERR_PTR(ret); return ERR_PTR(ret);
tk->nhit = alloc_percpu(unsigned long);
if (!tk->nhit)
goto error;
if (symbol) { if (symbol) {
tk->symbol = kstrdup(symbol, GFP_KERNEL); tk->symbol = kstrdup(symbol, GFP_KERNEL);
if (!tk->symbol) if (!tk->symbol)
...@@ -313,6 +317,7 @@ static struct trace_kprobe *alloc_trace_kprobe(const char *group, ...@@ -313,6 +317,7 @@ static struct trace_kprobe *alloc_trace_kprobe(const char *group,
error: error:
kfree(tk->tp.call.name); kfree(tk->tp.call.name);
kfree(tk->symbol); kfree(tk->symbol);
free_percpu(tk->nhit);
kfree(tk); kfree(tk);
return ERR_PTR(ret); return ERR_PTR(ret);
} }
...@@ -327,6 +332,7 @@ static void free_trace_kprobe(struct trace_kprobe *tk) ...@@ -327,6 +332,7 @@ static void free_trace_kprobe(struct trace_kprobe *tk)
kfree(tk->tp.call.class->system); kfree(tk->tp.call.class->system);
kfree(tk->tp.call.name); kfree(tk->tp.call.name);
kfree(tk->symbol); kfree(tk->symbol);
free_percpu(tk->nhit);
kfree(tk); kfree(tk);
} }
...@@ -874,9 +880,14 @@ static const struct file_operations kprobe_events_ops = { ...@@ -874,9 +880,14 @@ static const struct file_operations kprobe_events_ops = {
static int probes_profile_seq_show(struct seq_file *m, void *v) static int probes_profile_seq_show(struct seq_file *m, void *v)
{ {
struct trace_kprobe *tk = v; struct trace_kprobe *tk = v;
unsigned long nhit = 0;
int cpu;
for_each_possible_cpu(cpu)
nhit += *per_cpu_ptr(tk->nhit, cpu);
seq_printf(m, " %-44s %15lu %15lu\n", seq_printf(m, " %-44s %15lu %15lu\n",
trace_event_name(&tk->tp.call), tk->nhit, trace_event_name(&tk->tp.call), nhit,
tk->rp.kp.nmissed); tk->rp.kp.nmissed);
return 0; return 0;
...@@ -1225,7 +1236,7 @@ static int kprobe_dispatcher(struct kprobe *kp, struct pt_regs *regs) ...@@ -1225,7 +1236,7 @@ static int kprobe_dispatcher(struct kprobe *kp, struct pt_regs *regs)
{ {
struct trace_kprobe *tk = container_of(kp, struct trace_kprobe, rp.kp); struct trace_kprobe *tk = container_of(kp, struct trace_kprobe, rp.kp);
tk->nhit++; raw_cpu_inc(*tk->nhit);
if (tk->tp.flags & TP_FLAG_TRACE) if (tk->tp.flags & TP_FLAG_TRACE)
kprobe_trace_func(tk, regs); kprobe_trace_func(tk, regs);
...@@ -1242,7 +1253,7 @@ kretprobe_dispatcher(struct kretprobe_instance *ri, struct pt_regs *regs) ...@@ -1242,7 +1253,7 @@ kretprobe_dispatcher(struct kretprobe_instance *ri, struct pt_regs *regs)
{ {
struct trace_kprobe *tk = container_of(ri->rp, struct trace_kprobe, rp); struct trace_kprobe *tk = container_of(ri->rp, struct trace_kprobe, rp);
tk->nhit++; raw_cpu_inc(*tk->nhit);
if (tk->tp.flags & TP_FLAG_TRACE) if (tk->tp.flags & TP_FLAG_TRACE)
kretprobe_trace_func(tk, ri, regs); kretprobe_trace_func(tk, ri, regs);
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment