Commit b91e014f authored by Alexei Starovoitov's avatar Alexei Starovoitov

bpf: Make BPF trampoline use register_ftrace_direct() API

Make BPF trampoline attach its generated assembly code to kernel functions via
register_ftrace_direct() API. It helps ftrace-based tracers co-exist with BPF
trampoline on the same kernel function. It also switches attaching logic from
arch specific text_poke to generic ftrace that is available on many
architectures. text_poke is still necessary for bpf-to-bpf attach and for
bpf_tail_call optimization.
Signed-off-by: default avatarAlexei Starovoitov <ast@kernel.org>
Acked-by: default avatarDaniel Borkmann <daniel@iogearbox.net>
Link: https://lore.kernel.org/bpf/20191209000114.1876138-3-ast@kernel.org
parent 5b79bcdf
......@@ -461,6 +461,7 @@ struct bpf_trampoline {
struct {
struct btf_func_model model;
void *addr;
bool ftrace_managed;
} func;
/* list of BPF programs using this trampoline */
struct hlist_head progs_hlist[BPF_TRAMP_MAX];
......
......@@ -3,6 +3,7 @@
#include <linux/hash.h>
#include <linux/bpf.h>
#include <linux/filter.h>
#include <linux/ftrace.h>
/* btf_vmlinux has ~22k attachable functions. 1k htab is enough. */
#define TRAMPOLINE_HASH_BITS 10
......@@ -59,6 +60,60 @@ struct bpf_trampoline *bpf_trampoline_lookup(u64 key)
return tr;
}
static int is_ftrace_location(void *ip)
{
long addr;
addr = ftrace_location((long)ip);
if (!addr)
return 0;
if (WARN_ON_ONCE(addr != (long)ip))
return -EFAULT;
return 1;
}
static int unregister_fentry(struct bpf_trampoline *tr, void *old_addr)
{
void *ip = tr->func.addr;
int ret;
if (tr->func.ftrace_managed)
ret = unregister_ftrace_direct((long)ip, (long)old_addr);
else
ret = bpf_arch_text_poke(ip, BPF_MOD_CALL, old_addr, NULL);
return ret;
}
static int modify_fentry(struct bpf_trampoline *tr, void *old_addr, void *new_addr)
{
void *ip = tr->func.addr;
int ret;
if (tr->func.ftrace_managed)
ret = modify_ftrace_direct((long)ip, (long)old_addr, (long)new_addr);
else
ret = bpf_arch_text_poke(ip, BPF_MOD_CALL, old_addr, new_addr);
return ret;
}
/* first time registering */
static int register_fentry(struct bpf_trampoline *tr, void *new_addr)
{
void *ip = tr->func.addr;
int ret;
ret = is_ftrace_location(ip);
if (ret < 0)
return ret;
tr->func.ftrace_managed = ret;
if (tr->func.ftrace_managed)
ret = register_ftrace_direct((long)ip, (long)new_addr);
else
ret = bpf_arch_text_poke(ip, BPF_MOD_CALL, NULL, new_addr);
return ret;
}
/* Each call __bpf_prog_enter + call bpf_func + call __bpf_prog_exit is ~50
* bytes on x86. Pick a number to fit into PAGE_SIZE / 2
*/
......@@ -77,8 +132,7 @@ static int bpf_trampoline_update(struct bpf_trampoline *tr)
int err;
if (fentry_cnt + fexit_cnt == 0) {
err = bpf_arch_text_poke(tr->func.addr, BPF_MOD_CALL,
old_image, NULL);
err = unregister_fentry(tr, old_image);
tr->selector = 0;
goto out;
}
......@@ -105,12 +159,10 @@ static int bpf_trampoline_update(struct bpf_trampoline *tr)
if (tr->selector)
/* progs already running at this address */
err = bpf_arch_text_poke(tr->func.addr, BPF_MOD_CALL,
old_image, new_image);
err = modify_fentry(tr, old_image, new_image);
else
/* first time registering */
err = bpf_arch_text_poke(tr->func.addr, BPF_MOD_CALL, NULL,
new_image);
err = register_fentry(tr, new_image);
if (err)
goto out;
tr->selector++;
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment