Commit 7de16e3a authored by Jakub Kicinski's avatar Jakub Kicinski Committed by David S. Miller

bpf: split verifier and program ops

struct bpf_verifier_ops contains both verifier ops and operations
used later during program's lifetime (test_run).  Split the runtime
ops into a different structure.

BPF_PROG_TYPE() will now append ## _prog_ops or ## _verifier_ops
to the names.
Signed-off-by: default avatarJakub Kicinski <jakub.kicinski@netronome.com>
Acked-by: default avatarDaniel Borkmann <daniel@iogearbox.net>
Acked-by: default avatarAlexei Starovoitov <ast@kernel.org>
Signed-off-by: default avatarDavid S. Miller <davem@davemloft.net>
parent 386fd5da
...@@ -157,6 +157,11 @@ bpf_ctx_record_field_size(struct bpf_insn_access_aux *aux, u32 size) ...@@ -157,6 +157,11 @@ bpf_ctx_record_field_size(struct bpf_insn_access_aux *aux, u32 size)
aux->ctx_field_size = size; aux->ctx_field_size = size;
} }
struct bpf_prog_ops {
int (*test_run)(struct bpf_prog *prog, const union bpf_attr *kattr,
union bpf_attr __user *uattr);
};
struct bpf_verifier_ops { struct bpf_verifier_ops {
/* return eBPF function prototype for verification */ /* return eBPF function prototype for verification */
const struct bpf_func_proto *(*get_func_proto)(enum bpf_func_id func_id); const struct bpf_func_proto *(*get_func_proto)(enum bpf_func_id func_id);
...@@ -172,8 +177,6 @@ struct bpf_verifier_ops { ...@@ -172,8 +177,6 @@ struct bpf_verifier_ops {
const struct bpf_insn *src, const struct bpf_insn *src,
struct bpf_insn *dst, struct bpf_insn *dst,
struct bpf_prog *prog, u32 *target_size); struct bpf_prog *prog, u32 *target_size);
int (*test_run)(struct bpf_prog *prog, const union bpf_attr *kattr,
union bpf_attr __user *uattr);
}; };
struct bpf_prog_aux { struct bpf_prog_aux {
...@@ -184,7 +187,8 @@ struct bpf_prog_aux { ...@@ -184,7 +187,8 @@ struct bpf_prog_aux {
u32 id; u32 id;
struct latch_tree_node ksym_tnode; struct latch_tree_node ksym_tnode;
struct list_head ksym_lnode; struct list_head ksym_lnode;
const struct bpf_verifier_ops *ops; const struct bpf_prog_ops *ops;
const struct bpf_verifier_ops *vops;
struct bpf_map **used_maps; struct bpf_map **used_maps;
struct bpf_prog *prog; struct bpf_prog *prog;
struct user_struct *user; struct user_struct *user;
...@@ -279,8 +283,9 @@ int bpf_prog_array_copy_to_user(struct bpf_prog_array __rcu *progs, ...@@ -279,8 +283,9 @@ int bpf_prog_array_copy_to_user(struct bpf_prog_array __rcu *progs,
#ifdef CONFIG_BPF_SYSCALL #ifdef CONFIG_BPF_SYSCALL
DECLARE_PER_CPU(int, bpf_prog_active); DECLARE_PER_CPU(int, bpf_prog_active);
#define BPF_PROG_TYPE(_id, _ops) \ #define BPF_PROG_TYPE(_id, _name) \
extern const struct bpf_verifier_ops _ops; extern const struct bpf_prog_ops _name ## _prog_ops; \
extern const struct bpf_verifier_ops _name ## _verifier_ops;
#define BPF_MAP_TYPE(_id, _ops) \ #define BPF_MAP_TYPE(_id, _ops) \
extern const struct bpf_map_ops _ops; extern const struct bpf_map_ops _ops;
#include <linux/bpf_types.h> #include <linux/bpf_types.h>
......
/* internal file - do not include directly */ /* internal file - do not include directly */
#ifdef CONFIG_NET #ifdef CONFIG_NET
BPF_PROG_TYPE(BPF_PROG_TYPE_SOCKET_FILTER, sk_filter_prog_ops) BPF_PROG_TYPE(BPF_PROG_TYPE_SOCKET_FILTER, sk_filter)
BPF_PROG_TYPE(BPF_PROG_TYPE_SCHED_CLS, tc_cls_act_prog_ops) BPF_PROG_TYPE(BPF_PROG_TYPE_SCHED_CLS, tc_cls_act)
BPF_PROG_TYPE(BPF_PROG_TYPE_SCHED_ACT, tc_cls_act_prog_ops) BPF_PROG_TYPE(BPF_PROG_TYPE_SCHED_ACT, tc_cls_act)
BPF_PROG_TYPE(BPF_PROG_TYPE_XDP, xdp_prog_ops) BPF_PROG_TYPE(BPF_PROG_TYPE_XDP, xdp)
BPF_PROG_TYPE(BPF_PROG_TYPE_CGROUP_SKB, cg_skb_prog_ops) BPF_PROG_TYPE(BPF_PROG_TYPE_CGROUP_SKB, cg_skb)
BPF_PROG_TYPE(BPF_PROG_TYPE_CGROUP_SOCK, cg_sock_prog_ops) BPF_PROG_TYPE(BPF_PROG_TYPE_CGROUP_SOCK, cg_sock)
BPF_PROG_TYPE(BPF_PROG_TYPE_LWT_IN, lwt_inout_prog_ops) BPF_PROG_TYPE(BPF_PROG_TYPE_LWT_IN, lwt_inout)
BPF_PROG_TYPE(BPF_PROG_TYPE_LWT_OUT, lwt_inout_prog_ops) BPF_PROG_TYPE(BPF_PROG_TYPE_LWT_OUT, lwt_inout)
BPF_PROG_TYPE(BPF_PROG_TYPE_LWT_XMIT, lwt_xmit_prog_ops) BPF_PROG_TYPE(BPF_PROG_TYPE_LWT_XMIT, lwt_xmit)
BPF_PROG_TYPE(BPF_PROG_TYPE_SOCK_OPS, sock_ops_prog_ops) BPF_PROG_TYPE(BPF_PROG_TYPE_SOCK_OPS, sock_ops)
BPF_PROG_TYPE(BPF_PROG_TYPE_SK_SKB, sk_skb_prog_ops) BPF_PROG_TYPE(BPF_PROG_TYPE_SK_SKB, sk_skb)
#endif #endif
#ifdef CONFIG_BPF_EVENTS #ifdef CONFIG_BPF_EVENTS
BPF_PROG_TYPE(BPF_PROG_TYPE_KPROBE, kprobe_prog_ops) BPF_PROG_TYPE(BPF_PROG_TYPE_KPROBE, kprobe)
BPF_PROG_TYPE(BPF_PROG_TYPE_TRACEPOINT, tracepoint_prog_ops) BPF_PROG_TYPE(BPF_PROG_TYPE_TRACEPOINT, tracepoint)
BPF_PROG_TYPE(BPF_PROG_TYPE_PERF_EVENT, perf_event_prog_ops) BPF_PROG_TYPE(BPF_PROG_TYPE_PERF_EVENT, perf_event)
#endif #endif
BPF_MAP_TYPE(BPF_MAP_TYPE_ARRAY, array_map_ops) BPF_MAP_TYPE(BPF_MAP_TYPE_ARRAY, array_map_ops)
......
...@@ -739,9 +739,18 @@ static int map_get_next_key(union bpf_attr *attr) ...@@ -739,9 +739,18 @@ static int map_get_next_key(union bpf_attr *attr)
return err; return err;
} }
static const struct bpf_verifier_ops * const bpf_prog_types[] = { static const struct bpf_prog_ops * const bpf_prog_types[] = {
#define BPF_PROG_TYPE(_id, _ops) \ #define BPF_PROG_TYPE(_id, _name) \
[_id] = &_ops, [_id] = & _name ## _prog_ops,
#define BPF_MAP_TYPE(_id, _ops)
#include <linux/bpf_types.h>
#undef BPF_PROG_TYPE
#undef BPF_MAP_TYPE
};
static const struct bpf_verifier_ops * const bpf_verifier_ops[] = {
#define BPF_PROG_TYPE(_id, _name) \
[_id] = & _name ## _verifier_ops,
#define BPF_MAP_TYPE(_id, _ops) #define BPF_MAP_TYPE(_id, _ops)
#include <linux/bpf_types.h> #include <linux/bpf_types.h>
#undef BPF_PROG_TYPE #undef BPF_PROG_TYPE
...@@ -754,6 +763,7 @@ static int find_prog_type(enum bpf_prog_type type, struct bpf_prog *prog) ...@@ -754,6 +763,7 @@ static int find_prog_type(enum bpf_prog_type type, struct bpf_prog *prog)
return -EINVAL; return -EINVAL;
prog->aux->ops = bpf_prog_types[type]; prog->aux->ops = bpf_prog_types[type];
prog->aux->vops = bpf_verifier_ops[type];
prog->type = type; prog->type = type;
return 0; return 0;
} }
......
...@@ -856,8 +856,8 @@ static int check_ctx_access(struct bpf_verifier_env *env, int insn_idx, int off, ...@@ -856,8 +856,8 @@ static int check_ctx_access(struct bpf_verifier_env *env, int insn_idx, int off,
*reg_type = info.reg_type; *reg_type = info.reg_type;
return 0; return 0;
} }
} else if (env->prog->aux->ops->is_valid_access && } else if (env->prog->aux->vops->is_valid_access &&
env->prog->aux->ops->is_valid_access(off, size, t, &info)) { env->prog->aux->vops->is_valid_access(off, size, t, &info)) {
/* A non zero info.ctx_field_size indicates that this field is a /* A non zero info.ctx_field_size indicates that this field is a
* candidate for later verifier transformation to load the whole * candidate for later verifier transformation to load the whole
* field and then apply a mask when accessed with a narrower * field and then apply a mask when accessed with a narrower
...@@ -1565,8 +1565,8 @@ static int check_call(struct bpf_verifier_env *env, int func_id, int insn_idx) ...@@ -1565,8 +1565,8 @@ static int check_call(struct bpf_verifier_env *env, int func_id, int insn_idx)
return -EINVAL; return -EINVAL;
} }
if (env->prog->aux->ops->get_func_proto) if (env->prog->aux->vops->get_func_proto)
fn = env->prog->aux->ops->get_func_proto(func_id); fn = env->prog->aux->vops->get_func_proto(func_id);
if (!fn) { if (!fn) {
verbose(env, "unknown func %s#%d\n", func_id_name(func_id), verbose(env, "unknown func %s#%d\n", func_id_name(func_id),
...@@ -4035,7 +4035,7 @@ static struct bpf_prog *bpf_patch_insn_data(struct bpf_verifier_env *env, u32 of ...@@ -4035,7 +4035,7 @@ static struct bpf_prog *bpf_patch_insn_data(struct bpf_verifier_env *env, u32 of
*/ */
static int convert_ctx_accesses(struct bpf_verifier_env *env) static int convert_ctx_accesses(struct bpf_verifier_env *env)
{ {
const struct bpf_verifier_ops *ops = env->prog->aux->ops; const struct bpf_verifier_ops *ops = env->prog->aux->vops;
int i, cnt, size, ctx_field_size, delta = 0; int i, cnt, size, ctx_field_size, delta = 0;
const int insn_cnt = env->prog->len; const int insn_cnt = env->prog->len;
struct bpf_insn insn_buf[16], *insn; struct bpf_insn insn_buf[16], *insn;
...@@ -4236,7 +4236,7 @@ static int fixup_bpf_calls(struct bpf_verifier_env *env) ...@@ -4236,7 +4236,7 @@ static int fixup_bpf_calls(struct bpf_verifier_env *env)
insn = new_prog->insnsi + i + delta; insn = new_prog->insnsi + i + delta;
} }
patch_call_imm: patch_call_imm:
fn = prog->aux->ops->get_func_proto(insn->imm); fn = prog->aux->vops->get_func_proto(insn->imm);
/* all functions that have prototype and verifier allowed /* all functions that have prototype and verifier allowed
* programs to call them, must be real in-kernel functions * programs to call them, must be real in-kernel functions
*/ */
......
...@@ -561,11 +561,14 @@ static bool kprobe_prog_is_valid_access(int off, int size, enum bpf_access_type ...@@ -561,11 +561,14 @@ static bool kprobe_prog_is_valid_access(int off, int size, enum bpf_access_type
return true; return true;
} }
const struct bpf_verifier_ops kprobe_prog_ops = { const struct bpf_verifier_ops kprobe_verifier_ops = {
.get_func_proto = kprobe_prog_func_proto, .get_func_proto = kprobe_prog_func_proto,
.is_valid_access = kprobe_prog_is_valid_access, .is_valid_access = kprobe_prog_is_valid_access,
}; };
const struct bpf_prog_ops kprobe_prog_ops = {
};
BPF_CALL_5(bpf_perf_event_output_tp, void *, tp_buff, struct bpf_map *, map, BPF_CALL_5(bpf_perf_event_output_tp, void *, tp_buff, struct bpf_map *, map,
u64, flags, void *, data, u64, size) u64, flags, void *, data, u64, size)
{ {
...@@ -667,11 +670,14 @@ static bool tp_prog_is_valid_access(int off, int size, enum bpf_access_type type ...@@ -667,11 +670,14 @@ static bool tp_prog_is_valid_access(int off, int size, enum bpf_access_type type
return true; return true;
} }
const struct bpf_verifier_ops tracepoint_prog_ops = { const struct bpf_verifier_ops tracepoint_verifier_ops = {
.get_func_proto = tp_prog_func_proto, .get_func_proto = tp_prog_func_proto,
.is_valid_access = tp_prog_is_valid_access, .is_valid_access = tp_prog_is_valid_access,
}; };
const struct bpf_prog_ops tracepoint_prog_ops = {
};
static bool pe_prog_is_valid_access(int off, int size, enum bpf_access_type type, static bool pe_prog_is_valid_access(int off, int size, enum bpf_access_type type,
struct bpf_insn_access_aux *info) struct bpf_insn_access_aux *info)
{ {
...@@ -727,8 +733,11 @@ static u32 pe_prog_convert_ctx_access(enum bpf_access_type type, ...@@ -727,8 +733,11 @@ static u32 pe_prog_convert_ctx_access(enum bpf_access_type type,
return insn - insn_buf; return insn - insn_buf;
} }
const struct bpf_verifier_ops perf_event_prog_ops = { const struct bpf_verifier_ops perf_event_verifier_ops = {
.get_func_proto = tp_prog_func_proto, .get_func_proto = tp_prog_func_proto,
.is_valid_access = pe_prog_is_valid_access, .is_valid_access = pe_prog_is_valid_access,
.convert_ctx_access = pe_prog_convert_ctx_access, .convert_ctx_access = pe_prog_convert_ctx_access,
}; };
const struct bpf_prog_ops perf_event_prog_ops = {
};
...@@ -4395,68 +4395,95 @@ static u32 sock_ops_convert_ctx_access(enum bpf_access_type type, ...@@ -4395,68 +4395,95 @@ static u32 sock_ops_convert_ctx_access(enum bpf_access_type type,
return insn - insn_buf; return insn - insn_buf;
} }
const struct bpf_verifier_ops sk_filter_prog_ops = { const struct bpf_verifier_ops sk_filter_verifier_ops = {
.get_func_proto = sk_filter_func_proto, .get_func_proto = sk_filter_func_proto,
.is_valid_access = sk_filter_is_valid_access, .is_valid_access = sk_filter_is_valid_access,
.convert_ctx_access = bpf_convert_ctx_access, .convert_ctx_access = bpf_convert_ctx_access,
}; };
const struct bpf_verifier_ops tc_cls_act_prog_ops = { const struct bpf_prog_ops sk_filter_prog_ops = {
};
const struct bpf_verifier_ops tc_cls_act_verifier_ops = {
.get_func_proto = tc_cls_act_func_proto, .get_func_proto = tc_cls_act_func_proto,
.is_valid_access = tc_cls_act_is_valid_access, .is_valid_access = tc_cls_act_is_valid_access,
.convert_ctx_access = tc_cls_act_convert_ctx_access, .convert_ctx_access = tc_cls_act_convert_ctx_access,
.gen_prologue = tc_cls_act_prologue, .gen_prologue = tc_cls_act_prologue,
};
const struct bpf_prog_ops tc_cls_act_prog_ops = {
.test_run = bpf_prog_test_run_skb, .test_run = bpf_prog_test_run_skb,
}; };
const struct bpf_verifier_ops xdp_prog_ops = { const struct bpf_verifier_ops xdp_verifier_ops = {
.get_func_proto = xdp_func_proto, .get_func_proto = xdp_func_proto,
.is_valid_access = xdp_is_valid_access, .is_valid_access = xdp_is_valid_access,
.convert_ctx_access = xdp_convert_ctx_access, .convert_ctx_access = xdp_convert_ctx_access,
};
const struct bpf_prog_ops xdp_prog_ops = {
.test_run = bpf_prog_test_run_xdp, .test_run = bpf_prog_test_run_xdp,
}; };
const struct bpf_verifier_ops cg_skb_prog_ops = { const struct bpf_verifier_ops cg_skb_verifier_ops = {
.get_func_proto = sk_filter_func_proto, .get_func_proto = sk_filter_func_proto,
.is_valid_access = sk_filter_is_valid_access, .is_valid_access = sk_filter_is_valid_access,
.convert_ctx_access = bpf_convert_ctx_access, .convert_ctx_access = bpf_convert_ctx_access,
};
const struct bpf_prog_ops cg_skb_prog_ops = {
.test_run = bpf_prog_test_run_skb, .test_run = bpf_prog_test_run_skb,
}; };
const struct bpf_verifier_ops lwt_inout_prog_ops = { const struct bpf_verifier_ops lwt_inout_verifier_ops = {
.get_func_proto = lwt_inout_func_proto, .get_func_proto = lwt_inout_func_proto,
.is_valid_access = lwt_is_valid_access, .is_valid_access = lwt_is_valid_access,
.convert_ctx_access = bpf_convert_ctx_access, .convert_ctx_access = bpf_convert_ctx_access,
};
const struct bpf_prog_ops lwt_inout_prog_ops = {
.test_run = bpf_prog_test_run_skb, .test_run = bpf_prog_test_run_skb,
}; };
const struct bpf_verifier_ops lwt_xmit_prog_ops = { const struct bpf_verifier_ops lwt_xmit_verifier_ops = {
.get_func_proto = lwt_xmit_func_proto, .get_func_proto = lwt_xmit_func_proto,
.is_valid_access = lwt_is_valid_access, .is_valid_access = lwt_is_valid_access,
.convert_ctx_access = bpf_convert_ctx_access, .convert_ctx_access = bpf_convert_ctx_access,
.gen_prologue = tc_cls_act_prologue, .gen_prologue = tc_cls_act_prologue,
};
const struct bpf_prog_ops lwt_xmit_prog_ops = {
.test_run = bpf_prog_test_run_skb, .test_run = bpf_prog_test_run_skb,
}; };
const struct bpf_verifier_ops cg_sock_prog_ops = { const struct bpf_verifier_ops cg_sock_verifier_ops = {
.get_func_proto = sock_filter_func_proto, .get_func_proto = sock_filter_func_proto,
.is_valid_access = sock_filter_is_valid_access, .is_valid_access = sock_filter_is_valid_access,
.convert_ctx_access = sock_filter_convert_ctx_access, .convert_ctx_access = sock_filter_convert_ctx_access,
}; };
const struct bpf_verifier_ops sock_ops_prog_ops = { const struct bpf_prog_ops cg_sock_prog_ops = {
};
const struct bpf_verifier_ops sock_ops_verifier_ops = {
.get_func_proto = sock_ops_func_proto, .get_func_proto = sock_ops_func_proto,
.is_valid_access = sock_ops_is_valid_access, .is_valid_access = sock_ops_is_valid_access,
.convert_ctx_access = sock_ops_convert_ctx_access, .convert_ctx_access = sock_ops_convert_ctx_access,
}; };
const struct bpf_verifier_ops sk_skb_prog_ops = { const struct bpf_prog_ops sock_ops_prog_ops = {
};
const struct bpf_verifier_ops sk_skb_verifier_ops = {
.get_func_proto = sk_skb_func_proto, .get_func_proto = sk_skb_func_proto,
.is_valid_access = sk_skb_is_valid_access, .is_valid_access = sk_skb_is_valid_access,
.convert_ctx_access = bpf_convert_ctx_access, .convert_ctx_access = bpf_convert_ctx_access,
.gen_prologue = sk_skb_prologue, .gen_prologue = sk_skb_prologue,
}; };
const struct bpf_prog_ops sk_skb_prog_ops = {
};
int sk_detach_filter(struct sock *sk) int sk_detach_filter(struct sock *sk)
{ {
int ret = -ENOENT; int ret = -ENOENT;
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment