Commit e2ae4ca2 authored by Jakub Kicinski's avatar Jakub Kicinski Committed by Alexei Starovoitov

bpf: verifier: hard wire branches to dead code

Loading programs with dead code becomes more and more
common, as people begin to patch constants at load time.
Turn conditional jumps to unconditional ones, to avoid
potential branch misprediction penalty.

This optimization is enabled for privileged users only.

For branches which just fall through we could just mark
them as not seen and have dead code removal take care of
them, but that seems less clean.

v0.2:
 - don't call capable(CAP_SYS_ADMIN) twice (Jiong).
v3:
 - fix GCC warning;
Signed-off-by: default avatarJakub Kicinski <jakub.kicinski@netronome.com>
Acked-by: default avatarYonghong Song <yhs@fb.com>
Signed-off-by: default avatarAlexei Starovoitov <ast@kernel.org>
parent 2cbd95a5
...@@ -6458,6 +6458,40 @@ static void sanitize_dead_code(struct bpf_verifier_env *env) ...@@ -6458,6 +6458,40 @@ static void sanitize_dead_code(struct bpf_verifier_env *env)
} }
} }
static bool insn_is_cond_jump(u8 code)
{
u8 op;
if (BPF_CLASS(code) != BPF_JMP)
return false;
op = BPF_OP(code);
return op != BPF_JA && op != BPF_EXIT && op != BPF_CALL;
}
static void opt_hard_wire_dead_code_branches(struct bpf_verifier_env *env)
{
struct bpf_insn_aux_data *aux_data = env->insn_aux_data;
struct bpf_insn ja = BPF_JMP_IMM(BPF_JA, 0, 0, 0);
struct bpf_insn *insn = env->prog->insnsi;
const int insn_cnt = env->prog->len;
int i;
for (i = 0; i < insn_cnt; i++, insn++) {
if (!insn_is_cond_jump(insn->code))
continue;
if (!aux_data[i + 1].seen)
ja.off = insn->off;
else if (!aux_data[i + 1 + insn->off].seen)
ja.off = 0;
else
continue;
memcpy(insn, &ja, sizeof(ja));
}
}
/* convert load instructions that access fields of a context type into a /* convert load instructions that access fields of a context type into a
* sequence of instructions that access fields of the underlying structure: * sequence of instructions that access fields of the underlying structure:
* struct __sk_buff -> struct sk_buff * struct __sk_buff -> struct sk_buff
...@@ -7149,6 +7183,7 @@ int bpf_check(struct bpf_prog **prog, union bpf_attr *attr, ...@@ -7149,6 +7183,7 @@ int bpf_check(struct bpf_prog **prog, union bpf_attr *attr,
struct bpf_verifier_env *env; struct bpf_verifier_env *env;
struct bpf_verifier_log *log; struct bpf_verifier_log *log;
int ret = -EINVAL; int ret = -EINVAL;
bool is_priv;
/* no program is valid */ /* no program is valid */
if (ARRAY_SIZE(bpf_verifier_ops) == 0) if (ARRAY_SIZE(bpf_verifier_ops) == 0)
...@@ -7195,6 +7230,9 @@ int bpf_check(struct bpf_prog **prog, union bpf_attr *attr, ...@@ -7195,6 +7230,9 @@ int bpf_check(struct bpf_prog **prog, union bpf_attr *attr,
if (attr->prog_flags & BPF_F_ANY_ALIGNMENT) if (attr->prog_flags & BPF_F_ANY_ALIGNMENT)
env->strict_alignment = false; env->strict_alignment = false;
is_priv = capable(CAP_SYS_ADMIN);
env->allow_ptr_leaks = is_priv;
ret = replace_map_fd_with_map_ptr(env); ret = replace_map_fd_with_map_ptr(env);
if (ret < 0) if (ret < 0)
goto skip_full_check; goto skip_full_check;
...@@ -7212,8 +7250,6 @@ int bpf_check(struct bpf_prog **prog, union bpf_attr *attr, ...@@ -7212,8 +7250,6 @@ int bpf_check(struct bpf_prog **prog, union bpf_attr *attr,
if (!env->explored_states) if (!env->explored_states)
goto skip_full_check; goto skip_full_check;
env->allow_ptr_leaks = capable(CAP_SYS_ADMIN);
ret = check_subprogs(env); ret = check_subprogs(env);
if (ret < 0) if (ret < 0)
goto skip_full_check; goto skip_full_check;
...@@ -7243,6 +7279,11 @@ int bpf_check(struct bpf_prog **prog, union bpf_attr *attr, ...@@ -7243,6 +7279,11 @@ int bpf_check(struct bpf_prog **prog, union bpf_attr *attr,
ret = check_max_stack_depth(env); ret = check_max_stack_depth(env);
/* instruction rewrites happen after this point */ /* instruction rewrites happen after this point */
if (is_priv) {
if (ret == 0)
opt_hard_wire_dead_code_branches(env);
}
if (ret == 0) if (ret == 0)
sanitize_dead_code(env); sanitize_dead_code(env);
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment