Commit c237ee5e authored by Daniel Borkmann's avatar Daniel Borkmann Committed by David S. Miller

bpf: add bpf_patch_insn_single helper

Move the functionality to patch instructions out of the verifier
code and into the core as the new bpf_patch_insn_single() helper
will be needed later on for blinding as well. No changes in
functionality.
Signed-off-by: default avatarDaniel Borkmann <daniel@iogearbox.net>
Acked-by: default avatarAlexei Starovoitov <ast@kernel.org>
Signed-off-by: default avatarDavid S. Miller <davem@davemloft.net>
parent 93a73d44
...@@ -495,6 +495,9 @@ u64 __bpf_call_base(u64 r1, u64 r2, u64 r3, u64 r4, u64 r5); ...@@ -495,6 +495,9 @@ u64 __bpf_call_base(u64 r1, u64 r2, u64 r3, u64 r4, u64 r5);
void bpf_int_jit_compile(struct bpf_prog *fp); void bpf_int_jit_compile(struct bpf_prog *fp);
bool bpf_helper_changes_skb_data(void *func); bool bpf_helper_changes_skb_data(void *func);
struct bpf_prog *bpf_patch_insn_single(struct bpf_prog *prog, u32 off,
const struct bpf_insn *patch, u32 len);
#ifdef CONFIG_BPF_JIT #ifdef CONFIG_BPF_JIT
extern int bpf_jit_enable; extern int bpf_jit_enable;
......
...@@ -136,6 +136,77 @@ void __bpf_prog_free(struct bpf_prog *fp) ...@@ -136,6 +136,77 @@ void __bpf_prog_free(struct bpf_prog *fp)
vfree(fp); vfree(fp);
} }
static bool bpf_is_jmp_and_has_target(const struct bpf_insn *insn)
{
return BPF_CLASS(insn->code) == BPF_JMP &&
/* Call and Exit are both special jumps with no
* target inside the BPF instruction image.
*/
BPF_OP(insn->code) != BPF_CALL &&
BPF_OP(insn->code) != BPF_EXIT;
}
static void bpf_adj_branches(struct bpf_prog *prog, u32 pos, u32 delta)
{
struct bpf_insn *insn = prog->insnsi;
u32 i, insn_cnt = prog->len;
for (i = 0; i < insn_cnt; i++, insn++) {
if (!bpf_is_jmp_and_has_target(insn))
continue;
/* Adjust offset of jmps if we cross boundaries. */
if (i < pos && i + insn->off + 1 > pos)
insn->off += delta;
else if (i > pos + delta && i + insn->off + 1 <= pos + delta)
insn->off -= delta;
}
}
struct bpf_prog *bpf_patch_insn_single(struct bpf_prog *prog, u32 off,
const struct bpf_insn *patch, u32 len)
{
u32 insn_adj_cnt, insn_rest, insn_delta = len - 1;
struct bpf_prog *prog_adj;
/* Since our patchlet doesn't expand the image, we're done. */
if (insn_delta == 0) {
memcpy(prog->insnsi + off, patch, sizeof(*patch));
return prog;
}
insn_adj_cnt = prog->len + insn_delta;
/* Several new instructions need to be inserted. Make room
* for them. Likely, there's no need for a new allocation as
* last page could have large enough tailroom.
*/
prog_adj = bpf_prog_realloc(prog, bpf_prog_size(insn_adj_cnt),
GFP_USER);
if (!prog_adj)
return NULL;
prog_adj->len = insn_adj_cnt;
/* Patching happens in 3 steps:
*
* 1) Move over tail of insnsi from next instruction onwards,
* so we can patch the single target insn with one or more
* new ones (patching is always from 1 to n insns, n > 0).
* 2) Inject new instructions at the target location.
* 3) Adjust branch offsets if necessary.
*/
insn_rest = insn_adj_cnt - off - len;
memmove(prog_adj->insnsi + off + len, prog_adj->insnsi + off + 1,
sizeof(*patch) * insn_rest);
memcpy(prog_adj->insnsi + off, patch, sizeof(*patch) * len);
bpf_adj_branches(prog_adj, off, insn_delta);
return prog_adj;
}
#ifdef CONFIG_BPF_JIT #ifdef CONFIG_BPF_JIT
struct bpf_binary_header * struct bpf_binary_header *
bpf_jit_binary_alloc(unsigned int proglen, u8 **image_ptr, bpf_jit_binary_alloc(unsigned int proglen, u8 **image_ptr,
......
...@@ -2587,26 +2587,6 @@ static void convert_pseudo_ld_imm64(struct verifier_env *env) ...@@ -2587,26 +2587,6 @@ static void convert_pseudo_ld_imm64(struct verifier_env *env)
insn->src_reg = 0; insn->src_reg = 0;
} }
static void adjust_branches(struct bpf_prog *prog, int pos, int delta)
{
struct bpf_insn *insn = prog->insnsi;
int insn_cnt = prog->len;
int i;
for (i = 0; i < insn_cnt; i++, insn++) {
if (BPF_CLASS(insn->code) != BPF_JMP ||
BPF_OP(insn->code) == BPF_CALL ||
BPF_OP(insn->code) == BPF_EXIT)
continue;
/* adjust offset of jmps if necessary */
if (i < pos && i + insn->off + 1 > pos)
insn->off += delta;
else if (i > pos + delta && i + insn->off + 1 <= pos + delta)
insn->off -= delta;
}
}
/* convert load instructions that access fields of 'struct __sk_buff' /* convert load instructions that access fields of 'struct __sk_buff'
* into sequence of instructions that access fields of 'struct sk_buff' * into sequence of instructions that access fields of 'struct sk_buff'
*/ */
...@@ -2616,14 +2596,15 @@ static int convert_ctx_accesses(struct verifier_env *env) ...@@ -2616,14 +2596,15 @@ static int convert_ctx_accesses(struct verifier_env *env)
int insn_cnt = env->prog->len; int insn_cnt = env->prog->len;
struct bpf_insn insn_buf[16]; struct bpf_insn insn_buf[16];
struct bpf_prog *new_prog; struct bpf_prog *new_prog;
u32 cnt;
int i;
enum bpf_access_type type; enum bpf_access_type type;
int i;
if (!env->prog->aux->ops->convert_ctx_access) if (!env->prog->aux->ops->convert_ctx_access)
return 0; return 0;
for (i = 0; i < insn_cnt; i++, insn++) { for (i = 0; i < insn_cnt; i++, insn++) {
u32 insn_delta, cnt;
if (insn->code == (BPF_LDX | BPF_MEM | BPF_W)) if (insn->code == (BPF_LDX | BPF_MEM | BPF_W))
type = BPF_READ; type = BPF_READ;
else if (insn->code == (BPF_STX | BPF_MEM | BPF_W)) else if (insn->code == (BPF_STX | BPF_MEM | BPF_W))
...@@ -2645,34 +2626,18 @@ static int convert_ctx_accesses(struct verifier_env *env) ...@@ -2645,34 +2626,18 @@ static int convert_ctx_accesses(struct verifier_env *env)
return -EINVAL; return -EINVAL;
} }
if (cnt == 1) { new_prog = bpf_patch_insn_single(env->prog, i, insn_buf, cnt);
memcpy(insn, insn_buf, sizeof(*insn));
continue;
}
/* several new insns need to be inserted. Make room for them */
insn_cnt += cnt - 1;
new_prog = bpf_prog_realloc(env->prog,
bpf_prog_size(insn_cnt),
GFP_USER);
if (!new_prog) if (!new_prog)
return -ENOMEM; return -ENOMEM;
new_prog->len = insn_cnt; insn_delta = cnt - 1;
memmove(new_prog->insnsi + i + cnt, new_prog->insns + i + 1,
sizeof(*insn) * (insn_cnt - i - cnt));
/* copy substitute insns in place of load instruction */
memcpy(new_prog->insnsi + i, insn_buf, sizeof(*insn) * cnt);
/* adjust branches in the whole program */
adjust_branches(new_prog, i, cnt - 1);
/* keep walking new program and skip insns we just inserted */ /* keep walking new program and skip insns we just inserted */
env->prog = new_prog; env->prog = new_prog;
insn = new_prog->insnsi + i + cnt - 1; insn = new_prog->insnsi + i + insn_delta;
i += cnt - 1;
insn_cnt += insn_delta;
i += insn_delta;
} }
return 0; return 0;
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment