Skip to content

L
linux
Commit 7800a3d5 authored by Daniel Borkmann's avatar Daniel Borkmann
Browse files
Options

Merge branch 'bpf-riscv-jit-improvements' 

Björn Töpel says:

====================
This series contain one non-critical fix, support for far jumps, and
some optimizations for the BPF JIT.

Previously, the JIT only supported 12b branch targets for conditional
branches, and 21b for unconditional branches. Starting with this
series, 32b branching is supported.

As part of supporting far jumps, branch relaxation was introduced. The
idea is to start with a pessimistic jump (e.g. auipc/jalr) and for
each pass the JIT will have an opportunity to pick a better
instruction (e.g. jal) and shrink the image. Instead of two passes,
the JIT requires more passes. It typically converges after 3 passes.

The optimizations mentioned in the subject are for calls and tail
calls. In the tail call generation we can save one instruction by
using the offset in jalr. Calls are optimized by doing (auipc)/jal(r)
relative jumps instead of loading the entire absolute address and
doing jalr. This required that the JIT image allocator was made RISC-V
specific, so we can ensure that the JIT image and the kernel text are
in range (32b).

The last two patches of the series is not critical to the series, but
are two UAPI build issues for BPF events. A closer look from the
RV-folks would be much appreciated.

The test_bpf.ko module, selftests/bpf/test_verifier and
selftests/seccomp/seccomp_bpf pass all tests.

RISC-V is still missing proper kprobe and tracepoint support, so a lot
of BPF selftests cannot be run.

v1->v2: [1]
 * Removed unused function parameter from emit_branch()
 * Added patch to support far branch in tail call emit

[1] https://lore.kernel.org/bpf/20191209173136.29615-1-bjorn.topel@gmail.com/
====================
Signed-off-by: default avatarDaniel Borkmann <daniel@iogearbox.net>
parents a352a824 34bfc10a
Hide whitespace changes
Inline Side-by-side
Showing with 312 additions and 238 deletions
arch/riscv/include/asm/perf_event.h
View file @ 7800a3d5
......@@ -82,4 +82,8 @@ struct riscv_pmu {
int irq;
};
#ifdef CONFIG_PERF_EVENTS
#define perf_arch_bpf_user_pt_regs(regs) (struct user_regs_struct *)regs
#endif
#endif /* _ASM_RISCV_PERF_EVENT_H */
arch/riscv/include/asm/pgtable.h
View file @ 7800a3d5
......@@ -404,6 +404,10 @@ static inline int ptep_clear_flush_young(struct vm_area_struct *vma,
#define VMALLOC_END (PAGE_OFFSET - 1)
#define VMALLOC_START (PAGE_OFFSET - VMALLOC_SIZE)
#define BPF_JIT_REGION_SIZE (SZ_128M)
#define BPF_JIT_REGION_START (PAGE_OFFSET - BPF_JIT_REGION_SIZE)
#define BPF_JIT_REGION_END (VMALLOC_END)
/*
* Roughly size the vmemmap space to be large enough to fit enough
* struct pages to map half the virtual address space. Then
......
arch/riscv/include/uapi/asm/bpf_perf_event.h 0 → 100644
View file @ 7800a3d5
/* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */
#ifndef _UAPI__ASM_BPF_PERF_EVENT_H__
#define _UAPI__ASM_BPF_PERF_EVENT_H__
#include <asm/ptrace.h>
typedef struct user_regs_struct bpf_user_pt_regs_t;
#endif /* _UAPI__ASM_BPF_PERF_EVENT_H__ */
arch/riscv/net/bpf_jit_comp.c
View file @ 7800a3d5
......@@ -120,6 +120,11 @@ static bool seen_reg(int reg, struct rv_jit_context *ctx)
return false;
}
static void mark_fp(struct rv_jit_context *ctx)
{
__set_bit(RV_CTX_F_SEEN_S5, &ctx->flags);
}
static void mark_call(struct rv_jit_context *ctx)
{
__set_bit(RV_CTX_F_SEEN_CALL, &ctx->flags);
......@@ -456,6 +461,11 @@ static u32 rv_amoadd_d(u8 rd, u8 rs2, u8 rs1, u8 aq, u8 rl)
return rv_amo_insn(0, aq, rl, rs2, rs1, 3, rd, 0x2f);
}
static u32 rv_auipc(u8 rd, u32 imm31_12)
{
return rv_u_insn(imm31_12, rd, 0x17);
}
static bool is_12b_int(s64 val)
{
return -(1 << 11) <= val && val < (1 << 11);
......@@ -479,27 +489,7 @@ static bool is_32b_int(s64 val)
static int is_12b_check(int off, int insn)
{
if (!is_12b_int(off)) {
pr_err("bpf-jit: insn=%d offset=%d not supported yet!\n",
insn, (int)off);
return -1;
}
return 0;
}
static int is_13b_check(int off, int insn)
{
if (!is_13b_int(off)) {
pr_err("bpf-jit: insn=%d offset=%d not supported yet!\n",
insn, (int)off);
return -1;
}
return 0;
}
static int is_21b_check(int off, int insn)
{
if (!is_21b_int(off)) {
pr_err("bpf-jit: insn=%d offset=%d not supported yet!\n",
pr_err("bpf-jit: insn=%d 12b < offset=%d not supported yet!\n",
insn, (int)off);
return -1;
}
......@@ -545,10 +535,13 @@ static void emit_imm(u8 rd, s64 val, struct rv_jit_context *ctx)
emit(rv_addi(rd, rd, lower), ctx);
}
static int rv_offset(int bpf_to, int bpf_from, struct rv_jit_context *ctx)
static int rv_offset(int insn, int off, struct rv_jit_context *ctx)
{
int from = ctx->offset[bpf_from] - 1, to = ctx->offset[bpf_to];
int from, to;
off++; /* BPF branch is from PC+1, RV is from PC */
from = (insn > 0) ? ctx->offset[insn - 1] : 0;
to = (insn + off > 0) ? ctx->offset[insn + off - 1] : 0;
return (to - from) << 2;
}
......@@ -559,7 +552,7 @@ static int epilogue_offset(struct rv_jit_context *ctx)
return (to - from) << 2;
}
static void __build_epilogue(u8 reg, struct rv_jit_context *ctx)
static void __build_epilogue(bool is_tail_call, struct rv_jit_context *ctx)
{
int stack_adjust = ctx->stack_size, store_offset = stack_adjust - 8;
......@@ -596,8 +589,114 @@ static void __build_epilogue(u8 reg, struct rv_jit_context *ctx)
emit(rv_addi(RV_REG_SP, RV_REG_SP, stack_adjust), ctx);
/* Set return value. */
emit(rv_addi(RV_REG_A0, RV_REG_A5, 0), ctx);
emit(rv_jalr(RV_REG_ZERO, reg, 0), ctx);
if (!is_tail_call)
emit(rv_addi(RV_REG_A0, RV_REG_A5, 0), ctx);
emit(rv_jalr(RV_REG_ZERO, is_tail_call ? RV_REG_T3 : RV_REG_RA,
is_tail_call ? 4 : 0), /* skip TCC init */
ctx);
}
/* return -1 or inverted cond */
static int invert_bpf_cond(u8 cond)
{
switch (cond) {
case BPF_JEQ:
return BPF_JNE;
case BPF_JGT:
return BPF_JLE;
case BPF_JLT:
return BPF_JGE;
case BPF_JGE:
return BPF_JLT;
case BPF_JLE:
return BPF_JGT;
case BPF_JNE:
return BPF_JEQ;
case BPF_JSGT:
return BPF_JSLE;
case BPF_JSLT:
return BPF_JSGE;
case BPF_JSGE:
return BPF_JSLT;
case BPF_JSLE:
return BPF_JSGT;
}
return -1;
}
static void emit_bcc(u8 cond, u8 rd, u8 rs, int rvoff,
struct rv_jit_context *ctx)
{
switch (cond) {
case BPF_JEQ:
emit(rv_beq(rd, rs, rvoff >> 1), ctx);
return;
case BPF_JGT:
emit(rv_bltu(rs, rd, rvoff >> 1), ctx);
return;
case BPF_JLT:
emit(rv_bltu(rd, rs, rvoff >> 1), ctx);
return;
case BPF_JGE:
emit(rv_bgeu(rd, rs, rvoff >> 1), ctx);
return;
case BPF_JLE:
emit(rv_bgeu(rs, rd, rvoff >> 1), ctx);
return;
case BPF_JNE:
emit(rv_bne(rd, rs, rvoff >> 1), ctx);
return;
case BPF_JSGT:
emit(rv_blt(rs, rd, rvoff >> 1), ctx);
return;
case BPF_JSLT:
emit(rv_blt(rd, rs, rvoff >> 1), ctx);
return;
case BPF_JSGE:
emit(rv_bge(rd, rs, rvoff >> 1), ctx);
return;
case BPF_JSLE:
emit(rv_bge(rs, rd, rvoff >> 1), ctx);
}
}
static void emit_branch(u8 cond, u8 rd, u8 rs, int rvoff,
struct rv_jit_context *ctx)
{
s64 upper, lower;
if (is_13b_int(rvoff)) {
emit_bcc(cond, rd, rs, rvoff, ctx);
return;
}
/* Adjust for jal */
rvoff -= 4;
/* Transform, e.g.:
* bne rd,rs,foo
* to
* beq rd,rs,<.L1>
* (auipc foo)
* jal(r) foo
* .L1
*/
cond = invert_bpf_cond(cond);
if (is_21b_int(rvoff)) {
emit_bcc(cond, rd, rs, 8, ctx);
emit(rv_jal(RV_REG_ZERO, rvoff >> 1), ctx);
return;
}
/* 32b No need for an additional rvoff adjustment, since we
* get that from the auipc at PC', where PC = PC' + 4.
*/
upper = (rvoff + (1 << 11)) >> 12;
lower = rvoff & 0xfff;
emit_bcc(cond, rd, rs, 12, ctx);
emit(rv_auipc(RV_REG_T1, upper), ctx);
emit(rv_jalr(RV_REG_ZERO, RV_REG_T1, lower), ctx);
}
static void emit_zext_32(u8 reg, struct rv_jit_context *ctx)
......@@ -627,18 +726,14 @@ static int emit_bpf_tail_call(int insn, struct rv_jit_context *ctx)
return -1;
emit(rv_lwu(RV_REG_T1, off, RV_REG_A1), ctx);
off = (tc_ninsn - (ctx->ninsns - start_insn)) << 2;
if (is_13b_check(off, insn))
return -1;
emit(rv_bgeu(RV_REG_A2, RV_REG_T1, off >> 1), ctx);
emit_branch(BPF_JGE, RV_REG_A2, RV_REG_T1, off, ctx);
/* if (--TCC < 0)
* goto out;
*/
emit(rv_addi(RV_REG_T1, tcc, -1), ctx);
off = (tc_ninsn - (ctx->ninsns - start_insn)) << 2;
if (is_13b_check(off, insn))
return -1;
emit(rv_blt(RV_REG_T1, RV_REG_ZERO, off >> 1), ctx);
emit_branch(BPF_JSLT, RV_REG_T1, RV_REG_ZERO, off, ctx);
/* prog = array->ptrs[index];
* if (!prog)
......@@ -651,18 +746,15 @@ static int emit_bpf_tail_call(int insn, struct rv_jit_context *ctx)
return -1;
emit(rv_ld(RV_REG_T2, off, RV_REG_T2), ctx);
off = (tc_ninsn - (ctx->ninsns - start_insn)) << 2;
if (is_13b_check(off, insn))
return -1;
emit(rv_beq(RV_REG_T2, RV_REG_ZERO, off >> 1), ctx);
emit_branch(BPF_JEQ, RV_REG_T2, RV_REG_ZERO, off, ctx);
/* goto *(prog->bpf_func + 4); */
off = offsetof(struct bpf_prog, bpf_func);
if (is_12b_check(off, insn))
return -1;
emit(rv_ld(RV_REG_T3, off, RV_REG_T2), ctx);
emit(rv_addi(RV_REG_T3, RV_REG_T3, 4), ctx);
emit(rv_addi(RV_REG_TCC, RV_REG_T1, 0), ctx);
__build_epilogue(RV_REG_T3, ctx);
__build_epilogue(true, ctx);
return 0;
}
......@@ -687,13 +779,6 @@ static void init_regs(u8 *rd, u8 *rs, const struct bpf_insn *insn,
*rs = bpf_to_rv_reg(insn->src_reg, ctx);
}
static int rv_offset_check(int *rvoff, s16 off, int insn,
struct rv_jit_context *ctx)
{
*rvoff = rv_offset(insn + off, insn, ctx);
return is_13b_check(*rvoff, insn);
}
static void emit_zext_32_rd_rs(u8 *rd, u8 *rs, struct rv_jit_context *ctx)
{
emit(rv_addi(RV_REG_T2, *rd, 0), ctx);
......@@ -726,13 +811,57 @@ static void emit_sext_32_rd(u8 *rd, struct rv_jit_context *ctx)
*rd = RV_REG_T2;
}
static void emit_jump_and_link(u8 rd, s64 rvoff, bool force_jalr,
struct rv_jit_context *ctx)
{
s64 upper, lower;
if (rvoff && is_21b_int(rvoff) && !force_jalr) {
emit(rv_jal(rd, rvoff >> 1), ctx);
return;
}
upper = (rvoff + (1 << 11)) >> 12;
lower = rvoff & 0xfff;
emit(rv_auipc(RV_REG_T1, upper), ctx);
emit(rv_jalr(rd, RV_REG_T1, lower), ctx);
}
static bool is_signed_bpf_cond(u8 cond)
{
return cond == BPF_JSGT || cond == BPF_JSLT ||
cond == BPF_JSGE || cond == BPF_JSLE;
}
static int emit_call(bool fixed, u64 addr, struct rv_jit_context *ctx)
{
s64 off = 0;
u64 ip;
u8 rd;
if (addr && ctx->insns) {
ip = (u64)(long)(ctx->insns + ctx->ninsns);
off = addr - ip;
if (!is_32b_int(off)) {
pr_err("bpf-jit: target call addr %pK is out of range\n",
(void *)addr);
return -ERANGE;
}
}
emit_jump_and_link(RV_REG_RA, off, !fixed, ctx);
rd = bpf_to_rv_reg(BPF_REG_0, ctx);
emit(rv_addi(rd, RV_REG_A0, 0), ctx);
return 0;
}
static int emit_insn(const struct bpf_insn *insn, struct rv_jit_context *ctx,
bool extra_pass)
{
bool is64 = BPF_CLASS(insn->code) == BPF_ALU64 ||
BPF_CLASS(insn->code) == BPF_JMP;
int s, e, rvoff, i = insn - ctx->prog->insnsi;
struct bpf_prog_aux *aux = ctx->prog->aux;
int rvoff, i = insn - ctx->prog->insnsi;
u8 rd = -1, rs = -1, code = insn->code;
s16 off = insn->off;
s32 imm = insn->imm;
......@@ -1000,214 +1129,110 @@ static int emit_insn(const struct bpf_insn *insn, struct rv_jit_context *ctx,
/* JUMP off */
case BPF_JMP | BPF_JA:
rvoff = rv_offset(i + off, i, ctx);
if (!is_21b_int(rvoff)) {
pr_err("bpf-jit: insn=%d offset=%d not supported yet!\n",
i, rvoff);
return -1;
}
emit(rv_jal(RV_REG_ZERO, rvoff >> 1), ctx);
rvoff = rv_offset(i, off, ctx);
emit_jump_and_link(RV_REG_ZERO, rvoff, false, ctx);
break;
/* IF (dst COND src) JUMP off */
case BPF_JMP | BPF_JEQ | BPF_X:
case BPF_JMP32 | BPF_JEQ | BPF_X:
if (rv_offset_check(&rvoff, off, i, ctx))
return -1;
if (!is64)
emit_zext_32_rd_rs(&rd, &rs, ctx);
emit(rv_beq(rd, rs, rvoff >> 1), ctx);
break;
case BPF_JMP | BPF_JGT | BPF_X:
case BPF_JMP32 | BPF_JGT | BPF_X:
if (rv_offset_check(&rvoff, off, i, ctx))
return -1;
if (!is64)
emit_zext_32_rd_rs(&rd, &rs, ctx);
emit(rv_bltu(rs, rd, rvoff >> 1), ctx);
break;
case BPF_JMP | BPF_JLT | BPF_X:
case BPF_JMP32 | BPF_JLT | BPF_X:
if (rv_offset_check(&rvoff, off, i, ctx))
return -1;
if (!is64)
emit_zext_32_rd_rs(&rd, &rs, ctx);
emit(rv_bltu(rd, rs, rvoff >> 1), ctx);
break;
case BPF_JMP | BPF_JGE | BPF_X:
case BPF_JMP32 | BPF_JGE | BPF_X:
if (rv_offset_check(&rvoff, off, i, ctx))
return -1;
if (!is64)
emit_zext_32_rd_rs(&rd, &rs, ctx);
emit(rv_bgeu(rd, rs, rvoff >> 1), ctx);
break;
case BPF_JMP | BPF_JLE | BPF_X:
case BPF_JMP32 | BPF_JLE | BPF_X:
if (rv_offset_check(&rvoff, off, i, ctx))
return -1;
if (!is64)
emit_zext_32_rd_rs(&rd, &rs, ctx);
emit(rv_bgeu(rs, rd, rvoff >> 1), ctx);
break;
case BPF_JMP | BPF_JNE | BPF_X:
case BPF_JMP32 | BPF_JNE | BPF_X:
if (rv_offset_check(&rvoff, off, i, ctx))
return -1;
if (!is64)
emit_zext_32_rd_rs(&rd, &rs, ctx);
emit(rv_bne(rd, rs, rvoff >> 1), ctx);
break;
case BPF_JMP | BPF_JSGT | BPF_X:
case BPF_JMP32 | BPF_JSGT | BPF_X:
if (rv_offset_check(&rvoff, off, i, ctx))
return -1;
if (!is64)
emit_sext_32_rd_rs(&rd, &rs, ctx);
emit(rv_blt(rs, rd, rvoff >> 1), ctx);
break;
case BPF_JMP | BPF_JSLT | BPF_X:
case BPF_JMP32 | BPF_JSLT | BPF_X:
if (rv_offset_check(&rvoff, off, i, ctx))
return -1;
if (!is64)
emit_sext_32_rd_rs(&rd, &rs, ctx);
emit(rv_blt(rd, rs, rvoff >> 1), ctx);
break;
case BPF_JMP | BPF_JSGE | BPF_X:
case BPF_JMP32 | BPF_JSGE | BPF_X:
if (rv_offset_check(&rvoff, off, i, ctx))
return -1;
if (!is64)
emit_sext_32_rd_rs(&rd, &rs, ctx);
emit(rv_bge(rd, rs, rvoff >> 1), ctx);
break;
case BPF_JMP | BPF_JSLE | BPF_X:
case BPF_JMP32 | BPF_JSLE | BPF_X:
if (rv_offset_check(&rvoff, off, i, ctx))
return -1;
if (!is64)
emit_sext_32_rd_rs(&rd, &rs, ctx);
emit(rv_bge(rs, rd, rvoff >> 1), ctx);
break;
case BPF_JMP | BPF_JSET | BPF_X:
case BPF_JMP32 | BPF_JSET | BPF_X:
if (rv_offset_check(&rvoff, off, i, ctx))
return -1;
if (!is64)
emit_zext_32_rd_rs(&rd, &rs, ctx);
emit(rv_and(RV_REG_T1, rd, rs), ctx);
emit(rv_bne(RV_REG_T1, RV_REG_ZERO, rvoff >> 1), ctx);
rvoff = rv_offset(i, off, ctx);
if (!is64) {
s = ctx->ninsns;
if (is_signed_bpf_cond(BPF_OP(code)))
emit_sext_32_rd_rs(&rd, &rs, ctx);
else
emit_zext_32_rd_rs(&rd, &rs, ctx);
e = ctx->ninsns;
/* Adjust for extra insns */
rvoff -= (e - s) << 2;
}
if (BPF_OP(code) == BPF_JSET) {
/* Adjust for and */
rvoff -= 4;
emit(rv_and(RV_REG_T1, rd, rs), ctx);
emit_branch(BPF_JNE, RV_REG_T1, RV_REG_ZERO, rvoff,
ctx);
} else {
emit_branch(BPF_OP(code), rd, rs, rvoff, ctx);
}
break;
/* IF (dst COND imm) JUMP off */
case BPF_JMP | BPF_JEQ | BPF_K:
case BPF_JMP32 | BPF_JEQ | BPF_K:
if (rv_offset_check(&rvoff, off, i, ctx))
return -1;
emit_imm(RV_REG_T1, imm, ctx);
if (!is64)
emit_zext_32_rd_t1(&rd, ctx);
emit(rv_beq(rd, RV_REG_T1, rvoff >> 1), ctx);
break;
case BPF_JMP | BPF_JGT | BPF_K:
case BPF_JMP32 | BPF_JGT | BPF_K:
if (rv_offset_check(&rvoff, off, i, ctx))
return -1;
emit_imm(RV_REG_T1, imm, ctx);
if (!is64)
emit_zext_32_rd_t1(&rd, ctx);
emit(rv_bltu(RV_REG_T1, rd, rvoff >> 1), ctx);
break;
case BPF_JMP | BPF_JLT | BPF_K:
case BPF_JMP32 | BPF_JLT | BPF_K:
if (rv_offset_check(&rvoff, off, i, ctx))
return -1;
emit_imm(RV_REG_T1, imm, ctx);
if (!is64)
emit_zext_32_rd_t1(&rd, ctx);
emit(rv_bltu(rd, RV_REG_T1, rvoff >> 1), ctx);
break;
case BPF_JMP | BPF_JGE | BPF_K:
case BPF_JMP32 | BPF_JGE | BPF_K:
if (rv_offset_check(&rvoff, off, i, ctx))
return -1;
emit_imm(RV_REG_T1, imm, ctx);
if (!is64)
emit_zext_32_rd_t1(&rd, ctx);
emit(rv_bgeu(rd, RV_REG_T1, rvoff >> 1), ctx);
break;
case BPF_JMP | BPF_JLE | BPF_K:
case BPF_JMP32 | BPF_JLE | BPF_K:
if (rv_offset_check(&rvoff, off, i, ctx))
return -1;
emit_imm(RV_REG_T1, imm, ctx);
if (!is64)
emit_zext_32_rd_t1(&rd, ctx);
emit(rv_bgeu(RV_REG_T1, rd, rvoff >> 1), ctx);
break;
case BPF_JMP | BPF_JNE | BPF_K:
case BPF_JMP32 | BPF_JNE | BPF_K:
if (rv_offset_check(&rvoff, off, i, ctx))
return -1;
emit_imm(RV_REG_T1, imm, ctx);
if (!is64)
emit_zext_32_rd_t1(&rd, ctx);
emit(rv_bne(rd, RV_REG_T1, rvoff >> 1), ctx);
break;
case BPF_JMP | BPF_JSGT | BPF_K:
case BPF_JMP32 | BPF_JSGT | BPF_K:
if (rv_offset_check(&rvoff, off, i, ctx))
return -1;
emit_imm(RV_REG_T1, imm, ctx);
if (!is64)
emit_sext_32_rd(&rd, ctx);
emit(rv_blt(RV_REG_T1, rd, rvoff >> 1), ctx);
break;
case BPF_JMP | BPF_JSLT | BPF_K:
case BPF_JMP32 | BPF_JSLT | BPF_K:
if (rv_offset_check(&rvoff, off, i, ctx))
return -1;
emit_imm(RV_REG_T1, imm, ctx);
if (!is64)
emit_sext_32_rd(&rd, ctx);
emit(rv_blt(rd, RV_REG_T1, rvoff >> 1), ctx);
break;
case BPF_JMP | BPF_JSGE | BPF_K:
case BPF_JMP32 | BPF_JSGE | BPF_K:
if (rv_offset_check(&rvoff, off, i, ctx))
return -1;
emit_imm(RV_REG_T1, imm, ctx);
if (!is64)
emit_sext_32_rd(&rd, ctx);
emit(rv_bge(rd, RV_REG_T1, rvoff >> 1), ctx);
break;
case BPF_JMP | BPF_JSLE | BPF_K:
case BPF_JMP32 | BPF_JSLE | BPF_K:
if (rv_offset_check(&rvoff, off, i, ctx))
return -1;
emit_imm(RV_REG_T1, imm, ctx);
if (!is64)
emit_sext_32_rd(&rd, ctx);
emit(rv_bge(RV_REG_T1, rd, rvoff >> 1), ctx);
break;
case BPF_JMP | BPF_JSET | BPF_K:
case BPF_JMP32 | BPF_JSET | BPF_K:
if (rv_offset_check(&rvoff, off, i, ctx))
return -1;
rvoff = rv_offset(i, off, ctx);
s = ctx->ninsns;
emit_imm(RV_REG_T1, imm, ctx);
if (!is64)
emit_zext_32_rd_t1(&rd, ctx);
emit(rv_and(RV_REG_T1, rd, RV_REG_T1), ctx);
emit(rv_bne(RV_REG_T1, RV_REG_ZERO, rvoff >> 1), ctx);
if (!is64) {
if (is_signed_bpf_cond(BPF_OP(code)))
emit_sext_32_rd(&rd, ctx);
else
emit_zext_32_rd_t1(&rd, ctx);
}
e = ctx->ninsns;
/* Adjust for extra insns */
rvoff -= (e - s) << 2;
if (BPF_OP(code) == BPF_JSET) {
/* Adjust for and */
rvoff -= 4;
emit(rv_and(RV_REG_T1, rd, RV_REG_T1), ctx);
emit_branch(BPF_JNE, RV_REG_T1, RV_REG_ZERO, rvoff,
ctx);
} else {
emit_branch(BPF_OP(code), rd, RV_REG_T1, rvoff, ctx);
}
break;
/* function call */
case BPF_JMP | BPF_CALL:
{
bool fixed;
int i, ret;
int ret;
u64 addr;
mark_call(ctx);
......@@ -1215,20 +1240,9 @@ static int emit_insn(const struct bpf_insn *insn, struct rv_jit_context *ctx,
&fixed);
if (ret < 0)
return ret;
if (fixed) {
emit_imm(RV_REG_T1, addr, ctx);
} else {
i = ctx->ninsns;
emit_imm(RV_REG_T1, addr, ctx);
for (i = ctx->ninsns - i; i < 8; i++) {
/* nop */
emit(rv_addi(RV_REG_ZERO, RV_REG_ZERO, 0),
ctx);
}
}
emit(rv_jalr(RV_REG_RA, RV_REG_T1, 0), ctx);
rd = bpf_to_rv_reg(BPF_REG_0, ctx);
emit(rv_addi(rd, RV_REG_A0, 0), ctx);
ret = emit_call(fixed, addr, ctx);
if (ret)
return ret;
break;
}
/* tail call */
......@@ -1243,9 +1257,7 @@ static int emit_insn(const struct bpf_insn *insn, struct rv_jit_context *ctx,
break;
rvoff = epilogue_offset(ctx);
if (is_21b_check(rvoff, i))
return -1;
emit(rv_jal(RV_REG_ZERO, rvoff >> 1), ctx);
emit_jump_and_link(RV_REG_ZERO, rvoff, false, ctx);
break;
/* dst = imm64 */
......@@ -1426,6 +1438,10 @@ static void build_prologue(struct rv_jit_context *ctx)
{
int stack_adjust = 0, store_offset, bpf_stack_adjust;
bpf_stack_adjust = round_up(ctx->prog->aux->stack_depth, 16);
if (bpf_stack_adjust)
mark_fp(ctx);
if (seen_reg(RV_REG_RA, ctx))
stack_adjust += 8;
stack_adjust += 8; /* RV_REG_FP */
......@@ -1443,7 +1459,6 @@ static void build_prologue(struct rv_jit_context *ctx)
stack_adjust += 8;
stack_adjust = round_up(stack_adjust, 16);
bpf_stack_adjust = round_up(ctx->prog->aux->stack_depth, 16);
stack_adjust += bpf_stack_adjust;
store_offset = stack_adjust - 8;
......@@ -1502,10 +1517,10 @@ static void build_prologue(struct rv_jit_context *ctx)
static void build_epilogue(struct rv_jit_context *ctx)
{
__build_epilogue(RV_REG_RA, ctx);
__build_epilogue(false, ctx);
}
static int build_body(struct rv_jit_context *ctx, bool extra_pass)
static int build_body(struct rv_jit_context *ctx, bool extra_pass, int *offset)
{
const struct bpf_prog *prog = ctx->prog;
int i;
......@@ -1517,12 +1532,12 @@ static int build_body(struct rv_jit_context *ctx, bool extra_pass)
ret = emit_insn(insn, ctx, extra_pass);
if (ret > 0) {
i++;
if (ctx->insns == NULL)
ctx->offset[i] = ctx->ninsns;
if (offset)
offset[i] = ctx->ninsns;
continue;
}
if (ctx->insns == NULL)
ctx->offset[i] = ctx->ninsns;
if (offset)
offset[i] = ctx->ninsns;
if (ret)
return ret;
}
......@@ -1548,9 +1563,10 @@ struct bpf_prog *bpf_int_jit_compile(struct bpf_prog *prog)
{
bool tmp_blinded = false, extra_pass = false;
struct bpf_prog *tmp, *orig_prog = prog;
int pass = 0, prev_ninsns = 0, i;
struct rv_jit_data *jit_data;
unsigned int image_size = 0;
struct rv_jit_context *ctx;
unsigned int image_size;
if (!prog->jit_requested)
return orig_prog;
......@@ -1587,33 +1603,59 @@ struct bpf_prog *bpf_int_jit_compile(struct bpf_prog *prog)
prog = orig_prog;
goto out_offset;
}
for (i = 0; i < prog->len; i++) {
prev_ninsns += 32;
ctx->offset[i] = prev_ninsns;
}
/* First pass generates the ctx->offset, but does not emit an image. */
if (build_body(ctx, extra_pass)) {
prog = orig_prog;
goto out_offset;
for (i = 0; i < 16; i++) {
pass++;
ctx->ninsns = 0;
if (build_body(ctx, extra_pass, ctx->offset)) {
prog = orig_prog;
goto out_offset;
}
build_prologue(ctx);
ctx->epilogue_offset = ctx->ninsns;
build_epilogue(ctx);
if (ctx->ninsns == prev_ninsns) {
if (jit_data->header)
break;
image_size = sizeof(u32) * ctx->ninsns;
jit_data->header =
bpf_jit_binary_alloc(image_size,
&jit_data->image,
sizeof(u32),
bpf_fill_ill_insns);
if (!jit_data->header) {
prog = orig_prog;
goto out_offset;
}
ctx->insns = (u32 *)jit_data->image;
/* Now, when the image is allocated, the image
* can potentially shrink more (auipc/jalr ->
* jal).
*/
}
prev_ninsns = ctx->ninsns;
}
build_prologue(ctx);
ctx->epilogue_offset = ctx->ninsns;
build_epilogue(ctx);
/* Allocate image, now that we know the size. */
image_size = sizeof(u32) * ctx->ninsns;
jit_data->header = bpf_jit_binary_alloc(image_size, &jit_data->image,
sizeof(u32),
bpf_fill_ill_insns);
if (!jit_data->header) {
if (i == 16) {
pr_err("bpf-jit: image did not converge in <%d passes!\n", i);
bpf_jit_binary_free(jit_data->header);
prog = orig_prog;
goto out_offset;
}
/* Second, real pass, that acutally emits the image. */
ctx->insns = (u32 *)jit_data->image;
skip_init_ctx:
pass++;
ctx->ninsns = 0;
build_prologue(ctx);
if (build_body(ctx, extra_pass)) {
if (build_body(ctx, extra_pass, NULL)) {
bpf_jit_binary_free(jit_data->header);
prog = orig_prog;
goto out_offset;
......@@ -1621,7 +1663,7 @@ struct bpf_prog *bpf_int_jit_compile(struct bpf_prog *prog)
build_epilogue(ctx);
if (bpf_jit_enable > 1)
bpf_jit_dump(prog->len, image_size, 2, ctx->insns);
bpf_jit_dump(prog->len, image_size, pass, ctx->insns);
prog->bpf_func = (void *)ctx->insns;
prog->jited = 1;
......@@ -1641,3 +1683,16 @@ struct bpf_prog *bpf_int_jit_compile(struct bpf_prog *prog)
tmp : orig_prog);
return prog;
}
void *bpf_jit_alloc_exec(unsigned long size)
{
return __vmalloc_node_range(size, PAGE_SIZE, BPF_JIT_REGION_START,
BPF_JIT_REGION_END, GFP_KERNEL,
PAGE_KERNEL_EXEC, 0, NUMA_NO_NODE,
__builtin_return_address(0));
}
void bpf_jit_free_exec(void *addr)
{
return vfree(addr);
}
tools/include/uapi/asm/bpf_perf_event.h
View file @ 7800a3d5
......@@ -2,6 +2,8 @@
#include "../../arch/arm64/include/uapi/asm/bpf_perf_event.h"
#elif defined(__s390__)
#include "../../arch/s390/include/uapi/asm/bpf_perf_event.h"
#elif defined(__riscv)
#include "../../arch/riscv/include/uapi/asm/bpf_perf_event.h"
#else
#include <uapi/asm-generic/bpf_perf_event.h>
#endif
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment
GitLab Nexedi Edition | About GitLab | About Nexedi | 沪ICP备2021021310号-2 | 沪ICP备2021021310号-7