Skip to content

L
linux
Commit 9e3b47ab authored by Alexei Starovoitov's avatar Alexei Starovoitov
Browse files
Options

Merge branch 'add-support-cpu-v4-insns-for-rv64' 

Pu Lehui says:

====================
Add support cpu v4 insns for RV64

Add support cpu v4 instructions for RV64. The relevant tests have passed as show bellow:

Summary: 6/166 PASSED, 0 SKIPPED, 0 FAILED

NOTE: ldsx_insn testcase uses fentry and needs to rely on ftrace direct call [0].
[0] https://lore.kernel.org/all/20230627111612.761164-1-suagrfillet@gmail.com/

v2:
- Use temporary reg to avoid clobbering the source reg in movs_8/16 insns. (Björn)
- Add Acked-by

v1:
https://lore.kernel.org/bpf/20230823231059.3363698-1-pulehui@huaweicloud.com
====================
Tested-by: default avatarBjörn Töpel <bjorn@rivosinc.com>
Link: https://lore.kernel.org/r/20230824095001.3408573-1-pulehui@huaweicloud.comSigned-off-by: default avatarAlexei Starovoitov <ast@kernel.org>
parents 1b580c9b 0209fd51
Show whitespace changes
Inline Side-by-side
Showing with 122 additions and 28 deletions
arch/riscv/net/bpf_jit.h
View file @ 9e3b47ab
......@@ -431,11 +431,21 @@ static inline u32 rv_mulhu(u8 rd, u8 rs1, u8 rs2)
return rv_r_insn(1, rs2, rs1, 3, rd, 0x33);
}
static inline u32 rv_div(u8 rd, u8 rs1, u8 rs2)
{
return rv_r_insn(1, rs2, rs1, 4, rd, 0x33);
}
static inline u32 rv_divu(u8 rd, u8 rs1, u8 rs2)
{
return rv_r_insn(1, rs2, rs1, 5, rd, 0x33);
}
static inline u32 rv_rem(u8 rd, u8 rs1, u8 rs2)
{
return rv_r_insn(1, rs2, rs1, 6, rd, 0x33);
}
static inline u32 rv_remu(u8 rd, u8 rs1, u8 rs2)
{
return rv_r_insn(1, rs2, rs1, 7, rd, 0x33);
......@@ -501,6 +511,16 @@ static inline u32 rv_ble(u8 rs1, u8 rs2, u16 imm12_1)
return rv_bge(rs2, rs1, imm12_1);
}
static inline u32 rv_lb(u8 rd, u16 imm11_0, u8 rs1)
{
return rv_i_insn(imm11_0, rs1, 0, rd, 0x03);
}
static inline u32 rv_lh(u8 rd, u16 imm11_0, u8 rs1)
{
return rv_i_insn(imm11_0, rs1, 1, rd, 0x03);
}
static inline u32 rv_lw(u8 rd, u16 imm11_0, u8 rs1)
{
return rv_i_insn(imm11_0, rs1, 2, rd, 0x03);
......@@ -766,11 +786,21 @@ static inline u32 rv_mulw(u8 rd, u8 rs1, u8 rs2)
return rv_r_insn(1, rs2, rs1, 0, rd, 0x3b);
}
static inline u32 rv_divw(u8 rd, u8 rs1, u8 rs2)
{
return rv_r_insn(1, rs2, rs1, 4, rd, 0x3b);
}
static inline u32 rv_divuw(u8 rd, u8 rs1, u8 rs2)
{
return rv_r_insn(1, rs2, rs1, 5, rd, 0x3b);
}
static inline u32 rv_remw(u8 rd, u8 rs1, u8 rs2)
{
return rv_r_insn(1, rs2, rs1, 6, rd, 0x3b);
}
static inline u32 rv_remuw(u8 rd, u8 rs1, u8 rs2)
{
return rv_r_insn(1, rs2, rs1, 7, rd, 0x3b);
......
arch/riscv/net/bpf_jit_comp64.c
View file @ 9e3b47ab
......@@ -580,7 +580,8 @@ static int add_exception_handler(const struct bpf_insn *insn,
unsigned long pc;
off_t offset;
if (!ctx->insns || !ctx->prog->aux->extable || BPF_MODE(insn->code) != BPF_PROBE_MEM)
if (!ctx->insns || !ctx->prog->aux->extable ||
(BPF_MODE(insn->code) != BPF_PROBE_MEM && BPF_MODE(insn->code) != BPF_PROBE_MEMSX))
return 0;
if (WARN_ON_ONCE(ctx->nexentries >= ctx->prog->aux->num_exentries))
......@@ -1046,7 +1047,19 @@ int bpf_jit_emit_insn(const struct bpf_insn *insn, struct rv_jit_context *ctx,
emit_zext_32(rd, ctx);
break;
}
switch (insn->off) {
case 0:
emit_mv(rd, rs, ctx);
break;
case 8:
case 16:
emit_slli(RV_REG_T1, rs, 64 - insn->off, ctx);
emit_srai(rd, RV_REG_T1, 64 - insn->off, ctx);
break;
case 32:
emit_addiw(rd, rs, 0, ctx);
break;
}
if (!is64 && !aux->verifier_zext)
emit_zext_32(rd, ctx);
break;
......@@ -1094,12 +1107,18 @@ int bpf_jit_emit_insn(const struct bpf_insn *insn, struct rv_jit_context *ctx,
break;
case BPF_ALU | BPF_DIV | BPF_X:
case BPF_ALU64 | BPF_DIV | BPF_X:
if (off)
emit(is64 ? rv_div(rd, rd, rs) : rv_divw(rd, rd, rs), ctx);
else
emit(is64 ? rv_divu(rd, rd, rs) : rv_divuw(rd, rd, rs), ctx);
if (!is64 && !aux->verifier_zext)
emit_zext_32(rd, ctx);
break;
case BPF_ALU | BPF_MOD | BPF_X:
case BPF_ALU64 | BPF_MOD | BPF_X:
if (off)
emit(is64 ? rv_rem(rd, rd, rs) : rv_remw(rd, rd, rs), ctx);
else
emit(is64 ? rv_remu(rd, rd, rs) : rv_remuw(rd, rd, rs), ctx);
if (!is64 && !aux->verifier_zext)
emit_zext_32(rd, ctx);
......@@ -1149,6 +1168,7 @@ int bpf_jit_emit_insn(const struct bpf_insn *insn, struct rv_jit_context *ctx,
break;
case BPF_ALU | BPF_END | BPF_FROM_BE:
case BPF_ALU64 | BPF_END | BPF_FROM_LE:
emit_li(RV_REG_T2, 0, ctx);
emit_andi(RV_REG_T1, rd, 0xff, ctx);
......@@ -1271,6 +1291,10 @@ int bpf_jit_emit_insn(const struct bpf_insn *insn, struct rv_jit_context *ctx,
case BPF_ALU | BPF_DIV | BPF_K:
case BPF_ALU64 | BPF_DIV | BPF_K:
emit_imm(RV_REG_T1, imm, ctx);
if (off)
emit(is64 ? rv_div(rd, rd, RV_REG_T1) :
rv_divw(rd, rd, RV_REG_T1), ctx);
else
emit(is64 ? rv_divu(rd, rd, RV_REG_T1) :
rv_divuw(rd, rd, RV_REG_T1), ctx);
if (!is64 && !aux->verifier_zext)
......@@ -1279,6 +1303,10 @@ int bpf_jit_emit_insn(const struct bpf_insn *insn, struct rv_jit_context *ctx,
case BPF_ALU | BPF_MOD | BPF_K:
case BPF_ALU64 | BPF_MOD | BPF_K:
emit_imm(RV_REG_T1, imm, ctx);
if (off)
emit(is64 ? rv_rem(rd, rd, RV_REG_T1) :
rv_remw(rd, rd, RV_REG_T1), ctx);
else
emit(is64 ? rv_remu(rd, rd, RV_REG_T1) :
rv_remuw(rd, rd, RV_REG_T1), ctx);
if (!is64 && !aux->verifier_zext)
......@@ -1314,7 +1342,11 @@ int bpf_jit_emit_insn(const struct bpf_insn *insn, struct rv_jit_context *ctx,
/* JUMP off */
case BPF_JMP | BPF_JA:
case BPF_JMP32 | BPF_JA:
if (BPF_CLASS(code) == BPF_JMP)
rvoff = rv_offset(i, off, ctx);
else
rvoff = rv_offset(i, imm, ctx);
ret = emit_jump_and_link(RV_REG_ZERO, rvoff, true, ctx);
if (ret)
return ret;
......@@ -1486,7 +1518,7 @@ int bpf_jit_emit_insn(const struct bpf_insn *insn, struct rv_jit_context *ctx,
return 1;
}
/* LDX: dst = *(size *)(src + off) */
/* LDX: dst = *(unsigned size *)(src + off) */
case BPF_LDX | BPF_MEM | BPF_B:
case BPF_LDX | BPF_MEM | BPF_H:
case BPF_LDX | BPF_MEM | BPF_W:
......@@ -1495,13 +1527,27 @@ int bpf_jit_emit_insn(const struct bpf_insn *insn, struct rv_jit_context *ctx,
case BPF_LDX | BPF_PROBE_MEM | BPF_H:
case BPF_LDX | BPF_PROBE_MEM | BPF_W:
case BPF_LDX | BPF_PROBE_MEM | BPF_DW:
/* LDSX: dst = *(signed size *)(src + off) */
case BPF_LDX | BPF_MEMSX | BPF_B:
case BPF_LDX | BPF_MEMSX | BPF_H:
case BPF_LDX | BPF_MEMSX | BPF_W:
case BPF_LDX | BPF_PROBE_MEMSX | BPF_B:
case BPF_LDX | BPF_PROBE_MEMSX | BPF_H:
case BPF_LDX | BPF_PROBE_MEMSX | BPF_W:
{
int insn_len, insns_start;
bool sign_ext;
sign_ext = BPF_MODE(insn->code) == BPF_MEMSX ||
BPF_MODE(insn->code) == BPF_PROBE_MEMSX;
switch (BPF_SIZE(code)) {
case BPF_B:
if (is_12b_int(off)) {
insns_start = ctx->ninsns;
if (sign_ext)
emit(rv_lb(rd, off, rs), ctx);
else
emit(rv_lbu(rd, off, rs), ctx);
insn_len = ctx->ninsns - insns_start;
break;
......@@ -1510,14 +1556,18 @@ int bpf_jit_emit_insn(const struct bpf_insn *insn, struct rv_jit_context *ctx,
emit_imm(RV_REG_T1, off, ctx);
emit_add(RV_REG_T1, RV_REG_T1, rs, ctx);
insns_start = ctx->ninsns;
if (sign_ext)
emit(rv_lb(rd, 0, RV_REG_T1), ctx);
else
emit(rv_lbu(rd, 0, RV_REG_T1), ctx);
insn_len = ctx->ninsns - insns_start;
if (insn_is_zext(&insn[1]))
return 1;
break;
case BPF_H:
if (is_12b_int(off)) {
insns_start = ctx->ninsns;
if (sign_ext)
emit(rv_lh(rd, off, rs), ctx);
else
emit(rv_lhu(rd, off, rs), ctx);
insn_len = ctx->ninsns - insns_start;
break;
......@@ -1526,14 +1576,18 @@ int bpf_jit_emit_insn(const struct bpf_insn *insn, struct rv_jit_context *ctx,
emit_imm(RV_REG_T1, off, ctx);
emit_add(RV_REG_T1, RV_REG_T1, rs, ctx);
insns_start = ctx->ninsns;
if (sign_ext)
emit(rv_lh(rd, 0, RV_REG_T1), ctx);
else
emit(rv_lhu(rd, 0, RV_REG_T1), ctx);
insn_len = ctx->ninsns - insns_start;
if (insn_is_zext(&insn[1]))
return 1;
break;
case BPF_W:
if (is_12b_int(off)) {
insns_start = ctx->ninsns;
if (sign_ext)
emit(rv_lw(rd, off, rs), ctx);
else
emit(rv_lwu(rd, off, rs), ctx);
insn_len = ctx->ninsns - insns_start;
break;
......@@ -1542,10 +1596,11 @@ int bpf_jit_emit_insn(const struct bpf_insn *insn, struct rv_jit_context *ctx,
emit_imm(RV_REG_T1, off, ctx);
emit_add(RV_REG_T1, RV_REG_T1, rs, ctx);
insns_start = ctx->ninsns;
if (sign_ext)
emit(rv_lw(rd, 0, RV_REG_T1), ctx);
else
emit(rv_lwu(rd, 0, RV_REG_T1), ctx);
insn_len = ctx->ninsns - insns_start;
if (insn_is_zext(&insn[1]))
return 1;
break;
case BPF_DW:
if (is_12b_int(off)) {
......@@ -1566,6 +1621,9 @@ int bpf_jit_emit_insn(const struct bpf_insn *insn, struct rv_jit_context *ctx,
ret = add_exception_handler(insn, ctx, rd, insn_len);
if (ret)
return ret;
if (BPF_SIZE(code) != BPF_DW && insn_is_zext(&insn[1]))
return 1;
break;
}
/* speculation barrier */
......
tools/testing/selftests/bpf/progs/test_ldsx_insn.c
View file @ 9e3b47ab
......@@ -5,7 +5,8 @@
#include <bpf/bpf_helpers.h>
#include <bpf/bpf_tracing.h>
#if (defined(__TARGET_ARCH_arm64) || defined(__TARGET_ARCH_x86)) && __clang_major__ >= 18
#if (defined(__TARGET_ARCH_arm64) || defined(__TARGET_ARCH_x86) || \
(defined(__TARGET_ARCH_riscv) && __riscv_xlen == 64)) && __clang_major__ >= 18
const volatile int skip = 0;
#else
const volatile int skip = 1;
......
tools/testing/selftests/bpf/progs/verifier_bswap.c
View file @ 9e3b47ab
......@@ -4,7 +4,8 @@
#include <bpf/bpf_helpers.h>
#include "bpf_misc.h"
#if (defined(__TARGET_ARCH_arm64) || defined(__TARGET_ARCH_x86)) && __clang_major__ >= 18
#if (defined(__TARGET_ARCH_arm64) || defined(__TARGET_ARCH_x86) || \
(defined(__TARGET_ARCH_riscv) && __riscv_xlen == 64)) && __clang_major__ >= 18
SEC("socket")
__description("BSWAP, 16")
......
tools/testing/selftests/bpf/progs/verifier_gotol.c
View file @ 9e3b47ab
......@@ -4,7 +4,8 @@
#include <bpf/bpf_helpers.h>
#include "bpf_misc.h"
#if (defined(__TARGET_ARCH_arm64) || defined(__TARGET_ARCH_x86)) && __clang_major__ >= 18
#if (defined(__TARGET_ARCH_arm64) || defined(__TARGET_ARCH_x86) || \
(defined(__TARGET_ARCH_riscv) && __riscv_xlen == 64)) && __clang_major__ >= 18
SEC("socket")
__description("gotol, small_imm")
......
tools/testing/selftests/bpf/progs/verifier_ldsx.c
View file @ 9e3b47ab
......@@ -4,7 +4,8 @@
#include <bpf/bpf_helpers.h>
#include "bpf_misc.h"
#if (defined(__TARGET_ARCH_arm64) || defined(__TARGET_ARCH_x86)) && __clang_major__ >= 18
#if (defined(__TARGET_ARCH_arm64) || defined(__TARGET_ARCH_x86) || \
(defined(__TARGET_ARCH_riscv) && __riscv_xlen == 64)) && __clang_major__ >= 18
SEC("socket")
__description("LDSX, S8")
......
tools/testing/selftests/bpf/progs/verifier_movsx.c
View file @ 9e3b47ab
......@@ -4,7 +4,8 @@
#include <bpf/bpf_helpers.h>
#include "bpf_misc.h"
#if (defined(__TARGET_ARCH_arm64) || defined(__TARGET_ARCH_x86)) && __clang_major__ >= 18
#if (defined(__TARGET_ARCH_arm64) || defined(__TARGET_ARCH_x86) || \
(defined(__TARGET_ARCH_riscv) && __riscv_xlen == 64)) && __clang_major__ >= 18
SEC("socket")
__description("MOV32SX, S8")
......
tools/testing/selftests/bpf/progs/verifier_sdiv.c
View file @ 9e3b47ab
......@@ -4,7 +4,8 @@
#include <bpf/bpf_helpers.h>
#include "bpf_misc.h"
#if (defined(__TARGET_ARCH_arm64) || defined(__TARGET_ARCH_x86)) && __clang_major__ >= 18
#if (defined(__TARGET_ARCH_arm64) || defined(__TARGET_ARCH_x86) || \
(defined(__TARGET_ARCH_riscv) && __riscv_xlen == 64)) && __clang_major__ >= 18
SEC("socket")
__description("SDIV32, non-zero imm divisor, check 1")
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment
GitLab Nexedi Edition | About GitLab | About Nexedi | 沪ICP备2021021310号-2 | 沪ICP备2021021310号-7