nfp: bpf: relocate jump targets just before the load

Don't translate the program assuming it will be loaded at a given
address.  This will be required for sharing programs between ports
of the same NIC, tail calls and subprograms.  It will also make the
jump targets easier to understand when dumping the program to user
space.

Translate the program as if it was going to be loaded at address
zero.  When load happens add the load offset in and set addresses
of special branches.
Signed-off-by: Jakub Kicinski <jakub.kicinski@netronome.com>
Reviewed-by: Jiong Wang <jiong.wang@netronome.com>
Signed-off-by: Daniel Borkmann <daniel@iogearbox.net>

drivers/net/ethernet/netronome/nfp/bpf/jit.c

@@ -85,7 +85,7 @@ static void nfp_prog_push(struct nfp_prog *nfp_prog, u64 insn)
 
 static unsigned int nfp_prog_current_offset(struct nfp_prog *nfp_prog)
 {
-	return nfp_prog->start_off + nfp_prog->prog_len;
+	return nfp_prog->prog_len;
 }
 
 static bool
@@ -100,12 +100,6 @@ nfp_prog_confirm_current_offset(struct nfp_prog *nfp_prog, unsigned int off)
 	return !WARN_ON_ONCE(nfp_prog_current_offset(nfp_prog) != off);
 }
 
-static unsigned int
-nfp_prog_offset_to_index(struct nfp_prog *nfp_prog, unsigned int offset)
-{
-	return offset - nfp_prog->start_off;
-}
-
 /* --- Emitters --- */
 static void
 __emit_cmd(struct nfp_prog *nfp_prog, enum cmd_tgt_map op,
@@ -195,22 +189,28 @@ __emit_br(struct nfp_prog *nfp_prog, enum br_mask mask, enum br_ev_pip ev_pip,
 	nfp_prog_push(nfp_prog, insn);
 }
 
-static void emit_br_def(struct nfp_prog *nfp_prog, u16 addr, u8 defer)
+static void
+emit_br_relo(struct nfp_prog *nfp_prog, enum br_mask mask, u16 addr, u8 defer,
+	     enum nfp_relo_type relo)
 {
-	if (defer > 2) {
+	if (mask == BR_UNC && defer > 2) {
 		pr_err("BUG: branch defer out of bounds %d\n", defer);
 		nfp_prog->error = -EFAULT;
 		return;
 	}
-	__emit_br(nfp_prog, BR_UNC, BR_EV_PIP_UNCOND, BR_CSS_NONE, addr, defer);
+
+	__emit_br(nfp_prog, mask,
+		  mask != BR_UNC ? BR_EV_PIP_COND : BR_EV_PIP_UNCOND,
+		  BR_CSS_NONE, addr, defer);
+
+	nfp_prog->prog[nfp_prog->prog_len - 1] |=
+		FIELD_PREP(OP_RELO_TYPE, relo);
 }
 
 static void
 emit_br(struct nfp_prog *nfp_prog, enum br_mask mask, u16 addr, u8 defer)
 {
-	__emit_br(nfp_prog, mask,
-		  mask != BR_UNC ? BR_EV_PIP_COND : BR_EV_PIP_UNCOND,
-		  BR_CSS_NONE, addr, defer);
+	emit_br_relo(nfp_prog, mask, addr, defer, RELO_BR_REL);
 }
 
 static void
@@ -515,16 +515,6 @@ static void wrp_nops(struct nfp_prog *nfp_prog, unsigned int count)
 		emit_nop(nfp_prog);
 }
 
-static void
-wrp_br_special(struct nfp_prog *nfp_prog, enum br_mask mask,
-	       enum br_special special)
-{
-	emit_br(nfp_prog, mask, 0, 0);
-
-	nfp_prog->prog[nfp_prog->prog_len - 1] |=
-		FIELD_PREP(OP_BR_SPECIAL, special);
-}
-
 static void wrp_mov(struct nfp_prog *nfp_prog, swreg dst, swreg src)
 {
 	emit_alu(nfp_prog, dst, reg_none(), ALU_OP_NONE, src);
@@ -749,7 +739,7 @@ construct_data_ind_ld(struct nfp_prog *nfp_prog, u16 offset, u16 src, u8 size)
 		 imm_a(nfp_prog), ALU_OP_ADD, reg_imm(size));
 	emit_alu(nfp_prog, reg_none(),
 		 plen_reg(nfp_prog), ALU_OP_SUB, imm_a(nfp_prog));
-	wrp_br_special(nfp_prog, BR_BLO, OP_BR_GO_ABORT);
+	emit_br_relo(nfp_prog, BR_BLO, 0, 0, RELO_BR_GO_ABORT);
 
 	/* Load data */
 	return data_ld(nfp_prog, imm_b(nfp_prog), 0, size);
@@ -762,7 +752,7 @@ static int construct_data_ld(struct nfp_prog *nfp_prog, u16 offset, u8 size)
 	/* Check packet length */
 	tmp_reg = ur_load_imm_any(nfp_prog, offset + size, imm_a(nfp_prog));
 	emit_alu(nfp_prog, reg_none(), plen_reg(nfp_prog), ALU_OP_SUB, tmp_reg);
-	wrp_br_special(nfp_prog, BR_BLO, OP_BR_GO_ABORT);
+	emit_br_relo(nfp_prog, BR_BLO, 0, 0, RELO_BR_GO_ABORT);
 
 	/* Load data */
 	tmp_reg = re_load_imm_any(nfp_prog, offset, imm_b(nfp_prog));
@@ -1269,7 +1259,7 @@ static int adjust_head(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta)
 	emit_ld_field(nfp_prog, pptr_reg(nfp_prog), 0x3, tmp, SHF_SC_NONE, 0);
 
 	/* Skip over the -EINVAL ret code (defer 2) */
-	emit_br_def(nfp_prog, end, 2);
+	emit_br(nfp_prog, BR_UNC, end, 2);
 
 	emit_alu(nfp_prog, plen_reg(nfp_prog),
 		 plen_reg(nfp_prog), ALU_OP_SUB, reg_a(2 * 2));
@@ -2036,7 +2026,7 @@ static int call(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta)
 
 static int goto_out(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta)
 {
-	wrp_br_special(nfp_prog, BR_UNC, OP_BR_GO_OUT);
+	emit_br_relo(nfp_prog, BR_UNC, 0, 0, RELO_BR_GO_OUT);
 
 	return 0;
 }
@@ -2125,11 +2115,9 @@ static int nfp_fixup_branches(struct nfp_prog *nfp_prog)
 			continue;
 
 		if (list_is_last(&meta->l, &nfp_prog->insns))
-			idx = nfp_prog->last_bpf_off;
+			br_idx = nfp_prog->last_bpf_off;
 		else
-			idx = list_next_entry(meta, l)->off - 1;
-
-		br_idx = nfp_prog_offset_to_index(nfp_prog, idx);
+			br_idx = list_next_entry(meta, l)->off - 1;
 
 		if (!nfp_is_br(nfp_prog->prog[br_idx])) {
 			pr_err("Fixup found block not ending in branch %d %02x %016llx!!\n",
@@ -2137,7 +2125,8 @@ static int nfp_fixup_branches(struct nfp_prog *nfp_prog)
 			return -ELOOP;
 		}
 		/* Leave special branches for later */
-		if (FIELD_GET(OP_BR_SPECIAL, nfp_prog->prog[br_idx]))
+		if (FIELD_GET(OP_RELO_TYPE, nfp_prog->prog[br_idx]) !=
+		    RELO_BR_REL)
 			continue;
 
 		if (!meta->jmp_dst) {
@@ -2152,38 +2141,13 @@ static int nfp_fixup_branches(struct nfp_prog *nfp_prog)
 			return -ELOOP;
 		}
 
-		for (idx = nfp_prog_offset_to_index(nfp_prog, meta->off);
-		     idx <= br_idx; idx++) {
+		for (idx = meta->off; idx <= br_idx; idx++) {
 			if (!nfp_is_br(nfp_prog->prog[idx]))
 				continue;
 			br_set_offset(&nfp_prog->prog[idx], jmp_dst->off);
 		}
 	}
 
-	/* Fixup 'goto out's separately, they can be scattered around */
-	for (br_idx = 0; br_idx < nfp_prog->prog_len; br_idx++) {
-		enum br_special special;
-
-		if ((nfp_prog->prog[br_idx] & OP_BR_BASE_MASK) != OP_BR_BASE)
-			continue;
-
-		special = FIELD_GET(OP_BR_SPECIAL, nfp_prog->prog[br_idx]);
-		switch (special) {
-		case OP_BR_NORMAL:
-			break;
-		case OP_BR_GO_OUT:
-			br_set_offset(&nfp_prog->prog[br_idx],
-				      nfp_prog->tgt_out);
-			break;
-		case OP_BR_GO_ABORT:
-			br_set_offset(&nfp_prog->prog[br_idx],
-				      nfp_prog->tgt_abort);
-			break;
-		}
-
-		nfp_prog->prog[br_idx] &= ~OP_BR_SPECIAL;
-	}
-
 	return 0;
 }
 
@@ -2211,7 +2175,7 @@ static void nfp_outro_tc_da(struct nfp_prog *nfp_prog)
 	/* Target for aborts */
 	nfp_prog->tgt_abort = nfp_prog_current_offset(nfp_prog);
 
-	emit_br_def(nfp_prog, nfp_prog->tgt_done, 2);
+	emit_br_relo(nfp_prog, BR_UNC, 0, 2, RELO_BR_NEXT_PKT);
 
 	wrp_mov(nfp_prog, reg_a(0), NFP_BPF_ABI_FLAGS);
 	emit_ld_field(nfp_prog, reg_a(0), 0xc, reg_imm(0x11), SHF_SC_L_SHF, 16);
@@ -2238,7 +2202,7 @@ static void nfp_outro_tc_da(struct nfp_prog *nfp_prog)
 	emit_shf(nfp_prog, reg_b(2),
 		 reg_imm(0xf), SHF_OP_AND, reg_b(3), SHF_SC_R_SHF, 0);
 
-	emit_br_def(nfp_prog, nfp_prog->tgt_done, 2);
+	emit_br_relo(nfp_prog, BR_UNC, 0, 2, RELO_BR_NEXT_PKT);
 
 	emit_shf(nfp_prog, reg_b(2),
 		 reg_a(2), SHF_OP_OR, reg_b(2), SHF_SC_L_SHF, 4);
@@ -2257,7 +2221,7 @@ static void nfp_outro_xdp(struct nfp_prog *nfp_prog)
 	/* Target for aborts */
 	nfp_prog->tgt_abort = nfp_prog_current_offset(nfp_prog);
 
-	emit_br_def(nfp_prog, nfp_prog->tgt_done, 2);
+	emit_br_relo(nfp_prog, BR_UNC, 0, 2, RELO_BR_NEXT_PKT);
 
 	wrp_mov(nfp_prog, reg_a(0), NFP_BPF_ABI_FLAGS);
 	emit_ld_field(nfp_prog, reg_a(0), 0xc, reg_imm(0x82), SHF_SC_L_SHF, 16);
@@ -2278,7 +2242,7 @@ static void nfp_outro_xdp(struct nfp_prog *nfp_prog)
 	emit_shf(nfp_prog, reg_b(2),
 		 reg_imm(0xff), SHF_OP_AND, reg_b(2), SHF_SC_R_SHF, 0);
 
-	emit_br_def(nfp_prog, nfp_prog->tgt_done, 2);
+	emit_br_relo(nfp_prog, BR_UNC, 0, 2, RELO_BR_NEXT_PKT);
 
 	wrp_mov(nfp_prog, reg_a(0), NFP_BPF_ABI_FLAGS);
 	emit_ld_field(nfp_prog, reg_a(0), 0xc, reg_b(2), SHF_SC_L_SHF, 16);
@@ -2694,20 +2658,19 @@ static int nfp_bpf_optimize(struct nfp_prog *nfp_prog)
 	return 0;
 }
 
-static int nfp_bpf_ustore_calc(struct nfp_prog *nfp_prog, __le64 *ustore)
+static int nfp_bpf_ustore_calc(u64 *prog, unsigned int len)
 {
+	__le64 *ustore = (__force __le64 *)prog;
 	int i;
 
-	for (i = 0; i < nfp_prog->prog_len; i++) {
+	for (i = 0; i < len; i++) {
 		int err;
 
-		err = nfp_ustore_check_valid_no_ecc(nfp_prog->prog[i]);
+		err = nfp_ustore_check_valid_no_ecc(prog[i]);
 		if (err)
 			return err;
 
-		nfp_prog->prog[i] = nfp_ustore_calc_ecc_insn(nfp_prog->prog[i]);
-
-		ustore[i] = cpu_to_le64(nfp_prog->prog[i]);
+		ustore[i] = cpu_to_le64(nfp_ustore_calc_ecc_insn(prog[i]));
 	}
 
 	return 0;
@@ -2728,7 +2691,7 @@ int nfp_bpf_jit(struct nfp_prog *nfp_prog)
 		return -EINVAL;
 	}
 
-	return nfp_bpf_ustore_calc(nfp_prog, (__force __le64 *)nfp_prog->prog);
+	return ret;
 }
 
 void nfp_bpf_jit_prepare(struct nfp_prog *nfp_prog, unsigned int cnt)
@@ -2753,3 +2716,51 @@ void nfp_bpf_jit_prepare(struct nfp_prog *nfp_prog, unsigned int cnt)
 		}
 	}
 }
+
+void *nfp_bpf_relo_for_vnic(struct nfp_prog *nfp_prog, struct nfp_bpf_vnic *bv)
+{
+	unsigned int i;
+	u64 *prog;
+	int err;
+
+	prog = kmemdup(nfp_prog->prog, nfp_prog->prog_len * sizeof(u64),
+		       GFP_KERNEL);
+	if (!prog)
+		return ERR_PTR(-ENOMEM);
+
+	for (i = 0; i < nfp_prog->prog_len; i++) {
+		enum nfp_relo_type special;
+
+		special = FIELD_GET(OP_RELO_TYPE, prog[i]);
+		switch (special) {
+		case RELO_NONE:
+			continue;
+		case RELO_BR_REL:
+			br_add_offset(&prog[i], bv->start_off);
+			break;
+		case RELO_BR_GO_OUT:
+			br_set_offset(&prog[i],
+				      nfp_prog->tgt_out + bv->start_off);
+			break;
+		case RELO_BR_GO_ABORT:
+			br_set_offset(&prog[i],
+				      nfp_prog->tgt_abort + bv->start_off);
+			break;
+		case RELO_BR_NEXT_PKT:
+			br_set_offset(&prog[i], bv->tgt_done);
+			break;
+		}
+
+		prog[i] &= ~OP_RELO_TYPE;
+	}
+
+	err = nfp_bpf_ustore_calc(prog, nfp_prog->prog_len);
+	if (err)
+		goto err_free_prog;
+
+	return prog;
+
+err_free_prog:
+	kfree(prog);
+	return ERR_PTR(err);
+}

drivers/net/ethernet/netronome/nfp/bpf/main.c

@@ -87,16 +87,21 @@ static const char *nfp_bpf_extra_cap(struct nfp_app *app, struct nfp_net *nn)
 static int
 nfp_bpf_vnic_alloc(struct nfp_app *app, struct nfp_net *nn, unsigned int id)
 {
+	struct nfp_bpf_vnic *bv;
 	int err;
 
-	nn->app_priv = kzalloc(sizeof(struct nfp_bpf_vnic), GFP_KERNEL);
-	if (!nn->app_priv)
+	bv = kzalloc(sizeof(*bv), GFP_KERNEL);
+	if (!bv)
 		return -ENOMEM;
+	nn->app_priv = bv;
 
 	err = nfp_app_nic_vnic_alloc(app, nn, id);
 	if (err)
 		goto err_free_priv;
 
+	bv->start_off = nn_readw(nn, NFP_NET_CFG_BPF_START);
+	bv->tgt_done = nn_readw(nn, NFP_NET_CFG_BPF_DONE);
+
 	return 0;
 err_free_priv:
 	kfree(nn->app_priv);

drivers/net/ethernet/netronome/nfp/bpf/main.h

@@ -42,15 +42,20 @@
 
 #include "../nfp_asm.h"
 
-/* For branch fixup logic use up-most byte of branch instruction as scratch
+/* For relocation logic use up-most byte of branch instruction as scratch
  * area.  Remember to clear this before sending instructions to HW!
  */
-#define OP_BR_SPECIAL	0xff00000000000000ULL
-
-enum br_special {
-	OP_BR_NORMAL = 0,
-	OP_BR_GO_OUT,
-	OP_BR_GO_ABORT,
+#define OP_RELO_TYPE	0xff00000000000000ULL
+
+enum nfp_relo_type {
+	RELO_NONE = 0,
+	/* standard internal jumps */
+	RELO_BR_REL,
+	/* internal jumps to parts of the outro */
+	RELO_BR_GO_OUT,
+	RELO_BR_GO_ABORT,
+	/* external jumps to fixed addresses */
+	RELO_BR_NEXT_PKT,
 };
 
 enum static_regs {
@@ -191,11 +196,9 @@ static inline bool is_mbpf_store(const struct nfp_insn_meta *meta)
  * @__prog_alloc_len: alloc size of @prog array
  * @verifier_meta: temporary storage for verifier's insn meta
  * @type: BPF program type
- * @start_off: address of the first instruction in the memory
  * @last_bpf_off: address of the last instruction translated from BPF
  * @tgt_out: jump target for normal exit
  * @tgt_abort: jump target for abort (e.g. access outside of packet buffer)
- * @tgt_done: jump target to get the next packet
  * @n_translated: number of successfully translated instructions (for errors)
  * @error: error code if something went wrong
  * @stack_depth: max stack depth from the verifier
@@ -213,11 +216,9 @@ struct nfp_prog {
 
 	enum bpf_prog_type type;
 
-	unsigned int start_off;
 	unsigned int last_bpf_off;
 	unsigned int tgt_out;
 	unsigned int tgt_abort;
-	unsigned int tgt_done;
 
 	unsigned int n_translated;
 	int error;
@@ -231,9 +232,13 @@ struct nfp_prog {
 /**
  * struct nfp_bpf_vnic - per-vNIC BPF priv structure
  * @tc_prog:	currently loaded cls_bpf program
+ * @start_off:	address of the first instruction in the memory
+ * @tgt_done:	jump target to get the next packet
  */
 struct nfp_bpf_vnic {
 	struct bpf_prog *tc_prog;
+	unsigned int start_off;
+	unsigned int tgt_done;
 };
 
 void nfp_bpf_jit_prepare(struct nfp_prog *nfp_prog, unsigned int cnt);
@@ -257,4 +262,6 @@ int nfp_bpf_destroy(struct nfp_app *app, struct nfp_net *nn,
 struct nfp_insn_meta *
 nfp_bpf_goto_meta(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta,
 		  unsigned int insn_idx, unsigned int n_insns);
+
+void *nfp_bpf_relo_for_vnic(struct nfp_prog *nfp_prog, struct nfp_bpf_vnic *bv);
 #endif

drivers/net/ethernet/netronome/nfp/bpf/offload.c

@@ -130,10 +130,7 @@ int nfp_bpf_translate(struct nfp_app *app, struct nfp_net *nn,
 			prog->aux->stack_depth, stack_size);
 		return -EOPNOTSUPP;
 	}
-
 	nfp_prog->stack_depth = round_up(prog->aux->stack_depth, 4);
-	nfp_prog->start_off = nn_readw(nn, NFP_NET_CFG_BPF_START);
-	nfp_prog->tgt_done = nn_readw(nn, NFP_NET_CFG_BPF_DONE);
 
 	max_instr = nn_readw(nn, NFP_NET_CFG_BPF_MAX_LEN);
 	nfp_prog->__prog_alloc_len = max_instr * sizeof(u64);
@@ -161,6 +158,7 @@ static int nfp_net_bpf_load(struct nfp_net *nn, struct bpf_prog *prog)
 	struct nfp_prog *nfp_prog = prog->aux->offload->dev_priv;
 	unsigned int max_mtu;
 	dma_addr_t dma_addr;
+	void *img;
 	int err;
 
 	max_mtu = nn_readb(nn, NFP_NET_CFG_BPF_INL_MTU) * 64 - 32;
@@ -169,11 +167,17 @@ static int nfp_net_bpf_load(struct nfp_net *nn, struct bpf_prog *prog)
 		return -EOPNOTSUPP;
 	}
 
-	dma_addr = dma_map_single(nn->dp.dev, nfp_prog->prog,
+	img = nfp_bpf_relo_for_vnic(nfp_prog, nn->app_priv);
+	if (IS_ERR(img))
+		return PTR_ERR(img);
+
+	dma_addr = dma_map_single(nn->dp.dev, img,
 				  nfp_prog->prog_len * sizeof(u64),
 				  DMA_TO_DEVICE);
-	if (dma_mapping_error(nn->dp.dev, dma_addr))
+	if (dma_mapping_error(nn->dp.dev, dma_addr)) {
+		kfree(img);
 		return -ENOMEM;
+	}
 
 	nn_writew(nn, NFP_NET_CFG_BPF_SIZE, nfp_prog->prog_len);
 	nn_writeq(nn, NFP_NET_CFG_BPF_ADDR, dma_addr);
@@ -185,6 +189,7 @@ static int nfp_net_bpf_load(struct nfp_net *nn, struct bpf_prog *prog)
 
 	dma_unmap_single(nn->dp.dev, dma_addr, nfp_prog->prog_len * sizeof(u64),
 			 DMA_TO_DEVICE);
+	kfree(img);
 
 	return err;
 }