Commit 86be36f6 authored by Naveen N. Rao's avatar Naveen N. Rao Committed by Daniel Borkmann

powerpc: bpf: Fix generation of load/store DW instructions

Yauheni Kaliuta pointed out that PTR_TO_STACK store/load verifier test
was failing on powerpc64 BE, and rightfully indicated that the PPC_LD()
macro is not masking away the last two bits of the offset per the ISA,
resulting in the generation of 'lwa' instruction instead of the intended
'ld' instruction.

Segher also pointed out that we can't simply mask away the last two bits
as that will result in loading/storing from/to a memory location that
was not intended.

This patch addresses this by using ldx/stdx if the offset is not
word-aligned. We load the offset into a temporary register (TMP_REG_2)
and use that as the index register in a subsequent ldx/stdx. We fix
PPC_LD() macro to mask off the last two bits, but enhance PPC_BPF_LL()
and PPC_BPF_STL() to factor in the offset value and generate the proper
instruction sequence. We also convert all existing users of PPC_LD() and
PPC_STD() to use these macros. All existing uses of these macros have
been audited to ensure that TMP_REG_2 can be clobbered.

Fixes: 156d0e29 ("powerpc/ebpf/jit: Implement JIT compiler for extended BPF")
Cc: stable@vger.kernel.org # v4.9+
Reported-by: default avatarYauheni Kaliuta <yauheni.kaliuta@redhat.com>
Signed-off-by: default avatarNaveen N. Rao <naveen.n.rao@linux.vnet.ibm.com>
Signed-off-by: default avatarDaniel Borkmann <daniel@iogearbox.net>
parent 044175a0
...@@ -302,6 +302,7 @@ ...@@ -302,6 +302,7 @@
/* Misc instructions for BPF compiler */ /* Misc instructions for BPF compiler */
#define PPC_INST_LBZ 0x88000000 #define PPC_INST_LBZ 0x88000000
#define PPC_INST_LD 0xe8000000 #define PPC_INST_LD 0xe8000000
#define PPC_INST_LDX 0x7c00002a
#define PPC_INST_LHZ 0xa0000000 #define PPC_INST_LHZ 0xa0000000
#define PPC_INST_LWZ 0x80000000 #define PPC_INST_LWZ 0x80000000
#define PPC_INST_LHBRX 0x7c00062c #define PPC_INST_LHBRX 0x7c00062c
...@@ -309,6 +310,7 @@ ...@@ -309,6 +310,7 @@
#define PPC_INST_STB 0x98000000 #define PPC_INST_STB 0x98000000
#define PPC_INST_STH 0xb0000000 #define PPC_INST_STH 0xb0000000
#define PPC_INST_STD 0xf8000000 #define PPC_INST_STD 0xf8000000
#define PPC_INST_STDX 0x7c00012a
#define PPC_INST_STDU 0xf8000001 #define PPC_INST_STDU 0xf8000001
#define PPC_INST_STW 0x90000000 #define PPC_INST_STW 0x90000000
#define PPC_INST_STWU 0x94000000 #define PPC_INST_STWU 0x94000000
......
...@@ -51,6 +51,8 @@ ...@@ -51,6 +51,8 @@
#define PPC_LIS(r, i) PPC_ADDIS(r, 0, i) #define PPC_LIS(r, i) PPC_ADDIS(r, 0, i)
#define PPC_STD(r, base, i) EMIT(PPC_INST_STD | ___PPC_RS(r) | \ #define PPC_STD(r, base, i) EMIT(PPC_INST_STD | ___PPC_RS(r) | \
___PPC_RA(base) | ((i) & 0xfffc)) ___PPC_RA(base) | ((i) & 0xfffc))
#define PPC_STDX(r, base, b) EMIT(PPC_INST_STDX | ___PPC_RS(r) | \
___PPC_RA(base) | ___PPC_RB(b))
#define PPC_STDU(r, base, i) EMIT(PPC_INST_STDU | ___PPC_RS(r) | \ #define PPC_STDU(r, base, i) EMIT(PPC_INST_STDU | ___PPC_RS(r) | \
___PPC_RA(base) | ((i) & 0xfffc)) ___PPC_RA(base) | ((i) & 0xfffc))
#define PPC_STW(r, base, i) EMIT(PPC_INST_STW | ___PPC_RS(r) | \ #define PPC_STW(r, base, i) EMIT(PPC_INST_STW | ___PPC_RS(r) | \
...@@ -65,7 +67,9 @@ ...@@ -65,7 +67,9 @@
#define PPC_LBZ(r, base, i) EMIT(PPC_INST_LBZ | ___PPC_RT(r) | \ #define PPC_LBZ(r, base, i) EMIT(PPC_INST_LBZ | ___PPC_RT(r) | \
___PPC_RA(base) | IMM_L(i)) ___PPC_RA(base) | IMM_L(i))
#define PPC_LD(r, base, i) EMIT(PPC_INST_LD | ___PPC_RT(r) | \ #define PPC_LD(r, base, i) EMIT(PPC_INST_LD | ___PPC_RT(r) | \
___PPC_RA(base) | IMM_L(i)) ___PPC_RA(base) | ((i) & 0xfffc))
#define PPC_LDX(r, base, b) EMIT(PPC_INST_LDX | ___PPC_RT(r) | \
___PPC_RA(base) | ___PPC_RB(b))
#define PPC_LWZ(r, base, i) EMIT(PPC_INST_LWZ | ___PPC_RT(r) | \ #define PPC_LWZ(r, base, i) EMIT(PPC_INST_LWZ | ___PPC_RT(r) | \
___PPC_RA(base) | IMM_L(i)) ___PPC_RA(base) | IMM_L(i))
#define PPC_LHZ(r, base, i) EMIT(PPC_INST_LHZ | ___PPC_RT(r) | \ #define PPC_LHZ(r, base, i) EMIT(PPC_INST_LHZ | ___PPC_RT(r) | \
...@@ -85,17 +89,6 @@ ...@@ -85,17 +89,6 @@
___PPC_RA(a) | ___PPC_RB(b)) ___PPC_RA(a) | ___PPC_RB(b))
#define PPC_BPF_STDCX(s, a, b) EMIT(PPC_INST_STDCX | ___PPC_RS(s) | \ #define PPC_BPF_STDCX(s, a, b) EMIT(PPC_INST_STDCX | ___PPC_RS(s) | \
___PPC_RA(a) | ___PPC_RB(b)) ___PPC_RA(a) | ___PPC_RB(b))
#ifdef CONFIG_PPC64
#define PPC_BPF_LL(r, base, i) do { PPC_LD(r, base, i); } while(0)
#define PPC_BPF_STL(r, base, i) do { PPC_STD(r, base, i); } while(0)
#define PPC_BPF_STLU(r, base, i) do { PPC_STDU(r, base, i); } while(0)
#else
#define PPC_BPF_LL(r, base, i) do { PPC_LWZ(r, base, i); } while(0)
#define PPC_BPF_STL(r, base, i) do { PPC_STW(r, base, i); } while(0)
#define PPC_BPF_STLU(r, base, i) do { PPC_STWU(r, base, i); } while(0)
#endif
#define PPC_CMPWI(a, i) EMIT(PPC_INST_CMPWI | ___PPC_RA(a) | IMM_L(i)) #define PPC_CMPWI(a, i) EMIT(PPC_INST_CMPWI | ___PPC_RA(a) | IMM_L(i))
#define PPC_CMPDI(a, i) EMIT(PPC_INST_CMPDI | ___PPC_RA(a) | IMM_L(i)) #define PPC_CMPDI(a, i) EMIT(PPC_INST_CMPDI | ___PPC_RA(a) | IMM_L(i))
#define PPC_CMPW(a, b) EMIT(PPC_INST_CMPW | ___PPC_RA(a) | \ #define PPC_CMPW(a, b) EMIT(PPC_INST_CMPW | ___PPC_RA(a) | \
......
...@@ -123,6 +123,10 @@ DECLARE_LOAD_FUNC(sk_load_byte_msh); ...@@ -123,6 +123,10 @@ DECLARE_LOAD_FUNC(sk_load_byte_msh);
#define PPC_NTOHS_OFFS(r, base, i) PPC_LHZ_OFFS(r, base, i) #define PPC_NTOHS_OFFS(r, base, i) PPC_LHZ_OFFS(r, base, i)
#endif #endif
#define PPC_BPF_LL(r, base, i) do { PPC_LWZ(r, base, i); } while(0)
#define PPC_BPF_STL(r, base, i) do { PPC_STW(r, base, i); } while(0)
#define PPC_BPF_STLU(r, base, i) do { PPC_STWU(r, base, i); } while(0)
#define SEEN_DATAREF 0x10000 /* might call external helpers */ #define SEEN_DATAREF 0x10000 /* might call external helpers */
#define SEEN_XREG 0x20000 /* X reg is used */ #define SEEN_XREG 0x20000 /* X reg is used */
#define SEEN_MEM 0x40000 /* SEEN_MEM+(1<<n) = use mem[n] for temporary #define SEEN_MEM 0x40000 /* SEEN_MEM+(1<<n) = use mem[n] for temporary
......
...@@ -68,6 +68,26 @@ static const int b2p[] = { ...@@ -68,6 +68,26 @@ static const int b2p[] = {
/* PPC NVR range -- update this if we ever use NVRs below r27 */ /* PPC NVR range -- update this if we ever use NVRs below r27 */
#define BPF_PPC_NVR_MIN 27 #define BPF_PPC_NVR_MIN 27
/*
* WARNING: These can use TMP_REG_2 if the offset is not at word boundary,
* so ensure that it isn't in use already.
*/
#define PPC_BPF_LL(r, base, i) do { \
if ((i) % 4) { \
PPC_LI(b2p[TMP_REG_2], (i)); \
PPC_LDX(r, base, b2p[TMP_REG_2]); \
} else \
PPC_LD(r, base, i); \
} while(0)
#define PPC_BPF_STL(r, base, i) do { \
if ((i) % 4) { \
PPC_LI(b2p[TMP_REG_2], (i)); \
PPC_STDX(r, base, b2p[TMP_REG_2]); \
} else \
PPC_STD(r, base, i); \
} while(0)
#define PPC_BPF_STLU(r, base, i) do { PPC_STDU(r, base, i); } while(0)
#define SEEN_FUNC 0x1000 /* might call external helpers */ #define SEEN_FUNC 0x1000 /* might call external helpers */
#define SEEN_STACK 0x2000 /* uses BPF stack */ #define SEEN_STACK 0x2000 /* uses BPF stack */
#define SEEN_TAILCALL 0x4000 /* uses tail calls */ #define SEEN_TAILCALL 0x4000 /* uses tail calls */
......
...@@ -252,7 +252,7 @@ static void bpf_jit_emit_tail_call(u32 *image, struct codegen_context *ctx, u32 ...@@ -252,7 +252,7 @@ static void bpf_jit_emit_tail_call(u32 *image, struct codegen_context *ctx, u32
* if (tail_call_cnt > MAX_TAIL_CALL_CNT) * if (tail_call_cnt > MAX_TAIL_CALL_CNT)
* goto out; * goto out;
*/ */
PPC_LD(b2p[TMP_REG_1], 1, bpf_jit_stack_tailcallcnt(ctx)); PPC_BPF_LL(b2p[TMP_REG_1], 1, bpf_jit_stack_tailcallcnt(ctx));
PPC_CMPLWI(b2p[TMP_REG_1], MAX_TAIL_CALL_CNT); PPC_CMPLWI(b2p[TMP_REG_1], MAX_TAIL_CALL_CNT);
PPC_BCC(COND_GT, out); PPC_BCC(COND_GT, out);
...@@ -265,7 +265,7 @@ static void bpf_jit_emit_tail_call(u32 *image, struct codegen_context *ctx, u32 ...@@ -265,7 +265,7 @@ static void bpf_jit_emit_tail_call(u32 *image, struct codegen_context *ctx, u32
/* prog = array->ptrs[index]; */ /* prog = array->ptrs[index]; */
PPC_MULI(b2p[TMP_REG_1], b2p_index, 8); PPC_MULI(b2p[TMP_REG_1], b2p_index, 8);
PPC_ADD(b2p[TMP_REG_1], b2p[TMP_REG_1], b2p_bpf_array); PPC_ADD(b2p[TMP_REG_1], b2p[TMP_REG_1], b2p_bpf_array);
PPC_LD(b2p[TMP_REG_1], b2p[TMP_REG_1], offsetof(struct bpf_array, ptrs)); PPC_BPF_LL(b2p[TMP_REG_1], b2p[TMP_REG_1], offsetof(struct bpf_array, ptrs));
/* /*
* if (prog == NULL) * if (prog == NULL)
...@@ -275,7 +275,7 @@ static void bpf_jit_emit_tail_call(u32 *image, struct codegen_context *ctx, u32 ...@@ -275,7 +275,7 @@ static void bpf_jit_emit_tail_call(u32 *image, struct codegen_context *ctx, u32
PPC_BCC(COND_EQ, out); PPC_BCC(COND_EQ, out);
/* goto *(prog->bpf_func + prologue_size); */ /* goto *(prog->bpf_func + prologue_size); */
PPC_LD(b2p[TMP_REG_1], b2p[TMP_REG_1], offsetof(struct bpf_prog, bpf_func)); PPC_BPF_LL(b2p[TMP_REG_1], b2p[TMP_REG_1], offsetof(struct bpf_prog, bpf_func));
#ifdef PPC64_ELF_ABI_v1 #ifdef PPC64_ELF_ABI_v1
/* skip past the function descriptor */ /* skip past the function descriptor */
PPC_ADDI(b2p[TMP_REG_1], b2p[TMP_REG_1], PPC_ADDI(b2p[TMP_REG_1], b2p[TMP_REG_1],
...@@ -606,7 +606,7 @@ static int bpf_jit_build_body(struct bpf_prog *fp, u32 *image, ...@@ -606,7 +606,7 @@ static int bpf_jit_build_body(struct bpf_prog *fp, u32 *image,
* the instructions generated will remain the * the instructions generated will remain the
* same across all passes * same across all passes
*/ */
PPC_STD(dst_reg, 1, bpf_jit_stack_local(ctx)); PPC_BPF_STL(dst_reg, 1, bpf_jit_stack_local(ctx));
PPC_ADDI(b2p[TMP_REG_1], 1, bpf_jit_stack_local(ctx)); PPC_ADDI(b2p[TMP_REG_1], 1, bpf_jit_stack_local(ctx));
PPC_LDBRX(dst_reg, 0, b2p[TMP_REG_1]); PPC_LDBRX(dst_reg, 0, b2p[TMP_REG_1]);
break; break;
...@@ -662,7 +662,7 @@ static int bpf_jit_build_body(struct bpf_prog *fp, u32 *image, ...@@ -662,7 +662,7 @@ static int bpf_jit_build_body(struct bpf_prog *fp, u32 *image,
PPC_LI32(b2p[TMP_REG_1], imm); PPC_LI32(b2p[TMP_REG_1], imm);
src_reg = b2p[TMP_REG_1]; src_reg = b2p[TMP_REG_1];
} }
PPC_STD(src_reg, dst_reg, off); PPC_BPF_STL(src_reg, dst_reg, off);
break; break;
/* /*
...@@ -709,7 +709,7 @@ static int bpf_jit_build_body(struct bpf_prog *fp, u32 *image, ...@@ -709,7 +709,7 @@ static int bpf_jit_build_body(struct bpf_prog *fp, u32 *image,
break; break;
/* dst = *(u64 *)(ul) (src + off) */ /* dst = *(u64 *)(ul) (src + off) */
case BPF_LDX | BPF_MEM | BPF_DW: case BPF_LDX | BPF_MEM | BPF_DW:
PPC_LD(dst_reg, src_reg, off); PPC_BPF_LL(dst_reg, src_reg, off);
break; break;
/* /*
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment