Commit fe8d9571 authored by Alexei Starovoitov's avatar Alexei Starovoitov

bpf, x64: fix stack layout of JITed bpf code

Since commit 177366bf the %rbp stopped pointing to %rbp of the
previous stack frame. That broke frame pointer based stack unwinding.
This commit is a partial revert of it.
Note that the location of tail_call_cnt is fixed, since the verifier
enforces MAX_BPF_STACK stack size for programs with tail calls.

Fixes: 177366bf ("bpf: change x86 JITed program stack layout")
Signed-off-by: default avatarAlexei Starovoitov <ast@kernel.org>
parent 86723c86
...@@ -190,9 +190,7 @@ struct jit_context { ...@@ -190,9 +190,7 @@ struct jit_context {
#define BPF_MAX_INSN_SIZE 128 #define BPF_MAX_INSN_SIZE 128
#define BPF_INSN_SAFETY 64 #define BPF_INSN_SAFETY 64
#define AUX_STACK_SPACE 40 /* Space for RBX, R13, R14, R15, tailcnt */ #define PROLOGUE_SIZE 20
#define PROLOGUE_SIZE 37
/* /*
* Emit x86-64 prologue code for BPF program and check its size. * Emit x86-64 prologue code for BPF program and check its size.
...@@ -203,44 +201,19 @@ static void emit_prologue(u8 **pprog, u32 stack_depth, bool ebpf_from_cbpf) ...@@ -203,44 +201,19 @@ static void emit_prologue(u8 **pprog, u32 stack_depth, bool ebpf_from_cbpf)
u8 *prog = *pprog; u8 *prog = *pprog;
int cnt = 0; int cnt = 0;
/* push rbp */ EMIT1(0x55); /* push rbp */
EMIT1(0x55); EMIT3(0x48, 0x89, 0xE5); /* mov rbp, rsp */
/* sub rsp, rounded_stack_depth */
/* mov rbp,rsp */ EMIT3_off32(0x48, 0x81, 0xEC, round_up(stack_depth, 8));
EMIT3(0x48, 0x89, 0xE5); EMIT1(0x53); /* push rbx */
EMIT2(0x41, 0x55); /* push r13 */
/* sub rsp, rounded_stack_depth + AUX_STACK_SPACE */ EMIT2(0x41, 0x56); /* push r14 */
EMIT3_off32(0x48, 0x81, 0xEC, EMIT2(0x41, 0x57); /* push r15 */
round_up(stack_depth, 8) + AUX_STACK_SPACE);
/* sub rbp, AUX_STACK_SPACE */
EMIT4(0x48, 0x83, 0xED, AUX_STACK_SPACE);
/* mov qword ptr [rbp+0],rbx */
EMIT4(0x48, 0x89, 0x5D, 0);
/* mov qword ptr [rbp+8],r13 */
EMIT4(0x4C, 0x89, 0x6D, 8);
/* mov qword ptr [rbp+16],r14 */
EMIT4(0x4C, 0x89, 0x75, 16);
/* mov qword ptr [rbp+24],r15 */
EMIT4(0x4C, 0x89, 0x7D, 24);
if (!ebpf_from_cbpf) { if (!ebpf_from_cbpf) {
/* /* zero init tail_call_cnt */
* Clear the tail call counter (tail_call_cnt): for eBPF tail EMIT2(0x6a, 0x00);
* calls we need to reset the counter to 0. It's done in two
* instructions, resetting RAX register to 0, and moving it
* to the counter location.
*/
/* xor eax, eax */
EMIT2(0x31, 0xc0);
/* mov qword ptr [rbp+32], rax */
EMIT4(0x48, 0x89, 0x45, 32);
BUILD_BUG_ON(cnt != PROLOGUE_SIZE); BUILD_BUG_ON(cnt != PROLOGUE_SIZE);
} }
*pprog = prog; *pprog = prog;
} }
...@@ -285,13 +258,13 @@ static void emit_bpf_tail_call(u8 **pprog) ...@@ -285,13 +258,13 @@ static void emit_bpf_tail_call(u8 **pprog)
* if (tail_call_cnt > MAX_TAIL_CALL_CNT) * if (tail_call_cnt > MAX_TAIL_CALL_CNT)
* goto out; * goto out;
*/ */
EMIT2_off32(0x8B, 0x85, 36); /* mov eax, dword ptr [rbp + 36] */ EMIT2_off32(0x8B, 0x85, -36 - MAX_BPF_STACK); /* mov eax, dword ptr [rbp - 548] */
EMIT3(0x83, 0xF8, MAX_TAIL_CALL_CNT); /* cmp eax, MAX_TAIL_CALL_CNT */ EMIT3(0x83, 0xF8, MAX_TAIL_CALL_CNT); /* cmp eax, MAX_TAIL_CALL_CNT */
#define OFFSET2 (30 + RETPOLINE_RAX_BPF_JIT_SIZE) #define OFFSET2 (30 + RETPOLINE_RAX_BPF_JIT_SIZE)
EMIT2(X86_JA, OFFSET2); /* ja out */ EMIT2(X86_JA, OFFSET2); /* ja out */
label2 = cnt; label2 = cnt;
EMIT3(0x83, 0xC0, 0x01); /* add eax, 1 */ EMIT3(0x83, 0xC0, 0x01); /* add eax, 1 */
EMIT2_off32(0x89, 0x85, 36); /* mov dword ptr [rbp + 36], eax */ EMIT2_off32(0x89, 0x85, -36 - MAX_BPF_STACK); /* mov dword ptr [rbp -548], eax */
/* prog = array->ptrs[index]; */ /* prog = array->ptrs[index]; */
EMIT4_off32(0x48, 0x8B, 0x84, 0xD6, /* mov rax, [rsi + rdx * 8 + offsetof(...)] */ EMIT4_off32(0x48, 0x8B, 0x84, 0xD6, /* mov rax, [rsi + rdx * 8 + offsetof(...)] */
...@@ -1040,19 +1013,14 @@ xadd: if (is_imm8(insn->off)) ...@@ -1040,19 +1013,14 @@ xadd: if (is_imm8(insn->off))
seen_exit = true; seen_exit = true;
/* Update cleanup_addr */ /* Update cleanup_addr */
ctx->cleanup_addr = proglen; ctx->cleanup_addr = proglen;
/* mov rbx, qword ptr [rbp+0] */ if (!bpf_prog_was_classic(bpf_prog))
EMIT4(0x48, 0x8B, 0x5D, 0); EMIT1(0x5B); /* get rid of tail_call_cnt */
/* mov r13, qword ptr [rbp+8] */ EMIT2(0x41, 0x5F); /* pop r15 */
EMIT4(0x4C, 0x8B, 0x6D, 8); EMIT2(0x41, 0x5E); /* pop r14 */
/* mov r14, qword ptr [rbp+16] */ EMIT2(0x41, 0x5D); /* pop r13 */
EMIT4(0x4C, 0x8B, 0x75, 16); EMIT1(0x5B); /* pop rbx */
/* mov r15, qword ptr [rbp+24] */ EMIT1(0xC9); /* leave */
EMIT4(0x4C, 0x8B, 0x7D, 24); EMIT1(0xC3); /* ret */
/* add rbp, AUX_STACK_SPACE */
EMIT4(0x48, 0x83, 0xC5, AUX_STACK_SPACE);
EMIT1(0xC9); /* leave */
EMIT1(0xC3); /* ret */
break; break;
default: default:
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment