• Zi Shen Lim's avatar
    arm64: bpf: implement bpf_tail_call() helper · ddb55992
    Zi Shen Lim authored
    Add support for JMP_CALL_X (tail call) introduced by commit 04fd61ab
    ("bpf: allow bpf programs to tail-call other bpf programs").
    
    bpf_tail_call() arguments:
      ctx   - context pointer passed to next program
      array - pointer to map which type is BPF_MAP_TYPE_PROG_ARRAY
      index - index inside array that selects specific program to run
    
    In this implementation arm64 JIT jumps into callee program after prologue,
    so callee program reuses the same stack. For tail_call_cnt, we use the
    callee-saved R26 (which was already saved/restored but previously unused
    by JIT).
    
    With this patch a tail call generates the following code on arm64:
    
      if (index >= array->map.max_entries)
          goto out;
    
      34:   mov     x10, #0x10                      // #16
      38:   ldr     w10, [x1,x10]
      3c:   cmp     w2, w10
      40:   b.ge    0x0000000000000074
    
      if (tail_call_cnt > MAX_TAIL_CALL_CNT)
          goto out;
      tail_call_cnt++;
    
      44:   mov     x10, #0x20                      // #32
      48:   cmp     x26, x10
      4c:   b.gt    0x0000000000000074
      50:   add     x26, x26, #0x1
    
      prog = array->ptrs[index];
      if (prog == NULL)
          goto out;
    
      54:   mov     x10, #0x68                      // #104
      58:   ldr     x10, [x1,x10]
      5c:   ldr     x11, [x10,x2]
      60:   cbz     x11, 0x0000000000000074
    
      goto *(prog->bpf_func + prologue_size);
    
      64:   mov     x10, #0x20                      // #32
      68:   ldr     x10, [x11,x10]
      6c:   add     x10, x10, #0x20
      70:   br      x10
      74:
    Signed-off-by: default avatarZi Shen Lim <zlim.lnx@gmail.com>
    Signed-off-by: default avatarDavid S. Miller <davem@davemloft.net>
    ddb55992
bpf_jit_comp.c 23.3 KB