Commit fa9dd599 authored by Daniel Borkmann's avatar Daniel Borkmann Committed by Alexei Starovoitov

bpf: get rid of pure_initcall dependency to enable jits

Having a pure_initcall() callback just to permanently enable BPF
JITs under CONFIG_BPF_JIT_ALWAYS_ON is unnecessary and could leave
a small race window in future where JIT is still disabled on boot.
Since we know about the setting at compilation time anyway, just
initialize it properly there. Also consolidate all the individual
bpf_jit_enable variables into a single one and move them under one
location. Moreover, don't allow for setting unspecified garbage
values on them.
Signed-off-by: default avatarDaniel Borkmann <daniel@iogearbox.net>
Acked-by: default avatarAlexei Starovoitov <ast@kernel.org>
Signed-off-by: default avatarAlexei Starovoitov <ast@kernel.org>
parent 87c1793b
...@@ -25,8 +25,6 @@ ...@@ -25,8 +25,6 @@
#include "bpf_jit_32.h" #include "bpf_jit_32.h"
int bpf_jit_enable __read_mostly;
#define STACK_OFFSET(k) (k) #define STACK_OFFSET(k) (k)
#define TMP_REG_1 (MAX_BPF_JIT_REG + 0) /* TEMP Register 1 */ #define TMP_REG_1 (MAX_BPF_JIT_REG + 0) /* TEMP Register 1 */
#define TMP_REG_2 (MAX_BPF_JIT_REG + 1) /* TEMP Register 2 */ #define TMP_REG_2 (MAX_BPF_JIT_REG + 1) /* TEMP Register 2 */
......
...@@ -31,8 +31,6 @@ ...@@ -31,8 +31,6 @@
#include "bpf_jit.h" #include "bpf_jit.h"
int bpf_jit_enable __read_mostly;
#define TMP_REG_1 (MAX_BPF_JIT_REG + 0) #define TMP_REG_1 (MAX_BPF_JIT_REG + 0)
#define TMP_REG_2 (MAX_BPF_JIT_REG + 1) #define TMP_REG_2 (MAX_BPF_JIT_REG + 1)
#define TCALL_CNT (MAX_BPF_JIT_REG + 2) #define TCALL_CNT (MAX_BPF_JIT_REG + 2)
......
...@@ -1207,8 +1207,6 @@ static int build_body(struct jit_ctx *ctx) ...@@ -1207,8 +1207,6 @@ static int build_body(struct jit_ctx *ctx)
return 0; return 0;
} }
int bpf_jit_enable __read_mostly;
void bpf_jit_compile(struct bpf_prog *fp) void bpf_jit_compile(struct bpf_prog *fp)
{ {
struct jit_ctx ctx; struct jit_ctx ctx;
......
...@@ -177,8 +177,6 @@ static u32 b_imm(unsigned int tgt, struct jit_ctx *ctx) ...@@ -177,8 +177,6 @@ static u32 b_imm(unsigned int tgt, struct jit_ctx *ctx)
(ctx->idx * 4) - 4; (ctx->idx * 4) - 4;
} }
int bpf_jit_enable __read_mostly;
enum which_ebpf_reg { enum which_ebpf_reg {
src_reg, src_reg,
src_reg_no_fp, src_reg_no_fp,
......
...@@ -18,8 +18,6 @@ ...@@ -18,8 +18,6 @@
#include "bpf_jit32.h" #include "bpf_jit32.h"
int bpf_jit_enable __read_mostly;
static inline void bpf_flush_icache(void *start, void *end) static inline void bpf_flush_icache(void *start, void *end)
{ {
smp_wmb(); smp_wmb();
......
...@@ -21,8 +21,6 @@ ...@@ -21,8 +21,6 @@
#include "bpf_jit64.h" #include "bpf_jit64.h"
int bpf_jit_enable __read_mostly;
static void bpf_jit_fill_ill_insns(void *area, unsigned int size) static void bpf_jit_fill_ill_insns(void *area, unsigned int size)
{ {
memset32(area, BREAKPOINT_INSTRUCTION, size/4); memset32(area, BREAKPOINT_INSTRUCTION, size/4);
......
...@@ -28,8 +28,6 @@ ...@@ -28,8 +28,6 @@
#include <asm/set_memory.h> #include <asm/set_memory.h>
#include "bpf_jit.h" #include "bpf_jit.h"
int bpf_jit_enable __read_mostly;
struct bpf_jit { struct bpf_jit {
u32 seen; /* Flags to remember seen eBPF instructions */ u32 seen; /* Flags to remember seen eBPF instructions */
u32 seen_reg[16]; /* Array to remember which registers are used */ u32 seen_reg[16]; /* Array to remember which registers are used */
......
...@@ -11,8 +11,6 @@ ...@@ -11,8 +11,6 @@
#include "bpf_jit_32.h" #include "bpf_jit_32.h"
int bpf_jit_enable __read_mostly;
static inline bool is_simm13(unsigned int value) static inline bool is_simm13(unsigned int value)
{ {
return value + 0x1000 < 0x2000; return value + 0x1000 < 0x2000;
......
...@@ -12,8 +12,6 @@ ...@@ -12,8 +12,6 @@
#include "bpf_jit_64.h" #include "bpf_jit_64.h"
int bpf_jit_enable __read_mostly;
static inline bool is_simm13(unsigned int value) static inline bool is_simm13(unsigned int value)
{ {
return value + 0x1000 < 0x2000; return value + 0x1000 < 0x2000;
......
...@@ -15,8 +15,6 @@ ...@@ -15,8 +15,6 @@
#include <asm/set_memory.h> #include <asm/set_memory.h>
#include <linux/bpf.h> #include <linux/bpf.h>
int bpf_jit_enable __read_mostly;
/* /*
* assembly code in arch/x86/net/bpf_jit.S * assembly code in arch/x86/net/bpf_jit.S
*/ */
......
...@@ -300,6 +300,11 @@ struct bpf_prog *bpf_patch_insn_single(struct bpf_prog *prog, u32 off, ...@@ -300,6 +300,11 @@ struct bpf_prog *bpf_patch_insn_single(struct bpf_prog *prog, u32 off,
} }
#ifdef CONFIG_BPF_JIT #ifdef CONFIG_BPF_JIT
/* All BPF JIT sysctl knobs here. */
int bpf_jit_enable __read_mostly = IS_BUILTIN(CONFIG_BPF_JIT_ALWAYS_ON);
int bpf_jit_harden __read_mostly;
int bpf_jit_kallsyms __read_mostly;
static __always_inline void static __always_inline void
bpf_get_prog_addr_region(const struct bpf_prog *prog, bpf_get_prog_addr_region(const struct bpf_prog *prog,
unsigned long *symbol_start, unsigned long *symbol_start,
...@@ -381,8 +386,6 @@ static DEFINE_SPINLOCK(bpf_lock); ...@@ -381,8 +386,6 @@ static DEFINE_SPINLOCK(bpf_lock);
static LIST_HEAD(bpf_kallsyms); static LIST_HEAD(bpf_kallsyms);
static struct latch_tree_root bpf_tree __cacheline_aligned; static struct latch_tree_root bpf_tree __cacheline_aligned;
int bpf_jit_kallsyms __read_mostly;
static void bpf_prog_ksym_node_add(struct bpf_prog_aux *aux) static void bpf_prog_ksym_node_add(struct bpf_prog_aux *aux)
{ {
WARN_ON_ONCE(!list_empty(&aux->ksym_lnode)); WARN_ON_ONCE(!list_empty(&aux->ksym_lnode));
...@@ -563,8 +566,6 @@ void __weak bpf_jit_free(struct bpf_prog *fp) ...@@ -563,8 +566,6 @@ void __weak bpf_jit_free(struct bpf_prog *fp)
bpf_prog_unlock_free(fp); bpf_prog_unlock_free(fp);
} }
int bpf_jit_harden __read_mostly;
static int bpf_jit_blind_insn(const struct bpf_insn *from, static int bpf_jit_blind_insn(const struct bpf_insn *from,
const struct bpf_insn *aux, const struct bpf_insn *aux,
struct bpf_insn *to_buff) struct bpf_insn *to_buff)
...@@ -1379,9 +1380,13 @@ void bpf_patch_call_args(struct bpf_insn *insn, u32 stack_depth) ...@@ -1379,9 +1380,13 @@ void bpf_patch_call_args(struct bpf_insn *insn, u32 stack_depth)
} }
#else #else
static unsigned int __bpf_prog_ret0(const void *ctx, static unsigned int __bpf_prog_ret0_warn(const void *ctx,
const struct bpf_insn *insn) const struct bpf_insn *insn)
{ {
/* If this handler ever gets executed, then BPF_JIT_ALWAYS_ON
* is not working properly, so warn about it!
*/
WARN_ON_ONCE(1);
return 0; return 0;
} }
#endif #endif
...@@ -1441,7 +1446,7 @@ struct bpf_prog *bpf_prog_select_runtime(struct bpf_prog *fp, int *err) ...@@ -1441,7 +1446,7 @@ struct bpf_prog *bpf_prog_select_runtime(struct bpf_prog *fp, int *err)
fp->bpf_func = interpreters[(round_up(stack_depth, 32) / 32) - 1]; fp->bpf_func = interpreters[(round_up(stack_depth, 32) / 32) - 1];
#else #else
fp->bpf_func = __bpf_prog_ret0; fp->bpf_func = __bpf_prog_ret0_warn;
#endif #endif
/* eBPF JITs can rewrite the program in case constant /* eBPF JITs can rewrite the program in case constant
......
...@@ -25,6 +25,7 @@ ...@@ -25,6 +25,7 @@
static int zero = 0; static int zero = 0;
static int one = 1; static int one = 1;
static int two __maybe_unused = 2;
static int min_sndbuf = SOCK_MIN_SNDBUF; static int min_sndbuf = SOCK_MIN_SNDBUF;
static int min_rcvbuf = SOCK_MIN_RCVBUF; static int min_rcvbuf = SOCK_MIN_RCVBUF;
static int max_skb_frags = MAX_SKB_FRAGS; static int max_skb_frags = MAX_SKB_FRAGS;
...@@ -325,13 +326,14 @@ static struct ctl_table net_core_table[] = { ...@@ -325,13 +326,14 @@ static struct ctl_table net_core_table[] = {
.data = &bpf_jit_enable, .data = &bpf_jit_enable,
.maxlen = sizeof(int), .maxlen = sizeof(int),
.mode = 0644, .mode = 0644,
#ifndef CONFIG_BPF_JIT_ALWAYS_ON
.proc_handler = proc_dointvec
#else
.proc_handler = proc_dointvec_minmax, .proc_handler = proc_dointvec_minmax,
# ifdef CONFIG_BPF_JIT_ALWAYS_ON
.extra1 = &one, .extra1 = &one,
.extra2 = &one, .extra2 = &one,
#endif # else
.extra1 = &zero,
.extra2 = &two,
# endif
}, },
# ifdef CONFIG_HAVE_EBPF_JIT # ifdef CONFIG_HAVE_EBPF_JIT
{ {
...@@ -339,14 +341,18 @@ static struct ctl_table net_core_table[] = { ...@@ -339,14 +341,18 @@ static struct ctl_table net_core_table[] = {
.data = &bpf_jit_harden, .data = &bpf_jit_harden,
.maxlen = sizeof(int), .maxlen = sizeof(int),
.mode = 0600, .mode = 0600,
.proc_handler = proc_dointvec, .proc_handler = proc_dointvec_minmax,
.extra1 = &zero,
.extra2 = &two,
}, },
{ {
.procname = "bpf_jit_kallsyms", .procname = "bpf_jit_kallsyms",
.data = &bpf_jit_kallsyms, .data = &bpf_jit_kallsyms,
.maxlen = sizeof(int), .maxlen = sizeof(int),
.mode = 0600, .mode = 0600,
.proc_handler = proc_dointvec, .proc_handler = proc_dointvec_minmax,
.extra1 = &zero,
.extra2 = &one,
}, },
# endif # endif
#endif #endif
......
...@@ -2613,15 +2613,6 @@ static int __init sock_init(void) ...@@ -2613,15 +2613,6 @@ static int __init sock_init(void)
core_initcall(sock_init); /* early initcall */ core_initcall(sock_init); /* early initcall */
static int __init jit_init(void)
{
#ifdef CONFIG_BPF_JIT_ALWAYS_ON
bpf_jit_enable = 1;
#endif
return 0;
}
pure_initcall(jit_init);
#ifdef CONFIG_PROC_FS #ifdef CONFIG_PROC_FS
void socket_seq_show(struct seq_file *seq) void socket_seq_show(struct seq_file *seq)
{ {
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment