Commit a1efb01f authored by Peter Zijlstra's avatar Peter Zijlstra Committed by Ingo Molnar

jump_label, locking/static_keys: Rename JUMP_LABEL_TYPE_* and related helpers...

jump_label, locking/static_keys: Rename JUMP_LABEL_TYPE_* and related helpers to the static_key* pattern

Rename the JUMP_LABEL_TYPE_* macros to be JUMP_TYPE_* and move the
inline helpers into kernel/jump_label.c, since that's the only place
they're ever used.

Also rename the helpers where it's all about static keys.

This is the second step in removing the naming confusion that has led to
a stream of avoidable bugs such as:

  a833581e ("x86, perf: Fix static_key bug in load_mm_cr4()")
Signed-off-by: default avatarPeter Zijlstra (Intel) <peterz@infradead.org>
Cc: Andrew Morton <akpm@linux-foundation.org>
Cc: Linus Torvalds <torvalds@linux-foundation.org>
Cc: Paul E. McKenney <paulmck@linux.vnet.ibm.com>
Cc: Peter Zijlstra <peterz@infradead.org>
Cc: Thomas Gleixner <tglx@linutronix.de>
Cc: linux-kernel@vger.kernel.org
Signed-off-by: default avatarIngo Molnar <mingo@kernel.org>
parent 76b235c6
...@@ -101,24 +101,9 @@ static inline int static_key_count(struct static_key *key) ...@@ -101,24 +101,9 @@ static inline int static_key_count(struct static_key *key)
#ifdef HAVE_JUMP_LABEL #ifdef HAVE_JUMP_LABEL
#define JUMP_LABEL_TYPE_FALSE_BRANCH 0UL #define JUMP_TYPE_FALSE 0UL
#define JUMP_LABEL_TYPE_TRUE_BRANCH 1UL #define JUMP_TYPE_TRUE 1UL
#define JUMP_LABEL_TYPE_MASK 1UL #define JUMP_TYPE_MASK 1UL
static
inline struct jump_entry *jump_label_get_entries(struct static_key *key)
{
return (struct jump_entry *)((unsigned long)key->entries
& ~JUMP_LABEL_TYPE_MASK);
}
static inline bool jump_label_get_branch_default(struct static_key *key)
{
if (((unsigned long)key->entries & JUMP_LABEL_TYPE_MASK) ==
JUMP_LABEL_TYPE_TRUE_BRANCH)
return true;
return false;
}
static __always_inline bool static_key_false(struct static_key *key) static __always_inline bool static_key_false(struct static_key *key)
{ {
...@@ -147,10 +132,10 @@ extern void jump_label_apply_nops(struct module *mod); ...@@ -147,10 +132,10 @@ extern void jump_label_apply_nops(struct module *mod);
#define STATIC_KEY_INIT_TRUE ((struct static_key) \ #define STATIC_KEY_INIT_TRUE ((struct static_key) \
{ .enabled = ATOMIC_INIT(1), \ { .enabled = ATOMIC_INIT(1), \
.entries = (void *)JUMP_LABEL_TYPE_TRUE_BRANCH }) .entries = (void *)JUMP_TYPE_TRUE })
#define STATIC_KEY_INIT_FALSE ((struct static_key) \ #define STATIC_KEY_INIT_FALSE ((struct static_key) \
{ .enabled = ATOMIC_INIT(0), \ { .enabled = ATOMIC_INIT(0), \
.entries = (void *)JUMP_LABEL_TYPE_FALSE_BRANCH }) .entries = (void *)JUMP_TYPE_FALSE })
#else /* !HAVE_JUMP_LABEL */ #else /* !HAVE_JUMP_LABEL */
......
...@@ -56,6 +56,11 @@ jump_label_sort_entries(struct jump_entry *start, struct jump_entry *stop) ...@@ -56,6 +56,11 @@ jump_label_sort_entries(struct jump_entry *start, struct jump_entry *stop)
static void jump_label_update(struct static_key *key, int enable); static void jump_label_update(struct static_key *key, int enable);
static inline bool static_key_type(struct static_key *key)
{
return (unsigned long)key->entries & JUMP_TYPE_MASK;
}
void static_key_slow_inc(struct static_key *key) void static_key_slow_inc(struct static_key *key)
{ {
STATIC_KEY_CHECK_USE(); STATIC_KEY_CHECK_USE();
...@@ -64,7 +69,7 @@ void static_key_slow_inc(struct static_key *key) ...@@ -64,7 +69,7 @@ void static_key_slow_inc(struct static_key *key)
jump_label_lock(); jump_label_lock();
if (atomic_read(&key->enabled) == 0) { if (atomic_read(&key->enabled) == 0) {
if (!jump_label_get_branch_default(key)) if (!static_key_type(key))
jump_label_update(key, JUMP_LABEL_JMP); jump_label_update(key, JUMP_LABEL_JMP);
else else
jump_label_update(key, JUMP_LABEL_NOP); jump_label_update(key, JUMP_LABEL_NOP);
...@@ -87,7 +92,7 @@ static void __static_key_slow_dec(struct static_key *key, ...@@ -87,7 +92,7 @@ static void __static_key_slow_dec(struct static_key *key,
atomic_inc(&key->enabled); atomic_inc(&key->enabled);
schedule_delayed_work(work, rate_limit); schedule_delayed_work(work, rate_limit);
} else { } else {
if (!jump_label_get_branch_default(key)) if (!static_key_type(key))
jump_label_update(key, JUMP_LABEL_NOP); jump_label_update(key, JUMP_LABEL_NOP);
else else
jump_label_update(key, JUMP_LABEL_JMP); jump_label_update(key, JUMP_LABEL_JMP);
...@@ -178,15 +183,17 @@ static void __jump_label_update(struct static_key *key, ...@@ -178,15 +183,17 @@ static void __jump_label_update(struct static_key *key,
} }
} }
static enum jump_label_type jump_label_type(struct static_key *key) static inline struct jump_entry *static_key_entries(struct static_key *key)
{ {
bool true_branch = jump_label_get_branch_default(key); return (struct jump_entry *)((unsigned long)key->entries & ~JUMP_TYPE_MASK);
bool state = static_key_enabled(key); }
if ((!true_branch && state) || (true_branch && !state)) static enum jump_label_type jump_label_type(struct static_key *key)
return JUMP_LABEL_JMP; {
bool enabled = static_key_enabled(key);
bool type = static_key_type(key);
return JUMP_LABEL_NOP; return enabled ^ type;
} }
void __init jump_label_init(void) void __init jump_label_init(void)
...@@ -442,7 +449,7 @@ int jump_label_text_reserved(void *start, void *end) ...@@ -442,7 +449,7 @@ int jump_label_text_reserved(void *start, void *end)
static void jump_label_update(struct static_key *key, int enable) static void jump_label_update(struct static_key *key, int enable)
{ {
struct jump_entry *stop = __stop___jump_table; struct jump_entry *stop = __stop___jump_table;
struct jump_entry *entry = jump_label_get_entries(key); struct jump_entry *entry = static_key_entries(key);
#ifdef CONFIG_MODULES #ifdef CONFIG_MODULES
struct module *mod; struct module *mod;
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment