Commit efd9e03f authored by Catalin Marinas's avatar Catalin Marinas Committed by Will Deacon

arm64: Use static keys for CPU features

This patch adds static keys transparently for all the cpu_hwcaps
features by implementing an array of default-false static keys and
enabling them when detected. The cpus_have_cap() check uses the static
keys if the feature being checked is a constant, otherwise the compiler
generates the bitmap test.

Because of the early call to static_branch_enable() via
check_local_cpu_errata() -> update_cpu_capabilities(), the jump labels
are initialised in cpuinfo_store_boot_cpu().

Cc: Will Deacon <will.deacon@arm.com>
Cc: Suzuki K. Poulose <Suzuki.Poulose@arm.com>
Signed-off-by: default avatarCatalin Marinas <catalin.marinas@arm.com>
Signed-off-by: default avatarWill Deacon <will.deacon@arm.com>
parent ef0da55a
...@@ -9,6 +9,8 @@ ...@@ -9,6 +9,8 @@
#ifndef __ASM_CPUFEATURE_H #ifndef __ASM_CPUFEATURE_H
#define __ASM_CPUFEATURE_H #define __ASM_CPUFEATURE_H
#include <linux/jump_label.h>
#include <asm/hwcap.h> #include <asm/hwcap.h>
#include <asm/sysreg.h> #include <asm/sysreg.h>
...@@ -110,6 +112,7 @@ struct arm64_cpu_capabilities { ...@@ -110,6 +112,7 @@ struct arm64_cpu_capabilities {
}; };
extern DECLARE_BITMAP(cpu_hwcaps, ARM64_NCAPS); extern DECLARE_BITMAP(cpu_hwcaps, ARM64_NCAPS);
extern struct static_key_false cpu_hwcap_keys[ARM64_NCAPS];
bool this_cpu_has_cap(unsigned int cap); bool this_cpu_has_cap(unsigned int cap);
...@@ -122,16 +125,21 @@ static inline bool cpus_have_cap(unsigned int num) ...@@ -122,16 +125,21 @@ static inline bool cpus_have_cap(unsigned int num)
{ {
if (num >= ARM64_NCAPS) if (num >= ARM64_NCAPS)
return false; return false;
if (__builtin_constant_p(num))
return static_branch_unlikely(&cpu_hwcap_keys[num]);
else
return test_bit(num, cpu_hwcaps); return test_bit(num, cpu_hwcaps);
} }
static inline void cpus_set_cap(unsigned int num) static inline void cpus_set_cap(unsigned int num)
{ {
if (num >= ARM64_NCAPS) if (num >= ARM64_NCAPS) {
pr_warn("Attempt to set an illegal CPU capability (%d >= %d)\n", pr_warn("Attempt to set an illegal CPU capability (%d >= %d)\n",
num, ARM64_NCAPS); num, ARM64_NCAPS);
else } else {
__set_bit(num, cpu_hwcaps); __set_bit(num, cpu_hwcaps);
static_branch_enable(&cpu_hwcap_keys[num]);
}
} }
static inline int __attribute_const__ static inline int __attribute_const__
......
...@@ -46,6 +46,9 @@ unsigned int compat_elf_hwcap2 __read_mostly; ...@@ -46,6 +46,9 @@ unsigned int compat_elf_hwcap2 __read_mostly;
DECLARE_BITMAP(cpu_hwcaps, ARM64_NCAPS); DECLARE_BITMAP(cpu_hwcaps, ARM64_NCAPS);
DEFINE_STATIC_KEY_ARRAY_FALSE(cpu_hwcap_keys, ARM64_NCAPS);
EXPORT_SYMBOL(cpu_hwcap_keys);
#define __ARM64_FTR_BITS(SIGNED, STRICT, TYPE, SHIFT, WIDTH, SAFE_VAL) \ #define __ARM64_FTR_BITS(SIGNED, STRICT, TYPE, SHIFT, WIDTH, SAFE_VAL) \
{ \ { \
.sign = SIGNED, \ .sign = SIGNED, \
......
...@@ -437,6 +437,11 @@ void __init smp_cpus_done(unsigned int max_cpus) ...@@ -437,6 +437,11 @@ void __init smp_cpus_done(unsigned int max_cpus)
void __init smp_prepare_boot_cpu(void) void __init smp_prepare_boot_cpu(void)
{ {
set_my_cpu_offset(per_cpu_offset(smp_processor_id())); set_my_cpu_offset(per_cpu_offset(smp_processor_id()));
/*
* Initialise the static keys early as they may be enabled by the
* cpufeature code.
*/
jump_label_init();
cpuinfo_store_boot_cpu(); cpuinfo_store_boot_cpu();
save_boot_cpu_run_el(); save_boot_cpu_run_el();
} }
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment