Commit 865dad20 authored by Linus Torvalds's avatar Linus Torvalds

Merge tag 'kcfi-v6.1-rc1' of git://git.kernel.org/pub/scm/linux/kernel/git/kees/linux

Pull kcfi updates from Kees Cook:
 "This replaces the prior support for Clang's standard Control Flow
  Integrity (CFI) instrumentation, which has required a lot of special
  conditions (e.g. LTO) and work-arounds.

  The new implementation ("Kernel CFI") is specific to C, directly
  designed for the Linux kernel, and takes advantage of architectural
  features like x86's IBT. This series retains arm64 support and adds
  x86 support.

  GCC support is expected in the future[1], and additional "generic"
  architectural support is expected soon[2].

  Summary:

   - treewide: Remove old CFI support details

   - arm64: Replace Clang CFI support with Clang KCFI support

   - x86: Introduce Clang KCFI support"

Link: https://gcc.gnu.org/bugzilla/show_bug.cgi?id=107048 [1]
Link: https://github.com/samitolvanen/llvm-project/commits/kcfi_generic [2]

* tag 'kcfi-v6.1-rc1' of git://git.kernel.org/pub/scm/linux/kernel/git/kees/linux: (22 commits)
  x86: Add support for CONFIG_CFI_CLANG
  x86/purgatory: Disable CFI
  x86: Add types to indirectly called assembly functions
  x86/tools/relocs: Ignore __kcfi_typeid_ relocations
  kallsyms: Drop CONFIG_CFI_CLANG workarounds
  objtool: Disable CFI warnings
  objtool: Preserve special st_shndx indexes in elf_update_symbol
  treewide: Drop __cficanonical
  treewide: Drop WARN_ON_FUNCTION_MISMATCH
  treewide: Drop function_nocfi
  init: Drop __nocfi from __init
  arm64: Drop unneeded __nocfi attributes
  arm64: Add CFI error handling
  arm64: Add types to indirect called assembly functions
  psci: Fix the function type for psci_initcall_t
  lkdtm: Emit an indirect call for CFI tests
  cfi: Add type helper macros
  cfi: Switch to -fsanitize=kcfi
  cfi: Drop __CFI_ADDRESSABLE
  cfi: Remove CONFIG_CFI_CLANG_SHADOW
  ...
parents 12ed00ba 3c516f89
...@@ -1001,18 +1001,7 @@ export CC_FLAGS_LTO ...@@ -1001,18 +1001,7 @@ export CC_FLAGS_LTO
endif endif
ifdef CONFIG_CFI_CLANG ifdef CONFIG_CFI_CLANG
CC_FLAGS_CFI := -fsanitize=cfi \ CC_FLAGS_CFI := -fsanitize=kcfi
-fsanitize-cfi-cross-dso \
-fno-sanitize-cfi-canonical-jump-tables \
-fno-sanitize-trap=cfi \
-fno-sanitize-blacklist
ifdef CONFIG_CFI_PERMISSIVE
CC_FLAGS_CFI += -fsanitize-recover=cfi
endif
# If LTO flags are filtered out, we must also filter out CFI.
CC_FLAGS_LTO += $(CC_FLAGS_CFI)
KBUILD_CFLAGS += $(CC_FLAGS_CFI) KBUILD_CFLAGS += $(CC_FLAGS_CFI)
export CC_FLAGS_CFI export CC_FLAGS_CFI
endif endif
......
...@@ -744,11 +744,13 @@ config ARCH_SUPPORTS_CFI_CLANG ...@@ -744,11 +744,13 @@ config ARCH_SUPPORTS_CFI_CLANG
An architecture should select this option if it can support Clang's An architecture should select this option if it can support Clang's
Control-Flow Integrity (CFI) checking. Control-Flow Integrity (CFI) checking.
config ARCH_USES_CFI_TRAPS
bool
config CFI_CLANG config CFI_CLANG
bool "Use Clang's Control Flow Integrity (CFI)" bool "Use Clang's Control Flow Integrity (CFI)"
depends on LTO_CLANG && ARCH_SUPPORTS_CFI_CLANG depends on ARCH_SUPPORTS_CFI_CLANG
depends on CLANG_VERSION >= 140000 depends on $(cc-option,-fsanitize=kcfi)
select KALLSYMS
help help
This option enables Clangs forward-edge Control Flow Integrity This option enables Clangs forward-edge Control Flow Integrity
(CFI) checking, where the compiler injects a runtime check to each (CFI) checking, where the compiler injects a runtime check to each
...@@ -760,16 +762,6 @@ config CFI_CLANG ...@@ -760,16 +762,6 @@ config CFI_CLANG
https://clang.llvm.org/docs/ControlFlowIntegrity.html https://clang.llvm.org/docs/ControlFlowIntegrity.html
config CFI_CLANG_SHADOW
bool "Use CFI shadow to speed up cross-module checks"
default y
depends on CFI_CLANG && MODULES
help
If you select this option, the kernel builds a fast look-up table of
CFI check functions in loaded modules to reduce performance overhead.
If unsure, say Y.
config CFI_PERMISSIVE config CFI_PERMISSIVE
bool "Use CFI in permissive mode" bool "Use CFI in permissive mode"
depends on CFI_CLANG depends on CFI_CLANG
......
...@@ -6,6 +6,7 @@ ...@@ -6,6 +6,7 @@
*/ */
#include <linux/linkage.h> #include <linux/linkage.h>
#include <linux/cfi_types.h>
#include <asm/assembler.h> #include <asm/assembler.h>
SHASH .req v0 SHASH .req v0
...@@ -350,11 +351,11 @@ CPU_LE( rev64 T1.16b, T1.16b ) ...@@ -350,11 +351,11 @@ CPU_LE( rev64 T1.16b, T1.16b )
* void pmull_ghash_update(int blocks, u64 dg[], const char *src, * void pmull_ghash_update(int blocks, u64 dg[], const char *src,
* struct ghash_key const *k, const char *head) * struct ghash_key const *k, const char *head)
*/ */
SYM_FUNC_START(pmull_ghash_update_p64) SYM_TYPED_FUNC_START(pmull_ghash_update_p64)
__pmull_ghash p64 __pmull_ghash p64
SYM_FUNC_END(pmull_ghash_update_p64) SYM_FUNC_END(pmull_ghash_update_p64)
SYM_FUNC_START(pmull_ghash_update_p8) SYM_TYPED_FUNC_START(pmull_ghash_update_p8)
__pmull_ghash p8 __pmull_ghash p8
SYM_FUNC_END(pmull_ghash_update_p8) SYM_FUNC_END(pmull_ghash_update_p8)
......
...@@ -6,6 +6,7 @@ ...@@ -6,6 +6,7 @@
*/ */
#include <linux/linkage.h> #include <linux/linkage.h>
#include <linux/cfi_types.h>
#include <asm/assembler.h> #include <asm/assembler.h>
.irp b, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12 .irp b, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12
...@@ -73,7 +74,7 @@ ...@@ -73,7 +74,7 @@
* int blocks) * int blocks)
*/ */
.text .text
SYM_FUNC_START(sm3_ce_transform) SYM_TYPED_FUNC_START(sm3_ce_transform)
/* load state */ /* load state */
ld1 {v8.4s-v9.4s}, [x0] ld1 {v8.4s-v9.4s}, [x0]
rev64 v8.4s, v8.4s rev64 v8.4s, v8.4s
......
...@@ -17,6 +17,7 @@ ...@@ -17,6 +17,7 @@
* 0x401: for compile time BRK instruction * 0x401: for compile time BRK instruction
* 0x800: kernel-mode BUG() and WARN() traps * 0x800: kernel-mode BUG() and WARN() traps
* 0x9xx: tag-based KASAN trap (allowed values 0x900 - 0x9ff) * 0x9xx: tag-based KASAN trap (allowed values 0x900 - 0x9ff)
* 0x8xxx: Control-Flow Integrity traps
*/ */
#define KPROBES_BRK_IMM 0x004 #define KPROBES_BRK_IMM 0x004
#define UPROBES_BRK_IMM 0x005 #define UPROBES_BRK_IMM 0x005
...@@ -28,4 +29,9 @@ ...@@ -28,4 +29,9 @@
#define KASAN_BRK_IMM 0x900 #define KASAN_BRK_IMM 0x900
#define KASAN_BRK_MASK 0x0ff #define KASAN_BRK_MASK 0x0ff
#define CFI_BRK_IMM_TARGET GENMASK(4, 0)
#define CFI_BRK_IMM_TYPE GENMASK(9, 5)
#define CFI_BRK_IMM_BASE 0x8000
#define CFI_BRK_IMM_MASK (CFI_BRK_IMM_TARGET | CFI_BRK_IMM_TYPE)
#endif #endif
...@@ -26,7 +26,7 @@ ...@@ -26,7 +26,7 @@
#ifdef CONFIG_DYNAMIC_FTRACE_WITH_REGS #ifdef CONFIG_DYNAMIC_FTRACE_WITH_REGS
#define ARCH_SUPPORTS_FTRACE_OPS 1 #define ARCH_SUPPORTS_FTRACE_OPS 1
#else #else
#define MCOUNT_ADDR ((unsigned long)function_nocfi(_mcount)) #define MCOUNT_ADDR ((unsigned long)_mcount)
#endif #endif
/* The BL at the callsite's adjusted rec->ip */ /* The BL at the callsite's adjusted rec->ip */
......
...@@ -39,4 +39,8 @@ ...@@ -39,4 +39,8 @@
SYM_START(name, SYM_L_WEAK, SYM_A_NONE) \ SYM_START(name, SYM_L_WEAK, SYM_A_NONE) \
bti c ; bti c ;
#define SYM_TYPED_FUNC_START(name) \
SYM_TYPED_START(name, SYM_L_GLOBAL, SYM_A_ALIGN) \
bti c ;
#endif #endif
...@@ -147,7 +147,7 @@ static inline void cpu_install_ttbr0(phys_addr_t ttbr0, unsigned long t0sz) ...@@ -147,7 +147,7 @@ static inline void cpu_install_ttbr0(phys_addr_t ttbr0, unsigned long t0sz)
* Atomically replaces the active TTBR1_EL1 PGD with a new VA-compatible PGD, * Atomically replaces the active TTBR1_EL1 PGD with a new VA-compatible PGD,
* avoiding the possibility of conflicting TLB entries being allocated. * avoiding the possibility of conflicting TLB entries being allocated.
*/ */
static inline void __nocfi cpu_replace_ttbr1(pgd_t *pgdp, pgd_t *idmap) static inline void cpu_replace_ttbr1(pgd_t *pgdp, pgd_t *idmap)
{ {
typedef void (ttbr_replace_func)(phys_addr_t); typedef void (ttbr_replace_func)(phys_addr_t);
extern ttbr_replace_func idmap_cpu_replace_ttbr1; extern ttbr_replace_func idmap_cpu_replace_ttbr1;
...@@ -168,7 +168,7 @@ static inline void __nocfi cpu_replace_ttbr1(pgd_t *pgdp, pgd_t *idmap) ...@@ -168,7 +168,7 @@ static inline void __nocfi cpu_replace_ttbr1(pgd_t *pgdp, pgd_t *idmap)
ttbr1 |= TTBR_CNP_BIT; ttbr1 |= TTBR_CNP_BIT;
} }
replace_phys = (void *)__pa_symbol(function_nocfi(idmap_cpu_replace_ttbr1)); replace_phys = (void *)__pa_symbol(idmap_cpu_replace_ttbr1);
__cpu_install_idmap(idmap); __cpu_install_idmap(idmap);
replace_phys(ttbr1); replace_phys(ttbr1);
......
...@@ -99,7 +99,7 @@ static int acpi_parking_protocol_cpu_boot(unsigned int cpu) ...@@ -99,7 +99,7 @@ static int acpi_parking_protocol_cpu_boot(unsigned int cpu)
* that read this address need to convert this address to the * that read this address need to convert this address to the
* Boot-Loader's endianness before jumping. * Boot-Loader's endianness before jumping.
*/ */
writeq_relaxed(__pa_symbol(function_nocfi(secondary_entry)), writeq_relaxed(__pa_symbol(secondary_entry),
&mailbox->entry_point); &mailbox->entry_point);
writel_relaxed(cpu_entry->gic_cpu_id, &mailbox->cpu_id); writel_relaxed(cpu_entry->gic_cpu_id, &mailbox->cpu_id);
......
...@@ -133,7 +133,7 @@ static void clean_dcache_range_nopatch(u64 start, u64 end) ...@@ -133,7 +133,7 @@ static void clean_dcache_range_nopatch(u64 start, u64 end)
} while (cur += d_size, cur < end); } while (cur += d_size, cur < end);
} }
static void __nocfi __apply_alternatives(struct alt_region *region, bool is_module, static void __apply_alternatives(struct alt_region *region, bool is_module,
unsigned long *feature_mask) unsigned long *feature_mask)
{ {
struct alt_instr *alt; struct alt_instr *alt;
......
...@@ -8,6 +8,7 @@ ...@@ -8,6 +8,7 @@
*/ */
#include <linux/linkage.h> #include <linux/linkage.h>
#include <linux/cfi_types.h>
#include <asm/assembler.h> #include <asm/assembler.h>
#include <asm/sysreg.h> #include <asm/sysreg.h>
#include <asm/virt.h> #include <asm/virt.h>
...@@ -28,7 +29,7 @@ ...@@ -28,7 +29,7 @@
* branch to what would be the reset vector. It must be executed with the * branch to what would be the reset vector. It must be executed with the
* flat identity mapping. * flat identity mapping.
*/ */
SYM_CODE_START(cpu_soft_restart) SYM_TYPED_FUNC_START(cpu_soft_restart)
mov_q x12, INIT_SCTLR_EL1_MMU_OFF mov_q x12, INIT_SCTLR_EL1_MMU_OFF
pre_disable_mmu_workaround pre_disable_mmu_workaround
/* /*
...@@ -47,6 +48,6 @@ SYM_CODE_START(cpu_soft_restart) ...@@ -47,6 +48,6 @@ SYM_CODE_START(cpu_soft_restart)
mov x1, x3 // arg1 mov x1, x3 // arg1
mov x2, x4 // arg2 mov x2, x4 // arg2
br x8 br x8
SYM_CODE_END(cpu_soft_restart) SYM_FUNC_END(cpu_soft_restart)
.popsection .popsection
...@@ -1685,7 +1685,7 @@ static phys_addr_t kpti_ng_pgd_alloc(int shift) ...@@ -1685,7 +1685,7 @@ static phys_addr_t kpti_ng_pgd_alloc(int shift)
return kpti_ng_temp_alloc; return kpti_ng_temp_alloc;
} }
static void __nocfi static void
kpti_install_ng_mappings(const struct arm64_cpu_capabilities *__unused) kpti_install_ng_mappings(const struct arm64_cpu_capabilities *__unused)
{ {
typedef void (kpti_remap_fn)(int, int, phys_addr_t, unsigned long); typedef void (kpti_remap_fn)(int, int, phys_addr_t, unsigned long);
...@@ -1713,7 +1713,7 @@ kpti_install_ng_mappings(const struct arm64_cpu_capabilities *__unused) ...@@ -1713,7 +1713,7 @@ kpti_install_ng_mappings(const struct arm64_cpu_capabilities *__unused)
if (arm64_use_ng_mappings) if (arm64_use_ng_mappings)
return; return;
remap_fn = (void *)__pa_symbol(function_nocfi(idmap_kpti_install_ng_mappings)); remap_fn = (void *)__pa_symbol(idmap_kpti_install_ng_mappings);
if (!cpu) { if (!cpu) {
alloc = __get_free_pages(GFP_ATOMIC | __GFP_ZERO, order); alloc = __get_free_pages(GFP_ATOMIC | __GFP_ZERO, order);
......
...@@ -56,7 +56,7 @@ int ftrace_update_ftrace_func(ftrace_func_t func) ...@@ -56,7 +56,7 @@ int ftrace_update_ftrace_func(ftrace_func_t func)
unsigned long pc; unsigned long pc;
u32 new; u32 new;
pc = (unsigned long)function_nocfi(ftrace_call); pc = (unsigned long)ftrace_call;
new = aarch64_insn_gen_branch_imm(pc, (unsigned long)func, new = aarch64_insn_gen_branch_imm(pc, (unsigned long)func,
AARCH64_INSN_BRANCH_LINK); AARCH64_INSN_BRANCH_LINK);
......
...@@ -204,7 +204,7 @@ void machine_kexec(struct kimage *kimage) ...@@ -204,7 +204,7 @@ void machine_kexec(struct kimage *kimage)
typeof(cpu_soft_restart) *restart; typeof(cpu_soft_restart) *restart;
cpu_install_idmap(); cpu_install_idmap();
restart = (void *)__pa_symbol(function_nocfi(cpu_soft_restart)); restart = (void *)__pa_symbol(cpu_soft_restart);
restart(is_hyp_nvhe(), kimage->start, kimage->arch.dtb_mem, restart(is_hyp_nvhe(), kimage->start, kimage->arch.dtb_mem,
0, 0); 0, 0);
} else { } else {
......
...@@ -38,7 +38,7 @@ static int __init cpu_psci_cpu_prepare(unsigned int cpu) ...@@ -38,7 +38,7 @@ static int __init cpu_psci_cpu_prepare(unsigned int cpu)
static int cpu_psci_cpu_boot(unsigned int cpu) static int cpu_psci_cpu_boot(unsigned int cpu)
{ {
phys_addr_t pa_secondary_entry = __pa_symbol(function_nocfi(secondary_entry)); phys_addr_t pa_secondary_entry = __pa_symbol(secondary_entry);
int err = psci_ops.cpu_on(cpu_logical_map(cpu), pa_secondary_entry); int err = psci_ops.cpu_on(cpu_logical_map(cpu), pa_secondary_entry);
if (err) if (err)
pr_err("failed to boot CPU%d (%d)\n", cpu, err); pr_err("failed to boot CPU%d (%d)\n", cpu, err);
......
...@@ -66,7 +66,7 @@ static int smp_spin_table_cpu_init(unsigned int cpu) ...@@ -66,7 +66,7 @@ static int smp_spin_table_cpu_init(unsigned int cpu)
static int smp_spin_table_cpu_prepare(unsigned int cpu) static int smp_spin_table_cpu_prepare(unsigned int cpu)
{ {
__le64 __iomem *release_addr; __le64 __iomem *release_addr;
phys_addr_t pa_holding_pen = __pa_symbol(function_nocfi(secondary_holding_pen)); phys_addr_t pa_holding_pen = __pa_symbol(secondary_holding_pen);
if (!cpu_release_addr[cpu]) if (!cpu_release_addr[cpu])
return -ENODEV; return -ENODEV;
......
...@@ -26,6 +26,7 @@ ...@@ -26,6 +26,7 @@
#include <linux/syscalls.h> #include <linux/syscalls.h>
#include <linux/mm_types.h> #include <linux/mm_types.h>
#include <linux/kasan.h> #include <linux/kasan.h>
#include <linux/cfi.h>
#include <asm/atomic.h> #include <asm/atomic.h>
#include <asm/bug.h> #include <asm/bug.h>
...@@ -991,6 +992,38 @@ static struct break_hook bug_break_hook = { ...@@ -991,6 +992,38 @@ static struct break_hook bug_break_hook = {
.imm = BUG_BRK_IMM, .imm = BUG_BRK_IMM,
}; };
#ifdef CONFIG_CFI_CLANG
static int cfi_handler(struct pt_regs *regs, unsigned long esr)
{
unsigned long target;
u32 type;
target = pt_regs_read_reg(regs, FIELD_GET(CFI_BRK_IMM_TARGET, esr));
type = (u32)pt_regs_read_reg(regs, FIELD_GET(CFI_BRK_IMM_TYPE, esr));
switch (report_cfi_failure(regs, regs->pc, &target, type)) {
case BUG_TRAP_TYPE_BUG:
die("Oops - CFI", regs, 0);
break;
case BUG_TRAP_TYPE_WARN:
break;
default:
return DBG_HOOK_ERROR;
}
arm64_skip_faulting_instruction(regs, AARCH64_INSN_SIZE);
return DBG_HOOK_HANDLED;
}
static struct break_hook cfi_break_hook = {
.fn = cfi_handler,
.imm = CFI_BRK_IMM_BASE,
.mask = CFI_BRK_IMM_MASK,
};
#endif /* CONFIG_CFI_CLANG */
static int reserved_fault_handler(struct pt_regs *regs, unsigned long esr) static int reserved_fault_handler(struct pt_regs *regs, unsigned long esr)
{ {
pr_err("%s generated an invalid instruction at %pS!\n", pr_err("%s generated an invalid instruction at %pS!\n",
...@@ -1052,6 +1085,9 @@ static struct break_hook kasan_break_hook = { ...@@ -1052,6 +1085,9 @@ static struct break_hook kasan_break_hook = {
}; };
#endif #endif
#define esr_comment(esr) ((esr) & ESR_ELx_BRK64_ISS_COMMENT_MASK)
/* /*
* Initial handler for AArch64 BRK exceptions * Initial handler for AArch64 BRK exceptions
* This handler only used until debug_traps_init(). * This handler only used until debug_traps_init().
...@@ -1059,10 +1095,12 @@ static struct break_hook kasan_break_hook = { ...@@ -1059,10 +1095,12 @@ static struct break_hook kasan_break_hook = {
int __init early_brk64(unsigned long addr, unsigned long esr, int __init early_brk64(unsigned long addr, unsigned long esr,
struct pt_regs *regs) struct pt_regs *regs)
{ {
#ifdef CONFIG_CFI_CLANG
if ((esr_comment(esr) & ~CFI_BRK_IMM_MASK) == CFI_BRK_IMM_BASE)
return cfi_handler(regs, esr) != DBG_HOOK_HANDLED;
#endif
#ifdef CONFIG_KASAN_SW_TAGS #ifdef CONFIG_KASAN_SW_TAGS
unsigned long comment = esr & ESR_ELx_BRK64_ISS_COMMENT_MASK; if ((esr_comment(esr) & ~KASAN_BRK_MASK) == KASAN_BRK_IMM)
if ((comment & ~KASAN_BRK_MASK) == KASAN_BRK_IMM)
return kasan_handler(regs, esr) != DBG_HOOK_HANDLED; return kasan_handler(regs, esr) != DBG_HOOK_HANDLED;
#endif #endif
return bug_handler(regs, esr) != DBG_HOOK_HANDLED; return bug_handler(regs, esr) != DBG_HOOK_HANDLED;
...@@ -1071,6 +1109,9 @@ int __init early_brk64(unsigned long addr, unsigned long esr, ...@@ -1071,6 +1109,9 @@ int __init early_brk64(unsigned long addr, unsigned long esr,
void __init trap_init(void) void __init trap_init(void)
{ {
register_kernel_break_hook(&bug_break_hook); register_kernel_break_hook(&bug_break_hook);
#ifdef CONFIG_CFI_CLANG
register_kernel_break_hook(&cfi_break_hook);
#endif
register_kernel_break_hook(&fault_break_hook); register_kernel_break_hook(&fault_break_hook);
#ifdef CONFIG_KASAN_SW_TAGS #ifdef CONFIG_KASAN_SW_TAGS
register_kernel_break_hook(&kasan_break_hook); register_kernel_break_hook(&kasan_break_hook);
......
...@@ -40,7 +40,8 @@ ccflags-y += -DDISABLE_BRANCH_PROFILING -DBUILD_VDSO ...@@ -40,7 +40,8 @@ ccflags-y += -DDISABLE_BRANCH_PROFILING -DBUILD_VDSO
# kernel with CONFIG_WERROR enabled. # kernel with CONFIG_WERROR enabled.
CFLAGS_REMOVE_vgettimeofday.o = $(CC_FLAGS_FTRACE) -Os $(CC_FLAGS_SCS) \ CFLAGS_REMOVE_vgettimeofday.o = $(CC_FLAGS_FTRACE) -Os $(CC_FLAGS_SCS) \
$(RANDSTRUCT_CFLAGS) $(GCC_PLUGINS_CFLAGS) \ $(RANDSTRUCT_CFLAGS) $(GCC_PLUGINS_CFLAGS) \
$(CC_FLAGS_LTO) -Wmissing-prototypes -Wmissing-declarations $(CC_FLAGS_LTO) $(CC_FLAGS_CFI) \
-Wmissing-prototypes -Wmissing-declarations
KASAN_SANITIZE := n KASAN_SANITIZE := n
KCSAN_SANITIZE := n KCSAN_SANITIZE := n
UBSAN_SANITIZE := n UBSAN_SANITIZE := n
......
...@@ -10,6 +10,7 @@ ...@@ -10,6 +10,7 @@
#include <linux/init.h> #include <linux/init.h>
#include <linux/linkage.h> #include <linux/linkage.h>
#include <linux/pgtable.h> #include <linux/pgtable.h>
#include <linux/cfi_types.h>
#include <asm/assembler.h> #include <asm/assembler.h>
#include <asm/asm-offsets.h> #include <asm/asm-offsets.h>
#include <asm/asm_pointer_auth.h> #include <asm/asm_pointer_auth.h>
...@@ -185,7 +186,7 @@ SYM_FUNC_END(cpu_do_resume) ...@@ -185,7 +186,7 @@ SYM_FUNC_END(cpu_do_resume)
* This is the low-level counterpart to cpu_replace_ttbr1, and should not be * This is the low-level counterpart to cpu_replace_ttbr1, and should not be
* called by anything else. It can only be executed from a TTBR0 mapping. * called by anything else. It can only be executed from a TTBR0 mapping.
*/ */
SYM_FUNC_START(idmap_cpu_replace_ttbr1) SYM_TYPED_FUNC_START(idmap_cpu_replace_ttbr1)
save_and_disable_daif flags=x2 save_and_disable_daif flags=x2
__idmap_cpu_set_reserved_ttbr1 x1, x3 __idmap_cpu_set_reserved_ttbr1 x1, x3
...@@ -253,7 +254,7 @@ SYM_FUNC_END(idmap_cpu_replace_ttbr1) ...@@ -253,7 +254,7 @@ SYM_FUNC_END(idmap_cpu_replace_ttbr1)
SYM_DATA(__idmap_kpti_flag, .long 1) SYM_DATA(__idmap_kpti_flag, .long 1)
.popsection .popsection
SYM_FUNC_START(idmap_kpti_install_ng_mappings) SYM_TYPED_FUNC_START(idmap_kpti_install_ng_mappings)
cpu .req w0 cpu .req w0
temp_pte .req x0 temp_pte .req x0
num_cpus .req w1 num_cpus .req w1
......
...@@ -107,6 +107,8 @@ config X86 ...@@ -107,6 +107,8 @@ config X86
select ARCH_SUPPORTS_PAGE_TABLE_CHECK if X86_64 select ARCH_SUPPORTS_PAGE_TABLE_CHECK if X86_64
select ARCH_SUPPORTS_NUMA_BALANCING if X86_64 select ARCH_SUPPORTS_NUMA_BALANCING if X86_64
select ARCH_SUPPORTS_KMAP_LOCAL_FORCE_MAP if NR_CPUS <= 4096 select ARCH_SUPPORTS_KMAP_LOCAL_FORCE_MAP if NR_CPUS <= 4096
select ARCH_SUPPORTS_CFI_CLANG if X86_64
select ARCH_USES_CFI_TRAPS if X86_64 && CFI_CLANG
select ARCH_SUPPORTS_LTO_CLANG select ARCH_SUPPORTS_LTO_CLANG
select ARCH_SUPPORTS_LTO_CLANG_THIN select ARCH_SUPPORTS_LTO_CLANG_THIN
select ARCH_USE_BUILTIN_BSWAP select ARCH_USE_BUILTIN_BSWAP
......
...@@ -6,6 +6,7 @@ ...@@ -6,6 +6,7 @@
*/ */
#include <linux/linkage.h> #include <linux/linkage.h>
#include <linux/cfi_types.h>
.file "blowfish-x86_64-asm.S" .file "blowfish-x86_64-asm.S"
.text .text
...@@ -141,7 +142,7 @@ SYM_FUNC_START(__blowfish_enc_blk) ...@@ -141,7 +142,7 @@ SYM_FUNC_START(__blowfish_enc_blk)
RET; RET;
SYM_FUNC_END(__blowfish_enc_blk) SYM_FUNC_END(__blowfish_enc_blk)
SYM_FUNC_START(blowfish_dec_blk) SYM_TYPED_FUNC_START(blowfish_dec_blk)
/* input: /* input:
* %rdi: ctx * %rdi: ctx
* %rsi: dst * %rsi: dst
...@@ -332,7 +333,7 @@ SYM_FUNC_START(__blowfish_enc_blk_4way) ...@@ -332,7 +333,7 @@ SYM_FUNC_START(__blowfish_enc_blk_4way)
RET; RET;
SYM_FUNC_END(__blowfish_enc_blk_4way) SYM_FUNC_END(__blowfish_enc_blk_4way)
SYM_FUNC_START(blowfish_dec_blk_4way) SYM_TYPED_FUNC_START(blowfish_dec_blk_4way)
/* input: /* input:
* %rdi: ctx * %rdi: ctx
* %rsi: dst * %rsi: dst
......
...@@ -91,7 +91,7 @@ ifneq ($(RETPOLINE_VDSO_CFLAGS),) ...@@ -91,7 +91,7 @@ ifneq ($(RETPOLINE_VDSO_CFLAGS),)
endif endif
endif endif
$(vobjs): KBUILD_CFLAGS := $(filter-out $(CC_FLAGS_LTO) $(RANDSTRUCT_CFLAGS) $(GCC_PLUGINS_CFLAGS) $(RETPOLINE_CFLAGS),$(KBUILD_CFLAGS)) $(CFL) $(vobjs): KBUILD_CFLAGS := $(filter-out $(CC_FLAGS_LTO) $(CC_FLAGS_CFI) $(RANDSTRUCT_CFLAGS) $(GCC_PLUGINS_CFLAGS) $(RETPOLINE_CFLAGS),$(KBUILD_CFLAGS)) $(CFL)
$(vobjs): KBUILD_AFLAGS += -DBUILD_VDSO $(vobjs): KBUILD_AFLAGS += -DBUILD_VDSO
# #
...@@ -153,6 +153,7 @@ KBUILD_CFLAGS_32 := $(filter-out $(RANDSTRUCT_CFLAGS),$(KBUILD_CFLAGS_32)) ...@@ -153,6 +153,7 @@ KBUILD_CFLAGS_32 := $(filter-out $(RANDSTRUCT_CFLAGS),$(KBUILD_CFLAGS_32))
KBUILD_CFLAGS_32 := $(filter-out $(GCC_PLUGINS_CFLAGS),$(KBUILD_CFLAGS_32)) KBUILD_CFLAGS_32 := $(filter-out $(GCC_PLUGINS_CFLAGS),$(KBUILD_CFLAGS_32))
KBUILD_CFLAGS_32 := $(filter-out $(RETPOLINE_CFLAGS),$(KBUILD_CFLAGS_32)) KBUILD_CFLAGS_32 := $(filter-out $(RETPOLINE_CFLAGS),$(KBUILD_CFLAGS_32))
KBUILD_CFLAGS_32 := $(filter-out $(CC_FLAGS_LTO),$(KBUILD_CFLAGS_32)) KBUILD_CFLAGS_32 := $(filter-out $(CC_FLAGS_LTO),$(KBUILD_CFLAGS_32))
KBUILD_CFLAGS_32 := $(filter-out $(CC_FLAGS_CFI),$(KBUILD_CFLAGS_32))
KBUILD_CFLAGS_32 += -m32 -msoft-float -mregparm=0 -fpic KBUILD_CFLAGS_32 += -m32 -msoft-float -mregparm=0 -fpic
KBUILD_CFLAGS_32 += -fno-stack-protector KBUILD_CFLAGS_32 += -fno-stack-protector
KBUILD_CFLAGS_32 += $(call cc-option, -foptimize-sibling-calls) KBUILD_CFLAGS_32 += $(call cc-option, -foptimize-sibling-calls)
......
/* SPDX-License-Identifier: GPL-2.0 */
#ifndef _ASM_X86_CFI_H
#define _ASM_X86_CFI_H
/*
* Clang Control Flow Integrity (CFI) support.
*
* Copyright (C) 2022 Google LLC
*/
#include <linux/cfi.h>
#ifdef CONFIG_CFI_CLANG
enum bug_trap_type handle_cfi_failure(struct pt_regs *regs);
#else
static inline enum bug_trap_type handle_cfi_failure(struct pt_regs *regs)
{
return BUG_TRAP_TYPE_NONE;
}
#endif /* CONFIG_CFI_CLANG */
#endif /* _ASM_X86_CFI_H */
...@@ -43,6 +43,18 @@ ...@@ -43,6 +43,18 @@
#endif /* __ASSEMBLY__ */ #endif /* __ASSEMBLY__ */
#define __CFI_TYPE(name) \
SYM_START(__cfi_##name, SYM_L_LOCAL, SYM_A_NONE) \
.fill 11, 1, 0x90 ASM_NL \
.byte 0xb8 ASM_NL \
.long __kcfi_typeid_##name ASM_NL \
SYM_FUNC_END(__cfi_##name)
/* SYM_TYPED_FUNC_START -- use for indirectly called globals, w/ CFI type */
#define SYM_TYPED_FUNC_START(name) \
SYM_TYPED_START(name, SYM_L_GLOBAL, SYM_A_ALIGN) \
ENDBR
/* SYM_FUNC_START -- use for global functions */ /* SYM_FUNC_START -- use for global functions */
#define SYM_FUNC_START(name) \ #define SYM_FUNC_START(name) \
SYM_START(name, SYM_L_GLOBAL, SYM_A_ALIGN) \ SYM_START(name, SYM_L_GLOBAL, SYM_A_ALIGN) \
......
...@@ -139,6 +139,8 @@ obj-$(CONFIG_UNWINDER_GUESS) += unwind_guess.o ...@@ -139,6 +139,8 @@ obj-$(CONFIG_UNWINDER_GUESS) += unwind_guess.o
obj-$(CONFIG_AMD_MEM_ENCRYPT) += sev.o obj-$(CONFIG_AMD_MEM_ENCRYPT) += sev.o
obj-$(CONFIG_CFI_CLANG) += cfi.o
### ###
# 64 bit specific files # 64 bit specific files
ifeq ($(CONFIG_X86_64),y) ifeq ($(CONFIG_X86_64),y)
......
// SPDX-License-Identifier: GPL-2.0
/*
* Clang Control Flow Integrity (CFI) support.
*
* Copyright (C) 2022 Google LLC
*/
#include <asm/cfi.h>
#include <asm/insn.h>
#include <asm/insn-eval.h>
#include <linux/string.h>
/*
* Returns the target address and the expected type when regs->ip points
* to a compiler-generated CFI trap.
*/
static bool decode_cfi_insn(struct pt_regs *regs, unsigned long *target,
u32 *type)
{
char buffer[MAX_INSN_SIZE];
struct insn insn;
int offset = 0;
*target = *type = 0;
/*
* The compiler generates the following instruction sequence
* for indirect call checks:
*
*   movl -<id>, %r10d ; 6 bytes
* addl -4(%reg), %r10d ; 4 bytes
* je .Ltmp1 ; 2 bytes
* ud2 ; <- regs->ip
* .Ltmp1:
*
* We can decode the expected type and the target address from the
* movl/addl instructions.
*/
if (copy_from_kernel_nofault(buffer, (void *)regs->ip - 12, MAX_INSN_SIZE))
return false;
if (insn_decode_kernel(&insn, &buffer[offset]))
return false;
if (insn.opcode.value != 0xBA)
return false;
*type = -(u32)insn.immediate.value;
if (copy_from_kernel_nofault(buffer, (void *)regs->ip - 6, MAX_INSN_SIZE))
return false;
if (insn_decode_kernel(&insn, &buffer[offset]))
return false;
if (insn.opcode.value != 0x3)
return false;
/* Read the target address from the register. */
offset = insn_get_modrm_rm_off(&insn, regs);
if (offset < 0)
return false;
*target = *(unsigned long *)((void *)regs + offset);
return true;
}
/*
* Checks if a ud2 trap is because of a CFI failure, and handles the trap
* if needed. Returns a bug_trap_type value similarly to report_bug.
*/
enum bug_trap_type handle_cfi_failure(struct pt_regs *regs)
{
unsigned long target;
u32 type;
if (!is_cfi_trap(regs->ip))
return BUG_TRAP_TYPE_NONE;
if (!decode_cfi_insn(regs, &target, &type))
return report_cfi_failure_noaddr(regs, regs->ip);
return report_cfi_failure(regs, regs->ip, &target, type);
}
/*
* Ensure that __kcfi_typeid_ symbols are emitted for functions that may
* not be indirectly called with all configurations.
*/
__ADDRESSABLE(__memcpy)
...@@ -63,6 +63,7 @@ ...@@ -63,6 +63,7 @@
#include <asm/insn-eval.h> #include <asm/insn-eval.h>
#include <asm/vdso.h> #include <asm/vdso.h>
#include <asm/tdx.h> #include <asm/tdx.h>
#include <asm/cfi.h>
#ifdef CONFIG_X86_64 #ifdef CONFIG_X86_64
#include <asm/x86_init.h> #include <asm/x86_init.h>
...@@ -313,7 +314,8 @@ static noinstr bool handle_bug(struct pt_regs *regs) ...@@ -313,7 +314,8 @@ static noinstr bool handle_bug(struct pt_regs *regs)
*/ */
if (regs->flags & X86_EFLAGS_IF) if (regs->flags & X86_EFLAGS_IF)
raw_local_irq_enable(); raw_local_irq_enable();
if (report_bug(regs->ip, regs) == BUG_TRAP_TYPE_WARN) { if (report_bug(regs->ip, regs) == BUG_TRAP_TYPE_WARN ||
handle_cfi_failure(regs) == BUG_TRAP_TYPE_WARN) {
regs->ip += LEN_UD2; regs->ip += LEN_UD2;
handled = true; handled = true;
} }
......
...@@ -2,6 +2,7 @@ ...@@ -2,6 +2,7 @@
/* Copyright 2002 Andi Kleen */ /* Copyright 2002 Andi Kleen */
#include <linux/linkage.h> #include <linux/linkage.h>
#include <linux/cfi_types.h>
#include <asm/errno.h> #include <asm/errno.h>
#include <asm/cpufeatures.h> #include <asm/cpufeatures.h>
#include <asm/alternative.h> #include <asm/alternative.h>
...@@ -27,7 +28,7 @@ ...@@ -27,7 +28,7 @@
* Output: * Output:
* rax original destination * rax original destination
*/ */
SYM_FUNC_START(__memcpy) SYM_TYPED_FUNC_START(__memcpy)
ALTERNATIVE_2 "jmp memcpy_orig", "", X86_FEATURE_REP_GOOD, \ ALTERNATIVE_2 "jmp memcpy_orig", "", X86_FEATURE_REP_GOOD, \
"jmp memcpy_erms", X86_FEATURE_ERMS "jmp memcpy_erms", X86_FEATURE_ERMS
......
...@@ -55,6 +55,10 @@ ifdef CONFIG_RETPOLINE ...@@ -55,6 +55,10 @@ ifdef CONFIG_RETPOLINE
PURGATORY_CFLAGS_REMOVE += $(RETPOLINE_CFLAGS) PURGATORY_CFLAGS_REMOVE += $(RETPOLINE_CFLAGS)
endif endif
ifdef CONFIG_CFI_CLANG
PURGATORY_CFLAGS_REMOVE += $(CC_FLAGS_CFI)
endif
CFLAGS_REMOVE_purgatory.o += $(PURGATORY_CFLAGS_REMOVE) CFLAGS_REMOVE_purgatory.o += $(PURGATORY_CFLAGS_REMOVE)
CFLAGS_purgatory.o += $(PURGATORY_CFLAGS) CFLAGS_purgatory.o += $(PURGATORY_CFLAGS)
......
...@@ -56,6 +56,7 @@ static const char * const sym_regex_kernel[S_NSYMTYPES] = { ...@@ -56,6 +56,7 @@ static const char * const sym_regex_kernel[S_NSYMTYPES] = {
"^(xen_irq_disable_direct_reloc$|" "^(xen_irq_disable_direct_reloc$|"
"xen_save_fl_direct_reloc$|" "xen_save_fl_direct_reloc$|"
"VDSO|" "VDSO|"
"__kcfi_typeid_|"
"__crc_)", "__crc_)",
/* /*
......
...@@ -46,6 +46,8 @@ KBUILD_CFLAGS := $(filter-out $(RANDSTRUCT_CFLAGS), $(KBUILD_CFLAGS)) ...@@ -46,6 +46,8 @@ KBUILD_CFLAGS := $(filter-out $(RANDSTRUCT_CFLAGS), $(KBUILD_CFLAGS))
# remove SCS flags from all objects in this directory # remove SCS flags from all objects in this directory
KBUILD_CFLAGS := $(filter-out $(CC_FLAGS_SCS), $(KBUILD_CFLAGS)) KBUILD_CFLAGS := $(filter-out $(CC_FLAGS_SCS), $(KBUILD_CFLAGS))
# disable CFI
KBUILD_CFLAGS := $(filter-out $(CC_FLAGS_CFI), $(KBUILD_CFLAGS))
# disable LTO # disable LTO
KBUILD_CFLAGS := $(filter-out $(CC_FLAGS_LTO), $(KBUILD_CFLAGS)) KBUILD_CFLAGS := $(filter-out $(CC_FLAGS_LTO), $(KBUILD_CFLAGS))
......
...@@ -274,7 +274,7 @@ static void set_conduit(enum arm_smccc_conduit conduit) ...@@ -274,7 +274,7 @@ static void set_conduit(enum arm_smccc_conduit conduit)
psci_conduit = conduit; psci_conduit = conduit;
} }
static int get_set_conduit_method(struct device_node *np) static int get_set_conduit_method(const struct device_node *np)
{ {
const char *method; const char *method;
...@@ -334,7 +334,7 @@ static int __init psci_features(u32 psci_func_id) ...@@ -334,7 +334,7 @@ static int __init psci_features(u32 psci_func_id)
static int psci_suspend_finisher(unsigned long state) static int psci_suspend_finisher(unsigned long state)
{ {
u32 power_state = state; u32 power_state = state;
phys_addr_t pa_cpu_resume = __pa_symbol(function_nocfi(cpu_resume)); phys_addr_t pa_cpu_resume = __pa_symbol(cpu_resume);
return psci_ops.cpu_suspend(power_state, pa_cpu_resume); return psci_ops.cpu_suspend(power_state, pa_cpu_resume);
} }
...@@ -359,7 +359,7 @@ int psci_cpu_suspend_enter(u32 state) ...@@ -359,7 +359,7 @@ int psci_cpu_suspend_enter(u32 state)
static int psci_system_suspend(unsigned long unused) static int psci_system_suspend(unsigned long unused)
{ {
phys_addr_t pa_cpu_resume = __pa_symbol(function_nocfi(cpu_resume)); phys_addr_t pa_cpu_resume = __pa_symbol(cpu_resume);
return invoke_psci_fn(PSCI_FN_NATIVE(1_0, SYSTEM_SUSPEND), return invoke_psci_fn(PSCI_FN_NATIVE(1_0, SYSTEM_SUSPEND),
pa_cpu_resume, 0, 0); pa_cpu_resume, 0, 0);
...@@ -528,7 +528,7 @@ typedef int (*psci_initcall_t)(const struct device_node *); ...@@ -528,7 +528,7 @@ typedef int (*psci_initcall_t)(const struct device_node *);
* *
* Probe based on PSCI PSCI_VERSION function * Probe based on PSCI PSCI_VERSION function
*/ */
static int __init psci_0_2_init(struct device_node *np) static int __init psci_0_2_init(const struct device_node *np)
{ {
int err; int err;
...@@ -549,7 +549,7 @@ static int __init psci_0_2_init(struct device_node *np) ...@@ -549,7 +549,7 @@ static int __init psci_0_2_init(struct device_node *np)
/* /*
* PSCI < v0.2 get PSCI Function IDs via DT. * PSCI < v0.2 get PSCI Function IDs via DT.
*/ */
static int __init psci_0_1_init(struct device_node *np) static int __init psci_0_1_init(const struct device_node *np)
{ {
u32 id; u32 id;
int err; int err;
...@@ -585,7 +585,7 @@ static int __init psci_0_1_init(struct device_node *np) ...@@ -585,7 +585,7 @@ static int __init psci_0_1_init(struct device_node *np)
return 0; return 0;
} }
static int __init psci_1_0_init(struct device_node *np) static int __init psci_1_0_init(const struct device_node *np)
{ {
int err; int err;
......
...@@ -20,6 +20,13 @@ static noinline int lkdtm_increment_int(int *counter) ...@@ -20,6 +20,13 @@ static noinline int lkdtm_increment_int(int *counter)
return *counter; return *counter;
} }
/* Don't allow the compiler to inline the calls. */
static noinline void lkdtm_indirect_call(void (*func)(int *))
{
func(&called_count);
}
/* /*
* This tries to call an indirect function with a mismatched prototype. * This tries to call an indirect function with a mismatched prototype.
*/ */
...@@ -29,15 +36,11 @@ static void lkdtm_CFI_FORWARD_PROTO(void) ...@@ -29,15 +36,11 @@ static void lkdtm_CFI_FORWARD_PROTO(void)
* Matches lkdtm_increment_void()'s prototype, but not * Matches lkdtm_increment_void()'s prototype, but not
* lkdtm_increment_int()'s prototype. * lkdtm_increment_int()'s prototype.
*/ */
void (*func)(int *);
pr_info("Calling matched prototype ...\n"); pr_info("Calling matched prototype ...\n");
func = lkdtm_increment_void; lkdtm_indirect_call(lkdtm_increment_void);
func(&called_count);
pr_info("Calling mismatched prototype ...\n"); pr_info("Calling mismatched prototype ...\n");
func = (void *)lkdtm_increment_int; lkdtm_indirect_call((void *)lkdtm_increment_int);
func(&called_count);
pr_err("FAIL: survived mismatched prototype function call!\n"); pr_err("FAIL: survived mismatched prototype function call!\n");
pr_expected_config(CONFIG_CFI_CLANG); pr_expected_config(CONFIG_CFI_CLANG);
......
...@@ -330,7 +330,7 @@ static void lkdtm_USERCOPY_KERNEL(void) ...@@ -330,7 +330,7 @@ static void lkdtm_USERCOPY_KERNEL(void)
pr_info("attempting bad copy_to_user from kernel text: %px\n", pr_info("attempting bad copy_to_user from kernel text: %px\n",
vm_mmap); vm_mmap);
if (copy_to_user((void __user *)user_addr, function_nocfi(vm_mmap), if (copy_to_user((void __user *)user_addr, vm_mmap,
unconst + PAGE_SIZE)) { unconst + PAGE_SIZE)) {
pr_warn("copy_to_user failed, but lacked Oops\n"); pr_warn("copy_to_user failed, but lacked Oops\n");
goto free_user; goto free_user;
......
...@@ -220,22 +220,6 @@ extern __printf(1, 2) void __warn_printk(const char *fmt, ...); ...@@ -220,22 +220,6 @@ extern __printf(1, 2) void __warn_printk(const char *fmt, ...);
# define WARN_ON_SMP(x) ({0;}) # define WARN_ON_SMP(x) ({0;})
#endif #endif
/*
* WARN_ON_FUNCTION_MISMATCH() warns if a value doesn't match a
* function address, and can be useful for catching issues with
* callback functions, for example.
*
* With CONFIG_CFI_CLANG, the warning is disabled because the
* compiler replaces function addresses taken in C code with
* local jump table addresses, which breaks cross-module function
* address equality.
*/
#if defined(CONFIG_CFI_CLANG) && defined(CONFIG_MODULES)
# define WARN_ON_FUNCTION_MISMATCH(x, fn) ({ 0; })
#else
# define WARN_ON_FUNCTION_MISMATCH(x, fn) WARN_ON_ONCE((x) != (fn))
#endif
#endif /* __ASSEMBLY__ */ #endif /* __ASSEMBLY__ */
#endif #endif
...@@ -421,6 +421,22 @@ ...@@ -421,6 +421,22 @@
__end_ro_after_init = .; __end_ro_after_init = .;
#endif #endif
/*
* .kcfi_traps contains a list KCFI trap locations.
*/
#ifndef KCFI_TRAPS
#ifdef CONFIG_ARCH_USES_CFI_TRAPS
#define KCFI_TRAPS \
__kcfi_traps : AT(ADDR(__kcfi_traps) - LOAD_OFFSET) { \
__start___kcfi_traps = .; \
KEEP(*(.kcfi_traps)) \
__stop___kcfi_traps = .; \
}
#else
#define KCFI_TRAPS
#endif
#endif
/* /*
* Read only Data * Read only Data
*/ */
...@@ -529,6 +545,8 @@ ...@@ -529,6 +545,8 @@
__stop___modver = .; \ __stop___modver = .; \
} \ } \
\ \
KCFI_TRAPS \
\
RO_EXCEPTION_TABLE \ RO_EXCEPTION_TABLE \
NOTES \ NOTES \
BTF \ BTF \
...@@ -537,20 +555,6 @@ ...@@ -537,20 +555,6 @@
__end_rodata = .; __end_rodata = .;
/*
* .text..L.cfi.jumptable.* contain Control-Flow Integrity (CFI)
* jump table entries.
*/
#ifdef CONFIG_CFI_CLANG
#define TEXT_CFI_JT \
ALIGN_FUNCTION(); \
__cfi_jt_start = .; \
*(.text..L.cfi.jumptable .text..L.cfi.jumptable.*) \
__cfi_jt_end = .;
#else
#define TEXT_CFI_JT
#endif
/* /*
* Non-instrumentable text section * Non-instrumentable text section
*/ */
...@@ -578,7 +582,6 @@ ...@@ -578,7 +582,6 @@
*(.text..refcount) \ *(.text..refcount) \
*(.ref.text) \ *(.ref.text) \
*(.text.asan.* .text.tsan.*) \ *(.text.asan.* .text.tsan.*) \
TEXT_CFI_JT \
MEM_KEEP(init.text*) \ MEM_KEEP(init.text*) \
MEM_KEEP(exit.text*) \ MEM_KEEP(exit.text*) \
...@@ -1007,8 +1010,7 @@ ...@@ -1007,8 +1010,7 @@
* keep any .init_array.* sections. * keep any .init_array.* sections.
* https://bugs.llvm.org/show_bug.cgi?id=46478 * https://bugs.llvm.org/show_bug.cgi?id=46478
*/ */
#if defined(CONFIG_GCOV_KERNEL) || defined(CONFIG_KASAN_GENERIC) || defined(CONFIG_KCSAN) || \ #if defined(CONFIG_GCOV_KERNEL) || defined(CONFIG_KASAN_GENERIC) || defined(CONFIG_KCSAN)
defined(CONFIG_CFI_CLANG)
# ifdef CONFIG_CONSTRUCTORS # ifdef CONFIG_CONSTRUCTORS
# define SANITIZER_DISCARDS \ # define SANITIZER_DISCARDS \
*(.eh_frame) *(.eh_frame)
......
...@@ -2,49 +2,38 @@ ...@@ -2,49 +2,38 @@
/* /*
* Clang Control Flow Integrity (CFI) support. * Clang Control Flow Integrity (CFI) support.
* *
* Copyright (C) 2021 Google LLC * Copyright (C) 2022 Google LLC
*/ */
#ifndef _LINUX_CFI_H #ifndef _LINUX_CFI_H
#define _LINUX_CFI_H #define _LINUX_CFI_H
#ifdef CONFIG_CFI_CLANG #include <linux/bug.h>
typedef void (*cfi_check_fn)(uint64_t id, void *ptr, void *diag); #include <linux/module.h>
/* Compiler-generated function in each module, and the kernel */
extern void __cfi_check(uint64_t id, void *ptr, void *diag);
/*
* Force the compiler to generate a CFI jump table entry for a function
* and store the jump table address to __cfi_jt_<function>.
*/
#define __CFI_ADDRESSABLE(fn, __attr) \
const void *__cfi_jt_ ## fn __visible __attr = (void *)&fn
#ifdef CONFIG_CFI_CLANG_SHADOW
extern void cfi_module_add(struct module *mod, unsigned long base_addr);
extern void cfi_module_remove(struct module *mod, unsigned long base_addr);
#else
static inline void cfi_module_add(struct module *mod, unsigned long base_addr) {} #ifdef CONFIG_CFI_CLANG
static inline void cfi_module_remove(struct module *mod, unsigned long base_addr) {} enum bug_trap_type report_cfi_failure(struct pt_regs *regs, unsigned long addr,
unsigned long *target, u32 type);
#endif /* CONFIG_CFI_CLANG_SHADOW */
#else /* !CONFIG_CFI_CLANG */
#ifdef CONFIG_X86_KERNEL_IBT
#define __CFI_ADDRESSABLE(fn, __attr) \
const void *__cfi_jt_ ## fn __visible __attr = (void *)&fn
#endif /* CONFIG_X86_KERNEL_IBT */ static inline enum bug_trap_type report_cfi_failure_noaddr(struct pt_regs *regs,
unsigned long addr)
{
return report_cfi_failure(regs, addr, NULL, 0);
}
#ifdef CONFIG_ARCH_USES_CFI_TRAPS
bool is_cfi_trap(unsigned long addr);
#endif
#endif /* CONFIG_CFI_CLANG */ #endif /* CONFIG_CFI_CLANG */
#ifndef __CFI_ADDRESSABLE #ifdef CONFIG_MODULES
#define __CFI_ADDRESSABLE(fn, __attr) #ifdef CONFIG_ARCH_USES_CFI_TRAPS
#endif void module_cfi_finalize(const Elf_Ehdr *hdr, const Elf_Shdr *sechdrs,
struct module *mod);
#else
static inline void module_cfi_finalize(const Elf_Ehdr *hdr,
const Elf_Shdr *sechdrs,
struct module *mod) {}
#endif /* CONFIG_ARCH_USES_CFI_TRAPS */
#endif /* CONFIG_MODULES */
#endif /* _LINUX_CFI_H */ #endif /* _LINUX_CFI_H */
/* SPDX-License-Identifier: GPL-2.0 */
/*
* Clang Control Flow Integrity (CFI) type definitions.
*/
#ifndef _LINUX_CFI_TYPES_H
#define _LINUX_CFI_TYPES_H
#ifdef __ASSEMBLY__
#include <linux/linkage.h>
#ifdef CONFIG_CFI_CLANG
/*
* Use the __kcfi_typeid_<function> type identifier symbol to
* annotate indirectly called assembly functions. The compiler emits
* these symbols for all address-taken function declarations in C
* code.
*/
#ifndef __CFI_TYPE
#define __CFI_TYPE(name) \
.4byte __kcfi_typeid_##name
#endif
#define SYM_TYPED_ENTRY(name, linkage, align...) \
linkage(name) ASM_NL \
align ASM_NL \
__CFI_TYPE(name) ASM_NL \
name:
#define SYM_TYPED_START(name, linkage, align...) \
SYM_TYPED_ENTRY(name, linkage, align)
#else /* CONFIG_CFI_CLANG */
#define SYM_TYPED_START(name, linkage, align...) \
SYM_START(name, linkage, align)
#endif /* CONFIG_CFI_CLANG */
#ifndef SYM_TYPED_FUNC_START
#define SYM_TYPED_FUNC_START(name) \
SYM_TYPED_START(name, SYM_L_GLOBAL, SYM_A_ALIGN)
#endif
#endif /* __ASSEMBLY__ */
#endif /* _LINUX_CFI_TYPES_H */
...@@ -66,17 +66,9 @@ ...@@ -66,17 +66,9 @@
# define __noscs __attribute__((__no_sanitize__("shadow-call-stack"))) # define __noscs __attribute__((__no_sanitize__("shadow-call-stack")))
#endif #endif
#define __nocfi __attribute__((__no_sanitize__("cfi"))) #if __has_feature(kcfi)
#define __cficanonical __attribute__((__cfi_canonical_jump_table__)) /* Disable CFI checking inside a function. */
#define __nocfi __attribute__((__no_sanitize__("kcfi")))
#if defined(CONFIG_CFI_CLANG)
/*
* With CONFIG_CFI_CLANG, the compiler replaces function address
* references with the address of the function's CFI jump table
* entry. The function_nocfi macro always returns the address of the
* actual function instead.
*/
#define function_nocfi(x) __builtin_function_start(x)
#endif #endif
/* /*
......
...@@ -203,16 +203,6 @@ void ftrace_likely_update(struct ftrace_likely_data *f, int val, ...@@ -203,16 +203,6 @@ void ftrace_likely_update(struct ftrace_likely_data *f, int val,
__v; \ __v; \
}) })
/*
* With CONFIG_CFI_CLANG, the compiler replaces function addresses in
* instrumented C code with jump table addresses. Architectures that
* support CFI can define this macro to return the actual function address
* when needed.
*/
#ifndef function_nocfi
#define function_nocfi(x) (x)
#endif
#endif /* __KERNEL__ */ #endif /* __KERNEL__ */
/* /*
...@@ -221,9 +211,11 @@ void ftrace_likely_update(struct ftrace_likely_data *f, int val, ...@@ -221,9 +211,11 @@ void ftrace_likely_update(struct ftrace_likely_data *f, int val,
* otherwise, or eliminated entirely due to lack of references that are * otherwise, or eliminated entirely due to lack of references that are
* visible to the compiler. * visible to the compiler.
*/ */
#define __ADDRESSABLE(sym) \ #define ___ADDRESSABLE(sym, __attrs) \
static void * __section(".discard.addressable") __used \ static void * __used __attrs \
__UNIQUE_ID(__PASTE(__addressable_,sym)) = (void *)&sym; __UNIQUE_ID(__PASTE(__addressable_,sym)) = (void *)&sym;
#define __ADDRESSABLE(sym) \
___ADDRESSABLE(sym, __section(".discard.addressable"))
/** /**
* offset_to_ptr - convert a relative memory offset to an absolute pointer * offset_to_ptr - convert a relative memory offset to an absolute pointer
......
...@@ -269,10 +269,6 @@ struct ftrace_likely_data { ...@@ -269,10 +269,6 @@ struct ftrace_likely_data {
# define __nocfi # define __nocfi
#endif #endif
#ifndef __cficanonical
# define __cficanonical
#endif
/* /*
* Any place that could be marked with the "alloc_size" attribute is also * Any place that could be marked with the "alloc_size" attribute is also
* a place to be marked with the "malloc" attribute. Do this as part of the * a place to be marked with the "malloc" attribute. Do this as part of the
......
...@@ -47,7 +47,7 @@ ...@@ -47,7 +47,7 @@
/* These are for everybody (although not all archs will actually /* These are for everybody (although not all archs will actually
discard it in modules) */ discard it in modules) */
#define __init __section(".init.text") __cold __latent_entropy __noinitretpoline __nocfi #define __init __section(".init.text") __cold __latent_entropy __noinitretpoline
#define __initdata __section(".init.data") #define __initdata __section(".init.data")
#define __initconst __section(".init.rodata") #define __initconst __section(".init.rodata")
#define __exitdata __section(".exit.data") #define __exitdata __section(".exit.data")
...@@ -220,8 +220,8 @@ extern bool initcall_debug; ...@@ -220,8 +220,8 @@ extern bool initcall_debug;
__initcall_name(initstub, __iid, id) __initcall_name(initstub, __iid, id)
#define __define_initcall_stub(__stub, fn) \ #define __define_initcall_stub(__stub, fn) \
int __init __cficanonical __stub(void); \ int __init __stub(void); \
int __init __cficanonical __stub(void) \ int __init __stub(void) \
{ \ { \
return fn(); \ return fn(); \
} \ } \
......
...@@ -27,7 +27,6 @@ ...@@ -27,7 +27,6 @@
#include <linux/tracepoint-defs.h> #include <linux/tracepoint-defs.h>
#include <linux/srcu.h> #include <linux/srcu.h>
#include <linux/static_call_types.h> #include <linux/static_call_types.h>
#include <linux/cfi.h>
#include <linux/percpu.h> #include <linux/percpu.h>
#include <asm/module.h> #include <asm/module.h>
...@@ -132,7 +131,7 @@ extern void cleanup_module(void); ...@@ -132,7 +131,7 @@ extern void cleanup_module(void);
{ return initfn; } \ { return initfn; } \
int init_module(void) __copy(initfn) \ int init_module(void) __copy(initfn) \
__attribute__((alias(#initfn))); \ __attribute__((alias(#initfn))); \
__CFI_ADDRESSABLE(init_module, __initdata); ___ADDRESSABLE(init_module, __initdata);
/* This is only required if you want to be unloadable. */ /* This is only required if you want to be unloadable. */
#define module_exit(exitfn) \ #define module_exit(exitfn) \
...@@ -140,7 +139,7 @@ extern void cleanup_module(void); ...@@ -140,7 +139,7 @@ extern void cleanup_module(void);
{ return exitfn; } \ { return exitfn; } \
void cleanup_module(void) __copy(exitfn) \ void cleanup_module(void) __copy(exitfn) \
__attribute__((alias(#exitfn))); \ __attribute__((alias(#exitfn))); \
__CFI_ADDRESSABLE(cleanup_module, __exitdata); ___ADDRESSABLE(cleanup_module, __exitdata);
#endif #endif
...@@ -387,8 +386,9 @@ struct module { ...@@ -387,8 +386,9 @@ struct module {
const s32 *crcs; const s32 *crcs;
unsigned int num_syms; unsigned int num_syms;
#ifdef CONFIG_CFI_CLANG #ifdef CONFIG_ARCH_USES_CFI_TRAPS
cfi_check_fn cfi_check; s32 *kcfi_traps;
s32 *kcfi_traps_end;
#endif #endif
/* Kernel parameters. */ /* Kernel parameters. */
......
...@@ -2019,8 +2019,8 @@ enum pci_fixup_pass { ...@@ -2019,8 +2019,8 @@ enum pci_fixup_pass {
#ifdef CONFIG_LTO_CLANG #ifdef CONFIG_LTO_CLANG
#define __DECLARE_PCI_FIXUP_SECTION(sec, name, vendor, device, class, \ #define __DECLARE_PCI_FIXUP_SECTION(sec, name, vendor, device, class, \
class_shift, hook, stub) \ class_shift, hook, stub) \
void __cficanonical stub(struct pci_dev *dev); \ void stub(struct pci_dev *dev); \
void __cficanonical stub(struct pci_dev *dev) \ void stub(struct pci_dev *dev) \
{ \ { \
hook(dev); \ hook(dev); \
} \ } \
......
This diff is collapsed.
...@@ -177,7 +177,6 @@ static bool cleanup_symbol_name(char *s) ...@@ -177,7 +177,6 @@ static bool cleanup_symbol_name(char *s)
* character in an identifier in C. Suffixes observed: * character in an identifier in C. Suffixes observed:
* - foo.llvm.[0-9a-f]+ * - foo.llvm.[0-9a-f]+
* - foo.[0-9a-f]+ * - foo.[0-9a-f]+
* - foo.[0-9a-f]+.cfi_jt
*/ */
res = strchr(s, '.'); res = strchr(s, '.');
if (res) { if (res) {
...@@ -185,22 +184,6 @@ static bool cleanup_symbol_name(char *s) ...@@ -185,22 +184,6 @@ static bool cleanup_symbol_name(char *s)
return true; return true;
} }
if (!IS_ENABLED(CONFIG_CFI_CLANG) ||
!IS_ENABLED(CONFIG_LTO_CLANG_THIN) ||
CONFIG_CLANG_VERSION >= 130000)
return false;
/*
* Prior to LLVM 13, the following suffixes were observed when thinLTO
* and CFI are both enabled:
* - foo$[0-9]+
*/
res = strrchr(s, '$');
if (res) {
*res = '\0';
return true;
}
return false; return false;
} }
......
...@@ -1050,8 +1050,7 @@ static void __kthread_queue_delayed_work(struct kthread_worker *worker, ...@@ -1050,8 +1050,7 @@ static void __kthread_queue_delayed_work(struct kthread_worker *worker,
struct timer_list *timer = &dwork->timer; struct timer_list *timer = &dwork->timer;
struct kthread_work *work = &dwork->work; struct kthread_work *work = &dwork->work;
WARN_ON_FUNCTION_MISMATCH(timer->function, WARN_ON_ONCE(timer->function != kthread_delayed_work_timer_fn);
kthread_delayed_work_timer_fn);
/* /*
* If @delay is 0, queue @dwork->work immediately. This is for * If @delay is 0, queue @dwork->work immediately. This is for
......
...@@ -53,6 +53,7 @@ ...@@ -53,6 +53,7 @@
#include <linux/bsearch.h> #include <linux/bsearch.h>
#include <linux/dynamic_debug.h> #include <linux/dynamic_debug.h>
#include <linux/audit.h> #include <linux/audit.h>
#include <linux/cfi.h>
#include <uapi/linux/module.h> #include <uapi/linux/module.h>
#include "internal.h" #include "internal.h"
...@@ -1144,8 +1145,6 @@ void __weak module_arch_freeing_init(struct module *mod) ...@@ -1144,8 +1145,6 @@ void __weak module_arch_freeing_init(struct module *mod)
{ {
} }
static void cfi_cleanup(struct module *mod);
/* Free a module, remove from lists, etc. */ /* Free a module, remove from lists, etc. */
static void free_module(struct module *mod) static void free_module(struct module *mod)
{ {
...@@ -1190,9 +1189,6 @@ static void free_module(struct module *mod) ...@@ -1190,9 +1189,6 @@ static void free_module(struct module *mod)
mod->name); mod->name);
mutex_unlock(&module_mutex); mutex_unlock(&module_mutex);
/* Clean up CFI for the module. */
cfi_cleanup(mod);
/* This may be empty, but that's OK */ /* This may be empty, but that's OK */
module_arch_freeing_init(mod); module_arch_freeing_init(mod);
module_memfree(mod->init_layout.base); module_memfree(mod->init_layout.base);
...@@ -2602,8 +2598,9 @@ static int complete_formation(struct module *mod, struct load_info *info) ...@@ -2602,8 +2598,9 @@ static int complete_formation(struct module *mod, struct load_info *info)
if (err < 0) if (err < 0)
goto out; goto out;
/* This relies on module_mutex for list integrity. */ /* These rely on module_mutex for list integrity. */
module_bug_finalize(info->hdr, info->sechdrs, mod); module_bug_finalize(info->hdr, info->sechdrs, mod);
module_cfi_finalize(info->hdr, info->sechdrs, mod);
if (module_check_misalignment(mod)) if (module_check_misalignment(mod))
goto out_misaligned; goto out_misaligned;
...@@ -2665,8 +2662,6 @@ static int unknown_module_param_cb(char *param, char *val, const char *modname, ...@@ -2665,8 +2662,6 @@ static int unknown_module_param_cb(char *param, char *val, const char *modname,
return 0; return 0;
} }
static void cfi_init(struct module *mod);
/* /*
* Allocate and load the module: note that size of section 0 is always * Allocate and load the module: note that size of section 0 is always
* zero, and we rely on this for optional sections. * zero, and we rely on this for optional sections.
...@@ -2796,9 +2791,6 @@ static int load_module(struct load_info *info, const char __user *uargs, ...@@ -2796,9 +2791,6 @@ static int load_module(struct load_info *info, const char __user *uargs,
flush_module_icache(mod); flush_module_icache(mod);
/* Setup CFI for the module. */
cfi_init(mod);
/* Now copy in args */ /* Now copy in args */
mod->args = strndup_user(uargs, ~0UL >> 1); mod->args = strndup_user(uargs, ~0UL >> 1);
if (IS_ERR(mod->args)) { if (IS_ERR(mod->args)) {
...@@ -2875,7 +2867,6 @@ static int load_module(struct load_info *info, const char __user *uargs, ...@@ -2875,7 +2867,6 @@ static int load_module(struct load_info *info, const char __user *uargs,
synchronize_rcu(); synchronize_rcu();
kfree(mod->args); kfree(mod->args);
free_arch_cleanup: free_arch_cleanup:
cfi_cleanup(mod);
module_arch_cleanup(mod); module_arch_cleanup(mod);
free_modinfo: free_modinfo:
free_modinfo(mod); free_modinfo(mod);
...@@ -2961,41 +2952,6 @@ static inline int within(unsigned long addr, void *start, unsigned long size) ...@@ -2961,41 +2952,6 @@ static inline int within(unsigned long addr, void *start, unsigned long size)
return ((void *)addr >= start && (void *)addr < start + size); return ((void *)addr >= start && (void *)addr < start + size);
} }
static void cfi_init(struct module *mod)
{
#ifdef CONFIG_CFI_CLANG
initcall_t *init;
#ifdef CONFIG_MODULE_UNLOAD
exitcall_t *exit;
#endif
rcu_read_lock_sched();
mod->cfi_check = (cfi_check_fn)
find_kallsyms_symbol_value(mod, "__cfi_check");
init = (initcall_t *)
find_kallsyms_symbol_value(mod, "__cfi_jt_init_module");
/* Fix init/exit functions to point to the CFI jump table */
if (init)
mod->init = *init;
#ifdef CONFIG_MODULE_UNLOAD
exit = (exitcall_t *)
find_kallsyms_symbol_value(mod, "__cfi_jt_cleanup_module");
if (exit)
mod->exit = *exit;
#endif
rcu_read_unlock_sched();
cfi_module_add(mod, mod_tree.addr_min);
#endif
}
static void cfi_cleanup(struct module *mod)
{
#ifdef CONFIG_CFI_CLANG
cfi_module_remove(mod, mod_tree.addr_min);
#endif
}
/* Keep in sync with MODULE_FLAGS_BUF_SIZE !!! */ /* Keep in sync with MODULE_FLAGS_BUF_SIZE !!! */
char *module_flags(struct module *mod, char *buf, bool show_state) char *module_flags(struct module *mod, char *buf, bool show_state)
{ {
......
...@@ -1651,7 +1651,7 @@ static void __queue_delayed_work(int cpu, struct workqueue_struct *wq, ...@@ -1651,7 +1651,7 @@ static void __queue_delayed_work(int cpu, struct workqueue_struct *wq,
struct work_struct *work = &dwork->work; struct work_struct *work = &dwork->work;
WARN_ON_ONCE(!wq); WARN_ON_ONCE(!wq);
WARN_ON_FUNCTION_MISMATCH(timer->function, delayed_work_timer_fn); WARN_ON_ONCE(timer->function != delayed_work_timer_fn);
WARN_ON_ONCE(timer_pending(timer)); WARN_ON_ONCE(timer_pending(timer));
WARN_ON_ONCE(!list_empty(&work->entry)); WARN_ON_ONCE(!list_empty(&work->entry));
......
...@@ -135,6 +135,7 @@ static bool is_ignored_symbol(const char *name, char type) ...@@ -135,6 +135,7 @@ static bool is_ignored_symbol(const char *name, char type)
"__ThumbV7PILongThunk_", "__ThumbV7PILongThunk_",
"__LA25Thunk_", /* mips lld */ "__LA25Thunk_", /* mips lld */
"__microLA25Thunk_", "__microLA25Thunk_",
"__kcfi_typeid_", /* CFI type identifiers */
NULL NULL
}; };
......
...@@ -3,20 +3,10 @@ ...@@ -3,20 +3,10 @@
* Archs are free to supply their own linker scripts. ld will * Archs are free to supply their own linker scripts. ld will
* combine them automatically. * combine them automatically.
*/ */
#ifdef CONFIG_CFI_CLANG
# include <asm/page.h>
# define ALIGN_CFI ALIGN(PAGE_SIZE)
# define SANITIZER_DISCARDS *(.eh_frame)
#else
# define ALIGN_CFI
# define SANITIZER_DISCARDS
#endif
SECTIONS { SECTIONS {
/DISCARD/ : { /DISCARD/ : {
*(.discard) *(.discard)
*(.discard.*) *(.discard.*)
SANITIZER_DISCARDS
} }
__ksymtab 0 : { *(SORT(___ksymtab+*)) } __ksymtab 0 : { *(SORT(___ksymtab+*)) }
...@@ -33,6 +23,10 @@ SECTIONS { ...@@ -33,6 +23,10 @@ SECTIONS {
__patchable_function_entries : { *(__patchable_function_entries) } __patchable_function_entries : { *(__patchable_function_entries) }
#ifdef CONFIG_ARCH_USES_CFI_TRAPS
__kcfi_traps : { KEEP(*(.kcfi_traps)) }
#endif
#ifdef CONFIG_LTO_CLANG #ifdef CONFIG_LTO_CLANG
/* /*
* With CONFIG_LTO_CLANG, LLD always enables -fdata-sections and * With CONFIG_LTO_CLANG, LLD always enables -fdata-sections and
...@@ -53,15 +47,6 @@ SECTIONS { ...@@ -53,15 +47,6 @@ SECTIONS {
*(.rodata .rodata.[0-9a-zA-Z_]*) *(.rodata .rodata.[0-9a-zA-Z_]*)
*(.rodata..L*) *(.rodata..L*)
} }
/*
* With CONFIG_CFI_CLANG, we assume __cfi_check is at the beginning
* of the .text section, and is aligned to PAGE_SIZE.
*/
.text : ALIGN_CFI {
*(.text.__cfi_check)
*(.text .text.[0-9a-zA-Z_]* .text..L.cfi*)
}
#endif #endif
} }
......
...@@ -3316,6 +3316,10 @@ static int validate_branch(struct objtool_file *file, struct symbol *func, ...@@ -3316,6 +3316,10 @@ static int validate_branch(struct objtool_file *file, struct symbol *func,
next_insn = next_insn_to_validate(file, insn); next_insn = next_insn_to_validate(file, insn);
if (func && insn->func && func != insn->func->pfunc) { if (func && insn->func && func != insn->func->pfunc) {
/* Ignore KCFI type preambles, which always fall through */
if (!strncmp(func->name, "__cfi_", 6))
return 0;
WARN("%s() falls through to next function %s()", WARN("%s() falls through to next function %s()",
func->name, insn->func->name); func->name, insn->func->name);
return 1; return 1;
...@@ -4113,7 +4117,8 @@ static int validate_ibt(struct objtool_file *file) ...@@ -4113,7 +4117,8 @@ static int validate_ibt(struct objtool_file *file)
!strcmp(sec->name, "__bug_table") || !strcmp(sec->name, "__bug_table") ||
!strcmp(sec->name, "__ex_table") || !strcmp(sec->name, "__ex_table") ||
!strcmp(sec->name, "__jump_table") || !strcmp(sec->name, "__jump_table") ||
!strcmp(sec->name, "__mcount_loc")) !strcmp(sec->name, "__mcount_loc") ||
!strcmp(sec->name, ".kcfi_traps"))
continue; continue;
list_for_each_entry(reloc, &sec->reloc->reloc_list, list) list_for_each_entry(reloc, &sec->reloc->reloc_list, list)
......
...@@ -619,6 +619,11 @@ static int elf_update_symbol(struct elf *elf, struct section *symtab, ...@@ -619,6 +619,11 @@ static int elf_update_symbol(struct elf *elf, struct section *symtab,
Elf64_Xword entsize = symtab->sh.sh_entsize; Elf64_Xword entsize = symtab->sh.sh_entsize;
int max_idx, idx = sym->idx; int max_idx, idx = sym->idx;
Elf_Scn *s, *t = NULL; Elf_Scn *s, *t = NULL;
bool is_special_shndx = sym->sym.st_shndx >= SHN_LORESERVE &&
sym->sym.st_shndx != SHN_XINDEX;
if (is_special_shndx)
shndx = sym->sym.st_shndx;
s = elf_getscn(elf->elf, symtab->idx); s = elf_getscn(elf->elf, symtab->idx);
if (!s) { if (!s) {
...@@ -704,7 +709,7 @@ static int elf_update_symbol(struct elf *elf, struct section *symtab, ...@@ -704,7 +709,7 @@ static int elf_update_symbol(struct elf *elf, struct section *symtab,
} }
/* setup extended section index magic and write the symbol */ /* setup extended section index magic and write the symbol */
if (shndx >= SHN_UNDEF && shndx < SHN_LORESERVE) { if ((shndx >= SHN_UNDEF && shndx < SHN_LORESERVE) || is_special_shndx) {
sym->sym.st_shndx = shndx; sym->sym.st_shndx = shndx;
if (!shndx_data) if (!shndx_data)
shndx = 0; shndx = 0;
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment