Commit ed53a0d9 authored by Peter Zijlstra's avatar Peter Zijlstra

x86/alternative: Use .ibt_endbr_seal to seal indirect calls

Objtool's --ibt option generates .ibt_endbr_seal which lists
superfluous ENDBR instructions. That is those instructions for which
the function is never indirectly called.

Overwrite these ENDBR instructions with a NOP4 such that these
function can never be indirect called, reducing the number of viable
ENDBR targets in the kernel.
Signed-off-by: default avatarPeter Zijlstra (Intel) <peterz@infradead.org>
Acked-by: default avatarJosh Poimboeuf <jpoimboe@redhat.com>
Link: https://lore.kernel.org/r/20220308154319.822545231@infradead.org
parent 89bc853e
......@@ -424,6 +424,10 @@ void __init check_bugs(void)
os_check_bugs();
}
void apply_ibt_endbr(s32 *start, s32 *end)
{
}
void apply_retpolines(s32 *start, s32 *end)
{
}
......
......@@ -1873,7 +1873,7 @@ config CC_HAS_IBT
config X86_KERNEL_IBT
prompt "Indirect Branch Tracking"
bool
depends on X86_64 && CC_HAS_IBT
depends on X86_64 && CC_HAS_IBT && STACK_VALIDATION
help
Build the kernel with support for Indirect Branch Tracking, a
hardware support course-grain forward-edge Control Flow Integrity
......@@ -1881,6 +1881,13 @@ config X86_KERNEL_IBT
an ENDBR instruction, as such, the compiler will instrument the
code with them to make this happen.
In addition to building the kernel with IBT, seal all functions that
are not indirect call targets, avoiding them ever becomming one.
This requires LTO like objtool runs and will slow down the build. It
does significantly reduce the number of ENDBR instructions in the
kernel image.
config X86_INTEL_MEMORY_PROTECTION_KEYS
prompt "Memory Protection Keys"
def_bool y
......
......@@ -76,6 +76,7 @@ extern int alternatives_patched;
extern void alternative_instructions(void);
extern void apply_alternatives(struct alt_instr *start, struct alt_instr *end);
extern void apply_retpolines(s32 *start, s32 *end);
extern void apply_ibt_endbr(s32 *start, s32 *end);
struct module;
......
......@@ -46,8 +46,20 @@ static inline __attribute_const__ u32 gen_endbr(void)
return endbr;
}
static inline __attribute_const__ u32 gen_endbr_poison(void)
{
/*
* 4 byte NOP that isn't NOP4 (in fact it is OSP NOP3), such that it
* will be unique to (former) ENDBR sites.
*/
return 0x001f0f66; /* osp nopl (%rax) */
}
static inline bool is_endbr(u32 val)
{
if (val == gen_endbr_poison())
return true;
val &= ~0x01000000U; /* ENDBR32 -> ENDBR64 */
return val == gen_endbr();
}
......
......@@ -115,6 +115,7 @@ static void __init_or_module add_nops(void *insns, unsigned int len)
}
extern s32 __retpoline_sites[], __retpoline_sites_end[];
extern s32 __ibt_endbr_seal[], __ibt_endbr_seal_end[];
extern struct alt_instr __alt_instructions[], __alt_instructions_end[];
extern s32 __smp_locks[], __smp_locks_end[];
void text_poke_early(void *addr, const void *opcode, size_t len);
......@@ -512,6 +513,42 @@ void __init_or_module noinline apply_retpolines(s32 *start, s32 *end) { }
#endif /* CONFIG_RETPOLINE && CONFIG_STACK_VALIDATION */
#ifdef CONFIG_X86_KERNEL_IBT
/*
* Generated by: objtool --ibt
*/
void __init_or_module noinline apply_ibt_endbr(s32 *start, s32 *end)
{
s32 *s;
for (s = start; s < end; s++) {
u32 endbr, poison = gen_endbr_poison();
void *addr = (void *)s + *s;
if (WARN_ON_ONCE(get_kernel_nofault(endbr, addr)))
continue;
if (WARN_ON_ONCE(!is_endbr(endbr)))
continue;
DPRINTK("ENDBR at: %pS (%px)", addr, addr);
/*
* When we have IBT, the lack of ENDBR will trigger #CP
*/
DUMP_BYTES(((u8*)addr), 4, "%px: orig: ", addr);
DUMP_BYTES(((u8*)&poison), 4, "%px: repl: ", addr);
text_poke_early(addr, &poison, 4);
}
}
#else
void __init_or_module noinline apply_ibt_endbr(s32 *start, s32 *end) { }
#endif /* CONFIG_X86_KERNEL_IBT */
#ifdef CONFIG_SMP
static void alternatives_smp_lock(const s32 *start, const s32 *end,
u8 *text, u8 *text_end)
......@@ -830,6 +867,8 @@ void __init alternative_instructions(void)
*/
apply_alternatives(__alt_instructions, __alt_instructions_end);
apply_ibt_endbr(__ibt_endbr_seal, __ibt_endbr_seal_end);
#ifdef CONFIG_SMP
/* Patch to UP if other cpus not imminent. */
if (!noreplace_smp && (num_present_cpus() == 1 || setup_max_cpus <= 1)) {
......
......@@ -253,7 +253,7 @@ int module_finalize(const Elf_Ehdr *hdr,
{
const Elf_Shdr *s, *text = NULL, *alt = NULL, *locks = NULL,
*para = NULL, *orc = NULL, *orc_ip = NULL,
*retpolines = NULL;
*retpolines = NULL, *ibt_endbr = NULL;
char *secstrings = (void *)hdr + sechdrs[hdr->e_shstrndx].sh_offset;
for (s = sechdrs; s < sechdrs + hdr->e_shnum; s++) {
......@@ -271,6 +271,8 @@ int module_finalize(const Elf_Ehdr *hdr,
orc_ip = s;
if (!strcmp(".retpoline_sites", secstrings + s->sh_name))
retpolines = s;
if (!strcmp(".ibt_endbr_seal", secstrings + s->sh_name))
ibt_endbr = s;
}
/*
......@@ -290,6 +292,10 @@ int module_finalize(const Elf_Ehdr *hdr,
void *aseg = (void *)alt->sh_addr;
apply_alternatives(aseg, aseg + alt->sh_size);
}
if (ibt_endbr) {
void *iseg = (void *)ibt_endbr->sh_addr;
apply_ibt_endbr(iseg, iseg + ibt_endbr->sh_size);
}
if (locks && text) {
void *lseg = (void *)locks->sh_addr;
void *tseg = (void *)text->sh_addr;
......
......@@ -86,12 +86,18 @@ ifdef need-builtin
targets-for-builtin += $(obj)/built-in.a
endif
targets-for-modules := $(patsubst %.o, %.mod, $(filter %.o, $(obj-m)))
targets-for-modules :=
ifdef CONFIG_LTO_CLANG
targets-for-modules += $(patsubst %.o, %.lto.o, $(filter %.o, $(obj-m)))
endif
ifdef CONFIG_X86_KERNEL_IBT
targets-for-modules += $(patsubst %.o, %.objtool, $(filter %.o, $(obj-m)))
endif
targets-for-modules += $(patsubst %.o, %.mod, $(filter %.o, $(obj-m)))
ifdef need-modorder
targets-for-modules += $(obj)/modules.order
endif
......@@ -230,6 +236,7 @@ objtool := $(objtree)/tools/objtool/objtool
objtool_args = \
$(if $(CONFIG_UNWINDER_ORC),orc generate,check) \
$(if $(part-of-module), --module) \
$(if $(CONFIG_X86_KERNEL_IBT), --lto --ibt) \
$(if $(CONFIG_FRAME_POINTER),, --no-fp) \
$(if $(CONFIG_GCOV_KERNEL)$(CONFIG_LTO_CLANG), --no-unreachable)\
$(if $(CONFIG_RETPOLINE), --retpoline) \
......@@ -237,8 +244,8 @@ objtool_args = \
$(if $(CONFIG_FTRACE_MCOUNT_USE_OBJTOOL), --mcount) \
$(if $(CONFIG_SLS), --sls)
cmd_objtool = $(if $(objtool-enabled), ; $(objtool) $(objtool_args) $@)
cmd_gen_objtooldep = $(if $(objtool-enabled), { echo ; echo '$@: $$(wildcard $(objtool))' ; } >> $(dot-target).cmd)
cmd_objtool = $(if $(objtool-enabled), ; $(objtool) $(objtool_args) $(@:.objtool=.o))
cmd_gen_objtooldep = $(if $(objtool-enabled), { echo ; echo '$(@:.objtool=.o): $$(wildcard $(objtool))' ; } >> $(dot-target).cmd)
endif # CONFIG_STACK_VALIDATION
......@@ -247,6 +254,21 @@ ifdef CONFIG_LTO_CLANG
# Skip objtool for LLVM bitcode
$(obj)/%.o: objtool-enabled :=
# objtool was skipped for LLVM bitcode, run it now that we have compiled
# modules into native code
$(obj)/%.lto.o: objtool-enabled = y
$(obj)/%.lto.o: part-of-module := y
else ifdef CONFIG_X86_KERNEL_IBT
# Skip objtool on individual files
$(obj)/%.o: objtool-enabled :=
# instead run objtool on the module as a whole, right before
# the final link pass with the linker script.
$(obj)/%.objtool: objtool-enabled = y
$(obj)/%.objtool: part-of-module := y
else
# 'OBJECT_FILES_NON_STANDARD := y': skip objtool checking for a directory
......@@ -292,18 +314,13 @@ ifdef CONFIG_LTO_CLANG
# Module .o files may contain LLVM bitcode, compile them into native code
# before ELF processing
quiet_cmd_cc_lto_link_modules = LTO [M] $@
cmd_cc_lto_link_modules = \
cmd_cc_lto_link_modules = \
$(LD) $(ld_flags) -r -o $@ \
$(shell [ -s $(@:.lto.o=.o.symversions) ] && \
echo -T $(@:.lto.o=.o.symversions)) \
--whole-archive $(filter-out FORCE,$^) \
$(cmd_objtool)
# objtool was skipped for LLVM bitcode, run it now that we have compiled
# modules into native code
$(obj)/%.lto.o: objtool-enabled = y
$(obj)/%.lto.o: part-of-module := y
$(obj)/%.lto.o: $(obj)/%.o FORCE
$(call if_changed,cc_lto_link_modules)
endif
......@@ -316,6 +333,18 @@ cmd_mod = { \
$(obj)/%.mod: $(obj)/%$(mod-prelink-ext).o FORCE
$(call if_changed,mod)
#
# Since objtool will re-write the file it will change the timestamps, therefore
# it is critical that the %.objtool file gets a timestamp *after* objtool runs.
#
# Additionally, care must be had with ordering this rule against the other rules
# that take %.o as a dependency.
#
cmd_objtool_mod = true $(cmd_objtool) ; touch $@
$(obj)/%.objtool: $(obj)/%$(mod-prelink-ext).o FORCE
$(call if_changed,objtool_mod)
quiet_cmd_cc_lst_c = MKLST $@
cmd_cc_lst_c = $(CC) $(c_flags) -g -c -o $*.o $< && \
$(CONFIG_SHELL) $(srctree)/scripts/makelst $*.o \
......
......@@ -108,7 +108,9 @@ objtool_link()
local objtoolcmd;
local objtoolopt;
if is_enabled CONFIG_LTO_CLANG && is_enabled CONFIG_STACK_VALIDATION; then
if is_enabled CONFIG_STACK_VALIDATION && \
( is_enabled CONFIG_LTO_CLANG || is_enabled CONFIG_X86_KERNEL_IBT ); then
# Don't perform vmlinux validation unless explicitly requested,
# but run objtool on vmlinux.o now that we have an object file.
if is_enabled CONFIG_UNWINDER_ORC; then
......@@ -117,6 +119,10 @@ objtool_link()
objtoolopt="${objtoolopt} --lto"
if is_enabled CONFIG_X86_KERNEL_IBT; then
objtoolopt="${objtoolopt} --ibt"
fi
if is_enabled CONFIG_FTRACE_MCOUNT_USE_OBJTOOL; then
objtoolopt="${objtoolopt} --mcount"
fi
......@@ -168,7 +174,7 @@ vmlinux_link()
# skip output file argument
shift
if is_enabled CONFIG_LTO_CLANG; then
if is_enabled CONFIG_LTO_CLANG || is_enabled CONFIG_X86_KERNEL_IBT; then
# Use vmlinux.o instead of performing the slow LTO link again.
objs=vmlinux.o
libs=
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment