Skip to content

L
linux
Commit c6536676 authored by Linus Torvalds's avatar Linus Torvalds
Browse files
Options

Merge tag 'x86_core_for_v5.13' of git://git.kernel.org/pub/scm/linux/kernel/git/tip/tip 

Pull x86 updates from Borislav Petkov:

 - Turn the stack canary into a normal __percpu variable on 32-bit which
   gets rid of the LAZY_GS stuff and a lot of code.

 - Add an insn_decode() API which all users of the instruction decoder
   should preferrably use. Its goal is to keep the details of the
   instruction decoder away from its users and simplify and streamline
   how one decodes insns in the kernel. Convert its users to it.

 - kprobes improvements and fixes

 - Set the maximum DIE per package variable on Hygon

 - Rip out the dynamic NOP selection and simplify all the machinery
   around selecting NOPs. Use the simplified NOPs in objtool now too.

 - Add Xeon Sapphire Rapids to list of CPUs that support PPIN

 - Simplify the retpolines by folding the entire thing into an
   alternative now that objtool can handle alternatives with stack ops.
   Then, have objtool rewrite the call to the retpoline with the
   alternative which then will get patched at boot time.

 - Document Intel uarch per models in intel-family.h

 - Make Sub-NUMA Clustering topology the default and Cluster-on-Die the
   exception on Intel.

* tag 'x86_core_for_v5.13' of git://git.kernel.org/pub/scm/linux/kernel/git/tip/tip: (53 commits)
  x86, sched: Treat Intel SNC topology as default, COD as exception
  x86/cpu: Comment Skylake server stepping too
  x86/cpu: Resort and comment Intel models
  objtool/x86: Rewrite retpoline thunk calls
  objtool: Skip magical retpoline .altinstr_replacement
  objtool: Cache instruction relocs
  objtool: Keep track of retpoline call sites
  objtool: Add elf_create_undef_symbol()
  objtool: Extract elf_symbol_add()
  objtool: Extract elf_strtab_concat()
  objtool: Create reloc sections implicitly
  objtool: Add elf_create_reloc() helper
  objtool: Rework the elf_rebuild_reloc_section() logic
  objtool: Fix static_call list generation
  objtool: Handle per arch retpoline naming
  objtool: Correctly handle retpoline thunk calls
  x86/retpoline: Simplify retpolines
  x86/alternatives: Optimize optimize_nops()
  x86: Add insn_decode_kernel()
  x86/kprobes: Move 'inline' to the beginning of the kprobe_is_ss() declaration
  ...
parents e7c6e405 2c88d45e
Expand all Hide whitespace changes
Inline Side-by-side
Showing with 1918 additions and 1432 deletions
arch/x86/Kconfig
View file @ c6536676
......@@ -361,10 +361,6 @@ config X86_64_SMP
def_bool y
depends on X86_64 && SMP
config X86_32_LAZY_GS
def_bool y
depends on X86_32 && !STACKPROTECTOR
config ARCH_SUPPORTS_UPROBES
def_bool y
......@@ -387,7 +383,8 @@ config CC_HAS_SANE_STACKPROTECTOR
default $(success,$(srctree)/scripts/gcc-x86_32-has-stack-protector.sh $(CC))
help
We have to make sure stack protector is unconditionally disabled if
the compiler produces broken code.
the compiler produces broken code or if it does not let us control
the segment on 32-bit kernels.
menu "Processor type and features"
......
arch/x86/Makefile
View file @ c6536676
......@@ -80,6 +80,14 @@ ifeq ($(CONFIG_X86_32),y)
# temporary until string.h is fixed
KBUILD_CFLAGS += -ffreestanding
ifeq ($(CONFIG_STACKPROTECTOR),y)
ifeq ($(CONFIG_SMP),y)
KBUILD_CFLAGS += -mstack-protector-guard-reg=fs -mstack-protector-guard-symbol=__stack_chk_guard
else
KBUILD_CFLAGS += -mstack-protector-guard=global
endif
endif
else
BITS := 64
UTS_MACHINE := x86_64
......
arch/x86/boot/compressed/sev-es.c
View file @ c6536676
......@@ -78,16 +78,15 @@ static inline void sev_es_wr_ghcb_msr(u64 val)
static enum es_result vc_decode_insn(struct es_em_ctxt *ctxt)
{
char buffer[MAX_INSN_SIZE];
enum es_result ret;
int ret;
memcpy(buffer, (unsigned char *)ctxt->regs->ip, MAX_INSN_SIZE);
insn_init(&ctxt->insn, buffer, MAX_INSN_SIZE, 1);
insn_get_length(&ctxt->insn);
ret = insn_decode(&ctxt->insn, buffer, MAX_INSN_SIZE, INSN_MODE_64);
if (ret < 0)
return ES_DECODE_FAILED;
ret = ctxt->insn.immediate.got ? ES_OK : ES_DECODE_FAILED;
return ret;
return ES_OK;
}
static enum es_result vc_write_mem(struct es_em_ctxt *ctxt,
......
arch/x86/entry/entry_32.S
View file @ c6536676
......@@ -20,7 +20,7 @@
* 1C(%esp) - %ds
* 20(%esp) - %es
* 24(%esp) - %fs
* 28(%esp) - %gs saved iff !CONFIG_X86_32_LAZY_GS
* 28(%esp) - unused -- was %gs on old stackprotector kernels
* 2C(%esp) - orig_eax
* 30(%esp) - %eip
* 34(%esp) - %cs
......@@ -53,83 +53,6 @@
#define PTI_SWITCH_MASK (1 << PAGE_SHIFT)
/*
* User gs save/restore
*
* %gs is used for userland TLS and kernel only uses it for stack
* canary which is required to be at %gs:20 by gcc. Read the comment
* at the top of stackprotector.h for more info.
*
* Local labels 98 and 99 are used.
*/
#ifdef CONFIG_X86_32_LAZY_GS
/* unfortunately push/pop can't be no-op */
.macro PUSH_GS
pushl $0
.endm
.macro POP_GS pop=0
addl $(4 + \pop), %esp
.endm
.macro POP_GS_EX
.endm
/* all the rest are no-op */
.macro PTGS_TO_GS
.endm
.macro PTGS_TO_GS_EX
.endm
.macro GS_TO_REG reg
.endm
.macro REG_TO_PTGS reg
.endm
.macro SET_KERNEL_GS reg
.endm
#else /* CONFIG_X86_32_LAZY_GS */
.macro PUSH_GS
pushl %gs
.endm
.macro POP_GS pop=0
98: popl %gs
.if \pop <> 0
add $\pop, %esp
.endif
.endm
.macro POP_GS_EX
.pushsection .fixup, "ax"
99: movl $0, (%esp)
jmp 98b
.popsection
_ASM_EXTABLE(98b, 99b)
.endm
.macro PTGS_TO_GS
98: mov PT_GS(%esp), %gs
.endm
.macro PTGS_TO_GS_EX
.pushsection .fixup, "ax"
99: movl $0, PT_GS(%esp)
jmp 98b
.popsection
_ASM_EXTABLE(98b, 99b)
.endm
.macro GS_TO_REG reg
movl %gs, \reg
.endm
.macro REG_TO_PTGS reg
movl \reg, PT_GS(%esp)
.endm
.macro SET_KERNEL_GS reg
movl $(__KERNEL_STACK_CANARY), \reg
movl \reg, %gs
.endm
#endif /* CONFIG_X86_32_LAZY_GS */
/* Unconditionally switch to user cr3 */
.macro SWITCH_TO_USER_CR3 scratch_reg:req
ALTERNATIVE "jmp .Lend_\@", "", X86_FEATURE_PTI
......@@ -282,7 +205,7 @@
.macro SAVE_ALL pt_regs_ax=%eax switch_stacks=0 skip_gs=0 unwind_espfix=0
cld
.if \skip_gs == 0
PUSH_GS
pushl $0
.endif
pushl %fs
......@@ -307,9 +230,6 @@
movl $(__USER_DS), %edx
movl %edx, %ds
movl %edx, %es
.if \skip_gs == 0
SET_KERNEL_GS %edx
.endif
/* Switch to kernel stack if necessary */
.if \switch_stacks > 0
SWITCH_TO_KERNEL_STACK
......@@ -348,7 +268,7 @@
1: popl %ds
2: popl %es
3: popl %fs
POP_GS \pop
addl $(4 + \pop), %esp /* pop the unused "gs" slot */
IRET_FRAME
.pushsection .fixup, "ax"
4: movl $0, (%esp)
......@@ -361,7 +281,6 @@
_ASM_EXTABLE(1b, 4b)
_ASM_EXTABLE(2b, 5b)
_ASM_EXTABLE(3b, 6b)
POP_GS_EX
.endm
.macro RESTORE_ALL_NMI cr3_reg:req pop=0
......@@ -779,7 +698,7 @@ SYM_CODE_START(__switch_to_asm)
#ifdef CONFIG_STACKPROTECTOR
movl TASK_stack_canary(%edx), %ebx
movl %ebx, PER_CPU_VAR(stack_canary)+stack_canary_offset
movl %ebx, PER_CPU_VAR(__stack_chk_guard)
#endif
#ifdef CONFIG_RETPOLINE
......@@ -976,7 +895,6 @@ SYM_FUNC_START(entry_SYSENTER_32)
movl PT_EIP(%esp), %edx /* pt_regs->ip */
movl PT_OLDESP(%esp), %ecx /* pt_regs->sp */
1: mov PT_FS(%esp), %fs
PTGS_TO_GS
popl %ebx /* pt_regs->bx */
addl $2*4, %esp /* skip pt_regs->cx and pt_regs->dx */
......@@ -1012,7 +930,6 @@ SYM_FUNC_START(entry_SYSENTER_32)
jmp 1b
.popsection
_ASM_EXTABLE(1b, 2b)
PTGS_TO_GS_EX
.Lsysenter_fix_flags:
pushl $X86_EFLAGS_FIXED
......@@ -1154,11 +1071,7 @@ SYM_CODE_START_LOCAL_NOALIGN(handle_exception)
SAVE_ALL switch_stacks=1 skip_gs=1 unwind_espfix=1
ENCODE_FRAME_POINTER
/* fixup %gs */
GS_TO_REG %ecx
movl PT_GS(%esp), %edi # get the function address
REG_TO_PTGS %ecx
SET_KERNEL_GS %ecx
/* fixup orig %eax */
movl PT_ORIG_EAX(%esp), %edx # get the error code
......
arch/x86/events/intel/ds.c
View file @ c6536676
......@@ -1353,14 +1353,13 @@ static int intel_pmu_pebs_fixup_ip(struct pt_regs *regs)
is_64bit = kernel_ip(to) || any_64bit_mode(regs);
#endif
insn_init(&insn, kaddr, size, is_64bit);
insn_get_length(&insn);
/*
* Make sure there was not a problem decoding the
* instruction and getting the length. This is
* doubly important because we have an infinite
* loop if insn.length=0.
* Make sure there was not a problem decoding the instruction.
* This is doubly important because we have an infinite loop if
* insn.length=0.
*/
if (!insn.length)
if (insn_get_length(&insn))
break;
to += insn.length;
......
arch/x86/events/intel/lbr.c
View file @ c6536676
......@@ -1224,8 +1224,7 @@ static int branch_type(unsigned long from, unsigned long to, int abort)
is64 = kernel_ip((unsigned long)addr) || any_64bit_mode(current_pt_regs());
#endif
insn_init(&insn, addr, bytes_read, is64);
insn_get_opcode(&insn);
if (!insn.opcode.got)
if (insn_get_opcode(&insn))
return X86_BR_ABORT;
switch (insn.opcode.bytes[0]) {
......@@ -1262,8 +1261,7 @@ static int branch_type(unsigned long from, unsigned long to, int abort)
ret = X86_BR_INT;
break;
case 0xe8: /* call near rel */
insn_get_immediate(&insn);
if (insn.immediate1.value == 0) {
if (insn_get_immediate(&insn) || insn.immediate1.value == 0) {
/* zero length call */
ret = X86_BR_ZERO_CALL;
break;
......@@ -1279,7 +1277,9 @@ static int branch_type(unsigned long from, unsigned long to, int abort)
ret = X86_BR_JMP;
break;
case 0xff: /* call near absolute, call far absolute ind */
insn_get_modrm(&insn);
if (insn_get_modrm(&insn))
return X86_BR_ABORT;
ext = (insn.modrm.bytes[0] >> 3) & 0x7;
switch (ext) {
case 2: /* near ind call */
......
arch/x86/include/asm/alternative.h
View file @ c6536676
......@@ -65,7 +65,6 @@ struct alt_instr {
u16 cpuid; /* cpuid bit set for replacement */
u8 instrlen; /* length of original instruction */
u8 replacementlen; /* length of new instruction */
u8 padlen; /* length of build-time padding */
} __packed;
/*
......@@ -104,7 +103,6 @@ static inline int alternatives_text_reserved(void *start, void *end)
#define alt_end_marker "663"
#define alt_slen "662b-661b"
#define alt_pad_len alt_end_marker"b-662b"
#define alt_total_slen alt_end_marker"b-661b"
#define alt_rlen(num) e_replacement(num)"f-"b_replacement(num)"f"
......@@ -151,8 +149,7 @@ static inline int alternatives_text_reserved(void *start, void *end)
" .long " b_replacement(num)"f - .\n" /* new instruction */ \
" .word " __stringify(feature) "\n" /* feature bit */ \
" .byte " alt_total_slen "\n" /* source len */ \
" .byte " alt_rlen(num) "\n" /* replacement len */ \
" .byte " alt_pad_len "\n" /* pad len */
" .byte " alt_rlen(num) "\n" /* replacement len */
#define ALTINSTR_REPLACEMENT(newinstr, num) /* replacement */ \
"# ALT: replacement " #num "\n" \
......@@ -224,9 +221,6 @@ static inline int alternatives_text_reserved(void *start, void *end)
* Peculiarities:
* No memory clobber here.
* Argument numbers start with 1.
* Best is to use constraints that are fixed size (like (%1) ... "r")
* If you use variable sized constraints like "m" or "g" in the
* replacement make sure to pad to the worst case length.
* Leaving an unused argument 0 to keep API compatibility.
*/
#define alternative_input(oldinstr, newinstr, feature, input...) \
......@@ -315,13 +309,12 @@ static inline int alternatives_text_reserved(void *start, void *end)
* enough information for the alternatives patching code to patch an
* instruction. See apply_alternatives().
*/
.macro altinstruction_entry orig alt feature orig_len alt_len pad_len
.macro altinstruction_entry orig alt feature orig_len alt_len
.long \orig - .
.long \alt - .
.word \feature
.byte \orig_len
.byte \alt_len
.byte \pad_len
.endm
/*
......@@ -338,7 +331,7 @@ static inline int alternatives_text_reserved(void *start, void *end)
142:
.pushsection .altinstructions,"a"
altinstruction_entry 140b,143f,\feature,142b-140b,144f-143f,142b-141b
altinstruction_entry 140b,143f,\feature,142b-140b,144f-143f
.popsection
.pushsection .altinstr_replacement,"ax"
......@@ -375,8 +368,8 @@ static inline int alternatives_text_reserved(void *start, void *end)
142:
.pushsection .altinstructions,"a"
altinstruction_entry 140b,143f,\feature1,142b-140b,144f-143f,142b-141b
altinstruction_entry 140b,144f,\feature2,142b-140b,145f-144f,142b-141b
altinstruction_entry 140b,143f,\feature1,142b-140b,144f-143f
altinstruction_entry 140b,144f,\feature2,142b-140b,145f-144f
.popsection
.pushsection .altinstr_replacement,"ax"
......
arch/x86/include/asm/asm-prototypes.h
View file @ c6536676
......@@ -19,18 +19,19 @@ extern void cmpxchg8b_emu(void);
#ifdef CONFIG_RETPOLINE
#define DECL_INDIRECT_THUNK(reg) \
#undef GEN
#define GEN(reg) \
extern asmlinkage void __x86_indirect_thunk_ ## reg (void);
#define DECL_RETPOLINE(reg) \
extern asmlinkage void __x86_retpoline_ ## reg (void);
#include <asm/GEN-for-each-reg.h>
#undef GEN
#define GEN(reg) DECL_INDIRECT_THUNK(reg)
#define GEN(reg) \
extern asmlinkage void __x86_indirect_alt_call_ ## reg (void);
#include <asm/GEN-for-each-reg.h>
#undef GEN
#define GEN(reg) DECL_RETPOLINE(reg)
#define GEN(reg) \
extern asmlinkage void __x86_indirect_alt_jmp_ ## reg (void);
#include <asm/GEN-for-each-reg.h>
#endif /* CONFIG_RETPOLINE */
arch/x86/include/asm/cpufeatures.h
View file @ c6536676
......@@ -84,7 +84,7 @@
/* CPU types for specific tunings: */
#define X86_FEATURE_K8 ( 3*32+ 4) /* "" Opteron, Athlon64 */
#define X86_FEATURE_K7 ( 3*32+ 5) /* "" Athlon */
/* FREE, was #define X86_FEATURE_K7 ( 3*32+ 5) "" Athlon */
#define X86_FEATURE_P3 ( 3*32+ 6) /* "" P3 */
#define X86_FEATURE_P4 ( 3*32+ 7) /* "" P4 */
#define X86_FEATURE_CONSTANT_TSC ( 3*32+ 8) /* TSC ticks at a constant rate */
......
arch/x86/include/asm/inat.h
View file @ c6536676
......@@ -6,7 +6,7 @@
*
* Written by Masami Hiramatsu <mhiramat@redhat.com>
*/
#include <asm/inat_types.h>
#include <asm/inat_types.h> /* __ignore_sync_check__ */
/*
* Internal bits. Don't use bitmasks directly, because these bits are
......
arch/x86/include/asm/insn-eval.h
View file @ c6536676
......@@ -25,7 +25,7 @@ int insn_fetch_from_user(struct pt_regs *regs,
unsigned char buf[MAX_INSN_SIZE]);
int insn_fetch_from_user_inatomic(struct pt_regs *regs,
unsigned char buf[MAX_INSN_SIZE]);
bool insn_decode(struct insn *insn, struct pt_regs *regs,
unsigned char buf[MAX_INSN_SIZE], int buf_size);
bool insn_decode_from_regs(struct insn *insn, struct pt_regs *regs,
unsigned char buf[MAX_INSN_SIZE], int buf_size);
#endif /* _ASM_X86_INSN_EVAL_H */
arch/x86/include/asm/insn.h
View file @ c6536676
......@@ -9,7 +9,7 @@
#include <asm/byteorder.h>
/* insn_attr_t is defined in inat.h */
#include <asm/inat.h>
#include <asm/inat.h> /* __ignore_sync_check__ */
#if defined(__BYTE_ORDER) ? __BYTE_ORDER == __LITTLE_ENDIAN : defined(__LITTLE_ENDIAN)
......@@ -132,13 +132,25 @@ struct insn {
#define X86_VEX_M_MAX 0x1f /* VEX3.M Maximum value */
extern void insn_init(struct insn *insn, const void *kaddr, int buf_len, int x86_64);
extern void insn_get_prefixes(struct insn *insn);
extern void insn_get_opcode(struct insn *insn);
extern void insn_get_modrm(struct insn *insn);
extern void insn_get_sib(struct insn *insn);
extern void insn_get_displacement(struct insn *insn);
extern void insn_get_immediate(struct insn *insn);
extern void insn_get_length(struct insn *insn);
extern int insn_get_prefixes(struct insn *insn);
extern int insn_get_opcode(struct insn *insn);
extern int insn_get_modrm(struct insn *insn);
extern int insn_get_sib(struct insn *insn);
extern int insn_get_displacement(struct insn *insn);
extern int insn_get_immediate(struct insn *insn);
extern int insn_get_length(struct insn *insn);
enum insn_mode {
INSN_MODE_32,
INSN_MODE_64,
/* Mode is determined by the current kernel build. */
INSN_MODE_KERN,
INSN_NUM_MODES,
};
extern int insn_decode(struct insn *insn, const void *kaddr, int buf_len, enum insn_mode m);
#define insn_decode_kernel(_insn, _ptr) insn_decode((_insn), (_ptr), MAX_INSN_SIZE, INSN_MODE_KERN)
/* Attribute will be determined after getting ModRM (for opcode groups) */
static inline void insn_get_attribute(struct insn *insn)
......@@ -149,17 +161,6 @@ static inline void insn_get_attribute(struct insn *insn)
/* Instruction uses RIP-relative addressing */
extern int insn_rip_relative(struct insn *insn);
/* Init insn for kernel text */
static inline void kernel_insn_init(struct insn *insn,
const void *kaddr, int buf_len)
{
#ifdef CONFIG_X86_64
insn_init(insn, kaddr, buf_len, 1);
#else /* CONFIG_X86_32 */
insn_init(insn, kaddr, buf_len, 0);
#endif
}
static inline int insn_is_avx(struct insn *insn)
{
if (!insn->prefixes.got)
......@@ -179,13 +180,6 @@ static inline int insn_has_emulate_prefix(struct insn *insn)
return !!insn->emulate_prefix_size;
}
/* Ensure this instruction is decoded completely */
static inline int insn_complete(struct insn *insn)
{
return insn->opcode.got && insn->modrm.got && insn->sib.got &&
insn->displacement.got && insn->immediate.got;
}
static inline insn_byte_t insn_vex_m_bits(struct insn *insn)
{
if (insn->vex_prefix.nbytes == 2) /* 2 bytes VEX */
......
arch/x86/include/asm/intel-family.h
View file @ c6536676
......@@ -32,7 +32,9 @@
* _EP - 2 socket server parts
* _EX - 4+ socket server parts
*
* The #define line may optionally include a comment including platform names.
* The #define line may optionally include a comment including platform or core
* names. An exception is made for skylake/kabylake where steppings seem to have gotten
* their own names :-(
*/
/* Wildcard match for FAM6 so X86_MATCH_INTEL_FAM6_MODEL(ANY) works */
......@@ -69,35 +71,41 @@
#define INTEL_FAM6_BROADWELL_X 0x4F
#define INTEL_FAM6_BROADWELL_D 0x56
#define INTEL_FAM6_SKYLAKE_L 0x4E
#define INTEL_FAM6_SKYLAKE 0x5E
#define INTEL_FAM6_SKYLAKE_X 0x55
#define INTEL_FAM6_KABYLAKE_L 0x8E
#define INTEL_FAM6_KABYLAKE 0x9E
#define INTEL_FAM6_SKYLAKE_L 0x4E /* Sky Lake */
#define INTEL_FAM6_SKYLAKE 0x5E /* Sky Lake */
#define INTEL_FAM6_SKYLAKE_X 0x55 /* Sky Lake */
/* CASCADELAKE_X 0x55 Sky Lake -- s: 7 */
/* COOPERLAKE_X 0x55 Sky Lake -- s: 11 */
#define INTEL_FAM6_CANNONLAKE_L 0x66
#define INTEL_FAM6_KABYLAKE_L 0x8E /* Sky Lake */
/* AMBERLAKE_L 0x8E Sky Lake -- s: 9 */
/* COFFEELAKE_L 0x8E Sky Lake -- s: 10 */
/* WHISKEYLAKE_L 0x8E Sky Lake -- s: 11,12 */
#define INTEL_FAM6_ICELAKE_X 0x6A
#define INTEL_FAM6_ICELAKE_D 0x6C
#define INTEL_FAM6_ICELAKE 0x7D
#define INTEL_FAM6_ICELAKE_L 0x7E
#define INTEL_FAM6_ICELAKE_NNPI 0x9D
#define INTEL_FAM6_KABYLAKE 0x9E /* Sky Lake */
/* COFFEELAKE 0x9E Sky Lake -- s: 10-13 */
#define INTEL_FAM6_TIGERLAKE_L 0x8C
#define INTEL_FAM6_TIGERLAKE 0x8D
#define INTEL_FAM6_COMETLAKE 0xA5 /* Sky Lake */
#define INTEL_FAM6_COMETLAKE_L 0xA6 /* Sky Lake */
#define INTEL_FAM6_COMETLAKE 0xA5
#define INTEL_FAM6_COMETLAKE_L 0xA6
#define INTEL_FAM6_CANNONLAKE_L 0x66 /* Palm Cove */
#define INTEL_FAM6_ROCKETLAKE 0xA7
#define INTEL_FAM6_ICELAKE_X 0x6A /* Sunny Cove */
#define INTEL_FAM6_ICELAKE_D 0x6C /* Sunny Cove */
#define INTEL_FAM6_ICELAKE 0x7D /* Sunny Cove */
#define INTEL_FAM6_ICELAKE_L 0x7E /* Sunny Cove */
#define INTEL_FAM6_ICELAKE_NNPI 0x9D /* Sunny Cove */
#define INTEL_FAM6_SAPPHIRERAPIDS_X 0x8F
#define INTEL_FAM6_LAKEFIELD 0x8A /* Sunny Cove / Tremont */
/* Hybrid Core/Atom Processors */
#define INTEL_FAM6_ROCKETLAKE 0xA7 /* Cypress Cove */
#define INTEL_FAM6_LAKEFIELD 0x8A
#define INTEL_FAM6_ALDERLAKE 0x97
#define INTEL_FAM6_ALDERLAKE_L 0x9A
#define INTEL_FAM6_TIGERLAKE_L 0x8C /* Willow Cove */
#define INTEL_FAM6_TIGERLAKE 0x8D /* Willow Cove */
#define INTEL_FAM6_SAPPHIRERAPIDS_X 0x8F /* Willow Cove */
#define INTEL_FAM6_ALDERLAKE 0x97 /* Golden Cove / Gracemont */
#define INTEL_FAM6_ALDERLAKE_L 0x9A /* Golden Cove / Gracemont */
/* "Small Core" Processors (Atom) */
......
arch/x86/include/asm/jump_label.h
View file @ c6536676
......@@ -6,12 +6,6 @@
#define JUMP_LABEL_NOP_SIZE 5
#ifdef CONFIG_X86_64
# define STATIC_KEY_INIT_NOP P6_NOP5_ATOMIC
#else
# define STATIC_KEY_INIT_NOP GENERIC_NOP5_ATOMIC
#endif
#include <asm/asm.h>
#include <asm/nops.h>
......@@ -23,7 +17,7 @@
static __always_inline bool arch_static_branch(struct static_key *key, bool branch)
{
asm_volatile_goto("1:"
".byte " __stringify(STATIC_KEY_INIT_NOP) "\n\t"
".byte " __stringify(BYTES_NOP5) "\n\t"
".pushsection __jump_table, \"aw\" \n\t"
_ASM_ALIGN "\n\t"
".long 1b - ., %l[l_yes] - . \n\t"
......@@ -63,7 +57,7 @@ static __always_inline bool arch_static_branch_jump(struct static_key *key, bool
.long \target - .Lstatic_jump_after_\@
.Lstatic_jump_after_\@:
.else
.byte STATIC_KEY_INIT_NOP
.byte BYTES_NOP5
.endif
.pushsection __jump_table, "aw"
_ASM_ALIGN
......@@ -75,7 +69,7 @@ static __always_inline bool arch_static_branch_jump(struct static_key *key, bool
.macro STATIC_JUMP_IF_FALSE target, key, def
.Lstatic_jump_\@:
.if \def
.byte STATIC_KEY_INIT_NOP
.byte BYTES_NOP5
.else
/* Equivalent to "jmp.d32 \target" */
.byte 0xe9
......
arch/x86/include/asm/kprobes.h
View file @ c6536676
......@@ -65,10 +65,22 @@ struct arch_specific_insn {
* a post_handler).
*/
unsigned boostable:1;
unsigned if_modifier:1;
unsigned is_call:1;
unsigned is_pushf:1;
unsigned is_abs_ip:1;
unsigned char size; /* The size of insn */
union {
unsigned char opcode;
struct {
unsigned char type;
} jcc;
struct {
unsigned char type;
unsigned char asize;
} loop;
struct {
unsigned char reg;
} indirect;
};
s32 rel32; /* relative offset must be s32, s16, or s8 */
void (*emulate_op)(struct kprobe *p, struct pt_regs *regs);
/* Number of bytes of text poked */
int tp_len;
};
......@@ -107,7 +119,6 @@ extern int kprobe_fault_handler(struct pt_regs *regs, int trapnr);
extern int kprobe_exceptions_notify(struct notifier_block *self,
unsigned long val, void *data);
extern int kprobe_int3_handler(struct pt_regs *regs);
extern int kprobe_debug_handler(struct pt_regs *regs);
#else
......
arch/x86/include/asm/nops.h
View file @ c6536676
......@@ -4,89 +4,58 @@
/*
* Define nops for use with alternative() and for tracing.
*
* *_NOP5_ATOMIC must be a single instruction.
*/
#define NOP_DS_PREFIX 0x3e
#ifndef CONFIG_64BIT
/* generic versions from gas
1: nop
the following instructions are NOT nops in 64-bit mode,
for 64-bit mode use K8 or P6 nops instead
2: movl %esi,%esi
3: leal 0x00(%esi),%esi
4: leal 0x00(,%esi,1),%esi
6: leal 0x00000000(%esi),%esi
7: leal 0x00000000(,%esi,1),%esi
*/
#define GENERIC_NOP1 0x90
#define GENERIC_NOP2 0x89,0xf6
#define GENERIC_NOP3 0x8d,0x76,0x00
#define GENERIC_NOP4 0x8d,0x74,0x26,0x00
#define GENERIC_NOP5 GENERIC_NOP1,GENERIC_NOP4
#define GENERIC_NOP6 0x8d,0xb6,0x00,0x00,0x00,0x00
#define GENERIC_NOP7 0x8d,0xb4,0x26,0x00,0x00,0x00,0x00
#define GENERIC_NOP8 GENERIC_NOP1,GENERIC_NOP7
#define GENERIC_NOP5_ATOMIC NOP_DS_PREFIX,GENERIC_NOP4
/*
* Generic 32bit nops from GAS:
*
* 1: nop
* 2: movl %esi,%esi
* 3: leal 0x0(%esi),%esi
* 4: leal 0x0(%esi,%eiz,1),%esi
* 5: leal %ds:0x0(%esi,%eiz,1),%esi
* 6: leal 0x0(%esi),%esi
* 7: leal 0x0(%esi,%eiz,1),%esi
* 8: leal %ds:0x0(%esi,%eiz,1),%esi
*
* Except 5 and 8, which are DS prefixed 4 and 7 resp, where GAS would emit 2
* nop instructions.
*/
#define BYTES_NOP1 0x90
#define BYTES_NOP2 0x89,0xf6
#define BYTES_NOP3 0x8d,0x76,0x00
#define BYTES_NOP4 0x8d,0x74,0x26,0x00
#define BYTES_NOP5 0x3e,BYTES_NOP4
#define BYTES_NOP6 0x8d,0xb6,0x00,0x00,0x00,0x00
#define BYTES_NOP7 0x8d,0xb4,0x26,0x00,0x00,0x00,0x00
#define BYTES_NOP8 0x3e,BYTES_NOP7
/* Opteron 64bit nops
1: nop
2: osp nop
3: osp osp nop
4: osp osp osp nop
*/
#define K8_NOP1 GENERIC_NOP1
#define K8_NOP2 0x66,K8_NOP1
#define K8_NOP3 0x66,K8_NOP2
#define K8_NOP4 0x66,K8_NOP3
#define K8_NOP5 K8_NOP3,K8_NOP2
#define K8_NOP6 K8_NOP3,K8_NOP3
#define K8_NOP7 K8_NOP4,K8_NOP3
#define K8_NOP8 K8_NOP4,K8_NOP4
#define K8_NOP5_ATOMIC 0x66,K8_NOP4
#else
/* K7 nops
uses eax dependencies (arbitrary choice)
1: nop
2: movl %eax,%eax
3: leal (,%eax,1),%eax
4: leal 0x00(,%eax,1),%eax
6: leal 0x00000000(%eax),%eax
7: leal 0x00000000(,%eax,1),%eax
*/
#define K7_NOP1 GENERIC_NOP1
#define K7_NOP2 0x8b,0xc0
#define K7_NOP3 0x8d,0x04,0x20
#define K7_NOP4 0x8d,0x44,0x20,0x00
#define K7_NOP5 K7_NOP4,K7_NOP1
#define K7_NOP6 0x8d,0x80,0,0,0,0
#define K7_NOP7 0x8D,0x04,0x05,0,0,0,0
#define K7_NOP8 K7_NOP7,K7_NOP1
#define K7_NOP5_ATOMIC NOP_DS_PREFIX,K7_NOP4
/*
* Generic 64bit nops from GAS:
*
* 1: nop
* 2: osp nop
* 3: nopl (%eax)
* 4: nopl 0x00(%eax)
* 5: nopl 0x00(%eax,%eax,1)
* 6: osp nopl 0x00(%eax,%eax,1)
* 7: nopl 0x00000000(%eax)
* 8: nopl 0x00000000(%eax,%eax,1)
*/
#define BYTES_NOP1 0x90
#define BYTES_NOP2 0x66,BYTES_NOP1
#define BYTES_NOP3 0x0f,0x1f,0x00
#define BYTES_NOP4 0x0f,0x1f,0x40,0x00
#define BYTES_NOP5 0x0f,0x1f,0x44,0x00,0x00
#define BYTES_NOP6 0x66,BYTES_NOP5
#define BYTES_NOP7 0x0f,0x1f,0x80,0x00,0x00,0x00,0x00
#define BYTES_NOP8 0x0f,0x1f,0x84,0x00,0x00,0x00,0x00,0x00
/* P6 nops
uses eax dependencies (Intel-recommended choice)
1: nop
2: osp nop
3: nopl (%eax)
4: nopl 0x00(%eax)
5: nopl 0x00(%eax,%eax,1)
6: osp nopl 0x00(%eax,%eax,1)
7: nopl 0x00000000(%eax)
8: nopl 0x00000000(%eax,%eax,1)
Note: All the above are assumed to be a single instruction.
There is kernel code that depends on this.
*/
#define P6_NOP1 GENERIC_NOP1
#define P6_NOP2 0x66,0x90
#define P6_NOP3 0x0f,0x1f,0x00
#define P6_NOP4 0x0f,0x1f,0x40,0
#define P6_NOP5 0x0f,0x1f,0x44,0x00,0
#define P6_NOP6 0x66,0x0f,0x1f,0x44,0x00,0
#define P6_NOP7 0x0f,0x1f,0x80,0,0,0,0
#define P6_NOP8 0x0f,0x1f,0x84,0x00,0,0,0,0
#define P6_NOP5_ATOMIC P6_NOP5
#endif /* CONFIG_64BIT */
#ifdef __ASSEMBLY__
#define _ASM_MK_NOP(x) .byte x
......@@ -94,54 +63,19 @@
#define _ASM_MK_NOP(x) ".byte " __stringify(x) "\n"
#endif
#if defined(CONFIG_MK7)
#define ASM_NOP1 _ASM_MK_NOP(K7_NOP1)
#define ASM_NOP2 _ASM_MK_NOP(K7_NOP2)
#define ASM_NOP3 _ASM_MK_NOP(K7_NOP3)
#define ASM_NOP4 _ASM_MK_NOP(K7_NOP4)
#define ASM_NOP5 _ASM_MK_NOP(K7_NOP5)
#define ASM_NOP6 _ASM_MK_NOP(K7_NOP6)
#define ASM_NOP7 _ASM_MK_NOP(K7_NOP7)
#define ASM_NOP8 _ASM_MK_NOP(K7_NOP8)
#define ASM_NOP5_ATOMIC _ASM_MK_NOP(K7_NOP5_ATOMIC)
#elif defined(CONFIG_X86_P6_NOP)
#define ASM_NOP1 _ASM_MK_NOP(P6_NOP1)
#define ASM_NOP2 _ASM_MK_NOP(P6_NOP2)
#define ASM_NOP3 _ASM_MK_NOP(P6_NOP3)
#define ASM_NOP4 _ASM_MK_NOP(P6_NOP4)
#define ASM_NOP5 _ASM_MK_NOP(P6_NOP5)
#define ASM_NOP6 _ASM_MK_NOP(P6_NOP6)
#define ASM_NOP7 _ASM_MK_NOP(P6_NOP7)
#define ASM_NOP8 _ASM_MK_NOP(P6_NOP8)
#define ASM_NOP5_ATOMIC _ASM_MK_NOP(P6_NOP5_ATOMIC)
#elif defined(CONFIG_X86_64)
#define ASM_NOP1 _ASM_MK_NOP(K8_NOP1)
#define ASM_NOP2 _ASM_MK_NOP(K8_NOP2)
#define ASM_NOP3 _ASM_MK_NOP(K8_NOP3)
#define ASM_NOP4 _ASM_MK_NOP(K8_NOP4)
#define ASM_NOP5 _ASM_MK_NOP(K8_NOP5)
#define ASM_NOP6 _ASM_MK_NOP(K8_NOP6)
#define ASM_NOP7 _ASM_MK_NOP(K8_NOP7)
#define ASM_NOP8 _ASM_MK_NOP(K8_NOP8)
#define ASM_NOP5_ATOMIC _ASM_MK_NOP(K8_NOP5_ATOMIC)
#else
#define ASM_NOP1 _ASM_MK_NOP(GENERIC_NOP1)
#define ASM_NOP2 _ASM_MK_NOP(GENERIC_NOP2)
#define ASM_NOP3 _ASM_MK_NOP(GENERIC_NOP3)
#define ASM_NOP4 _ASM_MK_NOP(GENERIC_NOP4)
#define ASM_NOP5 _ASM_MK_NOP(GENERIC_NOP5)
#define ASM_NOP6 _ASM_MK_NOP(GENERIC_NOP6)
#define ASM_NOP7 _ASM_MK_NOP(GENERIC_NOP7)
#define ASM_NOP8 _ASM_MK_NOP(GENERIC_NOP8)
#define ASM_NOP5_ATOMIC _ASM_MK_NOP(GENERIC_NOP5_ATOMIC)
#endif
#define ASM_NOP1 _ASM_MK_NOP(BYTES_NOP1)
#define ASM_NOP2 _ASM_MK_NOP(BYTES_NOP2)
#define ASM_NOP3 _ASM_MK_NOP(BYTES_NOP3)
#define ASM_NOP4 _ASM_MK_NOP(BYTES_NOP4)
#define ASM_NOP5 _ASM_MK_NOP(BYTES_NOP5)
#define ASM_NOP6 _ASM_MK_NOP(BYTES_NOP6)
#define ASM_NOP7 _ASM_MK_NOP(BYTES_NOP7)
#define ASM_NOP8 _ASM_MK_NOP(BYTES_NOP8)
#define ASM_NOP_MAX 8
#define NOP_ATOMIC5 (ASM_NOP_MAX+1) /* Entry for the 5-byte atomic NOP */
#ifndef __ASSEMBLY__
extern const unsigned char * const *ideal_nops;
extern void arch_init_ideal_nops(void);
extern const unsigned char * const x86_nops[];
#endif
#endif /* _ASM_X86_NOPS_H */
arch/x86/include/asm/nospec-branch.h
View file @ c6536676
......@@ -80,7 +80,7 @@
.macro JMP_NOSPEC reg:req
#ifdef CONFIG_RETPOLINE
ALTERNATIVE_2 __stringify(ANNOTATE_RETPOLINE_SAFE; jmp *%\reg), \
__stringify(jmp __x86_retpoline_\reg), X86_FEATURE_RETPOLINE, \
__stringify(jmp __x86_indirect_thunk_\reg), X86_FEATURE_RETPOLINE, \
__stringify(lfence; ANNOTATE_RETPOLINE_SAFE; jmp *%\reg), X86_FEATURE_RETPOLINE_AMD
#else
jmp *%\reg
......@@ -90,7 +90,7 @@
.macro CALL_NOSPEC reg:req
#ifdef CONFIG_RETPOLINE
ALTERNATIVE_2 __stringify(ANNOTATE_RETPOLINE_SAFE; call *%\reg), \
__stringify(call __x86_retpoline_\reg), X86_FEATURE_RETPOLINE, \
__stringify(call __x86_indirect_thunk_\reg), X86_FEATURE_RETPOLINE, \
__stringify(lfence; ANNOTATE_RETPOLINE_SAFE; call *%\reg), X86_FEATURE_RETPOLINE_AMD
#else
call *%\reg
......@@ -128,7 +128,7 @@
ALTERNATIVE_2( \
ANNOTATE_RETPOLINE_SAFE \
"call *%[thunk_target]\n", \
"call __x86_retpoline_%V[thunk_target]\n", \
"call __x86_indirect_thunk_%V[thunk_target]\n", \
X86_FEATURE_RETPOLINE, \
"lfence;\n" \
ANNOTATE_RETPOLINE_SAFE \
......
arch/x86/include/asm/processor.h
View file @ c6536676
......@@ -429,6 +429,9 @@ struct fixed_percpu_data {
* GCC hardcodes the stack canary as %gs:40. Since the
* irq_stack is the object at %gs:0, we reserve the bottom
* 48 bytes of the irq stack for the canary.
*
* Once we are willing to require -mstack-protector-guard-symbol=
* support for x86_64 stackprotector, we can get rid of this.
*/
char gs_base[40];
unsigned long stack_canary;
......@@ -450,17 +453,7 @@ extern asmlinkage void ignore_sysret(void);
void current_save_fsgs(void);
#else /* X86_64 */
#ifdef CONFIG_STACKPROTECTOR
/*
* Make sure stack canary segment base is cached-aligned:
* "For Intel Atom processors, avoid non zero segment base address
* that is not aligned to cache line boundary at all cost."
* (Optim Ref Manual Assembly/Compiler Coding Rule 15.)
*/
struct stack_canary {
char __pad[20]; /* canary at %gs:20 */
unsigned long canary;
};
DECLARE_PER_CPU_ALIGNED(struct stack_canary, stack_canary);
DECLARE_PER_CPU(unsigned long, __stack_chk_guard);
#endif
DECLARE_PER_CPU(struct irq_stack *, hardirq_stack_ptr);
DECLARE_PER_CPU(struct irq_stack *, softirq_stack_ptr);
......
arch/x86/include/asm/ptrace.h
View file @ c6536676
......@@ -37,7 +37,10 @@ struct pt_regs {
unsigned short __esh;
unsigned short fs;
unsigned short __fsh;
/* On interrupt, gs and __gsh store the vector number. */
/*
* On interrupt, gs and __gsh store the vector number. They never
* store gs any more.
*/
unsigned short gs;
unsigned short __gsh;
/* On interrupt, this is the error code. */
......
arch/x86/include/asm/segment.h
View file @ c6536676
......@@ -95,7 +95,7 @@
*
* 26 - ESPFIX small SS
* 27 - per-cpu [ offset to per-cpu data area ]
* 28 - stack_canary-20 [ for stack protector ] <=== cacheline #8
* 28 - unused
* 29 - unused
* 30 - unused
* 31 - TSS for double fault handler
......@@ -118,7 +118,6 @@
#define GDT_ENTRY_ESPFIX_SS 26
#define GDT_ENTRY_PERCPU 27
#define GDT_ENTRY_STACK_CANARY 28
#define GDT_ENTRY_DOUBLEFAULT_TSS 31
......@@ -158,12 +157,6 @@
# define __KERNEL_PERCPU 0
#endif
#ifdef CONFIG_STACKPROTECTOR
# define __KERNEL_STACK_CANARY (GDT_ENTRY_STACK_CANARY*8)
#else
# define __KERNEL_STACK_CANARY 0
#endif
#else /* 64-bit: */
#include <asm/cache.h>
......@@ -364,22 +357,15 @@ static inline void __loadsegment_fs(unsigned short value)
asm("mov %%" #seg ",%0":"=r" (value) : : "memory")
/*
* x86-32 user GS accessors:
* x86-32 user GS accessors. This is ugly and could do with some cleaning up.
*/
#ifdef CONFIG_X86_32
# ifdef CONFIG_X86_32_LAZY_GS
# define get_user_gs(regs) (u16)({ unsigned long v; savesegment(gs, v); v; })
# define set_user_gs(regs, v) loadsegment(gs, (unsigned long)(v))
# define task_user_gs(tsk) ((tsk)->thread.gs)
# define lazy_save_gs(v) savesegment(gs, (v))
# define lazy_load_gs(v) loadsegment(gs, (v))
# else /* X86_32_LAZY_GS */
# define get_user_gs(regs) (u16)((regs)->gs)
# define set_user_gs(regs, v) do { (regs)->gs = (v); } while (0)
# define task_user_gs(tsk) (task_pt_regs(tsk)->gs)
# define lazy_save_gs(v) do { } while (0)
# define lazy_load_gs(v) do { } while (0)
# endif /* X86_32_LAZY_GS */
# define get_user_gs(regs) (u16)({ unsigned long v; savesegment(gs, v); v; })
# define set_user_gs(regs, v) loadsegment(gs, (unsigned long)(v))
# define task_user_gs(tsk) ((tsk)->thread.gs)
# define lazy_save_gs(v) savesegment(gs, (v))
# define lazy_load_gs(v) loadsegment(gs, (v))
# define load_gs_index(v) loadsegment(gs, (v))
#endif /* X86_32 */
#endif /* !__ASSEMBLY__ */
......
arch/x86/include/asm/special_insns.h
View file @ c6536676
......@@ -214,7 +214,7 @@ static inline void clflush(volatile void *__p)
static inline void clflushopt(volatile void *__p)
{
alternative_io(".byte " __stringify(NOP_DS_PREFIX) "; clflush %P0",
alternative_io(".byte 0x3e; clflush %P0",
".byte 0x66; clflush %P0",
X86_FEATURE_CLFLUSHOPT,
"+m" (*(volatile char __force *)__p));
......@@ -225,7 +225,7 @@ static inline void clwb(volatile void *__p)
volatile struct { char x[64]; } *p = __p;
asm volatile(ALTERNATIVE_2(
".byte " __stringify(NOP_DS_PREFIX) "; clflush (%[pax])",
".byte 0x3e; clflush (%[pax])",
".byte 0x66; clflush (%[pax])", /* clflushopt (%%rax) */
X86_FEATURE_CLFLUSHOPT,
".byte 0x66, 0x0f, 0xae, 0x30", /* clwb (%%rax) */
......
arch/x86/include/asm/stackprotector.h
View file @ c6536676
......@@ -5,30 +5,23 @@
* Stack protector works by putting predefined pattern at the start of
* the stack frame and verifying that it hasn't been overwritten when
* returning from the function. The pattern is called stack canary
* and unfortunately gcc requires it to be at a fixed offset from %gs.
* On x86_64, the offset is 40 bytes and on x86_32 20 bytes. x86_64
* and x86_32 use segment registers differently and thus handles this
* requirement differently.
* and unfortunately gcc historically required it to be at a fixed offset
* from the percpu segment base. On x86_64, the offset is 40 bytes.
*
* On x86_64, %gs is shared by percpu area and stack canary. All
* percpu symbols are zero based and %gs points to the base of percpu
* area. The first occupant of the percpu area is always
* fixed_percpu_data which contains stack_canary at offset 40. Userland
* %gs is always saved and restored on kernel entry and exit using
* swapgs, so stack protector doesn't add any complexity there.
* The same segment is shared by percpu area and stack canary. On
* x86_64, percpu symbols are zero based and %gs (64-bit) points to the
* base of percpu area. The first occupant of the percpu area is always
* fixed_percpu_data which contains stack_canary at the approproate
* offset. On x86_32, the stack canary is just a regular percpu
* variable.
*
* On x86_32, it's slightly more complicated. As in x86_64, %gs is
* used for userland TLS. Unfortunately, some processors are much
* slower at loading segment registers with different value when
* entering and leaving the kernel, so the kernel uses %fs for percpu
* area and manages %gs lazily so that %gs is switched only when
* necessary, usually during task switch.
* Putting percpu data in %fs on 32-bit is a minor optimization compared to
* using %gs. Since 32-bit userspace normally has %fs == 0, we are likely
* to load 0 into %fs on exit to usermode, whereas with percpu data in
* %gs, we are likely to load a non-null %gs on return to user mode.
*
* As gcc requires the stack canary at %gs:20, %gs can't be managed
* lazily if stack protector is enabled, so the kernel saves and
* restores userland %gs on kernel entry and exit. This behavior is
* controlled by CONFIG_X86_32_LAZY_GS and accessors are defined in
* system.h to hide the details.
* Once we are willing to require GCC 8.1 or better for 64-bit stackprotector
* support, we can remove some of this complexity.
*/
#ifndef _ASM_STACKPROTECTOR_H
......@@ -44,14 +37,6 @@
#include <linux/random.h>
#include <linux/sched.h>
/*
* 24 byte read-only segment initializer for stack canary. Linker
* can't handle the address bit shifting. Address will be set in
* head_32 for boot CPU and setup_per_cpu_areas() for others.
*/
#define GDT_STACK_CANARY_INIT \
[GDT_ENTRY_STACK_CANARY] = GDT_ENTRY_INIT(0x4090, 0, 0x18),
/*
* Initialize the stackprotector canary value.
*
......@@ -86,7 +71,7 @@ static __always_inline void boot_init_stack_canary(void)
#ifdef CONFIG_X86_64
this_cpu_write(fixed_percpu_data.stack_canary, canary);
#else
this_cpu_write(stack_canary.canary, canary);
this_cpu_write(__stack_chk_guard, canary);
#endif
}
......@@ -95,48 +80,16 @@ static inline void cpu_init_stack_canary(int cpu, struct task_struct *idle)
#ifdef CONFIG_X86_64
per_cpu(fixed_percpu_data.stack_canary, cpu) = idle->stack_canary;
#else
per_cpu(stack_canary.canary, cpu) = idle->stack_canary;
#endif
}
static inline void setup_stack_canary_segment(int cpu)
{
#ifdef CONFIG_X86_32
unsigned long canary = (unsigned long)&per_cpu(stack_canary, cpu);
struct desc_struct *gdt_table = get_cpu_gdt_rw(cpu);
struct desc_struct desc;
desc = gdt_table[GDT_ENTRY_STACK_CANARY];
set_desc_base(&desc, canary);
write_gdt_entry(gdt_table, GDT_ENTRY_STACK_CANARY, &desc, DESCTYPE_S);
#endif
}
static inline void load_stack_canary_segment(void)
{
#ifdef CONFIG_X86_32
asm("mov %0, %%gs" : : "r" (__KERNEL_STACK_CANARY) : "memory");
per_cpu(__stack_chk_guard, cpu) = idle->stack_canary;
#endif
}
#else /* STACKPROTECTOR */
#define GDT_STACK_CANARY_INIT
/* dummy boot_init_stack_canary() is defined in linux/stackprotector.h */
static inline void setup_stack_canary_segment(int cpu)
{ }
static inline void cpu_init_stack_canary(int cpu, struct task_struct *idle)
{ }
static inline void load_stack_canary_segment(void)
{
#ifdef CONFIG_X86_32
asm volatile ("mov %0, %%gs" : : "r" (0));
#endif
}
#endif /* STACKPROTECTOR */
#endif /* _ASM_STACKPROTECTOR_H */
arch/x86/include/asm/suspend_32.h
View file @ c6536676
......@@ -13,12 +13,10 @@
/* image of the saved processor state */
struct saved_context {
/*
* On x86_32, all segment registers, with the possible exception of
* gs, are saved at kernel entry in pt_regs.
* On x86_32, all segment registers except gs are saved at kernel
* entry in pt_regs.
*/
#ifdef CONFIG_X86_32_LAZY_GS
u16 gs;
#endif
unsigned long cr0, cr2, cr3, cr4;
u64 misc_enable;
bool misc_enable_saved;
......
arch/x86/kernel/alternative.c
View file @ c6536676
......@@ -75,186 +75,30 @@ do { \
} \
} while (0)
/*
* Each GENERIC_NOPX is of X bytes, and defined as an array of bytes
* that correspond to that nop. Getting from one nop to the next, we
* add to the array the offset that is equal to the sum of all sizes of
* nops preceding the one we are after.
*
* Note: The GENERIC_NOP5_ATOMIC is at the end, as it breaks the
* nice symmetry of sizes of the previous nops.
*/
#if defined(GENERIC_NOP1) && !defined(CONFIG_X86_64)
static const unsigned char intelnops[] =
{
GENERIC_NOP1,
GENERIC_NOP2,
GENERIC_NOP3,
GENERIC_NOP4,
GENERIC_NOP5,
GENERIC_NOP6,
GENERIC_NOP7,
GENERIC_NOP8,
GENERIC_NOP5_ATOMIC
};
static const unsigned char * const intel_nops[ASM_NOP_MAX+2] =
const unsigned char x86nops[] =
{
NULL,
intelnops,
intelnops + 1,
intelnops + 1 + 2,
intelnops + 1 + 2 + 3,
intelnops + 1 + 2 + 3 + 4,
intelnops + 1 + 2 + 3 + 4 + 5,
intelnops + 1 + 2 + 3 + 4 + 5 + 6,
intelnops + 1 + 2 + 3 + 4 + 5 + 6 + 7,
intelnops + 1 + 2 + 3 + 4 + 5 + 6 + 7 + 8,
BYTES_NOP1,
BYTES_NOP2,
BYTES_NOP3,
BYTES_NOP4,
BYTES_NOP5,
BYTES_NOP6,
BYTES_NOP7,
BYTES_NOP8,
};
#endif
#ifdef K8_NOP1
static const unsigned char k8nops[] =
{
K8_NOP1,
K8_NOP2,
K8_NOP3,
K8_NOP4,
K8_NOP5,
K8_NOP6,
K8_NOP7,
K8_NOP8,
K8_NOP5_ATOMIC
};
static const unsigned char * const k8_nops[ASM_NOP_MAX+2] =
const unsigned char * const x86_nops[ASM_NOP_MAX+1] =
{
NULL,
k8nops,
k8nops + 1,
k8nops + 1 + 2,
k8nops + 1 + 2 + 3,
k8nops + 1 + 2 + 3 + 4,
k8nops + 1 + 2 + 3 + 4 + 5,
k8nops + 1 + 2 + 3 + 4 + 5 + 6,
k8nops + 1 + 2 + 3 + 4 + 5 + 6 + 7,
k8nops + 1 + 2 + 3 + 4 + 5 + 6 + 7 + 8,
x86nops,
x86nops + 1,
x86nops + 1 + 2,
x86nops + 1 + 2 + 3,
x86nops + 1 + 2 + 3 + 4,
x86nops + 1 + 2 + 3 + 4 + 5,
x86nops + 1 + 2 + 3 + 4 + 5 + 6,
x86nops + 1 + 2 + 3 + 4 + 5 + 6 + 7,
};
#endif
#if defined(K7_NOP1) && !defined(CONFIG_X86_64)
static const unsigned char k7nops[] =
{
K7_NOP1,
K7_NOP2,
K7_NOP3,
K7_NOP4,
K7_NOP5,
K7_NOP6,
K7_NOP7,
K7_NOP8,
K7_NOP5_ATOMIC
};
static const unsigned char * const k7_nops[ASM_NOP_MAX+2] =
{
NULL,
k7nops,
k7nops + 1,
k7nops + 1 + 2,
k7nops + 1 + 2 + 3,
k7nops + 1 + 2 + 3 + 4,
k7nops + 1 + 2 + 3 + 4 + 5,
k7nops + 1 + 2 + 3 + 4 + 5 + 6,
k7nops + 1 + 2 + 3 + 4 + 5 + 6 + 7,
k7nops + 1 + 2 + 3 + 4 + 5 + 6 + 7 + 8,
};
#endif
#ifdef P6_NOP1
static const unsigned char p6nops[] =
{
P6_NOP1,
P6_NOP2,
P6_NOP3,
P6_NOP4,
P6_NOP5,
P6_NOP6,
P6_NOP7,
P6_NOP8,
P6_NOP5_ATOMIC
};
static const unsigned char * const p6_nops[ASM_NOP_MAX+2] =
{
NULL,
p6nops,
p6nops + 1,
p6nops + 1 + 2,
p6nops + 1 + 2 + 3,
p6nops + 1 + 2 + 3 + 4,
p6nops + 1 + 2 + 3 + 4 + 5,
p6nops + 1 + 2 + 3 + 4 + 5 + 6,
p6nops + 1 + 2 + 3 + 4 + 5 + 6 + 7,
p6nops + 1 + 2 + 3 + 4 + 5 + 6 + 7 + 8,
};
#endif
/* Initialize these to a safe default */
#ifdef CONFIG_X86_64
const unsigned char * const *ideal_nops = p6_nops;
#else
const unsigned char * const *ideal_nops = intel_nops;
#endif
void __init arch_init_ideal_nops(void)
{
switch (boot_cpu_data.x86_vendor) {
case X86_VENDOR_INTEL:
/*
* Due to a decoder implementation quirk, some
* specific Intel CPUs actually perform better with
* the "k8_nops" than with the SDM-recommended NOPs.
*/
if (boot_cpu_data.x86 == 6 &&
boot_cpu_data.x86_model >= 0x0f &&
boot_cpu_data.x86_model != 0x1c &&
boot_cpu_data.x86_model != 0x26 &&
boot_cpu_data.x86_model != 0x27 &&
boot_cpu_data.x86_model < 0x30) {
ideal_nops = k8_nops;
} else if (boot_cpu_has(X86_FEATURE_NOPL)) {
ideal_nops = p6_nops;
} else {
#ifdef CONFIG_X86_64
ideal_nops = k8_nops;
#else
ideal_nops = intel_nops;
#endif
}
break;
case X86_VENDOR_HYGON:
ideal_nops = p6_nops;
return;
case X86_VENDOR_AMD:
if (boot_cpu_data.x86 > 0xf) {
ideal_nops = p6_nops;
return;
}
fallthrough;
default:
#ifdef CONFIG_X86_64
ideal_nops = k8_nops;
#else
if (boot_cpu_has(X86_FEATURE_K8))
ideal_nops = k8_nops;
else if (boot_cpu_has(X86_FEATURE_K7))
ideal_nops = k7_nops;
else
ideal_nops = intel_nops;
#endif
}
}
/* Use this to add nops to a buffer, then text_poke the whole buffer. */
static void __init_or_module add_nops(void *insns, unsigned int len)
......@@ -263,7 +107,7 @@ static void __init_or_module add_nops(void *insns, unsigned int len)
unsigned int noplen = len;
if (noplen > ASM_NOP_MAX)
noplen = ASM_NOP_MAX;
memcpy(insns, ideal_nops[noplen], noplen);
memcpy(insns, x86_nops[noplen], noplen);
insns += noplen;
len -= noplen;
}
......@@ -345,19 +189,35 @@ recompute_jump(struct alt_instr *a, u8 *orig_insn, u8 *repl_insn, u8 *insn_buff)
static void __init_or_module noinline optimize_nops(struct alt_instr *a, u8 *instr)
{
unsigned long flags;
int i;
struct insn insn;
int nop, i = 0;
for (i = 0; i < a->padlen; i++) {
if (instr[i] != 0x90)
/*
* Jump over the non-NOP insns, the remaining bytes must be single-byte
* NOPs, optimize them.
*/
for (;;) {
if (insn_decode_kernel(&insn, &instr[i]))
return;
if (insn.length == 1 && insn.opcode.bytes[0] == 0x90)
break;
if ((i += insn.length) >= a->instrlen)
return;
}
for (nop = i; i < a->instrlen; i++) {
if (WARN_ONCE(instr[i] != 0x90, "Not a NOP at 0x%px\n", &instr[i]))
return;
}
local_irq_save(flags);
add_nops(instr + (a->instrlen - a->padlen), a->padlen);
add_nops(instr + nop, i - nop);
local_irq_restore(flags);
DUMP_BYTES(instr, a->instrlen, "%px: [%d:%d) optimized NOPs: ",
instr, a->instrlen - a->padlen, a->padlen);
instr, nop, a->instrlen);
}
/*
......@@ -403,19 +263,15 @@ void __init_or_module noinline apply_alternatives(struct alt_instr *start,
* - feature not present but ALTINSTR_FLAG_INV is set to mean,
* patch if feature is *NOT* present.
*/
if (!boot_cpu_has(feature) == !(a->cpuid & ALTINSTR_FLAG_INV)) {
if (a->padlen > 1)
optimize_nops(a, instr);
continue;
}
if (!boot_cpu_has(feature) == !(a->cpuid & ALTINSTR_FLAG_INV))
goto next;
DPRINTK("feat: %s%d*32+%d, old: (%pS (%px) len: %d), repl: (%px, len: %d), pad: %d",
DPRINTK("feat: %s%d*32+%d, old: (%pS (%px) len: %d), repl: (%px, len: %d)",
(a->cpuid & ALTINSTR_FLAG_INV) ? "!" : "",
feature >> 5,
feature & 0x1f,
instr, instr, a->instrlen,
replacement, a->replacementlen, a->padlen);
replacement, a->replacementlen);
DUMP_BYTES(instr, a->instrlen, "%px: old_insn: ", instr);
DUMP_BYTES(replacement, a->replacementlen, "%px: rpl_insn: ", replacement);
......@@ -439,14 +295,15 @@ void __init_or_module noinline apply_alternatives(struct alt_instr *start,
if (a->replacementlen && is_jmp(replacement[0]))
recompute_jump(a, instr, replacement, insn_buff);
if (a->instrlen > a->replacementlen) {
add_nops(insn_buff + a->replacementlen,
a->instrlen - a->replacementlen);
insn_buff_sz += a->instrlen - a->replacementlen;
}
for (; insn_buff_sz < a->instrlen; insn_buff_sz++)
insn_buff[insn_buff_sz] = 0x90;
DUMP_BYTES(insn_buff, insn_buff_sz, "%px: final_insn: ", instr);
text_poke_early(instr, insn_buff, insn_buff_sz);
next:
optimize_nops(a, instr);
}
}
......@@ -1310,15 +1167,15 @@ static void text_poke_loc_init(struct text_poke_loc *tp, void *addr,
const void *opcode, size_t len, const void *emulate)
{
struct insn insn;
int ret;
memcpy((void *)tp->text, opcode, len);
if (!emulate)
emulate = opcode;
kernel_insn_init(&insn, emulate, MAX_INSN_SIZE);
insn_get_length(&insn);
ret = insn_decode_kernel(&insn, emulate);
BUG_ON(!insn_complete(&insn));
BUG_ON(ret < 0);
BUG_ON(len != insn.length);
tp->rel_addr = addr - (void *)_stext;
......@@ -1338,13 +1195,13 @@ static void text_poke_loc_init(struct text_poke_loc *tp, void *addr,
default: /* assume NOP */
switch (len) {
case 2: /* NOP2 -- emulate as JMP8+0 */
BUG_ON(memcmp(emulate, ideal_nops[len], len));
BUG_ON(memcmp(emulate, x86_nops[len], len));
tp->opcode = JMP8_INSN_OPCODE;
tp->rel32 = 0;
break;
case 5: /* NOP5 -- emulate as JMP32+0 */
BUG_ON(memcmp(emulate, ideal_nops[NOP_ATOMIC5], len));
BUG_ON(memcmp(emulate, x86_nops[len], len));
tp->opcode = JMP32_INSN_OPCODE;
tp->rel32 = 0;
break;
......
arch/x86/kernel/asm-offsets_32.c
View file @ c6536676
......@@ -53,11 +53,6 @@ void foo(void)
offsetof(struct cpu_entry_area, tss.x86_tss.sp1) -
offsetofend(struct cpu_entry_area, entry_stack_page.stack));
#ifdef CONFIG_STACKPROTECTOR
BLANK();
OFFSET(stack_canary_offset, stack_canary, canary);
#endif
BLANK();
DEFINE(EFI_svam, offsetof(efi_runtime_services_t, set_virtual_address_map));
}
arch/x86/kernel/cpu/amd.c
View file @ c6536676
......@@ -628,11 +628,6 @@ static void early_init_amd(struct cpuinfo_x86 *c)
early_init_amd_mc(c);
#ifdef CONFIG_X86_32
if (c->x86 == 6)
set_cpu_cap(c, X86_FEATURE_K7);
#endif
if (c->x86 >= 0xf)
set_cpu_cap(c, X86_FEATURE_K8);
......
arch/x86/kernel/cpu/common.c
View file @ c6536676
......@@ -161,7 +161,6 @@ DEFINE_PER_CPU_PAGE_ALIGNED(struct gdt_page, gdt_page) = { .gdt = {
[GDT_ENTRY_ESPFIX_SS] = GDT_ENTRY_INIT(0xc092, 0, 0xfffff),
[GDT_ENTRY_PERCPU] = GDT_ENTRY_INIT(0xc092, 0, 0xfffff),
GDT_STACK_CANARY_INIT
#endif
} };
EXPORT_PER_CPU_SYMBOL_GPL(gdt_page);
......@@ -599,7 +598,6 @@ void load_percpu_segment(int cpu)
__loadsegment_simple(gs, 0);
wrmsrl(MSR_GS_BASE, cpu_kernelmode_gs_base(cpu));
#endif
load_stack_canary_segment();
}
#ifdef CONFIG_X86_32
......@@ -1798,7 +1796,8 @@ DEFINE_PER_CPU(unsigned long, cpu_current_top_of_stack) =
EXPORT_PER_CPU_SYMBOL(cpu_current_top_of_stack);
#ifdef CONFIG_STACKPROTECTOR
DEFINE_PER_CPU_ALIGNED(struct stack_canary, stack_canary);
DEFINE_PER_CPU(unsigned long, __stack_chk_guard);
EXPORT_PER_CPU_SYMBOL(__stack_chk_guard);
#endif
#endif /* CONFIG_X86_64 */
......
arch/x86/kernel/cpu/hygon.c
View file @ c6536676
......@@ -215,12 +215,12 @@ static void bsp_init_hygon(struct cpuinfo_x86 *c)
u32 ecx;
ecx = cpuid_ecx(0x8000001e);
nodes_per_socket = ((ecx >> 8) & 7) + 1;
__max_die_per_package = nodes_per_socket = ((ecx >> 8) & 7) + 1;
} else if (boot_cpu_has(X86_FEATURE_NODEID_MSR)) {
u64 value;
rdmsrl(MSR_FAM10H_NODE_ID, value);
nodes_per_socket = ((value >> 3) & 7) + 1;
__max_die_per_package = nodes_per_socket = ((value >> 3) & 7) + 1;
}
if (!boot_cpu_has(X86_FEATURE_AMD_SSBD) &&
......
arch/x86/kernel/cpu/mce/intel.c
View file @ c6536676
......@@ -486,6 +486,7 @@ static void intel_ppin_init(struct cpuinfo_x86 *c)
case INTEL_FAM6_BROADWELL_X:
case INTEL_FAM6_SKYLAKE_X:
case INTEL_FAM6_ICELAKE_X:
case INTEL_FAM6_SAPPHIRERAPIDS_X:
case INTEL_FAM6_XEON_PHI_KNL:
case INTEL_FAM6_XEON_PHI_KNM:
......
arch/x86/kernel/cpu/mce/severity.c
View file @ c6536676
......@@ -218,15 +218,15 @@ static struct severity {
static bool is_copy_from_user(struct pt_regs *regs)
{
u8 insn_buf[MAX_INSN_SIZE];
struct insn insn;
unsigned long addr;
struct insn insn;
int ret;
if (copy_from_kernel_nofault(insn_buf, (void *)regs->ip, MAX_INSN_SIZE))
return false;
kernel_insn_init(&insn, insn_buf, MAX_INSN_SIZE);
insn_get_opcode(&insn);
if (!insn.opcode.got)
ret = insn_decode_kernel(&insn, insn_buf);
if (ret < 0)
return false;
switch (insn.opcode.value) {
......@@ -234,10 +234,6 @@ static bool is_copy_from_user(struct pt_regs *regs)
case 0x8A: case 0x8B:
/* MOVZ mem,reg */
case 0xB60F: case 0xB70F:
insn_get_modrm(&insn);
insn_get_sib(&insn);
if (!insn.modrm.got || !insn.sib.got)
return false;
addr = (unsigned long)insn_get_addr_ref(&insn, regs);
break;
/* REP MOVS */
......
arch/x86/kernel/doublefault_32.c
View file @ c6536676
......@@ -100,9 +100,7 @@ DEFINE_PER_CPU_PAGE_ALIGNED(struct doublefault_stack, doublefault_stack) = {
.ss = __KERNEL_DS,
.ds = __USER_DS,
.fs = __KERNEL_PERCPU,
#ifndef CONFIG_X86_32_LAZY_GS
.gs = __KERNEL_STACK_CANARY,
#endif
.gs = 0,
.__cr3 = __pa_nodebug(swapper_pg_dir),
},
......
arch/x86/kernel/ftrace.c
View file @ c6536676
......@@ -66,7 +66,7 @@ int ftrace_arch_code_modify_post_process(void)
static const char *ftrace_nop_replace(void)
{
return ideal_nops[NOP_ATOMIC5];
return x86_nops[5];
}
static const char *ftrace_call_replace(unsigned long ip, unsigned long addr)
......@@ -377,7 +377,7 @@ create_trampoline(struct ftrace_ops *ops, unsigned int *tramp_size)
ip = trampoline + (jmp_offset - start_offset);
if (WARN_ON(*(char *)ip != 0x75))
goto fail;
ret = copy_from_kernel_nofault(ip, ideal_nops[2], 2);
ret = copy_from_kernel_nofault(ip, x86_nops[2], 2);
if (ret < 0)
goto fail;
}
......
arch/x86/kernel/head_32.S
View file @ c6536676
......@@ -318,8 +318,8 @@ SYM_FUNC_START(startup_32_smp)
movl $(__KERNEL_PERCPU), %eax
movl %eax,%fs # set this cpu's percpu
movl $(__KERNEL_STACK_CANARY),%eax
movl %eax,%gs
xorl %eax,%eax
movl %eax,%gs # clear possible garbage in %gs
xorl %eax,%eax # Clear LDT
lldt %ax
......@@ -339,20 +339,6 @@ SYM_FUNC_END(startup_32_smp)
*/
__INIT
setup_once:
#ifdef CONFIG_STACKPROTECTOR
/*
* Configure the stack canary. The linker can't handle this by
* relocation. Manually set base address in stack canary
* segment descriptor.
*/
movl $gdt_page,%eax
movl $stack_canary,%ecx
movw %cx, 8 * GDT_ENTRY_STACK_CANARY + 2(%eax)
shrl $16, %ecx
movb %cl, 8 * GDT_ENTRY_STACK_CANARY + 4(%eax)
movb %ch, 8 * GDT_ENTRY_STACK_CANARY + 7(%eax)
#endif
andl $0,setup_once_ref /* Once is enough, thanks */
ret
......
arch/x86/kernel/jump_label.c
View file @ c6536676
......@@ -28,10 +28,8 @@ static void bug_at(const void *ip, int line)
}
static const void *
__jump_label_set_jump_code(struct jump_entry *entry, enum jump_label_type type, int init)
__jump_label_set_jump_code(struct jump_entry *entry, enum jump_label_type type)
{
const unsigned char default_nop[] = { STATIC_KEY_INIT_NOP };
const unsigned char *ideal_nop = ideal_nops[NOP_ATOMIC5];
const void *expect, *code;
const void *addr, *dest;
int line;
......@@ -41,10 +39,8 @@ __jump_label_set_jump_code(struct jump_entry *entry, enum jump_label_type type,
code = text_gen_insn(JMP32_INSN_OPCODE, addr, dest);
if (init) {
expect = default_nop; line = __LINE__;
} else if (type == JUMP_LABEL_JMP) {
expect = ideal_nop; line = __LINE__;
if (type == JUMP_LABEL_JMP) {
expect = x86_nops[5]; line = __LINE__;
} else {
expect = code; line = __LINE__;
}
......@@ -53,7 +49,7 @@ __jump_label_set_jump_code(struct jump_entry *entry, enum jump_label_type type,
bug_at(addr, line);
if (type == JUMP_LABEL_NOP)
code = ideal_nop;
code = x86_nops[5];
return code;
}
......@@ -62,7 +58,7 @@ static inline void __jump_label_transform(struct jump_entry *entry,
enum jump_label_type type,
int init)
{
const void *opcode = __jump_label_set_jump_code(entry, type, init);
const void *opcode = __jump_label_set_jump_code(entry, type);
/*
* As long as only a single processor is running and the code is still
......@@ -113,7 +109,7 @@ bool arch_jump_label_transform_queue(struct jump_entry *entry,
}
mutex_lock(&text_mutex);
opcode = __jump_label_set_jump_code(entry, type, 0);
opcode = __jump_label_set_jump_code(entry, type);
text_poke_queue((void *)jump_entry_code(entry),
opcode, JUMP_LABEL_NOP_SIZE, NULL);
mutex_unlock(&text_mutex);
......@@ -136,22 +132,6 @@ static enum {
__init_or_module void arch_jump_label_transform_static(struct jump_entry *entry,
enum jump_label_type type)
{
/*
* This function is called at boot up and when modules are
* first loaded. Check if the default nop, the one that is
* inserted at compile time, is the ideal nop. If it is, then
* we do not need to update the nop, and we can leave it as is.
* If it is not, then we need to update the nop to the ideal nop.
*/
if (jlstate == JL_STATE_START) {
const unsigned char default_nop[] = { STATIC_KEY_INIT_NOP };
const unsigned char *ideal_nop = ideal_nops[NOP_ATOMIC5];
if (memcmp(ideal_nop, default_nop, 5) != 0)
jlstate = JL_STATE_UPDATE;
else
jlstate = JL_STATE_NO_UPDATE;
}
if (jlstate == JL_STATE_UPDATE)
jump_label_transform(entry, type, 1);
}
arch/x86/kernel/kprobes/core.c
View file @ c6536676
This diff is collapsed.
arch/x86/kernel/kprobes/opt.c
View file @ c6536676
......@@ -312,6 +312,8 @@ static int can_optimize(unsigned long paddr)
addr = paddr - offset;
while (addr < paddr - offset + size) { /* Decode until function end */
unsigned long recovered_insn;
int ret;
if (search_exception_tables(addr))
/*
* Since some fixup code will jumps into this function,
......@@ -321,8 +323,11 @@ static int can_optimize(unsigned long paddr)
recovered_insn = recover_probed_instruction(buf, addr);
if (!recovered_insn)
return 0;
kernel_insn_init(&insn, (void *)recovered_insn, MAX_INSN_SIZE);
insn_get_length(&insn);
ret = insn_decode_kernel(&insn, (void *)recovered_insn);
if (ret < 0)
return 0;
/*
* In the case of detecting unknown breakpoint, this could be
* a padding INT3 between functions. Let's check that all the
......
arch/x86/kernel/setup.c
View file @ c6536676
......@@ -822,7 +822,6 @@ void __init setup_arch(char **cmdline_p)
idt_setup_early_traps();
early_cpu_init();
arch_init_ideal_nops();
jump_label_init();
static_call_init();
early_ioremap_init();
......
arch/x86/kernel/setup_percpu.c
View file @ c6536676
......@@ -224,7 +224,6 @@ void __init setup_per_cpu_areas(void)
per_cpu(this_cpu_off, cpu) = per_cpu_offset(cpu);
per_cpu(cpu_number, cpu) = cpu;
setup_percpu_segment(cpu);
setup_stack_canary_segment(cpu);
/*
* Copy data used in early init routines from the
* initial arrays to the per cpu data areas. These
......
arch/x86/kernel/sev-es.c
View file @ c6536676
......@@ -263,39 +263,54 @@ static int vc_fetch_insn_kernel(struct es_em_ctxt *ctxt,
return copy_from_kernel_nofault(buffer, (unsigned char *)ctxt->regs->ip, MAX_INSN_SIZE);
}
static enum es_result vc_decode_insn(struct es_em_ctxt *ctxt)
static enum es_result __vc_decode_user_insn(struct es_em_ctxt *ctxt)
{
char buffer[MAX_INSN_SIZE];
enum es_result ret;
int res;
if (user_mode(ctxt->regs)) {
res = insn_fetch_from_user_inatomic(ctxt->regs, buffer);
if (!res) {
ctxt->fi.vector = X86_TRAP_PF;
ctxt->fi.error_code = X86_PF_INSTR | X86_PF_USER;
ctxt->fi.cr2 = ctxt->regs->ip;
return ES_EXCEPTION;
}
res = insn_fetch_from_user_inatomic(ctxt->regs, buffer);
if (!res) {
ctxt->fi.vector = X86_TRAP_PF;
ctxt->fi.error_code = X86_PF_INSTR | X86_PF_USER;
ctxt->fi.cr2 = ctxt->regs->ip;
return ES_EXCEPTION;
}
if (!insn_decode(&ctxt->insn, ctxt->regs, buffer, res))
return ES_DECODE_FAILED;
} else {
res = vc_fetch_insn_kernel(ctxt, buffer);
if (res) {
ctxt->fi.vector = X86_TRAP_PF;
ctxt->fi.error_code = X86_PF_INSTR;
ctxt->fi.cr2 = ctxt->regs->ip;
return ES_EXCEPTION;
}
if (!insn_decode_from_regs(&ctxt->insn, ctxt->regs, buffer, res))
return ES_DECODE_FAILED;
if (ctxt->insn.immediate.got)
return ES_OK;
else
return ES_DECODE_FAILED;
}
insn_init(&ctxt->insn, buffer, MAX_INSN_SIZE, 1);
insn_get_length(&ctxt->insn);
static enum es_result __vc_decode_kern_insn(struct es_em_ctxt *ctxt)
{
char buffer[MAX_INSN_SIZE];
int res, ret;
res = vc_fetch_insn_kernel(ctxt, buffer);
if (res) {
ctxt->fi.vector = X86_TRAP_PF;
ctxt->fi.error_code = X86_PF_INSTR;
ctxt->fi.cr2 = ctxt->regs->ip;
return ES_EXCEPTION;
}
ret = ctxt->insn.immediate.got ? ES_OK : ES_DECODE_FAILED;
ret = insn_decode(&ctxt->insn, buffer, MAX_INSN_SIZE, INSN_MODE_64);
if (ret < 0)
return ES_DECODE_FAILED;
else
return ES_OK;
}
return ret;
static enum es_result vc_decode_insn(struct es_em_ctxt *ctxt)
{
if (user_mode(ctxt->regs))
return __vc_decode_user_insn(ctxt);
else
return __vc_decode_kern_insn(ctxt);
}
static enum es_result vc_write_mem(struct es_em_ctxt *ctxt,
......
arch/x86/kernel/smpboot.c
View file @ c6536676
......@@ -458,29 +458,52 @@ static bool match_smt(struct cpuinfo_x86 *c, struct cpuinfo_x86 *o)
return false;
}
static bool match_die(struct cpuinfo_x86 *c, struct cpuinfo_x86 *o)
{
if (c->phys_proc_id == o->phys_proc_id &&
c->cpu_die_id == o->cpu_die_id)
return true;
return false;
}
/*
* Define snc_cpu[] for SNC (Sub-NUMA Cluster) CPUs.
* Unlike the other levels, we do not enforce keeping a
* multicore group inside a NUMA node. If this happens, we will
* discard the MC level of the topology later.
*/
static bool match_pkg(struct cpuinfo_x86 *c, struct cpuinfo_x86 *o)
{
if (c->phys_proc_id == o->phys_proc_id)
return true;
return false;
}
/*
* Define intel_cod_cpu[] for Intel COD (Cluster-on-Die) CPUs.
*
* These are Intel CPUs that enumerate an LLC that is shared by
* multiple NUMA nodes. The LLC on these systems is shared for
* off-package data access but private to the NUMA node (half
* of the package) for on-package access.
* Any Intel CPU that has multiple nodes per package and does not
* match intel_cod_cpu[] has the SNC (Sub-NUMA Cluster) topology.
*
* CPUID (the source of the information about the LLC) can only
* enumerate the cache as being shared *or* unshared, but not
* this particular configuration. The CPU in this case enumerates
* the cache to be shared across the entire package (spanning both
* NUMA nodes).
* When in SNC mode, these CPUs enumerate an LLC that is shared
* by multiple NUMA nodes. The LLC is shared for off-package data
* access but private to the NUMA node (half of the package) for
* on-package access. CPUID (the source of the information about
* the LLC) can only enumerate the cache as shared or unshared,
* but not this particular configuration.
*/
static const struct x86_cpu_id snc_cpu[] = {
X86_MATCH_INTEL_FAM6_MODEL(SKYLAKE_X, NULL),
static const struct x86_cpu_id intel_cod_cpu[] = {
X86_MATCH_INTEL_FAM6_MODEL(HASWELL_X, 0), /* COD */
X86_MATCH_INTEL_FAM6_MODEL(BROADWELL_X, 0), /* COD */
X86_MATCH_INTEL_FAM6_MODEL(ANY, 1), /* SNC */
{}
};
static bool match_llc(struct cpuinfo_x86 *c, struct cpuinfo_x86 *o)
{
const struct x86_cpu_id *id = x86_match_cpu(intel_cod_cpu);
int cpu1 = c->cpu_index, cpu2 = o->cpu_index;
bool intel_snc = id && id->driver_data;
/* Do not match if we do not have a valid APICID for cpu: */
if (per_cpu(cpu_llc_id, cpu1) == BAD_APICID)
......@@ -495,32 +518,12 @@ static bool match_llc(struct cpuinfo_x86 *c, struct cpuinfo_x86 *o)
* means 'c' does not share the LLC of 'o'. This will be
* reflected to userspace.
*/
if (!topology_same_node(c, o) && x86_match_cpu(snc_cpu))
if (match_pkg(c, o) && !topology_same_node(c, o) && intel_snc)
return false;
return topology_sane(c, o, "llc");
}
/*
* Unlike the other levels, we do not enforce keeping a
* multicore group inside a NUMA node. If this happens, we will
* discard the MC level of the topology later.
*/
static bool match_pkg(struct cpuinfo_x86 *c, struct cpuinfo_x86 *o)
{
if (c->phys_proc_id == o->phys_proc_id)
return true;
return false;
}
static bool match_die(struct cpuinfo_x86 *c, struct cpuinfo_x86 *o)
{
if ((c->phys_proc_id == o->phys_proc_id) &&
(c->cpu_die_id == o->cpu_die_id))
return true;
return false;
}
#if defined(CONFIG_SCHED_SMT) || defined(CONFIG_SCHED_MC)
static inline int x86_sched_itmt_flags(void)
......@@ -592,14 +595,23 @@ void set_cpu_sibling_map(int cpu)
for_each_cpu(i, cpu_sibling_setup_mask) {
o = &cpu_data(i);
if (match_pkg(c, o) && !topology_same_node(c, o))
x86_has_numa_in_package = true;
if ((i == cpu) || (has_smt && match_smt(c, o)))
link_mask(topology_sibling_cpumask, cpu, i);
if ((i == cpu) || (has_mp && match_llc(c, o)))
link_mask(cpu_llc_shared_mask, cpu, i);
if ((i == cpu) || (has_mp && match_die(c, o)))
link_mask(topology_die_cpumask, cpu, i);
}
threads = cpumask_weight(topology_sibling_cpumask(cpu));
if (threads > __max_smt_threads)
__max_smt_threads = threads;
/*
* This needs a separate iteration over the cpus because we rely on all
* topology_sibling_cpumask links to be set-up.
......@@ -613,8 +625,7 @@ void set_cpu_sibling_map(int cpu)
/*
* Does this new cpu bringup a new core?
*/
if (cpumask_weight(
topology_sibling_cpumask(cpu)) == 1) {
if (threads == 1) {
/*
* for each core in package, increment
* the booted_cores for this new cpu
......@@ -631,16 +642,7 @@ void set_cpu_sibling_map(int cpu)
} else if (i != cpu && !c->booted_cores)
c->booted_cores = cpu_data(i).booted_cores;
}
if (match_pkg(c, o) && !topology_same_node(c, o))
x86_has_numa_in_package = true;
if ((i == cpu) || (has_mp && match_die(c, o)))
link_mask(topology_die_cpumask, cpu, i);
}
threads = cpumask_weight(topology_sibling_cpumask(cpu));
if (threads > __max_smt_threads)
__max_smt_threads = threads;
}
/* maps the cpu to the sched domain representing multi-core */
......
arch/x86/kernel/static_call.c
View file @ c6536676
......@@ -34,7 +34,7 @@ static void __ref __static_call_transform(void *insn, enum insn_type type, void
break;
case NOP:
code = ideal_nops[NOP_ATOMIC5];
code = x86_nops[5];
break;
case JMP:
......@@ -66,7 +66,7 @@ static void __static_call_validate(void *insn, bool tail)
return;
} else {
if (opcode == CALL_INSN_OPCODE ||
!memcmp(insn, ideal_nops[NOP_ATOMIC5], 5) ||
!memcmp(insn, x86_nops[5], 5) ||
!memcmp(insn, xor5rax, 5))
return;
}
......
arch/x86/kernel/tls.c
View file @ c6536676
......@@ -164,17 +164,11 @@ int do_set_thread_area(struct task_struct *p, int idx,
savesegment(fs, sel);
if (sel == modified_sel)
loadsegment(fs, sel);
savesegment(gs, sel);
if (sel == modified_sel)
load_gs_index(sel);
#endif
#ifdef CONFIG_X86_32_LAZY_GS
savesegment(gs, sel);
if (sel == modified_sel)
loadsegment(gs, sel);
#endif
load_gs_index(sel);
} else {
#ifdef CONFIG_X86_64
if (p->thread.fsindex == modified_sel)
......
arch/x86/kernel/traps.c
View file @ c6536676
......@@ -498,14 +498,15 @@ static enum kernel_gp_hint get_kernel_gp_address(struct pt_regs *regs,
{
u8 insn_buf[MAX_INSN_SIZE];
struct insn insn;
int ret;
if (copy_from_kernel_nofault(insn_buf, (void *)regs->ip,
MAX_INSN_SIZE))
return GP_NO_HINT;
kernel_insn_init(&insn, insn_buf, MAX_INSN_SIZE);
insn_get_modrm(&insn);
insn_get_sib(&insn);
ret = insn_decode_kernel(&insn, insn_buf);
if (ret < 0)
return GP_NO_HINT;
*addr = (unsigned long)insn_get_addr_ref(&insn, regs);
if (*addr == -1UL)
......@@ -889,9 +890,6 @@ static __always_inline void exc_debug_kernel(struct pt_regs *regs,
if ((dr6 & DR_STEP) && is_sysenter_singlestep(regs))
dr6 &= ~DR_STEP;
if (kprobe_debug_handler(regs))
goto out;
/*
* The kernel doesn't use INT1
*/
......
arch/x86/kernel/umip.c
View file @ c6536676
......@@ -356,7 +356,7 @@ bool fixup_umip_exception(struct pt_regs *regs)
if (!nr_copied)
return false;
if (!insn_decode(&insn, regs, buf, nr_copied))
if (!insn_decode_from_regs(&insn, regs, buf, nr_copied))
return false;
umip_inst = identify_insn(&insn);
......
arch/x86/kernel/uprobes.c
View file @ c6536676
......@@ -276,12 +276,12 @@ static bool is_prefix_bad(struct insn *insn)
static int uprobe_init_insn(struct arch_uprobe *auprobe, struct insn *insn, bool x86_64)
{
enum insn_mode m = x86_64 ? INSN_MODE_64 : INSN_MODE_32;
u32 volatile *good_insns;
int ret;
insn_init(insn, auprobe->insn, sizeof(auprobe->insn), x86_64);
/* has the side-effect of processing the entire instruction */
insn_get_length(insn);
if (!insn_complete(insn))
ret = insn_decode(insn, auprobe->insn, sizeof(auprobe->insn), m);
if (ret < 0)
return -ENOEXEC;
if (is_prefix_bad(insn))
......
arch/x86/lib/inat.c
View file @ c6536676
......@@ -4,7 +4,7 @@
*
* Written by Masami Hiramatsu <mhiramat@redhat.com>
*/
#include <asm/insn.h>
#include <asm/insn.h> /* __ignore_sync_check__ */
/* Attribute tables are generated from opcode map */
#include "inat-tables.c"
......
arch/x86/lib/insn-eval.c
View file @ c6536676
......@@ -404,10 +404,6 @@ static short get_segment_selector(struct pt_regs *regs, int seg_reg_idx)
case INAT_SEG_REG_FS:
return (unsigned short)(regs->fs & 0xffff);
case INAT_SEG_REG_GS:
/*
* GS may or may not be in regs as per CONFIG_X86_32_LAZY_GS.
* The macro below takes care of both cases.
*/
return get_user_gs(regs);
case INAT_SEG_REG_IGNORE:
default:
......@@ -928,10 +924,11 @@ static int get_seg_base_limit(struct insn *insn, struct pt_regs *regs,
static int get_eff_addr_reg(struct insn *insn, struct pt_regs *regs,
int *regoff, long *eff_addr)
{
insn_get_modrm(insn);
int ret;
if (!insn->modrm.nbytes)
return -EINVAL;
ret = insn_get_modrm(insn);
if (ret)
return ret;
if (X86_MODRM_MOD(insn->modrm.value) != 3)
return -EINVAL;
......@@ -977,14 +974,14 @@ static int get_eff_addr_modrm(struct insn *insn, struct pt_regs *regs,
int *regoff, long *eff_addr)
{
long tmp;
int ret;
if (insn->addr_bytes != 8 && insn->addr_bytes != 4)
return -EINVAL;
insn_get_modrm(insn);
if (!insn->modrm.nbytes)
return -EINVAL;
ret = insn_get_modrm(insn);
if (ret)
return ret;
if (X86_MODRM_MOD(insn->modrm.value) > 2)
return -EINVAL;
......@@ -1106,18 +1103,21 @@ static int get_eff_addr_modrm_16(struct insn *insn, struct pt_regs *regs,
* @base_offset will have a register, as an offset from the base of pt_regs,
* that can be used to resolve the associated segment.
*
* -EINVAL on error.
* Negative value on error.
*/
static int get_eff_addr_sib(struct insn *insn, struct pt_regs *regs,
int *base_offset, long *eff_addr)
{
long base, indx;
int indx_offset;
int ret;
if (insn->addr_bytes != 8 && insn->addr_bytes != 4)
return -EINVAL;
insn_get_modrm(insn);
ret = insn_get_modrm(insn);
if (ret)
return ret;
if (!insn->modrm.nbytes)
return -EINVAL;
......@@ -1125,7 +1125,9 @@ static int get_eff_addr_sib(struct insn *insn, struct pt_regs *regs,
if (X86_MODRM_MOD(insn->modrm.value) > 2)
return -EINVAL;
insn_get_sib(insn);
ret = insn_get_sib(insn);
if (ret)
return ret;
if (!insn->sib.nbytes)
return -EINVAL;
......@@ -1194,8 +1196,8 @@ static void __user *get_addr_ref_16(struct insn *insn, struct pt_regs *regs)
short eff_addr;
long tmp;
insn_get_modrm(insn);
insn_get_displacement(insn);
if (insn_get_displacement(insn))
goto out;
if (insn->addr_bytes != 2)
goto out;
......@@ -1492,7 +1494,7 @@ int insn_fetch_from_user_inatomic(struct pt_regs *regs, unsigned char buf[MAX_IN
}
/**
* insn_decode() - Decode an instruction
* insn_decode_from_regs() - Decode an instruction
* @insn: Structure to store decoded instruction
* @regs: Structure with register values as seen when entering kernel mode
* @buf: Buffer containing the instruction bytes
......@@ -1505,8 +1507,8 @@ int insn_fetch_from_user_inatomic(struct pt_regs *regs, unsigned char buf[MAX_IN
*
* True if instruction was decoded, False otherwise.
*/
bool insn_decode(struct insn *insn, struct pt_regs *regs,
unsigned char buf[MAX_INSN_SIZE], int buf_size)
bool insn_decode_from_regs(struct insn *insn, struct pt_regs *regs,
unsigned char buf[MAX_INSN_SIZE], int buf_size)
{
int seg_defs;
......@@ -1529,7 +1531,9 @@ bool insn_decode(struct insn *insn, struct pt_regs *regs,
insn->addr_bytes = INSN_CODE_SEG_ADDR_SZ(seg_defs);
insn->opnd_bytes = INSN_CODE_SEG_OPND_SZ(seg_defs);
insn_get_length(insn);
if (insn_get_length(insn))
return false;
if (buf_size < insn->length)
return false;
......
arch/x86/lib/insn.c
View file @ c6536676
......@@ -11,10 +11,13 @@
#else
#include <string.h>
#endif
#include <asm/inat.h>
#include <asm/insn.h>
#include <asm/inat.h> /*__ignore_sync_check__ */
#include <asm/insn.h> /* __ignore_sync_check__ */
#include <asm/emulate_prefix.h>
#include <linux/errno.h>
#include <linux/kconfig.h>
#include <asm/emulate_prefix.h> /* __ignore_sync_check__ */
#define leXX_to_cpu(t, r) \
({ \
......@@ -51,6 +54,7 @@
* insn_init() - initialize struct insn
* @insn: &struct insn to be initialized
* @kaddr: address (in kernel memory) of instruction (or copy thereof)
* @buf_len: length of the insn buffer at @kaddr
* @x86_64: !0 for 64-bit kernel or 64-bit app
*/
void insn_init(struct insn *insn, const void *kaddr, int buf_len, int x86_64)
......@@ -111,8 +115,12 @@ static void insn_get_emulate_prefix(struct insn *insn)
* Populates the @insn->prefixes bitmap, and updates @insn->next_byte
* to point to the (first) opcode. No effect if @insn->prefixes.got
* is already set.
*
* * Returns:
* 0: on success
* < 0: on error
*/
void insn_get_prefixes(struct insn *insn)
int insn_get_prefixes(struct insn *insn)
{
struct insn_field *prefixes = &insn->prefixes;
insn_attr_t attr;
......@@ -120,7 +128,7 @@ void insn_get_prefixes(struct insn *insn)
int i, nb;
if (prefixes->got)
return;
return 0;
insn_get_emulate_prefix(insn);
......@@ -230,8 +238,10 @@ void insn_get_prefixes(struct insn *insn)
prefixes->got = 1;
return 0;
err_out:
return;
return -ENODATA;
}
/**
......@@ -243,16 +253,25 @@ void insn_get_prefixes(struct insn *insn)
* If necessary, first collects any preceding (prefix) bytes.
* Sets @insn->opcode.value = opcode1. No effect if @insn->opcode.got
* is already 1.
*
* Returns:
* 0: on success
* < 0: on error
*/
void insn_get_opcode(struct insn *insn)
int insn_get_opcode(struct insn *insn)
{
struct insn_field *opcode = &insn->opcode;
int pfx_id, ret;
insn_byte_t op;
int pfx_id;
if (opcode->got)
return;
if (!insn->prefixes.got)
insn_get_prefixes(insn);
return 0;
if (!insn->prefixes.got) {
ret = insn_get_prefixes(insn);
if (ret)
return ret;
}
/* Get first opcode */
op = get_next(insn_byte_t, insn);
......@@ -267,9 +286,13 @@ void insn_get_opcode(struct insn *insn)
insn->attr = inat_get_avx_attribute(op, m, p);
if ((inat_must_evex(insn->attr) && !insn_is_evex(insn)) ||
(!inat_accept_vex(insn->attr) &&
!inat_is_group(insn->attr)))
insn->attr = 0; /* This instruction is bad */
goto end; /* VEX has only 1 byte for opcode */
!inat_is_group(insn->attr))) {
/* This instruction is bad */
insn->attr = 0;
return -EINVAL;
}
/* VEX has only 1 byte for opcode */
goto end;
}
insn->attr = inat_get_opcode_attribute(op);
......@@ -280,13 +303,18 @@ void insn_get_opcode(struct insn *insn)
pfx_id = insn_last_prefix_id(insn);
insn->attr = inat_get_escape_attribute(op, pfx_id, insn->attr);
}
if (inat_must_vex(insn->attr))
insn->attr = 0; /* This instruction is bad */
if (inat_must_vex(insn->attr)) {
/* This instruction is bad */
insn->attr = 0;
return -EINVAL;
}
end:
opcode->got = 1;
return 0;
err_out:
return;
return -ENODATA;
}
/**
......@@ -296,15 +324,25 @@ void insn_get_opcode(struct insn *insn)
* Populates @insn->modrm and updates @insn->next_byte to point past the
* ModRM byte, if any. If necessary, first collects the preceding bytes
* (prefixes and opcode(s)). No effect if @insn->modrm.got is already 1.
*
* Returns:
* 0: on success
* < 0: on error
*/
void insn_get_modrm(struct insn *insn)
int insn_get_modrm(struct insn *insn)
{
struct insn_field *modrm = &insn->modrm;
insn_byte_t pfx_id, mod;
int ret;
if (modrm->got)
return;
if (!insn->opcode.got)
insn_get_opcode(insn);
return 0;
if (!insn->opcode.got) {
ret = insn_get_opcode(insn);
if (ret)
return ret;
}
if (inat_has_modrm(insn->attr)) {
mod = get_next(insn_byte_t, insn);
......@@ -313,17 +351,22 @@ void insn_get_modrm(struct insn *insn)
pfx_id = insn_last_prefix_id(insn);
insn->attr = inat_get_group_attribute(mod, pfx_id,
insn->attr);
if (insn_is_avx(insn) && !inat_accept_vex(insn->attr))
insn->attr = 0; /* This is bad */
if (insn_is_avx(insn) && !inat_accept_vex(insn->attr)) {
/* Bad insn */
insn->attr = 0;
return -EINVAL;
}
}
}
if (insn->x86_64 && inat_is_force64(insn->attr))
insn->opnd_bytes = 8;
modrm->got = 1;
return 0;
err_out:
return;
return -ENODATA;
}
......@@ -337,11 +380,16 @@ void insn_get_modrm(struct insn *insn)
int insn_rip_relative(struct insn *insn)
{
struct insn_field *modrm = &insn->modrm;
int ret;
if (!insn->x86_64)
return 0;
if (!modrm->got)
insn_get_modrm(insn);
if (!modrm->got) {
ret = insn_get_modrm(insn);
if (ret)
return 0;
}
/*
* For rip-relative instructions, the mod field (top 2 bits)
* is zero and the r/m field (bottom 3 bits) is 0x5.
......@@ -355,15 +403,25 @@ int insn_rip_relative(struct insn *insn)
*
* If necessary, first collects the instruction up to and including the
* ModRM byte.
*
* Returns:
* 0: if decoding succeeded
* < 0: otherwise.
*/
void insn_get_sib(struct insn *insn)
int insn_get_sib(struct insn *insn)
{
insn_byte_t modrm;
int ret;
if (insn->sib.got)
return;
if (!insn->modrm.got)
insn_get_modrm(insn);
return 0;
if (!insn->modrm.got) {
ret = insn_get_modrm(insn);
if (ret)
return ret;
}
if (insn->modrm.nbytes) {
modrm = insn->modrm.bytes[0];
if (insn->addr_bytes != 2 &&
......@@ -374,8 +432,10 @@ void insn_get_sib(struct insn *insn)
}
insn->sib.got = 1;
return 0;
err_out:
return;
return -ENODATA;
}
......@@ -386,15 +446,25 @@ void insn_get_sib(struct insn *insn)
* If necessary, first collects the instruction up to and including the
* SIB byte.
* Displacement value is sign-expanded.
*
* * Returns:
* 0: if decoding succeeded
* < 0: otherwise.
*/
void insn_get_displacement(struct insn *insn)
int insn_get_displacement(struct insn *insn)
{
insn_byte_t mod, rm, base;
int ret;
if (insn->displacement.got)
return;
if (!insn->sib.got)
insn_get_sib(insn);
return 0;
if (!insn->sib.got) {
ret = insn_get_sib(insn);
if (ret)
return ret;
}
if (insn->modrm.nbytes) {
/*
* Interpreting the modrm byte:
......@@ -436,9 +506,10 @@ void insn_get_displacement(struct insn *insn)
}
out:
insn->displacement.got = 1;
return 0;
err_out:
return;
return -ENODATA;
}
/* Decode moffset16/32/64. Return 0 if failed */
......@@ -537,20 +608,30 @@ static int __get_immptr(struct insn *insn)
}
/**
* insn_get_immediate() - Get the immediates of instruction
* insn_get_immediate() - Get the immediate in an instruction
* @insn: &struct insn containing instruction
*
* If necessary, first collects the instruction up to and including the
* displacement bytes.
* Basically, most of immediates are sign-expanded. Unsigned-value can be
* get by bit masking with ((1 << (nbytes * 8)) - 1)
* computed by bit masking with ((1 << (nbytes * 8)) - 1)
*
* Returns:
* 0: on success
* < 0: on error
*/
void insn_get_immediate(struct insn *insn)
int insn_get_immediate(struct insn *insn)
{
int ret;
if (insn->immediate.got)
return;
if (!insn->displacement.got)
insn_get_displacement(insn);
return 0;
if (!insn->displacement.got) {
ret = insn_get_displacement(insn);
if (ret)
return ret;
}
if (inat_has_moffset(insn->attr)) {
if (!__get_moffset(insn))
......@@ -597,9 +678,10 @@ void insn_get_immediate(struct insn *insn)
}
done:
insn->immediate.got = 1;
return 0;
err_out:
return;
return -ENODATA;
}
/**
......@@ -608,13 +690,65 @@ void insn_get_immediate(struct insn *insn)
*
* If necessary, first collects the instruction up to and including the
* immediates bytes.
*/
void insn_get_length(struct insn *insn)
*
* Returns:
* - 0 on success
* - < 0 on error
*/
int insn_get_length(struct insn *insn)
{
int ret;
if (insn->length)
return;
if (!insn->immediate.got)
insn_get_immediate(insn);
return 0;
if (!insn->immediate.got) {
ret = insn_get_immediate(insn);
if (ret)
return ret;
}
insn->length = (unsigned char)((unsigned long)insn->next_byte
- (unsigned long)insn->kaddr);
return 0;
}
/* Ensure this instruction is decoded completely */
static inline int insn_complete(struct insn *insn)
{
return insn->opcode.got && insn->modrm.got && insn->sib.got &&
insn->displacement.got && insn->immediate.got;
}
/**
* insn_decode() - Decode an x86 instruction
* @insn: &struct insn to be initialized
* @kaddr: address (in kernel memory) of instruction (or copy thereof)
* @buf_len: length of the insn buffer at @kaddr
* @m: insn mode, see enum insn_mode
*
* Returns:
* 0: if decoding succeeded
* < 0: otherwise.
*/
int insn_decode(struct insn *insn, const void *kaddr, int buf_len, enum insn_mode m)
{
int ret;
/* #define INSN_MODE_KERN -1 __ignore_sync_check__ mode is only valid in the kernel */
if (m == INSN_MODE_KERN)
insn_init(insn, kaddr, buf_len, IS_ENABLED(CONFIG_X86_64));
else
insn_init(insn, kaddr, buf_len, m == INSN_MODE_64);
ret = insn_get_length(insn);
if (ret)
return ret;
if (insn_complete(insn))
return 0;
return -EINVAL;
}
arch/x86/lib/retpoline.S
View file @ c6536676
......@@ -10,27 +10,59 @@
#include <asm/unwind_hints.h>
#include <asm/frame.h>
.macro THUNK reg
.section .text.__x86.indirect_thunk
.align 32
SYM_FUNC_START(__x86_indirect_thunk_\reg)
JMP_NOSPEC \reg
SYM_FUNC_END(__x86_indirect_thunk_\reg)
SYM_FUNC_START_NOALIGN(__x86_retpoline_\reg)
.macro RETPOLINE reg
ANNOTATE_INTRA_FUNCTION_CALL
call .Ldo_rop_\@
call .Ldo_rop_\@
.Lspec_trap_\@:
UNWIND_HINT_EMPTY
pause
lfence
jmp .Lspec_trap_\@
jmp .Lspec_trap_\@
.Ldo_rop_\@:
mov %\reg, (%_ASM_SP)
mov %\reg, (%_ASM_SP)
UNWIND_HINT_FUNC
ret
SYM_FUNC_END(__x86_retpoline_\reg)
.endm
.macro THUNK reg
.align 32
SYM_FUNC_START(__x86_indirect_thunk_\reg)
ALTERNATIVE_2 __stringify(ANNOTATE_RETPOLINE_SAFE; jmp *%\reg), \
__stringify(RETPOLINE \reg), X86_FEATURE_RETPOLINE, \
__stringify(lfence; ANNOTATE_RETPOLINE_SAFE; jmp *%\reg), X86_FEATURE_RETPOLINE_AMD
SYM_FUNC_END(__x86_indirect_thunk_\reg)
.endm
/*
* This generates .altinstr_replacement symbols for use by objtool. They,
* however, must not actually live in .altinstr_replacement since that will be
* discarded after init, but module alternatives will also reference these
* symbols.
*
* Their names matches the "__x86_indirect_" prefix to mark them as retpolines.
*/
.macro ALT_THUNK reg
.align 1
SYM_FUNC_START_NOALIGN(__x86_indirect_alt_call_\reg)
ANNOTATE_RETPOLINE_SAFE
1: call *%\reg
2: .skip 5-(2b-1b), 0x90
SYM_FUNC_END(__x86_indirect_alt_call_\reg)
SYM_FUNC_START_NOALIGN(__x86_indirect_alt_jmp_\reg)
ANNOTATE_RETPOLINE_SAFE
1: jmp *%\reg
2: .skip 5-(2b-1b), 0x90
SYM_FUNC_END(__x86_indirect_alt_jmp_\reg)
.endm
......@@ -48,7 +80,6 @@ SYM_FUNC_END(__x86_retpoline_\reg)
#define __EXPORT_THUNK(sym) _ASM_NOKPROBE(sym); EXPORT_SYMBOL(sym)
#define EXPORT_THUNK(reg) __EXPORT_THUNK(__x86_indirect_thunk_ ## reg)
#define EXPORT_RETPOLINE(reg) __EXPORT_THUNK(__x86_retpoline_ ## reg)
#undef GEN
#define GEN(reg) THUNK reg
......@@ -59,5 +90,13 @@ SYM_FUNC_END(__x86_retpoline_\reg)
#include <asm/GEN-for-each-reg.h>
#undef GEN
#define GEN(reg) EXPORT_RETPOLINE(reg)
#define GEN(reg) ALT_THUNK reg
#include <asm/GEN-for-each-reg.h>
#undef GEN
#define GEN(reg) __EXPORT_THUNK(__x86_indirect_alt_call_ ## reg)
#include <asm/GEN-for-each-reg.h>
#undef GEN
#define GEN(reg) __EXPORT_THUNK(__x86_indirect_alt_jmp_ ## reg)
#include <asm/GEN-for-each-reg.h>
arch/x86/net/bpf_jit_comp.c
View file @ c6536676
......@@ -282,7 +282,7 @@ static void emit_prologue(u8 **pprog, u32 stack_depth, bool ebpf_from_cbpf,
/* BPF trampoline can be made to work without these nops,
* but let's waste 5 bytes for now and optimize later
*/
memcpy(prog, ideal_nops[NOP_ATOMIC5], cnt);
memcpy(prog, x86_nops[5], cnt);
prog += cnt;
if (!ebpf_from_cbpf) {
if (tail_call_reachable && !is_subprog)
......@@ -330,7 +330,7 @@ static int __bpf_arch_text_poke(void *ip, enum bpf_text_poke_type t,
void *old_addr, void *new_addr,
const bool text_live)
{
const u8 *nop_insn = ideal_nops[NOP_ATOMIC5];
const u8 *nop_insn = x86_nops[5];
u8 old_insn[X86_PATCH_SIZE];
u8 new_insn[X86_PATCH_SIZE];
u8 *prog;
......@@ -560,7 +560,7 @@ static void emit_bpf_tail_call_direct(struct bpf_jit_poke_descriptor *poke,
if (stack_depth)
EMIT3_off32(0x48, 0x81, 0xC4, round_up(stack_depth, 8));
memcpy(prog, ideal_nops[NOP_ATOMIC5], X86_PATCH_SIZE);
memcpy(prog, x86_nops[5], X86_PATCH_SIZE);
prog += X86_PATCH_SIZE;
/* out: */
......@@ -881,7 +881,7 @@ static int emit_nops(u8 **pprog, int len)
noplen = ASM_NOP_MAX;
for (i = 0; i < noplen; i++)
EMIT1(ideal_nops[noplen][i]);
EMIT1(x86_nops[noplen][i]);
len -= noplen;
}
......@@ -2021,7 +2021,7 @@ int arch_prepare_bpf_trampoline(struct bpf_tramp_image *im, void *image, void *i
/* remember return value in a stack for bpf prog to access */
emit_stx(&prog, BPF_DW, BPF_REG_FP, BPF_REG_0, -8);
im->ip_after_call = prog;
memcpy(prog, ideal_nops[NOP_ATOMIC5], X86_PATCH_SIZE);
memcpy(prog, x86_nops[5], X86_PATCH_SIZE);
prog += X86_PATCH_SIZE;
}
......
arch/x86/platform/pvh/head.S
View file @ c6536676
......@@ -46,10 +46,8 @@
#define PVH_GDT_ENTRY_CS 1
#define PVH_GDT_ENTRY_DS 2
#define PVH_GDT_ENTRY_CANARY 3
#define PVH_CS_SEL (PVH_GDT_ENTRY_CS * 8)
#define PVH_DS_SEL (PVH_GDT_ENTRY_DS * 8)
#define PVH_CANARY_SEL (PVH_GDT_ENTRY_CANARY * 8)
SYM_CODE_START_LOCAL(pvh_start_xen)
cld
......@@ -111,17 +109,6 @@ SYM_CODE_START_LOCAL(pvh_start_xen)
#else /* CONFIG_X86_64 */
/* Set base address in stack canary descriptor. */
movl $_pa(gdt_start),%eax
movl $_pa(canary),%ecx
movw %cx, (PVH_GDT_ENTRY_CANARY * 8) + 2(%eax)
shrl $16, %ecx
movb %cl, (PVH_GDT_ENTRY_CANARY * 8) + 4(%eax)
movb %ch, (PVH_GDT_ENTRY_CANARY * 8) + 7(%eax)
mov $PVH_CANARY_SEL,%eax
mov %eax,%gs
call mk_early_pgtbl_32
mov $_pa(initial_page_table), %eax
......@@ -165,7 +152,6 @@ SYM_DATA_START_LOCAL(gdt_start)
.quad GDT_ENTRY(0xc09a, 0, 0xfffff) /* PVH_CS_SEL */
#endif
.quad GDT_ENTRY(0xc092, 0, 0xfffff) /* PVH_DS_SEL */
.quad GDT_ENTRY(0x4090, 0, 0x18) /* PVH_CANARY_SEL */
SYM_DATA_END_LABEL(gdt_start, SYM_L_LOCAL, gdt_end)
.balign 16
......
arch/x86/power/cpu.c
View file @ c6536676
......@@ -99,11 +99,8 @@ static void __save_processor_state(struct saved_context *ctxt)
/*
* segment registers
*/
#ifdef CONFIG_X86_32_LAZY_GS
savesegment(gs, ctxt->gs);
#endif
#ifdef CONFIG_X86_64
savesegment(gs, ctxt->gs);
savesegment(fs, ctxt->fs);
savesegment(ds, ctxt->ds);
savesegment(es, ctxt->es);
......@@ -232,7 +229,6 @@ static void notrace __restore_processor_state(struct saved_context *ctxt)
wrmsrl(MSR_GS_BASE, ctxt->kernelmode_gs_base);
#else
loadsegment(fs, __KERNEL_PERCPU);
loadsegment(gs, __KERNEL_STACK_CANARY);
#endif
/* Restore the TSS, RO GDT, LDT, and usermode-relevant MSRs. */
......@@ -255,7 +251,7 @@ static void notrace __restore_processor_state(struct saved_context *ctxt)
*/
wrmsrl(MSR_FS_BASE, ctxt->fs_base);
wrmsrl(MSR_KERNEL_GS_BASE, ctxt->usermode_gs_base);
#elif defined(CONFIG_X86_32_LAZY_GS)
#else
loadsegment(gs, ctxt->gs);
#endif
......
arch/x86/tools/insn_decoder_test.c
View file @ c6536676
......@@ -120,7 +120,7 @@ int main(int argc, char **argv)
while (fgets(line, BUFSIZE, stdin)) {
char copy[BUFSIZE], *s, *tab1, *tab2;
int nb = 0;
int nb = 0, ret;
unsigned int b;
if (line[0] == '<') {
......@@ -148,10 +148,12 @@ int main(int argc, char **argv)
} else
break;
}
/* Decode an instruction */
insn_init(&insn, insn_buff, sizeof(insn_buff), x86_64);
insn_get_length(&insn);
if (insn.length != nb) {
ret = insn_decode(&insn, insn_buff, sizeof(insn_buff),
x86_64 ? INSN_MODE_64 : INSN_MODE_32);
if (ret < 0 || insn.length != nb) {
warnings++;
pr_warn("Found an x86 instruction decoder bug, "
"please report this.\n", sym);
......
arch/x86/tools/insn_sanity.c
View file @ c6536676
......@@ -218,8 +218,8 @@ static void parse_args(int argc, char **argv)
int main(int argc, char **argv)
{
int insns = 0, ret;
struct insn insn;
int insns = 0;
int errors = 0;
unsigned long i;
unsigned char insn_buff[MAX_INSN_SIZE * 2];
......@@ -237,15 +237,15 @@ int main(int argc, char **argv)
continue;
/* Decode an instruction */
insn_init(&insn, insn_buff, sizeof(insn_buff), x86_64);
insn_get_length(&insn);
ret = insn_decode(&insn, insn_buff, sizeof(insn_buff),
x86_64 ? INSN_MODE_64 : INSN_MODE_32);
if (insn.next_byte <= insn.kaddr ||
insn.kaddr + MAX_INSN_SIZE < insn.next_byte) {
/* Access out-of-range memory */
dump_stream(stderr, "Error: Found an access violation", i, insn_buff, &insn);
errors++;
} else if (verbose && !insn_complete(&insn))
} else if (verbose && ret < 0)
dump_stream(stdout, "Info: Found an undecodable input", i, insn_buff, &insn);
else if (verbose >= 2)
dump_insn(stdout, &insn);
......
arch/x86/xen/enlighten_pv.c
View file @ c6536676
......@@ -1202,7 +1202,6 @@ static void __init xen_setup_gdt(int cpu)
pv_ops.cpu.write_gdt_entry = xen_write_gdt_entry_boot;
pv_ops.cpu.load_gdt = xen_load_gdt_boot;
setup_stack_canary_segment(cpu);
switch_to_new_gdt(cpu);
pv_ops.cpu.write_gdt_entry = xen_write_gdt_entry;
......
scripts/gcc-x86_32-has-stack-protector.sh
View file @ c6536676
#!/bin/sh
# SPDX-License-Identifier: GPL-2.0
echo "int foo(void) { char X[200]; return 3; }" | $* -S -x c -c -m32 -O0 -fstack-protector - -o - 2> /dev/null | grep -q "%gs"
# This requires GCC 8.1 or better. Specifically, we require
# -mstack-protector-guard-reg, added by
# https://gcc.gnu.org/bugzilla/show_bug.cgi?id=81708
echo "int foo(void) { char X[200]; return 3; }" | $* -S -x c -c -m32 -O0 -fstack-protector -mstack-protector-guard-reg=fs -mstack-protector-guard-symbol=__stack_chk_guard - -o - 2> /dev/null | grep -q "%fs"
tools/arch/x86/include/asm/inat.h
View file @ c6536676
......@@ -6,7 +6,7 @@
*
* Written by Masami Hiramatsu <mhiramat@redhat.com>
*/
#include "inat_types.h"
#include "inat_types.h" /* __ignore_sync_check__ */
/*
* Internal bits. Don't use bitmasks directly, because these bits are
......
tools/arch/x86/include/asm/insn.h
View file @ c6536676
......@@ -9,7 +9,7 @@
#include <asm/byteorder.h>
/* insn_attr_t is defined in inat.h */
#include "inat.h"
#include "inat.h" /* __ignore_sync_check__ */
#if defined(__BYTE_ORDER) ? __BYTE_ORDER == __LITTLE_ENDIAN : defined(__LITTLE_ENDIAN)
......@@ -132,13 +132,25 @@ struct insn {
#define X86_VEX_M_MAX 0x1f /* VEX3.M Maximum value */
extern void insn_init(struct insn *insn, const void *kaddr, int buf_len, int x86_64);
extern void insn_get_prefixes(struct insn *insn);
extern void insn_get_opcode(struct insn *insn);
extern void insn_get_modrm(struct insn *insn);
extern void insn_get_sib(struct insn *insn);
extern void insn_get_displacement(struct insn *insn);
extern void insn_get_immediate(struct insn *insn);
extern void insn_get_length(struct insn *insn);
extern int insn_get_prefixes(struct insn *insn);
extern int insn_get_opcode(struct insn *insn);
extern int insn_get_modrm(struct insn *insn);
extern int insn_get_sib(struct insn *insn);
extern int insn_get_displacement(struct insn *insn);
extern int insn_get_immediate(struct insn *insn);
extern int insn_get_length(struct insn *insn);
enum insn_mode {
INSN_MODE_32,
INSN_MODE_64,
/* Mode is determined by the current kernel build. */
INSN_MODE_KERN,
INSN_NUM_MODES,
};
extern int insn_decode(struct insn *insn, const void *kaddr, int buf_len, enum insn_mode m);
#define insn_decode_kernel(_insn, _ptr) insn_decode((_insn), (_ptr), MAX_INSN_SIZE, INSN_MODE_KERN)
/* Attribute will be determined after getting ModRM (for opcode groups) */
static inline void insn_get_attribute(struct insn *insn)
......@@ -149,17 +161,6 @@ static inline void insn_get_attribute(struct insn *insn)
/* Instruction uses RIP-relative addressing */
extern int insn_rip_relative(struct insn *insn);
/* Init insn for kernel text */
static inline void kernel_insn_init(struct insn *insn,
const void *kaddr, int buf_len)
{
#ifdef CONFIG_X86_64
insn_init(insn, kaddr, buf_len, 1);
#else /* CONFIG_X86_32 */
insn_init(insn, kaddr, buf_len, 0);
#endif
}
static inline int insn_is_avx(struct insn *insn)
{
if (!insn->prefixes.got)
......@@ -179,13 +180,6 @@ static inline int insn_has_emulate_prefix(struct insn *insn)
return !!insn->emulate_prefix_size;
}
/* Ensure this instruction is decoded completely */
static inline int insn_complete(struct insn *insn)
{
return insn->opcode.got && insn->modrm.got && insn->sib.got &&
insn->displacement.got && insn->immediate.got;
}
static inline insn_byte_t insn_vex_m_bits(struct insn *insn)
{
if (insn->vex_prefix.nbytes == 2) /* 2 bytes VEX */
......
tools/arch/x86/include/asm/nops.h 0 → 100644
View file @ c6536676
/* SPDX-License-Identifier: GPL-2.0 */
#ifndef _ASM_X86_NOPS_H
#define _ASM_X86_NOPS_H
/*
* Define nops for use with alternative() and for tracing.
*/
#ifndef CONFIG_64BIT
/*
* Generic 32bit nops from GAS:
*
* 1: nop
* 2: movl %esi,%esi
* 3: leal 0x0(%esi),%esi
* 4: leal 0x0(%esi,%eiz,1),%esi
* 5: leal %ds:0x0(%esi,%eiz,1),%esi
* 6: leal 0x0(%esi),%esi
* 7: leal 0x0(%esi,%eiz,1),%esi
* 8: leal %ds:0x0(%esi,%eiz,1),%esi
*
* Except 5 and 8, which are DS prefixed 4 and 7 resp, where GAS would emit 2
* nop instructions.
*/
#define BYTES_NOP1 0x90
#define BYTES_NOP2 0x89,0xf6
#define BYTES_NOP3 0x8d,0x76,0x00
#define BYTES_NOP4 0x8d,0x74,0x26,0x00
#define BYTES_NOP5 0x3e,BYTES_NOP4
#define BYTES_NOP6 0x8d,0xb6,0x00,0x00,0x00,0x00
#define BYTES_NOP7 0x8d,0xb4,0x26,0x00,0x00,0x00,0x00
#define BYTES_NOP8 0x3e,BYTES_NOP7
#else
/*
* Generic 64bit nops from GAS:
*
* 1: nop
* 2: osp nop
* 3: nopl (%eax)
* 4: nopl 0x00(%eax)
* 5: nopl 0x00(%eax,%eax,1)
* 6: osp nopl 0x00(%eax,%eax,1)
* 7: nopl 0x00000000(%eax)
* 8: nopl 0x00000000(%eax,%eax,1)
*/
#define BYTES_NOP1 0x90
#define BYTES_NOP2 0x66,BYTES_NOP1
#define BYTES_NOP3 0x0f,0x1f,0x00
#define BYTES_NOP4 0x0f,0x1f,0x40,0x00
#define BYTES_NOP5 0x0f,0x1f,0x44,0x00,0x00
#define BYTES_NOP6 0x66,BYTES_NOP5
#define BYTES_NOP7 0x0f,0x1f,0x80,0x00,0x00,0x00,0x00
#define BYTES_NOP8 0x0f,0x1f,0x84,0x00,0x00,0x00,0x00,0x00
#endif /* CONFIG_64BIT */
#ifdef __ASSEMBLY__
#define _ASM_MK_NOP(x) .byte x
#else
#define _ASM_MK_NOP(x) ".byte " __stringify(x) "\n"
#endif
#define ASM_NOP1 _ASM_MK_NOP(BYTES_NOP1)
#define ASM_NOP2 _ASM_MK_NOP(BYTES_NOP2)
#define ASM_NOP3 _ASM_MK_NOP(BYTES_NOP3)
#define ASM_NOP4 _ASM_MK_NOP(BYTES_NOP4)
#define ASM_NOP5 _ASM_MK_NOP(BYTES_NOP5)
#define ASM_NOP6 _ASM_MK_NOP(BYTES_NOP6)
#define ASM_NOP7 _ASM_MK_NOP(BYTES_NOP7)
#define ASM_NOP8 _ASM_MK_NOP(BYTES_NOP8)
#define ASM_NOP_MAX 8
#ifndef __ASSEMBLY__
extern const unsigned char * const x86_nops[];
#endif
#endif /* _ASM_X86_NOPS_H */
tools/arch/x86/lib/inat.c
View file @ c6536676
......@@ -4,7 +4,7 @@
*
* Written by Masami Hiramatsu <mhiramat@redhat.com>
*/
#include "../include/asm/insn.h"
#include "../include/asm/insn.h" /* __ignore_sync_check__ */
/* Attribute tables are generated from opcode map */
#include "inat-tables.c"
......
tools/arch/x86/lib/insn.c
View file @ c6536676
......@@ -11,10 +11,13 @@
#else
#include <string.h>
#endif
#include "../include/asm/inat.h"
#include "../include/asm/insn.h"
#include "../include/asm/inat.h" /* __ignore_sync_check__ */
#include "../include/asm/insn.h" /* __ignore_sync_check__ */
#include "../include/asm/emulate_prefix.h"
#include <linux/errno.h>
#include <linux/kconfig.h>
#include "../include/asm/emulate_prefix.h" /* __ignore_sync_check__ */
#define leXX_to_cpu(t, r) \
({ \
......@@ -51,6 +54,7 @@
* insn_init() - initialize struct insn
* @insn: &struct insn to be initialized
* @kaddr: address (in kernel memory) of instruction (or copy thereof)
* @buf_len: length of the insn buffer at @kaddr
* @x86_64: !0 for 64-bit kernel or 64-bit app
*/
void insn_init(struct insn *insn, const void *kaddr, int buf_len, int x86_64)
......@@ -111,8 +115,12 @@ static void insn_get_emulate_prefix(struct insn *insn)
* Populates the @insn->prefixes bitmap, and updates @insn->next_byte
* to point to the (first) opcode. No effect if @insn->prefixes.got
* is already set.
*
* * Returns:
* 0: on success
* < 0: on error
*/
void insn_get_prefixes(struct insn *insn)
int insn_get_prefixes(struct insn *insn)
{
struct insn_field *prefixes = &insn->prefixes;
insn_attr_t attr;
......@@ -120,7 +128,7 @@ void insn_get_prefixes(struct insn *insn)
int i, nb;
if (prefixes->got)
return;
return 0;
insn_get_emulate_prefix(insn);
......@@ -230,8 +238,10 @@ void insn_get_prefixes(struct insn *insn)
prefixes->got = 1;
return 0;
err_out:
return;
return -ENODATA;
}
/**
......@@ -243,16 +253,25 @@ void insn_get_prefixes(struct insn *insn)
* If necessary, first collects any preceding (prefix) bytes.
* Sets @insn->opcode.value = opcode1. No effect if @insn->opcode.got
* is already 1.
*
* Returns:
* 0: on success
* < 0: on error
*/
void insn_get_opcode(struct insn *insn)
int insn_get_opcode(struct insn *insn)
{
struct insn_field *opcode = &insn->opcode;
int pfx_id, ret;
insn_byte_t op;
int pfx_id;
if (opcode->got)
return;
if (!insn->prefixes.got)
insn_get_prefixes(insn);
return 0;
if (!insn->prefixes.got) {
ret = insn_get_prefixes(insn);
if (ret)
return ret;
}
/* Get first opcode */
op = get_next(insn_byte_t, insn);
......@@ -267,9 +286,13 @@ void insn_get_opcode(struct insn *insn)
insn->attr = inat_get_avx_attribute(op, m, p);
if ((inat_must_evex(insn->attr) && !insn_is_evex(insn)) ||
(!inat_accept_vex(insn->attr) &&
!inat_is_group(insn->attr)))
insn->attr = 0; /* This instruction is bad */
goto end; /* VEX has only 1 byte for opcode */
!inat_is_group(insn->attr))) {
/* This instruction is bad */
insn->attr = 0;
return -EINVAL;
}
/* VEX has only 1 byte for opcode */
goto end;
}
insn->attr = inat_get_opcode_attribute(op);
......@@ -280,13 +303,18 @@ void insn_get_opcode(struct insn *insn)
pfx_id = insn_last_prefix_id(insn);
insn->attr = inat_get_escape_attribute(op, pfx_id, insn->attr);
}
if (inat_must_vex(insn->attr))
insn->attr = 0; /* This instruction is bad */
if (inat_must_vex(insn->attr)) {
/* This instruction is bad */
insn->attr = 0;
return -EINVAL;
}
end:
opcode->got = 1;
return 0;
err_out:
return;
return -ENODATA;
}
/**
......@@ -296,15 +324,25 @@ void insn_get_opcode(struct insn *insn)
* Populates @insn->modrm and updates @insn->next_byte to point past the
* ModRM byte, if any. If necessary, first collects the preceding bytes
* (prefixes and opcode(s)). No effect if @insn->modrm.got is already 1.
*
* Returns:
* 0: on success
* < 0: on error
*/
void insn_get_modrm(struct insn *insn)
int insn_get_modrm(struct insn *insn)
{
struct insn_field *modrm = &insn->modrm;
insn_byte_t pfx_id, mod;
int ret;
if (modrm->got)
return;
if (!insn->opcode.got)
insn_get_opcode(insn);
return 0;
if (!insn->opcode.got) {
ret = insn_get_opcode(insn);
if (ret)
return ret;
}
if (inat_has_modrm(insn->attr)) {
mod = get_next(insn_byte_t, insn);
......@@ -313,17 +351,22 @@ void insn_get_modrm(struct insn *insn)
pfx_id = insn_last_prefix_id(insn);
insn->attr = inat_get_group_attribute(mod, pfx_id,
insn->attr);
if (insn_is_avx(insn) && !inat_accept_vex(insn->attr))
insn->attr = 0; /* This is bad */
if (insn_is_avx(insn) && !inat_accept_vex(insn->attr)) {
/* Bad insn */
insn->attr = 0;
return -EINVAL;
}
}
}
if (insn->x86_64 && inat_is_force64(insn->attr))
insn->opnd_bytes = 8;
modrm->got = 1;
return 0;
err_out:
return;
return -ENODATA;
}
......@@ -337,11 +380,16 @@ void insn_get_modrm(struct insn *insn)
int insn_rip_relative(struct insn *insn)
{
struct insn_field *modrm = &insn->modrm;
int ret;
if (!insn->x86_64)
return 0;
if (!modrm->got)
insn_get_modrm(insn);
if (!modrm->got) {
ret = insn_get_modrm(insn);
if (ret)
return 0;
}
/*
* For rip-relative instructions, the mod field (top 2 bits)
* is zero and the r/m field (bottom 3 bits) is 0x5.
......@@ -355,15 +403,25 @@ int insn_rip_relative(struct insn *insn)
*
* If necessary, first collects the instruction up to and including the
* ModRM byte.
*
* Returns:
* 0: if decoding succeeded
* < 0: otherwise.
*/
void insn_get_sib(struct insn *insn)
int insn_get_sib(struct insn *insn)
{
insn_byte_t modrm;
int ret;
if (insn->sib.got)
return;
if (!insn->modrm.got)
insn_get_modrm(insn);
return 0;
if (!insn->modrm.got) {
ret = insn_get_modrm(insn);
if (ret)
return ret;
}
if (insn->modrm.nbytes) {
modrm = insn->modrm.bytes[0];
if (insn->addr_bytes != 2 &&
......@@ -374,8 +432,10 @@ void insn_get_sib(struct insn *insn)
}
insn->sib.got = 1;
return 0;
err_out:
return;
return -ENODATA;
}
......@@ -386,15 +446,25 @@ void insn_get_sib(struct insn *insn)
* If necessary, first collects the instruction up to and including the
* SIB byte.
* Displacement value is sign-expanded.
*
* * Returns:
* 0: if decoding succeeded
* < 0: otherwise.
*/
void insn_get_displacement(struct insn *insn)
int insn_get_displacement(struct insn *insn)
{
insn_byte_t mod, rm, base;
int ret;
if (insn->displacement.got)
return;
if (!insn->sib.got)
insn_get_sib(insn);
return 0;
if (!insn->sib.got) {
ret = insn_get_sib(insn);
if (ret)
return ret;
}
if (insn->modrm.nbytes) {
/*
* Interpreting the modrm byte:
......@@ -436,9 +506,10 @@ void insn_get_displacement(struct insn *insn)
}
out:
insn->displacement.got = 1;
return 0;
err_out:
return;
return -ENODATA;
}
/* Decode moffset16/32/64. Return 0 if failed */
......@@ -537,20 +608,30 @@ static int __get_immptr(struct insn *insn)
}
/**
* insn_get_immediate() - Get the immediates of instruction
* insn_get_immediate() - Get the immediate in an instruction
* @insn: &struct insn containing instruction
*
* If necessary, first collects the instruction up to and including the
* displacement bytes.
* Basically, most of immediates are sign-expanded. Unsigned-value can be
* get by bit masking with ((1 << (nbytes * 8)) - 1)
* computed by bit masking with ((1 << (nbytes * 8)) - 1)
*
* Returns:
* 0: on success
* < 0: on error
*/
void insn_get_immediate(struct insn *insn)
int insn_get_immediate(struct insn *insn)
{
int ret;
if (insn->immediate.got)
return;
if (!insn->displacement.got)
insn_get_displacement(insn);
return 0;
if (!insn->displacement.got) {
ret = insn_get_displacement(insn);
if (ret)
return ret;
}
if (inat_has_moffset(insn->attr)) {
if (!__get_moffset(insn))
......@@ -597,9 +678,10 @@ void insn_get_immediate(struct insn *insn)
}
done:
insn->immediate.got = 1;
return 0;
err_out:
return;
return -ENODATA;
}
/**
......@@ -608,13 +690,65 @@ void insn_get_immediate(struct insn *insn)
*
* If necessary, first collects the instruction up to and including the
* immediates bytes.
*/
void insn_get_length(struct insn *insn)
*
* Returns:
* - 0 on success
* - < 0 on error
*/
int insn_get_length(struct insn *insn)
{
int ret;
if (insn->length)
return;
if (!insn->immediate.got)
insn_get_immediate(insn);
return 0;
if (!insn->immediate.got) {
ret = insn_get_immediate(insn);
if (ret)
return ret;
}
insn->length = (unsigned char)((unsigned long)insn->next_byte
- (unsigned long)insn->kaddr);
return 0;
}
/* Ensure this instruction is decoded completely */
static inline int insn_complete(struct insn *insn)
{
return insn->opcode.got && insn->modrm.got && insn->sib.got &&
insn->displacement.got && insn->immediate.got;
}
/**
* insn_decode() - Decode an x86 instruction
* @insn: &struct insn to be initialized
* @kaddr: address (in kernel memory) of instruction (or copy thereof)
* @buf_len: length of the insn buffer at @kaddr
* @m: insn mode, see enum insn_mode
*
* Returns:
* 0: if decoding succeeded
* < 0: otherwise.
*/
int insn_decode(struct insn *insn, const void *kaddr, int buf_len, enum insn_mode m)
{
int ret;
#define INSN_MODE_KERN (enum insn_mode)-1 /* __ignore_sync_check__ mode is only valid in the kernel */
if (m == INSN_MODE_KERN)
insn_init(insn, kaddr, buf_len, IS_ENABLED(CONFIG_X86_64));
else
insn_init(insn, kaddr, buf_len, m == INSN_MODE_64);
ret = insn_get_length(insn);
if (ret)
return ret;
if (insn_complete(insn))
return 0;
return -EINVAL;
}
tools/include/linux/kconfig.h 0 → 100644
View file @ c6536676
/* SPDX-License-Identifier: GPL-2.0 */
#ifndef _TOOLS_LINUX_KCONFIG_H
#define _TOOLS_LINUX_KCONFIG_H
/* CONFIG_CC_VERSION_TEXT (Do not delete this comment. See help in Kconfig) */
#ifdef CONFIG_CPU_BIG_ENDIAN
#define __BIG_ENDIAN 4321
#else
#define __LITTLE_ENDIAN 1234
#endif
#define __ARG_PLACEHOLDER_1 0,
#define __take_second_arg(__ignored, val, ...) val
/*
* The use of "&&" / "||" is limited in certain expressions.
* The following enable to calculate "and" / "or" with macro expansion only.
*/
#define __and(x, y) ___and(x, y)
#define ___and(x, y) ____and(__ARG_PLACEHOLDER_##x, y)
#define ____and(arg1_or_junk, y) __take_second_arg(arg1_or_junk y, 0)
#define __or(x, y) ___or(x, y)
#define ___or(x, y) ____or(__ARG_PLACEHOLDER_##x, y)
#define ____or(arg1_or_junk, y) __take_second_arg(arg1_or_junk 1, y)
/*
* Helper macros to use CONFIG_ options in C/CPP expressions. Note that
* these only work with boolean and tristate options.
*/
/*
* Getting something that works in C and CPP for an arg that may or may
* not be defined is tricky. Here, if we have "#define CONFIG_BOOGER 1"
* we match on the placeholder define, insert the "0," for arg1 and generate
* the triplet (0, 1, 0). Then the last step cherry picks the 2nd arg (a one).
* When CONFIG_BOOGER is not defined, we generate a (... 1, 0) pair, and when
* the last step cherry picks the 2nd arg, we get a zero.
*/
#define __is_defined(x) ___is_defined(x)
#define ___is_defined(val) ____is_defined(__ARG_PLACEHOLDER_##val)
#define ____is_defined(arg1_or_junk) __take_second_arg(arg1_or_junk 1, 0)
/*
* IS_BUILTIN(CONFIG_FOO) evaluates to 1 if CONFIG_FOO is set to 'y', 0
* otherwise. For boolean options, this is equivalent to
* IS_ENABLED(CONFIG_FOO).
*/
#define IS_BUILTIN(option) __is_defined(option)
/*
* IS_MODULE(CONFIG_FOO) evaluates to 1 if CONFIG_FOO is set to 'm', 0
* otherwise.
*/
#define IS_MODULE(option) __is_defined(option##_MODULE)
/*
* IS_REACHABLE(CONFIG_FOO) evaluates to 1 if the currently compiled
* code can call a function defined in code compiled based on CONFIG_FOO.
* This is similar to IS_ENABLED(), but returns false when invoked from
* built-in code when CONFIG_FOO is set to 'm'.
*/
#define IS_REACHABLE(option) __or(IS_BUILTIN(option), \
__and(IS_MODULE(option), __is_defined(MODULE)))
/*
* IS_ENABLED(CONFIG_FOO) evaluates to 1 if CONFIG_FOO is set to 'y' or 'm',
* 0 otherwise.
*/
#define IS_ENABLED(option) __or(IS_BUILTIN(option), IS_MODULE(option))
#endif /* _TOOLS_LINUX_KCONFIG_H */
tools/objtool/arch/x86/decode.c
View file @ c6536676
......@@ -11,11 +11,15 @@
#include "../../../arch/x86/lib/inat.c"
#include "../../../arch/x86/lib/insn.c"
#define CONFIG_64BIT 1
#include <asm/nops.h>
#include <asm/orc_types.h>
#include <objtool/check.h>
#include <objtool/elf.h>
#include <objtool/arch.h>
#include <objtool/warn.h>
#include <arch/elf.h>
static unsigned char op_to_cfi_reg[][2] = {
{CFI_AX, CFI_R8},
......@@ -90,7 +94,7 @@ int arch_decode_instruction(const struct elf *elf, const struct section *sec,
struct list_head *ops_list)
{
struct insn insn;
int x86_64, sign;
int x86_64, sign, ret;
unsigned char op1, op2, rex = 0, rex_b = 0, rex_r = 0, rex_w = 0,
rex_x = 0, modrm = 0, modrm_mod = 0, modrm_rm = 0,
modrm_reg = 0, sib = 0;
......@@ -101,10 +105,9 @@ int arch_decode_instruction(const struct elf *elf, const struct section *sec,
if (x86_64 == -1)
return -1;
insn_init(&insn, sec->data->d_buf + offset, maxlen, x86_64);
insn_get_length(&insn);
if (!insn_complete(&insn)) {
ret = insn_decode(&insn, sec->data->d_buf + offset, maxlen,
x86_64 ? INSN_MODE_64 : INSN_MODE_32);
if (ret < 0) {
WARN("can't decode instruction at %s:0x%lx", sec->name, offset);
return -1;
}
......@@ -596,11 +599,11 @@ void arch_initial_func_cfi_state(struct cfi_init_state *state)
const char *arch_nop_insn(int len)
{
static const char nops[5][5] = {
/* 1 */ { 0x90 },
/* 2 */ { 0x66, 0x90 },
/* 3 */ { 0x0f, 0x1f, 0x00 },
/* 4 */ { 0x0f, 0x1f, 0x40, 0x00 },
/* 5 */ { 0x0f, 0x1f, 0x44, 0x00, 0x00 },
{ BYTES_NOP1 },
{ BYTES_NOP2 },
{ BYTES_NOP3 },
{ BYTES_NOP4 },
{ BYTES_NOP5 },
};
if (len < 1 || len > 5) {
......@@ -611,6 +614,122 @@ const char *arch_nop_insn(int len)
return nops[len-1];
}
/* asm/alternative.h ? */
#define ALTINSTR_FLAG_INV (1 << 15)
#define ALT_NOT(feat) ((feat) | ALTINSTR_FLAG_INV)
struct alt_instr {
s32 instr_offset; /* original instruction */
s32 repl_offset; /* offset to replacement instruction */
u16 cpuid; /* cpuid bit set for replacement */
u8 instrlen; /* length of original instruction */
u8 replacementlen; /* length of new instruction */
} __packed;
static int elf_add_alternative(struct elf *elf,
struct instruction *orig, struct symbol *sym,
int cpuid, u8 orig_len, u8 repl_len)
{
const int size = sizeof(struct alt_instr);
struct alt_instr *alt;
struct section *sec;
Elf_Scn *s;
sec = find_section_by_name(elf, ".altinstructions");
if (!sec) {
sec = elf_create_section(elf, ".altinstructions",
SHF_WRITE, size, 0);
if (!sec) {
WARN_ELF("elf_create_section");
return -1;
}
}
s = elf_getscn(elf->elf, sec->idx);
if (!s) {
WARN_ELF("elf_getscn");
return -1;
}
sec->data = elf_newdata(s);
if (!sec->data) {
WARN_ELF("elf_newdata");
return -1;
}
sec->data->d_size = size;
sec->data->d_align = 1;
alt = sec->data->d_buf = malloc(size);
if (!sec->data->d_buf) {
perror("malloc");
return -1;
}
memset(sec->data->d_buf, 0, size);
if (elf_add_reloc_to_insn(elf, sec, sec->sh.sh_size,
R_X86_64_PC32, orig->sec, orig->offset)) {
WARN("elf_create_reloc: alt_instr::instr_offset");
return -1;
}
if (elf_add_reloc(elf, sec, sec->sh.sh_size + 4,
R_X86_64_PC32, sym, 0)) {
WARN("elf_create_reloc: alt_instr::repl_offset");
return -1;
}
alt->cpuid = cpuid;
alt->instrlen = orig_len;
alt->replacementlen = repl_len;
sec->sh.sh_size += size;
sec->changed = true;
return 0;
}
#define X86_FEATURE_RETPOLINE ( 7*32+12)
int arch_rewrite_retpolines(struct objtool_file *file)
{
struct instruction *insn;
struct reloc *reloc;
struct symbol *sym;
char name[32] = "";
list_for_each_entry(insn, &file->retpoline_call_list, call_node) {
if (!strcmp(insn->sec->name, ".text.__x86.indirect_thunk"))
continue;
reloc = insn->reloc;
sprintf(name, "__x86_indirect_alt_%s_%s",
insn->type == INSN_JUMP_DYNAMIC ? "jmp" : "call",
reloc->sym->name + 21);
sym = find_symbol_by_name(file->elf, name);
if (!sym) {
sym = elf_create_undef_symbol(file->elf, name);
if (!sym) {
WARN("elf_create_undef_symbol");
return -1;
}
}
if (elf_add_alternative(file->elf, insn, sym,
ALT_NOT(X86_FEATURE_RETPOLINE), 5, 5)) {
WARN("elf_add_alternative");
return -1;
}
}
return 0;
}
int arch_decode_hint_reg(struct instruction *insn, u8 sp_reg)
{
struct cfi_reg *cfa = &insn->cfi.cfa;
......@@ -646,3 +765,8 @@ int arch_decode_hint_reg(struct instruction *insn, u8 sp_reg)
return 0;
}
bool arch_is_retpoline(struct symbol *sym)
{
return !strncmp(sym->name, "__x86_indirect_", 15);
}
tools/objtool/arch/x86/include/arch/special.h
View file @ c6536676
......@@ -10,7 +10,7 @@
#define JUMP_ORIG_OFFSET 0
#define JUMP_NEW_OFFSET 4
#define ALT_ENTRY_SIZE 13
#define ALT_ENTRY_SIZE 12
#define ALT_ORIG_OFFSET 0
#define ALT_NEW_OFFSET 4
#define ALT_FEATURE_OFFSET 8
......
tools/objtool/check.c
View file @ c6536676
......@@ -433,8 +433,7 @@ static int add_dead_ends(struct objtool_file *file)
static int create_static_call_sections(struct objtool_file *file)
{
struct section *sec, *reloc_sec;
struct reloc *reloc;
struct section *sec;
struct static_call_site *site;
struct instruction *insn;
struct symbol *key_sym;
......@@ -452,7 +451,7 @@ static int create_static_call_sections(struct objtool_file *file)
return 0;
idx = 0;
list_for_each_entry(insn, &file->static_call_list, static_call_node)
list_for_each_entry(insn, &file->static_call_list, call_node)
idx++;
sec = elf_create_section(file->elf, ".static_call_sites", SHF_WRITE,
......@@ -460,36 +459,18 @@ static int create_static_call_sections(struct objtool_file *file)
if (!sec)
return -1;
reloc_sec = elf_create_reloc_section(file->elf, sec, SHT_RELA);
if (!reloc_sec)
return -1;
idx = 0;
list_for_each_entry(insn, &file->static_call_list, static_call_node) {
list_for_each_entry(insn, &file->static_call_list, call_node) {
site = (struct static_call_site *)sec->data->d_buf + idx;
memset(site, 0, sizeof(struct static_call_site));
/* populate reloc for 'addr' */
reloc = malloc(sizeof(*reloc));
if (!reloc) {
perror("malloc");
if (elf_add_reloc_to_insn(file->elf, sec,
idx * sizeof(struct static_call_site),
R_X86_64_PC32,
insn->sec, insn->offset))
return -1;
}
memset(reloc, 0, sizeof(*reloc));
insn_to_reloc_sym_addend(insn->sec, insn->offset, reloc);
if (!reloc->sym) {
WARN_FUNC("static call tramp: missing containing symbol",
insn->sec, insn->offset);
return -1;
}
reloc->type = R_X86_64_PC32;
reloc->offset = idx * sizeof(struct static_call_site);
reloc->sec = reloc_sec;
elf_add_reloc(file->elf, reloc);
/* find key symbol */
key_name = strdup(insn->call_dest->name);
......@@ -526,32 +507,21 @@ static int create_static_call_sections(struct objtool_file *file)
free(key_name);
/* populate reloc for 'key' */
reloc = malloc(sizeof(*reloc));
if (!reloc) {
perror("malloc");
if (elf_add_reloc(file->elf, sec,
idx * sizeof(struct static_call_site) + 4,
R_X86_64_PC32, key_sym,
is_sibling_call(insn) * STATIC_CALL_SITE_TAIL))
return -1;
}
memset(reloc, 0, sizeof(*reloc));
reloc->sym = key_sym;
reloc->addend = is_sibling_call(insn) ? STATIC_CALL_SITE_TAIL : 0;
reloc->type = R_X86_64_PC32;
reloc->offset = idx * sizeof(struct static_call_site) + 4;
reloc->sec = reloc_sec;
elf_add_reloc(file->elf, reloc);
idx++;
}
if (elf_rebuild_reloc_section(file->elf, reloc_sec))
return -1;
return 0;
}
static int create_mcount_loc_sections(struct objtool_file *file)
{
struct section *sec, *reloc_sec;
struct reloc *reloc;
struct section *sec;
unsigned long *loc;
struct instruction *insn;
int idx;
......@@ -574,49 +544,21 @@ static int create_mcount_loc_sections(struct objtool_file *file)
if (!sec)
return -1;
reloc_sec = elf_create_reloc_section(file->elf, sec, SHT_RELA);
if (!reloc_sec)
return -1;
idx = 0;
list_for_each_entry(insn, &file->mcount_loc_list, mcount_loc_node) {
loc = (unsigned long *)sec->data->d_buf + idx;
memset(loc, 0, sizeof(unsigned long));
reloc = malloc(sizeof(*reloc));
if (!reloc) {
perror("malloc");
if (elf_add_reloc_to_insn(file->elf, sec,
idx * sizeof(unsigned long),
R_X86_64_64,
insn->sec, insn->offset))
return -1;
}
memset(reloc, 0, sizeof(*reloc));
if (insn->sec->sym) {
reloc->sym = insn->sec->sym;
reloc->addend = insn->offset;
} else {
reloc->sym = find_symbol_containing(insn->sec, insn->offset);
if (!reloc->sym) {
WARN("missing symbol for insn at offset 0x%lx\n",
insn->offset);
return -1;
}
reloc->addend = insn->offset - reloc->sym->offset;
}
reloc->type = R_X86_64_64;
reloc->offset = idx * sizeof(unsigned long);
reloc->sec = reloc_sec;
elf_add_reloc(file->elf, reloc);
idx++;
}
if (elf_rebuild_reloc_section(file->elf, reloc_sec))
return -1;
return 0;
}
......@@ -850,6 +792,30 @@ static int add_ignore_alternatives(struct objtool_file *file)
return 0;
}
__weak bool arch_is_retpoline(struct symbol *sym)
{
return false;
}
#define NEGATIVE_RELOC ((void *)-1L)
static struct reloc *insn_reloc(struct objtool_file *file, struct instruction *insn)
{
if (insn->reloc == NEGATIVE_RELOC)
return NULL;
if (!insn->reloc) {
insn->reloc = find_reloc_by_dest_range(file->elf, insn->sec,
insn->offset, insn->len);
if (!insn->reloc) {
insn->reloc = NEGATIVE_RELOC;
return NULL;
}
}
return insn->reloc;
}
/*
* Find the destination instructions for all jumps.
*/
......@@ -864,16 +830,14 @@ static int add_jump_destinations(struct objtool_file *file)
if (!is_static_jump(insn))
continue;
reloc = find_reloc_by_dest_range(file->elf, insn->sec,
insn->offset, insn->len);
reloc = insn_reloc(file, insn);
if (!reloc) {
dest_sec = insn->sec;
dest_off = arch_jump_destination(insn);
} else if (reloc->sym->type == STT_SECTION) {
dest_sec = reloc->sym->sec;
dest_off = arch_dest_reloc_offset(reloc->addend);
} else if (!strncmp(reloc->sym->name, "__x86_indirect_thunk_", 21) ||
!strncmp(reloc->sym->name, "__x86_retpoline_", 16)) {
} else if (arch_is_retpoline(reloc->sym)) {
/*
* Retpoline jumps are really dynamic jumps in
* disguise, so convert them accordingly.
......@@ -883,13 +847,16 @@ static int add_jump_destinations(struct objtool_file *file)
else
insn->type = INSN_JUMP_DYNAMIC_CONDITIONAL;
list_add_tail(&insn->call_node,
&file->retpoline_call_list);
insn->retpoline_safe = true;
continue;
} else if (insn->func) {
/* internal or external sibling call (with reloc) */
insn->call_dest = reloc->sym;
if (insn->call_dest->static_call_tramp) {
list_add_tail(&insn->static_call_node,
list_add_tail(&insn->call_node,
&file->static_call_list);
}
continue;
......@@ -951,7 +918,7 @@ static int add_jump_destinations(struct objtool_file *file)
/* internal sibling call (without reloc) */
insn->call_dest = insn->jump_dest->func;
if (insn->call_dest->static_call_tramp) {
list_add_tail(&insn->static_call_node,
list_add_tail(&insn->call_node,
&file->static_call_list);
}
}
......@@ -995,8 +962,7 @@ static int add_call_destinations(struct objtool_file *file)
if (insn->type != INSN_CALL)
continue;
reloc = find_reloc_by_dest_range(file->elf, insn->sec,
insn->offset, insn->len);
reloc = insn_reloc(file, insn);
if (!reloc) {
dest_off = arch_jump_destination(insn);
insn->call_dest = find_call_destination(insn->sec, dest_off);
......@@ -1026,9 +992,29 @@ static int add_call_destinations(struct objtool_file *file)
dest_off);
return -1;
}
} else if (arch_is_retpoline(reloc->sym)) {
/*
* Retpoline calls are really dynamic calls in
* disguise, so convert them accordingly.
*/
insn->type = INSN_CALL_DYNAMIC;
insn->retpoline_safe = true;
list_add_tail(&insn->call_node,
&file->retpoline_call_list);
remove_insn_ops(insn);
continue;
} else
insn->call_dest = reloc->sym;
if (insn->call_dest && insn->call_dest->static_call_tramp) {
list_add_tail(&insn->call_node,
&file->static_call_list);
}
/*
* Many compilers cannot disable KCOV with a function attribute
* so they need a little help, NOP out any KCOV calls from noinstr
......@@ -1175,8 +1161,7 @@ static int handle_group_alt(struct objtool_file *file,
* alternatives code can adjust the relative offsets
* accordingly.
*/
alt_reloc = find_reloc_by_dest_range(file->elf, insn->sec,
insn->offset, insn->len);
alt_reloc = insn_reloc(file, insn);
if (alt_reloc &&
!arch_support_alt_relocation(special_alt, insn, alt_reloc)) {
......@@ -1751,6 +1736,11 @@ static void mark_rodata(struct objtool_file *file)
file->rodata = found;
}
__weak int arch_rewrite_retpolines(struct objtool_file *file)
{
return 0;
}
static int decode_sections(struct objtool_file *file)
{
int ret;
......@@ -1772,10 +1762,17 @@ static int decode_sections(struct objtool_file *file)
if (ret)
return ret;
/*
* Must be before add_{jump_call}_destination.
*/
ret = read_static_call_tramps(file);
if (ret)
return ret;
/*
* Must be before add_special_section_alts() as that depends on
* jump_dest being set.
*/
ret = add_jump_destinations(file);
if (ret)
return ret;
......@@ -1784,6 +1781,10 @@ static int decode_sections(struct objtool_file *file)
if (ret)
return ret;
/*
* Must be before add_call_destination(); it changes INSN_CALL to
* INSN_JUMP.
*/
ret = read_intra_function_calls(file);
if (ret)
return ret;
......@@ -1808,6 +1809,15 @@ static int decode_sections(struct objtool_file *file)
if (ret)
return ret;
/*
* Must be after add_special_section_alts(), since this will emit
* alternatives. Must be after add_{jump,call}_destination(), since
* those create the call insn lists.
*/
ret = arch_rewrite_retpolines(file);
if (ret)
return ret;
return 0;
}
......@@ -2746,11 +2756,6 @@ static int validate_branch(struct objtool_file *file, struct symbol *func,
if (dead_end_function(file, insn->call_dest))
return 0;
if (insn->type == INSN_CALL && insn->call_dest->static_call_tramp) {
list_add_tail(&insn->static_call_node,
&file->static_call_list);
}
break;
case INSN_JUMP_CONDITIONAL:
......
tools/objtool/elf.c
View file @ c6536676
......@@ -211,32 +211,6 @@ struct reloc *find_reloc_by_dest(const struct elf *elf, struct section *sec, uns
return find_reloc_by_dest_range(elf, sec, offset, 1);
}
void insn_to_reloc_sym_addend(struct section *sec, unsigned long offset,
struct reloc *reloc)
{
if (sec->sym) {
reloc->sym = sec->sym;
reloc->addend = offset;
return;
}
/*
* The Clang assembler strips section symbols, so we have to reference
* the function symbol instead:
*/
reloc->sym = find_symbol_containing(sec, offset);
if (!reloc->sym) {
/*
* Hack alert. This happens when we need to reference the NOP
* pad insn immediately after the function.
*/
reloc->sym = find_symbol_containing(sec, offset - 1);
}
if (reloc->sym)
reloc->addend = offset - reloc->sym->offset;
}
static int read_sections(struct elf *elf)
{
Elf_Scn *s = NULL;
......@@ -316,12 +290,39 @@ static int read_sections(struct elf *elf)
return 0;
}
static void elf_add_symbol(struct elf *elf, struct symbol *sym)
{
struct list_head *entry;
struct rb_node *pnode;
sym->type = GELF_ST_TYPE(sym->sym.st_info);
sym->bind = GELF_ST_BIND(sym->sym.st_info);
sym->offset = sym->sym.st_value;
sym->len = sym->sym.st_size;
rb_add(&sym->node, &sym->sec->symbol_tree, symbol_to_offset);
pnode = rb_prev(&sym->node);
if (pnode)
entry = &rb_entry(pnode, struct symbol, node)->list;
else
entry = &sym->sec->symbol_list;
list_add(&sym->list, entry);
elf_hash_add(elf->symbol_hash, &sym->hash, sym->idx);
elf_hash_add(elf->symbol_name_hash, &sym->name_hash, str_hash(sym->name));
/*
* Don't store empty STT_NOTYPE symbols in the rbtree. They
* can exist within a function, confusing the sorting.
*/
if (!sym->len)
rb_erase(&sym->node, &sym->sec->symbol_tree);
}
static int read_symbols(struct elf *elf)
{
struct section *symtab, *symtab_shndx, *sec;
struct symbol *sym, *pfunc;
struct list_head *entry;
struct rb_node *pnode;
int symbols_nr, i;
char *coldstr;
Elf_Data *shndx_data = NULL;
......@@ -366,9 +367,6 @@ static int read_symbols(struct elf *elf)
goto err;
}
sym->type = GELF_ST_TYPE(sym->sym.st_info);
sym->bind = GELF_ST_BIND(sym->sym.st_info);
if ((sym->sym.st_shndx > SHN_UNDEF &&
sym->sym.st_shndx < SHN_LORESERVE) ||
(shndx_data && sym->sym.st_shndx == SHN_XINDEX)) {
......@@ -381,32 +379,14 @@ static int read_symbols(struct elf *elf)
sym->name);
goto err;
}
if (sym->type == STT_SECTION) {
if (GELF_ST_TYPE(sym->sym.st_info) == STT_SECTION) {
sym->name = sym->sec->name;
sym->sec->sym = sym;
}
} else
sym->sec = find_section_by_index(elf, 0);
sym->offset = sym->sym.st_value;
sym->len = sym->sym.st_size;
rb_add(&sym->node, &sym->sec->symbol_tree, symbol_to_offset);
pnode = rb_prev(&sym->node);
if (pnode)
entry = &rb_entry(pnode, struct symbol, node)->list;
else
entry = &sym->sec->symbol_list;
list_add(&sym->list, entry);
elf_hash_add(elf->symbol_hash, &sym->hash, sym->idx);
elf_hash_add(elf->symbol_name_hash, &sym->name_hash, str_hash(sym->name));
/*
* Don't store empty STT_NOTYPE symbols in the rbtree. They
* can exist within a function, confusing the sorting.
*/
if (!sym->len)
rb_erase(&sym->node, &sym->sec->symbol_tree);
elf_add_symbol(elf, sym);
}
if (stats)
......@@ -473,12 +453,73 @@ static int read_symbols(struct elf *elf)
return -1;
}
void elf_add_reloc(struct elf *elf, struct reloc *reloc)
static struct section *elf_create_reloc_section(struct elf *elf,
struct section *base,
int reltype);
int elf_add_reloc(struct elf *elf, struct section *sec, unsigned long offset,
unsigned int type, struct symbol *sym, int addend)
{
struct section *sec = reloc->sec;
struct reloc *reloc;
list_add_tail(&reloc->list, &sec->reloc_list);
if (!sec->reloc && !elf_create_reloc_section(elf, sec, SHT_RELA))
return -1;
reloc = malloc(sizeof(*reloc));
if (!reloc) {
perror("malloc");
return -1;
}
memset(reloc, 0, sizeof(*reloc));
reloc->sec = sec->reloc;
reloc->offset = offset;
reloc->type = type;
reloc->sym = sym;
reloc->addend = addend;
list_add_tail(&reloc->list, &sec->reloc->reloc_list);
elf_hash_add(elf->reloc_hash, &reloc->hash, reloc_hash(reloc));
sec->reloc->changed = true;
return 0;
}
int elf_add_reloc_to_insn(struct elf *elf, struct section *sec,
unsigned long offset, unsigned int type,
struct section *insn_sec, unsigned long insn_off)
{
struct symbol *sym;
int addend;
if (insn_sec->sym) {
sym = insn_sec->sym;
addend = insn_off;
} else {
/*
* The Clang assembler strips section symbols, so we have to
* reference the function symbol instead:
*/
sym = find_symbol_containing(insn_sec, insn_off);
if (!sym) {
/*
* Hack alert. This happens when we need to reference
* the NOP pad insn immediately after the function.
*/
sym = find_symbol_containing(insn_sec, insn_off - 1);
}
if (!sym) {
WARN("can't find symbol containing %s+0x%lx", insn_sec->name, insn_off);
return -1;
}
addend = insn_off - sym->offset;
}
return elf_add_reloc(elf, sec, offset, type, sym, addend);
}
static int read_rel_reloc(struct section *sec, int i, struct reloc *reloc, unsigned int *symndx)
......@@ -558,7 +599,9 @@ static int read_relocs(struct elf *elf)
return -1;
}
elf_add_reloc(elf, reloc);
list_add_tail(&reloc->list, &sec->reloc_list);
elf_hash_add(elf->reloc_hash, &reloc->hash, reloc_hash(reloc));
nr_reloc++;
}
max_reloc = max(max_reloc, nr_reloc);
......@@ -636,13 +679,108 @@ struct elf *elf_open_read(const char *name, int flags)
return NULL;
}
static int elf_add_string(struct elf *elf, struct section *strtab, char *str)
{
Elf_Data *data;
Elf_Scn *s;
int len;
if (!strtab)
strtab = find_section_by_name(elf, ".strtab");
if (!strtab) {
WARN("can't find .strtab section");
return -1;
}
s = elf_getscn(elf->elf, strtab->idx);
if (!s) {
WARN_ELF("elf_getscn");
return -1;
}
data = elf_newdata(s);
if (!data) {
WARN_ELF("elf_newdata");
return -1;
}
data->d_buf = str;
data->d_size = strlen(str) + 1;
data->d_align = 1;
len = strtab->len;
strtab->len += data->d_size;
strtab->changed = true;
return len;
}
struct symbol *elf_create_undef_symbol(struct elf *elf, const char *name)
{
struct section *symtab;
struct symbol *sym;
Elf_Data *data;
Elf_Scn *s;
sym = malloc(sizeof(*sym));
if (!sym) {
perror("malloc");
return NULL;
}
memset(sym, 0, sizeof(*sym));
sym->name = strdup(name);
sym->sym.st_name = elf_add_string(elf, NULL, sym->name);
if (sym->sym.st_name == -1)
return NULL;
sym->sym.st_info = GELF_ST_INFO(STB_GLOBAL, STT_NOTYPE);
// st_other 0
// st_shndx 0
// st_value 0
// st_size 0
symtab = find_section_by_name(elf, ".symtab");
if (!symtab) {
WARN("can't find .symtab");
return NULL;
}
s = elf_getscn(elf->elf, symtab->idx);
if (!s) {
WARN_ELF("elf_getscn");
return NULL;
}
data = elf_newdata(s);
if (!data) {
WARN_ELF("elf_newdata");
return NULL;
}
data->d_buf = &sym->sym;
data->d_size = sizeof(sym->sym);
data->d_align = 1;
sym->idx = symtab->len / sizeof(sym->sym);
symtab->len += data->d_size;
symtab->changed = true;
sym->sec = find_section_by_index(elf, 0);
elf_add_symbol(elf, sym);
return sym;
}
struct section *elf_create_section(struct elf *elf, const char *name,
unsigned int sh_flags, size_t entsize, int nr)
{
struct section *sec, *shstrtab;
size_t size = entsize * nr;
Elf_Scn *s;
Elf_Data *data;
sec = malloc(sizeof(*sec));
if (!sec) {
......@@ -699,7 +837,6 @@ struct section *elf_create_section(struct elf *elf, const char *name,
sec->sh.sh_addralign = 1;
sec->sh.sh_flags = SHF_ALLOC | sh_flags;
/* Add section name to .shstrtab (or .strtab for Clang) */
shstrtab = find_section_by_name(elf, ".shstrtab");
if (!shstrtab)
......@@ -708,27 +845,9 @@ struct section *elf_create_section(struct elf *elf, const char *name,
WARN("can't find .shstrtab or .strtab section");
return NULL;
}
s = elf_getscn(elf->elf, shstrtab->idx);
if (!s) {
WARN_ELF("elf_getscn");
return NULL;
}
data = elf_newdata(s);
if (!data) {
WARN_ELF("elf_newdata");
sec->sh.sh_name = elf_add_string(elf, shstrtab, sec->name);
if (sec->sh.sh_name == -1)
return NULL;
}
data->d_buf = sec->name;
data->d_size = strlen(name) + 1;
data->d_align = 1;
sec->sh.sh_name = shstrtab->len;
shstrtab->len += strlen(name) + 1;
shstrtab->changed = true;
list_add_tail(&sec->list, &elf->sections);
elf_hash_add(elf->section_hash, &sec->hash, sec->idx);
......@@ -799,7 +918,7 @@ static struct section *elf_create_rela_reloc_section(struct elf *elf, struct sec
return sec;
}
struct section *elf_create_reloc_section(struct elf *elf,
static struct section *elf_create_reloc_section(struct elf *elf,
struct section *base,
int reltype)
{
......@@ -873,14 +992,11 @@ static int elf_rebuild_rela_reloc_section(struct section *sec, int nr)
return 0;
}
int elf_rebuild_reloc_section(struct elf *elf, struct section *sec)
static int elf_rebuild_reloc_section(struct elf *elf, struct section *sec)
{
struct reloc *reloc;
int nr;
sec->changed = true;
elf->changed = true;
nr = 0;
list_for_each_entry(reloc, &sec->reloc_list, list)
nr++;
......@@ -944,9 +1060,15 @@ int elf_write(struct elf *elf)
struct section *sec;
Elf_Scn *s;
/* Update section headers for changed sections: */
/* Update changed relocation sections and section headers: */
list_for_each_entry(sec, &elf->sections, list) {
if (sec->changed) {
if (sec->base &&
elf_rebuild_reloc_section(elf, sec)) {
WARN("elf_rebuild_reloc_section");
return -1;
}
s = elf_getscn(elf->elf, sec->idx);
if (!s) {
WARN_ELF("elf_getscn");
......@@ -958,6 +1080,7 @@ int elf_write(struct elf *elf)
}
sec->changed = false;
elf->changed = true;
}
}
......
tools/objtool/include/objtool/arch.h
View file @ c6536676
This diff is collapsed.
tools/objtool/include/objtool/check.h
View file @ c6536676
This diff is collapsed.
tools/objtool/include/objtool/elf.h
View file @ c6536676
This diff is collapsed.
tools/objtool/include/objtool/objtool.h
View file @ c6536676
This diff is collapsed.
tools/objtool/objtool.c
View file @ c6536676
This diff is collapsed.
tools/objtool/orc_gen.c
View file @ c6536676
This diff is collapsed.
tools/objtool/special.c
View file @ c6536676
This diff is collapsed.
tools/objtool/sync-check.sh
View file @ c6536676
This diff is collapsed.
tools/perf/arch/x86/tests/insn-x86.c
View file @ c6536676
This diff is collapsed.
tools/perf/arch/x86/util/archinsn.c
View file @ c6536676
This diff is collapsed.
tools/perf/check-headers.sh
View file @ c6536676
This diff is collapsed.
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment
GitLab Nexedi Edition | About GitLab | About Nexedi | 沪ICP备2021021310号-2 | 沪ICP备2021021310号-7