Commit e5a0fc4e authored by Linus Torvalds's avatar Linus Torvalds

Merge tag 'x86-apic-2021-06-28' of git://git.kernel.org/pub/scm/linux/kernel/git/tip/tip

Pull x86 exception handling updates from Ingo Molnar:

 - Clean up & simplify AP exception handling setup.

 - Consolidate the disjoint IDT setup code living in idt_setup_traps()
   and idt_setup_ist_traps() into a single idt_setup_traps()
   initialization function and call it before cpu_init().

* tag 'x86-apic-2021-06-28' of git://git.kernel.org/pub/scm/linux/kernel/git/tip/tip:
  x86/idt: Rework IDT setup for boot CPU
  x86/cpu: Init AP exception handling from cpu_init_secondary()
parents 9269d27e 1dcc917a
...@@ -421,10 +421,8 @@ extern bool idt_is_f00f_address(unsigned long address); ...@@ -421,10 +421,8 @@ extern bool idt_is_f00f_address(unsigned long address);
#ifdef CONFIG_X86_64 #ifdef CONFIG_X86_64
extern void idt_setup_early_pf(void); extern void idt_setup_early_pf(void);
extern void idt_setup_ist_traps(void);
#else #else
static inline void idt_setup_early_pf(void) { } static inline void idt_setup_early_pf(void) { }
static inline void idt_setup_ist_traps(void) { }
#endif #endif
extern void idt_invalidate(void *addr); extern void idt_invalidate(void *addr);
......
...@@ -663,6 +663,7 @@ extern void load_direct_gdt(int); ...@@ -663,6 +663,7 @@ extern void load_direct_gdt(int);
extern void load_fixmap_gdt(int); extern void load_fixmap_gdt(int);
extern void load_percpu_segment(int); extern void load_percpu_segment(int);
extern void cpu_init(void); extern void cpu_init(void);
extern void cpu_init_secondary(void);
extern void cpu_init_exception_handling(void); extern void cpu_init_exception_handling(void);
extern void cr4_init(void); extern void cr4_init(void);
......
...@@ -1938,13 +1938,12 @@ void cpu_init_exception_handling(void) ...@@ -1938,13 +1938,12 @@ void cpu_init_exception_handling(void)
/* /*
* cpu_init() initializes state that is per-CPU. Some data is already * cpu_init() initializes state that is per-CPU. Some data is already
* initialized (naturally) in the bootstrap process, such as the GDT * initialized (naturally) in the bootstrap process, such as the GDT. We
* and IDT. We reload them nevertheless, this function acts as a * reload it nevertheless, this function acts as a 'CPU state barrier',
* 'CPU state barrier', nothing should get across. * nothing should get across.
*/ */
void cpu_init(void) void cpu_init(void)
{ {
struct tss_struct *tss = this_cpu_ptr(&cpu_tss_rw);
struct task_struct *cur = current; struct task_struct *cur = current;
int cpu = raw_smp_processor_id(); int cpu = raw_smp_processor_id();
...@@ -1957,8 +1956,6 @@ void cpu_init(void) ...@@ -1957,8 +1956,6 @@ void cpu_init(void)
early_cpu_to_node(cpu) != NUMA_NO_NODE) early_cpu_to_node(cpu) != NUMA_NO_NODE)
set_numa_node(early_cpu_to_node(cpu)); set_numa_node(early_cpu_to_node(cpu));
#endif #endif
setup_getcpu(cpu);
pr_debug("Initializing CPU#%d\n", cpu); pr_debug("Initializing CPU#%d\n", cpu);
if (IS_ENABLED(CONFIG_X86_64) || cpu_feature_enabled(X86_FEATURE_VME) || if (IS_ENABLED(CONFIG_X86_64) || cpu_feature_enabled(X86_FEATURE_VME) ||
...@@ -1970,7 +1967,6 @@ void cpu_init(void) ...@@ -1970,7 +1967,6 @@ void cpu_init(void)
* and set up the GDT descriptor: * and set up the GDT descriptor:
*/ */
switch_to_new_gdt(cpu); switch_to_new_gdt(cpu);
load_current_idt();
if (IS_ENABLED(CONFIG_X86_64)) { if (IS_ENABLED(CONFIG_X86_64)) {
loadsegment(fs, 0); loadsegment(fs, 0);
...@@ -1990,12 +1986,6 @@ void cpu_init(void) ...@@ -1990,12 +1986,6 @@ void cpu_init(void)
initialize_tlbstate_and_flush(); initialize_tlbstate_and_flush();
enter_lazy_tlb(&init_mm, cur); enter_lazy_tlb(&init_mm, cur);
/* Initialize the TSS. */
tss_setup_ist(tss);
tss_setup_io_bitmap(tss);
set_tss_desc(cpu, &get_cpu_entry_area(cpu)->tss.x86_tss);
load_TR_desc();
/* /*
* sp0 points to the entry trampoline stack regardless of what task * sp0 points to the entry trampoline stack regardless of what task
* is running. * is running.
...@@ -2017,6 +2007,18 @@ void cpu_init(void) ...@@ -2017,6 +2007,18 @@ void cpu_init(void)
load_fixmap_gdt(cpu); load_fixmap_gdt(cpu);
} }
#ifdef CONFIG_SMP
void cpu_init_secondary(void)
{
/*
* Relies on the BP having set-up the IDT tables, which are loaded
* on this CPU in cpu_init_exception_handling().
*/
cpu_init_exception_handling();
cpu_init();
}
#endif
/* /*
* The microcode loader calls this upon late microcode load to recheck features, * The microcode loader calls this upon late microcode load to recheck features,
* only when microcode has been updated. Caller holds microcode_mutex and CPU * only when microcode has been updated. Caller holds microcode_mutex and CPU
......
...@@ -35,12 +35,16 @@ ...@@ -35,12 +35,16 @@
#define SYSG(_vector, _addr) \ #define SYSG(_vector, _addr) \
G(_vector, _addr, DEFAULT_STACK, GATE_INTERRUPT, DPL3, __KERNEL_CS) G(_vector, _addr, DEFAULT_STACK, GATE_INTERRUPT, DPL3, __KERNEL_CS)
#ifdef CONFIG_X86_64
/* /*
* Interrupt gate with interrupt stack. The _ist index is the index in * Interrupt gate with interrupt stack. The _ist index is the index in
* the tss.ist[] array, but for the descriptor it needs to start at 1. * the tss.ist[] array, but for the descriptor it needs to start at 1.
*/ */
#define ISTG(_vector, _addr, _ist) \ #define ISTG(_vector, _addr, _ist) \
G(_vector, _addr, _ist + 1, GATE_INTERRUPT, DPL0, __KERNEL_CS) G(_vector, _addr, _ist + 1, GATE_INTERRUPT, DPL0, __KERNEL_CS)
#else
#define ISTG(_vector, _addr, _ist) INTG(_vector, _addr)
#endif
/* Task gate */ /* Task gate */
#define TSKG(_vector, _gdt) \ #define TSKG(_vector, _gdt) \
...@@ -74,7 +78,7 @@ static const __initconst struct idt_data early_idts[] = { ...@@ -74,7 +78,7 @@ static const __initconst struct idt_data early_idts[] = {
*/ */
static const __initconst struct idt_data def_idts[] = { static const __initconst struct idt_data def_idts[] = {
INTG(X86_TRAP_DE, asm_exc_divide_error), INTG(X86_TRAP_DE, asm_exc_divide_error),
INTG(X86_TRAP_NMI, asm_exc_nmi), ISTG(X86_TRAP_NMI, asm_exc_nmi, IST_INDEX_NMI),
INTG(X86_TRAP_BR, asm_exc_bounds), INTG(X86_TRAP_BR, asm_exc_bounds),
INTG(X86_TRAP_UD, asm_exc_invalid_op), INTG(X86_TRAP_UD, asm_exc_invalid_op),
INTG(X86_TRAP_NM, asm_exc_device_not_available), INTG(X86_TRAP_NM, asm_exc_device_not_available),
...@@ -91,12 +95,16 @@ static const __initconst struct idt_data def_idts[] = { ...@@ -91,12 +95,16 @@ static const __initconst struct idt_data def_idts[] = {
#ifdef CONFIG_X86_32 #ifdef CONFIG_X86_32
TSKG(X86_TRAP_DF, GDT_ENTRY_DOUBLEFAULT_TSS), TSKG(X86_TRAP_DF, GDT_ENTRY_DOUBLEFAULT_TSS),
#else #else
INTG(X86_TRAP_DF, asm_exc_double_fault), ISTG(X86_TRAP_DF, asm_exc_double_fault, IST_INDEX_DF),
#endif #endif
INTG(X86_TRAP_DB, asm_exc_debug), ISTG(X86_TRAP_DB, asm_exc_debug, IST_INDEX_DB),
#ifdef CONFIG_X86_MCE #ifdef CONFIG_X86_MCE
INTG(X86_TRAP_MC, asm_exc_machine_check), ISTG(X86_TRAP_MC, asm_exc_machine_check, IST_INDEX_MCE),
#endif
#ifdef CONFIG_AMD_MEM_ENCRYPT
ISTG(X86_TRAP_VC, asm_exc_vmm_communication, IST_INDEX_VC),
#endif #endif
SYSG(X86_TRAP_OF, asm_exc_overflow), SYSG(X86_TRAP_OF, asm_exc_overflow),
...@@ -221,22 +229,6 @@ static const __initconst struct idt_data early_pf_idts[] = { ...@@ -221,22 +229,6 @@ static const __initconst struct idt_data early_pf_idts[] = {
INTG(X86_TRAP_PF, asm_exc_page_fault), INTG(X86_TRAP_PF, asm_exc_page_fault),
}; };
/*
* The exceptions which use Interrupt stacks. They are setup after
* cpu_init() when the TSS has been initialized.
*/
static const __initconst struct idt_data ist_idts[] = {
ISTG(X86_TRAP_DB, asm_exc_debug, IST_INDEX_DB),
ISTG(X86_TRAP_NMI, asm_exc_nmi, IST_INDEX_NMI),
ISTG(X86_TRAP_DF, asm_exc_double_fault, IST_INDEX_DF),
#ifdef CONFIG_X86_MCE
ISTG(X86_TRAP_MC, asm_exc_machine_check, IST_INDEX_MCE),
#endif
#ifdef CONFIG_AMD_MEM_ENCRYPT
ISTG(X86_TRAP_VC, asm_exc_vmm_communication, IST_INDEX_VC),
#endif
};
/** /**
* idt_setup_early_pf - Initialize the idt table with early pagefault handler * idt_setup_early_pf - Initialize the idt table with early pagefault handler
* *
...@@ -254,14 +246,6 @@ void __init idt_setup_early_pf(void) ...@@ -254,14 +246,6 @@ void __init idt_setup_early_pf(void)
idt_setup_from_table(idt_table, early_pf_idts, idt_setup_from_table(idt_table, early_pf_idts,
ARRAY_SIZE(early_pf_idts), true); ARRAY_SIZE(early_pf_idts), true);
} }
/**
* idt_setup_ist_traps - Initialize the idt table with traps using IST
*/
void __init idt_setup_ist_traps(void)
{
idt_setup_from_table(idt_table, ist_idts, ARRAY_SIZE(ist_idts), true);
}
#endif #endif
static void __init idt_map_in_cea(void) static void __init idt_map_in_cea(void)
......
...@@ -232,8 +232,7 @@ static void notrace start_secondary(void *unused) ...@@ -232,8 +232,7 @@ static void notrace start_secondary(void *unused)
load_cr3(swapper_pg_dir); load_cr3(swapper_pg_dir);
__flush_tlb_all(); __flush_tlb_all();
#endif #endif
cpu_init_exception_handling(); cpu_init_secondary();
cpu_init();
rcu_cpu_starting(raw_smp_processor_id()); rcu_cpu_starting(raw_smp_processor_id());
x86_cpuinit.early_percpu_clock_init(); x86_cpuinit.early_percpu_clock_init();
smp_callin(); smp_callin();
......
...@@ -1160,12 +1160,9 @@ void __init trap_init(void) ...@@ -1160,12 +1160,9 @@ void __init trap_init(void)
/* Init GHCB memory pages when running as an SEV-ES guest */ /* Init GHCB memory pages when running as an SEV-ES guest */
sev_es_init_vc_handling(); sev_es_init_vc_handling();
/* Initialize TSS before setting up traps so ISTs work */
cpu_init_exception_handling();
/* Setup traps as cpu_init() might #GP */
idt_setup_traps(); idt_setup_traps();
/*
* Should be a barrier for any external CPU state:
*/
cpu_init(); cpu_init();
idt_setup_ist_traps();
} }
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment