Commit d0af9eed authored by Suresh Siddha's avatar Suresh Siddha Committed by H. Peter Anvin

x86, pat/mtrr: Rendezvous all the cpus for MTRR/PAT init

SDM Vol 3a section titled "MTRR considerations in MP systems" specifies
the need for synchronizing the logical cpu's while initializing/updating
MTRR.

Currently Linux kernel does the synchronization of all cpu's only when
a single MTRR register is programmed/updated. During an AP online
(during boot/cpu-online/resume)  where we initialize all the MTRR/PAT registers,
we don't follow this synchronization algorithm.

This can lead to scenarios where during a dynamic cpu online, that logical cpu
is initializing MTRR/PAT with cache disabled (cr0.cd=1) etc while other logical
HT sibling continue to run (also with cache disabled because of cr0.cd=1
on its sibling).

Starting from Westmere, VMX transitions with cr0.cd=1 don't work properly
(because of some VMX performance optimizations) and the above scenario
(with one logical cpu doing VMX activity and another logical cpu coming online)
can result in system crash.

Fix the MTRR initialization by doing rendezvous of all the cpus. During
boot and resume, we delay the MTRR/PAT init for APs till all the
logical cpu's come online and the rendezvous process at the end of AP's bringup,
will initialize the MTRR/PAT for all AP's.

For dynamic single cpu online, we synchronize all the logical cpus and
do the MTRR/PAT init on the AP that is coming online.
Signed-off-by: default avatarSuresh Siddha <suresh.b.siddha@intel.com>
Signed-off-by: default avatarH. Peter Anvin <hpa@zytor.com>
parent 269c861b
...@@ -121,8 +121,12 @@ extern int mtrr_del_page(int reg, unsigned long base, unsigned long size); ...@@ -121,8 +121,12 @@ extern int mtrr_del_page(int reg, unsigned long base, unsigned long size);
extern void mtrr_centaur_report_mcr(int mcr, u32 lo, u32 hi); extern void mtrr_centaur_report_mcr(int mcr, u32 lo, u32 hi);
extern void mtrr_ap_init(void); extern void mtrr_ap_init(void);
extern void mtrr_bp_init(void); extern void mtrr_bp_init(void);
extern void set_mtrr_aps_delayed_init(void);
extern void mtrr_aps_init(void);
extern void mtrr_bp_restore(void);
extern int mtrr_trim_uncached_memory(unsigned long end_pfn); extern int mtrr_trim_uncached_memory(unsigned long end_pfn);
extern int amd_special_default_mtrr(void); extern int amd_special_default_mtrr(void);
extern u32 mtrr_aps_delayed_init;
# else # else
static inline u8 mtrr_type_lookup(u64 addr, u64 end) static inline u8 mtrr_type_lookup(u64 addr, u64 end)
{ {
...@@ -161,6 +165,9 @@ static inline void mtrr_centaur_report_mcr(int mcr, u32 lo, u32 hi) ...@@ -161,6 +165,9 @@ static inline void mtrr_centaur_report_mcr(int mcr, u32 lo, u32 hi)
#define mtrr_ap_init() do {} while (0) #define mtrr_ap_init() do {} while (0)
#define mtrr_bp_init() do {} while (0) #define mtrr_bp_init() do {} while (0)
#define set_mtrr_aps_delayed_init() do {} while (0)
#define mtrr_aps_init() do {} while (0)
#define mtrr_bp_restore() do {} while (0)
# endif # endif
#ifdef CONFIG_COMPAT #ifdef CONFIG_COMPAT
......
...@@ -58,6 +58,7 @@ unsigned int mtrr_usage_table[MTRR_MAX_VAR_RANGES]; ...@@ -58,6 +58,7 @@ unsigned int mtrr_usage_table[MTRR_MAX_VAR_RANGES];
static DEFINE_MUTEX(mtrr_mutex); static DEFINE_MUTEX(mtrr_mutex);
u64 size_or_mask, size_and_mask; u64 size_or_mask, size_and_mask;
u32 mtrr_aps_delayed_init;
static struct mtrr_ops *mtrr_ops[X86_VENDOR_NUM]; static struct mtrr_ops *mtrr_ops[X86_VENDOR_NUM];
...@@ -163,7 +164,10 @@ static void ipi_handler(void *info) ...@@ -163,7 +164,10 @@ static void ipi_handler(void *info)
if (data->smp_reg != ~0U) { if (data->smp_reg != ~0U) {
mtrr_if->set(data->smp_reg, data->smp_base, mtrr_if->set(data->smp_reg, data->smp_base,
data->smp_size, data->smp_type); data->smp_size, data->smp_type);
} else { } else if (mtrr_aps_delayed_init) {
/*
* Initialize the MTRRs inaddition to the synchronisation.
*/
mtrr_if->set_all(); mtrr_if->set_all();
} }
...@@ -265,6 +269,8 @@ set_mtrr(unsigned int reg, unsigned long base, unsigned long size, mtrr_type typ ...@@ -265,6 +269,8 @@ set_mtrr(unsigned int reg, unsigned long base, unsigned long size, mtrr_type typ
*/ */
if (reg != ~0U) if (reg != ~0U)
mtrr_if->set(reg, base, size, type); mtrr_if->set(reg, base, size, type);
else if (!mtrr_aps_delayed_init)
mtrr_if->set_all();
/* Wait for the others */ /* Wait for the others */
while (atomic_read(&data.count)) while (atomic_read(&data.count))
...@@ -721,9 +727,7 @@ void __init mtrr_bp_init(void) ...@@ -721,9 +727,7 @@ void __init mtrr_bp_init(void)
void mtrr_ap_init(void) void mtrr_ap_init(void)
{ {
unsigned long flags; if (!use_intel() || mtrr_aps_delayed_init)
if (!mtrr_if || !use_intel())
return; return;
/* /*
* Ideally we should hold mtrr_mutex here to avoid mtrr entries * Ideally we should hold mtrr_mutex here to avoid mtrr entries
...@@ -738,11 +742,7 @@ void mtrr_ap_init(void) ...@@ -738,11 +742,7 @@ void mtrr_ap_init(void)
* 2. cpu hotadd time. We let mtrr_add/del_page hold cpuhotplug * 2. cpu hotadd time. We let mtrr_add/del_page hold cpuhotplug
* lock to prevent mtrr entry changes * lock to prevent mtrr entry changes
*/ */
local_irq_save(flags); set_mtrr(~0U, 0, 0, 0);
mtrr_if->set_all();
local_irq_restore(flags);
} }
/** /**
...@@ -753,6 +753,34 @@ void mtrr_save_state(void) ...@@ -753,6 +753,34 @@ void mtrr_save_state(void)
smp_call_function_single(0, mtrr_save_fixed_ranges, NULL, 1); smp_call_function_single(0, mtrr_save_fixed_ranges, NULL, 1);
} }
void set_mtrr_aps_delayed_init(void)
{
if (!use_intel())
return;
mtrr_aps_delayed_init = 1;
}
/*
* MTRR initialization for all AP's
*/
void mtrr_aps_init(void)
{
if (!use_intel())
return;
set_mtrr(~0U, 0, 0, 0);
mtrr_aps_delayed_init = 0;
}
void mtrr_bp_restore(void)
{
if (!use_intel())
return;
mtrr_if->set_all();
}
static int __init mtrr_init_finialize(void) static int __init mtrr_init_finialize(void)
{ {
if (!mtrr_if) if (!mtrr_if)
......
...@@ -1116,9 +1116,22 @@ void __init native_smp_prepare_cpus(unsigned int max_cpus) ...@@ -1116,9 +1116,22 @@ void __init native_smp_prepare_cpus(unsigned int max_cpus)
if (is_uv_system()) if (is_uv_system())
uv_system_init(); uv_system_init();
set_mtrr_aps_delayed_init();
out: out:
preempt_enable(); preempt_enable();
} }
void arch_enable_nonboot_cpus_begin(void)
{
set_mtrr_aps_delayed_init();
}
void arch_enable_nonboot_cpus_end(void)
{
mtrr_aps_init();
}
/* /*
* Early setup to make printk work. * Early setup to make printk work.
*/ */
...@@ -1140,6 +1153,7 @@ void __init native_smp_cpus_done(unsigned int max_cpus) ...@@ -1140,6 +1153,7 @@ void __init native_smp_cpus_done(unsigned int max_cpus)
setup_ioapic_dest(); setup_ioapic_dest();
#endif #endif
check_nmi_watchdog(); check_nmi_watchdog();
mtrr_aps_init();
} }
static int __initdata setup_possible_cpus = -1; static int __initdata setup_possible_cpus = -1;
......
...@@ -242,7 +242,7 @@ static void __restore_processor_state(struct saved_context *ctxt) ...@@ -242,7 +242,7 @@ static void __restore_processor_state(struct saved_context *ctxt)
fix_processor_context(); fix_processor_context();
do_fpu_end(); do_fpu_end();
mtrr_ap_init(); mtrr_bp_restore();
#ifdef CONFIG_X86_OLD_MCE #ifdef CONFIG_X86_OLD_MCE
mcheck_init(&boot_cpu_data); mcheck_init(&boot_cpu_data);
......
...@@ -413,6 +413,14 @@ int disable_nonboot_cpus(void) ...@@ -413,6 +413,14 @@ int disable_nonboot_cpus(void)
return error; return error;
} }
void __weak arch_enable_nonboot_cpus_begin(void)
{
}
void __weak arch_enable_nonboot_cpus_end(void)
{
}
void __ref enable_nonboot_cpus(void) void __ref enable_nonboot_cpus(void)
{ {
int cpu, error; int cpu, error;
...@@ -424,6 +432,9 @@ void __ref enable_nonboot_cpus(void) ...@@ -424,6 +432,9 @@ void __ref enable_nonboot_cpus(void)
goto out; goto out;
printk("Enabling non-boot CPUs ...\n"); printk("Enabling non-boot CPUs ...\n");
arch_enable_nonboot_cpus_begin();
for_each_cpu(cpu, frozen_cpus) { for_each_cpu(cpu, frozen_cpus) {
error = _cpu_up(cpu, 1); error = _cpu_up(cpu, 1);
if (!error) { if (!error) {
...@@ -432,6 +443,9 @@ void __ref enable_nonboot_cpus(void) ...@@ -432,6 +443,9 @@ void __ref enable_nonboot_cpus(void)
} }
printk(KERN_WARNING "Error taking CPU%d up: %d\n", cpu, error); printk(KERN_WARNING "Error taking CPU%d up: %d\n", cpu, error);
} }
arch_enable_nonboot_cpus_end();
cpumask_clear(frozen_cpus); cpumask_clear(frozen_cpus);
out: out:
cpu_maps_update_done(); cpu_maps_update_done();
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment