Commit 68bb7bfb authored by K. Y. Srinivasan's avatar K. Y. Srinivasan Committed by Thomas Gleixner

X86/Hyper-V: Enable IPI enlightenments

Hyper-V supports hypercalls to implement IPI; use them.
Signed-off-by: default avatarK. Y. Srinivasan <kys@microsoft.com>
Signed-off-by: default avatarThomas Gleixner <tglx@linutronix.de>
Reviewed-by: default avatarMichael Kelley <mikelley@microsoft.com>
Cc: olaf@aepfle.de
Cc: sthemmin@microsoft.com
Cc: gregkh@linuxfoundation.org
Cc: jasowang@redhat.com
Cc: Michael.H.Kelley@microsoft.com
Cc: hpa@zytor.com
Cc: apw@canonical.com
Cc: devel@linuxdriverproject.org
Cc: vkuznets@redhat.com
Link: https://lkml.kernel.org/r/20180516215334.6547-2-kys@linuxonhyperv.com
parent 6b48cb5f
...@@ -33,6 +33,8 @@ ...@@ -33,6 +33,8 @@
#ifdef CONFIG_X86_64 #ifdef CONFIG_X86_64
#if IS_ENABLED(CONFIG_HYPERV) #if IS_ENABLED(CONFIG_HYPERV)
static struct apic orig_apic;
static u64 hv_apic_icr_read(void) static u64 hv_apic_icr_read(void)
{ {
u64 reg_val; u64 reg_val;
...@@ -88,8 +90,123 @@ static void hv_apic_eoi_write(u32 reg, u32 val) ...@@ -88,8 +90,123 @@ static void hv_apic_eoi_write(u32 reg, u32 val)
wrmsr(HV_X64_MSR_EOI, val, 0); wrmsr(HV_X64_MSR_EOI, val, 0);
} }
/*
* IPI implementation on Hyper-V.
*/
static bool __send_ipi_mask(const struct cpumask *mask, int vector)
{
int cur_cpu, vcpu;
struct ipi_arg_non_ex **arg;
struct ipi_arg_non_ex *ipi_arg;
int ret = 1;
unsigned long flags;
if (cpumask_empty(mask))
return true;
if (!hv_hypercall_pg)
return false;
if ((vector < HV_IPI_LOW_VECTOR) || (vector > HV_IPI_HIGH_VECTOR))
return false;
local_irq_save(flags);
arg = (struct ipi_arg_non_ex **)this_cpu_ptr(hyperv_pcpu_input_arg);
ipi_arg = *arg;
if (unlikely(!ipi_arg))
goto ipi_mask_done;
ipi_arg->vector = vector;
ipi_arg->reserved = 0;
ipi_arg->cpu_mask = 0;
for_each_cpu(cur_cpu, mask) {
vcpu = hv_cpu_number_to_vp_number(cur_cpu);
/*
* This particular version of the IPI hypercall can
* only target upto 64 CPUs.
*/
if (vcpu >= 64)
goto ipi_mask_done;
__set_bit(vcpu, (unsigned long *)&ipi_arg->cpu_mask);
}
ret = hv_do_hypercall(HVCALL_SEND_IPI, ipi_arg, NULL);
ipi_mask_done:
local_irq_restore(flags);
return ((ret == 0) ? true : false);
}
static bool __send_ipi_one(int cpu, int vector)
{
struct cpumask mask = CPU_MASK_NONE;
cpumask_set_cpu(cpu, &mask);
return __send_ipi_mask(&mask, vector);
}
static void hv_send_ipi(int cpu, int vector)
{
if (!__send_ipi_one(cpu, vector))
orig_apic.send_IPI(cpu, vector);
}
static void hv_send_ipi_mask(const struct cpumask *mask, int vector)
{
if (!__send_ipi_mask(mask, vector))
orig_apic.send_IPI_mask(mask, vector);
}
static void hv_send_ipi_mask_allbutself(const struct cpumask *mask, int vector)
{
unsigned int this_cpu = smp_processor_id();
struct cpumask new_mask;
const struct cpumask *local_mask;
cpumask_copy(&new_mask, mask);
cpumask_clear_cpu(this_cpu, &new_mask);
local_mask = &new_mask;
if (!__send_ipi_mask(local_mask, vector))
orig_apic.send_IPI_mask_allbutself(mask, vector);
}
static void hv_send_ipi_allbutself(int vector)
{
hv_send_ipi_mask_allbutself(cpu_online_mask, vector);
}
static void hv_send_ipi_all(int vector)
{
if (!__send_ipi_mask(cpu_online_mask, vector))
orig_apic.send_IPI_all(vector);
}
static void hv_send_ipi_self(int vector)
{
if (!__send_ipi_one(smp_processor_id(), vector))
orig_apic.send_IPI_self(vector);
}
void __init hv_apic_init(void) void __init hv_apic_init(void)
{ {
if (ms_hyperv.hints & HV_X64_CLUSTER_IPI_RECOMMENDED) {
pr_info("Hyper-V: Using IPI hypercalls\n");
/*
* Set the IPI entry points.
*/
orig_apic = *apic;
apic->send_IPI = hv_send_ipi;
apic->send_IPI_mask = hv_send_ipi_mask;
apic->send_IPI_mask_allbutself = hv_send_ipi_mask_allbutself;
apic->send_IPI_allbutself = hv_send_ipi_allbutself;
apic->send_IPI_all = hv_send_ipi_all;
apic->send_IPI_self = hv_send_ipi_self;
}
if (ms_hyperv.hints & HV_X64_APIC_ACCESS_RECOMMENDED) { if (ms_hyperv.hints & HV_X64_APIC_ACCESS_RECOMMENDED) {
pr_info("Hyper-V: Using MSR based APIC access\n"); pr_info("Hyper-V: Using MSR based APIC access\n");
apic_set_eoi_write(hv_apic_eoi_write); apic_set_eoi_write(hv_apic_eoi_write);
......
...@@ -91,12 +91,19 @@ EXPORT_SYMBOL_GPL(hv_vp_index); ...@@ -91,12 +91,19 @@ EXPORT_SYMBOL_GPL(hv_vp_index);
struct hv_vp_assist_page **hv_vp_assist_page; struct hv_vp_assist_page **hv_vp_assist_page;
EXPORT_SYMBOL_GPL(hv_vp_assist_page); EXPORT_SYMBOL_GPL(hv_vp_assist_page);
void __percpu **hyperv_pcpu_input_arg;
EXPORT_SYMBOL_GPL(hyperv_pcpu_input_arg);
u32 hv_max_vp_index; u32 hv_max_vp_index;
static int hv_cpu_init(unsigned int cpu) static int hv_cpu_init(unsigned int cpu)
{ {
u64 msr_vp_index; u64 msr_vp_index;
struct hv_vp_assist_page **hvp = &hv_vp_assist_page[smp_processor_id()]; struct hv_vp_assist_page **hvp = &hv_vp_assist_page[smp_processor_id()];
void **input_arg;
input_arg = (void **)this_cpu_ptr(hyperv_pcpu_input_arg);
*input_arg = page_address(alloc_page(GFP_KERNEL));
hv_get_vp_index(msr_vp_index); hv_get_vp_index(msr_vp_index);
...@@ -217,6 +224,16 @@ static int hv_cpu_die(unsigned int cpu) ...@@ -217,6 +224,16 @@ static int hv_cpu_die(unsigned int cpu)
{ {
struct hv_reenlightenment_control re_ctrl; struct hv_reenlightenment_control re_ctrl;
unsigned int new_cpu; unsigned int new_cpu;
unsigned long flags;
void **input_arg;
void *input_pg = NULL;
local_irq_save(flags);
input_arg = (void **)this_cpu_ptr(hyperv_pcpu_input_arg);
input_pg = *input_arg;
*input_arg = NULL;
local_irq_restore(flags);
free_page((unsigned long)input_pg);
if (hv_vp_assist_page && hv_vp_assist_page[cpu]) if (hv_vp_assist_page && hv_vp_assist_page[cpu])
wrmsrl(HV_X64_MSR_VP_ASSIST_PAGE, 0); wrmsrl(HV_X64_MSR_VP_ASSIST_PAGE, 0);
...@@ -260,6 +277,16 @@ void __init hyperv_init(void) ...@@ -260,6 +277,16 @@ void __init hyperv_init(void)
if ((ms_hyperv.features & required_msrs) != required_msrs) if ((ms_hyperv.features & required_msrs) != required_msrs)
return; return;
/*
* Allocate the per-CPU state for the hypercall input arg.
* If this allocation fails, we will not be able to setup
* (per-CPU) hypercall input page and thus this failure is
* fatal on Hyper-V.
*/
hyperv_pcpu_input_arg = alloc_percpu(void *);
BUG_ON(hyperv_pcpu_input_arg == NULL);
/* Allocate percpu VP index */ /* Allocate percpu VP index */
hv_vp_index = kmalloc_array(num_possible_cpus(), sizeof(*hv_vp_index), hv_vp_index = kmalloc_array(num_possible_cpus(), sizeof(*hv_vp_index),
GFP_KERNEL); GFP_KERNEL);
......
...@@ -164,6 +164,11 @@ ...@@ -164,6 +164,11 @@
*/ */
#define HV_X64_DEPRECATING_AEOI_RECOMMENDED (1 << 9) #define HV_X64_DEPRECATING_AEOI_RECOMMENDED (1 << 9)
/*
* Recommend using cluster IPI hypercalls.
*/
#define HV_X64_CLUSTER_IPI_RECOMMENDED (1 << 10)
/* Recommend using the newer ExProcessorMasks interface */ /* Recommend using the newer ExProcessorMasks interface */
#define HV_X64_EX_PROCESSOR_MASKS_RECOMMENDED (1 << 11) #define HV_X64_EX_PROCESSOR_MASKS_RECOMMENDED (1 << 11)
...@@ -329,10 +334,14 @@ struct hv_tsc_emulation_status { ...@@ -329,10 +334,14 @@ struct hv_tsc_emulation_status {
#define HV_X64_MSR_HYPERCALL_PAGE_ADDRESS_MASK \ #define HV_X64_MSR_HYPERCALL_PAGE_ADDRESS_MASK \
(~((1ull << HV_X64_MSR_HYPERCALL_PAGE_ADDRESS_SHIFT) - 1)) (~((1ull << HV_X64_MSR_HYPERCALL_PAGE_ADDRESS_SHIFT) - 1))
#define HV_IPI_LOW_VECTOR 0x10
#define HV_IPI_HIGH_VECTOR 0xff
/* Declare the various hypercall operations. */ /* Declare the various hypercall operations. */
#define HVCALL_FLUSH_VIRTUAL_ADDRESS_SPACE 0x0002 #define HVCALL_FLUSH_VIRTUAL_ADDRESS_SPACE 0x0002
#define HVCALL_FLUSH_VIRTUAL_ADDRESS_LIST 0x0003 #define HVCALL_FLUSH_VIRTUAL_ADDRESS_LIST 0x0003
#define HVCALL_NOTIFY_LONG_SPIN_WAIT 0x0008 #define HVCALL_NOTIFY_LONG_SPIN_WAIT 0x0008
#define HVCALL_SEND_IPI 0x000b
#define HVCALL_FLUSH_VIRTUAL_ADDRESS_SPACE_EX 0x0013 #define HVCALL_FLUSH_VIRTUAL_ADDRESS_SPACE_EX 0x0013
#define HVCALL_FLUSH_VIRTUAL_ADDRESS_LIST_EX 0x0014 #define HVCALL_FLUSH_VIRTUAL_ADDRESS_LIST_EX 0x0014
#define HVCALL_POST_MESSAGE 0x005c #define HVCALL_POST_MESSAGE 0x005c
...@@ -706,4 +715,10 @@ struct hv_enlightened_vmcs { ...@@ -706,4 +715,10 @@ struct hv_enlightened_vmcs {
#define HV_STIMER_AUTOENABLE (1ULL << 3) #define HV_STIMER_AUTOENABLE (1ULL << 3)
#define HV_STIMER_SINT(config) (__u8)(((config) >> 16) & 0x0F) #define HV_STIMER_SINT(config) (__u8)(((config) >> 16) & 0x0F)
struct ipi_arg_non_ex {
u32 vector;
u32 reserved;
u64 cpu_mask;
};
#endif #endif
...@@ -122,6 +122,7 @@ static inline void hv_disable_stimer0_percpu_irq(int irq) {} ...@@ -122,6 +122,7 @@ static inline void hv_disable_stimer0_percpu_irq(int irq) {}
#if IS_ENABLED(CONFIG_HYPERV) #if IS_ENABLED(CONFIG_HYPERV)
extern struct clocksource *hyperv_cs; extern struct clocksource *hyperv_cs;
extern void *hv_hypercall_pg; extern void *hv_hypercall_pg;
extern void __percpu **hyperv_pcpu_input_arg;
static inline u64 hv_do_hypercall(u64 control, void *input, void *output) static inline u64 hv_do_hypercall(u64 control, void *input, void *output)
{ {
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment