Commit 2ecd9d29 authored by Peter Zijlstra's avatar Peter Zijlstra Committed by Paolo Bonzini

sched, preempt_notifier: separate notifier registration from static_key inc/dec

Commit 1cde2930 ("sched/preempt: Add static_key() to preempt_notifiers")
had two problems.  First, the preempt-notifier API needs to sleep with the
addition of the static_key, we do however need to hold off preemption
while modifying the preempt notifier list, otherwise a preemption could
observe an inconsistent list state.  KVM correctly registers and
unregisters preempt notifiers with preemption disabled, so the sleep
caused dmesg splats.

Second, KVM registers and unregisters preemption notifiers very often
(in vcpu_load/vcpu_put).  With a single uniprocessor guest the static key
would move between 0 and 1 continuously, hitting the slow path on every
userspace exit.

To fix this, wrap the static_key inc/dec in a new API, and call it from
KVM.

Fixes: 1cde2930 ("sched/preempt: Add static_key() to preempt_notifiers")
Reported-by: default avatarPontus Fuchs <pontus.fuchs@gmail.com>
Reported-by: default avatarTakashi Iwai <tiwai@suse.de>
Tested-by: default avatarTakashi Iwai <tiwai@suse.de>
Signed-off-by: default avatarPeter Zijlstra (Intel) <peterz@infradead.org>
Signed-off-by: default avatarPaolo Bonzini <pbonzini@redhat.com>
parent 9bdc771f
...@@ -293,6 +293,8 @@ struct preempt_notifier { ...@@ -293,6 +293,8 @@ struct preempt_notifier {
struct preempt_ops *ops; struct preempt_ops *ops;
}; };
void preempt_notifier_inc(void);
void preempt_notifier_dec(void);
void preempt_notifier_register(struct preempt_notifier *notifier); void preempt_notifier_register(struct preempt_notifier *notifier);
void preempt_notifier_unregister(struct preempt_notifier *notifier); void preempt_notifier_unregister(struct preempt_notifier *notifier);
......
...@@ -2320,13 +2320,27 @@ void wake_up_new_task(struct task_struct *p) ...@@ -2320,13 +2320,27 @@ void wake_up_new_task(struct task_struct *p)
static struct static_key preempt_notifier_key = STATIC_KEY_INIT_FALSE; static struct static_key preempt_notifier_key = STATIC_KEY_INIT_FALSE;
void preempt_notifier_inc(void)
{
static_key_slow_inc(&preempt_notifier_key);
}
EXPORT_SYMBOL_GPL(preempt_notifier_inc);
void preempt_notifier_dec(void)
{
static_key_slow_dec(&preempt_notifier_key);
}
EXPORT_SYMBOL_GPL(preempt_notifier_dec);
/** /**
* preempt_notifier_register - tell me when current is being preempted & rescheduled * preempt_notifier_register - tell me when current is being preempted & rescheduled
* @notifier: notifier struct to register * @notifier: notifier struct to register
*/ */
void preempt_notifier_register(struct preempt_notifier *notifier) void preempt_notifier_register(struct preempt_notifier *notifier)
{ {
static_key_slow_inc(&preempt_notifier_key); if (!static_key_false(&preempt_notifier_key))
WARN(1, "registering preempt_notifier while notifiers disabled\n");
hlist_add_head(&notifier->link, &current->preempt_notifiers); hlist_add_head(&notifier->link, &current->preempt_notifiers);
} }
EXPORT_SYMBOL_GPL(preempt_notifier_register); EXPORT_SYMBOL_GPL(preempt_notifier_register);
...@@ -2340,7 +2354,6 @@ EXPORT_SYMBOL_GPL(preempt_notifier_register); ...@@ -2340,7 +2354,6 @@ EXPORT_SYMBOL_GPL(preempt_notifier_register);
void preempt_notifier_unregister(struct preempt_notifier *notifier) void preempt_notifier_unregister(struct preempt_notifier *notifier)
{ {
hlist_del(&notifier->link); hlist_del(&notifier->link);
static_key_slow_dec(&preempt_notifier_key);
} }
EXPORT_SYMBOL_GPL(preempt_notifier_unregister); EXPORT_SYMBOL_GPL(preempt_notifier_unregister);
......
...@@ -553,6 +553,8 @@ static struct kvm *kvm_create_vm(unsigned long type) ...@@ -553,6 +553,8 @@ static struct kvm *kvm_create_vm(unsigned long type)
list_add(&kvm->vm_list, &vm_list); list_add(&kvm->vm_list, &vm_list);
spin_unlock(&kvm_lock); spin_unlock(&kvm_lock);
preempt_notifier_inc();
return kvm; return kvm;
out_err: out_err:
...@@ -620,6 +622,7 @@ static void kvm_destroy_vm(struct kvm *kvm) ...@@ -620,6 +622,7 @@ static void kvm_destroy_vm(struct kvm *kvm)
cleanup_srcu_struct(&kvm->irq_srcu); cleanup_srcu_struct(&kvm->irq_srcu);
cleanup_srcu_struct(&kvm->srcu); cleanup_srcu_struct(&kvm->srcu);
kvm_arch_free_vm(kvm); kvm_arch_free_vm(kvm);
preempt_notifier_dec();
hardware_disable_all(); hardware_disable_all();
mmdrop(mm); mmdrop(mm);
} }
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment