Commit 90b5363a authored by Peter Zijlstra (Intel)'s avatar Peter Zijlstra (Intel) Committed by Thomas Gleixner

sched: Clean up scheduler_ipi()

The scheduler IPI has grown weird and wonderful over the years, time
for spring cleaning.

Move all the non-trivial stuff out of it and into a regular smp function
call IPI. This then reduces the schedule_ipi() to most of it's former NOP
glory and ensures to keep the interrupt vector lean and mean.

Aside of that avoiding the full irq_enter() in the x86 IPI implementation
is incorrect as scheduler_ipi() can be instrumented. To work around that
scheduler_ipi() had an irq_enter/exit() hack when heavy work was
pending. This is gone now.
Signed-off-by: default avatarPeter Zijlstra (Intel) <peterz@infradead.org>
Signed-off-by: default avatarThomas Gleixner <tglx@linutronix.de>
Reviewed-by: default avatarAlexandre Chartre <alexandre.chartre@oracle.com>
Link: https://lkml.kernel.org/r/20200505134058.361859938@linutronix.de
parent b1d1779e
...@@ -219,6 +219,13 @@ void update_rq_clock(struct rq *rq) ...@@ -219,6 +219,13 @@ void update_rq_clock(struct rq *rq)
update_rq_clock_task(rq, delta); update_rq_clock_task(rq, delta);
} }
static inline void
rq_csd_init(struct rq *rq, call_single_data_t *csd, smp_call_func_t func)
{
csd->flags = 0;
csd->func = func;
csd->info = rq;
}
#ifdef CONFIG_SCHED_HRTICK #ifdef CONFIG_SCHED_HRTICK
/* /*
...@@ -314,16 +321,14 @@ void hrtick_start(struct rq *rq, u64 delay) ...@@ -314,16 +321,14 @@ void hrtick_start(struct rq *rq, u64 delay)
hrtimer_start(&rq->hrtick_timer, ns_to_ktime(delay), hrtimer_start(&rq->hrtick_timer, ns_to_ktime(delay),
HRTIMER_MODE_REL_PINNED_HARD); HRTIMER_MODE_REL_PINNED_HARD);
} }
#endif /* CONFIG_SMP */ #endif /* CONFIG_SMP */
static void hrtick_rq_init(struct rq *rq) static void hrtick_rq_init(struct rq *rq)
{ {
#ifdef CONFIG_SMP #ifdef CONFIG_SMP
rq->hrtick_csd.flags = 0; rq_csd_init(rq, &rq->hrtick_csd, __hrtick_start);
rq->hrtick_csd.func = __hrtick_start;
rq->hrtick_csd.info = rq;
#endif #endif
hrtimer_init(&rq->hrtick_timer, CLOCK_MONOTONIC, HRTIMER_MODE_REL_HARD); hrtimer_init(&rq->hrtick_timer, CLOCK_MONOTONIC, HRTIMER_MODE_REL_HARD);
rq->hrtick_timer.function = hrtick; rq->hrtick_timer.function = hrtick;
} }
...@@ -650,6 +655,16 @@ static inline bool got_nohz_idle_kick(void) ...@@ -650,6 +655,16 @@ static inline bool got_nohz_idle_kick(void)
return false; return false;
} }
static void nohz_csd_func(void *info)
{
struct rq *rq = info;
if (got_nohz_idle_kick()) {
rq->idle_balance = 1;
raise_softirq_irqoff(SCHED_SOFTIRQ);
}
}
#else /* CONFIG_NO_HZ_COMMON */ #else /* CONFIG_NO_HZ_COMMON */
static inline bool got_nohz_idle_kick(void) static inline bool got_nohz_idle_kick(void)
...@@ -2292,6 +2307,11 @@ void sched_ttwu_pending(void) ...@@ -2292,6 +2307,11 @@ void sched_ttwu_pending(void)
rq_unlock_irqrestore(rq, &rf); rq_unlock_irqrestore(rq, &rf);
} }
static void wake_csd_func(void *info)
{
sched_ttwu_pending();
}
void scheduler_ipi(void) void scheduler_ipi(void)
{ {
/* /*
...@@ -2300,34 +2320,6 @@ void scheduler_ipi(void) ...@@ -2300,34 +2320,6 @@ void scheduler_ipi(void)
* this IPI. * this IPI.
*/ */
preempt_fold_need_resched(); preempt_fold_need_resched();
if (llist_empty(&this_rq()->wake_list) && !got_nohz_idle_kick())
return;
/*
* Not all reschedule IPI handlers call irq_enter/irq_exit, since
* traditionally all their work was done from the interrupt return
* path. Now that we actually do some work, we need to make sure
* we do call them.
*
* Some archs already do call them, luckily irq_enter/exit nest
* properly.
*
* Arguably we should visit all archs and update all handlers,
* however a fair share of IPIs are still resched only so this would
* somewhat pessimize the simple resched case.
*/
irq_enter();
sched_ttwu_pending();
/*
* Check if someone kicked us for doing the nohz idle load balance.
*/
if (unlikely(got_nohz_idle_kick())) {
this_rq()->idle_balance = 1;
raise_softirq_irqoff(SCHED_SOFTIRQ);
}
irq_exit();
} }
static void ttwu_queue_remote(struct task_struct *p, int cpu, int wake_flags) static void ttwu_queue_remote(struct task_struct *p, int cpu, int wake_flags)
...@@ -2336,9 +2328,9 @@ static void ttwu_queue_remote(struct task_struct *p, int cpu, int wake_flags) ...@@ -2336,9 +2328,9 @@ static void ttwu_queue_remote(struct task_struct *p, int cpu, int wake_flags)
p->sched_remote_wakeup = !!(wake_flags & WF_MIGRATED); p->sched_remote_wakeup = !!(wake_flags & WF_MIGRATED);
if (llist_add(&p->wake_entry, &cpu_rq(cpu)->wake_list)) { if (llist_add(&p->wake_entry, &rq->wake_list)) {
if (!set_nr_if_polling(rq->idle)) if (!set_nr_if_polling(rq->idle))
smp_send_reschedule(cpu); smp_call_function_single_async(cpu, &rq->wake_csd);
else else
trace_sched_wake_idle_without_ipi(cpu); trace_sched_wake_idle_without_ipi(cpu);
} }
...@@ -6693,12 +6685,16 @@ void __init sched_init(void) ...@@ -6693,12 +6685,16 @@ void __init sched_init(void)
rq->avg_idle = 2*sysctl_sched_migration_cost; rq->avg_idle = 2*sysctl_sched_migration_cost;
rq->max_idle_balance_cost = sysctl_sched_migration_cost; rq->max_idle_balance_cost = sysctl_sched_migration_cost;
rq_csd_init(rq, &rq->wake_csd, wake_csd_func);
INIT_LIST_HEAD(&rq->cfs_tasks); INIT_LIST_HEAD(&rq->cfs_tasks);
rq_attach_root(rq, &def_root_domain); rq_attach_root(rq, &def_root_domain);
#ifdef CONFIG_NO_HZ_COMMON #ifdef CONFIG_NO_HZ_COMMON
rq->last_blocked_load_update_tick = jiffies; rq->last_blocked_load_update_tick = jiffies;
atomic_set(&rq->nohz_flags, 0); atomic_set(&rq->nohz_flags, 0);
rq_csd_init(rq, &rq->nohz_csd, nohz_csd_func);
#endif #endif
#endif /* CONFIG_SMP */ #endif /* CONFIG_SMP */
hrtick_rq_init(rq); hrtick_rq_init(rq);
......
...@@ -10000,12 +10000,11 @@ static void kick_ilb(unsigned int flags) ...@@ -10000,12 +10000,11 @@ static void kick_ilb(unsigned int flags)
return; return;
/* /*
* Use smp_send_reschedule() instead of resched_cpu(). * This way we generate an IPI on the target CPU which
* This way we generate a sched IPI on the target CPU which
* is idle. And the softirq performing nohz idle load balance * is idle. And the softirq performing nohz idle load balance
* will be run before returning from the IPI. * will be run before returning from the IPI.
*/ */
smp_send_reschedule(ilb_cpu); smp_call_function_single_async(ilb_cpu, &cpu_rq(ilb_cpu)->nohz_csd);
} }
/* /*
......
...@@ -889,6 +889,7 @@ struct rq { ...@@ -889,6 +889,7 @@ struct rq {
#ifdef CONFIG_SMP #ifdef CONFIG_SMP
unsigned long last_blocked_load_update_tick; unsigned long last_blocked_load_update_tick;
unsigned int has_blocked_load; unsigned int has_blocked_load;
call_single_data_t nohz_csd;
#endif /* CONFIG_SMP */ #endif /* CONFIG_SMP */
unsigned int nohz_tick_stopped; unsigned int nohz_tick_stopped;
atomic_t nohz_flags; atomic_t nohz_flags;
...@@ -978,7 +979,7 @@ struct rq { ...@@ -978,7 +979,7 @@ struct rq {
/* This is used to determine avg_idle's max value */ /* This is used to determine avg_idle's max value */
u64 max_idle_balance_cost; u64 max_idle_balance_cost;
#endif #endif /* CONFIG_SMP */
#ifdef CONFIG_IRQ_TIME_ACCOUNTING #ifdef CONFIG_IRQ_TIME_ACCOUNTING
u64 prev_irq_time; u64 prev_irq_time;
...@@ -1020,6 +1021,7 @@ struct rq { ...@@ -1020,6 +1021,7 @@ struct rq {
#endif #endif
#ifdef CONFIG_SMP #ifdef CONFIG_SMP
call_single_data_t wake_csd;
struct llist_head wake_list; struct llist_head wake_list;
#endif #endif
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment