Commit c0ad4aa4 authored by Peter Zijlstra's avatar Peter Zijlstra Committed by Ingo Molnar

sched/fair: Robustify CFS-bandwidth timer locking

Traditionally hrtimer callbacks were run with IRQs disabled, but with
the introduction of HRTIMER_MODE_SOFT it is possible they run from
SoftIRQ context, which does _NOT_ have IRQs disabled.

Allow for the CFS bandwidth timers (period_timer and slack_timer) to
be ran from SoftIRQ context; this entails removing the assumption that
IRQs are already disabled from the locking.

While mainline doesn't strictly need this, -RT forces all timers not
explicitly marked with MODE_HARD into MODE_SOFT and trips over this.
And marking these timers as MODE_HARD doesn't make sense as they're
not required for RT operation and can potentially be quite expensive.
Reported-by: default avatarTom Putzeys <tom.putzeys@be.atlascopco.com>
Tested-by: default avatarMike Galbraith <efault@gmx.de>
Signed-off-by: default avatarPeter Zijlstra (Intel) <peterz@infradead.org>
Cc: Linus Torvalds <torvalds@linux-foundation.org>
Cc: Peter Zijlstra <peterz@infradead.org>
Cc: Sebastian Andrzej Siewior <bigeasy@linutronix.de>
Cc: Thomas Gleixner <tglx@linutronix.de>
Link: https://lkml.kernel.org/r/20190107125231.GE14122@hirez.programming.kicks-ass.netSigned-off-by: default avatarIngo Molnar <mingo@kernel.org>
parent f8a696f2
...@@ -4565,7 +4565,7 @@ static u64 distribute_cfs_runtime(struct cfs_bandwidth *cfs_b, ...@@ -4565,7 +4565,7 @@ static u64 distribute_cfs_runtime(struct cfs_bandwidth *cfs_b,
struct rq *rq = rq_of(cfs_rq); struct rq *rq = rq_of(cfs_rq);
struct rq_flags rf; struct rq_flags rf;
rq_lock(rq, &rf); rq_lock_irqsave(rq, &rf);
if (!cfs_rq_throttled(cfs_rq)) if (!cfs_rq_throttled(cfs_rq))
goto next; goto next;
...@@ -4582,7 +4582,7 @@ static u64 distribute_cfs_runtime(struct cfs_bandwidth *cfs_b, ...@@ -4582,7 +4582,7 @@ static u64 distribute_cfs_runtime(struct cfs_bandwidth *cfs_b,
unthrottle_cfs_rq(cfs_rq); unthrottle_cfs_rq(cfs_rq);
next: next:
rq_unlock(rq, &rf); rq_unlock_irqrestore(rq, &rf);
if (!remaining) if (!remaining)
break; break;
...@@ -4598,7 +4598,7 @@ static u64 distribute_cfs_runtime(struct cfs_bandwidth *cfs_b, ...@@ -4598,7 +4598,7 @@ static u64 distribute_cfs_runtime(struct cfs_bandwidth *cfs_b,
* period the timer is deactivated until scheduling resumes; cfs_b->idle is * period the timer is deactivated until scheduling resumes; cfs_b->idle is
* used to track this state. * used to track this state.
*/ */
static int do_sched_cfs_period_timer(struct cfs_bandwidth *cfs_b, int overrun) static int do_sched_cfs_period_timer(struct cfs_bandwidth *cfs_b, int overrun, unsigned long flags)
{ {
u64 runtime, runtime_expires; u64 runtime, runtime_expires;
int throttled; int throttled;
...@@ -4640,11 +4640,11 @@ static int do_sched_cfs_period_timer(struct cfs_bandwidth *cfs_b, int overrun) ...@@ -4640,11 +4640,11 @@ static int do_sched_cfs_period_timer(struct cfs_bandwidth *cfs_b, int overrun)
while (throttled && cfs_b->runtime > 0 && !cfs_b->distribute_running) { while (throttled && cfs_b->runtime > 0 && !cfs_b->distribute_running) {
runtime = cfs_b->runtime; runtime = cfs_b->runtime;
cfs_b->distribute_running = 1; cfs_b->distribute_running = 1;
raw_spin_unlock(&cfs_b->lock); raw_spin_unlock_irqrestore(&cfs_b->lock, flags);
/* we can't nest cfs_b->lock while distributing bandwidth */ /* we can't nest cfs_b->lock while distributing bandwidth */
runtime = distribute_cfs_runtime(cfs_b, runtime, runtime = distribute_cfs_runtime(cfs_b, runtime,
runtime_expires); runtime_expires);
raw_spin_lock(&cfs_b->lock); raw_spin_lock_irqsave(&cfs_b->lock, flags);
cfs_b->distribute_running = 0; cfs_b->distribute_running = 0;
throttled = !list_empty(&cfs_b->throttled_cfs_rq); throttled = !list_empty(&cfs_b->throttled_cfs_rq);
...@@ -4753,17 +4753,18 @@ static __always_inline void return_cfs_rq_runtime(struct cfs_rq *cfs_rq) ...@@ -4753,17 +4753,18 @@ static __always_inline void return_cfs_rq_runtime(struct cfs_rq *cfs_rq)
static void do_sched_cfs_slack_timer(struct cfs_bandwidth *cfs_b) static void do_sched_cfs_slack_timer(struct cfs_bandwidth *cfs_b)
{ {
u64 runtime = 0, slice = sched_cfs_bandwidth_slice(); u64 runtime = 0, slice = sched_cfs_bandwidth_slice();
unsigned long flags;
u64 expires; u64 expires;
/* confirm we're still not at a refresh boundary */ /* confirm we're still not at a refresh boundary */
raw_spin_lock(&cfs_b->lock); raw_spin_lock_irqsave(&cfs_b->lock, flags);
if (cfs_b->distribute_running) { if (cfs_b->distribute_running) {
raw_spin_unlock(&cfs_b->lock); raw_spin_unlock_irqrestore(&cfs_b->lock, flags);
return; return;
} }
if (runtime_refresh_within(cfs_b, min_bandwidth_expiration)) { if (runtime_refresh_within(cfs_b, min_bandwidth_expiration)) {
raw_spin_unlock(&cfs_b->lock); raw_spin_unlock_irqrestore(&cfs_b->lock, flags);
return; return;
} }
...@@ -4774,18 +4775,18 @@ static void do_sched_cfs_slack_timer(struct cfs_bandwidth *cfs_b) ...@@ -4774,18 +4775,18 @@ static void do_sched_cfs_slack_timer(struct cfs_bandwidth *cfs_b)
if (runtime) if (runtime)
cfs_b->distribute_running = 1; cfs_b->distribute_running = 1;
raw_spin_unlock(&cfs_b->lock); raw_spin_unlock_irqrestore(&cfs_b->lock, flags);
if (!runtime) if (!runtime)
return; return;
runtime = distribute_cfs_runtime(cfs_b, runtime, expires); runtime = distribute_cfs_runtime(cfs_b, runtime, expires);
raw_spin_lock(&cfs_b->lock); raw_spin_lock_irqsave(&cfs_b->lock, flags);
if (expires == cfs_b->runtime_expires) if (expires == cfs_b->runtime_expires)
lsub_positive(&cfs_b->runtime, runtime); lsub_positive(&cfs_b->runtime, runtime);
cfs_b->distribute_running = 0; cfs_b->distribute_running = 0;
raw_spin_unlock(&cfs_b->lock); raw_spin_unlock_irqrestore(&cfs_b->lock, flags);
} }
/* /*
...@@ -4863,20 +4864,21 @@ static enum hrtimer_restart sched_cfs_period_timer(struct hrtimer *timer) ...@@ -4863,20 +4864,21 @@ static enum hrtimer_restart sched_cfs_period_timer(struct hrtimer *timer)
{ {
struct cfs_bandwidth *cfs_b = struct cfs_bandwidth *cfs_b =
container_of(timer, struct cfs_bandwidth, period_timer); container_of(timer, struct cfs_bandwidth, period_timer);
unsigned long flags;
int overrun; int overrun;
int idle = 0; int idle = 0;
raw_spin_lock(&cfs_b->lock); raw_spin_lock_irqsave(&cfs_b->lock, flags);
for (;;) { for (;;) {
overrun = hrtimer_forward_now(timer, cfs_b->period); overrun = hrtimer_forward_now(timer, cfs_b->period);
if (!overrun) if (!overrun)
break; break;
idle = do_sched_cfs_period_timer(cfs_b, overrun); idle = do_sched_cfs_period_timer(cfs_b, overrun, flags);
} }
if (idle) if (idle)
cfs_b->period_active = 0; cfs_b->period_active = 0;
raw_spin_unlock(&cfs_b->lock); raw_spin_unlock_irqrestore(&cfs_b->lock, flags);
return idle ? HRTIMER_NORESTART : HRTIMER_RESTART; return idle ? HRTIMER_NORESTART : HRTIMER_RESTART;
} }
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment