Commit 59b60185 authored by Linus Torvalds's avatar Linus Torvalds

Merge branch 'timers-nohz-for-linus' of git://git.kernel.org/pub/scm/linux/kernel/git/tip/tip

Pull nohz updates from Ingo Molnar:
 "The main changes in this cycle relate to fixing another bad (but
  sporadic and hard to detect) interaction between the dynticks
  scheduler tick and hrtimers, plus related improvements to better
  detection and handling of similar problems - by Frédéric Weisbecker"

* 'timers-nohz-for-linus' of git://git.kernel.org/pub/scm/linux/kernel/git/tip/tip:
  nohz: Fix spurious warning when hrtimer and clockevent get out of sync
  nohz: Fix buggy tick delay on IRQ storms
  nohz: Reset next_tick cache even when the timer has no regs
  nohz: Fix collision between tick and other hrtimers, again
  nohz: Add hrtimer sanity check
parents 9bd42183 d4af6d93
...@@ -150,6 +150,12 @@ static void tick_sched_handle(struct tick_sched *ts, struct pt_regs *regs) ...@@ -150,6 +150,12 @@ static void tick_sched_handle(struct tick_sched *ts, struct pt_regs *regs)
touch_softlockup_watchdog_sched(); touch_softlockup_watchdog_sched();
if (is_idle_task(current)) if (is_idle_task(current))
ts->idle_jiffies++; ts->idle_jiffies++;
/*
* In case the current tick fired too early past its expected
* expiration, make sure we don't bypass the next clock reprogramming
* to the same deadline.
*/
ts->next_tick = 0;
} }
#endif #endif
update_process_times(user_mode(regs)); update_process_times(user_mode(regs));
...@@ -660,6 +666,12 @@ static void tick_nohz_restart(struct tick_sched *ts, ktime_t now) ...@@ -660,6 +666,12 @@ static void tick_nohz_restart(struct tick_sched *ts, ktime_t now)
hrtimer_start_expires(&ts->sched_timer, HRTIMER_MODE_ABS_PINNED); hrtimer_start_expires(&ts->sched_timer, HRTIMER_MODE_ABS_PINNED);
else else
tick_program_event(hrtimer_get_expires(&ts->sched_timer), 1); tick_program_event(hrtimer_get_expires(&ts->sched_timer), 1);
/*
* Reset to make sure next tick stop doesn't get fooled by past
* cached clock deadline.
*/
ts->next_tick = 0;
} }
static ktime_t tick_nohz_stop_sched_tick(struct tick_sched *ts, static ktime_t tick_nohz_stop_sched_tick(struct tick_sched *ts,
...@@ -701,8 +713,6 @@ static ktime_t tick_nohz_stop_sched_tick(struct tick_sched *ts, ...@@ -701,8 +713,6 @@ static ktime_t tick_nohz_stop_sched_tick(struct tick_sched *ts,
*/ */
delta = next_tick - basemono; delta = next_tick - basemono;
if (delta <= (u64)TICK_NSEC) { if (delta <= (u64)TICK_NSEC) {
tick = 0;
/* /*
* Tell the timer code that the base is not idle, i.e. undo * Tell the timer code that the base is not idle, i.e. undo
* the effect of get_next_timer_interrupt(): * the effect of get_next_timer_interrupt():
...@@ -712,23 +722,8 @@ static ktime_t tick_nohz_stop_sched_tick(struct tick_sched *ts, ...@@ -712,23 +722,8 @@ static ktime_t tick_nohz_stop_sched_tick(struct tick_sched *ts,
* We've not stopped the tick yet, and there's a timer in the * We've not stopped the tick yet, and there's a timer in the
* next period, so no point in stopping it either, bail. * next period, so no point in stopping it either, bail.
*/ */
if (!ts->tick_stopped) if (!ts->tick_stopped) {
goto out; tick = 0;
/*
* If, OTOH, we did stop it, but there's a pending (expired)
* timer reprogram the timer hardware to fire now.
*
* We will not restart the tick proper, just prod the timer
* hardware into firing an interrupt to process the pending
* timers. Just like tick_irq_exit() will not restart the tick
* for 'normal' interrupts.
*
* Only once we exit the idle loop will we re-enable the tick,
* see tick_nohz_idle_exit().
*/
if (delta == 0) {
tick_nohz_restart(ts, now);
goto out; goto out;
} }
} }
...@@ -771,8 +766,16 @@ static ktime_t tick_nohz_stop_sched_tick(struct tick_sched *ts, ...@@ -771,8 +766,16 @@ static ktime_t tick_nohz_stop_sched_tick(struct tick_sched *ts,
tick = expires; tick = expires;
/* Skip reprogram of event if its not changed */ /* Skip reprogram of event if its not changed */
if (ts->tick_stopped && (expires == dev->next_event)) if (ts->tick_stopped && (expires == ts->next_tick)) {
goto out; /* Sanity check: make sure clockevent is actually programmed */
if (tick == KTIME_MAX || ts->next_tick == hrtimer_get_expires(&ts->sched_timer))
goto out;
WARN_ON_ONCE(1);
printk_once("basemono: %llu ts->next_tick: %llu dev->next_event: %llu timer->active: %d timer->expires: %llu\n",
basemono, ts->next_tick, dev->next_event,
hrtimer_active(&ts->sched_timer), hrtimer_get_expires(&ts->sched_timer));
}
/* /*
* nohz_stop_sched_tick can be called several times before * nohz_stop_sched_tick can be called several times before
...@@ -790,6 +793,8 @@ static ktime_t tick_nohz_stop_sched_tick(struct tick_sched *ts, ...@@ -790,6 +793,8 @@ static ktime_t tick_nohz_stop_sched_tick(struct tick_sched *ts,
trace_tick_stop(1, TICK_DEP_MASK_NONE); trace_tick_stop(1, TICK_DEP_MASK_NONE);
} }
ts->next_tick = tick;
/* /*
* If the expiration time == KTIME_MAX, then we simply stop * If the expiration time == KTIME_MAX, then we simply stop
* the tick timer. * the tick timer.
...@@ -800,12 +805,17 @@ static ktime_t tick_nohz_stop_sched_tick(struct tick_sched *ts, ...@@ -800,12 +805,17 @@ static ktime_t tick_nohz_stop_sched_tick(struct tick_sched *ts,
goto out; goto out;
} }
hrtimer_set_expires(&ts->sched_timer, tick);
if (ts->nohz_mode == NOHZ_MODE_HIGHRES) if (ts->nohz_mode == NOHZ_MODE_HIGHRES)
hrtimer_start(&ts->sched_timer, tick, HRTIMER_MODE_ABS_PINNED); hrtimer_start_expires(&ts->sched_timer, HRTIMER_MODE_ABS_PINNED);
else else
tick_program_event(tick, 1); tick_program_event(tick, 1);
out: out:
/* Update the estimated sleep length */ /*
* Update the estimated sleep length until the next timer
* (not only the tick).
*/
ts->sleep_length = ktime_sub(dev->next_event, now); ts->sleep_length = ktime_sub(dev->next_event, now);
return tick; return tick;
} }
...@@ -863,6 +873,11 @@ static bool can_stop_idle_tick(int cpu, struct tick_sched *ts) ...@@ -863,6 +873,11 @@ static bool can_stop_idle_tick(int cpu, struct tick_sched *ts)
if (unlikely(!cpu_online(cpu))) { if (unlikely(!cpu_online(cpu))) {
if (cpu == tick_do_timer_cpu) if (cpu == tick_do_timer_cpu)
tick_do_timer_cpu = TICK_DO_TIMER_NONE; tick_do_timer_cpu = TICK_DO_TIMER_NONE;
/*
* Make sure the CPU doesn't get fooled by obsolete tick
* deadline if it comes back online later.
*/
ts->next_tick = 0;
return false; return false;
} }
...@@ -1173,6 +1188,8 @@ static enum hrtimer_restart tick_sched_timer(struct hrtimer *timer) ...@@ -1173,6 +1188,8 @@ static enum hrtimer_restart tick_sched_timer(struct hrtimer *timer)
*/ */
if (regs) if (regs)
tick_sched_handle(ts, regs); tick_sched_handle(ts, regs);
else
ts->next_tick = 0;
/* No need to reprogram if we are in idle or full dynticks mode */ /* No need to reprogram if we are in idle or full dynticks mode */
if (unlikely(ts->tick_stopped)) if (unlikely(ts->tick_stopped))
......
...@@ -27,6 +27,7 @@ enum tick_nohz_mode { ...@@ -27,6 +27,7 @@ enum tick_nohz_mode {
* timer is modified for nohz sleeps. This is necessary * timer is modified for nohz sleeps. This is necessary
* to resume the tick timer operation in the timeline * to resume the tick timer operation in the timeline
* when the CPU returns from nohz sleep. * when the CPU returns from nohz sleep.
* @next_tick: Next tick to be fired when in dynticks mode.
* @tick_stopped: Indicator that the idle tick has been stopped * @tick_stopped: Indicator that the idle tick has been stopped
* @idle_jiffies: jiffies at the entry to idle for idle time accounting * @idle_jiffies: jiffies at the entry to idle for idle time accounting
* @idle_calls: Total number of idle calls * @idle_calls: Total number of idle calls
...@@ -44,6 +45,7 @@ struct tick_sched { ...@@ -44,6 +45,7 @@ struct tick_sched {
unsigned long check_clocks; unsigned long check_clocks;
enum tick_nohz_mode nohz_mode; enum tick_nohz_mode nohz_mode;
ktime_t last_tick; ktime_t last_tick;
ktime_t next_tick;
int inidle; int inidle;
int tick_stopped; int tick_stopped;
unsigned long idle_jiffies; unsigned long idle_jiffies;
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment