Commit 08ae95f4 authored by Nicholas Piggin's avatar Nicholas Piggin Committed by Ingo Molnar

nohz_full: Allow the boot CPU to be nohz_full

Allow the boot CPU/CPU0 to be nohz_full. Have the boot CPU take the
do_timer duty during boot until a housekeeping CPU can take over.

This is supported when CONFIG_PM_SLEEP_SMP is not configured, or when
it is configured and the arch allows suspend on non-zero CPUs.

nohz_full has been trialed at a large supercomputer site and found to
significantly reduce jitter. In order to deploy it in production, they
need CPU0 to be nohz_full because their job control system requires
the application CPUs to start from 0, and the housekeeping CPUs are
placed higher. An equivalent job scheduling that uses CPU0 for
housekeeping could be achieved by modifying their system, but it is
preferable if nohz_full can support their environment without
modification.
Signed-off-by: default avatarNicholas Piggin <npiggin@gmail.com>
Signed-off-by: default avatarPeter Zijlstra (Intel) <peterz@infradead.org>
Cc: Frederic Weisbecker <fweisbec@gmail.com>
Cc: Linus Torvalds <torvalds@linux-foundation.org>
Cc: Peter Zijlstra <peterz@infradead.org>
Cc: Rafael J . Wysocki <rafael.j.wysocki@intel.com>
Cc: Thomas Gleixner <tglx@linutronix.de>
Cc: linuxppc-dev@lists.ozlabs.org
Link: https://lkml.kernel.org/r/20190411033448.20842-6-npiggin@gmail.comSigned-off-by: default avatarIngo Molnar <mingo@kernel.org>
parent 9219565a
...@@ -46,6 +46,14 @@ ktime_t tick_period; ...@@ -46,6 +46,14 @@ ktime_t tick_period;
* procedure also covers cpu hotplug. * procedure also covers cpu hotplug.
*/ */
int tick_do_timer_cpu __read_mostly = TICK_DO_TIMER_BOOT; int tick_do_timer_cpu __read_mostly = TICK_DO_TIMER_BOOT;
#ifdef CONFIG_NO_HZ_FULL
/*
* tick_do_timer_boot_cpu indicates the boot CPU temporarily owns
* tick_do_timer_cpu and it should be taken over by an eligible secondary
* when one comes online.
*/
static int tick_do_timer_boot_cpu __read_mostly = -1;
#endif
/* /*
* Debugging: see timer_list.c * Debugging: see timer_list.c
...@@ -167,6 +175,26 @@ void tick_setup_periodic(struct clock_event_device *dev, int broadcast) ...@@ -167,6 +175,26 @@ void tick_setup_periodic(struct clock_event_device *dev, int broadcast)
} }
} }
#ifdef CONFIG_NO_HZ_FULL
static void giveup_do_timer(void *info)
{
int cpu = *(unsigned int *)info;
WARN_ON(tick_do_timer_cpu != smp_processor_id());
tick_do_timer_cpu = cpu;
}
static void tick_take_do_timer_from_boot(void)
{
int cpu = smp_processor_id();
int from = tick_do_timer_boot_cpu;
if (from >= 0 && from != cpu)
smp_call_function_single(from, giveup_do_timer, &cpu, 1);
}
#endif
/* /*
* Setup the tick device * Setup the tick device
*/ */
...@@ -186,12 +214,26 @@ static void tick_setup_device(struct tick_device *td, ...@@ -186,12 +214,26 @@ static void tick_setup_device(struct tick_device *td,
* this cpu: * this cpu:
*/ */
if (tick_do_timer_cpu == TICK_DO_TIMER_BOOT) { if (tick_do_timer_cpu == TICK_DO_TIMER_BOOT) {
if (!tick_nohz_full_cpu(cpu)) tick_do_timer_cpu = cpu;
tick_do_timer_cpu = cpu;
else
tick_do_timer_cpu = TICK_DO_TIMER_NONE;
tick_next_period = ktime_get(); tick_next_period = ktime_get();
tick_period = NSEC_PER_SEC / HZ; tick_period = NSEC_PER_SEC / HZ;
#ifdef CONFIG_NO_HZ_FULL
/*
* The boot CPU may be nohz_full, in which case set
* tick_do_timer_boot_cpu so the first housekeeping
* secondary that comes up will take do_timer from
* us.
*/
if (tick_nohz_full_cpu(cpu))
tick_do_timer_boot_cpu = cpu;
} else if (tick_do_timer_boot_cpu != -1 &&
!tick_nohz_full_cpu(cpu)) {
tick_take_do_timer_from_boot();
tick_do_timer_boot_cpu = -1;
WARN_ON(tick_do_timer_cpu != cpu);
#endif
} }
/* /*
......
...@@ -121,10 +121,16 @@ static void tick_sched_do_timer(struct tick_sched *ts, ktime_t now) ...@@ -121,10 +121,16 @@ static void tick_sched_do_timer(struct tick_sched *ts, ktime_t now)
* into a long sleep. If two CPUs happen to assign themselves to * into a long sleep. If two CPUs happen to assign themselves to
* this duty, then the jiffies update is still serialized by * this duty, then the jiffies update is still serialized by
* jiffies_lock. * jiffies_lock.
*
* If nohz_full is enabled, this should not happen because the
* tick_do_timer_cpu never relinquishes.
*/ */
if (unlikely(tick_do_timer_cpu == TICK_DO_TIMER_NONE) if (unlikely(tick_do_timer_cpu == TICK_DO_TIMER_NONE)) {
&& !tick_nohz_full_cpu(cpu)) #ifdef CONFIG_NO_HZ_FULL
WARN_ON(tick_nohz_full_running);
#endif
tick_do_timer_cpu = cpu; tick_do_timer_cpu = cpu;
}
#endif #endif
/* Check, if the jiffies need an update */ /* Check, if the jiffies need an update */
...@@ -395,8 +401,8 @@ void __init tick_nohz_full_setup(cpumask_var_t cpumask) ...@@ -395,8 +401,8 @@ void __init tick_nohz_full_setup(cpumask_var_t cpumask)
static int tick_nohz_cpu_down(unsigned int cpu) static int tick_nohz_cpu_down(unsigned int cpu)
{ {
/* /*
* The boot CPU handles housekeeping duty (unbound timers, * The tick_do_timer_cpu CPU handles housekeeping duty (unbound
* workqueues, timekeeping, ...) on behalf of full dynticks * timers, workqueues, timekeeping, ...) on behalf of full dynticks
* CPUs. It must remain online when nohz full is enabled. * CPUs. It must remain online when nohz full is enabled.
*/ */
if (tick_nohz_full_running && tick_do_timer_cpu == cpu) if (tick_nohz_full_running && tick_do_timer_cpu == cpu)
...@@ -423,12 +429,15 @@ void __init tick_nohz_init(void) ...@@ -423,12 +429,15 @@ void __init tick_nohz_init(void)
return; return;
} }
cpu = smp_processor_id(); if (IS_ENABLED(CONFIG_PM_SLEEP_SMP) &&
!IS_ENABLED(CONFIG_PM_SLEEP_SMP_NONZERO_CPU)) {
cpu = smp_processor_id();
if (cpumask_test_cpu(cpu, tick_nohz_full_mask)) { if (cpumask_test_cpu(cpu, tick_nohz_full_mask)) {
pr_warn("NO_HZ: Clearing %d from nohz_full range for timekeeping\n", pr_warn("NO_HZ: Clearing %d from nohz_full range "
cpu); "for timekeeping\n", cpu);
cpumask_clear_cpu(cpu, tick_nohz_full_mask); cpumask_clear_cpu(cpu, tick_nohz_full_mask);
}
} }
for_each_cpu(cpu, tick_nohz_full_mask) for_each_cpu(cpu, tick_nohz_full_mask)
...@@ -904,8 +913,13 @@ static bool can_stop_idle_tick(int cpu, struct tick_sched *ts) ...@@ -904,8 +913,13 @@ static bool can_stop_idle_tick(int cpu, struct tick_sched *ts)
/* /*
* Boot safety: make sure the timekeeping duty has been * Boot safety: make sure the timekeeping duty has been
* assigned before entering dyntick-idle mode, * assigned before entering dyntick-idle mode,
* tick_do_timer_cpu is TICK_DO_TIMER_BOOT
*/ */
if (tick_do_timer_cpu == TICK_DO_TIMER_NONE) if (unlikely(tick_do_timer_cpu == TICK_DO_TIMER_BOOT))
return false;
/* Should not happen for nohz-full */
if (WARN_ON_ONCE(tick_do_timer_cpu == TICK_DO_TIMER_NONE))
return false; return false;
} }
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment