Commit 69e1e811 authored by Suresh Siddha's avatar Suresh Siddha Committed by Ingo Molnar

sched, nohz: Track nr_busy_cpus in the sched_group_power

Introduce nr_busy_cpus in the struct sched_group_power [Not in sched_group
because sched groups are duplicated for the SD_OVERLAP scheduler domain]
and for each cpu that enters and exits idle, this parameter will
be updated in each scheduler group of the scheduler domain that this cpu
belongs to.

To avoid the frequent update of this state as the cpu enters
and exits idle, the update of the stat during idle exit is
delayed to the first timer tick that happens after the cpu becomes busy.
This is done using NOHZ_IDLE flag in the struct rq's nohz_flags.
Signed-off-by: default avatarSuresh Siddha <suresh.b.siddha@intel.com>
Signed-off-by: default avatarPeter Zijlstra <a.p.zijlstra@chello.nl>
Cc: Thomas Gleixner <tglx@linutronix.de>
Link: http://lkml.kernel.org/r/20111202010832.555984323@sbsiddha-desk.sc.intel.comSigned-off-by: default avatarIngo Molnar <mingo@elte.hu>
parent 1c792db7
...@@ -273,9 +273,11 @@ extern int runqueue_is_locked(int cpu); ...@@ -273,9 +273,11 @@ extern int runqueue_is_locked(int cpu);
#if defined(CONFIG_SMP) && defined(CONFIG_NO_HZ) #if defined(CONFIG_SMP) && defined(CONFIG_NO_HZ)
extern void select_nohz_load_balancer(int stop_tick); extern void select_nohz_load_balancer(int stop_tick);
extern void set_cpu_sd_state_idle(void);
extern int get_nohz_timer_target(void); extern int get_nohz_timer_target(void);
#else #else
static inline void select_nohz_load_balancer(int stop_tick) { } static inline void select_nohz_load_balancer(int stop_tick) { }
static inline void set_cpu_sd_state_idle(void);
#endif #endif
/* /*
...@@ -901,6 +903,10 @@ struct sched_group_power { ...@@ -901,6 +903,10 @@ struct sched_group_power {
* single CPU. * single CPU.
*/ */
unsigned int power, power_orig; unsigned int power, power_orig;
/*
* Number of busy cpus in this group.
*/
atomic_t nr_busy_cpus;
}; };
struct sched_group { struct sched_group {
......
...@@ -6024,6 +6024,7 @@ static void init_sched_groups_power(int cpu, struct sched_domain *sd) ...@@ -6024,6 +6024,7 @@ static void init_sched_groups_power(int cpu, struct sched_domain *sd)
return; return;
update_group_power(sd, cpu); update_group_power(sd, cpu);
atomic_set(&sg->sgp->nr_busy_cpus, sg->group_weight);
} }
int __weak arch_sd_sibling_asym_packing(void) int __weak arch_sd_sibling_asym_packing(void)
......
...@@ -4901,6 +4901,36 @@ static void nohz_balancer_kick(int cpu) ...@@ -4901,6 +4901,36 @@ static void nohz_balancer_kick(int cpu)
return; return;
} }
static inline void set_cpu_sd_state_busy(void)
{
struct sched_domain *sd;
int cpu = smp_processor_id();
if (!test_bit(NOHZ_IDLE, nohz_flags(cpu)))
return;
clear_bit(NOHZ_IDLE, nohz_flags(cpu));
rcu_read_lock();
for_each_domain(cpu, sd)
atomic_inc(&sd->groups->sgp->nr_busy_cpus);
rcu_read_unlock();
}
void set_cpu_sd_state_idle(void)
{
struct sched_domain *sd;
int cpu = smp_processor_id();
if (test_bit(NOHZ_IDLE, nohz_flags(cpu)))
return;
set_bit(NOHZ_IDLE, nohz_flags(cpu));
rcu_read_lock();
for_each_domain(cpu, sd)
atomic_dec(&sd->groups->sgp->nr_busy_cpus);
rcu_read_unlock();
}
/* /*
* This routine will try to nominate the ilb (idle load balancing) * This routine will try to nominate the ilb (idle load balancing)
* owner among the cpus whose ticks are stopped. ilb owner will do the idle * owner among the cpus whose ticks are stopped. ilb owner will do the idle
...@@ -5135,6 +5165,7 @@ static inline int nohz_kick_needed(struct rq *rq, int cpu) ...@@ -5135,6 +5165,7 @@ static inline int nohz_kick_needed(struct rq *rq, int cpu)
* We may be recently in ticked or tickless idle mode. At the first * We may be recently in ticked or tickless idle mode. At the first
* busy tick after returning from idle, we will update the busy stats. * busy tick after returning from idle, we will update the busy stats.
*/ */
set_cpu_sd_state_busy();
if (unlikely(test_bit(NOHZ_TICK_STOPPED, nohz_flags(cpu)))) if (unlikely(test_bit(NOHZ_TICK_STOPPED, nohz_flags(cpu))))
clear_bit(NOHZ_TICK_STOPPED, nohz_flags(cpu)); clear_bit(NOHZ_TICK_STOPPED, nohz_flags(cpu));
......
...@@ -1069,6 +1069,7 @@ extern void account_cfs_bandwidth_used(int enabled, int was_enabled); ...@@ -1069,6 +1069,7 @@ extern void account_cfs_bandwidth_used(int enabled, int was_enabled);
enum rq_nohz_flag_bits { enum rq_nohz_flag_bits {
NOHZ_TICK_STOPPED, NOHZ_TICK_STOPPED,
NOHZ_BALANCE_KICK, NOHZ_BALANCE_KICK,
NOHZ_IDLE,
}; };
#define nohz_flags(cpu) (&cpu_rq(cpu)->nohz_flags) #define nohz_flags(cpu) (&cpu_rq(cpu)->nohz_flags)
......
...@@ -296,6 +296,15 @@ void tick_nohz_stop_sched_tick(int inidle) ...@@ -296,6 +296,15 @@ void tick_nohz_stop_sched_tick(int inidle)
cpu = smp_processor_id(); cpu = smp_processor_id();
ts = &per_cpu(tick_cpu_sched, cpu); ts = &per_cpu(tick_cpu_sched, cpu);
/*
* Update the idle state in the scheduler domain hierarchy
* when tick_nohz_stop_sched_tick() is called from the idle loop.
* State will be updated to busy during the first busy tick after
* exiting idle.
*/
if (inidle)
set_cpu_sd_state_idle();
/* /*
* Call to tick_nohz_start_idle stops the last_update_time from being * Call to tick_nohz_start_idle stops the last_update_time from being
* updated. Thus, it must not be called in the event we are called from * updated. Thus, it must not be called in the event we are called from
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment