Commit 62214039 authored by Rafael J. Wysocki's avatar Rafael J. Wysocki

Merge branch 'pm-cpuidle'

* pm-cpuidle:
  cpuidle: Pass exit latency limit to cpuidle_use_deepest_state()
  cpuidle: Allow idle injection to apply exit latency limit
  cpuidle: Introduce cpuidle_driver_state_disabled() for driver quirks
  cpuidle: teo: Avoid code duplication in conditionals
  cpuidle: teo: Avoid using "early hits" incorrectly
  cpuidle: teo: Exclude cpuidle overhead from computations
  cpuidle: Use nanoseconds as the unit of time
  cpuidle: Consolidate disabled state checks
  ACPI: processor_idle: Skip dummy wait if kernel is in guest
  cpuidle: Do not unset the driver if it is there already
  cpuidle: teo: Fix "early hits" handling for disabled idle states
  cpuidle: teo: Consider hits and misses metrics of disabled states
  cpuidle: teo: Rename local variable in teo_select()
  cpuidle: teo: Ignore disabled idle states that are too deep
parents 05ff1ba4 5aa9ba63
...@@ -62,13 +62,13 @@ static struct cpuidle_driver imx6q_cpuidle_driver = { ...@@ -62,13 +62,13 @@ static struct cpuidle_driver imx6q_cpuidle_driver = {
*/ */
void imx6q_cpuidle_fec_irqs_used(void) void imx6q_cpuidle_fec_irqs_used(void)
{ {
imx6q_cpuidle_driver.states[1].disabled = true; cpuidle_driver_state_disabled(&imx6q_cpuidle_driver, 1, true);
} }
EXPORT_SYMBOL_GPL(imx6q_cpuidle_fec_irqs_used); EXPORT_SYMBOL_GPL(imx6q_cpuidle_fec_irqs_used);
void imx6q_cpuidle_fec_irqs_unused(void) void imx6q_cpuidle_fec_irqs_unused(void)
{ {
imx6q_cpuidle_driver.states[1].disabled = false; cpuidle_driver_state_disabled(&imx6q_cpuidle_driver, 1, false);
} }
EXPORT_SYMBOL_GPL(imx6q_cpuidle_fec_irqs_unused); EXPORT_SYMBOL_GPL(imx6q_cpuidle_fec_irqs_unused);
......
...@@ -203,7 +203,7 @@ void tegra20_cpuidle_pcie_irqs_in_use(void) ...@@ -203,7 +203,7 @@ void tegra20_cpuidle_pcie_irqs_in_use(void)
{ {
pr_info_once( pr_info_once(
"Disabling cpuidle LP2 state, since PCIe IRQs are in use\n"); "Disabling cpuidle LP2 state, since PCIe IRQs are in use\n");
tegra_idle_driver.states[1].disabled = true; cpuidle_driver_state_disabled(&tegra_idle_driver, 1, true);
} }
int __init tegra20_cpuidle_init(void) int __init tegra20_cpuidle_init(void)
......
...@@ -642,6 +642,19 @@ static int acpi_idle_bm_check(void) ...@@ -642,6 +642,19 @@ static int acpi_idle_bm_check(void)
return bm_status; return bm_status;
} }
static void wait_for_freeze(void)
{
#ifdef CONFIG_X86
/* No delay is needed if we are in guest */
if (boot_cpu_has(X86_FEATURE_HYPERVISOR))
return;
#endif
/* Dummy wait op - must do something useless after P_LVL2 read
because chipsets cannot guarantee that STPCLK# signal
gets asserted in time to freeze execution properly. */
inl(acpi_gbl_FADT.xpm_timer_block.address);
}
/** /**
* acpi_idle_do_entry - enter idle state using the appropriate method * acpi_idle_do_entry - enter idle state using the appropriate method
* @cx: cstate data * @cx: cstate data
...@@ -658,10 +671,7 @@ static void __cpuidle acpi_idle_do_entry(struct acpi_processor_cx *cx) ...@@ -658,10 +671,7 @@ static void __cpuidle acpi_idle_do_entry(struct acpi_processor_cx *cx)
} else { } else {
/* IO port based C-state */ /* IO port based C-state */
inb(cx->address); inb(cx->address);
/* Dummy wait op - must do something useless after P_LVL2 read wait_for_freeze();
because chipsets cannot guarantee that STPCLK# signal
gets asserted in time to freeze execution properly. */
inl(acpi_gbl_FADT.xpm_timer_block.address);
} }
} }
...@@ -682,8 +692,7 @@ static int acpi_idle_play_dead(struct cpuidle_device *dev, int index) ...@@ -682,8 +692,7 @@ static int acpi_idle_play_dead(struct cpuidle_device *dev, int index)
safe_halt(); safe_halt();
else if (cx->entry_method == ACPI_CSTATE_SYSTEMIO) { else if (cx->entry_method == ACPI_CSTATE_SYSTEMIO) {
inb(cx->address); inb(cx->address);
/* See comment in acpi_idle_do_entry() */ wait_for_freeze();
inl(acpi_gbl_FADT.xpm_timer_block.address);
} else } else
return -ENODEV; return -ENODEV;
} }
......
...@@ -56,13 +56,10 @@ static u64 get_snooze_timeout(struct cpuidle_device *dev, ...@@ -56,13 +56,10 @@ static u64 get_snooze_timeout(struct cpuidle_device *dev,
return default_snooze_timeout; return default_snooze_timeout;
for (i = index + 1; i < drv->state_count; i++) { for (i = index + 1; i < drv->state_count; i++) {
struct cpuidle_state *s = &drv->states[i]; if (dev->states_usage[i].disable)
struct cpuidle_state_usage *su = &dev->states_usage[i];
if (s->disabled || su->disable)
continue; continue;
return s->target_residency * tb_ticks_per_usec; return drv->states[i].target_residency * tb_ticks_per_usec;
} }
return default_snooze_timeout; return default_snooze_timeout;
......
...@@ -75,44 +75,45 @@ int cpuidle_play_dead(void) ...@@ -75,44 +75,45 @@ int cpuidle_play_dead(void)
static int find_deepest_state(struct cpuidle_driver *drv, static int find_deepest_state(struct cpuidle_driver *drv,
struct cpuidle_device *dev, struct cpuidle_device *dev,
unsigned int max_latency, u64 max_latency_ns,
unsigned int forbidden_flags, unsigned int forbidden_flags,
bool s2idle) bool s2idle)
{ {
unsigned int latency_req = 0; u64 latency_req = 0;
int i, ret = 0; int i, ret = 0;
for (i = 1; i < drv->state_count; i++) { for (i = 1; i < drv->state_count; i++) {
struct cpuidle_state *s = &drv->states[i]; struct cpuidle_state *s = &drv->states[i];
struct cpuidle_state_usage *su = &dev->states_usage[i];
if (s->disabled || su->disable || s->exit_latency <= latency_req if (dev->states_usage[i].disable ||
|| s->exit_latency > max_latency s->exit_latency_ns <= latency_req ||
|| (s->flags & forbidden_flags) s->exit_latency_ns > max_latency_ns ||
|| (s2idle && !s->enter_s2idle)) (s->flags & forbidden_flags) ||
(s2idle && !s->enter_s2idle))
continue; continue;
latency_req = s->exit_latency; latency_req = s->exit_latency_ns;
ret = i; ret = i;
} }
return ret; return ret;
} }
/** /**
* cpuidle_use_deepest_state - Set/clear governor override flag. * cpuidle_use_deepest_state - Set/unset governor override mode.
* @enable: New value of the flag. * @latency_limit_ns: Idle state exit latency limit (or no override if 0).
* *
* Set/unset the current CPU to use the deepest idle state (override governors * If @latency_limit_ns is nonzero, set the current CPU to use the deepest idle
* going forward if set). * state with exit latency within @latency_limit_ns (override governors going
* forward), or do not override governors if it is zero.
*/ */
void cpuidle_use_deepest_state(bool enable) void cpuidle_use_deepest_state(u64 latency_limit_ns)
{ {
struct cpuidle_device *dev; struct cpuidle_device *dev;
preempt_disable(); preempt_disable();
dev = cpuidle_get_device(); dev = cpuidle_get_device();
if (dev) if (dev)
dev->use_deepest_state = enable; dev->forced_idle_latency_limit_ns = latency_limit_ns;
preempt_enable(); preempt_enable();
} }
...@@ -122,9 +123,10 @@ void cpuidle_use_deepest_state(bool enable) ...@@ -122,9 +123,10 @@ void cpuidle_use_deepest_state(bool enable)
* @dev: cpuidle device for the given CPU. * @dev: cpuidle device for the given CPU.
*/ */
int cpuidle_find_deepest_state(struct cpuidle_driver *drv, int cpuidle_find_deepest_state(struct cpuidle_driver *drv,
struct cpuidle_device *dev) struct cpuidle_device *dev,
u64 latency_limit_ns)
{ {
return find_deepest_state(drv, dev, UINT_MAX, 0, false); return find_deepest_state(drv, dev, latency_limit_ns, 0, false);
} }
#ifdef CONFIG_SUSPEND #ifdef CONFIG_SUSPEND
...@@ -180,7 +182,7 @@ int cpuidle_enter_s2idle(struct cpuidle_driver *drv, struct cpuidle_device *dev) ...@@ -180,7 +182,7 @@ int cpuidle_enter_s2idle(struct cpuidle_driver *drv, struct cpuidle_device *dev)
* that interrupts won't be enabled when it exits and allows the tick to * that interrupts won't be enabled when it exits and allows the tick to
* be frozen safely. * be frozen safely.
*/ */
index = find_deepest_state(drv, dev, UINT_MAX, 0, true); index = find_deepest_state(drv, dev, U64_MAX, 0, true);
if (index > 0) if (index > 0)
enter_s2idle_proper(drv, dev, index); enter_s2idle_proper(drv, dev, index);
...@@ -209,7 +211,7 @@ int cpuidle_enter_state(struct cpuidle_device *dev, struct cpuidle_driver *drv, ...@@ -209,7 +211,7 @@ int cpuidle_enter_state(struct cpuidle_device *dev, struct cpuidle_driver *drv,
* CPU as a broadcast timer, this call may fail if it is not available. * CPU as a broadcast timer, this call may fail if it is not available.
*/ */
if (broadcast && tick_broadcast_enter()) { if (broadcast && tick_broadcast_enter()) {
index = find_deepest_state(drv, dev, target_state->exit_latency, index = find_deepest_state(drv, dev, target_state->exit_latency_ns,
CPUIDLE_FLAG_TIMER_STOP, false); CPUIDLE_FLAG_TIMER_STOP, false);
if (index < 0) { if (index < 0) {
default_idle_call(); default_idle_call();
...@@ -247,7 +249,7 @@ int cpuidle_enter_state(struct cpuidle_device *dev, struct cpuidle_driver *drv, ...@@ -247,7 +249,7 @@ int cpuidle_enter_state(struct cpuidle_device *dev, struct cpuidle_driver *drv,
local_irq_enable(); local_irq_enable();
if (entered_state >= 0) { if (entered_state >= 0) {
s64 diff, delay = drv->states[entered_state].exit_latency; s64 diff, delay = drv->states[entered_state].exit_latency_ns;
int i; int i;
/* /*
...@@ -255,18 +257,15 @@ int cpuidle_enter_state(struct cpuidle_device *dev, struct cpuidle_driver *drv, ...@@ -255,18 +257,15 @@ int cpuidle_enter_state(struct cpuidle_device *dev, struct cpuidle_driver *drv,
* This can be moved to within driver enter routine, * This can be moved to within driver enter routine,
* but that results in multiple copies of same code. * but that results in multiple copies of same code.
*/ */
diff = ktime_us_delta(time_end, time_start); diff = ktime_sub(time_end, time_start);
if (diff > INT_MAX)
diff = INT_MAX;
dev->last_residency = (int)diff; dev->last_residency_ns = diff;
dev->states_usage[entered_state].time += dev->last_residency; dev->states_usage[entered_state].time_ns += diff;
dev->states_usage[entered_state].usage++; dev->states_usage[entered_state].usage++;
if (diff < drv->states[entered_state].target_residency) { if (diff < drv->states[entered_state].target_residency_ns) {
for (i = entered_state - 1; i >= 0; i--) { for (i = entered_state - 1; i >= 0; i--) {
if (drv->states[i].disabled || if (dev->states_usage[i].disable)
dev->states_usage[i].disable)
continue; continue;
/* Shallower states are enabled, so update. */ /* Shallower states are enabled, so update. */
...@@ -275,22 +274,21 @@ int cpuidle_enter_state(struct cpuidle_device *dev, struct cpuidle_driver *drv, ...@@ -275,22 +274,21 @@ int cpuidle_enter_state(struct cpuidle_device *dev, struct cpuidle_driver *drv,
} }
} else if (diff > delay) { } else if (diff > delay) {
for (i = entered_state + 1; i < drv->state_count; i++) { for (i = entered_state + 1; i < drv->state_count; i++) {
if (drv->states[i].disabled || if (dev->states_usage[i].disable)
dev->states_usage[i].disable)
continue; continue;
/* /*
* Update if a deeper state would have been a * Update if a deeper state would have been a
* better match for the observed idle duration. * better match for the observed idle duration.
*/ */
if (diff - delay >= drv->states[i].target_residency) if (diff - delay >= drv->states[i].target_residency_ns)
dev->states_usage[entered_state].below++; dev->states_usage[entered_state].below++;
break; break;
} }
} }
} else { } else {
dev->last_residency = 0; dev->last_residency_ns = 0;
} }
return entered_state; return entered_state;
...@@ -380,10 +378,10 @@ u64 cpuidle_poll_time(struct cpuidle_driver *drv, ...@@ -380,10 +378,10 @@ u64 cpuidle_poll_time(struct cpuidle_driver *drv,
limit_ns = TICK_NSEC; limit_ns = TICK_NSEC;
for (i = 1; i < drv->state_count; i++) { for (i = 1; i < drv->state_count; i++) {
if (drv->states[i].disabled || dev->states_usage[i].disable) if (dev->states_usage[i].disable)
continue; continue;
limit_ns = (u64)drv->states[i].target_residency * NSEC_PER_USEC; limit_ns = (u64)drv->states[i].target_residency_ns;
} }
dev->poll_limit_ns = limit_ns; dev->poll_limit_ns = limit_ns;
...@@ -554,7 +552,7 @@ static void __cpuidle_unregister_device(struct cpuidle_device *dev) ...@@ -554,7 +552,7 @@ static void __cpuidle_unregister_device(struct cpuidle_device *dev)
static void __cpuidle_device_init(struct cpuidle_device *dev) static void __cpuidle_device_init(struct cpuidle_device *dev)
{ {
memset(dev->states_usage, 0, sizeof(dev->states_usage)); memset(dev->states_usage, 0, sizeof(dev->states_usage));
dev->last_residency = 0; dev->last_residency_ns = 0;
dev->next_hrtimer = 0; dev->next_hrtimer = 0;
} }
...@@ -567,12 +565,16 @@ static void __cpuidle_device_init(struct cpuidle_device *dev) ...@@ -567,12 +565,16 @@ static void __cpuidle_device_init(struct cpuidle_device *dev)
*/ */
static int __cpuidle_register_device(struct cpuidle_device *dev) static int __cpuidle_register_device(struct cpuidle_device *dev)
{ {
int ret;
struct cpuidle_driver *drv = cpuidle_get_cpu_driver(dev); struct cpuidle_driver *drv = cpuidle_get_cpu_driver(dev);
int i, ret;
if (!try_module_get(drv->owner)) if (!try_module_get(drv->owner))
return -EINVAL; return -EINVAL;
for (i = 0; i < drv->state_count; i++)
if (drv->states[i].disabled)
dev->states_usage[i].disable |= CPUIDLE_STATE_DISABLED_BY_DRIVER;
per_cpu(cpuidle_devices, dev->cpu) = dev; per_cpu(cpuidle_devices, dev->cpu) = dev;
list_add(&dev->device_list, &cpuidle_detected_devices); list_add(&dev->device_list, &cpuidle_detected_devices);
......
...@@ -62,24 +62,23 @@ static inline void __cpuidle_unset_driver(struct cpuidle_driver *drv) ...@@ -62,24 +62,23 @@ static inline void __cpuidle_unset_driver(struct cpuidle_driver *drv)
* __cpuidle_set_driver - set per CPU driver variables for the given driver. * __cpuidle_set_driver - set per CPU driver variables for the given driver.
* @drv: a valid pointer to a struct cpuidle_driver * @drv: a valid pointer to a struct cpuidle_driver
* *
* For each CPU in the driver's cpumask, unset the registered driver per CPU * Returns 0 on success, -EBUSY if any CPU in the cpumask have a driver
* to @drv. * different from drv already.
*
* Returns 0 on success, -EBUSY if the CPUs have driver(s) already.
*/ */
static inline int __cpuidle_set_driver(struct cpuidle_driver *drv) static inline int __cpuidle_set_driver(struct cpuidle_driver *drv)
{ {
int cpu; int cpu;
for_each_cpu(cpu, drv->cpumask) { for_each_cpu(cpu, drv->cpumask) {
struct cpuidle_driver *old_drv;
if (__cpuidle_get_cpu_driver(cpu)) { old_drv = __cpuidle_get_cpu_driver(cpu);
__cpuidle_unset_driver(drv); if (old_drv && old_drv != drv)
return -EBUSY; return -EBUSY;
} }
for_each_cpu(cpu, drv->cpumask)
per_cpu(cpuidle_drivers, cpu) = drv; per_cpu(cpuidle_drivers, cpu) = drv;
}
return 0; return 0;
} }
...@@ -166,16 +165,27 @@ static void __cpuidle_driver_init(struct cpuidle_driver *drv) ...@@ -166,16 +165,27 @@ static void __cpuidle_driver_init(struct cpuidle_driver *drv)
if (!drv->cpumask) if (!drv->cpumask)
drv->cpumask = (struct cpumask *)cpu_possible_mask; drv->cpumask = (struct cpumask *)cpu_possible_mask;
for (i = 0; i < drv->state_count; i++) {
struct cpuidle_state *s = &drv->states[i];
/* /*
* Look for the timer stop flag in the different states, so that we know * Look for the timer stop flag in the different states and if
* if the broadcast timer has to be set up. The loop is in the reverse * it is found, indicate that the broadcast timer has to be set
* order, because usually one of the deeper states have this flag set. * up.
*/ */
for (i = drv->state_count - 1; i >= 0 ; i--) { if (s->flags & CPUIDLE_FLAG_TIMER_STOP)
if (drv->states[i].flags & CPUIDLE_FLAG_TIMER_STOP) {
drv->bctimer = 1; drv->bctimer = 1;
break;
} /*
* The core will use the target residency and exit latency
* values in nanoseconds, but allow drivers to provide them in
* microseconds too.
*/
if (s->target_residency > 0)
s->target_residency_ns = s->target_residency * NSEC_PER_USEC;
if (s->exit_latency > 0)
s->exit_latency_ns = s->exit_latency * NSEC_PER_USEC;
} }
} }
...@@ -379,3 +389,31 @@ void cpuidle_driver_unref(void) ...@@ -379,3 +389,31 @@ void cpuidle_driver_unref(void)
spin_unlock(&cpuidle_driver_lock); spin_unlock(&cpuidle_driver_lock);
} }
/**
* cpuidle_driver_state_disabled - Disable or enable an idle state
* @drv: cpuidle driver owning the state
* @idx: State index
* @disable: Whether or not to disable the state
*/
void cpuidle_driver_state_disabled(struct cpuidle_driver *drv, int idx,
bool disable)
{
unsigned int cpu;
mutex_lock(&cpuidle_lock);
for_each_cpu(cpu, drv->cpumask) {
struct cpuidle_device *dev = per_cpu(cpuidle_devices, cpu);
if (!dev)
continue;
if (disable)
dev->states_usage[idx].disable |= CPUIDLE_STATE_DISABLED_BY_DRIVER;
else
dev->states_usage[idx].disable &= ~CPUIDLE_STATE_DISABLED_BY_DRIVER;
}
mutex_unlock(&cpuidle_lock);
}
...@@ -107,11 +107,14 @@ int cpuidle_register_governor(struct cpuidle_governor *gov) ...@@ -107,11 +107,14 @@ int cpuidle_register_governor(struct cpuidle_governor *gov)
* cpuidle_governor_latency_req - Compute a latency constraint for CPU * cpuidle_governor_latency_req - Compute a latency constraint for CPU
* @cpu: Target CPU * @cpu: Target CPU
*/ */
int cpuidle_governor_latency_req(unsigned int cpu) s64 cpuidle_governor_latency_req(unsigned int cpu)
{ {
int global_req = pm_qos_request(PM_QOS_CPU_DMA_LATENCY); int global_req = pm_qos_request(PM_QOS_CPU_DMA_LATENCY);
struct device *device = get_cpu_device(cpu); struct device *device = get_cpu_device(cpu);
int device_req = dev_pm_qos_raw_resume_latency(device); int device_req = dev_pm_qos_raw_resume_latency(device);
return device_req < global_req ? device_req : global_req; if (device_req > global_req)
device_req = global_req;
return (s64)device_req * NSEC_PER_USEC;
} }
...@@ -49,7 +49,7 @@ static int haltpoll_select(struct cpuidle_driver *drv, ...@@ -49,7 +49,7 @@ static int haltpoll_select(struct cpuidle_driver *drv,
struct cpuidle_device *dev, struct cpuidle_device *dev,
bool *stop_tick) bool *stop_tick)
{ {
int latency_req = cpuidle_governor_latency_req(dev->cpu); s64 latency_req = cpuidle_governor_latency_req(dev->cpu);
if (!drv->state_count || latency_req == 0) { if (!drv->state_count || latency_req == 0) {
*stop_tick = false; *stop_tick = false;
...@@ -75,10 +75,9 @@ static int haltpoll_select(struct cpuidle_driver *drv, ...@@ -75,10 +75,9 @@ static int haltpoll_select(struct cpuidle_driver *drv,
return 0; return 0;
} }
static void adjust_poll_limit(struct cpuidle_device *dev, unsigned int block_us) static void adjust_poll_limit(struct cpuidle_device *dev, u64 block_ns)
{ {
unsigned int val; unsigned int val;
u64 block_ns = block_us*NSEC_PER_USEC;
/* Grow cpu_halt_poll_us if /* Grow cpu_halt_poll_us if
* cpu_halt_poll_us < block_ns < guest_halt_poll_us * cpu_halt_poll_us < block_ns < guest_halt_poll_us
...@@ -115,7 +114,7 @@ static void haltpoll_reflect(struct cpuidle_device *dev, int index) ...@@ -115,7 +114,7 @@ static void haltpoll_reflect(struct cpuidle_device *dev, int index)
dev->last_state_idx = index; dev->last_state_idx = index;
if (index != 0) if (index != 0)
adjust_poll_limit(dev, dev->last_residency); adjust_poll_limit(dev, dev->last_residency_ns);
} }
/** /**
......
...@@ -27,8 +27,8 @@ struct ladder_device_state { ...@@ -27,8 +27,8 @@ struct ladder_device_state {
struct { struct {
u32 promotion_count; u32 promotion_count;
u32 demotion_count; u32 demotion_count;
u32 promotion_time; u64 promotion_time_ns;
u32 demotion_time; u64 demotion_time_ns;
} threshold; } threshold;
struct { struct {
int promotion_count; int promotion_count;
...@@ -68,9 +68,10 @@ static int ladder_select_state(struct cpuidle_driver *drv, ...@@ -68,9 +68,10 @@ static int ladder_select_state(struct cpuidle_driver *drv,
{ {
struct ladder_device *ldev = this_cpu_ptr(&ladder_devices); struct ladder_device *ldev = this_cpu_ptr(&ladder_devices);
struct ladder_device_state *last_state; struct ladder_device_state *last_state;
int last_residency, last_idx = dev->last_state_idx; int last_idx = dev->last_state_idx;
int first_idx = drv->states[0].flags & CPUIDLE_FLAG_POLLING ? 1 : 0; int first_idx = drv->states[0].flags & CPUIDLE_FLAG_POLLING ? 1 : 0;
int latency_req = cpuidle_governor_latency_req(dev->cpu); s64 latency_req = cpuidle_governor_latency_req(dev->cpu);
s64 last_residency;
/* Special case when user has set very strict latency requirement */ /* Special case when user has set very strict latency requirement */
if (unlikely(latency_req == 0)) { if (unlikely(latency_req == 0)) {
...@@ -80,14 +81,13 @@ static int ladder_select_state(struct cpuidle_driver *drv, ...@@ -80,14 +81,13 @@ static int ladder_select_state(struct cpuidle_driver *drv,
last_state = &ldev->states[last_idx]; last_state = &ldev->states[last_idx];
last_residency = dev->last_residency - drv->states[last_idx].exit_latency; last_residency = dev->last_residency_ns - drv->states[last_idx].exit_latency_ns;
/* consider promotion */ /* consider promotion */
if (last_idx < drv->state_count - 1 && if (last_idx < drv->state_count - 1 &&
!drv->states[last_idx + 1].disabled &&
!dev->states_usage[last_idx + 1].disable && !dev->states_usage[last_idx + 1].disable &&
last_residency > last_state->threshold.promotion_time && last_residency > last_state->threshold.promotion_time_ns &&
drv->states[last_idx + 1].exit_latency <= latency_req) { drv->states[last_idx + 1].exit_latency_ns <= latency_req) {
last_state->stats.promotion_count++; last_state->stats.promotion_count++;
last_state->stats.demotion_count = 0; last_state->stats.demotion_count = 0;
if (last_state->stats.promotion_count >= last_state->threshold.promotion_count) { if (last_state->stats.promotion_count >= last_state->threshold.promotion_count) {
...@@ -98,13 +98,12 @@ static int ladder_select_state(struct cpuidle_driver *drv, ...@@ -98,13 +98,12 @@ static int ladder_select_state(struct cpuidle_driver *drv,
/* consider demotion */ /* consider demotion */
if (last_idx > first_idx && if (last_idx > first_idx &&
(drv->states[last_idx].disabled || (dev->states_usage[last_idx].disable ||
dev->states_usage[last_idx].disable || drv->states[last_idx].exit_latency_ns > latency_req)) {
drv->states[last_idx].exit_latency > latency_req)) {
int i; int i;
for (i = last_idx - 1; i > first_idx; i--) { for (i = last_idx - 1; i > first_idx; i--) {
if (drv->states[i].exit_latency <= latency_req) if (drv->states[i].exit_latency_ns <= latency_req)
break; break;
} }
ladder_do_selection(dev, ldev, last_idx, i); ladder_do_selection(dev, ldev, last_idx, i);
...@@ -112,7 +111,7 @@ static int ladder_select_state(struct cpuidle_driver *drv, ...@@ -112,7 +111,7 @@ static int ladder_select_state(struct cpuidle_driver *drv,
} }
if (last_idx > first_idx && if (last_idx > first_idx &&
last_residency < last_state->threshold.demotion_time) { last_residency < last_state->threshold.demotion_time_ns) {
last_state->stats.demotion_count++; last_state->stats.demotion_count++;
last_state->stats.promotion_count = 0; last_state->stats.promotion_count = 0;
if (last_state->stats.demotion_count >= last_state->threshold.demotion_count) { if (last_state->stats.demotion_count >= last_state->threshold.demotion_count) {
...@@ -152,9 +151,9 @@ static int ladder_enable_device(struct cpuidle_driver *drv, ...@@ -152,9 +151,9 @@ static int ladder_enable_device(struct cpuidle_driver *drv,
lstate->threshold.demotion_count = DEMOTION_COUNT; lstate->threshold.demotion_count = DEMOTION_COUNT;
if (i < drv->state_count - 1) if (i < drv->state_count - 1)
lstate->threshold.promotion_time = state->exit_latency; lstate->threshold.promotion_time_ns = state->exit_latency_ns;
if (i > first_idx) if (i > first_idx)
lstate->threshold.demotion_time = state->exit_latency; lstate->threshold.demotion_time_ns = state->exit_latency_ns;
} }
return 0; return 0;
......
This diff is collapsed.
This diff is collapsed.
...@@ -49,6 +49,8 @@ void cpuidle_poll_state_init(struct cpuidle_driver *drv) ...@@ -49,6 +49,8 @@ void cpuidle_poll_state_init(struct cpuidle_driver *drv)
snprintf(state->desc, CPUIDLE_DESC_LEN, "CPUIDLE CORE POLL IDLE"); snprintf(state->desc, CPUIDLE_DESC_LEN, "CPUIDLE CORE POLL IDLE");
state->exit_latency = 0; state->exit_latency = 0;
state->target_residency = 0; state->target_residency = 0;
state->exit_latency_ns = 0;
state->target_residency_ns = 0;
state->power_usage = -1; state->power_usage = -1;
state->enter = poll_idle; state->enter = poll_idle;
state->disabled = false; state->disabled = false;
......
...@@ -255,25 +255,6 @@ static ssize_t show_state_##_name(struct cpuidle_state *state, \ ...@@ -255,25 +255,6 @@ static ssize_t show_state_##_name(struct cpuidle_state *state, \
return sprintf(buf, "%u\n", state->_name);\ return sprintf(buf, "%u\n", state->_name);\
} }
#define define_store_state_ull_function(_name) \
static ssize_t store_state_##_name(struct cpuidle_state *state, \
struct cpuidle_state_usage *state_usage, \
const char *buf, size_t size) \
{ \
unsigned long long value; \
int err; \
if (!capable(CAP_SYS_ADMIN)) \
return -EPERM; \
err = kstrtoull(buf, 0, &value); \
if (err) \
return err; \
if (value) \
state_usage->_name = 1; \
else \
state_usage->_name = 0; \
return size; \
}
#define define_show_state_ull_function(_name) \ #define define_show_state_ull_function(_name) \
static ssize_t show_state_##_name(struct cpuidle_state *state, \ static ssize_t show_state_##_name(struct cpuidle_state *state, \
struct cpuidle_state_usage *state_usage, \ struct cpuidle_state_usage *state_usage, \
...@@ -292,18 +273,60 @@ static ssize_t show_state_##_name(struct cpuidle_state *state, \ ...@@ -292,18 +273,60 @@ static ssize_t show_state_##_name(struct cpuidle_state *state, \
return sprintf(buf, "%s\n", state->_name);\ return sprintf(buf, "%s\n", state->_name);\
} }
define_show_state_function(exit_latency) #define define_show_state_time_function(_name) \
define_show_state_function(target_residency) static ssize_t show_state_##_name(struct cpuidle_state *state, \
struct cpuidle_state_usage *state_usage, \
char *buf) \
{ \
return sprintf(buf, "%llu\n", ktime_to_us(state->_name##_ns)); \
}
define_show_state_time_function(exit_latency)
define_show_state_time_function(target_residency)
define_show_state_function(power_usage) define_show_state_function(power_usage)
define_show_state_ull_function(usage) define_show_state_ull_function(usage)
define_show_state_ull_function(time)
define_show_state_str_function(name) define_show_state_str_function(name)
define_show_state_str_function(desc) define_show_state_str_function(desc)
define_show_state_ull_function(disable)
define_store_state_ull_function(disable)
define_show_state_ull_function(above) define_show_state_ull_function(above)
define_show_state_ull_function(below) define_show_state_ull_function(below)
static ssize_t show_state_time(struct cpuidle_state *state,
struct cpuidle_state_usage *state_usage,
char *buf)
{
return sprintf(buf, "%llu\n", ktime_to_us(state_usage->time_ns));
}
static ssize_t show_state_disable(struct cpuidle_state *state,
struct cpuidle_state_usage *state_usage,
char *buf)
{
return sprintf(buf, "%llu\n",
state_usage->disable & CPUIDLE_STATE_DISABLED_BY_USER);
}
static ssize_t store_state_disable(struct cpuidle_state *state,
struct cpuidle_state_usage *state_usage,
const char *buf, size_t size)
{
unsigned int value;
int err;
if (!capable(CAP_SYS_ADMIN))
return -EPERM;
err = kstrtouint(buf, 0, &value);
if (err)
return err;
if (value)
state_usage->disable |= CPUIDLE_STATE_DISABLED_BY_USER;
else
state_usage->disable &= ~CPUIDLE_STATE_DISABLED_BY_USER;
return size;
}
define_one_state_ro(name, show_state_name); define_one_state_ro(name, show_state_name);
define_one_state_ro(desc, show_state_desc); define_one_state_ro(desc, show_state_desc);
define_one_state_ro(latency, show_state_exit_latency); define_one_state_ro(latency, show_state_exit_latency);
......
...@@ -184,7 +184,12 @@ void arch_cpu_idle_dead(void); ...@@ -184,7 +184,12 @@ void arch_cpu_idle_dead(void);
int cpu_report_state(int cpu); int cpu_report_state(int cpu);
int cpu_check_up_prepare(int cpu); int cpu_check_up_prepare(int cpu);
void cpu_set_state_online(int cpu); void cpu_set_state_online(int cpu);
void play_idle(unsigned long duration_us); void play_idle_precise(u64 duration_ns, u64 latency_ns);
static inline void play_idle(unsigned long duration_us)
{
play_idle_precise(duration_us * NSEC_PER_USEC, U64_MAX);
}
#ifdef CONFIG_HOTPLUG_CPU #ifdef CONFIG_HOTPLUG_CPU
bool cpu_wait_death(unsigned int cpu, int seconds); bool cpu_wait_death(unsigned int cpu, int seconds);
......
...@@ -29,10 +29,13 @@ struct cpuidle_driver; ...@@ -29,10 +29,13 @@ struct cpuidle_driver;
* CPUIDLE DEVICE INTERFACE * * CPUIDLE DEVICE INTERFACE *
****************************/ ****************************/
#define CPUIDLE_STATE_DISABLED_BY_USER BIT(0)
#define CPUIDLE_STATE_DISABLED_BY_DRIVER BIT(1)
struct cpuidle_state_usage { struct cpuidle_state_usage {
unsigned long long disable; unsigned long long disable;
unsigned long long usage; unsigned long long usage;
unsigned long long time; /* in US */ u64 time_ns;
unsigned long long above; /* Number of times it's been too deep */ unsigned long long above; /* Number of times it's been too deep */
unsigned long long below; /* Number of times it's been too shallow */ unsigned long long below; /* Number of times it's been too shallow */
#ifdef CONFIG_SUSPEND #ifdef CONFIG_SUSPEND
...@@ -45,6 +48,8 @@ struct cpuidle_state { ...@@ -45,6 +48,8 @@ struct cpuidle_state {
char name[CPUIDLE_NAME_LEN]; char name[CPUIDLE_NAME_LEN];
char desc[CPUIDLE_DESC_LEN]; char desc[CPUIDLE_DESC_LEN];
u64 exit_latency_ns;
u64 target_residency_ns;
unsigned int flags; unsigned int flags;
unsigned int exit_latency; /* in US */ unsigned int exit_latency; /* in US */
int power_usage; /* in mW */ int power_usage; /* in mW */
...@@ -80,14 +85,14 @@ struct cpuidle_driver_kobj; ...@@ -80,14 +85,14 @@ struct cpuidle_driver_kobj;
struct cpuidle_device { struct cpuidle_device {
unsigned int registered:1; unsigned int registered:1;
unsigned int enabled:1; unsigned int enabled:1;
unsigned int use_deepest_state:1;
unsigned int poll_time_limit:1; unsigned int poll_time_limit:1;
unsigned int cpu; unsigned int cpu;
ktime_t next_hrtimer; ktime_t next_hrtimer;
int last_state_idx; int last_state_idx;
int last_residency; u64 last_residency_ns;
u64 poll_limit_ns; u64 poll_limit_ns;
u64 forced_idle_latency_limit_ns;
struct cpuidle_state_usage states_usage[CPUIDLE_STATE_MAX]; struct cpuidle_state_usage states_usage[CPUIDLE_STATE_MAX];
struct cpuidle_state_kobj *kobjs[CPUIDLE_STATE_MAX]; struct cpuidle_state_kobj *kobjs[CPUIDLE_STATE_MAX];
struct cpuidle_driver_kobj *kobj_driver; struct cpuidle_driver_kobj *kobj_driver;
...@@ -144,6 +149,8 @@ extern int cpuidle_register_driver(struct cpuidle_driver *drv); ...@@ -144,6 +149,8 @@ extern int cpuidle_register_driver(struct cpuidle_driver *drv);
extern struct cpuidle_driver *cpuidle_get_driver(void); extern struct cpuidle_driver *cpuidle_get_driver(void);
extern struct cpuidle_driver *cpuidle_driver_ref(void); extern struct cpuidle_driver *cpuidle_driver_ref(void);
extern void cpuidle_driver_unref(void); extern void cpuidle_driver_unref(void);
extern void cpuidle_driver_state_disabled(struct cpuidle_driver *drv, int idx,
bool disable);
extern void cpuidle_unregister_driver(struct cpuidle_driver *drv); extern void cpuidle_unregister_driver(struct cpuidle_driver *drv);
extern int cpuidle_register_device(struct cpuidle_device *dev); extern int cpuidle_register_device(struct cpuidle_device *dev);
extern void cpuidle_unregister_device(struct cpuidle_device *dev); extern void cpuidle_unregister_device(struct cpuidle_device *dev);
...@@ -181,6 +188,8 @@ static inline int cpuidle_register_driver(struct cpuidle_driver *drv) ...@@ -181,6 +188,8 @@ static inline int cpuidle_register_driver(struct cpuidle_driver *drv)
static inline struct cpuidle_driver *cpuidle_get_driver(void) {return NULL; } static inline struct cpuidle_driver *cpuidle_get_driver(void) {return NULL; }
static inline struct cpuidle_driver *cpuidle_driver_ref(void) {return NULL; } static inline struct cpuidle_driver *cpuidle_driver_ref(void) {return NULL; }
static inline void cpuidle_driver_unref(void) {} static inline void cpuidle_driver_unref(void) {}
static inline void cpuidle_driver_state_disabled(struct cpuidle_driver *drv,
int idx, bool disable) { }
static inline void cpuidle_unregister_driver(struct cpuidle_driver *drv) { } static inline void cpuidle_unregister_driver(struct cpuidle_driver *drv) { }
static inline int cpuidle_register_device(struct cpuidle_device *dev) static inline int cpuidle_register_device(struct cpuidle_device *dev)
{return -ENODEV; } {return -ENODEV; }
...@@ -204,18 +213,20 @@ static inline struct cpuidle_device *cpuidle_get_device(void) {return NULL; } ...@@ -204,18 +213,20 @@ static inline struct cpuidle_device *cpuidle_get_device(void) {return NULL; }
#ifdef CONFIG_CPU_IDLE #ifdef CONFIG_CPU_IDLE
extern int cpuidle_find_deepest_state(struct cpuidle_driver *drv, extern int cpuidle_find_deepest_state(struct cpuidle_driver *drv,
struct cpuidle_device *dev); struct cpuidle_device *dev,
u64 latency_limit_ns);
extern int cpuidle_enter_s2idle(struct cpuidle_driver *drv, extern int cpuidle_enter_s2idle(struct cpuidle_driver *drv,
struct cpuidle_device *dev); struct cpuidle_device *dev);
extern void cpuidle_use_deepest_state(bool enable); extern void cpuidle_use_deepest_state(u64 latency_limit_ns);
#else #else
static inline int cpuidle_find_deepest_state(struct cpuidle_driver *drv, static inline int cpuidle_find_deepest_state(struct cpuidle_driver *drv,
struct cpuidle_device *dev) struct cpuidle_device *dev,
u64 latency_limit_ns)
{return -ENODEV; } {return -ENODEV; }
static inline int cpuidle_enter_s2idle(struct cpuidle_driver *drv, static inline int cpuidle_enter_s2idle(struct cpuidle_driver *drv,
struct cpuidle_device *dev) struct cpuidle_device *dev)
{return -ENODEV; } {return -ENODEV; }
static inline void cpuidle_use_deepest_state(bool enable) static inline void cpuidle_use_deepest_state(u64 latency_limit_ns)
{ {
} }
#endif #endif
...@@ -260,7 +271,7 @@ struct cpuidle_governor { ...@@ -260,7 +271,7 @@ struct cpuidle_governor {
#ifdef CONFIG_CPU_IDLE #ifdef CONFIG_CPU_IDLE
extern int cpuidle_register_governor(struct cpuidle_governor *gov); extern int cpuidle_register_governor(struct cpuidle_governor *gov);
extern int cpuidle_governor_latency_req(unsigned int cpu); extern s64 cpuidle_governor_latency_req(unsigned int cpu);
#else #else
static inline int cpuidle_register_governor(struct cpuidle_governor *gov) static inline int cpuidle_register_governor(struct cpuidle_governor *gov)
{return 0;} {return 0;}
......
...@@ -104,7 +104,7 @@ static int call_cpuidle(struct cpuidle_driver *drv, struct cpuidle_device *dev, ...@@ -104,7 +104,7 @@ static int call_cpuidle(struct cpuidle_driver *drv, struct cpuidle_device *dev,
* update no idle residency and return. * update no idle residency and return.
*/ */
if (current_clr_polling_and_test()) { if (current_clr_polling_and_test()) {
dev->last_residency = 0; dev->last_residency_ns = 0;
local_irq_enable(); local_irq_enable();
return -EBUSY; return -EBUSY;
} }
...@@ -165,7 +165,9 @@ static void cpuidle_idle_call(void) ...@@ -165,7 +165,9 @@ static void cpuidle_idle_call(void)
* until a proper wakeup interrupt happens. * until a proper wakeup interrupt happens.
*/ */
if (idle_should_enter_s2idle() || dev->use_deepest_state) { if (idle_should_enter_s2idle() || dev->forced_idle_latency_limit_ns) {
u64 max_latency_ns;
if (idle_should_enter_s2idle()) { if (idle_should_enter_s2idle()) {
rcu_idle_enter(); rcu_idle_enter();
...@@ -176,12 +178,16 @@ static void cpuidle_idle_call(void) ...@@ -176,12 +178,16 @@ static void cpuidle_idle_call(void)
} }
rcu_idle_exit(); rcu_idle_exit();
max_latency_ns = U64_MAX;
} else {
max_latency_ns = dev->forced_idle_latency_limit_ns;
} }
tick_nohz_idle_stop_tick(); tick_nohz_idle_stop_tick();
rcu_idle_enter(); rcu_idle_enter();
next_state = cpuidle_find_deepest_state(drv, dev); next_state = cpuidle_find_deepest_state(drv, dev, max_latency_ns);
call_cpuidle(drv, dev, next_state); call_cpuidle(drv, dev, next_state);
} else { } else {
bool stop_tick = true; bool stop_tick = true;
...@@ -311,7 +317,7 @@ static enum hrtimer_restart idle_inject_timer_fn(struct hrtimer *timer) ...@@ -311,7 +317,7 @@ static enum hrtimer_restart idle_inject_timer_fn(struct hrtimer *timer)
return HRTIMER_NORESTART; return HRTIMER_NORESTART;
} }
void play_idle(unsigned long duration_us) void play_idle_precise(u64 duration_ns, u64 latency_ns)
{ {
struct idle_timer it; struct idle_timer it;
...@@ -323,29 +329,29 @@ void play_idle(unsigned long duration_us) ...@@ -323,29 +329,29 @@ void play_idle(unsigned long duration_us)
WARN_ON_ONCE(current->nr_cpus_allowed != 1); WARN_ON_ONCE(current->nr_cpus_allowed != 1);
WARN_ON_ONCE(!(current->flags & PF_KTHREAD)); WARN_ON_ONCE(!(current->flags & PF_KTHREAD));
WARN_ON_ONCE(!(current->flags & PF_NO_SETAFFINITY)); WARN_ON_ONCE(!(current->flags & PF_NO_SETAFFINITY));
WARN_ON_ONCE(!duration_us); WARN_ON_ONCE(!duration_ns);
rcu_sleep_check(); rcu_sleep_check();
preempt_disable(); preempt_disable();
current->flags |= PF_IDLE; current->flags |= PF_IDLE;
cpuidle_use_deepest_state(true); cpuidle_use_deepest_state(latency_ns);
it.done = 0; it.done = 0;
hrtimer_init_on_stack(&it.timer, CLOCK_MONOTONIC, HRTIMER_MODE_REL); hrtimer_init_on_stack(&it.timer, CLOCK_MONOTONIC, HRTIMER_MODE_REL);
it.timer.function = idle_inject_timer_fn; it.timer.function = idle_inject_timer_fn;
hrtimer_start(&it.timer, ns_to_ktime(duration_us * NSEC_PER_USEC), hrtimer_start(&it.timer, ns_to_ktime(duration_ns),
HRTIMER_MODE_REL_PINNED); HRTIMER_MODE_REL_PINNED);
while (!READ_ONCE(it.done)) while (!READ_ONCE(it.done))
do_idle(); do_idle();
cpuidle_use_deepest_state(false); cpuidle_use_deepest_state(0);
current->flags &= ~PF_IDLE; current->flags &= ~PF_IDLE;
preempt_fold_need_resched(); preempt_fold_need_resched();
preempt_enable(); preempt_enable();
} }
EXPORT_SYMBOL_GPL(play_idle); EXPORT_SYMBOL_GPL(play_idle_precise);
void cpu_startup_entry(enum cpuhp_state state) void cpu_startup_entry(enum cpuhp_state state)
{ {
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment