Commit 2dee4b0e authored by Rafael J. Wysocki's avatar Rafael J. Wysocki

Merge intel_pstate driver updates for v4.12.

parents 939dc6f5 630e5757
...@@ -37,6 +37,9 @@ ...@@ -37,6 +37,9 @@
#include <asm/cpufeature.h> #include <asm/cpufeature.h>
#include <asm/intel-family.h> #include <asm/intel-family.h>
#define INTEL_PSTATE_DEFAULT_SAMPLING_INTERVAL (10 * NSEC_PER_MSEC)
#define INTEL_PSTATE_HWP_SAMPLING_INTERVAL (50 * NSEC_PER_MSEC)
#define INTEL_CPUFREQ_TRANSITION_LATENCY 20000 #define INTEL_CPUFREQ_TRANSITION_LATENCY 20000
#ifdef CONFIG_ACPI #ifdef CONFIG_ACPI
...@@ -74,6 +77,11 @@ static inline int ceiling_fp(int32_t x) ...@@ -74,6 +77,11 @@ static inline int ceiling_fp(int32_t x)
return ret; return ret;
} }
static inline int32_t percent_fp(int percent)
{
return div_fp(percent, 100);
}
static inline u64 mul_ext_fp(u64 x, u64 y) static inline u64 mul_ext_fp(u64 x, u64 y)
{ {
return (x * y) >> EXT_FRAC_BITS; return (x * y) >> EXT_FRAC_BITS;
...@@ -186,45 +194,22 @@ struct _pid { ...@@ -186,45 +194,22 @@ struct _pid {
}; };
/** /**
* struct perf_limits - Store user and policy limits * struct global_params - Global parameters, mostly tunable via sysfs.
* @no_turbo: User requested turbo state from intel_pstate sysfs * @no_turbo: Whether or not to use turbo P-states.
* @turbo_disabled: Platform turbo status either from msr * @turbo_disabled: Whethet or not turbo P-states are available at all,
* MSR_IA32_MISC_ENABLE or when maximum available pstate * based on the MSR_IA32_MISC_ENABLE value and whether or
* matches the maximum turbo pstate * not the maximum reported turbo P-state is different from
* @max_perf_pct: Effective maximum performance limit in percentage, this * the maximum reported non-turbo one.
* is minimum of either limits enforced by cpufreq policy * @min_perf_pct: Minimum capacity limit in percent of the maximum turbo
* or limits from user set limits via intel_pstate sysfs * P-state capacity.
* @min_perf_pct: Effective minimum performance limit in percentage, this * @max_perf_pct: Maximum capacity limit in percent of the maximum turbo
* is maximum of either limits enforced by cpufreq policy * P-state capacity.
* or limits from user set limits via intel_pstate sysfs
* @max_perf: This is a scaled value between 0 to 255 for max_perf_pct
* This value is used to limit max pstate
* @min_perf: This is a scaled value between 0 to 255 for min_perf_pct
* This value is used to limit min pstate
* @max_policy_pct: The maximum performance in percentage enforced by
* cpufreq setpolicy interface
* @max_sysfs_pct: The maximum performance in percentage enforced by
* intel pstate sysfs interface, unused when per cpu
* controls are enforced
* @min_policy_pct: The minimum performance in percentage enforced by
* cpufreq setpolicy interface
* @min_sysfs_pct: The minimum performance in percentage enforced by
* intel pstate sysfs interface, unused when per cpu
* controls are enforced
*
* Storage for user and policy defined limits.
*/ */
struct perf_limits { struct global_params {
int no_turbo; bool no_turbo;
int turbo_disabled; bool turbo_disabled;
int max_perf_pct; int max_perf_pct;
int min_perf_pct; int min_perf_pct;
int32_t max_perf;
int32_t min_perf;
int max_policy_pct;
int max_sysfs_pct;
int min_policy_pct;
int min_sysfs_pct;
}; };
/** /**
...@@ -245,9 +230,10 @@ struct perf_limits { ...@@ -245,9 +230,10 @@ struct perf_limits {
* @prev_cummulative_iowait: IO Wait time difference from last and * @prev_cummulative_iowait: IO Wait time difference from last and
* current sample * current sample
* @sample: Storage for storing last Sample data * @sample: Storage for storing last Sample data
* @perf_limits: Pointer to perf_limit unique to this CPU * @min_perf: Minimum capacity limit as a fraction of the maximum
* Not all field in the structure are applicable * turbo P-state capacity.
* when per cpu controls are enforced * @max_perf: Maximum capacity limit as a fraction of the maximum
* turbo P-state capacity.
* @acpi_perf_data: Stores ACPI perf information read from _PSS * @acpi_perf_data: Stores ACPI perf information read from _PSS
* @valid_pss_table: Set to true for valid ACPI _PSS entries found * @valid_pss_table: Set to true for valid ACPI _PSS entries found
* @epp_powersave: Last saved HWP energy performance preference * @epp_powersave: Last saved HWP energy performance preference
...@@ -279,7 +265,8 @@ struct cpudata { ...@@ -279,7 +265,8 @@ struct cpudata {
u64 prev_tsc; u64 prev_tsc;
u64 prev_cummulative_iowait; u64 prev_cummulative_iowait;
struct sample sample; struct sample sample;
struct perf_limits *perf_limits; int32_t min_perf;
int32_t max_perf;
#ifdef CONFIG_ACPI #ifdef CONFIG_ACPI
struct acpi_processor_performance acpi_perf_data; struct acpi_processor_performance acpi_perf_data;
bool valid_pss_table; bool valid_pss_table;
...@@ -324,7 +311,7 @@ struct pstate_adjust_policy { ...@@ -324,7 +311,7 @@ struct pstate_adjust_policy {
* @get_scaling: Callback to get frequency scaling factor * @get_scaling: Callback to get frequency scaling factor
* @get_val: Callback to convert P state to actual MSR write value * @get_val: Callback to convert P state to actual MSR write value
* @get_vid: Callback to get VID data for Atom platforms * @get_vid: Callback to get VID data for Atom platforms
* @get_target_pstate: Callback to a function to calculate next P state to use * @update_util: Active mode utilization update callback.
* *
* Core and Atom CPU models have different way to get P State limits. This * Core and Atom CPU models have different way to get P State limits. This
* structure is used to store those callbacks. * structure is used to store those callbacks.
...@@ -337,43 +324,31 @@ struct pstate_funcs { ...@@ -337,43 +324,31 @@ struct pstate_funcs {
int (*get_scaling)(void); int (*get_scaling)(void);
u64 (*get_val)(struct cpudata*, int pstate); u64 (*get_val)(struct cpudata*, int pstate);
void (*get_vid)(struct cpudata *); void (*get_vid)(struct cpudata *);
int32_t (*get_target_pstate)(struct cpudata *); void (*update_util)(struct update_util_data *data, u64 time,
unsigned int flags);
}; };
/** static struct pstate_funcs pstate_funcs __read_mostly;
* struct cpu_defaults- Per CPU model default config data static struct pstate_adjust_policy pid_params __read_mostly = {
* @pid_policy: PID config data .sample_rate_ms = 10,
* @funcs: Callback function data .sample_rate_ns = 10 * NSEC_PER_MSEC,
*/ .deadband = 0,
struct cpu_defaults { .setpoint = 97,
struct pstate_adjust_policy pid_policy; .p_gain_pct = 20,
struct pstate_funcs funcs; .d_gain_pct = 0,
.i_gain_pct = 0,
}; };
static inline int32_t get_target_pstate_use_performance(struct cpudata *cpu);
static inline int32_t get_target_pstate_use_cpu_load(struct cpudata *cpu);
static struct pstate_adjust_policy pid_params __read_mostly;
static struct pstate_funcs pstate_funcs __read_mostly;
static int hwp_active __read_mostly; static int hwp_active __read_mostly;
static bool per_cpu_limits __read_mostly; static bool per_cpu_limits __read_mostly;
static bool driver_registered __read_mostly; static struct cpufreq_driver *intel_pstate_driver __read_mostly;
#ifdef CONFIG_ACPI #ifdef CONFIG_ACPI
static bool acpi_ppc; static bool acpi_ppc;
#endif #endif
static struct perf_limits global; static struct global_params global;
static void intel_pstate_init_limits(struct perf_limits *limits)
{
memset(limits, 0, sizeof(*limits));
limits->max_perf_pct = 100;
limits->max_perf = int_ext_tofp(1);
limits->max_policy_pct = 100;
limits->max_sysfs_pct = 100;
}
static DEFINE_MUTEX(intel_pstate_driver_lock); static DEFINE_MUTEX(intel_pstate_driver_lock);
static DEFINE_MUTEX(intel_pstate_limits_lock); static DEFINE_MUTEX(intel_pstate_limits_lock);
...@@ -530,29 +505,6 @@ static inline void intel_pstate_exit_perf_limits(struct cpufreq_policy *policy) ...@@ -530,29 +505,6 @@ static inline void intel_pstate_exit_perf_limits(struct cpufreq_policy *policy)
} }
#endif #endif
static inline void pid_reset(struct _pid *pid, int setpoint, int busy,
int deadband, int integral) {
pid->setpoint = int_tofp(setpoint);
pid->deadband = int_tofp(deadband);
pid->integral = int_tofp(integral);
pid->last_err = int_tofp(setpoint) - int_tofp(busy);
}
static inline void pid_p_gain_set(struct _pid *pid, int percent)
{
pid->p_gain = div_fp(percent, 100);
}
static inline void pid_i_gain_set(struct _pid *pid, int percent)
{
pid->i_gain = div_fp(percent, 100);
}
static inline void pid_d_gain_set(struct _pid *pid, int percent)
{
pid->d_gain = div_fp(percent, 100);
}
static signed int pid_calc(struct _pid *pid, int32_t busy) static signed int pid_calc(struct _pid *pid, int32_t busy)
{ {
signed int result; signed int result;
...@@ -590,23 +542,17 @@ static signed int pid_calc(struct _pid *pid, int32_t busy) ...@@ -590,23 +542,17 @@ static signed int pid_calc(struct _pid *pid, int32_t busy)
return (signed int)fp_toint(result); return (signed int)fp_toint(result);
} }
static inline void intel_pstate_busy_pid_reset(struct cpudata *cpu) static inline void intel_pstate_pid_reset(struct cpudata *cpu)
{ {
pid_p_gain_set(&cpu->pid, pid_params.p_gain_pct); struct _pid *pid = &cpu->pid;
pid_d_gain_set(&cpu->pid, pid_params.d_gain_pct);
pid_i_gain_set(&cpu->pid, pid_params.i_gain_pct);
pid_reset(&cpu->pid, pid_params.setpoint, 100, pid_params.deadband, 0); pid->p_gain = percent_fp(pid_params.p_gain_pct);
} pid->d_gain = percent_fp(pid_params.d_gain_pct);
pid->i_gain = percent_fp(pid_params.i_gain_pct);
static inline void intel_pstate_reset_all_pid(void) pid->setpoint = int_tofp(pid_params.setpoint);
{ pid->last_err = pid->setpoint - int_tofp(100);
unsigned int cpu; pid->deadband = int_tofp(pid_params.deadband);
pid->integral = 0;
for_each_online_cpu(cpu) {
if (all_cpu_data[cpu])
intel_pstate_busy_pid_reset(all_cpu_data[cpu]);
}
} }
static inline void update_turbo_state(void) static inline void update_turbo_state(void)
...@@ -621,6 +567,14 @@ static inline void update_turbo_state(void) ...@@ -621,6 +567,14 @@ static inline void update_turbo_state(void)
cpu->pstate.max_pstate == cpu->pstate.turbo_pstate); cpu->pstate.max_pstate == cpu->pstate.turbo_pstate);
} }
static int min_perf_pct_min(void)
{
struct cpudata *cpu = all_cpu_data[0];
return DIV_ROUND_UP(cpu->pstate.min_pstate * 100,
cpu->pstate.turbo_pstate);
}
static s16 intel_pstate_get_epb(struct cpudata *cpu_data) static s16 intel_pstate_get_epb(struct cpudata *cpu_data)
{ {
u64 epb; u64 epb;
...@@ -838,19 +792,13 @@ static struct freq_attr *hwp_cpufreq_attrs[] = { ...@@ -838,19 +792,13 @@ static struct freq_attr *hwp_cpufreq_attrs[] = {
NULL, NULL,
}; };
static void intel_pstate_hwp_set(struct cpufreq_policy *policy) static void intel_pstate_hwp_set(unsigned int cpu)
{ {
int min, hw_min, max, hw_max, cpu;
struct perf_limits *perf_limits = &global;
u64 value, cap;
for_each_cpu(cpu, policy->cpus) {
struct cpudata *cpu_data = all_cpu_data[cpu]; struct cpudata *cpu_data = all_cpu_data[cpu];
int min, hw_min, max, hw_max;
u64 value, cap;
s16 epp; s16 epp;
if (per_cpu_limits)
perf_limits = all_cpu_data[cpu]->perf_limits;
rdmsrl_on_cpu(cpu, MSR_HWP_CAPABILITIES, &cap); rdmsrl_on_cpu(cpu, MSR_HWP_CAPABILITIES, &cap);
hw_min = HWP_LOWEST_PERF(cap); hw_min = HWP_LOWEST_PERF(cap);
if (global.no_turbo) if (global.no_turbo)
...@@ -858,11 +806,11 @@ static void intel_pstate_hwp_set(struct cpufreq_policy *policy) ...@@ -858,11 +806,11 @@ static void intel_pstate_hwp_set(struct cpufreq_policy *policy)
else else
hw_max = HWP_HIGHEST_PERF(cap); hw_max = HWP_HIGHEST_PERF(cap);
max = fp_ext_toint(hw_max * perf_limits->max_perf); max = fp_ext_toint(hw_max * cpu_data->max_perf);
if (cpu_data->policy == CPUFREQ_POLICY_PERFORMANCE) if (cpu_data->policy == CPUFREQ_POLICY_PERFORMANCE)
min = max; min = max;
else else
min = fp_ext_toint(hw_max * perf_limits->min_perf); min = fp_ext_toint(hw_max * cpu_data->min_perf);
rdmsrl_on_cpu(cpu, MSR_HWP_REQUEST, &value); rdmsrl_on_cpu(cpu, MSR_HWP_REQUEST, &value);
...@@ -890,7 +838,6 @@ static void intel_pstate_hwp_set(struct cpufreq_policy *policy) ...@@ -890,7 +838,6 @@ static void intel_pstate_hwp_set(struct cpufreq_policy *policy)
if (epp < 0) if (epp < 0)
goto skip_epp; goto skip_epp;
epp = 0; epp = 0;
} else { } else {
/* skip setting EPP, when saved value is invalid */ /* skip setting EPP, when saved value is invalid */
...@@ -919,15 +866,6 @@ static void intel_pstate_hwp_set(struct cpufreq_policy *policy) ...@@ -919,15 +866,6 @@ static void intel_pstate_hwp_set(struct cpufreq_policy *policy)
} }
skip_epp: skip_epp:
wrmsrl_on_cpu(cpu, MSR_HWP_REQUEST, value); wrmsrl_on_cpu(cpu, MSR_HWP_REQUEST, value);
}
}
static int intel_pstate_hwp_set_policy(struct cpufreq_policy *policy)
{
if (hwp_active)
intel_pstate_hwp_set(policy);
return 0;
} }
static int intel_pstate_hwp_save_state(struct cpufreq_policy *policy) static int intel_pstate_hwp_save_state(struct cpufreq_policy *policy)
...@@ -944,20 +882,17 @@ static int intel_pstate_hwp_save_state(struct cpufreq_policy *policy) ...@@ -944,20 +882,17 @@ static int intel_pstate_hwp_save_state(struct cpufreq_policy *policy)
static int intel_pstate_resume(struct cpufreq_policy *policy) static int intel_pstate_resume(struct cpufreq_policy *policy)
{ {
int ret;
if (!hwp_active) if (!hwp_active)
return 0; return 0;
mutex_lock(&intel_pstate_limits_lock); mutex_lock(&intel_pstate_limits_lock);
all_cpu_data[policy->cpu]->epp_policy = 0; all_cpu_data[policy->cpu]->epp_policy = 0;
intel_pstate_hwp_set(policy->cpu);
ret = intel_pstate_hwp_set_policy(policy);
mutex_unlock(&intel_pstate_limits_lock); mutex_unlock(&intel_pstate_limits_lock);
return ret; return 0;
} }
static void intel_pstate_update_policies(void) static void intel_pstate_update_policies(void)
...@@ -971,9 +906,14 @@ static void intel_pstate_update_policies(void) ...@@ -971,9 +906,14 @@ static void intel_pstate_update_policies(void)
/************************** debugfs begin ************************/ /************************** debugfs begin ************************/
static int pid_param_set(void *data, u64 val) static int pid_param_set(void *data, u64 val)
{ {
unsigned int cpu;
*(u32 *)data = val; *(u32 *)data = val;
pid_params.sample_rate_ns = pid_params.sample_rate_ms * NSEC_PER_MSEC; pid_params.sample_rate_ns = pid_params.sample_rate_ms * NSEC_PER_MSEC;
intel_pstate_reset_all_pid(); for_each_possible_cpu(cpu)
if (all_cpu_data[cpu])
intel_pstate_pid_reset(all_cpu_data[cpu]);
return 0; return 0;
} }
...@@ -1084,7 +1024,7 @@ static ssize_t show_turbo_pct(struct kobject *kobj, ...@@ -1084,7 +1024,7 @@ static ssize_t show_turbo_pct(struct kobject *kobj,
mutex_lock(&intel_pstate_driver_lock); mutex_lock(&intel_pstate_driver_lock);
if (!driver_registered) { if (!intel_pstate_driver) {
mutex_unlock(&intel_pstate_driver_lock); mutex_unlock(&intel_pstate_driver_lock);
return -EAGAIN; return -EAGAIN;
} }
...@@ -1109,7 +1049,7 @@ static ssize_t show_num_pstates(struct kobject *kobj, ...@@ -1109,7 +1049,7 @@ static ssize_t show_num_pstates(struct kobject *kobj,
mutex_lock(&intel_pstate_driver_lock); mutex_lock(&intel_pstate_driver_lock);
if (!driver_registered) { if (!intel_pstate_driver) {
mutex_unlock(&intel_pstate_driver_lock); mutex_unlock(&intel_pstate_driver_lock);
return -EAGAIN; return -EAGAIN;
} }
...@@ -1129,7 +1069,7 @@ static ssize_t show_no_turbo(struct kobject *kobj, ...@@ -1129,7 +1069,7 @@ static ssize_t show_no_turbo(struct kobject *kobj,
mutex_lock(&intel_pstate_driver_lock); mutex_lock(&intel_pstate_driver_lock);
if (!driver_registered) { if (!intel_pstate_driver) {
mutex_unlock(&intel_pstate_driver_lock); mutex_unlock(&intel_pstate_driver_lock);
return -EAGAIN; return -EAGAIN;
} }
...@@ -1157,7 +1097,7 @@ static ssize_t store_no_turbo(struct kobject *a, struct attribute *b, ...@@ -1157,7 +1097,7 @@ static ssize_t store_no_turbo(struct kobject *a, struct attribute *b,
mutex_lock(&intel_pstate_driver_lock); mutex_lock(&intel_pstate_driver_lock);
if (!driver_registered) { if (!intel_pstate_driver) {
mutex_unlock(&intel_pstate_driver_lock); mutex_unlock(&intel_pstate_driver_lock);
return -EAGAIN; return -EAGAIN;
} }
...@@ -1174,6 +1114,15 @@ static ssize_t store_no_turbo(struct kobject *a, struct attribute *b, ...@@ -1174,6 +1114,15 @@ static ssize_t store_no_turbo(struct kobject *a, struct attribute *b,
global.no_turbo = clamp_t(int, input, 0, 1); global.no_turbo = clamp_t(int, input, 0, 1);
if (global.no_turbo) {
struct cpudata *cpu = all_cpu_data[0];
int pct = cpu->pstate.max_pstate * 100 / cpu->pstate.turbo_pstate;
/* Squash the global minimum into the permitted range. */
if (global.min_perf_pct > pct)
global.min_perf_pct = pct;
}
mutex_unlock(&intel_pstate_limits_lock); mutex_unlock(&intel_pstate_limits_lock);
intel_pstate_update_policies(); intel_pstate_update_policies();
...@@ -1195,18 +1144,14 @@ static ssize_t store_max_perf_pct(struct kobject *a, struct attribute *b, ...@@ -1195,18 +1144,14 @@ static ssize_t store_max_perf_pct(struct kobject *a, struct attribute *b,
mutex_lock(&intel_pstate_driver_lock); mutex_lock(&intel_pstate_driver_lock);
if (!driver_registered) { if (!intel_pstate_driver) {
mutex_unlock(&intel_pstate_driver_lock); mutex_unlock(&intel_pstate_driver_lock);
return -EAGAIN; return -EAGAIN;
} }
mutex_lock(&intel_pstate_limits_lock); mutex_lock(&intel_pstate_limits_lock);
global.max_sysfs_pct = clamp_t(int, input, 0 , 100); global.max_perf_pct = clamp_t(int, input, global.min_perf_pct, 100);
global.max_perf_pct = min(global.max_policy_pct, global.max_sysfs_pct);
global.max_perf_pct = max(global.min_policy_pct, global.max_perf_pct);
global.max_perf_pct = max(global.min_perf_pct, global.max_perf_pct);
global.max_perf = percent_ext_fp(global.max_perf_pct);
mutex_unlock(&intel_pstate_limits_lock); mutex_unlock(&intel_pstate_limits_lock);
...@@ -1229,18 +1174,15 @@ static ssize_t store_min_perf_pct(struct kobject *a, struct attribute *b, ...@@ -1229,18 +1174,15 @@ static ssize_t store_min_perf_pct(struct kobject *a, struct attribute *b,
mutex_lock(&intel_pstate_driver_lock); mutex_lock(&intel_pstate_driver_lock);
if (!driver_registered) { if (!intel_pstate_driver) {
mutex_unlock(&intel_pstate_driver_lock); mutex_unlock(&intel_pstate_driver_lock);
return -EAGAIN; return -EAGAIN;
} }
mutex_lock(&intel_pstate_limits_lock); mutex_lock(&intel_pstate_limits_lock);
global.min_sysfs_pct = clamp_t(int, input, 0 , 100); global.min_perf_pct = clamp_t(int, input,
global.min_perf_pct = max(global.min_policy_pct, global.min_sysfs_pct); min_perf_pct_min(), global.max_perf_pct);
global.min_perf_pct = min(global.max_policy_pct, global.min_perf_pct);
global.min_perf_pct = min(global.max_perf_pct, global.min_perf_pct);
global.min_perf = percent_ext_fp(global.min_perf_pct);
mutex_unlock(&intel_pstate_limits_lock); mutex_unlock(&intel_pstate_limits_lock);
...@@ -1554,132 +1496,10 @@ static int knl_get_turbo_pstate(void) ...@@ -1554,132 +1496,10 @@ static int knl_get_turbo_pstate(void)
return ret; return ret;
} }
static struct cpu_defaults core_params = { static int intel_pstate_get_base_pstate(struct cpudata *cpu)
.pid_policy = {
.sample_rate_ms = 10,
.deadband = 0,
.setpoint = 97,
.p_gain_pct = 20,
.d_gain_pct = 0,
.i_gain_pct = 0,
},
.funcs = {
.get_max = core_get_max_pstate,
.get_max_physical = core_get_max_pstate_physical,
.get_min = core_get_min_pstate,
.get_turbo = core_get_turbo_pstate,
.get_scaling = core_get_scaling,
.get_val = core_get_val,
.get_target_pstate = get_target_pstate_use_performance,
},
};
static const struct cpu_defaults silvermont_params = {
.pid_policy = {
.sample_rate_ms = 10,
.deadband = 0,
.setpoint = 60,
.p_gain_pct = 14,
.d_gain_pct = 0,
.i_gain_pct = 4,
},
.funcs = {
.get_max = atom_get_max_pstate,
.get_max_physical = atom_get_max_pstate,
.get_min = atom_get_min_pstate,
.get_turbo = atom_get_turbo_pstate,
.get_val = atom_get_val,
.get_scaling = silvermont_get_scaling,
.get_vid = atom_get_vid,
.get_target_pstate = get_target_pstate_use_cpu_load,
},
};
static const struct cpu_defaults airmont_params = {
.pid_policy = {
.sample_rate_ms = 10,
.deadband = 0,
.setpoint = 60,
.p_gain_pct = 14,
.d_gain_pct = 0,
.i_gain_pct = 4,
},
.funcs = {
.get_max = atom_get_max_pstate,
.get_max_physical = atom_get_max_pstate,
.get_min = atom_get_min_pstate,
.get_turbo = atom_get_turbo_pstate,
.get_val = atom_get_val,
.get_scaling = airmont_get_scaling,
.get_vid = atom_get_vid,
.get_target_pstate = get_target_pstate_use_cpu_load,
},
};
static const struct cpu_defaults knl_params = {
.pid_policy = {
.sample_rate_ms = 10,
.deadband = 0,
.setpoint = 97,
.p_gain_pct = 20,
.d_gain_pct = 0,
.i_gain_pct = 0,
},
.funcs = {
.get_max = core_get_max_pstate,
.get_max_physical = core_get_max_pstate_physical,
.get_min = core_get_min_pstate,
.get_turbo = knl_get_turbo_pstate,
.get_scaling = core_get_scaling,
.get_val = core_get_val,
.get_target_pstate = get_target_pstate_use_performance,
},
};
static const struct cpu_defaults bxt_params = {
.pid_policy = {
.sample_rate_ms = 10,
.deadband = 0,
.setpoint = 60,
.p_gain_pct = 14,
.d_gain_pct = 0,
.i_gain_pct = 4,
},
.funcs = {
.get_max = core_get_max_pstate,
.get_max_physical = core_get_max_pstate_physical,
.get_min = core_get_min_pstate,
.get_turbo = core_get_turbo_pstate,
.get_scaling = core_get_scaling,
.get_val = core_get_val,
.get_target_pstate = get_target_pstate_use_cpu_load,
},
};
static void intel_pstate_get_min_max(struct cpudata *cpu, int *min, int *max)
{ {
int max_perf = cpu->pstate.turbo_pstate; return global.no_turbo || global.turbo_disabled ?
int max_perf_adj; cpu->pstate.max_pstate : cpu->pstate.turbo_pstate;
int min_perf;
struct perf_limits *perf_limits = &global;
if (global.no_turbo || global.turbo_disabled)
max_perf = cpu->pstate.max_pstate;
if (per_cpu_limits)
perf_limits = cpu->perf_limits;
/*
* performance can be limited by user through sysfs, by cpufreq
* policy, or by cpu specific default values determined through
* experimentation.
*/
max_perf_adj = fp_ext_toint(max_perf * perf_limits->max_perf);
*max = clamp_t(int, max_perf_adj,
cpu->pstate.min_pstate, cpu->pstate.turbo_pstate);
min_perf = fp_ext_toint(max_perf * perf_limits->min_perf);
*min = clamp_t(int, min_perf, cpu->pstate.min_pstate, max_perf);
} }
static void intel_pstate_set_pstate(struct cpudata *cpu, int pstate) static void intel_pstate_set_pstate(struct cpudata *cpu, int pstate)
...@@ -1702,11 +1522,13 @@ static void intel_pstate_set_min_pstate(struct cpudata *cpu) ...@@ -1702,11 +1522,13 @@ static void intel_pstate_set_min_pstate(struct cpudata *cpu)
static void intel_pstate_max_within_limits(struct cpudata *cpu) static void intel_pstate_max_within_limits(struct cpudata *cpu)
{ {
int min_pstate, max_pstate; int pstate;
update_turbo_state(); update_turbo_state();
intel_pstate_get_min_max(cpu, &min_pstate, &max_pstate); pstate = intel_pstate_get_base_pstate(cpu);
intel_pstate_set_pstate(cpu, max_pstate); pstate = max(cpu->pstate.min_pstate,
fp_ext_toint(pstate * cpu->max_perf));
intel_pstate_set_pstate(cpu, pstate);
} }
static void intel_pstate_get_cpu_pstates(struct cpudata *cpu) static void intel_pstate_get_cpu_pstates(struct cpudata *cpu)
...@@ -1767,7 +1589,11 @@ static inline bool intel_pstate_sample(struct cpudata *cpu, u64 time) ...@@ -1767,7 +1589,11 @@ static inline bool intel_pstate_sample(struct cpudata *cpu, u64 time)
* that sample.time will always be reset before setting the utilization * that sample.time will always be reset before setting the utilization
* update hook and make the caller skip the sample then. * update hook and make the caller skip the sample then.
*/ */
return !!cpu->last_sample_time; if (cpu->last_sample_time) {
intel_pstate_calc_avg_perf(cpu);
return true;
}
return false;
} }
static inline int32_t get_avg_frequency(struct cpudata *cpu) static inline int32_t get_avg_frequency(struct cpudata *cpu)
...@@ -1788,6 +1614,9 @@ static inline int32_t get_target_pstate_use_cpu_load(struct cpudata *cpu) ...@@ -1788,6 +1614,9 @@ static inline int32_t get_target_pstate_use_cpu_load(struct cpudata *cpu)
int32_t busy_frac, boost; int32_t busy_frac, boost;
int target, avg_pstate; int target, avg_pstate;
if (cpu->policy == CPUFREQ_POLICY_PERFORMANCE)
return cpu->pstate.turbo_pstate;
busy_frac = div_fp(sample->mperf, sample->tsc); busy_frac = div_fp(sample->mperf, sample->tsc);
boost = cpu->iowait_boost; boost = cpu->iowait_boost;
...@@ -1824,6 +1653,9 @@ static inline int32_t get_target_pstate_use_performance(struct cpudata *cpu) ...@@ -1824,6 +1653,9 @@ static inline int32_t get_target_pstate_use_performance(struct cpudata *cpu)
int32_t perf_scaled, max_pstate, current_pstate, sample_ratio; int32_t perf_scaled, max_pstate, current_pstate, sample_ratio;
u64 duration_ns; u64 duration_ns;
if (cpu->policy == CPUFREQ_POLICY_PERFORMANCE)
return cpu->pstate.turbo_pstate;
/* /*
* perf_scaled is the ratio of the average P-state during the last * perf_scaled is the ratio of the average P-state during the last
* sampling period to the P-state requested last time (in percent). * sampling period to the P-state requested last time (in percent).
...@@ -1858,11 +1690,13 @@ static inline int32_t get_target_pstate_use_performance(struct cpudata *cpu) ...@@ -1858,11 +1690,13 @@ static inline int32_t get_target_pstate_use_performance(struct cpudata *cpu)
static int intel_pstate_prepare_request(struct cpudata *cpu, int pstate) static int intel_pstate_prepare_request(struct cpudata *cpu, int pstate)
{ {
int max_perf, min_perf; int max_pstate = intel_pstate_get_base_pstate(cpu);
int min_pstate;
intel_pstate_get_min_max(cpu, &min_perf, &max_perf); min_pstate = max(cpu->pstate.min_pstate,
pstate = clamp_t(int, pstate, min_perf, max_perf); fp_ext_toint(max_pstate * cpu->min_perf));
return pstate; max_pstate = max(min_pstate, fp_ext_toint(max_pstate * cpu->max_perf));
return clamp_t(int, pstate, min_pstate, max_pstate);
} }
static void intel_pstate_update_pstate(struct cpudata *cpu, int pstate) static void intel_pstate_update_pstate(struct cpudata *cpu, int pstate)
...@@ -1874,16 +1708,11 @@ static void intel_pstate_update_pstate(struct cpudata *cpu, int pstate) ...@@ -1874,16 +1708,11 @@ static void intel_pstate_update_pstate(struct cpudata *cpu, int pstate)
wrmsrl(MSR_IA32_PERF_CTL, pstate_funcs.get_val(cpu, pstate)); wrmsrl(MSR_IA32_PERF_CTL, pstate_funcs.get_val(cpu, pstate));
} }
static inline void intel_pstate_adjust_busy_pstate(struct cpudata *cpu) static void intel_pstate_adjust_pstate(struct cpudata *cpu, int target_pstate)
{ {
int from, target_pstate; int from = cpu->pstate.current_pstate;
struct sample *sample; struct sample *sample;
from = cpu->pstate.current_pstate;
target_pstate = cpu->policy == CPUFREQ_POLICY_PERFORMANCE ?
cpu->pstate.turbo_pstate : pstate_funcs.get_target_pstate(cpu);
update_turbo_state(); update_turbo_state();
target_pstate = intel_pstate_prepare_request(cpu, target_pstate); target_pstate = intel_pstate_prepare_request(cpu, target_pstate);
...@@ -1902,13 +1731,39 @@ static inline void intel_pstate_adjust_busy_pstate(struct cpudata *cpu) ...@@ -1902,13 +1731,39 @@ static inline void intel_pstate_adjust_busy_pstate(struct cpudata *cpu)
fp_toint(cpu->iowait_boost * 100)); fp_toint(cpu->iowait_boost * 100));
} }
static void intel_pstate_update_util_hwp(struct update_util_data *data,
u64 time, unsigned int flags)
{
struct cpudata *cpu = container_of(data, struct cpudata, update_util);
u64 delta_ns = time - cpu->sample.time;
if ((s64)delta_ns >= INTEL_PSTATE_HWP_SAMPLING_INTERVAL)
intel_pstate_sample(cpu, time);
}
static void intel_pstate_update_util_pid(struct update_util_data *data,
u64 time, unsigned int flags)
{
struct cpudata *cpu = container_of(data, struct cpudata, update_util);
u64 delta_ns = time - cpu->sample.time;
if ((s64)delta_ns < pid_params.sample_rate_ns)
return;
if (intel_pstate_sample(cpu, time)) {
int target_pstate;
target_pstate = get_target_pstate_use_performance(cpu);
intel_pstate_adjust_pstate(cpu, target_pstate);
}
}
static void intel_pstate_update_util(struct update_util_data *data, u64 time, static void intel_pstate_update_util(struct update_util_data *data, u64 time,
unsigned int flags) unsigned int flags)
{ {
struct cpudata *cpu = container_of(data, struct cpudata, update_util); struct cpudata *cpu = container_of(data, struct cpudata, update_util);
u64 delta_ns; u64 delta_ns;
if (pstate_funcs.get_target_pstate == get_target_pstate_use_cpu_load) {
if (flags & SCHED_CPUFREQ_IOWAIT) { if (flags & SCHED_CPUFREQ_IOWAIT) {
cpu->iowait_boost = int_tofp(1); cpu->iowait_boost = int_tofp(1);
} else if (cpu->iowait_boost) { } else if (cpu->iowait_boost) {
...@@ -1918,60 +1773,113 @@ static void intel_pstate_update_util(struct update_util_data *data, u64 time, ...@@ -1918,60 +1773,113 @@ static void intel_pstate_update_util(struct update_util_data *data, u64 time,
cpu->iowait_boost = 0; cpu->iowait_boost = 0;
} }
cpu->last_update = time; cpu->last_update = time;
}
delta_ns = time - cpu->sample.time; delta_ns = time - cpu->sample.time;
if ((s64)delta_ns >= pid_params.sample_rate_ns) { if ((s64)delta_ns < INTEL_PSTATE_DEFAULT_SAMPLING_INTERVAL)
bool sample_taken = intel_pstate_sample(cpu, time); return;
if (sample_taken) { if (intel_pstate_sample(cpu, time)) {
intel_pstate_calc_avg_perf(cpu); int target_pstate;
if (!hwp_active)
intel_pstate_adjust_busy_pstate(cpu); target_pstate = get_target_pstate_use_cpu_load(cpu);
} intel_pstate_adjust_pstate(cpu, target_pstate);
} }
} }
static struct pstate_funcs core_funcs = {
.get_max = core_get_max_pstate,
.get_max_physical = core_get_max_pstate_physical,
.get_min = core_get_min_pstate,
.get_turbo = core_get_turbo_pstate,
.get_scaling = core_get_scaling,
.get_val = core_get_val,
.update_util = intel_pstate_update_util_pid,
};
static const struct pstate_funcs silvermont_funcs = {
.get_max = atom_get_max_pstate,
.get_max_physical = atom_get_max_pstate,
.get_min = atom_get_min_pstate,
.get_turbo = atom_get_turbo_pstate,
.get_val = atom_get_val,
.get_scaling = silvermont_get_scaling,
.get_vid = atom_get_vid,
.update_util = intel_pstate_update_util,
};
static const struct pstate_funcs airmont_funcs = {
.get_max = atom_get_max_pstate,
.get_max_physical = atom_get_max_pstate,
.get_min = atom_get_min_pstate,
.get_turbo = atom_get_turbo_pstate,
.get_val = atom_get_val,
.get_scaling = airmont_get_scaling,
.get_vid = atom_get_vid,
.update_util = intel_pstate_update_util,
};
static const struct pstate_funcs knl_funcs = {
.get_max = core_get_max_pstate,
.get_max_physical = core_get_max_pstate_physical,
.get_min = core_get_min_pstate,
.get_turbo = knl_get_turbo_pstate,
.get_scaling = core_get_scaling,
.get_val = core_get_val,
.update_util = intel_pstate_update_util_pid,
};
static const struct pstate_funcs bxt_funcs = {
.get_max = core_get_max_pstate,
.get_max_physical = core_get_max_pstate_physical,
.get_min = core_get_min_pstate,
.get_turbo = core_get_turbo_pstate,
.get_scaling = core_get_scaling,
.get_val = core_get_val,
.update_util = intel_pstate_update_util,
};
#define ICPU(model, policy) \ #define ICPU(model, policy) \
{ X86_VENDOR_INTEL, 6, model, X86_FEATURE_APERFMPERF,\ { X86_VENDOR_INTEL, 6, model, X86_FEATURE_APERFMPERF,\
(unsigned long)&policy } (unsigned long)&policy }
static const struct x86_cpu_id intel_pstate_cpu_ids[] = { static const struct x86_cpu_id intel_pstate_cpu_ids[] = {
ICPU(INTEL_FAM6_SANDYBRIDGE, core_params), ICPU(INTEL_FAM6_SANDYBRIDGE, core_funcs),
ICPU(INTEL_FAM6_SANDYBRIDGE_X, core_params), ICPU(INTEL_FAM6_SANDYBRIDGE_X, core_funcs),
ICPU(INTEL_FAM6_ATOM_SILVERMONT1, silvermont_params), ICPU(INTEL_FAM6_ATOM_SILVERMONT1, silvermont_funcs),
ICPU(INTEL_FAM6_IVYBRIDGE, core_params), ICPU(INTEL_FAM6_IVYBRIDGE, core_funcs),
ICPU(INTEL_FAM6_HASWELL_CORE, core_params), ICPU(INTEL_FAM6_HASWELL_CORE, core_funcs),
ICPU(INTEL_FAM6_BROADWELL_CORE, core_params), ICPU(INTEL_FAM6_BROADWELL_CORE, core_funcs),
ICPU(INTEL_FAM6_IVYBRIDGE_X, core_params), ICPU(INTEL_FAM6_IVYBRIDGE_X, core_funcs),
ICPU(INTEL_FAM6_HASWELL_X, core_params), ICPU(INTEL_FAM6_HASWELL_X, core_funcs),
ICPU(INTEL_FAM6_HASWELL_ULT, core_params), ICPU(INTEL_FAM6_HASWELL_ULT, core_funcs),
ICPU(INTEL_FAM6_HASWELL_GT3E, core_params), ICPU(INTEL_FAM6_HASWELL_GT3E, core_funcs),
ICPU(INTEL_FAM6_BROADWELL_GT3E, core_params), ICPU(INTEL_FAM6_BROADWELL_GT3E, core_funcs),
ICPU(INTEL_FAM6_ATOM_AIRMONT, airmont_params), ICPU(INTEL_FAM6_ATOM_AIRMONT, airmont_funcs),
ICPU(INTEL_FAM6_SKYLAKE_MOBILE, core_params), ICPU(INTEL_FAM6_SKYLAKE_MOBILE, core_funcs),
ICPU(INTEL_FAM6_BROADWELL_X, core_params), ICPU(INTEL_FAM6_BROADWELL_X, core_funcs),
ICPU(INTEL_FAM6_SKYLAKE_DESKTOP, core_params), ICPU(INTEL_FAM6_SKYLAKE_DESKTOP, core_funcs),
ICPU(INTEL_FAM6_BROADWELL_XEON_D, core_params), ICPU(INTEL_FAM6_BROADWELL_XEON_D, core_funcs),
ICPU(INTEL_FAM6_XEON_PHI_KNL, knl_params), ICPU(INTEL_FAM6_XEON_PHI_KNL, knl_funcs),
ICPU(INTEL_FAM6_XEON_PHI_KNM, knl_params), ICPU(INTEL_FAM6_XEON_PHI_KNM, knl_funcs),
ICPU(INTEL_FAM6_ATOM_GOLDMONT, bxt_params), ICPU(INTEL_FAM6_ATOM_GOLDMONT, bxt_funcs),
ICPU(INTEL_FAM6_ATOM_GEMINI_LAKE, bxt_funcs),
{} {}
}; };
MODULE_DEVICE_TABLE(x86cpu, intel_pstate_cpu_ids); MODULE_DEVICE_TABLE(x86cpu, intel_pstate_cpu_ids);
static const struct x86_cpu_id intel_pstate_cpu_oob_ids[] __initconst = { static const struct x86_cpu_id intel_pstate_cpu_oob_ids[] __initconst = {
ICPU(INTEL_FAM6_BROADWELL_XEON_D, core_params), ICPU(INTEL_FAM6_BROADWELL_XEON_D, core_funcs),
ICPU(INTEL_FAM6_BROADWELL_X, core_params), ICPU(INTEL_FAM6_BROADWELL_X, core_funcs),
ICPU(INTEL_FAM6_SKYLAKE_X, core_params), ICPU(INTEL_FAM6_SKYLAKE_X, core_funcs),
{} {}
}; };
static const struct x86_cpu_id intel_pstate_cpu_ee_disable_ids[] = { static const struct x86_cpu_id intel_pstate_cpu_ee_disable_ids[] = {
ICPU(INTEL_FAM6_KABYLAKE_DESKTOP, core_params), ICPU(INTEL_FAM6_KABYLAKE_DESKTOP, core_funcs),
{} {}
}; };
static bool pid_in_use(void);
static int intel_pstate_init_cpu(unsigned int cpunum) static int intel_pstate_init_cpu(unsigned int cpunum)
{ {
struct cpudata *cpu; struct cpudata *cpu;
...@@ -1979,18 +1887,11 @@ static int intel_pstate_init_cpu(unsigned int cpunum) ...@@ -1979,18 +1887,11 @@ static int intel_pstate_init_cpu(unsigned int cpunum)
cpu = all_cpu_data[cpunum]; cpu = all_cpu_data[cpunum];
if (!cpu) { if (!cpu) {
unsigned int size = sizeof(struct cpudata); cpu = kzalloc(sizeof(*cpu), GFP_KERNEL);
if (per_cpu_limits)
size += sizeof(struct perf_limits);
cpu = kzalloc(size, GFP_KERNEL);
if (!cpu) if (!cpu)
return -ENOMEM; return -ENOMEM;
all_cpu_data[cpunum] = cpu; all_cpu_data[cpunum] = cpu;
if (per_cpu_limits)
cpu->perf_limits = (struct perf_limits *)(cpu + 1);
cpu->epp_default = -EINVAL; cpu->epp_default = -EINVAL;
cpu->epp_powersave = -EINVAL; cpu->epp_powersave = -EINVAL;
...@@ -2009,14 +1910,12 @@ static int intel_pstate_init_cpu(unsigned int cpunum) ...@@ -2009,14 +1910,12 @@ static int intel_pstate_init_cpu(unsigned int cpunum)
intel_pstate_disable_ee(cpunum); intel_pstate_disable_ee(cpunum);
intel_pstate_hwp_enable(cpu); intel_pstate_hwp_enable(cpu);
pid_params.sample_rate_ms = 50; } else if (pid_in_use()) {
pid_params.sample_rate_ns = 50 * NSEC_PER_MSEC; intel_pstate_pid_reset(cpu);
} }
intel_pstate_get_cpu_pstates(cpu); intel_pstate_get_cpu_pstates(cpu);
intel_pstate_busy_pid_reset(cpu);
pr_debug("controlling: cpu %d\n", cpunum); pr_debug("controlling: cpu %d\n", cpunum);
return 0; return 0;
...@@ -2039,7 +1938,7 @@ static void intel_pstate_set_update_util_hook(unsigned int cpu_num) ...@@ -2039,7 +1938,7 @@ static void intel_pstate_set_update_util_hook(unsigned int cpu_num)
/* Prevent intel_pstate_update_util() from using stale data. */ /* Prevent intel_pstate_update_util() from using stale data. */
cpu->sample.time = 0; cpu->sample.time = 0;
cpufreq_add_update_util_hook(cpu_num, &cpu->update_util, cpufreq_add_update_util_hook(cpu_num, &cpu->update_util,
intel_pstate_update_util); pstate_funcs.update_util);
cpu->update_util_set = true; cpu->update_util_set = true;
} }
...@@ -2055,46 +1954,68 @@ static void intel_pstate_clear_update_util_hook(unsigned int cpu) ...@@ -2055,46 +1954,68 @@ static void intel_pstate_clear_update_util_hook(unsigned int cpu)
synchronize_sched(); synchronize_sched();
} }
static int intel_pstate_get_max_freq(struct cpudata *cpu)
{
return global.turbo_disabled || global.no_turbo ?
cpu->pstate.max_freq : cpu->pstate.turbo_freq;
}
static void intel_pstate_update_perf_limits(struct cpufreq_policy *policy, static void intel_pstate_update_perf_limits(struct cpufreq_policy *policy,
struct perf_limits *limits) struct cpudata *cpu)
{ {
int max_freq = intel_pstate_get_max_freq(cpu);
int32_t max_policy_perf, min_policy_perf; int32_t max_policy_perf, min_policy_perf;
max_policy_perf = div_ext_fp(policy->max, policy->cpuinfo.max_freq); max_policy_perf = div_ext_fp(policy->max, max_freq);
max_policy_perf = clamp_t(int32_t, max_policy_perf, 0, int_ext_tofp(1)); max_policy_perf = clamp_t(int32_t, max_policy_perf, 0, int_ext_tofp(1));
if (policy->max == policy->min) { if (policy->max == policy->min) {
min_policy_perf = max_policy_perf; min_policy_perf = max_policy_perf;
} else { } else {
min_policy_perf = div_ext_fp(policy->min, min_policy_perf = div_ext_fp(policy->min, max_freq);
policy->cpuinfo.max_freq);
min_policy_perf = clamp_t(int32_t, min_policy_perf, min_policy_perf = clamp_t(int32_t, min_policy_perf,
0, max_policy_perf); 0, max_policy_perf);
} }
/* Normalize user input to [min_perf, max_perf] */ /* Normalize user input to [min_perf, max_perf] */
limits->min_perf = max(min_policy_perf, if (per_cpu_limits) {
percent_ext_fp(limits->min_sysfs_pct)); cpu->min_perf = min_policy_perf;
limits->min_perf = min(limits->min_perf, max_policy_perf); cpu->max_perf = max_policy_perf;
limits->max_perf = min(max_policy_perf, } else {
percent_ext_fp(limits->max_sysfs_pct)); int32_t global_min, global_max;
limits->max_perf = max(min_policy_perf, limits->max_perf);
/* Global limits are in percent of the maximum turbo P-state. */
global_max = percent_ext_fp(global.max_perf_pct);
global_min = percent_ext_fp(global.min_perf_pct);
if (max_freq != cpu->pstate.turbo_freq) {
int32_t turbo_factor;
turbo_factor = div_ext_fp(cpu->pstate.turbo_pstate,
cpu->pstate.max_pstate);
global_min = mul_ext_fp(global_min, turbo_factor);
global_max = mul_ext_fp(global_max, turbo_factor);
}
global_min = clamp_t(int32_t, global_min, 0, global_max);
cpu->min_perf = max(min_policy_perf, global_min);
cpu->min_perf = min(cpu->min_perf, max_policy_perf);
cpu->max_perf = min(max_policy_perf, global_max);
cpu->max_perf = max(min_policy_perf, cpu->max_perf);
/* Make sure min_perf <= max_perf */ /* Make sure min_perf <= max_perf */
limits->min_perf = min(limits->min_perf, limits->max_perf); cpu->min_perf = min(cpu->min_perf, cpu->max_perf);
}
limits->max_perf = round_up(limits->max_perf, EXT_FRAC_BITS); cpu->max_perf = round_up(cpu->max_perf, EXT_FRAC_BITS);
limits->min_perf = round_up(limits->min_perf, EXT_FRAC_BITS); cpu->min_perf = round_up(cpu->min_perf, EXT_FRAC_BITS);
limits->max_perf_pct = fp_ext_toint(limits->max_perf * 100);
limits->min_perf_pct = fp_ext_toint(limits->min_perf * 100);
pr_debug("cpu:%d max_perf_pct:%d min_perf_pct:%d\n", policy->cpu, pr_debug("cpu:%d max_perf_pct:%d min_perf_pct:%d\n", policy->cpu,
limits->max_perf_pct, limits->min_perf_pct); fp_ext_toint(cpu->max_perf * 100),
fp_ext_toint(cpu->min_perf * 100));
} }
static int intel_pstate_set_policy(struct cpufreq_policy *policy) static int intel_pstate_set_policy(struct cpufreq_policy *policy)
{ {
struct cpudata *cpu; struct cpudata *cpu;
struct perf_limits *perf_limits = &global;
if (!policy->cpuinfo.max_freq) if (!policy->cpuinfo.max_freq)
return -ENODEV; return -ENODEV;
...@@ -2105,19 +2026,9 @@ static int intel_pstate_set_policy(struct cpufreq_policy *policy) ...@@ -2105,19 +2026,9 @@ static int intel_pstate_set_policy(struct cpufreq_policy *policy)
cpu = all_cpu_data[policy->cpu]; cpu = all_cpu_data[policy->cpu];
cpu->policy = policy->policy; cpu->policy = policy->policy;
if (cpu->pstate.max_pstate_physical > cpu->pstate.max_pstate &&
policy->max < policy->cpuinfo.max_freq &&
policy->max > cpu->pstate.max_pstate * cpu->pstate.scaling) {
pr_debug("policy->max > max non turbo frequency\n");
policy->max = policy->cpuinfo.max_freq;
}
if (per_cpu_limits)
perf_limits = cpu->perf_limits;
mutex_lock(&intel_pstate_limits_lock); mutex_lock(&intel_pstate_limits_lock);
intel_pstate_update_perf_limits(policy, perf_limits); intel_pstate_update_perf_limits(policy, cpu);
if (cpu->policy == CPUFREQ_POLICY_PERFORMANCE) { if (cpu->policy == CPUFREQ_POLICY_PERFORMANCE) {
/* /*
...@@ -2130,38 +2041,38 @@ static int intel_pstate_set_policy(struct cpufreq_policy *policy) ...@@ -2130,38 +2041,38 @@ static int intel_pstate_set_policy(struct cpufreq_policy *policy)
intel_pstate_set_update_util_hook(policy->cpu); intel_pstate_set_update_util_hook(policy->cpu);
intel_pstate_hwp_set_policy(policy); if (hwp_active)
intel_pstate_hwp_set(policy->cpu);
mutex_unlock(&intel_pstate_limits_lock); mutex_unlock(&intel_pstate_limits_lock);
return 0; return 0;
} }
static void intel_pstate_adjust_policy_max(struct cpufreq_policy *policy,
struct cpudata *cpu)
{
if (cpu->pstate.max_pstate_physical > cpu->pstate.max_pstate &&
policy->max < policy->cpuinfo.max_freq &&
policy->max > cpu->pstate.max_freq) {
pr_debug("policy->max > max non turbo frequency\n");
policy->max = policy->cpuinfo.max_freq;
}
}
static int intel_pstate_verify_policy(struct cpufreq_policy *policy) static int intel_pstate_verify_policy(struct cpufreq_policy *policy)
{ {
struct cpudata *cpu = all_cpu_data[policy->cpu]; struct cpudata *cpu = all_cpu_data[policy->cpu];
update_turbo_state(); update_turbo_state();
policy->cpuinfo.max_freq = global.turbo_disabled || global.no_turbo ? cpufreq_verify_within_limits(policy, policy->cpuinfo.min_freq,
cpu->pstate.max_freq : intel_pstate_get_max_freq(cpu));
cpu->pstate.turbo_freq;
cpufreq_verify_within_cpu_limits(policy);
if (policy->policy != CPUFREQ_POLICY_POWERSAVE && if (policy->policy != CPUFREQ_POLICY_POWERSAVE &&
policy->policy != CPUFREQ_POLICY_PERFORMANCE) policy->policy != CPUFREQ_POLICY_PERFORMANCE)
return -EINVAL; return -EINVAL;
/* When per-CPU limits are used, sysfs limits are not used */ intel_pstate_adjust_policy_max(policy, cpu);
if (!per_cpu_limits) {
unsigned int max_freq, min_freq;
max_freq = policy->cpuinfo.max_freq *
global.max_sysfs_pct / 100;
min_freq = policy->cpuinfo.max_freq *
global.min_sysfs_pct / 100;
cpufreq_verify_within_limits(policy, min_freq, max_freq);
}
return 0; return 0;
} }
...@@ -2202,8 +2113,8 @@ static int __intel_pstate_cpu_init(struct cpufreq_policy *policy) ...@@ -2202,8 +2113,8 @@ static int __intel_pstate_cpu_init(struct cpufreq_policy *policy)
cpu = all_cpu_data[policy->cpu]; cpu = all_cpu_data[policy->cpu];
if (per_cpu_limits) cpu->max_perf = int_ext_tofp(1);
intel_pstate_init_limits(cpu->perf_limits); cpu->min_perf = 0;
policy->min = cpu->pstate.min_pstate * cpu->pstate.scaling; policy->min = cpu->pstate.min_pstate * cpu->pstate.scaling;
policy->max = cpu->pstate.turbo_pstate * cpu->pstate.scaling; policy->max = cpu->pstate.turbo_pstate * cpu->pstate.scaling;
...@@ -2257,10 +2168,12 @@ static int intel_cpufreq_verify_policy(struct cpufreq_policy *policy) ...@@ -2257,10 +2168,12 @@ static int intel_cpufreq_verify_policy(struct cpufreq_policy *policy)
struct cpudata *cpu = all_cpu_data[policy->cpu]; struct cpudata *cpu = all_cpu_data[policy->cpu];
update_turbo_state(); update_turbo_state();
policy->cpuinfo.max_freq = global.no_turbo || global.turbo_disabled ? cpufreq_verify_within_limits(policy, policy->cpuinfo.min_freq,
cpu->pstate.max_freq : cpu->pstate.turbo_freq; intel_pstate_get_max_freq(cpu));
cpufreq_verify_within_cpu_limits(policy); intel_pstate_adjust_policy_max(policy, cpu);
intel_pstate_update_perf_limits(policy, cpu);
return 0; return 0;
} }
...@@ -2341,7 +2254,13 @@ static struct cpufreq_driver intel_cpufreq = { ...@@ -2341,7 +2254,13 @@ static struct cpufreq_driver intel_cpufreq = {
.name = "intel_cpufreq", .name = "intel_cpufreq",
}; };
static struct cpufreq_driver *intel_pstate_driver = &intel_pstate; static struct cpufreq_driver *default_driver = &intel_pstate;
static bool pid_in_use(void)
{
return intel_pstate_driver == &intel_pstate &&
pstate_funcs.update_util == intel_pstate_update_util_pid;
}
static void intel_pstate_driver_cleanup(void) static void intel_pstate_driver_cleanup(void)
{ {
...@@ -2358,26 +2277,26 @@ static void intel_pstate_driver_cleanup(void) ...@@ -2358,26 +2277,26 @@ static void intel_pstate_driver_cleanup(void)
} }
} }
put_online_cpus(); put_online_cpus();
intel_pstate_driver = NULL;
} }
static int intel_pstate_register_driver(void) static int intel_pstate_register_driver(struct cpufreq_driver *driver)
{ {
int ret; int ret;
intel_pstate_init_limits(&global); memset(&global, 0, sizeof(global));
global.max_perf_pct = 100;
intel_pstate_driver = driver;
ret = cpufreq_register_driver(intel_pstate_driver); ret = cpufreq_register_driver(intel_pstate_driver);
if (ret) { if (ret) {
intel_pstate_driver_cleanup(); intel_pstate_driver_cleanup();
return ret; return ret;
} }
mutex_lock(&intel_pstate_limits_lock); global.min_perf_pct = min_perf_pct_min();
driver_registered = true;
mutex_unlock(&intel_pstate_limits_lock);
if (intel_pstate_driver == &intel_pstate && !hwp_active && if (pid_in_use())
pstate_funcs.get_target_pstate != get_target_pstate_use_cpu_load)
intel_pstate_debug_expose_params(); intel_pstate_debug_expose_params();
return 0; return 0;
...@@ -2388,14 +2307,9 @@ static int intel_pstate_unregister_driver(void) ...@@ -2388,14 +2307,9 @@ static int intel_pstate_unregister_driver(void)
if (hwp_active) if (hwp_active)
return -EBUSY; return -EBUSY;
if (intel_pstate_driver == &intel_pstate && !hwp_active && if (pid_in_use())
pstate_funcs.get_target_pstate != get_target_pstate_use_cpu_load)
intel_pstate_debug_hide_params(); intel_pstate_debug_hide_params();
mutex_lock(&intel_pstate_limits_lock);
driver_registered = false;
mutex_unlock(&intel_pstate_limits_lock);
cpufreq_unregister_driver(intel_pstate_driver); cpufreq_unregister_driver(intel_pstate_driver);
intel_pstate_driver_cleanup(); intel_pstate_driver_cleanup();
...@@ -2404,7 +2318,7 @@ static int intel_pstate_unregister_driver(void) ...@@ -2404,7 +2318,7 @@ static int intel_pstate_unregister_driver(void)
static ssize_t intel_pstate_show_status(char *buf) static ssize_t intel_pstate_show_status(char *buf)
{ {
if (!driver_registered) if (!intel_pstate_driver)
return sprintf(buf, "off\n"); return sprintf(buf, "off\n");
return sprintf(buf, "%s\n", intel_pstate_driver == &intel_pstate ? return sprintf(buf, "%s\n", intel_pstate_driver == &intel_pstate ?
...@@ -2416,11 +2330,11 @@ static int intel_pstate_update_status(const char *buf, size_t size) ...@@ -2416,11 +2330,11 @@ static int intel_pstate_update_status(const char *buf, size_t size)
int ret; int ret;
if (size == 3 && !strncmp(buf, "off", size)) if (size == 3 && !strncmp(buf, "off", size))
return driver_registered ? return intel_pstate_driver ?
intel_pstate_unregister_driver() : -EINVAL; intel_pstate_unregister_driver() : -EINVAL;
if (size == 6 && !strncmp(buf, "active", size)) { if (size == 6 && !strncmp(buf, "active", size)) {
if (driver_registered) { if (intel_pstate_driver) {
if (intel_pstate_driver == &intel_pstate) if (intel_pstate_driver == &intel_pstate)
return 0; return 0;
...@@ -2429,13 +2343,12 @@ static int intel_pstate_update_status(const char *buf, size_t size) ...@@ -2429,13 +2343,12 @@ static int intel_pstate_update_status(const char *buf, size_t size)
return ret; return ret;
} }
intel_pstate_driver = &intel_pstate; return intel_pstate_register_driver(&intel_pstate);
return intel_pstate_register_driver();
} }
if (size == 7 && !strncmp(buf, "passive", size)) { if (size == 7 && !strncmp(buf, "passive", size)) {
if (driver_registered) { if (intel_pstate_driver) {
if (intel_pstate_driver != &intel_pstate) if (intel_pstate_driver == &intel_cpufreq)
return 0; return 0;
ret = intel_pstate_unregister_driver(); ret = intel_pstate_unregister_driver();
...@@ -2443,8 +2356,7 @@ static int intel_pstate_update_status(const char *buf, size_t size) ...@@ -2443,8 +2356,7 @@ static int intel_pstate_update_status(const char *buf, size_t size)
return ret; return ret;
} }
intel_pstate_driver = &intel_cpufreq; return intel_pstate_register_driver(&intel_cpufreq);
return intel_pstate_register_driver();
} }
return -EINVAL; return -EINVAL;
...@@ -2465,23 +2377,17 @@ static int __init intel_pstate_msrs_not_valid(void) ...@@ -2465,23 +2377,17 @@ static int __init intel_pstate_msrs_not_valid(void)
return 0; return 0;
} }
static void __init copy_pid_params(struct pstate_adjust_policy *policy)
{
pid_params.sample_rate_ms = policy->sample_rate_ms;
pid_params.sample_rate_ns = pid_params.sample_rate_ms * NSEC_PER_MSEC;
pid_params.p_gain_pct = policy->p_gain_pct;
pid_params.i_gain_pct = policy->i_gain_pct;
pid_params.d_gain_pct = policy->d_gain_pct;
pid_params.deadband = policy->deadband;
pid_params.setpoint = policy->setpoint;
}
#ifdef CONFIG_ACPI #ifdef CONFIG_ACPI
static void intel_pstate_use_acpi_profile(void) static void intel_pstate_use_acpi_profile(void)
{ {
if (acpi_gbl_FADT.preferred_profile == PM_MOBILE) switch (acpi_gbl_FADT.preferred_profile) {
pstate_funcs.get_target_pstate = case PM_MOBILE:
get_target_pstate_use_cpu_load; case PM_TABLET:
case PM_APPLIANCE_PC:
case PM_DESKTOP:
case PM_WORKSTATION:
pstate_funcs.update_util = intel_pstate_update_util;
}
} }
#else #else
static void intel_pstate_use_acpi_profile(void) static void intel_pstate_use_acpi_profile(void)
...@@ -2498,7 +2404,7 @@ static void __init copy_cpu_funcs(struct pstate_funcs *funcs) ...@@ -2498,7 +2404,7 @@ static void __init copy_cpu_funcs(struct pstate_funcs *funcs)
pstate_funcs.get_scaling = funcs->get_scaling; pstate_funcs.get_scaling = funcs->get_scaling;
pstate_funcs.get_val = funcs->get_val; pstate_funcs.get_val = funcs->get_val;
pstate_funcs.get_vid = funcs->get_vid; pstate_funcs.get_vid = funcs->get_vid;
pstate_funcs.get_target_pstate = funcs->get_target_pstate; pstate_funcs.update_util = funcs->update_util;
intel_pstate_use_acpi_profile(); intel_pstate_use_acpi_profile();
} }
...@@ -2637,28 +2543,30 @@ static const struct x86_cpu_id hwp_support_ids[] __initconst = { ...@@ -2637,28 +2543,30 @@ static const struct x86_cpu_id hwp_support_ids[] __initconst = {
static int __init intel_pstate_init(void) static int __init intel_pstate_init(void)
{ {
const struct x86_cpu_id *id; int rc;
struct cpu_defaults *cpu_def;
int rc = 0;
if (no_load) if (no_load)
return -ENODEV; return -ENODEV;
if (x86_match_cpu(hwp_support_ids) && !no_hwp) { if (x86_match_cpu(hwp_support_ids)) {
copy_cpu_funcs(&core_params.funcs); copy_cpu_funcs(&core_funcs);
if (no_hwp) {
pstate_funcs.update_util = intel_pstate_update_util;
} else {
hwp_active++; hwp_active++;
intel_pstate.attr = hwp_cpufreq_attrs; intel_pstate.attr = hwp_cpufreq_attrs;
pstate_funcs.update_util = intel_pstate_update_util_hwp;
goto hwp_cpu_matched; goto hwp_cpu_matched;
} }
} else {
const struct x86_cpu_id *id;
id = x86_match_cpu(intel_pstate_cpu_ids); id = x86_match_cpu(intel_pstate_cpu_ids);
if (!id) if (!id)
return -ENODEV; return -ENODEV;
cpu_def = (struct cpu_defaults *)id->driver_data; copy_cpu_funcs((struct pstate_funcs *)id->driver_data);
}
copy_pid_params(&cpu_def->pid_policy);
copy_cpu_funcs(&cpu_def->funcs);
if (intel_pstate_msrs_not_valid()) if (intel_pstate_msrs_not_valid())
return -ENODEV; return -ENODEV;
...@@ -2685,7 +2593,7 @@ static int __init intel_pstate_init(void) ...@@ -2685,7 +2593,7 @@ static int __init intel_pstate_init(void)
intel_pstate_sysfs_expose_params(); intel_pstate_sysfs_expose_params();
mutex_lock(&intel_pstate_driver_lock); mutex_lock(&intel_pstate_driver_lock);
rc = intel_pstate_register_driver(); rc = intel_pstate_register_driver(default_driver);
mutex_unlock(&intel_pstate_driver_lock); mutex_unlock(&intel_pstate_driver_lock);
if (rc) if (rc)
return rc; return rc;
...@@ -2706,7 +2614,7 @@ static int __init intel_pstate_setup(char *str) ...@@ -2706,7 +2614,7 @@ static int __init intel_pstate_setup(char *str)
no_load = 1; no_load = 1;
} else if (!strcmp(str, "passive")) { } else if (!strcmp(str, "passive")) {
pr_info("Passive mode enabled\n"); pr_info("Passive mode enabled\n");
intel_pstate_driver = &intel_cpufreq; default_driver = &intel_cpufreq;
no_hwp = 1; no_hwp = 1;
} }
if (!strcmp(str, "no_hwp")) { if (!strcmp(str, "no_hwp")) {
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment