cpufreq.c 64.6 KB
Newer Older
Linus Torvalds's avatar
Linus Torvalds committed
1 2 3 4 5
/*
 *  linux/drivers/cpufreq/cpufreq.c
 *
 *  Copyright (C) 2001 Russell King
 *            (C) 2002 - 2003 Dominik Brodowski <linux@brodo.de>
6
 *            (C) 2013 Viresh Kumar <viresh.kumar@linaro.org>
Linus Torvalds's avatar
Linus Torvalds committed
7
 *
8
 *  Oct 2005 - Ashok Raj <ashok.raj@intel.com>
9
 *	Added handling for CPU hotplug
10 11
 *  Feb 2006 - Jacob Shin <jacob.shin@amd.com>
 *	Fix handling for CPU hotplug -- affected CPUs
12
 *
Linus Torvalds's avatar
Linus Torvalds committed
13 14 15 16 17
 * This program is free software; you can redistribute it and/or modify
 * it under the terms of the GNU General Public License version 2 as
 * published by the Free Software Foundation.
 */

Viresh Kumar's avatar
Viresh Kumar committed
18 19
#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt

20
#include <linux/cpu.h>
Linus Torvalds's avatar
Linus Torvalds committed
21 22 23
#include <linux/cpufreq.h>
#include <linux/delay.h>
#include <linux/device.h>
24 25 26
#include <linux/init.h>
#include <linux/kernel_stat.h>
#include <linux/module.h>
27
#include <linux/mutex.h>
28
#include <linux/slab.h>
29
#include <linux/suspend.h>
30
#include <linux/syscore_ops.h>
31
#include <linux/tick.h>
32 33
#include <trace/events/power.h>

34
static LIST_HEAD(cpufreq_policy_list);
35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89

static inline bool policy_is_inactive(struct cpufreq_policy *policy)
{
	return cpumask_empty(policy->cpus);
}

static bool suitable_policy(struct cpufreq_policy *policy, bool active)
{
	return active == !policy_is_inactive(policy);
}

/* Finds Next Acive/Inactive policy */
static struct cpufreq_policy *next_policy(struct cpufreq_policy *policy,
					  bool active)
{
	do {
		policy = list_next_entry(policy, policy_list);

		/* No more policies in the list */
		if (&policy->policy_list == &cpufreq_policy_list)
			return NULL;
	} while (!suitable_policy(policy, active));

	return policy;
}

static struct cpufreq_policy *first_policy(bool active)
{
	struct cpufreq_policy *policy;

	/* No policies in the list */
	if (list_empty(&cpufreq_policy_list))
		return NULL;

	policy = list_first_entry(&cpufreq_policy_list, typeof(*policy),
				  policy_list);

	if (!suitable_policy(policy, active))
		policy = next_policy(policy, active);

	return policy;
}

/* Macros to iterate over CPU policies */
#define for_each_suitable_policy(__policy, __active)	\
	for (__policy = first_policy(__active);		\
	     __policy;					\
	     __policy = next_policy(__policy, __active))

#define for_each_active_policy(__policy)		\
	for_each_suitable_policy(__policy, true)
#define for_each_inactive_policy(__policy)		\
	for_each_suitable_policy(__policy, false)

#define for_each_policy(__policy)			\
90 91
	list_for_each_entry(__policy, &cpufreq_policy_list, policy_list)

92 93 94 95 96
/* Iterate over governors */
static LIST_HEAD(cpufreq_governor_list);
#define for_each_governor(__governor)				\
	list_for_each_entry(__governor, &cpufreq_governor_list, governor_list)

Linus Torvalds's avatar
Linus Torvalds committed
97
/**
Dave Jones's avatar
Dave Jones committed
98
 * The "cpufreq driver" - the arch- or hardware-dependent low
Linus Torvalds's avatar
Linus Torvalds committed
99 100 101
 * level driver of CPUFreq support, and its spinlock. This lock
 * also protects the cpufreq_cpu_data array.
 */
102
static struct cpufreq_driver *cpufreq_driver;
103
static DEFINE_PER_CPU(struct cpufreq_policy *, cpufreq_cpu_data);
104
static DEFINE_RWLOCK(cpufreq_driver_lock);
105
DEFINE_MUTEX(cpufreq_governor_lock);
106

107 108
/* Flag to suspend/resume CPUFreq governors */
static bool cpufreq_suspended;
Linus Torvalds's avatar
Linus Torvalds committed
109

110 111 112 113 114
static inline bool has_target(void)
{
	return cpufreq_driver->target_index || cpufreq_driver->target;
}

Linus Torvalds's avatar
Linus Torvalds committed
115
/* internal prototypes */
116 117
static int __cpufreq_governor(struct cpufreq_policy *policy,
		unsigned int event);
118
static unsigned int __cpufreq_get(struct cpufreq_policy *policy);
119
static void handle_update(struct work_struct *work);
Linus Torvalds's avatar
Linus Torvalds committed
120 121

/**
122 123
 * Two notifier lists: the "policy" list is involved in the
 * validation process for a new CPU frequency policy; the
Linus Torvalds's avatar
Linus Torvalds committed
124 125 126 127
 * "transition" list for kernel code that needs to handle
 * changes to devices when the CPU clock speed changes.
 * The mutex locks both lists.
 */
128
static BLOCKING_NOTIFIER_HEAD(cpufreq_policy_notifier_list);
129
static struct srcu_notifier_head cpufreq_transition_notifier_list;
Linus Torvalds's avatar
Linus Torvalds committed
130

131
static bool init_cpufreq_transition_notifier_list_called;
132 133 134
static int __init init_cpufreq_transition_notifier_list(void)
{
	srcu_init_notifier_head(&cpufreq_transition_notifier_list);
135
	init_cpufreq_transition_notifier_list_called = true;
136 137
	return 0;
}
138
pure_initcall(init_cpufreq_transition_notifier_list);
Linus Torvalds's avatar
Linus Torvalds committed
139

140
static int off __read_mostly;
141
static int cpufreq_disabled(void)
142 143 144 145 146 147 148
{
	return off;
}
void disable_cpufreq(void)
{
	off = 1;
}
149
static DEFINE_MUTEX(cpufreq_governor_mutex);
Linus Torvalds's avatar
Linus Torvalds committed
150

151 152
bool have_governor_per_policy(void)
{
153
	return !!(cpufreq_driver->flags & CPUFREQ_HAVE_GOVERNOR_PER_POLICY);
154
}
155
EXPORT_SYMBOL_GPL(have_governor_per_policy);
156

157 158 159 160 161 162 163 164 165
struct kobject *get_governor_parent_kobj(struct cpufreq_policy *policy)
{
	if (have_governor_per_policy())
		return &policy->kobj;
	else
		return cpufreq_global_kobject;
}
EXPORT_SYMBOL_GPL(get_governor_parent_kobj);

166 167 168 169 170 171 172 173 174
struct cpufreq_frequency_table *cpufreq_frequency_get_table(unsigned int cpu)
{
	struct cpufreq_policy *policy = per_cpu(cpufreq_cpu_data, cpu);

	return policy && !policy_is_inactive(policy) ?
		policy->freq_table : NULL;
}
EXPORT_SYMBOL_GPL(cpufreq_frequency_get_table);

175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209
static inline u64 get_cpu_idle_time_jiffy(unsigned int cpu, u64 *wall)
{
	u64 idle_time;
	u64 cur_wall_time;
	u64 busy_time;

	cur_wall_time = jiffies64_to_cputime64(get_jiffies_64());

	busy_time = kcpustat_cpu(cpu).cpustat[CPUTIME_USER];
	busy_time += kcpustat_cpu(cpu).cpustat[CPUTIME_SYSTEM];
	busy_time += kcpustat_cpu(cpu).cpustat[CPUTIME_IRQ];
	busy_time += kcpustat_cpu(cpu).cpustat[CPUTIME_SOFTIRQ];
	busy_time += kcpustat_cpu(cpu).cpustat[CPUTIME_STEAL];
	busy_time += kcpustat_cpu(cpu).cpustat[CPUTIME_NICE];

	idle_time = cur_wall_time - busy_time;
	if (wall)
		*wall = cputime_to_usecs(cur_wall_time);

	return cputime_to_usecs(idle_time);
}

u64 get_cpu_idle_time(unsigned int cpu, u64 *wall, int io_busy)
{
	u64 idle_time = get_cpu_idle_time_us(cpu, io_busy ? wall : NULL);

	if (idle_time == -1ULL)
		return get_cpu_idle_time_jiffy(cpu, wall);
	else if (!io_busy)
		idle_time += get_cpu_iowait_time_us(cpu, wall);

	return idle_time;
}
EXPORT_SYMBOL_GPL(get_cpu_idle_time);

210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231
/*
 * This is a generic cpufreq init() routine which can be used by cpufreq
 * drivers of SMP systems. It will do following:
 * - validate & show freq table passed
 * - set policies transition latency
 * - policy->cpus with all possible CPUs
 */
int cpufreq_generic_init(struct cpufreq_policy *policy,
		struct cpufreq_frequency_table *table,
		unsigned int transition_latency)
{
	int ret;

	ret = cpufreq_table_validate_and_show(policy, table);
	if (ret) {
		pr_err("%s: invalid frequency table: %d\n", __func__, ret);
		return ret;
	}

	policy->cpuinfo.transition_latency = transition_latency;

	/*
232
	 * The driver only supports the SMP configuration where all processors
233 234 235 236 237 238 239 240
	 * share the clock and voltage and clock.
	 */
	cpumask_setall(policy->cpus);

	return 0;
}
EXPORT_SYMBOL_GPL(cpufreq_generic_init);

241
struct cpufreq_policy *cpufreq_cpu_get_raw(unsigned int cpu)
242 243 244
{
	struct cpufreq_policy *policy = per_cpu(cpufreq_cpu_data, cpu);

245 246
	return policy && cpumask_test_cpu(cpu, policy->cpus) ? policy : NULL;
}
247
EXPORT_SYMBOL_GPL(cpufreq_cpu_get_raw);
248 249 250 251 252

unsigned int cpufreq_generic_get(unsigned int cpu)
{
	struct cpufreq_policy *policy = cpufreq_cpu_get_raw(cpu);

253
	if (!policy || IS_ERR(policy->clk)) {
254 255
		pr_err("%s: No %s associated to cpu: %d\n",
		       __func__, policy ? "clk" : "policy", cpu);
256 257 258 259 260 261 262
		return 0;
	}

	return clk_get_rate(policy->clk) / 1000;
}
EXPORT_SYMBOL_GPL(cpufreq_generic_get);

263 264 265 266 267 268 269 270 271 272 273 274 275
/**
 * cpufreq_cpu_get: returns policy for a cpu and marks it busy.
 *
 * @cpu: cpu to find policy for.
 *
 * This returns policy for 'cpu', returns NULL if it doesn't exist.
 * It also increments the kobject reference count to mark it busy and so would
 * require a corresponding call to cpufreq_cpu_put() to decrement it back.
 * If corresponding call cpufreq_cpu_put() isn't made, the policy wouldn't be
 * freed as that depends on the kobj count.
 *
 * Return: A valid policy on success, otherwise NULL on failure.
 */
276
struct cpufreq_policy *cpufreq_cpu_get(unsigned int cpu)
Linus Torvalds's avatar
Linus Torvalds committed
277
{
278
	struct cpufreq_policy *policy = NULL;
Linus Torvalds's avatar
Linus Torvalds committed
279 280
	unsigned long flags;

281
	if (WARN_ON(cpu >= nr_cpu_ids))
282 283
		return NULL;

Linus Torvalds's avatar
Linus Torvalds committed
284
	/* get the cpufreq driver */
285
	read_lock_irqsave(&cpufreq_driver_lock, flags);
Linus Torvalds's avatar
Linus Torvalds committed
286

287 288
	if (cpufreq_driver) {
		/* get the CPU */
289
		policy = cpufreq_cpu_get_raw(cpu);
290 291 292
		if (policy)
			kobject_get(&policy->kobj);
	}
Linus Torvalds's avatar
Linus Torvalds committed
293

294
	read_unlock_irqrestore(&cpufreq_driver_lock, flags);
Linus Torvalds's avatar
Linus Torvalds committed
295

296
	return policy;
297
}
Linus Torvalds's avatar
Linus Torvalds committed
298 299
EXPORT_SYMBOL_GPL(cpufreq_cpu_get);

300 301 302 303 304 305 306 307
/**
 * cpufreq_cpu_put: Decrements the usage count of a policy
 *
 * @policy: policy earlier returned by cpufreq_cpu_get().
 *
 * This decrements the kobject reference count incremented earlier by calling
 * cpufreq_cpu_get().
 */
308
void cpufreq_cpu_put(struct cpufreq_policy *policy)
Linus Torvalds's avatar
Linus Torvalds committed
309
{
310
	kobject_put(&policy->kobj);
Linus Torvalds's avatar
Linus Torvalds committed
311 312 313 314 315 316 317 318 319 320 321 322
}
EXPORT_SYMBOL_GPL(cpufreq_cpu_put);

/*********************************************************************
 *            EXTERNALLY AFFECTING FREQUENCY CHANGES                 *
 *********************************************************************/

/**
 * adjust_jiffies - adjust the system "loops_per_jiffy"
 *
 * This function alters the system "loops_per_jiffy" for the clock
 * speed change. Note that loops_per_jiffy cannot be updated on SMP
323
 * systems as each CPU might be scaled differently. So, use the arch
Linus Torvalds's avatar
Linus Torvalds committed
324 325
 * per-CPU loops_per_jiffy value wherever possible.
 */
326
static void adjust_jiffies(unsigned long val, struct cpufreq_freqs *ci)
Linus Torvalds's avatar
Linus Torvalds committed
327
{
328 329 330 331
#ifndef CONFIG_SMP
	static unsigned long l_p_j_ref;
	static unsigned int l_p_j_ref_freq;

Linus Torvalds's avatar
Linus Torvalds committed
332 333 334 335 336 337
	if (ci->flags & CPUFREQ_CONST_LOOPS)
		return;

	if (!l_p_j_ref_freq) {
		l_p_j_ref = loops_per_jiffy;
		l_p_j_ref_freq = ci->old;
338 339
		pr_debug("saving %lu as reference value for loops_per_jiffy; freq is %u kHz\n",
			 l_p_j_ref, l_p_j_ref_freq);
Linus Torvalds's avatar
Linus Torvalds committed
340
	}
341
	if (val == CPUFREQ_POSTCHANGE && ci->old != ci->new) {
342 343
		loops_per_jiffy = cpufreq_scale(l_p_j_ref, l_p_j_ref_freq,
								ci->new);
344 345
		pr_debug("scaling loops_per_jiffy to %lu for frequency %u kHz\n",
			 loops_per_jiffy, ci->new);
Linus Torvalds's avatar
Linus Torvalds committed
346 347
	}
#endif
348
}
Linus Torvalds's avatar
Linus Torvalds committed
349

350
static void __cpufreq_notify_transition(struct cpufreq_policy *policy,
351
		struct cpufreq_freqs *freqs, unsigned int state)
Linus Torvalds's avatar
Linus Torvalds committed
352 353 354
{
	BUG_ON(irqs_disabled());

355 356 357
	if (cpufreq_disabled())
		return;

358
	freqs->flags = cpufreq_driver->flags;
359
	pr_debug("notification %u of frequency transition to %u kHz\n",
360
		 state, freqs->new);
Linus Torvalds's avatar
Linus Torvalds committed
361 362

	switch (state) {
363

Linus Torvalds's avatar
Linus Torvalds committed
364
	case CPUFREQ_PRECHANGE:
365
		/* detect if the driver reported a value as "old frequency"
366 367
		 * which is not equal to what the cpufreq core thinks is
		 * "old frequency".
Linus Torvalds's avatar
Linus Torvalds committed
368
		 */
369
		if (!(cpufreq_driver->flags & CPUFREQ_CONST_LOOPS)) {
370 371
			if ((policy) && (policy->cpu == freqs->cpu) &&
			    (policy->cur) && (policy->cur != freqs->old)) {
372 373
				pr_debug("Warning: CPU frequency is %u, cpufreq assumed %u kHz\n",
					 freqs->old, policy->cur);
374
				freqs->old = policy->cur;
Linus Torvalds's avatar
Linus Torvalds committed
375 376
			}
		}
377
		srcu_notifier_call_chain(&cpufreq_transition_notifier_list,
378
				CPUFREQ_PRECHANGE, freqs);
Linus Torvalds's avatar
Linus Torvalds committed
379 380
		adjust_jiffies(CPUFREQ_PRECHANGE, freqs);
		break;
381

Linus Torvalds's avatar
Linus Torvalds committed
382 383
	case CPUFREQ_POSTCHANGE:
		adjust_jiffies(CPUFREQ_POSTCHANGE, freqs);
384 385
		pr_debug("FREQ: %lu - CPU: %lu\n",
			 (unsigned long)freqs->new, (unsigned long)freqs->cpu);
386
		trace_cpu_frequency(freqs->new, freqs->cpu);
387
		srcu_notifier_call_chain(&cpufreq_transition_notifier_list,
388
				CPUFREQ_POSTCHANGE, freqs);
389 390
		if (likely(policy) && likely(policy->cpu == freqs->cpu))
			policy->cur = freqs->new;
Linus Torvalds's avatar
Linus Torvalds committed
391 392 393
		break;
	}
}
394

395 396 397 398 399 400 401 402
/**
 * cpufreq_notify_transition - call notifier chain and adjust_jiffies
 * on frequency transition.
 *
 * This function calls the transition notifiers and the "adjust_jiffies"
 * function. It is called twice on all CPU frequency changes that have
 * external effects.
 */
403
static void cpufreq_notify_transition(struct cpufreq_policy *policy,
404 405 406 407 408
		struct cpufreq_freqs *freqs, unsigned int state)
{
	for_each_cpu(freqs->cpu, policy->cpus)
		__cpufreq_notify_transition(policy, freqs, state);
}
Linus Torvalds's avatar
Linus Torvalds committed
409

410
/* Do post notifications when there are chances that transition has failed */
411
static void cpufreq_notify_post_transition(struct cpufreq_policy *policy,
412 413 414 415 416 417 418 419 420 421 422
		struct cpufreq_freqs *freqs, int transition_failed)
{
	cpufreq_notify_transition(policy, freqs, CPUFREQ_POSTCHANGE);
	if (!transition_failed)
		return;

	swap(freqs->old, freqs->new);
	cpufreq_notify_transition(policy, freqs, CPUFREQ_PRECHANGE);
	cpufreq_notify_transition(policy, freqs, CPUFREQ_POSTCHANGE);
}

423 424 425
void cpufreq_freq_transition_begin(struct cpufreq_policy *policy,
		struct cpufreq_freqs *freqs)
{
426 427 428 429 430 431 432 433 434 435 436 437

	/*
	 * Catch double invocations of _begin() which lead to self-deadlock.
	 * ASYNC_NOTIFICATION drivers are left out because the cpufreq core
	 * doesn't invoke _begin() on their behalf, and hence the chances of
	 * double invocations are very low. Moreover, there are scenarios
	 * where these checks can emit false-positive warnings in these
	 * drivers; so we avoid that by skipping them altogether.
	 */
	WARN_ON(!(cpufreq_driver->flags & CPUFREQ_ASYNC_NOTIFICATION)
				&& current == policy->transition_task);

438 439 440 441 442 443 444 445 446 447 448
wait:
	wait_event(policy->transition_wait, !policy->transition_ongoing);

	spin_lock(&policy->transition_lock);

	if (unlikely(policy->transition_ongoing)) {
		spin_unlock(&policy->transition_lock);
		goto wait;
	}

	policy->transition_ongoing = true;
449
	policy->transition_task = current;
450 451 452 453 454 455 456 457 458 459 460 461 462 463 464 465

	spin_unlock(&policy->transition_lock);

	cpufreq_notify_transition(policy, freqs, CPUFREQ_PRECHANGE);
}
EXPORT_SYMBOL_GPL(cpufreq_freq_transition_begin);

void cpufreq_freq_transition_end(struct cpufreq_policy *policy,
		struct cpufreq_freqs *freqs, int transition_failed)
{
	if (unlikely(WARN_ON(!policy->transition_ongoing)))
		return;

	cpufreq_notify_post_transition(policy, freqs, transition_failed);

	policy->transition_ongoing = false;
466
	policy->transition_task = NULL;
467 468 469 470 471

	wake_up(&policy->transition_wait);
}
EXPORT_SYMBOL_GPL(cpufreq_freq_transition_end);

Linus Torvalds's avatar
Linus Torvalds committed
472 473 474 475

/*********************************************************************
 *                          SYSFS INTERFACE                          *
 *********************************************************************/
476
static ssize_t show_boost(struct kobject *kobj,
477 478 479 480 481 482 483 484 485 486 487 488 489 490 491
				 struct attribute *attr, char *buf)
{
	return sprintf(buf, "%d\n", cpufreq_driver->boost_enabled);
}

static ssize_t store_boost(struct kobject *kobj, struct attribute *attr,
				  const char *buf, size_t count)
{
	int ret, enable;

	ret = sscanf(buf, "%d", &enable);
	if (ret != 1 || enable < 0 || enable > 1)
		return -EINVAL;

	if (cpufreq_boost_trigger_state(enable)) {
492 493
		pr_err("%s: Cannot %s BOOST!\n",
		       __func__, enable ? "enable" : "disable");
494 495 496
		return -EINVAL;
	}

497 498
	pr_debug("%s: cpufreq BOOST %s\n",
		 __func__, enable ? "enabled" : "disabled");
499 500 501 502

	return count;
}
define_one_global_rw(boost);
Linus Torvalds's avatar
Linus Torvalds committed
503

504
static struct cpufreq_governor *find_governor(const char *str_governor)
505 506 507
{
	struct cpufreq_governor *t;

508
	for_each_governor(t)
509
		if (!strncasecmp(str_governor, t->name, CPUFREQ_NAME_LEN))
510 511 512 513 514
			return t;

	return NULL;
}

Linus Torvalds's avatar
Linus Torvalds committed
515 516 517
/**
 * cpufreq_parse_governor - parse a governor string
 */
Dave Jones's avatar
Dave Jones committed
518
static int cpufreq_parse_governor(char *str_governor, unsigned int *policy,
Linus Torvalds's avatar
Linus Torvalds committed
519 520
				struct cpufreq_governor **governor)
{
521
	int err = -EINVAL;
522 523

	if (cpufreq_driver->setpolicy) {
524
		if (!strncasecmp(str_governor, "performance", CPUFREQ_NAME_LEN)) {
Linus Torvalds's avatar
Linus Torvalds committed
525
			*policy = CPUFREQ_POLICY_PERFORMANCE;
526
			err = 0;
527
		} else if (!strncasecmp(str_governor, "powersave",
528
						CPUFREQ_NAME_LEN)) {
Linus Torvalds's avatar
Linus Torvalds committed
529
			*policy = CPUFREQ_POLICY_POWERSAVE;
530
			err = 0;
Linus Torvalds's avatar
Linus Torvalds committed
531
		}
532
	} else {
Linus Torvalds's avatar
Linus Torvalds committed
533
		struct cpufreq_governor *t;
534

535
		mutex_lock(&cpufreq_governor_mutex);
536

537
		t = find_governor(str_governor);
538

539
		if (t == NULL) {
540
			int ret;
541

542 543 544
			mutex_unlock(&cpufreq_governor_mutex);
			ret = request_module("cpufreq_%s", str_governor);
			mutex_lock(&cpufreq_governor_mutex);
545

546
			if (ret == 0)
547
				t = find_governor(str_governor);
548 549
		}

550 551 552
		if (t != NULL) {
			*governor = t;
			err = 0;
Linus Torvalds's avatar
Linus Torvalds committed
553
		}
554

555
		mutex_unlock(&cpufreq_governor_mutex);
Linus Torvalds's avatar
Linus Torvalds committed
556
	}
557
	return err;
Linus Torvalds's avatar
Linus Torvalds committed
558 559 560
}

/**
561 562
 * cpufreq_per_cpu_attr_read() / show_##file_name() -
 * print out cpufreq information
Linus Torvalds's avatar
Linus Torvalds committed
563 564 565 566 567
 *
 * Write out information from cpufreq_driver->policy[cpu]; object must be
 * "unsigned int".
 */

568 569
#define show_one(file_name, object)			\
static ssize_t show_##file_name				\
Dave Jones's avatar
Dave Jones committed
570
(struct cpufreq_policy *policy, char *buf)		\
571
{							\
572
	return sprintf(buf, "%u\n", policy->object);	\
Linus Torvalds's avatar
Linus Torvalds committed
573 574 575 576
}

show_one(cpuinfo_min_freq, cpuinfo.min_freq);
show_one(cpuinfo_max_freq, cpuinfo.max_freq);
577
show_one(cpuinfo_transition_latency, cpuinfo.transition_latency);
Linus Torvalds's avatar
Linus Torvalds committed
578 579
show_one(scaling_min_freq, min);
show_one(scaling_max_freq, max);
580

581
static ssize_t show_scaling_cur_freq(struct cpufreq_policy *policy, char *buf)
582 583 584 585 586 587 588 589 590
{
	ssize_t ret;

	if (cpufreq_driver && cpufreq_driver->setpolicy && cpufreq_driver->get)
		ret = sprintf(buf, "%u\n", cpufreq_driver->get(policy->cpu));
	else
		ret = sprintf(buf, "%u\n", policy->cur);
	return ret;
}
Linus Torvalds's avatar
Linus Torvalds committed
591

592
static int cpufreq_set_policy(struct cpufreq_policy *policy,
593
				struct cpufreq_policy *new_policy);
594

Linus Torvalds's avatar
Linus Torvalds committed
595 596 597 598 599
/**
 * cpufreq_per_cpu_attr_write() / store_##file_name() - sysfs write access
 */
#define store_one(file_name, object)			\
static ssize_t store_##file_name					\
Dave Jones's avatar
Dave Jones committed
600
(struct cpufreq_policy *policy, const char *buf, size_t count)		\
Linus Torvalds's avatar
Linus Torvalds committed
601
{									\
602
	int ret, temp;							\
Linus Torvalds's avatar
Linus Torvalds committed
603 604
	struct cpufreq_policy new_policy;				\
									\
605
	memcpy(&new_policy, policy, sizeof(*policy));			\
Linus Torvalds's avatar
Linus Torvalds committed
606
									\
607
	ret = sscanf(buf, "%u", &new_policy.object);			\
Linus Torvalds's avatar
Linus Torvalds committed
608 609 610
	if (ret != 1)							\
		return -EINVAL;						\
									\
611
	temp = new_policy.object;					\
612
	ret = cpufreq_set_policy(policy, &new_policy);		\
613 614
	if (!ret)							\
		policy->user_policy.object = temp;			\
Linus Torvalds's avatar
Linus Torvalds committed
615 616 617 618
									\
	return ret ? ret : count;					\
}

619 620
store_one(scaling_min_freq, min);
store_one(scaling_max_freq, max);
Linus Torvalds's avatar
Linus Torvalds committed
621 622 623 624

/**
 * show_cpuinfo_cur_freq - current CPU frequency as detected by hardware
 */
Dave Jones's avatar
Dave Jones committed
625 626
static ssize_t show_cpuinfo_cur_freq(struct cpufreq_policy *policy,
					char *buf)
Linus Torvalds's avatar
Linus Torvalds committed
627
{
628
	unsigned int cur_freq = __cpufreq_get(policy);
Linus Torvalds's avatar
Linus Torvalds committed
629 630 631 632 633 634 635 636
	if (!cur_freq)
		return sprintf(buf, "<unknown>");
	return sprintf(buf, "%u\n", cur_freq);
}

/**
 * show_scaling_governor - show the current policy for the specified CPU
 */
Dave Jones's avatar
Dave Jones committed
637
static ssize_t show_scaling_governor(struct cpufreq_policy *policy, char *buf)
Linus Torvalds's avatar
Linus Torvalds committed
638
{
639
	if (policy->policy == CPUFREQ_POLICY_POWERSAVE)
Linus Torvalds's avatar
Linus Torvalds committed
640 641 642 643
		return sprintf(buf, "powersave\n");
	else if (policy->policy == CPUFREQ_POLICY_PERFORMANCE)
		return sprintf(buf, "performance\n");
	else if (policy->governor)
644
		return scnprintf(buf, CPUFREQ_NAME_PLEN, "%s\n",
645
				policy->governor->name);
Linus Torvalds's avatar
Linus Torvalds committed
646 647 648 649 650 651
	return -EINVAL;
}

/**
 * store_scaling_governor - store policy for the specified CPU
 */
Dave Jones's avatar
Dave Jones committed
652 653
static ssize_t store_scaling_governor(struct cpufreq_policy *policy,
					const char *buf, size_t count)
Linus Torvalds's avatar
Linus Torvalds committed
654
{
655
	int ret;
Linus Torvalds's avatar
Linus Torvalds committed
656 657 658
	char	str_governor[16];
	struct cpufreq_policy new_policy;

659
	memcpy(&new_policy, policy, sizeof(*policy));
Linus Torvalds's avatar
Linus Torvalds committed
660

661
	ret = sscanf(buf, "%15s", str_governor);
Linus Torvalds's avatar
Linus Torvalds committed
662 663 664
	if (ret != 1)
		return -EINVAL;

665 666
	if (cpufreq_parse_governor(str_governor, &new_policy.policy,
						&new_policy.governor))
Linus Torvalds's avatar
Linus Torvalds committed
667 668
		return -EINVAL;

669
	ret = cpufreq_set_policy(policy, &new_policy);
670
	return ret ? ret : count;
Linus Torvalds's avatar
Linus Torvalds committed
671 672 673 674 675
}

/**
 * show_scaling_driver - show the cpufreq driver currently loaded
 */
Dave Jones's avatar
Dave Jones committed
676
static ssize_t show_scaling_driver(struct cpufreq_policy *policy, char *buf)
Linus Torvalds's avatar
Linus Torvalds committed
677
{
678
	return scnprintf(buf, CPUFREQ_NAME_PLEN, "%s\n", cpufreq_driver->name);
Linus Torvalds's avatar
Linus Torvalds committed
679 680 681 682 683
}

/**
 * show_scaling_available_governors - show the available CPUfreq governors
 */
Dave Jones's avatar
Dave Jones committed
684 685
static ssize_t show_scaling_available_governors(struct cpufreq_policy *policy,
						char *buf)
Linus Torvalds's avatar
Linus Torvalds committed
686 687 688 689
{
	ssize_t i = 0;
	struct cpufreq_governor *t;

690
	if (!has_target()) {
Linus Torvalds's avatar
Linus Torvalds committed
691 692 693 694
		i += sprintf(buf, "performance powersave");
		goto out;
	}

695
	for_each_governor(t) {
696 697
		if (i >= (ssize_t) ((PAGE_SIZE / sizeof(char))
		    - (CPUFREQ_NAME_LEN + 2)))
Linus Torvalds's avatar
Linus Torvalds committed
698
			goto out;
699
		i += scnprintf(&buf[i], CPUFREQ_NAME_PLEN, "%s ", t->name);
Linus Torvalds's avatar
Linus Torvalds committed
700
	}
701
out:
Linus Torvalds's avatar
Linus Torvalds committed
702 703 704
	i += sprintf(&buf[i], "\n");
	return i;
}
705

706
ssize_t cpufreq_show_cpus(const struct cpumask *mask, char *buf)
Linus Torvalds's avatar
Linus Torvalds committed
707 708 709 710
{
	ssize_t i = 0;
	unsigned int cpu;

711
	for_each_cpu(cpu, mask) {
Linus Torvalds's avatar
Linus Torvalds committed
712 713 714 715
		if (i)
			i += scnprintf(&buf[i], (PAGE_SIZE - i - 2), " ");
		i += scnprintf(&buf[i], (PAGE_SIZE - i - 2), "%u", cpu);
		if (i >= (PAGE_SIZE - 5))
716
			break;
Linus Torvalds's avatar
Linus Torvalds committed
717 718 719 720
	}
	i += sprintf(&buf[i], "\n");
	return i;
}
721
EXPORT_SYMBOL_GPL(cpufreq_show_cpus);
Linus Torvalds's avatar
Linus Torvalds committed
722

723 724 725 726 727 728
/**
 * show_related_cpus - show the CPUs affected by each transition even if
 * hw coordination is in use
 */
static ssize_t show_related_cpus(struct cpufreq_policy *policy, char *buf)
{
729
	return cpufreq_show_cpus(policy->related_cpus, buf);
730 731 732 733 734 735 736
}

/**
 * show_affected_cpus - show the CPUs affected by each transition
 */
static ssize_t show_affected_cpus(struct cpufreq_policy *policy, char *buf)
{
737
	return cpufreq_show_cpus(policy->cpus, buf);
738 739
}

740
static ssize_t store_scaling_setspeed(struct cpufreq_policy *policy,
Dave Jones's avatar
Dave Jones committed
741
					const char *buf, size_t count)
742 743 744 745
{
	unsigned int freq = 0;
	unsigned int ret;

746
	if (!policy->governor || !policy->governor->store_setspeed)
747 748 749 750 751 752 753 754 755 756 757 758 759
		return -EINVAL;

	ret = sscanf(buf, "%u", &freq);
	if (ret != 1)
		return -EINVAL;

	policy->governor->store_setspeed(policy, freq);

	return count;
}

static ssize_t show_scaling_setspeed(struct cpufreq_policy *policy, char *buf)
{
760
	if (!policy->governor || !policy->governor->show_setspeed)
761 762 763 764
		return sprintf(buf, "<unsupported>\n");

	return policy->governor->show_setspeed(policy, buf);
}
Linus Torvalds's avatar
Linus Torvalds committed
765

766
/**
767
 * show_bios_limit - show the current cpufreq HW/BIOS limitation
768 769 770 771 772
 */
static ssize_t show_bios_limit(struct cpufreq_policy *policy, char *buf)
{
	unsigned int limit;
	int ret;
773 774
	if (cpufreq_driver->bios_limit) {
		ret = cpufreq_driver->bios_limit(policy->cpu, &limit);
775 776 777 778 779 780
		if (!ret)
			return sprintf(buf, "%u\n", limit);
	}
	return sprintf(buf, "%u\n", policy->cpuinfo.max_freq);
}

781 782 783 784 785 786 787 788 789 790 791 792 793 794
cpufreq_freq_attr_ro_perm(cpuinfo_cur_freq, 0400);
cpufreq_freq_attr_ro(cpuinfo_min_freq);
cpufreq_freq_attr_ro(cpuinfo_max_freq);
cpufreq_freq_attr_ro(cpuinfo_transition_latency);
cpufreq_freq_attr_ro(scaling_available_governors);
cpufreq_freq_attr_ro(scaling_driver);
cpufreq_freq_attr_ro(scaling_cur_freq);
cpufreq_freq_attr_ro(bios_limit);
cpufreq_freq_attr_ro(related_cpus);
cpufreq_freq_attr_ro(affected_cpus);
cpufreq_freq_attr_rw(scaling_min_freq);
cpufreq_freq_attr_rw(scaling_max_freq);
cpufreq_freq_attr_rw(scaling_governor);
cpufreq_freq_attr_rw(scaling_setspeed);
Linus Torvalds's avatar
Linus Torvalds committed
795

Dave Jones's avatar
Dave Jones committed
796
static struct attribute *default_attrs[] = {
Linus Torvalds's avatar
Linus Torvalds committed
797 798
	&cpuinfo_min_freq.attr,
	&cpuinfo_max_freq.attr,
799
	&cpuinfo_transition_latency.attr,
Linus Torvalds's avatar
Linus Torvalds committed
800 801 802
	&scaling_min_freq.attr,
	&scaling_max_freq.attr,
	&affected_cpus.attr,
803
	&related_cpus.attr,
Linus Torvalds's avatar
Linus Torvalds committed
804 805 806
	&scaling_governor.attr,
	&scaling_driver.attr,
	&scaling_available_governors.attr,
807
	&scaling_setspeed.attr,
Linus Torvalds's avatar
Linus Torvalds committed
808 809 810
	NULL
};

811 812
#define to_policy(k) container_of(k, struct cpufreq_policy, kobj)
#define to_attr(a) container_of(a, struct freq_attr, attr)
Linus Torvalds's avatar
Linus Torvalds committed
813

814
static ssize_t show(struct kobject *kobj, struct attribute *attr, char *buf)
Linus Torvalds's avatar
Linus Torvalds committed
815
{
Dave Jones's avatar
Dave Jones committed
816 817
	struct cpufreq_policy *policy = to_policy(kobj);
	struct freq_attr *fattr = to_attr(attr);
818
	ssize_t ret;
819

820
	down_read(&policy->rwsem);
821

822 823 824 825 826
	if (fattr->show)
		ret = fattr->show(policy, buf);
	else
		ret = -EIO;

827
	up_read(&policy->rwsem);
828

Linus Torvalds's avatar
Linus Torvalds committed
829 830 831
	return ret;
}

Dave Jones's avatar
Dave Jones committed
832 833
static ssize_t store(struct kobject *kobj, struct attribute *attr,
		     const char *buf, size_t count)
Linus Torvalds's avatar
Linus Torvalds committed
834
{
Dave Jones's avatar
Dave Jones committed
835 836
	struct cpufreq_policy *policy = to_policy(kobj);
	struct freq_attr *fattr = to_attr(attr);
837
	ssize_t ret = -EINVAL;
838

839 840 841 842 843
	get_online_cpus();

	if (!cpu_online(policy->cpu))
		goto unlock;

844
	down_write(&policy->rwsem);
845

846 847 848 849 850
	if (fattr->store)
		ret = fattr->store(policy, buf, count);
	else
		ret = -EIO;

851
	up_write(&policy->rwsem);
852 853 854
unlock:
	put_online_cpus();

Linus Torvalds's avatar
Linus Torvalds committed
855 856 857
	return ret;
}

Dave Jones's avatar
Dave Jones committed
858
static void cpufreq_sysfs_release(struct kobject *kobj)
Linus Torvalds's avatar
Linus Torvalds committed
859
{
Dave Jones's avatar
Dave Jones committed
860
	struct cpufreq_policy *policy = to_policy(kobj);
861
	pr_debug("last reference is dropped\n");
Linus Torvalds's avatar
Linus Torvalds committed
862 863 864
	complete(&policy->kobj_unregister);
}

865
static const struct sysfs_ops sysfs_ops = {
Linus Torvalds's avatar
Linus Torvalds committed
866 867 868 869 870 871 872 873 874 875
	.show	= show,
	.store	= store,
};

static struct kobj_type ktype_cpufreq = {
	.sysfs_ops	= &sysfs_ops,
	.default_attrs	= default_attrs,
	.release	= cpufreq_sysfs_release,
};

876 877 878 879 880 881 882 883 884 885 886 887 888 889 890 891 892 893 894 895 896 897 898 899 900 901 902 903 904 905
static int add_cpu_dev_symlink(struct cpufreq_policy *policy, int cpu)
{
	struct device *cpu_dev;

	pr_debug("%s: Adding symlink for CPU: %u\n", __func__, cpu);

	if (!policy)
		return 0;

	cpu_dev = get_cpu_device(cpu);
	if (WARN_ON(!cpu_dev))
		return 0;

	return sysfs_create_link(&cpu_dev->kobj, &policy->kobj, "cpufreq");
}

static void remove_cpu_dev_symlink(struct cpufreq_policy *policy, int cpu)
{
	struct device *cpu_dev;

	pr_debug("%s: Removing symlink for CPU: %u\n", __func__, cpu);

	cpu_dev = get_cpu_device(cpu);
	if (WARN_ON(!cpu_dev))
		return;

	sysfs_remove_link(&cpu_dev->kobj, "cpufreq");
}

/* Add/remove symlinks for all related CPUs */
906
static int cpufreq_add_dev_symlink(struct cpufreq_policy *policy)
907 908 909 910
{
	unsigned int j;
	int ret = 0;

911
	/* Some related CPUs might not be present (physically hotplugged) */
912
	for_each_cpu(j, policy->real_cpus) {
913
		ret = add_cpu_dev_symlink(policy, j);
914 915
		if (ret)
			break;
916
	}
917

918 919 920
	return ret;
}

921 922 923 924 925
static void cpufreq_remove_dev_symlink(struct cpufreq_policy *policy)
{
	unsigned int j;

	/* Some related CPUs might not be present (physically hotplugged) */
926
	for_each_cpu(j, policy->real_cpus)
927 928 929
		remove_cpu_dev_symlink(policy, j);
}

930
static int cpufreq_add_dev_interface(struct cpufreq_policy *policy)
931 932 933 934 935
{
	struct freq_attr **drv_attr;
	int ret = 0;

	/* set up files for this cpu device */
936
	drv_attr = cpufreq_driver->attr;
937
	while (drv_attr && *drv_attr) {
938 939
		ret = sysfs_create_file(&policy->kobj, &((*drv_attr)->attr));
		if (ret)
940
			return ret;
941 942
		drv_attr++;
	}
943
	if (cpufreq_driver->get) {
944 945
		ret = sysfs_create_file(&policy->kobj, &cpuinfo_cur_freq.attr);
		if (ret)
946
			return ret;
947
	}
948 949 950

	ret = sysfs_create_file(&policy->kobj, &scaling_cur_freq.attr);
	if (ret)
951
		return ret;
952

953
	if (cpufreq_driver->bios_limit) {
954 955
		ret = sysfs_create_file(&policy->kobj, &bios_limit.attr);
		if (ret)
956
			return ret;
957
	}
958

959
	return cpufreq_add_dev_symlink(policy);
960 961
}

962
static int cpufreq_init_policy(struct cpufreq_policy *policy)
963
{
964
	struct cpufreq_governor *gov = NULL;
965 966
	struct cpufreq_policy new_policy;

967
	memcpy(&new_policy, policy, sizeof(*policy));
968

969
	/* Update governor of new_policy to the governor used before hotplug */
970
	gov = find_governor(policy->last_governor);
971 972 973 974 975 976 977 978
	if (gov)
		pr_debug("Restoring governor %s for cpu %d\n",
				policy->governor->name, policy->cpu);
	else
		gov = CPUFREQ_DEFAULT_GOVERNOR;

	new_policy.governor = gov;

979 980
	/* Use the default policy if its valid. */
	if (cpufreq_driver->setpolicy)
981
		cpufreq_parse_governor(gov->name, &new_policy.policy, NULL);
982 983

	/* set default policy */
984
	return cpufreq_set_policy(policy, &new_policy);
985 986
}

987
static int cpufreq_add_policy_cpu(struct cpufreq_policy *policy, unsigned int cpu)
988
{
989
	int ret = 0;
990

991 992 993 994
	/* Has this CPU been taken care of already? */
	if (cpumask_test_cpu(cpu, policy->cpus))
		return 0;

995
	if (has_target()) {
996 997 998 999 1000 1001
		ret = __cpufreq_governor(policy, CPUFREQ_GOV_STOP);
		if (ret) {
			pr_err("%s: Failed to stop governor\n", __func__);
			return ret;
		}
	}
1002

1003
	down_write(&policy->rwsem);
1004
	cpumask_set_cpu(cpu, policy->cpus);
1005
	up_write(&policy->rwsem);
Viresh Kumar's avatar
Viresh Kumar committed
1006

1007
	if (has_target()) {
1008 1009 1010 1011 1012
		ret = __cpufreq_governor(policy, CPUFREQ_GOV_START);
		if (!ret)
			ret = __cpufreq_governor(policy, CPUFREQ_GOV_LIMITS);

		if (ret) {
1013 1014 1015
			pr_err("%s: Failed to start governor\n", __func__);
			return ret;
		}
1016
	}
1017

1018
	return 0;
1019
}
Linus Torvalds's avatar
Linus Torvalds committed
1020

1021
static struct cpufreq_policy *cpufreq_policy_alloc(unsigned int cpu)
1022
{
1023
	struct device *dev = get_cpu_device(cpu);
1024 1025
	struct cpufreq_policy *policy;

1026 1027 1028
	if (WARN_ON(!dev))
		return NULL;

1029 1030 1031 1032 1033 1034 1035 1036 1037 1038
	policy = kzalloc(sizeof(*policy), GFP_KERNEL);
	if (!policy)
		return NULL;

	if (!alloc_cpumask_var(&policy->cpus, GFP_KERNEL))
		goto err_free_policy;

	if (!zalloc_cpumask_var(&policy->related_cpus, GFP_KERNEL))
		goto err_free_cpumask;

1039 1040 1041
	if (!zalloc_cpumask_var(&policy->real_cpus, GFP_KERNEL))
		goto err_free_rcpumask;

1042
	kobject_init(&policy->kobj, &ktype_cpufreq);
1043
	INIT_LIST_HEAD(&policy->policy_list);
1044
	init_rwsem(&policy->rwsem);
1045 1046
	spin_lock_init(&policy->transition_lock);
	init_waitqueue_head(&policy->transition_wait);
1047 1048
	init_completion(&policy->kobj_unregister);
	INIT_WORK(&policy->update, handle_update);
1049

1050
	policy->cpu = cpu;
1051 1052
	return policy;

1053 1054
err_free_rcpumask:
	free_cpumask_var(policy->related_cpus);
1055 1056 1057 1058 1059 1060 1061 1062
err_free_cpumask:
	free_cpumask_var(policy->cpus);
err_free_policy:
	kfree(policy);

	return NULL;
}

1063
static void cpufreq_policy_put_kobj(struct cpufreq_policy *policy, bool notify)
1064 1065 1066 1067
{
	struct kobject *kobj;
	struct completion *cmp;

1068 1069 1070
	if (notify)
		blocking_notifier_call_chain(&cpufreq_policy_notifier_list,
					     CPUFREQ_REMOVE_POLICY, policy);
1071

1072 1073
	down_write(&policy->rwsem);
	cpufreq_remove_dev_symlink(policy);
1074 1075
	kobj = &policy->kobj;
	cmp = &policy->kobj_unregister;
1076
	up_write(&policy->rwsem);
1077 1078 1079 1080 1081 1082 1083 1084 1085 1086 1087 1088
	kobject_put(kobj);

	/*
	 * We need to make sure that the underlying kobj is
	 * actually not referenced anymore by anybody before we
	 * proceed with unloading.
	 */
	pr_debug("waiting for dropping of refcount\n");
	wait_for_completion(cmp);
	pr_debug("wait complete\n");
}

1089
static void cpufreq_policy_free(struct cpufreq_policy *policy, bool notify)
1090
{
1091 1092 1093 1094 1095 1096 1097 1098 1099 1100 1101
	unsigned long flags;
	int cpu;

	/* Remove policy from list */
	write_lock_irqsave(&cpufreq_driver_lock, flags);
	list_del(&policy->policy_list);

	for_each_cpu(cpu, policy->related_cpus)
		per_cpu(cpufreq_cpu_data, cpu) = NULL;
	write_unlock_irqrestore(&cpufreq_driver_lock, flags);

1102
	cpufreq_policy_put_kobj(policy, notify);
1103
	free_cpumask_var(policy->real_cpus);
1104 1105 1106 1107 1108
	free_cpumask_var(policy->related_cpus);
	free_cpumask_var(policy->cpus);
	kfree(policy);
}

1109
static int cpufreq_online(unsigned int cpu)
Linus Torvalds's avatar
Linus Torvalds committed
1110
{
1111
	struct cpufreq_policy *policy;
1112
	bool new_policy;
Linus Torvalds's avatar
Linus Torvalds committed
1113
	unsigned long flags;
1114 1115
	unsigned int j;
	int ret;
1116

1117
	pr_debug("%s: bringing CPU%u online\n", __func__, cpu);
1118

1119
	/* Check if this CPU already has a policy to manage it */
1120
	policy = per_cpu(cpufreq_cpu_data, cpu);
1121
	if (policy) {
1122
		WARN_ON(!cpumask_test_cpu(cpu, policy->related_cpus));
1123
		if (!policy_is_inactive(policy))
1124
			return cpufreq_add_policy_cpu(policy, cpu);
Linus Torvalds's avatar
Linus Torvalds committed
1125

1126
		/* This is the only online CPU for the policy.  Start over. */
1127
		new_policy = false;
1128 1129 1130 1131 1132
		down_write(&policy->rwsem);
		policy->cpu = cpu;
		policy->governor = NULL;
		up_write(&policy->rwsem);
	} else {
1133
		new_policy = true;
1134
		policy = cpufreq_policy_alloc(cpu);
1135
		if (!policy)
1136
			return -ENOMEM;
1137
	}
1138

1139
	cpumask_copy(policy->cpus, cpumask_of(cpu));
Linus Torvalds's avatar
Linus Torvalds committed
1140 1141 1142 1143

	/* call driver. From then on the cpufreq must be able
	 * to accept all calls to ->verify and ->setpolicy for this CPU
	 */
1144
	ret = cpufreq_driver->init(policy);
Linus Torvalds's avatar
Linus Torvalds committed
1145
	if (ret) {
1146
		pr_debug("initialization failed\n");
1147
		goto out_free_policy;
Linus Torvalds's avatar
Linus Torvalds committed
1148
	}
1149

1150 1151
	down_write(&policy->rwsem);

1152
	if (new_policy) {
1153
		/* related_cpus should at least include policy->cpus. */
1154
		cpumask_copy(policy->related_cpus, policy->cpus);
1155
		/* Remember CPUs present at the policy creation time. */
1156
		cpumask_and(policy->real_cpus, policy->cpus, cpu_present_mask);
1157 1158 1159 1160 1161 1162 1163 1164 1165 1166

		/* Name and add the kobject */
		ret = kobject_add(&policy->kobj, cpufreq_global_kobject,
				  "policy%u",
				  cpumask_first(policy->related_cpus));
		if (ret) {
			pr_err("%s: failed to add policy->kobj: %d\n", __func__,
			       ret);
			goto out_exit_policy;
		}
1167
	}
1168

1169 1170 1171 1172 1173 1174
	/*
	 * affected cpus must always be the one, which are online. We aren't
	 * managing offline cpus here.
	 */
	cpumask_and(policy->cpus, policy->cpus, cpu_online_mask);

1175
	if (new_policy) {
1176 1177
		policy->user_policy.min = policy->min;
		policy->user_policy.max = policy->max;
1178

1179 1180 1181 1182 1183
		write_lock_irqsave(&cpufreq_driver_lock, flags);
		for_each_cpu(j, policy->related_cpus)
			per_cpu(cpufreq_cpu_data, j) = policy;
		write_unlock_irqrestore(&cpufreq_driver_lock, flags);
	}
1184

1185
	if (cpufreq_driver->get && !cpufreq_driver->setpolicy) {
1186 1187 1188
		policy->cur = cpufreq_driver->get(policy->cpu);
		if (!policy->cur) {
			pr_err("%s: ->get() failed\n", __func__);
1189
			goto out_exit_policy;
1190 1191 1192
		}
	}

1193 1194 1195 1196 1197 1198 1199 1200 1201 1202 1203 1204 1205 1206 1207 1208 1209 1210 1211 1212 1213 1214 1215 1216 1217 1218 1219 1220 1221 1222 1223 1224 1225 1226 1227 1228 1229 1230 1231 1232
	/*
	 * Sometimes boot loaders set CPU frequency to a value outside of
	 * frequency table present with cpufreq core. In such cases CPU might be
	 * unstable if it has to run on that frequency for long duration of time
	 * and so its better to set it to a frequency which is specified in
	 * freq-table. This also makes cpufreq stats inconsistent as
	 * cpufreq-stats would fail to register because current frequency of CPU
	 * isn't found in freq-table.
	 *
	 * Because we don't want this change to effect boot process badly, we go
	 * for the next freq which is >= policy->cur ('cur' must be set by now,
	 * otherwise we will end up setting freq to lowest of the table as 'cur'
	 * is initialized to zero).
	 *
	 * We are passing target-freq as "policy->cur - 1" otherwise
	 * __cpufreq_driver_target() would simply fail, as policy->cur will be
	 * equal to target-freq.
	 */
	if ((cpufreq_driver->flags & CPUFREQ_NEED_INITIAL_FREQ_CHECK)
	    && has_target()) {
		/* Are we running at unknown frequency ? */
		ret = cpufreq_frequency_table_get_index(policy, policy->cur);
		if (ret == -EINVAL) {
			/* Warn user and fix it */
			pr_warn("%s: CPU%d: Running at unlisted freq: %u KHz\n",
				__func__, policy->cpu, policy->cur);
			ret = __cpufreq_driver_target(policy, policy->cur - 1,
				CPUFREQ_RELATION_L);

			/*
			 * Reaching here after boot in a few seconds may not
			 * mean that system will remain stable at "unknown"
			 * frequency for longer duration. Hence, a BUG_ON().
			 */
			BUG_ON(ret);
			pr_warn("%s: CPU%d: Unlisted initial frequency changed to: %u KHz\n",
				__func__, policy->cpu, policy->cur);
		}
	}

1233 1234 1235
	blocking_notifier_call_chain(&cpufreq_policy_notifier_list,
				     CPUFREQ_START, policy);

1236
	if (new_policy) {
1237
		ret = cpufreq_add_dev_interface(policy);
1238
		if (ret)
1239
			goto out_exit_policy;
1240 1241
		blocking_notifier_call_chain(&cpufreq_policy_notifier_list,
				CPUFREQ_CREATE_POLICY, policy);
1242

1243 1244 1245 1246
		write_lock_irqsave(&cpufreq_driver_lock, flags);
		list_add(&policy->policy_list, &cpufreq_policy_list);
		write_unlock_irqrestore(&cpufreq_driver_lock, flags);
	}
1247

1248 1249 1250 1251
	ret = cpufreq_init_policy(policy);
	if (ret) {
		pr_err("%s: Failed to initialize policy for cpu: %d (%d)\n",
		       __func__, cpu, ret);
1252 1253 1254
		/* cpufreq_policy_free() will notify based on this */
		new_policy = false;
		goto out_exit_policy;
1255
	}
1256

1257
	up_write(&policy->rwsem);
1258

1259
	kobject_uevent(&policy->kobj, KOBJ_ADD);
1260 1261 1262 1263 1264

	/* Callback for handling stuff after policy is ready */
	if (cpufreq_driver->ready)
		cpufreq_driver->ready(policy);

1265
	pr_debug("initialization complete\n");
1266

Linus Torvalds's avatar
Linus Torvalds committed
1267 1268
	return 0;

1269
out_exit_policy:
1270 1271
	up_write(&policy->rwsem);

1272 1273
	if (cpufreq_driver->exit)
		cpufreq_driver->exit(policy);
1274
out_free_policy:
1275
	cpufreq_policy_free(policy, !new_policy);
Linus Torvalds's avatar
Linus Torvalds committed
1276 1277 1278
	return ret;
}

1279 1280 1281 1282 1283 1284 1285 1286 1287 1288 1289 1290 1291 1292 1293 1294 1295 1296 1297 1298 1299 1300 1301 1302 1303
/**
 * cpufreq_add_dev - the cpufreq interface for a CPU device.
 * @dev: CPU device.
 * @sif: Subsystem interface structure pointer (not used)
 */
static int cpufreq_add_dev(struct device *dev, struct subsys_interface *sif)
{
	unsigned cpu = dev->id;
	int ret;

	dev_dbg(dev, "%s: adding CPU%u\n", __func__, cpu);

	if (cpu_online(cpu)) {
		ret = cpufreq_online(cpu);
	} else {
		/*
		 * A hotplug notifier will follow and we will handle it as CPU
		 * online then.  For now, just create the sysfs link, unless
		 * there is no policy or the link is already present.
		 */
		struct cpufreq_policy *policy = per_cpu(cpufreq_cpu_data, cpu);

		ret = policy && !cpumask_test_and_set_cpu(cpu, policy->real_cpus)
			? add_cpu_dev_symlink(policy, cpu) : 0;
	}
1304

Linus Torvalds's avatar
Linus Torvalds committed
1305 1306 1307
	return ret;
}

1308
static void cpufreq_offline_prepare(unsigned int cpu)
Linus Torvalds's avatar
Linus Torvalds committed
1309
{
1310
	struct cpufreq_policy *policy;
Linus Torvalds's avatar
Linus Torvalds committed
1311

1312
	pr_debug("%s: unregistering CPU %u\n", __func__, cpu);
Linus Torvalds's avatar
Linus Torvalds committed
1313

1314
	policy = cpufreq_cpu_get_raw(cpu);
1315
	if (!policy) {
1316
		pr_debug("%s: No cpu_data found\n", __func__);
1317
		return;
Linus Torvalds's avatar
Linus Torvalds committed
1318 1319
	}

1320
	if (has_target()) {
1321
		int ret = __cpufreq_governor(policy, CPUFREQ_GOV_STOP);
1322
		if (ret)
1323
			pr_err("%s: Failed to stop governor\n", __func__);
1324
	}
Linus Torvalds's avatar
Linus Torvalds committed
1325

1326
	down_write(&policy->rwsem);
1327
	cpumask_clear_cpu(cpu, policy->cpus);
1328

1329 1330 1331 1332 1333 1334 1335 1336
	if (policy_is_inactive(policy)) {
		if (has_target())
			strncpy(policy->last_governor, policy->governor->name,
				CPUFREQ_NAME_LEN);
	} else if (cpu == policy->cpu) {
		/* Nominate new CPU */
		policy->cpu = cpumask_any(policy->cpus);
	}
1337
	up_write(&policy->rwsem);
1338

1339 1340 1341
	/* Start governor again for active policy */
	if (!policy_is_inactive(policy)) {
		if (has_target()) {
1342
			int ret = __cpufreq_governor(policy, CPUFREQ_GOV_START);
1343 1344
			if (!ret)
				ret = __cpufreq_governor(policy, CPUFREQ_GOV_LIMITS);
1345

1346 1347 1348 1349
			if (ret)
				pr_err("%s: Failed to start governor\n", __func__);
		}
	} else if (cpufreq_driver->stop_cpu) {
1350
		cpufreq_driver->stop_cpu(policy);
1351
	}
1352 1353
}

1354
static void cpufreq_offline_finish(unsigned int cpu)
1355
{
1356
	struct cpufreq_policy *policy = per_cpu(cpufreq_cpu_data, cpu);
1357 1358 1359

	if (!policy) {
		pr_debug("%s: No cpu_data found\n", __func__);
1360
		return;
1361 1362
	}

1363 1364
	/* Only proceed for inactive policies */
	if (!policy_is_inactive(policy))
1365
		return;
1366 1367 1368

	/* If cpu is last user of policy, free policy */
	if (has_target()) {
1369
		int ret = __cpufreq_governor(policy, CPUFREQ_GOV_POLICY_EXIT);
1370
		if (ret)
1371
			pr_err("%s: Failed to exit governor\n", __func__);
1372
	}
Linus Torvalds's avatar
Linus Torvalds committed
1373

1374 1375 1376 1377 1378
	/*
	 * Perform the ->exit() even during light-weight tear-down,
	 * since this is a core component, and is essential for the
	 * subsequent light-weight ->init() to succeed.
	 */
1379
	if (cpufreq_driver->exit) {
1380
		cpufreq_driver->exit(policy);
1381 1382
		policy->freq_table = NULL;
	}
Linus Torvalds's avatar
Linus Torvalds committed
1383 1384
}

1385
/**
1386
 * cpufreq_remove_dev - remove a CPU device
1387 1388 1389
 *
 * Removes the cpufreq interface for a CPU device.
 */
1390
static void cpufreq_remove_dev(struct device *dev, struct subsys_interface *sif)
1391
{
1392
	unsigned int cpu = dev->id;
1393
	struct cpufreq_policy *policy = per_cpu(cpufreq_cpu_data, cpu);
1394

1395
	if (!policy)
1396
		return;
1397

1398
	if (cpu_online(cpu)) {
1399 1400
		cpufreq_offline_prepare(cpu);
		cpufreq_offline_finish(cpu);
1401
	}
1402

1403
	cpumask_clear_cpu(cpu, policy->real_cpus);
1404
	remove_cpu_dev_symlink(policy, cpu);
1405

1406
	if (cpumask_empty(policy->real_cpus))
1407
		cpufreq_policy_free(policy, true);
1408 1409
}

1410
static void handle_update(struct work_struct *work)
Linus Torvalds's avatar
Linus Torvalds committed
1411
{
1412 1413 1414
	struct cpufreq_policy *policy =
		container_of(work, struct cpufreq_policy, update);
	unsigned int cpu = policy->cpu;
1415
	pr_debug("handle_update for cpu %u called\n", cpu);
Linus Torvalds's avatar
Linus Torvalds committed
1416 1417 1418 1419
	cpufreq_update_policy(cpu);
}

/**
1420 1421
 *	cpufreq_out_of_sync - If actual and saved CPU frequency differs, we're
 *	in deep trouble.
1422
 *	@policy: policy managing CPUs
Linus Torvalds's avatar
Linus Torvalds committed
1423 1424
 *	@new_freq: CPU frequency the CPU actually runs at
 *
1425 1426
 *	We adjust to current frequency first, and need to clean up later.
 *	So either call to cpufreq_update_policy() or schedule handle_update()).
Linus Torvalds's avatar
Linus Torvalds committed
1427
 */
1428
static void cpufreq_out_of_sync(struct cpufreq_policy *policy,
1429
				unsigned int new_freq)
Linus Torvalds's avatar
Linus Torvalds committed
1430 1431
{
	struct cpufreq_freqs freqs;
1432

1433
	pr_debug("Warning: CPU frequency out of sync: cpufreq and timing core thinks of %u, is %u kHz\n",
1434
		 policy->cur, new_freq);
Linus Torvalds's avatar
Linus Torvalds committed
1435

1436
	freqs.old = policy->cur;
Linus Torvalds's avatar
Linus Torvalds committed
1437
	freqs.new = new_freq;
1438

1439 1440
	cpufreq_freq_transition_begin(policy, &freqs);
	cpufreq_freq_transition_end(policy, &freqs, 0);
Linus Torvalds's avatar
Linus Torvalds committed
1441 1442
}

1443
/**
1444
 * cpufreq_quick_get - get the CPU frequency (in kHz) from policy->cur
1445 1446 1447 1448 1449 1450 1451
 * @cpu: CPU number
 *
 * This is the last known freq, without actually getting it from the driver.
 * Return value will be same as what is shown in scaling_cur_freq in sysfs.
 */
unsigned int cpufreq_quick_get(unsigned int cpu)
{
1452
	struct cpufreq_policy *policy;
1453
	unsigned int ret_freq = 0;
1454

1455 1456
	if (cpufreq_driver && cpufreq_driver->setpolicy && cpufreq_driver->get)
		return cpufreq_driver->get(cpu);
1457 1458

	policy = cpufreq_cpu_get(cpu);
1459
	if (policy) {
1460
		ret_freq = policy->cur;
1461 1462 1463
		cpufreq_cpu_put(policy);
	}

Dave Jones's avatar
Dave Jones committed
1464
	return ret_freq;
1465 1466 1467
}
EXPORT_SYMBOL(cpufreq_quick_get);

1468 1469 1470 1471 1472 1473 1474 1475 1476 1477 1478 1479 1480 1481 1482 1483 1484 1485 1486 1487
/**
 * cpufreq_quick_get_max - get the max reported CPU frequency for this CPU
 * @cpu: CPU number
 *
 * Just return the max possible frequency for a given CPU.
 */
unsigned int cpufreq_quick_get_max(unsigned int cpu)
{
	struct cpufreq_policy *policy = cpufreq_cpu_get(cpu);
	unsigned int ret_freq = 0;

	if (policy) {
		ret_freq = policy->max;
		cpufreq_cpu_put(policy);
	}

	return ret_freq;
}
EXPORT_SYMBOL(cpufreq_quick_get_max);

1488
static unsigned int __cpufreq_get(struct cpufreq_policy *policy)
Linus Torvalds's avatar
Linus Torvalds committed
1489
{
1490
	unsigned int ret_freq = 0;
1491

1492
	if (!cpufreq_driver->get)
Dave Jones's avatar
Dave Jones committed
1493
		return ret_freq;
Linus Torvalds's avatar
Linus Torvalds committed
1494

1495
	ret_freq = cpufreq_driver->get(policy->cpu);
Linus Torvalds's avatar
Linus Torvalds committed
1496

1497 1498 1499 1500
	/* Updating inactive policies is invalid, so avoid doing that. */
	if (unlikely(policy_is_inactive(policy)))
		return ret_freq;

1501
	if (ret_freq && policy->cur &&
1502
		!(cpufreq_driver->flags & CPUFREQ_CONST_LOOPS)) {
1503 1504 1505
		/* verify no discrepancy between actual and
					saved value exists */
		if (unlikely(ret_freq != policy->cur)) {
1506
			cpufreq_out_of_sync(policy, ret_freq);
Linus Torvalds's avatar
Linus Torvalds committed
1507 1508 1509 1510
			schedule_work(&policy->update);
		}
	}

Dave Jones's avatar
Dave Jones committed
1511
	return ret_freq;
1512
}
Linus Torvalds's avatar
Linus Torvalds committed
1513

1514 1515 1516 1517 1518 1519 1520 1521
/**
 * cpufreq_get - get the current CPU frequency (in kHz)
 * @cpu: CPU number
 *
 * Get the CPU current (static) CPU frequency
 */
unsigned int cpufreq_get(unsigned int cpu)
{
1522
	struct cpufreq_policy *policy = cpufreq_cpu_get(cpu);
1523 1524
	unsigned int ret_freq = 0;

1525 1526
	if (policy) {
		down_read(&policy->rwsem);
1527
		ret_freq = __cpufreq_get(policy);
1528
		up_read(&policy->rwsem);
1529

1530 1531
		cpufreq_cpu_put(policy);
	}
1532

Dave Jones's avatar
Dave Jones committed
1533
	return ret_freq;
Linus Torvalds's avatar
Linus Torvalds committed
1534 1535 1536
}
EXPORT_SYMBOL(cpufreq_get);

1537 1538 1539 1540 1541
static struct subsys_interface cpufreq_interface = {
	.name		= "cpufreq",
	.subsys		= &cpu_subsys,
	.add_dev	= cpufreq_add_dev,
	.remove_dev	= cpufreq_remove_dev,
1542 1543
};

1544 1545 1546 1547 1548 1549 1550 1551 1552
/*
 * In case platform wants some specific frequency to be configured
 * during suspend..
 */
int cpufreq_generic_suspend(struct cpufreq_policy *policy)
{
	int ret;

	if (!policy->suspend_freq) {
1553 1554
		pr_debug("%s: suspend_freq not defined\n", __func__);
		return 0;
1555 1556 1557 1558 1559 1560 1561 1562 1563 1564 1565 1566 1567 1568 1569
	}

	pr_debug("%s: Setting suspend-freq: %u\n", __func__,
			policy->suspend_freq);

	ret = __cpufreq_driver_target(policy, policy->suspend_freq,
			CPUFREQ_RELATION_H);
	if (ret)
		pr_err("%s: unable to set suspend-freq: %u. err: %d\n",
				__func__, policy->suspend_freq, ret);

	return ret;
}
EXPORT_SYMBOL(cpufreq_generic_suspend);

1570
/**
1571
 * cpufreq_suspend() - Suspend CPUFreq governors
1572
 *
1573 1574 1575 1576
 * Called during system wide Suspend/Hibernate cycles for suspending governors
 * as some platforms can't change frequency after this point in suspend cycle.
 * Because some of the devices (like: i2c, regulators, etc) they use for
 * changing frequency are suspended quickly after this point.
1577
 */
1578
void cpufreq_suspend(void)
1579
{
1580
	struct cpufreq_policy *policy;
1581

1582 1583
	if (!cpufreq_driver)
		return;
1584

1585
	if (!has_target())
1586
		goto suspend;
1587

1588 1589
	pr_debug("%s: Suspending Governors\n", __func__);

1590
	for_each_active_policy(policy) {
1591 1592 1593 1594 1595 1596 1597
		if (__cpufreq_governor(policy, CPUFREQ_GOV_STOP))
			pr_err("%s: Failed to stop governor for policy: %p\n",
				__func__, policy);
		else if (cpufreq_driver->suspend
		    && cpufreq_driver->suspend(policy))
			pr_err("%s: Failed to suspend driver: %p\n", __func__,
				policy);
1598
	}
1599 1600 1601

suspend:
	cpufreq_suspended = true;
1602 1603
}

Linus Torvalds's avatar
Linus Torvalds committed
1604
/**
1605
 * cpufreq_resume() - Resume CPUFreq governors
Linus Torvalds's avatar
Linus Torvalds committed
1606
 *
1607 1608
 * Called during system wide Suspend/Hibernate cycle for resuming governors that
 * are suspended with cpufreq_suspend().
Linus Torvalds's avatar
Linus Torvalds committed
1609
 */
1610
void cpufreq_resume(void)
Linus Torvalds's avatar
Linus Torvalds committed
1611
{
1612
	struct cpufreq_policy *policy;
Linus Torvalds's avatar
Linus Torvalds committed
1613

1614 1615
	if (!cpufreq_driver)
		return;
Linus Torvalds's avatar
Linus Torvalds committed
1616

1617 1618
	cpufreq_suspended = false;

1619
	if (!has_target())
1620
		return;
Linus Torvalds's avatar
Linus Torvalds committed
1621

1622
	pr_debug("%s: Resuming Governors\n", __func__);
Linus Torvalds's avatar
Linus Torvalds committed
1623

1624
	for_each_active_policy(policy) {
1625 1626 1627 1628
		if (cpufreq_driver->resume && cpufreq_driver->resume(policy))
			pr_err("%s: Failed to resume driver: %p\n", __func__,
				policy);
		else if (__cpufreq_governor(policy, CPUFREQ_GOV_START)
1629 1630 1631 1632
		    || __cpufreq_governor(policy, CPUFREQ_GOV_LIMITS))
			pr_err("%s: Failed to start governor for policy: %p\n",
				__func__, policy);
	}
1633 1634 1635 1636 1637 1638 1639 1640 1641 1642 1643

	/*
	 * schedule call cpufreq_update_policy() for first-online CPU, as that
	 * wouldn't be hotplugged-out on suspend. It will verify that the
	 * current freq is in sync with what we believe it to be.
	 */
	policy = cpufreq_cpu_get_raw(cpumask_first(cpu_online_mask));
	if (WARN_ON(!policy))
		return;

	schedule_work(&policy->update);
1644
}
Linus Torvalds's avatar
Linus Torvalds committed
1645

1646 1647 1648 1649 1650 1651 1652 1653
/**
 *	cpufreq_get_current_driver - return current driver's name
 *
 *	Return the name string of the currently loaded cpufreq driver
 *	or NULL, if none.
 */
const char *cpufreq_get_current_driver(void)
{
1654 1655 1656 1657
	if (cpufreq_driver)
		return cpufreq_driver->name;

	return NULL;
1658 1659
}
EXPORT_SYMBOL_GPL(cpufreq_get_current_driver);
Linus Torvalds's avatar
Linus Torvalds committed
1660

1661 1662 1663 1664 1665 1666 1667 1668 1669 1670 1671 1672 1673 1674 1675
/**
 *	cpufreq_get_driver_data - return current driver data
 *
 *	Return the private data of the currently loaded cpufreq
 *	driver, or NULL if no cpufreq driver is loaded.
 */
void *cpufreq_get_driver_data(void)
{
	if (cpufreq_driver)
		return cpufreq_driver->driver_data;

	return NULL;
}
EXPORT_SYMBOL_GPL(cpufreq_get_driver_data);

Linus Torvalds's avatar
Linus Torvalds committed
1676 1677 1678 1679 1680 1681 1682 1683 1684
/*********************************************************************
 *                     NOTIFIER LISTS INTERFACE                      *
 *********************************************************************/

/**
 *	cpufreq_register_notifier - register a driver with cpufreq
 *	@nb: notifier function to register
 *      @list: CPUFREQ_TRANSITION_NOTIFIER or CPUFREQ_POLICY_NOTIFIER
 *
1685
 *	Add a driver to one of two lists: either a list of drivers that
Linus Torvalds's avatar
Linus Torvalds committed
1686 1687 1688 1689 1690
 *      are notified about clock rate changes (once before and once after
 *      the transition), or a list of drivers that are notified about
 *      changes in cpufreq policy.
 *
 *	This function may sleep, and has the same return conditions as
1691
 *	blocking_notifier_chain_register.
Linus Torvalds's avatar
Linus Torvalds committed
1692 1693 1694 1695 1696
 */
int cpufreq_register_notifier(struct notifier_block *nb, unsigned int list)
{
	int ret;

1697 1698 1699
	if (cpufreq_disabled())
		return -EINVAL;

1700 1701
	WARN_ON(!init_cpufreq_transition_notifier_list_called);

Linus Torvalds's avatar
Linus Torvalds committed
1702 1703
	switch (list) {
	case CPUFREQ_TRANSITION_NOTIFIER:
1704
		ret = srcu_notifier_chain_register(
1705
				&cpufreq_transition_notifier_list, nb);
Linus Torvalds's avatar
Linus Torvalds committed
1706 1707
		break;
	case CPUFREQ_POLICY_NOTIFIER:
1708 1709
		ret = blocking_notifier_chain_register(
				&cpufreq_policy_notifier_list, nb);
Linus Torvalds's avatar
Linus Torvalds committed
1710 1711 1712 1713 1714 1715 1716 1717 1718 1719 1720 1721
		break;
	default:
		ret = -EINVAL;
	}

	return ret;
}
EXPORT_SYMBOL(cpufreq_register_notifier);

/**
 *	cpufreq_unregister_notifier - unregister a driver with cpufreq
 *	@nb: notifier block to be unregistered
1722
 *	@list: CPUFREQ_TRANSITION_NOTIFIER or CPUFREQ_POLICY_NOTIFIER
Linus Torvalds's avatar
Linus Torvalds committed
1723 1724 1725 1726
 *
 *	Remove a driver from the CPU frequency notifier list.
 *
 *	This function may sleep, and has the same return conditions as
1727
 *	blocking_notifier_chain_unregister.
Linus Torvalds's avatar
Linus Torvalds committed
1728 1729 1730 1731 1732
 */
int cpufreq_unregister_notifier(struct notifier_block *nb, unsigned int list)
{
	int ret;

1733 1734 1735
	if (cpufreq_disabled())
		return -EINVAL;

Linus Torvalds's avatar
Linus Torvalds committed
1736 1737
	switch (list) {
	case CPUFREQ_TRANSITION_NOTIFIER:
1738
		ret = srcu_notifier_chain_unregister(
1739
				&cpufreq_transition_notifier_list, nb);
Linus Torvalds's avatar
Linus Torvalds committed
1740 1741
		break;
	case CPUFREQ_POLICY_NOTIFIER:
1742 1743
		ret = blocking_notifier_chain_unregister(
				&cpufreq_policy_notifier_list, nb);
Linus Torvalds's avatar
Linus Torvalds committed
1744 1745 1746 1747 1748 1749 1750 1751 1752 1753 1754 1755 1756 1757
		break;
	default:
		ret = -EINVAL;
	}

	return ret;
}
EXPORT_SYMBOL(cpufreq_unregister_notifier);


/*********************************************************************
 *                              GOVERNORS                            *
 *********************************************************************/

1758 1759 1760 1761 1762 1763 1764 1765 1766 1767 1768 1769 1770 1771 1772 1773 1774 1775 1776 1777 1778 1779 1780 1781 1782 1783
/* Must set freqs->new to intermediate frequency */
static int __target_intermediate(struct cpufreq_policy *policy,
				 struct cpufreq_freqs *freqs, int index)
{
	int ret;

	freqs->new = cpufreq_driver->get_intermediate(policy, index);

	/* We don't need to switch to intermediate freq */
	if (!freqs->new)
		return 0;

	pr_debug("%s: cpu: %d, switching to intermediate freq: oldfreq: %u, intermediate freq: %u\n",
		 __func__, policy->cpu, freqs->old, freqs->new);

	cpufreq_freq_transition_begin(policy, freqs);
	ret = cpufreq_driver->target_intermediate(policy, index);
	cpufreq_freq_transition_end(policy, freqs, ret);

	if (ret)
		pr_err("%s: Failed to change to intermediate frequency: %d\n",
		       __func__, ret);

	return ret;
}

1784 1785 1786
static int __target_index(struct cpufreq_policy *policy,
			  struct cpufreq_frequency_table *freq_table, int index)
{
1787 1788
	struct cpufreq_freqs freqs = {.old = policy->cur, .flags = 0};
	unsigned int intermediate_freq = 0;
1789 1790 1791 1792 1793
	int retval = -EINVAL;
	bool notify;

	notify = !(cpufreq_driver->flags & CPUFREQ_ASYNC_NOTIFICATION);
	if (notify) {
1794 1795 1796 1797 1798 1799 1800 1801 1802 1803 1804
		/* Handle switching to intermediate frequency */
		if (cpufreq_driver->get_intermediate) {
			retval = __target_intermediate(policy, &freqs, index);
			if (retval)
				return retval;

			intermediate_freq = freqs.new;
			/* Set old freq to intermediate */
			if (intermediate_freq)
				freqs.old = freqs.new;
		}
1805

1806
		freqs.new = freq_table[index].frequency;
1807 1808 1809 1810 1811 1812 1813 1814 1815 1816 1817
		pr_debug("%s: cpu: %d, oldfreq: %u, new freq: %u\n",
			 __func__, policy->cpu, freqs.old, freqs.new);

		cpufreq_freq_transition_begin(policy, &freqs);
	}

	retval = cpufreq_driver->target_index(policy, index);
	if (retval)
		pr_err("%s: Failed to change cpu frequency: %d\n", __func__,
		       retval);

1818
	if (notify) {
1819 1820
		cpufreq_freq_transition_end(policy, &freqs, retval);

1821 1822 1823 1824
		/*
		 * Failed after setting to intermediate freq? Driver should have
		 * reverted back to initial frequency and so should we. Check
		 * here for intermediate_freq instead of get_intermediate, in
1825
		 * case we haven't switched to intermediate freq at all.
1826 1827 1828 1829 1830 1831 1832 1833 1834
		 */
		if (unlikely(retval && intermediate_freq)) {
			freqs.old = intermediate_freq;
			freqs.new = policy->restore_freq;
			cpufreq_freq_transition_begin(policy, &freqs);
			cpufreq_freq_transition_end(policy, &freqs, 0);
		}
	}

1835 1836 1837
	return retval;
}

Linus Torvalds's avatar
Linus Torvalds committed
1838 1839 1840 1841
int __cpufreq_driver_target(struct cpufreq_policy *policy,
			    unsigned int target_freq,
			    unsigned int relation)
{
1842
	unsigned int old_target_freq = target_freq;
1843
	int retval = -EINVAL;
1844

1845 1846 1847
	if (cpufreq_disabled())
		return -ENODEV;

1848 1849 1850 1851 1852 1853 1854
	/* Make sure that target_freq is within supported range */
	if (target_freq > policy->max)
		target_freq = policy->max;
	if (target_freq < policy->min)
		target_freq = policy->min;

	pr_debug("target for CPU %u: %u kHz, relation %u, requested %u kHz\n",
1855
		 policy->cpu, target_freq, relation, old_target_freq);
1856

1857 1858 1859 1860 1861 1862
	/*
	 * This might look like a redundant call as we are checking it again
	 * after finding index. But it is left intentionally for cases where
	 * exactly same freq is called again and so we can save on few function
	 * calls.
	 */
1863 1864 1865
	if (target_freq == policy->cur)
		return 0;

1866 1867 1868
	/* Save last value to restore later on errors */
	policy->restore_freq = policy->cur;

1869 1870
	if (cpufreq_driver->target)
		retval = cpufreq_driver->target(policy, target_freq, relation);
1871 1872 1873
	else if (cpufreq_driver->target_index) {
		struct cpufreq_frequency_table *freq_table;
		int index;
1874

1875 1876 1877 1878 1879 1880 1881 1882 1883 1884 1885 1886 1887
		freq_table = cpufreq_frequency_get_table(policy->cpu);
		if (unlikely(!freq_table)) {
			pr_err("%s: Unable to find freq_table\n", __func__);
			goto out;
		}

		retval = cpufreq_frequency_table_target(policy, freq_table,
				target_freq, relation, &index);
		if (unlikely(retval)) {
			pr_err("%s: Unable to find matching freq\n", __func__);
			goto out;
		}

1888
		if (freq_table[index].frequency == policy->cur) {
1889
			retval = 0;
1890 1891 1892
			goto out;
		}

1893
		retval = __target_index(policy, freq_table, index);
1894 1895 1896
	}

out:
Linus Torvalds's avatar
Linus Torvalds committed
1897 1898 1899 1900 1901 1902 1903 1904
	return retval;
}
EXPORT_SYMBOL_GPL(__cpufreq_driver_target);

int cpufreq_driver_target(struct cpufreq_policy *policy,
			  unsigned int target_freq,
			  unsigned int relation)
{
1905
	int ret = -EINVAL;
Linus Torvalds's avatar
Linus Torvalds committed
1906

1907
	down_write(&policy->rwsem);
Linus Torvalds's avatar
Linus Torvalds committed
1908 1909 1910

	ret = __cpufreq_driver_target(policy, target_freq, relation);

1911
	up_write(&policy->rwsem);
Linus Torvalds's avatar
Linus Torvalds committed
1912 1913 1914 1915 1916

	return ret;
}
EXPORT_SYMBOL_GPL(cpufreq_driver_target);

1917 1918
static int __cpufreq_governor(struct cpufreq_policy *policy,
					unsigned int event)
Linus Torvalds's avatar
Linus Torvalds committed
1919
{
1920
	int ret;
1921 1922 1923 1924 1925 1926 1927 1928 1929 1930

	/* Only must be defined when default governor is known to have latency
	   restrictions, like e.g. conservative or ondemand.
	   That this is the case is already ensured in Kconfig
	*/
#ifdef CONFIG_CPU_FREQ_GOV_PERFORMANCE
	struct cpufreq_governor *gov = &cpufreq_gov_performance;
#else
	struct cpufreq_governor *gov = NULL;
#endif
1931

1932 1933 1934
	/* Don't start any governor operations if we are entering suspend */
	if (cpufreq_suspended)
		return 0;
1935 1936 1937 1938 1939 1940
	/*
	 * Governor might not be initiated here if ACPI _PPC changed
	 * notification happened, so check it.
	 */
	if (!policy->governor)
		return -EINVAL;
1941

1942 1943 1944
	if (policy->governor->max_transition_latency &&
	    policy->cpuinfo.transition_latency >
	    policy->governor->max_transition_latency) {
1945 1946 1947
		if (!gov)
			return -EINVAL;
		else {
1948 1949
			pr_warn("%s governor failed, too long transition latency of HW, fallback to %s governor\n",
				policy->governor->name, gov->name);
1950 1951
			policy->governor = gov;
		}
1952
	}
Linus Torvalds's avatar
Linus Torvalds committed
1953

1954 1955 1956
	if (event == CPUFREQ_GOV_POLICY_INIT)
		if (!try_module_get(policy->governor->owner))
			return -EINVAL;
Linus Torvalds's avatar
Linus Torvalds committed
1957

1958
	pr_debug("%s: for CPU %u, event %u\n", __func__, policy->cpu, event);
1959 1960

	mutex_lock(&cpufreq_governor_lock);
1961
	if ((policy->governor_enabled && event == CPUFREQ_GOV_START)
1962 1963
	    || (!policy->governor_enabled
	    && (event == CPUFREQ_GOV_LIMITS || event == CPUFREQ_GOV_STOP))) {
1964 1965 1966 1967 1968 1969 1970 1971 1972 1973 1974
		mutex_unlock(&cpufreq_governor_lock);
		return -EBUSY;
	}

	if (event == CPUFREQ_GOV_STOP)
		policy->governor_enabled = false;
	else if (event == CPUFREQ_GOV_START)
		policy->governor_enabled = true;

	mutex_unlock(&cpufreq_governor_lock);

Linus Torvalds's avatar
Linus Torvalds committed
1975 1976
	ret = policy->governor->governor(policy, event);

1977 1978 1979 1980 1981
	if (!ret) {
		if (event == CPUFREQ_GOV_POLICY_INIT)
			policy->governor->initialized++;
		else if (event == CPUFREQ_GOV_POLICY_EXIT)
			policy->governor->initialized--;
1982 1983 1984 1985 1986 1987 1988 1989
	} else {
		/* Restore original values */
		mutex_lock(&cpufreq_governor_lock);
		if (event == CPUFREQ_GOV_STOP)
			policy->governor_enabled = true;
		else if (event == CPUFREQ_GOV_START)
			policy->governor_enabled = false;
		mutex_unlock(&cpufreq_governor_lock);
1990
	}
1991

1992 1993
	if (((event == CPUFREQ_GOV_POLICY_INIT) && ret) ||
			((event == CPUFREQ_GOV_POLICY_EXIT) && !ret))
Linus Torvalds's avatar
Linus Torvalds committed
1994 1995 1996 1997 1998 1999 2000
		module_put(policy->governor->owner);

	return ret;
}

int cpufreq_register_governor(struct cpufreq_governor *governor)
{
2001
	int err;
Linus Torvalds's avatar
Linus Torvalds committed
2002 2003 2004 2005

	if (!governor)
		return -EINVAL;

2006 2007 2008
	if (cpufreq_disabled())
		return -ENODEV;

2009
	mutex_lock(&cpufreq_governor_mutex);
2010

2011
	governor->initialized = 0;
2012
	err = -EBUSY;
2013
	if (!find_governor(governor->name)) {
2014 2015
		err = 0;
		list_add(&governor->governor_list, &cpufreq_governor_list);
Linus Torvalds's avatar
Linus Torvalds committed
2016 2017
	}

2018
	mutex_unlock(&cpufreq_governor_mutex);
2019
	return err;
Linus Torvalds's avatar
Linus Torvalds committed
2020 2021 2022 2023 2024
}
EXPORT_SYMBOL_GPL(cpufreq_register_governor);

void cpufreq_unregister_governor(struct cpufreq_governor *governor)
{
2025 2026
	struct cpufreq_policy *policy;
	unsigned long flags;
2027

Linus Torvalds's avatar
Linus Torvalds committed
2028 2029 2030
	if (!governor)
		return;

2031 2032 2033
	if (cpufreq_disabled())
		return;

2034 2035 2036
	/* clear last_governor for all inactive policies */
	read_lock_irqsave(&cpufreq_driver_lock, flags);
	for_each_inactive_policy(policy) {
2037 2038
		if (!strcmp(policy->last_governor, governor->name)) {
			policy->governor = NULL;
2039
			strcpy(policy->last_governor, "\0");
2040
		}
2041
	}
2042
	read_unlock_irqrestore(&cpufreq_driver_lock, flags);
2043

2044
	mutex_lock(&cpufreq_governor_mutex);
Linus Torvalds's avatar
Linus Torvalds committed
2045
	list_del(&governor->governor_list);
2046
	mutex_unlock(&cpufreq_governor_mutex);
Linus Torvalds's avatar
Linus Torvalds committed
2047 2048 2049 2050 2051 2052 2053 2054 2055 2056 2057
	return;
}
EXPORT_SYMBOL_GPL(cpufreq_unregister_governor);


/*********************************************************************
 *                          POLICY INTERFACE                         *
 *********************************************************************/

/**
 * cpufreq_get_policy - get the current cpufreq_policy
2058 2059
 * @policy: struct cpufreq_policy into which the current cpufreq_policy
 *	is written
Linus Torvalds's avatar
Linus Torvalds committed
2060 2061 2062 2063 2064 2065 2066 2067 2068 2069 2070 2071 2072
 *
 * Reads the current cpufreq policy.
 */
int cpufreq_get_policy(struct cpufreq_policy *policy, unsigned int cpu)
{
	struct cpufreq_policy *cpu_policy;
	if (!policy)
		return -EINVAL;

	cpu_policy = cpufreq_cpu_get(cpu);
	if (!cpu_policy)
		return -EINVAL;

2073
	memcpy(policy, cpu_policy, sizeof(*policy));
Linus Torvalds's avatar
Linus Torvalds committed
2074 2075 2076 2077 2078 2079

	cpufreq_cpu_put(cpu_policy);
	return 0;
}
EXPORT_SYMBOL(cpufreq_get_policy);

2080
/*
2081 2082
 * policy : current policy.
 * new_policy: policy to be set.
2083
 */
2084
static int cpufreq_set_policy(struct cpufreq_policy *policy,
2085
				struct cpufreq_policy *new_policy)
Linus Torvalds's avatar
Linus Torvalds committed
2086
{
2087 2088
	struct cpufreq_governor *old_gov;
	int ret;
Linus Torvalds's avatar
Linus Torvalds committed
2089

2090 2091
	pr_debug("setting new policy for CPU %u: %u - %u kHz\n",
		 new_policy->cpu, new_policy->min, new_policy->max);
Linus Torvalds's avatar
Linus Torvalds committed
2092

2093
	memcpy(&new_policy->cpuinfo, &policy->cpuinfo, sizeof(policy->cpuinfo));
Linus Torvalds's avatar
Linus Torvalds committed
2094

2095 2096 2097 2098 2099
	/*
	* This check works well when we store new min/max freq attributes,
	* because new_policy is a copy of policy with one field updated.
	*/
	if (new_policy->min > new_policy->max)
2100
		return -EINVAL;
2101

Linus Torvalds's avatar
Linus Torvalds committed
2102
	/* verify the cpu speed can be set within this limit */
2103
	ret = cpufreq_driver->verify(new_policy);
Linus Torvalds's avatar
Linus Torvalds committed
2104
	if (ret)
2105
		return ret;
Linus Torvalds's avatar
Linus Torvalds committed
2106 2107

	/* adjust if necessary - all reasons */
2108
	blocking_notifier_call_chain(&cpufreq_policy_notifier_list,
2109
			CPUFREQ_ADJUST, new_policy);
Linus Torvalds's avatar
Linus Torvalds committed
2110

2111 2112 2113 2114
	/*
	 * verify the cpu speed can be set within this limit, which might be
	 * different to the first one
	 */
2115
	ret = cpufreq_driver->verify(new_policy);
2116
	if (ret)
2117
		return ret;
Linus Torvalds's avatar
Linus Torvalds committed
2118 2119

	/* notification of the new policy */
2120
	blocking_notifier_call_chain(&cpufreq_policy_notifier_list,
2121
			CPUFREQ_NOTIFY, new_policy);
Linus Torvalds's avatar
Linus Torvalds committed
2122

2123 2124
	policy->min = new_policy->min;
	policy->max = new_policy->max;
Linus Torvalds's avatar
Linus Torvalds committed
2125

2126
	pr_debug("new min and max freqs are %u - %u kHz\n",
2127
		 policy->min, policy->max);
Linus Torvalds's avatar
Linus Torvalds committed
2128

2129
	if (cpufreq_driver->setpolicy) {
2130
		policy->policy = new_policy->policy;
2131
		pr_debug("setting range\n");
2132 2133
		return cpufreq_driver->setpolicy(new_policy);
	}
Linus Torvalds's avatar
Linus Torvalds committed
2134

2135 2136
	if (new_policy->governor == policy->governor)
		goto out;
2137

2138 2139 2140 2141 2142 2143
	pr_debug("governor switch\n");

	/* save old, working values */
	old_gov = policy->governor;
	/* end old governor */
	if (old_gov) {
2144 2145 2146 2147 2148 2149 2150 2151
		ret = __cpufreq_governor(policy, CPUFREQ_GOV_STOP);
		if (ret) {
			/* This can happen due to race with other operations */
			pr_debug("%s: Failed to Stop Governor: %s (%d)\n",
				 __func__, old_gov->name, ret);
			return ret;
		}

2152
		up_write(&policy->rwsem);
2153
		ret = __cpufreq_governor(policy, CPUFREQ_GOV_POLICY_EXIT);
2154
		down_write(&policy->rwsem);
2155 2156 2157 2158 2159 2160

		if (ret) {
			pr_err("%s: Failed to Exit Governor: %s (%d)\n",
			       __func__, old_gov->name, ret);
			return ret;
		}
Linus Torvalds's avatar
Linus Torvalds committed
2161 2162
	}

2163 2164
	/* start new governor */
	policy->governor = new_policy->governor;
2165 2166 2167 2168
	ret = __cpufreq_governor(policy, CPUFREQ_GOV_POLICY_INIT);
	if (!ret) {
		ret = __cpufreq_governor(policy, CPUFREQ_GOV_START);
		if (!ret)
2169 2170 2171 2172 2173 2174 2175 2176 2177 2178 2179
			goto out;

		up_write(&policy->rwsem);
		__cpufreq_governor(policy, CPUFREQ_GOV_POLICY_EXIT);
		down_write(&policy->rwsem);
	}

	/* new governor failed, so re-start old one */
	pr_debug("starting governor %s failed\n", policy->governor->name);
	if (old_gov) {
		policy->governor = old_gov;
2180 2181 2182 2183
		if (__cpufreq_governor(policy, CPUFREQ_GOV_POLICY_INIT))
			policy->governor = NULL;
		else
			__cpufreq_governor(policy, CPUFREQ_GOV_START);
2184 2185
	}

2186
	return ret;
2187 2188 2189 2190

 out:
	pr_debug("governor: change or update limits\n");
	return __cpufreq_governor(policy, CPUFREQ_GOV_LIMITS);
Linus Torvalds's avatar
Linus Torvalds committed
2191 2192 2193 2194 2195 2196
}

/**
 *	cpufreq_update_policy - re-evaluate an existing cpufreq policy
 *	@cpu: CPU which shall be re-evaluated
 *
Lucas De Marchi's avatar
Lucas De Marchi committed
2197
 *	Useful for policy notifiers which have different necessities
Linus Torvalds's avatar
Linus Torvalds committed
2198 2199 2200 2201
 *	at different times.
 */
int cpufreq_update_policy(unsigned int cpu)
{
2202 2203
	struct cpufreq_policy *policy = cpufreq_cpu_get(cpu);
	struct cpufreq_policy new_policy;
2204
	int ret;
Linus Torvalds's avatar
Linus Torvalds committed
2205

2206 2207
	if (!policy)
		return -ENODEV;
Linus Torvalds's avatar
Linus Torvalds committed
2208

2209
	down_write(&policy->rwsem);
Linus Torvalds's avatar
Linus Torvalds committed
2210

2211
	pr_debug("updating policy for CPU %u\n", cpu);
2212
	memcpy(&new_policy, policy, sizeof(*policy));
2213 2214
	new_policy.min = policy->user_policy.min;
	new_policy.max = policy->user_policy.max;
Linus Torvalds's avatar
Linus Torvalds committed
2215

2216 2217 2218 2219
	/*
	 * BIOS might change freq behind our back
	 * -> ask driver for current freq and notify governors about a change
	 */
2220
	if (cpufreq_driver->get && !cpufreq_driver->setpolicy) {
2221
		new_policy.cur = cpufreq_driver->get(cpu);
2222 2223
		if (WARN_ON(!new_policy.cur)) {
			ret = -EIO;
2224
			goto unlock;
2225 2226
		}

2227
		if (!policy->cur) {
2228
			pr_debug("Driver did not initialize current freq\n");
2229
			policy->cur = new_policy.cur;
2230
		} else {
2231
			if (policy->cur != new_policy.cur && has_target())
2232
				cpufreq_out_of_sync(policy, new_policy.cur);
2233
		}
2234 2235
	}

2236
	ret = cpufreq_set_policy(policy, &new_policy);
Linus Torvalds's avatar
Linus Torvalds committed
2237

2238
unlock:
2239
	up_write(&policy->rwsem);
2240

2241
	cpufreq_cpu_put(policy);
Linus Torvalds's avatar
Linus Torvalds committed
2242 2243 2244 2245
	return ret;
}
EXPORT_SYMBOL(cpufreq_update_policy);

2246
static int cpufreq_cpu_callback(struct notifier_block *nfb,
2247 2248 2249 2250
					unsigned long action, void *hcpu)
{
	unsigned int cpu = (unsigned long)hcpu;

2251 2252 2253 2254
	switch (action & ~CPU_TASKS_FROZEN) {
	case CPU_ONLINE:
		cpufreq_online(cpu);
		break;
2255

2256 2257 2258
	case CPU_DOWN_PREPARE:
		cpufreq_offline_prepare(cpu);
		break;
2259

2260 2261 2262
	case CPU_POST_DEAD:
		cpufreq_offline_finish(cpu);
		break;
2263

2264 2265 2266
	case CPU_DOWN_FAILED:
		cpufreq_online(cpu);
		break;
2267 2268 2269 2270
	}
	return NOTIFY_OK;
}

2271
static struct notifier_block __refdata cpufreq_cpu_notifier = {
2272
	.notifier_call = cpufreq_cpu_callback,
2273
};
Linus Torvalds's avatar
Linus Torvalds committed
2274

2275 2276 2277 2278 2279 2280 2281 2282 2283
/*********************************************************************
 *               BOOST						     *
 *********************************************************************/
static int cpufreq_boost_set_sw(int state)
{
	struct cpufreq_frequency_table *freq_table;
	struct cpufreq_policy *policy;
	int ret = -EINVAL;

2284
	for_each_active_policy(policy) {
2285 2286 2287 2288 2289 2290 2291 2292 2293 2294 2295 2296 2297 2298 2299 2300 2301 2302 2303 2304 2305 2306 2307 2308 2309 2310 2311 2312 2313 2314 2315 2316 2317 2318 2319
		freq_table = cpufreq_frequency_get_table(policy->cpu);
		if (freq_table) {
			ret = cpufreq_frequency_table_cpuinfo(policy,
							freq_table);
			if (ret) {
				pr_err("%s: Policy frequency update failed\n",
				       __func__);
				break;
			}
			policy->user_policy.max = policy->max;
			__cpufreq_governor(policy, CPUFREQ_GOV_LIMITS);
		}
	}

	return ret;
}

int cpufreq_boost_trigger_state(int state)
{
	unsigned long flags;
	int ret = 0;

	if (cpufreq_driver->boost_enabled == state)
		return 0;

	write_lock_irqsave(&cpufreq_driver_lock, flags);
	cpufreq_driver->boost_enabled = state;
	write_unlock_irqrestore(&cpufreq_driver_lock, flags);

	ret = cpufreq_driver->set_boost(state);
	if (ret) {
		write_lock_irqsave(&cpufreq_driver_lock, flags);
		cpufreq_driver->boost_enabled = !state;
		write_unlock_irqrestore(&cpufreq_driver_lock, flags);

2320 2321
		pr_err("%s: Cannot %s BOOST\n",
		       __func__, state ? "enable" : "disable");
2322 2323 2324 2325 2326 2327 2328 2329 2330 2331 2332 2333 2334 2335
	}

	return ret;
}

int cpufreq_boost_supported(void)
{
	if (likely(cpufreq_driver))
		return cpufreq_driver->boost_supported;

	return 0;
}
EXPORT_SYMBOL_GPL(cpufreq_boost_supported);

2336 2337 2338 2339 2340 2341 2342 2343 2344 2345 2346 2347 2348 2349
static int create_boost_sysfs_file(void)
{
	int ret;

	if (!cpufreq_boost_supported())
		return 0;

	/*
	 * Check if driver provides function to enable boost -
	 * if not, use cpufreq_boost_set_sw as default
	 */
	if (!cpufreq_driver->set_boost)
		cpufreq_driver->set_boost = cpufreq_boost_set_sw;

2350
	ret = sysfs_create_file(cpufreq_global_kobject, &boost.attr);
2351 2352 2353 2354 2355 2356 2357 2358 2359 2360
	if (ret)
		pr_err("%s: cannot register global BOOST sysfs file\n",
		       __func__);

	return ret;
}

static void remove_boost_sysfs_file(void)
{
	if (cpufreq_boost_supported())
2361
		sysfs_remove_file(cpufreq_global_kobject, &boost.attr);
2362 2363 2364 2365 2366 2367 2368 2369 2370 2371 2372 2373 2374 2375 2376 2377 2378
}

int cpufreq_enable_boost_support(void)
{
	if (!cpufreq_driver)
		return -EINVAL;

	if (cpufreq_boost_supported())
		return 0;

	cpufreq_driver->boost_supported = true;

	/* This will get removed on driver unregister */
	return create_boost_sysfs_file();
}
EXPORT_SYMBOL_GPL(cpufreq_enable_boost_support);

2379 2380 2381 2382 2383 2384
int cpufreq_boost_enabled(void)
{
	return cpufreq_driver->boost_enabled;
}
EXPORT_SYMBOL_GPL(cpufreq_boost_enabled);

Linus Torvalds's avatar
Linus Torvalds committed
2385 2386 2387 2388 2389 2390 2391 2392 2393
/*********************************************************************
 *               REGISTER / UNREGISTER CPUFREQ DRIVER                *
 *********************************************************************/

/**
 * cpufreq_register_driver - register a CPU Frequency driver
 * @driver_data: A struct cpufreq_driver containing the values#
 * submitted by the CPU Frequency driver.
 *
2394
 * Registers a CPU Frequency driver to this core code. This code
Linus Torvalds's avatar
Linus Torvalds committed
2395
 * returns zero on success, -EBUSY when another driver got here first
2396
 * (and isn't unregistered in the meantime).
Linus Torvalds's avatar
Linus Torvalds committed
2397 2398
 *
 */
2399
int cpufreq_register_driver(struct cpufreq_driver *driver_data)
Linus Torvalds's avatar
Linus Torvalds committed
2400 2401 2402 2403
{
	unsigned long flags;
	int ret;

2404 2405 2406
	if (cpufreq_disabled())
		return -ENODEV;

Linus Torvalds's avatar
Linus Torvalds committed
2407
	if (!driver_data || !driver_data->verify || !driver_data->init ||
2408
	    !(driver_data->setpolicy || driver_data->target_index ||
2409 2410
		    driver_data->target) ||
	     (driver_data->setpolicy && (driver_data->target_index ||
2411 2412
		    driver_data->target)) ||
	     (!!driver_data->get_intermediate != !!driver_data->target_intermediate))
Linus Torvalds's avatar
Linus Torvalds committed
2413 2414
		return -EINVAL;

2415
	pr_debug("trying to register driver %s\n", driver_data->name);
Linus Torvalds's avatar
Linus Torvalds committed
2416

2417 2418 2419
	/* Protect against concurrent CPU online/offline. */
	get_online_cpus();

2420
	write_lock_irqsave(&cpufreq_driver_lock, flags);
2421
	if (cpufreq_driver) {
2422
		write_unlock_irqrestore(&cpufreq_driver_lock, flags);
2423 2424
		ret = -EEXIST;
		goto out;
Linus Torvalds's avatar
Linus Torvalds committed
2425
	}
2426
	cpufreq_driver = driver_data;
2427
	write_unlock_irqrestore(&cpufreq_driver_lock, flags);
Linus Torvalds's avatar
Linus Torvalds committed
2428

2429 2430 2431
	if (driver_data->setpolicy)
		driver_data->flags |= CPUFREQ_CONST_LOOPS;

2432 2433 2434
	ret = create_boost_sysfs_file();
	if (ret)
		goto err_null_driver;
2435

2436
	ret = subsys_interface_register(&cpufreq_interface);
2437
	if (ret)
2438
		goto err_boost_unreg;
Linus Torvalds's avatar
Linus Torvalds committed
2439

2440 2441
	if (!(cpufreq_driver->flags & CPUFREQ_STICKY) &&
	    list_empty(&cpufreq_policy_list)) {
Linus Torvalds's avatar
Linus Torvalds committed
2442
		/* if all ->init() calls failed, unregister */
2443 2444 2445
		pr_debug("%s: No CPU initialized for driver %s\n", __func__,
			 driver_data->name);
		goto err_if_unreg;
Linus Torvalds's avatar
Linus Torvalds committed
2446 2447
	}

2448
	register_hotcpu_notifier(&cpufreq_cpu_notifier);
2449
	pr_debug("driver %s up and running\n", driver_data->name);
Linus Torvalds's avatar
Linus Torvalds committed
2450

2451 2452 2453 2454
out:
	put_online_cpus();
	return ret;

2455 2456
err_if_unreg:
	subsys_interface_unregister(&cpufreq_interface);
2457
err_boost_unreg:
2458
	remove_boost_sysfs_file();
2459
err_null_driver:
2460
	write_lock_irqsave(&cpufreq_driver_lock, flags);
2461
	cpufreq_driver = NULL;
2462
	write_unlock_irqrestore(&cpufreq_driver_lock, flags);
2463
	goto out;
Linus Torvalds's avatar
Linus Torvalds committed
2464 2465 2466 2467 2468 2469
}
EXPORT_SYMBOL_GPL(cpufreq_register_driver);

/**
 * cpufreq_unregister_driver - unregister the current CPUFreq driver
 *
2470
 * Unregister the current CPUFreq driver. Only call this if you have
Linus Torvalds's avatar
Linus Torvalds committed
2471 2472 2473 2474
 * the right to do so, i.e. if you have succeeded in initialising before!
 * Returns zero if successful, and -EINVAL if the cpufreq_driver is
 * currently not initialised.
 */
2475
int cpufreq_unregister_driver(struct cpufreq_driver *driver)
Linus Torvalds's avatar
Linus Torvalds committed
2476 2477 2478
{
	unsigned long flags;

2479
	if (!cpufreq_driver || (driver != cpufreq_driver))
Linus Torvalds's avatar
Linus Torvalds committed
2480 2481
		return -EINVAL;

2482
	pr_debug("unregistering driver %s\n", driver->name);
Linus Torvalds's avatar
Linus Torvalds committed
2483

2484 2485
	/* Protect against concurrent cpu hotplug */
	get_online_cpus();
2486
	subsys_interface_unregister(&cpufreq_interface);
2487
	remove_boost_sysfs_file();
2488
	unregister_hotcpu_notifier(&cpufreq_cpu_notifier);
Linus Torvalds's avatar
Linus Torvalds committed
2489

2490
	write_lock_irqsave(&cpufreq_driver_lock, flags);
2491

2492
	cpufreq_driver = NULL;
2493

2494
	write_unlock_irqrestore(&cpufreq_driver_lock, flags);
2495
	put_online_cpus();
Linus Torvalds's avatar
Linus Torvalds committed
2496 2497 2498 2499

	return 0;
}
EXPORT_SYMBOL_GPL(cpufreq_unregister_driver);
2500

2501 2502 2503 2504 2505 2506 2507 2508
/*
 * Stop cpufreq at shutdown to make sure it isn't holding any locks
 * or mutexes when secondary CPUs are halted.
 */
static struct syscore_ops cpufreq_syscore_ops = {
	.shutdown = cpufreq_suspend,
};

2509 2510 2511
struct kobject *cpufreq_global_kobject;
EXPORT_SYMBOL(cpufreq_global_kobject);

2512 2513
static int __init cpufreq_core_init(void)
{
2514 2515 2516
	if (cpufreq_disabled())
		return -ENODEV;

2517
	cpufreq_global_kobject = kobject_create_and_add("cpufreq", &cpu_subsys.dev_root->kobj);
2518 2519
	BUG_ON(!cpufreq_global_kobject);

2520 2521
	register_syscore_ops(&cpufreq_syscore_ops);

2522 2523 2524
	return 0;
}
core_initcall(cpufreq_core_init);