Merge tag 'pm-5.1-rc1-2' of git://git.kernel.org/pub/scm/linux/kernel/git/rafael...
[muen/linux.git] / drivers / cpufreq / cpufreq.c
1 /*
2  *  linux/drivers/cpufreq/cpufreq.c
3  *
4  *  Copyright (C) 2001 Russell King
5  *            (C) 2002 - 2003 Dominik Brodowski <linux@brodo.de>
6  *            (C) 2013 Viresh Kumar <viresh.kumar@linaro.org>
7  *
8  *  Oct 2005 - Ashok Raj <ashok.raj@intel.com>
9  *      Added handling for CPU hotplug
10  *  Feb 2006 - Jacob Shin <jacob.shin@amd.com>
11  *      Fix handling for CPU hotplug -- affected CPUs
12  *
13  * This program is free software; you can redistribute it and/or modify
14  * it under the terms of the GNU General Public License version 2 as
15  * published by the Free Software Foundation.
16  */
17
18 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
19
20 #include <linux/cpu.h>
21 #include <linux/cpufreq.h>
22 #include <linux/cpu_cooling.h>
23 #include <linux/delay.h>
24 #include <linux/device.h>
25 #include <linux/init.h>
26 #include <linux/kernel_stat.h>
27 #include <linux/module.h>
28 #include <linux/mutex.h>
29 #include <linux/slab.h>
30 #include <linux/suspend.h>
31 #include <linux/syscore_ops.h>
32 #include <linux/tick.h>
33 #include <trace/events/power.h>
34
35 static LIST_HEAD(cpufreq_policy_list);
36
37 static inline bool policy_is_inactive(struct cpufreq_policy *policy)
38 {
39         return cpumask_empty(policy->cpus);
40 }
41
42 /* Macros to iterate over CPU policies */
43 #define for_each_suitable_policy(__policy, __active)                     \
44         list_for_each_entry(__policy, &cpufreq_policy_list, policy_list) \
45                 if ((__active) == !policy_is_inactive(__policy))
46
47 #define for_each_active_policy(__policy)                \
48         for_each_suitable_policy(__policy, true)
49 #define for_each_inactive_policy(__policy)              \
50         for_each_suitable_policy(__policy, false)
51
52 #define for_each_policy(__policy)                       \
53         list_for_each_entry(__policy, &cpufreq_policy_list, policy_list)
54
55 /* Iterate over governors */
56 static LIST_HEAD(cpufreq_governor_list);
57 #define for_each_governor(__governor)                           \
58         list_for_each_entry(__governor, &cpufreq_governor_list, governor_list)
59
60 /**
61  * The "cpufreq driver" - the arch- or hardware-dependent low
62  * level driver of CPUFreq support, and its spinlock. This lock
63  * also protects the cpufreq_cpu_data array.
64  */
65 static struct cpufreq_driver *cpufreq_driver;
66 static DEFINE_PER_CPU(struct cpufreq_policy *, cpufreq_cpu_data);
67 static DEFINE_RWLOCK(cpufreq_driver_lock);
68
69 /* Flag to suspend/resume CPUFreq governors */
70 static bool cpufreq_suspended;
71
72 static inline bool has_target(void)
73 {
74         return cpufreq_driver->target_index || cpufreq_driver->target;
75 }
76
77 /* internal prototypes */
78 static unsigned int __cpufreq_get(struct cpufreq_policy *policy);
79 static int cpufreq_init_governor(struct cpufreq_policy *policy);
80 static void cpufreq_exit_governor(struct cpufreq_policy *policy);
81 static int cpufreq_start_governor(struct cpufreq_policy *policy);
82 static void cpufreq_stop_governor(struct cpufreq_policy *policy);
83 static void cpufreq_governor_limits(struct cpufreq_policy *policy);
84
85 /**
86  * Two notifier lists: the "policy" list is involved in the
87  * validation process for a new CPU frequency policy; the
88  * "transition" list for kernel code that needs to handle
89  * changes to devices when the CPU clock speed changes.
90  * The mutex locks both lists.
91  */
92 static BLOCKING_NOTIFIER_HEAD(cpufreq_policy_notifier_list);
93 SRCU_NOTIFIER_HEAD_STATIC(cpufreq_transition_notifier_list);
94
95 static int off __read_mostly;
96 static int cpufreq_disabled(void)
97 {
98         return off;
99 }
100 void disable_cpufreq(void)
101 {
102         off = 1;
103 }
104 static DEFINE_MUTEX(cpufreq_governor_mutex);
105
106 bool have_governor_per_policy(void)
107 {
108         return !!(cpufreq_driver->flags & CPUFREQ_HAVE_GOVERNOR_PER_POLICY);
109 }
110 EXPORT_SYMBOL_GPL(have_governor_per_policy);
111
112 struct kobject *get_governor_parent_kobj(struct cpufreq_policy *policy)
113 {
114         if (have_governor_per_policy())
115                 return &policy->kobj;
116         else
117                 return cpufreq_global_kobject;
118 }
119 EXPORT_SYMBOL_GPL(get_governor_parent_kobj);
120
121 static inline u64 get_cpu_idle_time_jiffy(unsigned int cpu, u64 *wall)
122 {
123         u64 idle_time;
124         u64 cur_wall_time;
125         u64 busy_time;
126
127         cur_wall_time = jiffies64_to_nsecs(get_jiffies_64());
128
129         busy_time = kcpustat_cpu(cpu).cpustat[CPUTIME_USER];
130         busy_time += kcpustat_cpu(cpu).cpustat[CPUTIME_SYSTEM];
131         busy_time += kcpustat_cpu(cpu).cpustat[CPUTIME_IRQ];
132         busy_time += kcpustat_cpu(cpu).cpustat[CPUTIME_SOFTIRQ];
133         busy_time += kcpustat_cpu(cpu).cpustat[CPUTIME_STEAL];
134         busy_time += kcpustat_cpu(cpu).cpustat[CPUTIME_NICE];
135
136         idle_time = cur_wall_time - busy_time;
137         if (wall)
138                 *wall = div_u64(cur_wall_time, NSEC_PER_USEC);
139
140         return div_u64(idle_time, NSEC_PER_USEC);
141 }
142
143 u64 get_cpu_idle_time(unsigned int cpu, u64 *wall, int io_busy)
144 {
145         u64 idle_time = get_cpu_idle_time_us(cpu, io_busy ? wall : NULL);
146
147         if (idle_time == -1ULL)
148                 return get_cpu_idle_time_jiffy(cpu, wall);
149         else if (!io_busy)
150                 idle_time += get_cpu_iowait_time_us(cpu, wall);
151
152         return idle_time;
153 }
154 EXPORT_SYMBOL_GPL(get_cpu_idle_time);
155
156 __weak void arch_set_freq_scale(struct cpumask *cpus, unsigned long cur_freq,
157                 unsigned long max_freq)
158 {
159 }
160 EXPORT_SYMBOL_GPL(arch_set_freq_scale);
161
162 /*
163  * This is a generic cpufreq init() routine which can be used by cpufreq
164  * drivers of SMP systems. It will do following:
165  * - validate & show freq table passed
166  * - set policies transition latency
167  * - policy->cpus with all possible CPUs
168  */
169 int cpufreq_generic_init(struct cpufreq_policy *policy,
170                 struct cpufreq_frequency_table *table,
171                 unsigned int transition_latency)
172 {
173         policy->freq_table = table;
174         policy->cpuinfo.transition_latency = transition_latency;
175
176         /*
177          * The driver only supports the SMP configuration where all processors
178          * share the clock and voltage and clock.
179          */
180         cpumask_setall(policy->cpus);
181
182         return 0;
183 }
184 EXPORT_SYMBOL_GPL(cpufreq_generic_init);
185
186 struct cpufreq_policy *cpufreq_cpu_get_raw(unsigned int cpu)
187 {
188         struct cpufreq_policy *policy = per_cpu(cpufreq_cpu_data, cpu);
189
190         return policy && cpumask_test_cpu(cpu, policy->cpus) ? policy : NULL;
191 }
192 EXPORT_SYMBOL_GPL(cpufreq_cpu_get_raw);
193
194 unsigned int cpufreq_generic_get(unsigned int cpu)
195 {
196         struct cpufreq_policy *policy = cpufreq_cpu_get_raw(cpu);
197
198         if (!policy || IS_ERR(policy->clk)) {
199                 pr_err("%s: No %s associated to cpu: %d\n",
200                        __func__, policy ? "clk" : "policy", cpu);
201                 return 0;
202         }
203
204         return clk_get_rate(policy->clk) / 1000;
205 }
206 EXPORT_SYMBOL_GPL(cpufreq_generic_get);
207
208 /**
209  * cpufreq_cpu_get - Return policy for a CPU and mark it as busy.
210  * @cpu: CPU to find the policy for.
211  *
212  * Call cpufreq_cpu_get_raw() to obtain a cpufreq policy for @cpu and increment
213  * the kobject reference counter of that policy.  Return a valid policy on
214  * success or NULL on failure.
215  *
216  * The policy returned by this function has to be released with the help of
217  * cpufreq_cpu_put() to balance its kobject reference counter properly.
218  */
219 struct cpufreq_policy *cpufreq_cpu_get(unsigned int cpu)
220 {
221         struct cpufreq_policy *policy = NULL;
222         unsigned long flags;
223
224         if (WARN_ON(cpu >= nr_cpu_ids))
225                 return NULL;
226
227         /* get the cpufreq driver */
228         read_lock_irqsave(&cpufreq_driver_lock, flags);
229
230         if (cpufreq_driver) {
231                 /* get the CPU */
232                 policy = cpufreq_cpu_get_raw(cpu);
233                 if (policy)
234                         kobject_get(&policy->kobj);
235         }
236
237         read_unlock_irqrestore(&cpufreq_driver_lock, flags);
238
239         return policy;
240 }
241 EXPORT_SYMBOL_GPL(cpufreq_cpu_get);
242
243 /**
244  * cpufreq_cpu_put - Decrement kobject usage counter for cpufreq policy.
245  * @policy: cpufreq policy returned by cpufreq_cpu_get().
246  */
247 void cpufreq_cpu_put(struct cpufreq_policy *policy)
248 {
249         kobject_put(&policy->kobj);
250 }
251 EXPORT_SYMBOL_GPL(cpufreq_cpu_put);
252
253 /*********************************************************************
254  *            EXTERNALLY AFFECTING FREQUENCY CHANGES                 *
255  *********************************************************************/
256
257 /**
258  * adjust_jiffies - adjust the system "loops_per_jiffy"
259  *
260  * This function alters the system "loops_per_jiffy" for the clock
261  * speed change. Note that loops_per_jiffy cannot be updated on SMP
262  * systems as each CPU might be scaled differently. So, use the arch
263  * per-CPU loops_per_jiffy value wherever possible.
264  */
265 static void adjust_jiffies(unsigned long val, struct cpufreq_freqs *ci)
266 {
267 #ifndef CONFIG_SMP
268         static unsigned long l_p_j_ref;
269         static unsigned int l_p_j_ref_freq;
270
271         if (ci->flags & CPUFREQ_CONST_LOOPS)
272                 return;
273
274         if (!l_p_j_ref_freq) {
275                 l_p_j_ref = loops_per_jiffy;
276                 l_p_j_ref_freq = ci->old;
277                 pr_debug("saving %lu as reference value for loops_per_jiffy; freq is %u kHz\n",
278                          l_p_j_ref, l_p_j_ref_freq);
279         }
280         if (val == CPUFREQ_POSTCHANGE && ci->old != ci->new) {
281                 loops_per_jiffy = cpufreq_scale(l_p_j_ref, l_p_j_ref_freq,
282                                                                 ci->new);
283                 pr_debug("scaling loops_per_jiffy to %lu for frequency %u kHz\n",
284                          loops_per_jiffy, ci->new);
285         }
286 #endif
287 }
288
289 /**
290  * cpufreq_notify_transition - Notify frequency transition and adjust_jiffies.
291  * @policy: cpufreq policy to enable fast frequency switching for.
292  * @freqs: contain details of the frequency update.
293  * @state: set to CPUFREQ_PRECHANGE or CPUFREQ_POSTCHANGE.
294  *
295  * This function calls the transition notifiers and the "adjust_jiffies"
296  * function. It is called twice on all CPU frequency changes that have
297  * external effects.
298  */
299 static void cpufreq_notify_transition(struct cpufreq_policy *policy,
300                                       struct cpufreq_freqs *freqs,
301                                       unsigned int state)
302 {
303         BUG_ON(irqs_disabled());
304
305         if (cpufreq_disabled())
306                 return;
307
308         freqs->flags = cpufreq_driver->flags;
309         pr_debug("notification %u of frequency transition to %u kHz\n",
310                  state, freqs->new);
311
312         switch (state) {
313         case CPUFREQ_PRECHANGE:
314                 /*
315                  * Detect if the driver reported a value as "old frequency"
316                  * which is not equal to what the cpufreq core thinks is
317                  * "old frequency".
318                  */
319                 if (!(cpufreq_driver->flags & CPUFREQ_CONST_LOOPS)) {
320                         if (policy->cur && (policy->cur != freqs->old)) {
321                                 pr_debug("Warning: CPU frequency is %u, cpufreq assumed %u kHz\n",
322                                          freqs->old, policy->cur);
323                                 freqs->old = policy->cur;
324                         }
325                 }
326
327                 for_each_cpu(freqs->cpu, policy->cpus) {
328                         srcu_notifier_call_chain(&cpufreq_transition_notifier_list,
329                                                  CPUFREQ_PRECHANGE, freqs);
330                 }
331
332                 adjust_jiffies(CPUFREQ_PRECHANGE, freqs);
333                 break;
334
335         case CPUFREQ_POSTCHANGE:
336                 adjust_jiffies(CPUFREQ_POSTCHANGE, freqs);
337                 pr_debug("FREQ: %u - CPUs: %*pbl\n", freqs->new,
338                          cpumask_pr_args(policy->cpus));
339
340                 for_each_cpu(freqs->cpu, policy->cpus) {
341                         trace_cpu_frequency(freqs->new, freqs->cpu);
342                         srcu_notifier_call_chain(&cpufreq_transition_notifier_list,
343                                                  CPUFREQ_POSTCHANGE, freqs);
344                 }
345
346                 cpufreq_stats_record_transition(policy, freqs->new);
347                 policy->cur = freqs->new;
348         }
349 }
350
351 /* Do post notifications when there are chances that transition has failed */
352 static void cpufreq_notify_post_transition(struct cpufreq_policy *policy,
353                 struct cpufreq_freqs *freqs, int transition_failed)
354 {
355         cpufreq_notify_transition(policy, freqs, CPUFREQ_POSTCHANGE);
356         if (!transition_failed)
357                 return;
358
359         swap(freqs->old, freqs->new);
360         cpufreq_notify_transition(policy, freqs, CPUFREQ_PRECHANGE);
361         cpufreq_notify_transition(policy, freqs, CPUFREQ_POSTCHANGE);
362 }
363
364 void cpufreq_freq_transition_begin(struct cpufreq_policy *policy,
365                 struct cpufreq_freqs *freqs)
366 {
367
368         /*
369          * Catch double invocations of _begin() which lead to self-deadlock.
370          * ASYNC_NOTIFICATION drivers are left out because the cpufreq core
371          * doesn't invoke _begin() on their behalf, and hence the chances of
372          * double invocations are very low. Moreover, there are scenarios
373          * where these checks can emit false-positive warnings in these
374          * drivers; so we avoid that by skipping them altogether.
375          */
376         WARN_ON(!(cpufreq_driver->flags & CPUFREQ_ASYNC_NOTIFICATION)
377                                 && current == policy->transition_task);
378
379 wait:
380         wait_event(policy->transition_wait, !policy->transition_ongoing);
381
382         spin_lock(&policy->transition_lock);
383
384         if (unlikely(policy->transition_ongoing)) {
385                 spin_unlock(&policy->transition_lock);
386                 goto wait;
387         }
388
389         policy->transition_ongoing = true;
390         policy->transition_task = current;
391
392         spin_unlock(&policy->transition_lock);
393
394         cpufreq_notify_transition(policy, freqs, CPUFREQ_PRECHANGE);
395 }
396 EXPORT_SYMBOL_GPL(cpufreq_freq_transition_begin);
397
398 void cpufreq_freq_transition_end(struct cpufreq_policy *policy,
399                 struct cpufreq_freqs *freqs, int transition_failed)
400 {
401         if (WARN_ON(!policy->transition_ongoing))
402                 return;
403
404         cpufreq_notify_post_transition(policy, freqs, transition_failed);
405
406         policy->transition_ongoing = false;
407         policy->transition_task = NULL;
408
409         wake_up(&policy->transition_wait);
410 }
411 EXPORT_SYMBOL_GPL(cpufreq_freq_transition_end);
412
413 /*
414  * Fast frequency switching status count.  Positive means "enabled", negative
415  * means "disabled" and 0 means "not decided yet".
416  */
417 static int cpufreq_fast_switch_count;
418 static DEFINE_MUTEX(cpufreq_fast_switch_lock);
419
420 static void cpufreq_list_transition_notifiers(void)
421 {
422         struct notifier_block *nb;
423
424         pr_info("Registered transition notifiers:\n");
425
426         mutex_lock(&cpufreq_transition_notifier_list.mutex);
427
428         for (nb = cpufreq_transition_notifier_list.head; nb; nb = nb->next)
429                 pr_info("%pF\n", nb->notifier_call);
430
431         mutex_unlock(&cpufreq_transition_notifier_list.mutex);
432 }
433
434 /**
435  * cpufreq_enable_fast_switch - Enable fast frequency switching for policy.
436  * @policy: cpufreq policy to enable fast frequency switching for.
437  *
438  * Try to enable fast frequency switching for @policy.
439  *
440  * The attempt will fail if there is at least one transition notifier registered
441  * at this point, as fast frequency switching is quite fundamentally at odds
442  * with transition notifiers.  Thus if successful, it will make registration of
443  * transition notifiers fail going forward.
444  */
445 void cpufreq_enable_fast_switch(struct cpufreq_policy *policy)
446 {
447         lockdep_assert_held(&policy->rwsem);
448
449         if (!policy->fast_switch_possible)
450                 return;
451
452         mutex_lock(&cpufreq_fast_switch_lock);
453         if (cpufreq_fast_switch_count >= 0) {
454                 cpufreq_fast_switch_count++;
455                 policy->fast_switch_enabled = true;
456         } else {
457                 pr_warn("CPU%u: Fast frequency switching not enabled\n",
458                         policy->cpu);
459                 cpufreq_list_transition_notifiers();
460         }
461         mutex_unlock(&cpufreq_fast_switch_lock);
462 }
463 EXPORT_SYMBOL_GPL(cpufreq_enable_fast_switch);
464
465 /**
466  * cpufreq_disable_fast_switch - Disable fast frequency switching for policy.
467  * @policy: cpufreq policy to disable fast frequency switching for.
468  */
469 void cpufreq_disable_fast_switch(struct cpufreq_policy *policy)
470 {
471         mutex_lock(&cpufreq_fast_switch_lock);
472         if (policy->fast_switch_enabled) {
473                 policy->fast_switch_enabled = false;
474                 if (!WARN_ON(cpufreq_fast_switch_count <= 0))
475                         cpufreq_fast_switch_count--;
476         }
477         mutex_unlock(&cpufreq_fast_switch_lock);
478 }
479 EXPORT_SYMBOL_GPL(cpufreq_disable_fast_switch);
480
481 /**
482  * cpufreq_driver_resolve_freq - Map a target frequency to a driver-supported
483  * one.
484  * @target_freq: target frequency to resolve.
485  *
486  * The target to driver frequency mapping is cached in the policy.
487  *
488  * Return: Lowest driver-supported frequency greater than or equal to the
489  * given target_freq, subject to policy (min/max) and driver limitations.
490  */
491 unsigned int cpufreq_driver_resolve_freq(struct cpufreq_policy *policy,
492                                          unsigned int target_freq)
493 {
494         target_freq = clamp_val(target_freq, policy->min, policy->max);
495         policy->cached_target_freq = target_freq;
496
497         if (cpufreq_driver->target_index) {
498                 int idx;
499
500                 idx = cpufreq_frequency_table_target(policy, target_freq,
501                                                      CPUFREQ_RELATION_L);
502                 policy->cached_resolved_idx = idx;
503                 return policy->freq_table[idx].frequency;
504         }
505
506         if (cpufreq_driver->resolve_freq)
507                 return cpufreq_driver->resolve_freq(policy, target_freq);
508
509         return target_freq;
510 }
511 EXPORT_SYMBOL_GPL(cpufreq_driver_resolve_freq);
512
513 unsigned int cpufreq_policy_transition_delay_us(struct cpufreq_policy *policy)
514 {
515         unsigned int latency;
516
517         if (policy->transition_delay_us)
518                 return policy->transition_delay_us;
519
520         latency = policy->cpuinfo.transition_latency / NSEC_PER_USEC;
521         if (latency) {
522                 /*
523                  * For platforms that can change the frequency very fast (< 10
524                  * us), the above formula gives a decent transition delay. But
525                  * for platforms where transition_latency is in milliseconds, it
526                  * ends up giving unrealistic values.
527                  *
528                  * Cap the default transition delay to 10 ms, which seems to be
529                  * a reasonable amount of time after which we should reevaluate
530                  * the frequency.
531                  */
532                 return min(latency * LATENCY_MULTIPLIER, (unsigned int)10000);
533         }
534
535         return LATENCY_MULTIPLIER;
536 }
537 EXPORT_SYMBOL_GPL(cpufreq_policy_transition_delay_us);
538
539 /*********************************************************************
540  *                          SYSFS INTERFACE                          *
541  *********************************************************************/
542 static ssize_t show_boost(struct kobject *kobj,
543                           struct kobj_attribute *attr, char *buf)
544 {
545         return sprintf(buf, "%d\n", cpufreq_driver->boost_enabled);
546 }
547
548 static ssize_t store_boost(struct kobject *kobj, struct kobj_attribute *attr,
549                            const char *buf, size_t count)
550 {
551         int ret, enable;
552
553         ret = sscanf(buf, "%d", &enable);
554         if (ret != 1 || enable < 0 || enable > 1)
555                 return -EINVAL;
556
557         if (cpufreq_boost_trigger_state(enable)) {
558                 pr_err("%s: Cannot %s BOOST!\n",
559                        __func__, enable ? "enable" : "disable");
560                 return -EINVAL;
561         }
562
563         pr_debug("%s: cpufreq BOOST %s\n",
564                  __func__, enable ? "enabled" : "disabled");
565
566         return count;
567 }
568 define_one_global_rw(boost);
569
570 static struct cpufreq_governor *find_governor(const char *str_governor)
571 {
572         struct cpufreq_governor *t;
573
574         for_each_governor(t)
575                 if (!strncasecmp(str_governor, t->name, CPUFREQ_NAME_LEN))
576                         return t;
577
578         return NULL;
579 }
580
581 /**
582  * cpufreq_parse_governor - parse a governor string
583  */
584 static int cpufreq_parse_governor(char *str_governor,
585                                   struct cpufreq_policy *policy)
586 {
587         if (cpufreq_driver->setpolicy) {
588                 if (!strncasecmp(str_governor, "performance", CPUFREQ_NAME_LEN)) {
589                         policy->policy = CPUFREQ_POLICY_PERFORMANCE;
590                         return 0;
591                 }
592
593                 if (!strncasecmp(str_governor, "powersave", CPUFREQ_NAME_LEN)) {
594                         policy->policy = CPUFREQ_POLICY_POWERSAVE;
595                         return 0;
596                 }
597         } else {
598                 struct cpufreq_governor *t;
599
600                 mutex_lock(&cpufreq_governor_mutex);
601
602                 t = find_governor(str_governor);
603                 if (!t) {
604                         int ret;
605
606                         mutex_unlock(&cpufreq_governor_mutex);
607
608                         ret = request_module("cpufreq_%s", str_governor);
609                         if (ret)
610                                 return -EINVAL;
611
612                         mutex_lock(&cpufreq_governor_mutex);
613
614                         t = find_governor(str_governor);
615                 }
616                 if (t && !try_module_get(t->owner))
617                         t = NULL;
618
619                 mutex_unlock(&cpufreq_governor_mutex);
620
621                 if (t) {
622                         policy->governor = t;
623                         return 0;
624                 }
625         }
626
627         return -EINVAL;
628 }
629
630 /**
631  * cpufreq_per_cpu_attr_read() / show_##file_name() -
632  * print out cpufreq information
633  *
634  * Write out information from cpufreq_driver->policy[cpu]; object must be
635  * "unsigned int".
636  */
637
638 #define show_one(file_name, object)                     \
639 static ssize_t show_##file_name                         \
640 (struct cpufreq_policy *policy, char *buf)              \
641 {                                                       \
642         return sprintf(buf, "%u\n", policy->object);    \
643 }
644
645 show_one(cpuinfo_min_freq, cpuinfo.min_freq);
646 show_one(cpuinfo_max_freq, cpuinfo.max_freq);
647 show_one(cpuinfo_transition_latency, cpuinfo.transition_latency);
648 show_one(scaling_min_freq, min);
649 show_one(scaling_max_freq, max);
650
651 __weak unsigned int arch_freq_get_on_cpu(int cpu)
652 {
653         return 0;
654 }
655
656 static ssize_t show_scaling_cur_freq(struct cpufreq_policy *policy, char *buf)
657 {
658         ssize_t ret;
659         unsigned int freq;
660
661         freq = arch_freq_get_on_cpu(policy->cpu);
662         if (freq)
663                 ret = sprintf(buf, "%u\n", freq);
664         else if (cpufreq_driver && cpufreq_driver->setpolicy &&
665                         cpufreq_driver->get)
666                 ret = sprintf(buf, "%u\n", cpufreq_driver->get(policy->cpu));
667         else
668                 ret = sprintf(buf, "%u\n", policy->cur);
669         return ret;
670 }
671
672 static int cpufreq_set_policy(struct cpufreq_policy *policy,
673                                 struct cpufreq_policy *new_policy);
674
675 /**
676  * cpufreq_per_cpu_attr_write() / store_##file_name() - sysfs write access
677  */
678 #define store_one(file_name, object)                    \
679 static ssize_t store_##file_name                                        \
680 (struct cpufreq_policy *policy, const char *buf, size_t count)          \
681 {                                                                       \
682         int ret, temp;                                                  \
683         struct cpufreq_policy new_policy;                               \
684                                                                         \
685         memcpy(&new_policy, policy, sizeof(*policy));                   \
686         new_policy.min = policy->user_policy.min;                       \
687         new_policy.max = policy->user_policy.max;                       \
688                                                                         \
689         ret = sscanf(buf, "%u", &new_policy.object);                    \
690         if (ret != 1)                                                   \
691                 return -EINVAL;                                         \
692                                                                         \
693         temp = new_policy.object;                                       \
694         ret = cpufreq_set_policy(policy, &new_policy);          \
695         if (!ret)                                                       \
696                 policy->user_policy.object = temp;                      \
697                                                                         \
698         return ret ? ret : count;                                       \
699 }
700
701 store_one(scaling_min_freq, min);
702 store_one(scaling_max_freq, max);
703
704 /**
705  * show_cpuinfo_cur_freq - current CPU frequency as detected by hardware
706  */
707 static ssize_t show_cpuinfo_cur_freq(struct cpufreq_policy *policy,
708                                         char *buf)
709 {
710         unsigned int cur_freq = __cpufreq_get(policy);
711
712         if (cur_freq)
713                 return sprintf(buf, "%u\n", cur_freq);
714
715         return sprintf(buf, "<unknown>\n");
716 }
717
718 /**
719  * show_scaling_governor - show the current policy for the specified CPU
720  */
721 static ssize_t show_scaling_governor(struct cpufreq_policy *policy, char *buf)
722 {
723         if (policy->policy == CPUFREQ_POLICY_POWERSAVE)
724                 return sprintf(buf, "powersave\n");
725         else if (policy->policy == CPUFREQ_POLICY_PERFORMANCE)
726                 return sprintf(buf, "performance\n");
727         else if (policy->governor)
728                 return scnprintf(buf, CPUFREQ_NAME_PLEN, "%s\n",
729                                 policy->governor->name);
730         return -EINVAL;
731 }
732
733 /**
734  * store_scaling_governor - store policy for the specified CPU
735  */
736 static ssize_t store_scaling_governor(struct cpufreq_policy *policy,
737                                         const char *buf, size_t count)
738 {
739         int ret;
740         char    str_governor[16];
741         struct cpufreq_policy new_policy;
742
743         memcpy(&new_policy, policy, sizeof(*policy));
744
745         ret = sscanf(buf, "%15s", str_governor);
746         if (ret != 1)
747                 return -EINVAL;
748
749         if (cpufreq_parse_governor(str_governor, &new_policy))
750                 return -EINVAL;
751
752         ret = cpufreq_set_policy(policy, &new_policy);
753
754         if (new_policy.governor)
755                 module_put(new_policy.governor->owner);
756
757         return ret ? ret : count;
758 }
759
760 /**
761  * show_scaling_driver - show the cpufreq driver currently loaded
762  */
763 static ssize_t show_scaling_driver(struct cpufreq_policy *policy, char *buf)
764 {
765         return scnprintf(buf, CPUFREQ_NAME_PLEN, "%s\n", cpufreq_driver->name);
766 }
767
768 /**
769  * show_scaling_available_governors - show the available CPUfreq governors
770  */
771 static ssize_t show_scaling_available_governors(struct cpufreq_policy *policy,
772                                                 char *buf)
773 {
774         ssize_t i = 0;
775         struct cpufreq_governor *t;
776
777         if (!has_target()) {
778                 i += sprintf(buf, "performance powersave");
779                 goto out;
780         }
781
782         for_each_governor(t) {
783                 if (i >= (ssize_t) ((PAGE_SIZE / sizeof(char))
784                     - (CPUFREQ_NAME_LEN + 2)))
785                         goto out;
786                 i += scnprintf(&buf[i], CPUFREQ_NAME_PLEN, "%s ", t->name);
787         }
788 out:
789         i += sprintf(&buf[i], "\n");
790         return i;
791 }
792
793 ssize_t cpufreq_show_cpus(const struct cpumask *mask, char *buf)
794 {
795         ssize_t i = 0;
796         unsigned int cpu;
797
798         for_each_cpu(cpu, mask) {
799                 if (i)
800                         i += scnprintf(&buf[i], (PAGE_SIZE - i - 2), " ");
801                 i += scnprintf(&buf[i], (PAGE_SIZE - i - 2), "%u", cpu);
802                 if (i >= (PAGE_SIZE - 5))
803                         break;
804         }
805         i += sprintf(&buf[i], "\n");
806         return i;
807 }
808 EXPORT_SYMBOL_GPL(cpufreq_show_cpus);
809
810 /**
811  * show_related_cpus - show the CPUs affected by each transition even if
812  * hw coordination is in use
813  */
814 static ssize_t show_related_cpus(struct cpufreq_policy *policy, char *buf)
815 {
816         return cpufreq_show_cpus(policy->related_cpus, buf);
817 }
818
819 /**
820  * show_affected_cpus - show the CPUs affected by each transition
821  */
822 static ssize_t show_affected_cpus(struct cpufreq_policy *policy, char *buf)
823 {
824         return cpufreq_show_cpus(policy->cpus, buf);
825 }
826
827 static ssize_t store_scaling_setspeed(struct cpufreq_policy *policy,
828                                         const char *buf, size_t count)
829 {
830         unsigned int freq = 0;
831         unsigned int ret;
832
833         if (!policy->governor || !policy->governor->store_setspeed)
834                 return -EINVAL;
835
836         ret = sscanf(buf, "%u", &freq);
837         if (ret != 1)
838                 return -EINVAL;
839
840         policy->governor->store_setspeed(policy, freq);
841
842         return count;
843 }
844
845 static ssize_t show_scaling_setspeed(struct cpufreq_policy *policy, char *buf)
846 {
847         if (!policy->governor || !policy->governor->show_setspeed)
848                 return sprintf(buf, "<unsupported>\n");
849
850         return policy->governor->show_setspeed(policy, buf);
851 }
852
853 /**
854  * show_bios_limit - show the current cpufreq HW/BIOS limitation
855  */
856 static ssize_t show_bios_limit(struct cpufreq_policy *policy, char *buf)
857 {
858         unsigned int limit;
859         int ret;
860         if (cpufreq_driver->bios_limit) {
861                 ret = cpufreq_driver->bios_limit(policy->cpu, &limit);
862                 if (!ret)
863                         return sprintf(buf, "%u\n", limit);
864         }
865         return sprintf(buf, "%u\n", policy->cpuinfo.max_freq);
866 }
867
868 cpufreq_freq_attr_ro_perm(cpuinfo_cur_freq, 0400);
869 cpufreq_freq_attr_ro(cpuinfo_min_freq);
870 cpufreq_freq_attr_ro(cpuinfo_max_freq);
871 cpufreq_freq_attr_ro(cpuinfo_transition_latency);
872 cpufreq_freq_attr_ro(scaling_available_governors);
873 cpufreq_freq_attr_ro(scaling_driver);
874 cpufreq_freq_attr_ro(scaling_cur_freq);
875 cpufreq_freq_attr_ro(bios_limit);
876 cpufreq_freq_attr_ro(related_cpus);
877 cpufreq_freq_attr_ro(affected_cpus);
878 cpufreq_freq_attr_rw(scaling_min_freq);
879 cpufreq_freq_attr_rw(scaling_max_freq);
880 cpufreq_freq_attr_rw(scaling_governor);
881 cpufreq_freq_attr_rw(scaling_setspeed);
882
883 static struct attribute *default_attrs[] = {
884         &cpuinfo_min_freq.attr,
885         &cpuinfo_max_freq.attr,
886         &cpuinfo_transition_latency.attr,
887         &scaling_min_freq.attr,
888         &scaling_max_freq.attr,
889         &affected_cpus.attr,
890         &related_cpus.attr,
891         &scaling_governor.attr,
892         &scaling_driver.attr,
893         &scaling_available_governors.attr,
894         &scaling_setspeed.attr,
895         NULL
896 };
897
898 #define to_policy(k) container_of(k, struct cpufreq_policy, kobj)
899 #define to_attr(a) container_of(a, struct freq_attr, attr)
900
901 static ssize_t show(struct kobject *kobj, struct attribute *attr, char *buf)
902 {
903         struct cpufreq_policy *policy = to_policy(kobj);
904         struct freq_attr *fattr = to_attr(attr);
905         ssize_t ret;
906
907         down_read(&policy->rwsem);
908         ret = fattr->show(policy, buf);
909         up_read(&policy->rwsem);
910
911         return ret;
912 }
913
914 static ssize_t store(struct kobject *kobj, struct attribute *attr,
915                      const char *buf, size_t count)
916 {
917         struct cpufreq_policy *policy = to_policy(kobj);
918         struct freq_attr *fattr = to_attr(attr);
919         ssize_t ret = -EINVAL;
920
921         /*
922          * cpus_read_trylock() is used here to work around a circular lock
923          * dependency problem with respect to the cpufreq_register_driver().
924          */
925         if (!cpus_read_trylock())
926                 return -EBUSY;
927
928         if (cpu_online(policy->cpu)) {
929                 down_write(&policy->rwsem);
930                 ret = fattr->store(policy, buf, count);
931                 up_write(&policy->rwsem);
932         }
933
934         cpus_read_unlock();
935
936         return ret;
937 }
938
939 static void cpufreq_sysfs_release(struct kobject *kobj)
940 {
941         struct cpufreq_policy *policy = to_policy(kobj);
942         pr_debug("last reference is dropped\n");
943         complete(&policy->kobj_unregister);
944 }
945
946 static const struct sysfs_ops sysfs_ops = {
947         .show   = show,
948         .store  = store,
949 };
950
951 static struct kobj_type ktype_cpufreq = {
952         .sysfs_ops      = &sysfs_ops,
953         .default_attrs  = default_attrs,
954         .release        = cpufreq_sysfs_release,
955 };
956
957 static void add_cpu_dev_symlink(struct cpufreq_policy *policy, unsigned int cpu)
958 {
959         struct device *dev = get_cpu_device(cpu);
960
961         if (!dev)
962                 return;
963
964         if (cpumask_test_and_set_cpu(cpu, policy->real_cpus))
965                 return;
966
967         dev_dbg(dev, "%s: Adding symlink\n", __func__);
968         if (sysfs_create_link(&dev->kobj, &policy->kobj, "cpufreq"))
969                 dev_err(dev, "cpufreq symlink creation failed\n");
970 }
971
972 static void remove_cpu_dev_symlink(struct cpufreq_policy *policy,
973                                    struct device *dev)
974 {
975         dev_dbg(dev, "%s: Removing symlink\n", __func__);
976         sysfs_remove_link(&dev->kobj, "cpufreq");
977 }
978
979 static int cpufreq_add_dev_interface(struct cpufreq_policy *policy)
980 {
981         struct freq_attr **drv_attr;
982         int ret = 0;
983
984         /* set up files for this cpu device */
985         drv_attr = cpufreq_driver->attr;
986         while (drv_attr && *drv_attr) {
987                 ret = sysfs_create_file(&policy->kobj, &((*drv_attr)->attr));
988                 if (ret)
989                         return ret;
990                 drv_attr++;
991         }
992         if (cpufreq_driver->get) {
993                 ret = sysfs_create_file(&policy->kobj, &cpuinfo_cur_freq.attr);
994                 if (ret)
995                         return ret;
996         }
997
998         ret = sysfs_create_file(&policy->kobj, &scaling_cur_freq.attr);
999         if (ret)
1000                 return ret;
1001
1002         if (cpufreq_driver->bios_limit) {
1003                 ret = sysfs_create_file(&policy->kobj, &bios_limit.attr);
1004                 if (ret)
1005                         return ret;
1006         }
1007
1008         return 0;
1009 }
1010
1011 __weak struct cpufreq_governor *cpufreq_default_governor(void)
1012 {
1013         return NULL;
1014 }
1015
1016 static int cpufreq_init_policy(struct cpufreq_policy *policy)
1017 {
1018         struct cpufreq_governor *gov = NULL;
1019         struct cpufreq_policy new_policy;
1020
1021         memcpy(&new_policy, policy, sizeof(*policy));
1022
1023         /* Update governor of new_policy to the governor used before hotplug */
1024         gov = find_governor(policy->last_governor);
1025         if (gov) {
1026                 pr_debug("Restoring governor %s for cpu %d\n",
1027                                 policy->governor->name, policy->cpu);
1028         } else {
1029                 gov = cpufreq_default_governor();
1030                 if (!gov)
1031                         return -ENODATA;
1032         }
1033
1034         new_policy.governor = gov;
1035
1036         /* Use the default policy if there is no last_policy. */
1037         if (cpufreq_driver->setpolicy) {
1038                 if (policy->last_policy)
1039                         new_policy.policy = policy->last_policy;
1040                 else
1041                         cpufreq_parse_governor(gov->name, &new_policy);
1042         }
1043         /* set default policy */
1044         return cpufreq_set_policy(policy, &new_policy);
1045 }
1046
1047 static int cpufreq_add_policy_cpu(struct cpufreq_policy *policy, unsigned int cpu)
1048 {
1049         int ret = 0;
1050
1051         /* Has this CPU been taken care of already? */
1052         if (cpumask_test_cpu(cpu, policy->cpus))
1053                 return 0;
1054
1055         down_write(&policy->rwsem);
1056         if (has_target())
1057                 cpufreq_stop_governor(policy);
1058
1059         cpumask_set_cpu(cpu, policy->cpus);
1060
1061         if (has_target()) {
1062                 ret = cpufreq_start_governor(policy);
1063                 if (ret)
1064                         pr_err("%s: Failed to start governor\n", __func__);
1065         }
1066         up_write(&policy->rwsem);
1067         return ret;
1068 }
1069
1070 static void handle_update(struct work_struct *work)
1071 {
1072         struct cpufreq_policy *policy =
1073                 container_of(work, struct cpufreq_policy, update);
1074         unsigned int cpu = policy->cpu;
1075         pr_debug("handle_update for cpu %u called\n", cpu);
1076         cpufreq_update_policy(cpu);
1077 }
1078
1079 static struct cpufreq_policy *cpufreq_policy_alloc(unsigned int cpu)
1080 {
1081         struct cpufreq_policy *policy;
1082         int ret;
1083
1084         policy = kzalloc(sizeof(*policy), GFP_KERNEL);
1085         if (!policy)
1086                 return NULL;
1087
1088         if (!alloc_cpumask_var(&policy->cpus, GFP_KERNEL))
1089                 goto err_free_policy;
1090
1091         if (!zalloc_cpumask_var(&policy->related_cpus, GFP_KERNEL))
1092                 goto err_free_cpumask;
1093
1094         if (!zalloc_cpumask_var(&policy->real_cpus, GFP_KERNEL))
1095                 goto err_free_rcpumask;
1096
1097         ret = kobject_init_and_add(&policy->kobj, &ktype_cpufreq,
1098                                    cpufreq_global_kobject, "policy%u", cpu);
1099         if (ret) {
1100                 pr_err("%s: failed to init policy->kobj: %d\n", __func__, ret);
1101                 goto err_free_real_cpus;
1102         }
1103
1104         INIT_LIST_HEAD(&policy->policy_list);
1105         init_rwsem(&policy->rwsem);
1106         spin_lock_init(&policy->transition_lock);
1107         init_waitqueue_head(&policy->transition_wait);
1108         init_completion(&policy->kobj_unregister);
1109         INIT_WORK(&policy->update, handle_update);
1110
1111         policy->cpu = cpu;
1112         return policy;
1113
1114 err_free_real_cpus:
1115         free_cpumask_var(policy->real_cpus);
1116 err_free_rcpumask:
1117         free_cpumask_var(policy->related_cpus);
1118 err_free_cpumask:
1119         free_cpumask_var(policy->cpus);
1120 err_free_policy:
1121         kfree(policy);
1122
1123         return NULL;
1124 }
1125
1126 static void cpufreq_policy_put_kobj(struct cpufreq_policy *policy)
1127 {
1128         struct kobject *kobj;
1129         struct completion *cmp;
1130
1131         down_write(&policy->rwsem);
1132         cpufreq_stats_free_table(policy);
1133         kobj = &policy->kobj;
1134         cmp = &policy->kobj_unregister;
1135         up_write(&policy->rwsem);
1136         kobject_put(kobj);
1137
1138         /*
1139          * We need to make sure that the underlying kobj is
1140          * actually not referenced anymore by anybody before we
1141          * proceed with unloading.
1142          */
1143         pr_debug("waiting for dropping of refcount\n");
1144         wait_for_completion(cmp);
1145         pr_debug("wait complete\n");
1146 }
1147
1148 static void cpufreq_policy_free(struct cpufreq_policy *policy)
1149 {
1150         unsigned long flags;
1151         int cpu;
1152
1153         /* Remove policy from list */
1154         write_lock_irqsave(&cpufreq_driver_lock, flags);
1155         list_del(&policy->policy_list);
1156
1157         for_each_cpu(cpu, policy->related_cpus)
1158                 per_cpu(cpufreq_cpu_data, cpu) = NULL;
1159         write_unlock_irqrestore(&cpufreq_driver_lock, flags);
1160
1161         cpufreq_policy_put_kobj(policy);
1162         free_cpumask_var(policy->real_cpus);
1163         free_cpumask_var(policy->related_cpus);
1164         free_cpumask_var(policy->cpus);
1165         kfree(policy);
1166 }
1167
1168 static int cpufreq_online(unsigned int cpu)
1169 {
1170         struct cpufreq_policy *policy;
1171         bool new_policy;
1172         unsigned long flags;
1173         unsigned int j;
1174         int ret;
1175
1176         pr_debug("%s: bringing CPU%u online\n", __func__, cpu);
1177
1178         /* Check if this CPU already has a policy to manage it */
1179         policy = per_cpu(cpufreq_cpu_data, cpu);
1180         if (policy) {
1181                 WARN_ON(!cpumask_test_cpu(cpu, policy->related_cpus));
1182                 if (!policy_is_inactive(policy))
1183                         return cpufreq_add_policy_cpu(policy, cpu);
1184
1185                 /* This is the only online CPU for the policy.  Start over. */
1186                 new_policy = false;
1187                 down_write(&policy->rwsem);
1188                 policy->cpu = cpu;
1189                 policy->governor = NULL;
1190                 up_write(&policy->rwsem);
1191         } else {
1192                 new_policy = true;
1193                 policy = cpufreq_policy_alloc(cpu);
1194                 if (!policy)
1195                         return -ENOMEM;
1196         }
1197
1198         if (!new_policy && cpufreq_driver->online) {
1199                 ret = cpufreq_driver->online(policy);
1200                 if (ret) {
1201                         pr_debug("%s: %d: initialization failed\n", __func__,
1202                                  __LINE__);
1203                         goto out_exit_policy;
1204                 }
1205
1206                 /* Recover policy->cpus using related_cpus */
1207                 cpumask_copy(policy->cpus, policy->related_cpus);
1208         } else {
1209                 cpumask_copy(policy->cpus, cpumask_of(cpu));
1210
1211                 /*
1212                  * Call driver. From then on the cpufreq must be able
1213                  * to accept all calls to ->verify and ->setpolicy for this CPU.
1214                  */
1215                 ret = cpufreq_driver->init(policy);
1216                 if (ret) {
1217                         pr_debug("%s: %d: initialization failed\n", __func__,
1218                                  __LINE__);
1219                         goto out_free_policy;
1220                 }
1221
1222                 ret = cpufreq_table_validate_and_sort(policy);
1223                 if (ret)
1224                         goto out_exit_policy;
1225
1226                 /* related_cpus should at least include policy->cpus. */
1227                 cpumask_copy(policy->related_cpus, policy->cpus);
1228         }
1229
1230         down_write(&policy->rwsem);
1231         /*
1232          * affected cpus must always be the one, which are online. We aren't
1233          * managing offline cpus here.
1234          */
1235         cpumask_and(policy->cpus, policy->cpus, cpu_online_mask);
1236
1237         if (new_policy) {
1238                 policy->user_policy.min = policy->min;
1239                 policy->user_policy.max = policy->max;
1240
1241                 for_each_cpu(j, policy->related_cpus) {
1242                         per_cpu(cpufreq_cpu_data, j) = policy;
1243                         add_cpu_dev_symlink(policy, j);
1244                 }
1245         } else {
1246                 policy->min = policy->user_policy.min;
1247                 policy->max = policy->user_policy.max;
1248         }
1249
1250         if (cpufreq_driver->get && !cpufreq_driver->setpolicy) {
1251                 policy->cur = cpufreq_driver->get(policy->cpu);
1252                 if (!policy->cur) {
1253                         pr_err("%s: ->get() failed\n", __func__);
1254                         goto out_destroy_policy;
1255                 }
1256         }
1257
1258         /*
1259          * Sometimes boot loaders set CPU frequency to a value outside of
1260          * frequency table present with cpufreq core. In such cases CPU might be
1261          * unstable if it has to run on that frequency for long duration of time
1262          * and so its better to set it to a frequency which is specified in
1263          * freq-table. This also makes cpufreq stats inconsistent as
1264          * cpufreq-stats would fail to register because current frequency of CPU
1265          * isn't found in freq-table.
1266          *
1267          * Because we don't want this change to effect boot process badly, we go
1268          * for the next freq which is >= policy->cur ('cur' must be set by now,
1269          * otherwise we will end up setting freq to lowest of the table as 'cur'
1270          * is initialized to zero).
1271          *
1272          * We are passing target-freq as "policy->cur - 1" otherwise
1273          * __cpufreq_driver_target() would simply fail, as policy->cur will be
1274          * equal to target-freq.
1275          */
1276         if ((cpufreq_driver->flags & CPUFREQ_NEED_INITIAL_FREQ_CHECK)
1277             && has_target()) {
1278                 /* Are we running at unknown frequency ? */
1279                 ret = cpufreq_frequency_table_get_index(policy, policy->cur);
1280                 if (ret == -EINVAL) {
1281                         /* Warn user and fix it */
1282                         pr_warn("%s: CPU%d: Running at unlisted freq: %u KHz\n",
1283                                 __func__, policy->cpu, policy->cur);
1284                         ret = __cpufreq_driver_target(policy, policy->cur - 1,
1285                                 CPUFREQ_RELATION_L);
1286
1287                         /*
1288                          * Reaching here after boot in a few seconds may not
1289                          * mean that system will remain stable at "unknown"
1290                          * frequency for longer duration. Hence, a BUG_ON().
1291                          */
1292                         BUG_ON(ret);
1293                         pr_warn("%s: CPU%d: Unlisted initial frequency changed to: %u KHz\n",
1294                                 __func__, policy->cpu, policy->cur);
1295                 }
1296         }
1297
1298         if (new_policy) {
1299                 ret = cpufreq_add_dev_interface(policy);
1300                 if (ret)
1301                         goto out_destroy_policy;
1302
1303                 cpufreq_stats_create_table(policy);
1304
1305                 write_lock_irqsave(&cpufreq_driver_lock, flags);
1306                 list_add(&policy->policy_list, &cpufreq_policy_list);
1307                 write_unlock_irqrestore(&cpufreq_driver_lock, flags);
1308         }
1309
1310         ret = cpufreq_init_policy(policy);
1311         if (ret) {
1312                 pr_err("%s: Failed to initialize policy for cpu: %d (%d)\n",
1313                        __func__, cpu, ret);
1314                 goto out_destroy_policy;
1315         }
1316
1317         up_write(&policy->rwsem);
1318
1319         kobject_uevent(&policy->kobj, KOBJ_ADD);
1320
1321         /* Callback for handling stuff after policy is ready */
1322         if (cpufreq_driver->ready)
1323                 cpufreq_driver->ready(policy);
1324
1325         if (IS_ENABLED(CONFIG_CPU_THERMAL) &&
1326             cpufreq_driver->flags & CPUFREQ_IS_COOLING_DEV)
1327                 policy->cdev = of_cpufreq_cooling_register(policy);
1328
1329         pr_debug("initialization complete\n");
1330
1331         return 0;
1332
1333 out_destroy_policy:
1334         for_each_cpu(j, policy->real_cpus)
1335                 remove_cpu_dev_symlink(policy, get_cpu_device(j));
1336
1337         up_write(&policy->rwsem);
1338
1339 out_exit_policy:
1340         if (cpufreq_driver->exit)
1341                 cpufreq_driver->exit(policy);
1342
1343 out_free_policy:
1344         cpufreq_policy_free(policy);
1345         return ret;
1346 }
1347
1348 /**
1349  * cpufreq_add_dev - the cpufreq interface for a CPU device.
1350  * @dev: CPU device.
1351  * @sif: Subsystem interface structure pointer (not used)
1352  */
1353 static int cpufreq_add_dev(struct device *dev, struct subsys_interface *sif)
1354 {
1355         struct cpufreq_policy *policy;
1356         unsigned cpu = dev->id;
1357         int ret;
1358
1359         dev_dbg(dev, "%s: adding CPU%u\n", __func__, cpu);
1360
1361         if (cpu_online(cpu)) {
1362                 ret = cpufreq_online(cpu);
1363                 if (ret)
1364                         return ret;
1365         }
1366
1367         /* Create sysfs link on CPU registration */
1368         policy = per_cpu(cpufreq_cpu_data, cpu);
1369         if (policy)
1370                 add_cpu_dev_symlink(policy, cpu);
1371
1372         return 0;
1373 }
1374
1375 static int cpufreq_offline(unsigned int cpu)
1376 {
1377         struct cpufreq_policy *policy;
1378         int ret;
1379
1380         pr_debug("%s: unregistering CPU %u\n", __func__, cpu);
1381
1382         policy = cpufreq_cpu_get_raw(cpu);
1383         if (!policy) {
1384                 pr_debug("%s: No cpu_data found\n", __func__);
1385                 return 0;
1386         }
1387
1388         down_write(&policy->rwsem);
1389         if (has_target())
1390                 cpufreq_stop_governor(policy);
1391
1392         cpumask_clear_cpu(cpu, policy->cpus);
1393
1394         if (policy_is_inactive(policy)) {
1395                 if (has_target())
1396                         strncpy(policy->last_governor, policy->governor->name,
1397                                 CPUFREQ_NAME_LEN);
1398                 else
1399                         policy->last_policy = policy->policy;
1400         } else if (cpu == policy->cpu) {
1401                 /* Nominate new CPU */
1402                 policy->cpu = cpumask_any(policy->cpus);
1403         }
1404
1405         /* Start governor again for active policy */
1406         if (!policy_is_inactive(policy)) {
1407                 if (has_target()) {
1408                         ret = cpufreq_start_governor(policy);
1409                         if (ret)
1410                                 pr_err("%s: Failed to start governor\n", __func__);
1411                 }
1412
1413                 goto unlock;
1414         }
1415
1416         if (IS_ENABLED(CONFIG_CPU_THERMAL) &&
1417             cpufreq_driver->flags & CPUFREQ_IS_COOLING_DEV) {
1418                 cpufreq_cooling_unregister(policy->cdev);
1419                 policy->cdev = NULL;
1420         }
1421
1422         if (cpufreq_driver->stop_cpu)
1423                 cpufreq_driver->stop_cpu(policy);
1424
1425         if (has_target())
1426                 cpufreq_exit_governor(policy);
1427
1428         /*
1429          * Perform the ->offline() during light-weight tear-down, as
1430          * that allows fast recovery when the CPU comes back.
1431          */
1432         if (cpufreq_driver->offline) {
1433                 cpufreq_driver->offline(policy);
1434         } else if (cpufreq_driver->exit) {
1435                 cpufreq_driver->exit(policy);
1436                 policy->freq_table = NULL;
1437         }
1438
1439 unlock:
1440         up_write(&policy->rwsem);
1441         return 0;
1442 }
1443
1444 /**
1445  * cpufreq_remove_dev - remove a CPU device
1446  *
1447  * Removes the cpufreq interface for a CPU device.
1448  */
1449 static void cpufreq_remove_dev(struct device *dev, struct subsys_interface *sif)
1450 {
1451         unsigned int cpu = dev->id;
1452         struct cpufreq_policy *policy = per_cpu(cpufreq_cpu_data, cpu);
1453
1454         if (!policy)
1455                 return;
1456
1457         if (cpu_online(cpu))
1458                 cpufreq_offline(cpu);
1459
1460         cpumask_clear_cpu(cpu, policy->real_cpus);
1461         remove_cpu_dev_symlink(policy, dev);
1462
1463         if (cpumask_empty(policy->real_cpus)) {
1464                 /* We did light-weight exit earlier, do full tear down now */
1465                 if (cpufreq_driver->offline)
1466                         cpufreq_driver->exit(policy);
1467
1468                 cpufreq_policy_free(policy);
1469         }
1470 }
1471
1472 /**
1473  *      cpufreq_out_of_sync - If actual and saved CPU frequency differs, we're
1474  *      in deep trouble.
1475  *      @policy: policy managing CPUs
1476  *      @new_freq: CPU frequency the CPU actually runs at
1477  *
1478  *      We adjust to current frequency first, and need to clean up later.
1479  *      So either call to cpufreq_update_policy() or schedule handle_update()).
1480  */
1481 static void cpufreq_out_of_sync(struct cpufreq_policy *policy,
1482                                 unsigned int new_freq)
1483 {
1484         struct cpufreq_freqs freqs;
1485
1486         pr_debug("Warning: CPU frequency out of sync: cpufreq and timing core thinks of %u, is %u kHz\n",
1487                  policy->cur, new_freq);
1488
1489         freqs.old = policy->cur;
1490         freqs.new = new_freq;
1491
1492         cpufreq_freq_transition_begin(policy, &freqs);
1493         cpufreq_freq_transition_end(policy, &freqs, 0);
1494 }
1495
1496 /**
1497  * cpufreq_quick_get - get the CPU frequency (in kHz) from policy->cur
1498  * @cpu: CPU number
1499  *
1500  * This is the last known freq, without actually getting it from the driver.
1501  * Return value will be same as what is shown in scaling_cur_freq in sysfs.
1502  */
1503 unsigned int cpufreq_quick_get(unsigned int cpu)
1504 {
1505         struct cpufreq_policy *policy;
1506         unsigned int ret_freq = 0;
1507         unsigned long flags;
1508
1509         read_lock_irqsave(&cpufreq_driver_lock, flags);
1510
1511         if (cpufreq_driver && cpufreq_driver->setpolicy && cpufreq_driver->get) {
1512                 ret_freq = cpufreq_driver->get(cpu);
1513                 read_unlock_irqrestore(&cpufreq_driver_lock, flags);
1514                 return ret_freq;
1515         }
1516
1517         read_unlock_irqrestore(&cpufreq_driver_lock, flags);
1518
1519         policy = cpufreq_cpu_get(cpu);
1520         if (policy) {
1521                 ret_freq = policy->cur;
1522                 cpufreq_cpu_put(policy);
1523         }
1524
1525         return ret_freq;
1526 }
1527 EXPORT_SYMBOL(cpufreq_quick_get);
1528
1529 /**
1530  * cpufreq_quick_get_max - get the max reported CPU frequency for this CPU
1531  * @cpu: CPU number
1532  *
1533  * Just return the max possible frequency for a given CPU.
1534  */
1535 unsigned int cpufreq_quick_get_max(unsigned int cpu)
1536 {
1537         struct cpufreq_policy *policy = cpufreq_cpu_get(cpu);
1538         unsigned int ret_freq = 0;
1539
1540         if (policy) {
1541                 ret_freq = policy->max;
1542                 cpufreq_cpu_put(policy);
1543         }
1544
1545         return ret_freq;
1546 }
1547 EXPORT_SYMBOL(cpufreq_quick_get_max);
1548
1549 static unsigned int __cpufreq_get(struct cpufreq_policy *policy)
1550 {
1551         unsigned int ret_freq = 0;
1552
1553         if (unlikely(policy_is_inactive(policy)) || !cpufreq_driver->get)
1554                 return ret_freq;
1555
1556         ret_freq = cpufreq_driver->get(policy->cpu);
1557
1558         /*
1559          * If fast frequency switching is used with the given policy, the check
1560          * against policy->cur is pointless, so skip it in that case too.
1561          */
1562         if (policy->fast_switch_enabled)
1563                 return ret_freq;
1564
1565         if (ret_freq && policy->cur &&
1566                 !(cpufreq_driver->flags & CPUFREQ_CONST_LOOPS)) {
1567                 /* verify no discrepancy between actual and
1568                                         saved value exists */
1569                 if (unlikely(ret_freq != policy->cur)) {
1570                         cpufreq_out_of_sync(policy, ret_freq);
1571                         schedule_work(&policy->update);
1572                 }
1573         }
1574
1575         return ret_freq;
1576 }
1577
1578 /**
1579  * cpufreq_get - get the current CPU frequency (in kHz)
1580  * @cpu: CPU number
1581  *
1582  * Get the CPU current (static) CPU frequency
1583  */
1584 unsigned int cpufreq_get(unsigned int cpu)
1585 {
1586         struct cpufreq_policy *policy = cpufreq_cpu_get(cpu);
1587         unsigned int ret_freq = 0;
1588
1589         if (policy) {
1590                 down_read(&policy->rwsem);
1591                 ret_freq = __cpufreq_get(policy);
1592                 up_read(&policy->rwsem);
1593
1594                 cpufreq_cpu_put(policy);
1595         }
1596
1597         return ret_freq;
1598 }
1599 EXPORT_SYMBOL(cpufreq_get);
1600
1601 static unsigned int cpufreq_update_current_freq(struct cpufreq_policy *policy)
1602 {
1603         unsigned int new_freq;
1604
1605         new_freq = cpufreq_driver->get(policy->cpu);
1606         if (!new_freq)
1607                 return 0;
1608
1609         if (!policy->cur) {
1610                 pr_debug("cpufreq: Driver did not initialize current freq\n");
1611                 policy->cur = new_freq;
1612         } else if (policy->cur != new_freq && has_target()) {
1613                 cpufreq_out_of_sync(policy, new_freq);
1614         }
1615
1616         return new_freq;
1617 }
1618
1619 static struct subsys_interface cpufreq_interface = {
1620         .name           = "cpufreq",
1621         .subsys         = &cpu_subsys,
1622         .add_dev        = cpufreq_add_dev,
1623         .remove_dev     = cpufreq_remove_dev,
1624 };
1625
1626 /*
1627  * In case platform wants some specific frequency to be configured
1628  * during suspend..
1629  */
1630 int cpufreq_generic_suspend(struct cpufreq_policy *policy)
1631 {
1632         int ret;
1633
1634         if (!policy->suspend_freq) {
1635                 pr_debug("%s: suspend_freq not defined\n", __func__);
1636                 return 0;
1637         }
1638
1639         pr_debug("%s: Setting suspend-freq: %u\n", __func__,
1640                         policy->suspend_freq);
1641
1642         ret = __cpufreq_driver_target(policy, policy->suspend_freq,
1643                         CPUFREQ_RELATION_H);
1644         if (ret)
1645                 pr_err("%s: unable to set suspend-freq: %u. err: %d\n",
1646                                 __func__, policy->suspend_freq, ret);
1647
1648         return ret;
1649 }
1650 EXPORT_SYMBOL(cpufreq_generic_suspend);
1651
1652 /**
1653  * cpufreq_suspend() - Suspend CPUFreq governors
1654  *
1655  * Called during system wide Suspend/Hibernate cycles for suspending governors
1656  * as some platforms can't change frequency after this point in suspend cycle.
1657  * Because some of the devices (like: i2c, regulators, etc) they use for
1658  * changing frequency are suspended quickly after this point.
1659  */
1660 void cpufreq_suspend(void)
1661 {
1662         struct cpufreq_policy *policy;
1663
1664         if (!cpufreq_driver)
1665                 return;
1666
1667         if (!has_target() && !cpufreq_driver->suspend)
1668                 goto suspend;
1669
1670         pr_debug("%s: Suspending Governors\n", __func__);
1671
1672         for_each_active_policy(policy) {
1673                 if (has_target()) {
1674                         down_write(&policy->rwsem);
1675                         cpufreq_stop_governor(policy);
1676                         up_write(&policy->rwsem);
1677                 }
1678
1679                 if (cpufreq_driver->suspend && cpufreq_driver->suspend(policy))
1680                         pr_err("%s: Failed to suspend driver: %p\n", __func__,
1681                                 policy);
1682         }
1683
1684 suspend:
1685         cpufreq_suspended = true;
1686 }
1687
1688 /**
1689  * cpufreq_resume() - Resume CPUFreq governors
1690  *
1691  * Called during system wide Suspend/Hibernate cycle for resuming governors that
1692  * are suspended with cpufreq_suspend().
1693  */
1694 void cpufreq_resume(void)
1695 {
1696         struct cpufreq_policy *policy;
1697         int ret;
1698
1699         if (!cpufreq_driver)
1700                 return;
1701
1702         if (unlikely(!cpufreq_suspended))
1703                 return;
1704
1705         cpufreq_suspended = false;
1706
1707         if (!has_target() && !cpufreq_driver->resume)
1708                 return;
1709
1710         pr_debug("%s: Resuming Governors\n", __func__);
1711
1712         for_each_active_policy(policy) {
1713                 if (cpufreq_driver->resume && cpufreq_driver->resume(policy)) {
1714                         pr_err("%s: Failed to resume driver: %p\n", __func__,
1715                                 policy);
1716                 } else if (has_target()) {
1717                         down_write(&policy->rwsem);
1718                         ret = cpufreq_start_governor(policy);
1719                         up_write(&policy->rwsem);
1720
1721                         if (ret)
1722                                 pr_err("%s: Failed to start governor for policy: %p\n",
1723                                        __func__, policy);
1724                 }
1725         }
1726 }
1727
1728 /**
1729  *      cpufreq_get_current_driver - return current driver's name
1730  *
1731  *      Return the name string of the currently loaded cpufreq driver
1732  *      or NULL, if none.
1733  */
1734 const char *cpufreq_get_current_driver(void)
1735 {
1736         if (cpufreq_driver)
1737                 return cpufreq_driver->name;
1738
1739         return NULL;
1740 }
1741 EXPORT_SYMBOL_GPL(cpufreq_get_current_driver);
1742
1743 /**
1744  *      cpufreq_get_driver_data - return current driver data
1745  *
1746  *      Return the private data of the currently loaded cpufreq
1747  *      driver, or NULL if no cpufreq driver is loaded.
1748  */
1749 void *cpufreq_get_driver_data(void)
1750 {
1751         if (cpufreq_driver)
1752                 return cpufreq_driver->driver_data;
1753
1754         return NULL;
1755 }
1756 EXPORT_SYMBOL_GPL(cpufreq_get_driver_data);
1757
1758 /*********************************************************************
1759  *                     NOTIFIER LISTS INTERFACE                      *
1760  *********************************************************************/
1761
1762 /**
1763  *      cpufreq_register_notifier - register a driver with cpufreq
1764  *      @nb: notifier function to register
1765  *      @list: CPUFREQ_TRANSITION_NOTIFIER or CPUFREQ_POLICY_NOTIFIER
1766  *
1767  *      Add a driver to one of two lists: either a list of drivers that
1768  *      are notified about clock rate changes (once before and once after
1769  *      the transition), or a list of drivers that are notified about
1770  *      changes in cpufreq policy.
1771  *
1772  *      This function may sleep, and has the same return conditions as
1773  *      blocking_notifier_chain_register.
1774  */
1775 int cpufreq_register_notifier(struct notifier_block *nb, unsigned int list)
1776 {
1777         int ret;
1778
1779         if (cpufreq_disabled())
1780                 return -EINVAL;
1781
1782         switch (list) {
1783         case CPUFREQ_TRANSITION_NOTIFIER:
1784                 mutex_lock(&cpufreq_fast_switch_lock);
1785
1786                 if (cpufreq_fast_switch_count > 0) {
1787                         mutex_unlock(&cpufreq_fast_switch_lock);
1788                         return -EBUSY;
1789                 }
1790                 ret = srcu_notifier_chain_register(
1791                                 &cpufreq_transition_notifier_list, nb);
1792                 if (!ret)
1793                         cpufreq_fast_switch_count--;
1794
1795                 mutex_unlock(&cpufreq_fast_switch_lock);
1796                 break;
1797         case CPUFREQ_POLICY_NOTIFIER:
1798                 ret = blocking_notifier_chain_register(
1799                                 &cpufreq_policy_notifier_list, nb);
1800                 break;
1801         default:
1802                 ret = -EINVAL;
1803         }
1804
1805         return ret;
1806 }
1807 EXPORT_SYMBOL(cpufreq_register_notifier);
1808
1809 /**
1810  *      cpufreq_unregister_notifier - unregister a driver with cpufreq
1811  *      @nb: notifier block to be unregistered
1812  *      @list: CPUFREQ_TRANSITION_NOTIFIER or CPUFREQ_POLICY_NOTIFIER
1813  *
1814  *      Remove a driver from the CPU frequency notifier list.
1815  *
1816  *      This function may sleep, and has the same return conditions as
1817  *      blocking_notifier_chain_unregister.
1818  */
1819 int cpufreq_unregister_notifier(struct notifier_block *nb, unsigned int list)
1820 {
1821         int ret;
1822
1823         if (cpufreq_disabled())
1824                 return -EINVAL;
1825
1826         switch (list) {
1827         case CPUFREQ_TRANSITION_NOTIFIER:
1828                 mutex_lock(&cpufreq_fast_switch_lock);
1829
1830                 ret = srcu_notifier_chain_unregister(
1831                                 &cpufreq_transition_notifier_list, nb);
1832                 if (!ret && !WARN_ON(cpufreq_fast_switch_count >= 0))
1833                         cpufreq_fast_switch_count++;
1834
1835                 mutex_unlock(&cpufreq_fast_switch_lock);
1836                 break;
1837         case CPUFREQ_POLICY_NOTIFIER:
1838                 ret = blocking_notifier_chain_unregister(
1839                                 &cpufreq_policy_notifier_list, nb);
1840                 break;
1841         default:
1842                 ret = -EINVAL;
1843         }
1844
1845         return ret;
1846 }
1847 EXPORT_SYMBOL(cpufreq_unregister_notifier);
1848
1849
1850 /*********************************************************************
1851  *                              GOVERNORS                            *
1852  *********************************************************************/
1853
1854 /**
1855  * cpufreq_driver_fast_switch - Carry out a fast CPU frequency switch.
1856  * @policy: cpufreq policy to switch the frequency for.
1857  * @target_freq: New frequency to set (may be approximate).
1858  *
1859  * Carry out a fast frequency switch without sleeping.
1860  *
1861  * The driver's ->fast_switch() callback invoked by this function must be
1862  * suitable for being called from within RCU-sched read-side critical sections
1863  * and it is expected to select the minimum available frequency greater than or
1864  * equal to @target_freq (CPUFREQ_RELATION_L).
1865  *
1866  * This function must not be called if policy->fast_switch_enabled is unset.
1867  *
1868  * Governors calling this function must guarantee that it will never be invoked
1869  * twice in parallel for the same policy and that it will never be called in
1870  * parallel with either ->target() or ->target_index() for the same policy.
1871  *
1872  * Returns the actual frequency set for the CPU.
1873  *
1874  * If 0 is returned by the driver's ->fast_switch() callback to indicate an
1875  * error condition, the hardware configuration must be preserved.
1876  */
1877 unsigned int cpufreq_driver_fast_switch(struct cpufreq_policy *policy,
1878                                         unsigned int target_freq)
1879 {
1880         target_freq = clamp_val(target_freq, policy->min, policy->max);
1881
1882         return cpufreq_driver->fast_switch(policy, target_freq);
1883 }
1884 EXPORT_SYMBOL_GPL(cpufreq_driver_fast_switch);
1885
1886 /* Must set freqs->new to intermediate frequency */
1887 static int __target_intermediate(struct cpufreq_policy *policy,
1888                                  struct cpufreq_freqs *freqs, int index)
1889 {
1890         int ret;
1891
1892         freqs->new = cpufreq_driver->get_intermediate(policy, index);
1893
1894         /* We don't need to switch to intermediate freq */
1895         if (!freqs->new)
1896                 return 0;
1897
1898         pr_debug("%s: cpu: %d, switching to intermediate freq: oldfreq: %u, intermediate freq: %u\n",
1899                  __func__, policy->cpu, freqs->old, freqs->new);
1900
1901         cpufreq_freq_transition_begin(policy, freqs);
1902         ret = cpufreq_driver->target_intermediate(policy, index);
1903         cpufreq_freq_transition_end(policy, freqs, ret);
1904
1905         if (ret)
1906                 pr_err("%s: Failed to change to intermediate frequency: %d\n",
1907                        __func__, ret);
1908
1909         return ret;
1910 }
1911
1912 static int __target_index(struct cpufreq_policy *policy, int index)
1913 {
1914         struct cpufreq_freqs freqs = {.old = policy->cur, .flags = 0};
1915         unsigned int intermediate_freq = 0;
1916         unsigned int newfreq = policy->freq_table[index].frequency;
1917         int retval = -EINVAL;
1918         bool notify;
1919
1920         if (newfreq == policy->cur)
1921                 return 0;
1922
1923         notify = !(cpufreq_driver->flags & CPUFREQ_ASYNC_NOTIFICATION);
1924         if (notify) {
1925                 /* Handle switching to intermediate frequency */
1926                 if (cpufreq_driver->get_intermediate) {
1927                         retval = __target_intermediate(policy, &freqs, index);
1928                         if (retval)
1929                                 return retval;
1930
1931                         intermediate_freq = freqs.new;
1932                         /* Set old freq to intermediate */
1933                         if (intermediate_freq)
1934                                 freqs.old = freqs.new;
1935                 }
1936
1937                 freqs.new = newfreq;
1938                 pr_debug("%s: cpu: %d, oldfreq: %u, new freq: %u\n",
1939                          __func__, policy->cpu, freqs.old, freqs.new);
1940
1941                 cpufreq_freq_transition_begin(policy, &freqs);
1942         }
1943
1944         retval = cpufreq_driver->target_index(policy, index);
1945         if (retval)
1946                 pr_err("%s: Failed to change cpu frequency: %d\n", __func__,
1947                        retval);
1948
1949         if (notify) {
1950                 cpufreq_freq_transition_end(policy, &freqs, retval);
1951
1952                 /*
1953                  * Failed after setting to intermediate freq? Driver should have
1954                  * reverted back to initial frequency and so should we. Check
1955                  * here for intermediate_freq instead of get_intermediate, in
1956                  * case we haven't switched to intermediate freq at all.
1957                  */
1958                 if (unlikely(retval && intermediate_freq)) {
1959                         freqs.old = intermediate_freq;
1960                         freqs.new = policy->restore_freq;
1961                         cpufreq_freq_transition_begin(policy, &freqs);
1962                         cpufreq_freq_transition_end(policy, &freqs, 0);
1963                 }
1964         }
1965
1966         return retval;
1967 }
1968
1969 int __cpufreq_driver_target(struct cpufreq_policy *policy,
1970                             unsigned int target_freq,
1971                             unsigned int relation)
1972 {
1973         unsigned int old_target_freq = target_freq;
1974         int index;
1975
1976         if (cpufreq_disabled())
1977                 return -ENODEV;
1978
1979         /* Make sure that target_freq is within supported range */
1980         target_freq = clamp_val(target_freq, policy->min, policy->max);
1981
1982         pr_debug("target for CPU %u: %u kHz, relation %u, requested %u kHz\n",
1983                  policy->cpu, target_freq, relation, old_target_freq);
1984
1985         /*
1986          * This might look like a redundant call as we are checking it again
1987          * after finding index. But it is left intentionally for cases where
1988          * exactly same freq is called again and so we can save on few function
1989          * calls.
1990          */
1991         if (target_freq == policy->cur)
1992                 return 0;
1993
1994         /* Save last value to restore later on errors */
1995         policy->restore_freq = policy->cur;
1996
1997         if (cpufreq_driver->target)
1998                 return cpufreq_driver->target(policy, target_freq, relation);
1999
2000         if (!cpufreq_driver->target_index)
2001                 return -EINVAL;
2002
2003         index = cpufreq_frequency_table_target(policy, target_freq, relation);
2004
2005         return __target_index(policy, index);
2006 }
2007 EXPORT_SYMBOL_GPL(__cpufreq_driver_target);
2008
2009 int cpufreq_driver_target(struct cpufreq_policy *policy,
2010                           unsigned int target_freq,
2011                           unsigned int relation)
2012 {
2013         int ret = -EINVAL;
2014
2015         down_write(&policy->rwsem);
2016
2017         ret = __cpufreq_driver_target(policy, target_freq, relation);
2018
2019         up_write(&policy->rwsem);
2020
2021         return ret;
2022 }
2023 EXPORT_SYMBOL_GPL(cpufreq_driver_target);
2024
2025 __weak struct cpufreq_governor *cpufreq_fallback_governor(void)
2026 {
2027         return NULL;
2028 }
2029
2030 static int cpufreq_init_governor(struct cpufreq_policy *policy)
2031 {
2032         int ret;
2033
2034         /* Don't start any governor operations if we are entering suspend */
2035         if (cpufreq_suspended)
2036                 return 0;
2037         /*
2038          * Governor might not be initiated here if ACPI _PPC changed
2039          * notification happened, so check it.
2040          */
2041         if (!policy->governor)
2042                 return -EINVAL;
2043
2044         /* Platform doesn't want dynamic frequency switching ? */
2045         if (policy->governor->dynamic_switching &&
2046             cpufreq_driver->flags & CPUFREQ_NO_AUTO_DYNAMIC_SWITCHING) {
2047                 struct cpufreq_governor *gov = cpufreq_fallback_governor();
2048
2049                 if (gov) {
2050                         pr_warn("Can't use %s governor as dynamic switching is disallowed. Fallback to %s governor\n",
2051                                 policy->governor->name, gov->name);
2052                         policy->governor = gov;
2053                 } else {
2054                         return -EINVAL;
2055                 }
2056         }
2057
2058         if (!try_module_get(policy->governor->owner))
2059                 return -EINVAL;
2060
2061         pr_debug("%s: for CPU %u\n", __func__, policy->cpu);
2062
2063         if (policy->governor->init) {
2064                 ret = policy->governor->init(policy);
2065                 if (ret) {
2066                         module_put(policy->governor->owner);
2067                         return ret;
2068                 }
2069         }
2070
2071         return 0;
2072 }
2073
2074 static void cpufreq_exit_governor(struct cpufreq_policy *policy)
2075 {
2076         if (cpufreq_suspended || !policy->governor)
2077                 return;
2078
2079         pr_debug("%s: for CPU %u\n", __func__, policy->cpu);
2080
2081         if (policy->governor->exit)
2082                 policy->governor->exit(policy);
2083
2084         module_put(policy->governor->owner);
2085 }
2086
2087 static int cpufreq_start_governor(struct cpufreq_policy *policy)
2088 {
2089         int ret;
2090
2091         if (cpufreq_suspended)
2092                 return 0;
2093
2094         if (!policy->governor)
2095                 return -EINVAL;
2096
2097         pr_debug("%s: for CPU %u\n", __func__, policy->cpu);
2098
2099         if (cpufreq_driver->get && !cpufreq_driver->setpolicy)
2100                 cpufreq_update_current_freq(policy);
2101
2102         if (policy->governor->start) {
2103                 ret = policy->governor->start(policy);
2104                 if (ret)
2105                         return ret;
2106         }
2107
2108         if (policy->governor->limits)
2109                 policy->governor->limits(policy);
2110
2111         return 0;
2112 }
2113
2114 static void cpufreq_stop_governor(struct cpufreq_policy *policy)
2115 {
2116         if (cpufreq_suspended || !policy->governor)
2117                 return;
2118
2119         pr_debug("%s: for CPU %u\n", __func__, policy->cpu);
2120
2121         if (policy->governor->stop)
2122                 policy->governor->stop(policy);
2123 }
2124
2125 static void cpufreq_governor_limits(struct cpufreq_policy *policy)
2126 {
2127         if (cpufreq_suspended || !policy->governor)
2128                 return;
2129
2130         pr_debug("%s: for CPU %u\n", __func__, policy->cpu);
2131
2132         if (policy->governor->limits)
2133                 policy->governor->limits(policy);
2134 }
2135
2136 int cpufreq_register_governor(struct cpufreq_governor *governor)
2137 {
2138         int err;
2139
2140         if (!governor)
2141                 return -EINVAL;
2142
2143         if (cpufreq_disabled())
2144                 return -ENODEV;
2145
2146         mutex_lock(&cpufreq_governor_mutex);
2147
2148         err = -EBUSY;
2149         if (!find_governor(governor->name)) {
2150                 err = 0;
2151                 list_add(&governor->governor_list, &cpufreq_governor_list);
2152         }
2153
2154         mutex_unlock(&cpufreq_governor_mutex);
2155         return err;
2156 }
2157 EXPORT_SYMBOL_GPL(cpufreq_register_governor);
2158
2159 void cpufreq_unregister_governor(struct cpufreq_governor *governor)
2160 {
2161         struct cpufreq_policy *policy;
2162         unsigned long flags;
2163
2164         if (!governor)
2165                 return;
2166
2167         if (cpufreq_disabled())
2168                 return;
2169
2170         /* clear last_governor for all inactive policies */
2171         read_lock_irqsave(&cpufreq_driver_lock, flags);
2172         for_each_inactive_policy(policy) {
2173                 if (!strcmp(policy->last_governor, governor->name)) {
2174                         policy->governor = NULL;
2175                         strcpy(policy->last_governor, "\0");
2176                 }
2177         }
2178         read_unlock_irqrestore(&cpufreq_driver_lock, flags);
2179
2180         mutex_lock(&cpufreq_governor_mutex);
2181         list_del(&governor->governor_list);
2182         mutex_unlock(&cpufreq_governor_mutex);
2183 }
2184 EXPORT_SYMBOL_GPL(cpufreq_unregister_governor);
2185
2186
2187 /*********************************************************************
2188  *                          POLICY INTERFACE                         *
2189  *********************************************************************/
2190
2191 /**
2192  * cpufreq_get_policy - get the current cpufreq_policy
2193  * @policy: struct cpufreq_policy into which the current cpufreq_policy
2194  *      is written
2195  *
2196  * Reads the current cpufreq policy.
2197  */
2198 int cpufreq_get_policy(struct cpufreq_policy *policy, unsigned int cpu)
2199 {
2200         struct cpufreq_policy *cpu_policy;
2201         if (!policy)
2202                 return -EINVAL;
2203
2204         cpu_policy = cpufreq_cpu_get(cpu);
2205         if (!cpu_policy)
2206                 return -EINVAL;
2207
2208         memcpy(policy, cpu_policy, sizeof(*policy));
2209
2210         cpufreq_cpu_put(cpu_policy);
2211         return 0;
2212 }
2213 EXPORT_SYMBOL(cpufreq_get_policy);
2214
2215 /**
2216  * cpufreq_set_policy - Modify cpufreq policy parameters.
2217  * @policy: Policy object to modify.
2218  * @new_policy: New policy data.
2219  *
2220  * Pass @new_policy to the cpufreq driver's ->verify() callback, run the
2221  * installed policy notifiers for it with the CPUFREQ_ADJUST value, pass it to
2222  * the driver's ->verify() callback again and run the notifiers for it again
2223  * with the CPUFREQ_NOTIFY value.  Next, copy the min and max parameters
2224  * of @new_policy to @policy and either invoke the driver's ->setpolicy()
2225  * callback (if present) or carry out a governor update for @policy.  That is,
2226  * run the current governor's ->limits() callback (if the governor field in
2227  * @new_policy points to the same object as the one in @policy) or replace the
2228  * governor for @policy with the new one stored in @new_policy.
2229  *
2230  * The cpuinfo part of @policy is not updated by this function.
2231  */
2232 static int cpufreq_set_policy(struct cpufreq_policy *policy,
2233                               struct cpufreq_policy *new_policy)
2234 {
2235         struct cpufreq_governor *old_gov;
2236         int ret;
2237
2238         pr_debug("setting new policy for CPU %u: %u - %u kHz\n",
2239                  new_policy->cpu, new_policy->min, new_policy->max);
2240
2241         memcpy(&new_policy->cpuinfo, &policy->cpuinfo, sizeof(policy->cpuinfo));
2242
2243         /*
2244         * This check works well when we store new min/max freq attributes,
2245         * because new_policy is a copy of policy with one field updated.
2246         */
2247         if (new_policy->min > new_policy->max)
2248                 return -EINVAL;
2249
2250         /* verify the cpu speed can be set within this limit */
2251         ret = cpufreq_driver->verify(new_policy);
2252         if (ret)
2253                 return ret;
2254
2255         /* adjust if necessary - all reasons */
2256         blocking_notifier_call_chain(&cpufreq_policy_notifier_list,
2257                         CPUFREQ_ADJUST, new_policy);
2258
2259         /*
2260          * verify the cpu speed can be set within this limit, which might be
2261          * different to the first one
2262          */
2263         ret = cpufreq_driver->verify(new_policy);
2264         if (ret)
2265                 return ret;
2266
2267         /* notification of the new policy */
2268         blocking_notifier_call_chain(&cpufreq_policy_notifier_list,
2269                         CPUFREQ_NOTIFY, new_policy);
2270
2271         policy->min = new_policy->min;
2272         policy->max = new_policy->max;
2273         trace_cpu_frequency_limits(policy);
2274
2275         policy->cached_target_freq = UINT_MAX;
2276
2277         pr_debug("new min and max freqs are %u - %u kHz\n",
2278                  policy->min, policy->max);
2279
2280         if (cpufreq_driver->setpolicy) {
2281                 policy->policy = new_policy->policy;
2282                 pr_debug("setting range\n");
2283                 return cpufreq_driver->setpolicy(policy);
2284         }
2285
2286         if (new_policy->governor == policy->governor) {
2287                 pr_debug("governor limits update\n");
2288                 cpufreq_governor_limits(policy);
2289                 return 0;
2290         }
2291
2292         pr_debug("governor switch\n");
2293
2294         /* save old, working values */
2295         old_gov = policy->governor;
2296         /* end old governor */
2297         if (old_gov) {
2298                 cpufreq_stop_governor(policy);
2299                 cpufreq_exit_governor(policy);
2300         }
2301
2302         /* start new governor */
2303         policy->governor = new_policy->governor;
2304         ret = cpufreq_init_governor(policy);
2305         if (!ret) {
2306                 ret = cpufreq_start_governor(policy);
2307                 if (!ret) {
2308                         pr_debug("governor change\n");
2309                         sched_cpufreq_governor_change(policy, old_gov);
2310                         return 0;
2311                 }
2312                 cpufreq_exit_governor(policy);
2313         }
2314
2315         /* new governor failed, so re-start old one */
2316         pr_debug("starting governor %s failed\n", policy->governor->name);
2317         if (old_gov) {
2318                 policy->governor = old_gov;
2319                 if (cpufreq_init_governor(policy))
2320                         policy->governor = NULL;
2321                 else
2322                         cpufreq_start_governor(policy);
2323         }
2324
2325         return ret;
2326 }
2327
2328 /**
2329  * cpufreq_update_policy - Re-evaluate an existing cpufreq policy.
2330  * @cpu: CPU to re-evaluate the policy for.
2331  *
2332  * Update the current frequency for the cpufreq policy of @cpu and use
2333  * cpufreq_set_policy() to re-apply the min and max limits saved in the
2334  * user_policy sub-structure of that policy, which triggers the evaluation
2335  * of policy notifiers and the cpufreq driver's ->verify() callback for the
2336  * policy in question, among other things.
2337  */
2338 void cpufreq_update_policy(unsigned int cpu)
2339 {
2340         struct cpufreq_policy *policy = cpufreq_cpu_get(cpu);
2341         struct cpufreq_policy new_policy;
2342
2343         if (!policy)
2344                 return;
2345
2346         down_write(&policy->rwsem);
2347
2348         if (policy_is_inactive(policy))
2349                 goto unlock;
2350
2351         /*
2352          * BIOS might change freq behind our back
2353          * -> ask driver for current freq and notify governors about a change
2354          */
2355         if (cpufreq_driver->get && !cpufreq_driver->setpolicy &&
2356             (cpufreq_suspended || WARN_ON(!cpufreq_update_current_freq(policy))))
2357                 goto unlock;
2358
2359         pr_debug("updating policy for CPU %u\n", cpu);
2360         memcpy(&new_policy, policy, sizeof(*policy));
2361         new_policy.min = policy->user_policy.min;
2362         new_policy.max = policy->user_policy.max;
2363
2364         cpufreq_set_policy(policy, &new_policy);
2365
2366 unlock:
2367         up_write(&policy->rwsem);
2368
2369         cpufreq_cpu_put(policy);
2370 }
2371 EXPORT_SYMBOL(cpufreq_update_policy);
2372
2373 /*********************************************************************
2374  *               BOOST                                               *
2375  *********************************************************************/
2376 static int cpufreq_boost_set_sw(int state)
2377 {
2378         struct cpufreq_policy *policy;
2379         int ret = -EINVAL;
2380
2381         for_each_active_policy(policy) {
2382                 if (!policy->freq_table)
2383                         continue;
2384
2385                 ret = cpufreq_frequency_table_cpuinfo(policy,
2386                                                       policy->freq_table);
2387                 if (ret) {
2388                         pr_err("%s: Policy frequency update failed\n",
2389                                __func__);
2390                         break;
2391                 }
2392
2393                 down_write(&policy->rwsem);
2394                 policy->user_policy.max = policy->max;
2395                 cpufreq_governor_limits(policy);
2396                 up_write(&policy->rwsem);
2397         }
2398
2399         return ret;
2400 }
2401
2402 int cpufreq_boost_trigger_state(int state)
2403 {
2404         unsigned long flags;
2405         int ret = 0;
2406
2407         if (cpufreq_driver->boost_enabled == state)
2408                 return 0;
2409
2410         write_lock_irqsave(&cpufreq_driver_lock, flags);
2411         cpufreq_driver->boost_enabled = state;
2412         write_unlock_irqrestore(&cpufreq_driver_lock, flags);
2413
2414         ret = cpufreq_driver->set_boost(state);
2415         if (ret) {
2416                 write_lock_irqsave(&cpufreq_driver_lock, flags);
2417                 cpufreq_driver->boost_enabled = !state;
2418                 write_unlock_irqrestore(&cpufreq_driver_lock, flags);
2419
2420                 pr_err("%s: Cannot %s BOOST\n",
2421                        __func__, state ? "enable" : "disable");
2422         }
2423
2424         return ret;
2425 }
2426
2427 static bool cpufreq_boost_supported(void)
2428 {
2429         return likely(cpufreq_driver) && cpufreq_driver->set_boost;
2430 }
2431
2432 static int create_boost_sysfs_file(void)
2433 {
2434         int ret;
2435
2436         ret = sysfs_create_file(cpufreq_global_kobject, &boost.attr);
2437         if (ret)
2438                 pr_err("%s: cannot register global BOOST sysfs file\n",
2439                        __func__);
2440
2441         return ret;
2442 }
2443
2444 static void remove_boost_sysfs_file(void)
2445 {
2446         if (cpufreq_boost_supported())
2447                 sysfs_remove_file(cpufreq_global_kobject, &boost.attr);
2448 }
2449
2450 int cpufreq_enable_boost_support(void)
2451 {
2452         if (!cpufreq_driver)
2453                 return -EINVAL;
2454
2455         if (cpufreq_boost_supported())
2456                 return 0;
2457
2458         cpufreq_driver->set_boost = cpufreq_boost_set_sw;
2459
2460         /* This will get removed on driver unregister */
2461         return create_boost_sysfs_file();
2462 }
2463 EXPORT_SYMBOL_GPL(cpufreq_enable_boost_support);
2464
2465 int cpufreq_boost_enabled(void)
2466 {
2467         return cpufreq_driver->boost_enabled;
2468 }
2469 EXPORT_SYMBOL_GPL(cpufreq_boost_enabled);
2470
2471 /*********************************************************************
2472  *               REGISTER / UNREGISTER CPUFREQ DRIVER                *
2473  *********************************************************************/
2474 static enum cpuhp_state hp_online;
2475
2476 static int cpuhp_cpufreq_online(unsigned int cpu)
2477 {
2478         cpufreq_online(cpu);
2479
2480         return 0;
2481 }
2482
2483 static int cpuhp_cpufreq_offline(unsigned int cpu)
2484 {
2485         cpufreq_offline(cpu);
2486
2487         return 0;
2488 }
2489
2490 /**
2491  * cpufreq_register_driver - register a CPU Frequency driver
2492  * @driver_data: A struct cpufreq_driver containing the values#
2493  * submitted by the CPU Frequency driver.
2494  *
2495  * Registers a CPU Frequency driver to this core code. This code
2496  * returns zero on success, -EEXIST when another driver got here first
2497  * (and isn't unregistered in the meantime).
2498  *
2499  */
2500 int cpufreq_register_driver(struct cpufreq_driver *driver_data)
2501 {
2502         unsigned long flags;
2503         int ret;
2504
2505         if (cpufreq_disabled())
2506                 return -ENODEV;
2507
2508         if (!driver_data || !driver_data->verify || !driver_data->init ||
2509             !(driver_data->setpolicy || driver_data->target_index ||
2510                     driver_data->target) ||
2511              (driver_data->setpolicy && (driver_data->target_index ||
2512                     driver_data->target)) ||
2513              (!driver_data->get_intermediate != !driver_data->target_intermediate) ||
2514              (!driver_data->online != !driver_data->offline))
2515                 return -EINVAL;
2516
2517         pr_debug("trying to register driver %s\n", driver_data->name);
2518
2519         /* Protect against concurrent CPU online/offline. */
2520         cpus_read_lock();
2521
2522         write_lock_irqsave(&cpufreq_driver_lock, flags);
2523         if (cpufreq_driver) {
2524                 write_unlock_irqrestore(&cpufreq_driver_lock, flags);
2525                 ret = -EEXIST;
2526                 goto out;
2527         }
2528         cpufreq_driver = driver_data;
2529         write_unlock_irqrestore(&cpufreq_driver_lock, flags);
2530
2531         if (driver_data->setpolicy)
2532                 driver_data->flags |= CPUFREQ_CONST_LOOPS;
2533
2534         if (cpufreq_boost_supported()) {
2535                 ret = create_boost_sysfs_file();
2536                 if (ret)
2537                         goto err_null_driver;
2538         }
2539
2540         ret = subsys_interface_register(&cpufreq_interface);
2541         if (ret)
2542                 goto err_boost_unreg;
2543
2544         if (!(cpufreq_driver->flags & CPUFREQ_STICKY) &&
2545             list_empty(&cpufreq_policy_list)) {
2546                 /* if all ->init() calls failed, unregister */
2547                 ret = -ENODEV;
2548                 pr_debug("%s: No CPU initialized for driver %s\n", __func__,
2549                          driver_data->name);
2550                 goto err_if_unreg;
2551         }
2552
2553         ret = cpuhp_setup_state_nocalls_cpuslocked(CPUHP_AP_ONLINE_DYN,
2554                                                    "cpufreq:online",
2555                                                    cpuhp_cpufreq_online,
2556                                                    cpuhp_cpufreq_offline);
2557         if (ret < 0)
2558                 goto err_if_unreg;
2559         hp_online = ret;
2560         ret = 0;
2561
2562         pr_debug("driver %s up and running\n", driver_data->name);
2563         goto out;
2564
2565 err_if_unreg:
2566         subsys_interface_unregister(&cpufreq_interface);
2567 err_boost_unreg:
2568         remove_boost_sysfs_file();
2569 err_null_driver:
2570         write_lock_irqsave(&cpufreq_driver_lock, flags);
2571         cpufreq_driver = NULL;
2572         write_unlock_irqrestore(&cpufreq_driver_lock, flags);
2573 out:
2574         cpus_read_unlock();
2575         return ret;
2576 }
2577 EXPORT_SYMBOL_GPL(cpufreq_register_driver);
2578
2579 /**
2580  * cpufreq_unregister_driver - unregister the current CPUFreq driver
2581  *
2582  * Unregister the current CPUFreq driver. Only call this if you have
2583  * the right to do so, i.e. if you have succeeded in initialising before!
2584  * Returns zero if successful, and -EINVAL if the cpufreq_driver is
2585  * currently not initialised.
2586  */
2587 int cpufreq_unregister_driver(struct cpufreq_driver *driver)
2588 {
2589         unsigned long flags;
2590
2591         if (!cpufreq_driver || (driver != cpufreq_driver))
2592                 return -EINVAL;
2593
2594         pr_debug("unregistering driver %s\n", driver->name);
2595
2596         /* Protect against concurrent cpu hotplug */
2597         cpus_read_lock();
2598         subsys_interface_unregister(&cpufreq_interface);
2599         remove_boost_sysfs_file();
2600         cpuhp_remove_state_nocalls_cpuslocked(hp_online);
2601
2602         write_lock_irqsave(&cpufreq_driver_lock, flags);
2603
2604         cpufreq_driver = NULL;
2605
2606         write_unlock_irqrestore(&cpufreq_driver_lock, flags);
2607         cpus_read_unlock();
2608
2609         return 0;
2610 }
2611 EXPORT_SYMBOL_GPL(cpufreq_unregister_driver);
2612
2613 /*
2614  * Stop cpufreq at shutdown to make sure it isn't holding any locks
2615  * or mutexes when secondary CPUs are halted.
2616  */
2617 static struct syscore_ops cpufreq_syscore_ops = {
2618         .shutdown = cpufreq_suspend,
2619 };
2620
2621 struct kobject *cpufreq_global_kobject;
2622 EXPORT_SYMBOL(cpufreq_global_kobject);
2623
2624 static int __init cpufreq_core_init(void)
2625 {
2626         if (cpufreq_disabled())
2627                 return -ENODEV;
2628
2629         cpufreq_global_kobject = kobject_create_and_add("cpufreq", &cpu_subsys.dev_root->kobj);
2630         BUG_ON(!cpufreq_global_kobject);
2631
2632         register_syscore_ops(&cpufreq_syscore_ops);
2633
2634         return 0;
2635 }
2636 module_param(off, int, 0444);
2637 core_initcall(cpufreq_core_init);