[PATCH 55/70] cpufreq: Interactive: Implement per policy instances of governor

From: BÃlint Czobor
Date: Tue Oct 27 2015 - 13:33:36 EST

From: Viresh Kumar <viresh.kumar@xxxxxxxxxx>

If we have a multi-package system, where we have multiple instances of struct
policy (per package), currently we can't have multiple instances of same
governor. i.e. We can't have multiple instances of Interactive governor for
multiple packages.

This is a bottleneck for multicluster system, where we want different packages
to use Interactive governor, but with different tunables.

This patch uses the infrastructure provided by earlier patches pushed in
Mainline in v3.10-rc1/rc2 and implements per policy instances of Interactive
governor.

Change-Id: I70436d4a5a45c6cb6edf37f3e46d0b9fbc930982
[toddpoynor@xxxxxxxxxx: merge with later code, minor changes]
Signed-off-by: Viresh Kumar <viresh.kumar@xxxxxxxxxx>
Signed-off-by: BÃlint Czobor <czoborbalint@xxxxxxxxx>
---
drivers/cpufreq/cpufreq_interactive.c | 629 ++++++++++++++++++++-------------
1 file changed, 383 insertions(+), 246 deletions(-)

diff --git a/drivers/cpufreq/cpufreq_interactive.c b/drivers/cpufreq/cpufreq_interactive.c
index b0685c7..66c096d 100644
--- a/drivers/cpufreq/cpufreq_interactive.c
+++ b/drivers/cpufreq/cpufreq_interactive.c
@@ -36,8 +36,6 @@
#define CREATE_TRACE_POINTS
#include <trace/events/cpufreq_interactive.h>

-static int active_count;
-
struct cpufreq_interactive_cpuinfo {
struct timer_list cpu_timer;
struct timer_list cpu_slack_timer;
@@ -64,58 +62,62 @@ static cpumask_t speedchange_cpumask;
static spinlock_t speedchange_cpumask_lock;
static struct mutex gov_lock;

-/* Hi speed to bump to from lo speed when load burst (default max) */
-static unsigned int hispeed_freq;
-
-/* Go to hi speed when CPU load at or above this value. */
-#define DEFAULT_GO_HISPEED_LOAD 99
-static unsigned long go_hispeed_load = DEFAULT_GO_HISPEED_LOAD;
-
/* Target load. Lower values result in higher CPU speeds. */
#define DEFAULT_TARGET_LOAD 90
static unsigned int default_target_loads[] = {DEFAULT_TARGET_LOAD};
-static spinlock_t target_loads_lock;
-static unsigned int *target_loads = default_target_loads;
-static int ntarget_loads = ARRAY_SIZE(default_target_loads);
-
-/*
- * The minimum amount of time to spend at a frequency before we can ramp down.
- */
-#define DEFAULT_MIN_SAMPLE_TIME (80 * USEC_PER_MSEC)
-static unsigned long min_sample_time = DEFAULT_MIN_SAMPLE_TIME;

-/*
- * The sample rate of the timer used to increase frequency
- */
#define DEFAULT_TIMER_RATE (20 * USEC_PER_MSEC)
-static unsigned long timer_rate = DEFAULT_TIMER_RATE;
-
-/*
- * Wait this long before raising speed above hispeed, by default a single
- * timer interval.
- */
#define DEFAULT_ABOVE_HISPEED_DELAY DEFAULT_TIMER_RATE
static unsigned int default_above_hispeed_delay[] = {
DEFAULT_ABOVE_HISPEED_DELAY };
-static spinlock_t above_hispeed_delay_lock;
-static unsigned int *above_hispeed_delay = default_above_hispeed_delay;
-static int nabove_hispeed_delay = ARRAY_SIZE(default_above_hispeed_delay);
-
-/* Non-zero means indefinite speed boost active */
-static int boost_val;
-/* Duration of a boot pulse in usecs */
-static int boostpulse_duration_val = DEFAULT_MIN_SAMPLE_TIME;
-/* End time of boost pulse in ktime converted to usecs */
-static u64 boostpulse_endtime;

-/*
- * Max additional time to wait in idle, beyond timer_rate, at speeds above
- * minimum before wakeup to reduce speed, or -1 if unnecessary.
- */
+struct cpufreq_interactive_tunables {
+ int usage_count;
+ /* Hi speed to bump to from lo speed when load burst (default max) */
+ unsigned int hispeed_freq;
+ /* Go to hi speed when CPU load at or above this value. */
+#define DEFAULT_GO_HISPEED_LOAD 99
+ unsigned long go_hispeed_load;
+ /* Target load. Lower values result in higher CPU speeds. */
+ spinlock_t target_loads_lock;
+ unsigned int *target_loads;
+ int ntarget_loads;
+ /*
+ * The minimum amount of time to spend at a frequency before we can ramp
+ * down.
+ */
+#define DEFAULT_MIN_SAMPLE_TIME (80 * USEC_PER_MSEC)
+ unsigned long min_sample_time;
+ /*
+ * The sample rate of the timer used to increase frequency
+ */
+ unsigned long timer_rate;
+ /*
+ * Wait this long before raising speed above hispeed, by default a
+ * single timer interval.
+ */
+ spinlock_t above_hispeed_delay_lock;
+ unsigned int *above_hispeed_delay;
+ int nabove_hispeed_delay;
+ /* Non-zero means indefinite speed boost active */
+ int boost_val;
+ /* Duration of a boot pulse in usecs */
+ int boostpulse_duration_val;
+ /* End time of boost pulse in ktime converted to usecs */
+ u64 boostpulse_endtime;
+ /*
+ * Max additional time to wait in idle, beyond timer_rate, at speeds
+ * above minimum before wakeup to reduce speed, or -1 if unnecessary.
+ */
#define DEFAULT_TIMER_SLACK (4 * DEFAULT_TIMER_RATE)
-static int timer_slack_val = DEFAULT_TIMER_SLACK;
+ int timer_slack_val;
+ bool io_is_busy;
+};
+
+/* For cases where we have single governor instance for system */
+struct cpufreq_interactive_tunables *common_tunables;

-static bool io_is_busy;
+static struct attribute_group *get_sysfs_attr(void);

static inline cputime64_t get_cpu_idle_time_jiffy(unsigned int cpu,
cputime64_t *wall)
@@ -140,8 +142,10 @@ static inline cputime64_t get_cpu_idle_time_jiffy(unsigned int cpu,
return jiffies_to_usecs(idle_time);
}

-static inline cputime64_t get_cpu_idle_time(unsigned int cpu,
- cputime64_t *wall)
+static inline cputime64_t get_cpu_idle_time(
+ unsigned int cpu,
+ cputime64_t *wall,
+ bool io_is_busy)
{
u64 idle_time = get_cpu_idle_time_us(cpu, wall);

@@ -156,20 +160,24 @@ static inline cputime64_t get_cpu_idle_time(unsigned int cpu,
static void cpufreq_interactive_timer_resched(
struct cpufreq_interactive_cpuinfo *pcpu)
{
+ struct cpufreq_interactive_tunables *tunables =
+ pcpu->policy->governor_data;
unsigned long expires;
unsigned long flags;

spin_lock_irqsave(&pcpu->load_lock, flags);
pcpu->time_in_idle =
get_cpu_idle_time(smp_processor_id(),
- &pcpu->time_in_idle_timestamp);
+ &pcpu->time_in_idle_timestamp,
+ tunables->io_is_busy);
pcpu->cputime_speedadj = 0;
pcpu->cputime_speedadj_timestamp = pcpu->time_in_idle_timestamp;
- expires = jiffies + usecs_to_jiffies(timer_rate);
+ expires = jiffies + usecs_to_jiffies(tunables->timer_rate);
mod_timer_pinned(&pcpu->cpu_timer, expires);

- if (timer_slack_val >= 0 && pcpu->target_freq > pcpu->policy->min) {
- expires += usecs_to_jiffies(timer_slack_val);
+ if (tunables->timer_slack_val >= 0 &&
+ pcpu->target_freq > pcpu->policy->min) {
+ expires += usecs_to_jiffies(tunables->timer_slack_val);
mod_timer_pinned(&pcpu->cpu_slack_timer, expires);
}

@@ -180,58 +188,66 @@ static void cpufreq_interactive_timer_resched(
* The cpu_timer and cpu_slack_timer must be deactivated when calling this
* function.
*/
-static void cpufreq_interactive_timer_start(int cpu)
+static void cpufreq_interactive_timer_start(
+ struct cpufreq_interactive_tunables *tunables, int cpu)
{
struct cpufreq_interactive_cpuinfo *pcpu = &per_cpu(cpuinfo, cpu);
- unsigned long expires = jiffies + usecs_to_jiffies(timer_rate);
+ unsigned long expires = jiffies +
+ usecs_to_jiffies(tunables->timer_rate);
unsigned long flags;

pcpu->cpu_timer.expires = expires;
add_timer_on(&pcpu->cpu_timer, cpu);
- if (timer_slack_val >= 0 && pcpu->target_freq > pcpu->policy->min) {
- expires += usecs_to_jiffies(timer_slack_val);
+ if (tunables->timer_slack_val >= 0 &&
+ pcpu->target_freq > pcpu->policy->min) {
+ expires += usecs_to_jiffies(tunables->timer_slack_val);
pcpu->cpu_slack_timer.expires = expires;
add_timer_on(&pcpu->cpu_slack_timer, cpu);
}

spin_lock_irqsave(&pcpu->load_lock, flags);
pcpu->time_in_idle =
- get_cpu_idle_time(cpu, &pcpu->time_in_idle_timestamp);
+ get_cpu_idle_time(cpu, &pcpu->time_in_idle_timestamp,
+ tunables->io_is_busy);
pcpu->cputime_speedadj = 0;
pcpu->cputime_speedadj_timestamp = pcpu->time_in_idle_timestamp;
spin_unlock_irqrestore(&pcpu->load_lock, flags);
}

-static unsigned int freq_to_above_hispeed_delay(unsigned int freq)
+static unsigned int freq_to_above_hispeed_delay(
+ struct cpufreq_interactive_tunables *tunables,
+ unsigned int freq)
{
int i;
unsigned int ret;
unsigned long flags;

- spin_lock_irqsave(&above_hispeed_delay_lock, flags);
+ spin_lock_irqsave(&tunables->above_hispeed_delay_lock, flags);

- for (i = 0; i < nabove_hispeed_delay - 1 &&
- freq >= above_hispeed_delay[i+1]; i += 2)
+ for (i = 0; i < tunables->nabove_hispeed_delay - 1 &&
+ freq >= tunables->above_hispeed_delay[i+1]; i += 2)
;

- ret = above_hispeed_delay[i];
- spin_unlock_irqrestore(&above_hispeed_delay_lock, flags);
+ ret = tunables->above_hispeed_delay[i];
+ spin_unlock_irqrestore(&tunables->above_hispeed_delay_lock, flags);
return ret;
}

-static unsigned int freq_to_targetload(unsigned int freq)
+static unsigned int freq_to_targetload(
+ struct cpufreq_interactive_tunables *tunables, unsigned int freq)
{
int i;
unsigned int ret;
unsigned long flags;

- spin_lock_irqsave(&target_loads_lock, flags);
+ spin_lock_irqsave(&tunables->target_loads_lock, flags);

- for (i = 0; i < ntarget_loads - 1 && freq >= target_loads[i+1]; i += 2)
+ for (i = 0; i < tunables->ntarget_loads - 1 &&
+ freq >= tunables->target_loads[i+1]; i += 2)
;

- ret = target_loads[i];
- spin_unlock_irqrestore(&target_loads_lock, flags);
+ ret = tunables->target_loads[i];
+ spin_unlock_irqrestore(&tunables->target_loads_lock, flags);
return ret;
}

@@ -240,9 +256,8 @@ static unsigned int freq_to_targetload(unsigned int freq)
* choose_freq() will find the minimum frequency that does not exceed its
* target load given the current load.
*/
-
-static unsigned int choose_freq(
- struct cpufreq_interactive_cpuinfo *pcpu, unsigned int loadadjfreq)
+static unsigned int choose_freq(struct cpufreq_interactive_cpuinfo *pcpu,
+ unsigned int loadadjfreq)
{
unsigned int freq = pcpu->policy->cur;
unsigned int prevfreq, freqmin, freqmax;
@@ -254,7 +269,7 @@ static unsigned int choose_freq(

do {
prevfreq = freq;
- tl = freq_to_targetload(freq);
+ tl = freq_to_targetload(pcpu->policy->governor_data, freq);

/*
* Find the lowest frequency where the computed load is less
@@ -329,13 +344,15 @@ static unsigned int choose_freq(
static u64 update_load(int cpu)
{
struct cpufreq_interactive_cpuinfo *pcpu = &per_cpu(cpuinfo, cpu);
+ struct cpufreq_interactive_tunables *tunables =
+ pcpu->policy->governor_data;
u64 now;
u64 now_idle;
unsigned int delta_idle;
unsigned int delta_time;
u64 active_time;

- now_idle = get_cpu_idle_time(cpu, &now);
+ now_idle = get_cpu_idle_time(cpu, &now, tunables->io_is_busy);
delta_idle = (unsigned int)(now_idle - pcpu->time_in_idle);
delta_time = (unsigned int)(now - pcpu->time_in_idle_timestamp);

@@ -359,6 +376,8 @@ static void cpufreq_interactive_timer(unsigned long data)
int cpu_load;
struct cpufreq_interactive_cpuinfo *pcpu =
&per_cpu(cpuinfo, data);
+ struct cpufreq_interactive_tunables *tunables =
+ pcpu->policy->governor_data;
unsigned int new_freq;
unsigned int loadadjfreq;
unsigned int index;
@@ -382,25 +401,25 @@ static void cpufreq_interactive_timer(unsigned long data)
do_div(cputime_speedadj, delta_time);
loadadjfreq = (unsigned int)cputime_speedadj * 100;
cpu_load = loadadjfreq / pcpu->target_freq;
- boosted = boost_val || now < boostpulse_endtime;
+ boosted = tunables->boost_val || now < tunables->boostpulse_endtime;

- if (cpu_load >= go_hispeed_load || boosted) {
- if (pcpu->target_freq < hispeed_freq) {
- new_freq = hispeed_freq;
+ if (cpu_load >= tunables->go_hispeed_load || boosted) {
+ if (pcpu->target_freq < tunables->hispeed_freq) {
+ new_freq = tunables->hispeed_freq;
} else {
new_freq = choose_freq(pcpu, loadadjfreq);

- if (new_freq < hispeed_freq)
- new_freq = hispeed_freq;
+ if (new_freq < tunables->hispeed_freq)
+ new_freq = tunables->hispeed_freq;
}
} else {
new_freq = choose_freq(pcpu, loadadjfreq);
}

- if (pcpu->target_freq >= hispeed_freq &&
+ if (pcpu->target_freq >= tunables->hispeed_freq &&
new_freq > pcpu->target_freq &&
now - pcpu->hispeed_validate_time <
- freq_to_above_hispeed_delay(pcpu->target_freq)) {
+ freq_to_above_hispeed_delay(tunables, pcpu->target_freq)) {
trace_cpufreq_interactive_notyet(
data, cpu_load, pcpu->target_freq,
pcpu->policy->cur, new_freq);
@@ -421,7 +440,8 @@ static void cpufreq_interactive_timer(unsigned long data)
* floor frequency for the minimum sample time since last validated.
*/
if (new_freq < pcpu->floor_freq) {
- if (now - pcpu->floor_validate_time < min_sample_time) {
+ if (now - pcpu->floor_validate_time <
+ tunables->min_sample_time) {
trace_cpufreq_interactive_notyet(
data, cpu_load, pcpu->target_freq,
pcpu->policy->cur, new_freq);
@@ -437,7 +457,7 @@ static void cpufreq_interactive_timer(unsigned long data)
* (or the indefinite boost is turned off).
*/

- if (!boosted || new_freq > hispeed_freq) {
+ if (!boosted || new_freq > tunables->hispeed_freq) {
pcpu->floor_freq = new_freq;
pcpu->floor_validate_time = now;
}
@@ -598,14 +618,16 @@ static void cpufreq_interactive_boost(void)
int anyboost = 0;
unsigned long flags;
struct cpufreq_interactive_cpuinfo *pcpu;
+ struct cpufreq_interactive_tunables *tunables;

spin_lock_irqsave(&speedchange_cpumask_lock, flags);

for_each_online_cpu(i) {
pcpu = &per_cpu(cpuinfo, i);
+ tunables = pcpu->policy->governor_data;

- if (pcpu->target_freq < hispeed_freq) {
- pcpu->target_freq = hispeed_freq;
+ if (pcpu->target_freq < tunables->hispeed_freq) {
+ pcpu->target_freq = tunables->hispeed_freq;
cpumask_set_cpu(i, &speedchange_cpumask);
pcpu->hispeed_validate_time =
ktime_to_us(ktime_get());
@@ -617,7 +639,7 @@ static void cpufreq_interactive_boost(void)
* validated.
*/

- pcpu->floor_freq = hispeed_freq;
+ pcpu->floor_freq = tunables->hispeed_freq;
pcpu->floor_validate_time = ktime_to_us(ktime_get());
}

@@ -717,26 +739,27 @@ err:
}

static ssize_t show_target_loads(
- struct kobject *kobj, struct attribute *attr, char *buf)
+ struct cpufreq_interactive_tunables *tunables,
+ char *buf)
{
int i;
ssize_t ret = 0;
unsigned long flags;

- spin_lock_irqsave(&target_loads_lock, flags);
+ spin_lock_irqsave(&tunables->target_loads_lock, flags);

- for (i = 0; i < ntarget_loads; i++)
- ret += sprintf(buf + ret, "%u%s", target_loads[i],
+ for (i = 0; i < tunables->ntarget_loads; i++)
+ ret += sprintf(buf + ret, "%u%s", tunables->target_loads[i],
i & 0x1 ? ":" : " ");

ret += sprintf(buf + --ret, "\n");
- spin_unlock_irqrestore(&target_loads_lock, flags);
+ spin_unlock_irqrestore(&tunables->target_loads_lock, flags);
return ret;
}

static ssize_t store_target_loads(
- struct kobject *kobj, struct attribute *attr, const char *buf,
- size_t count)
+ struct cpufreq_interactive_tunables *tunables,
+ const char *buf, size_t count)
{
int ntokens;
unsigned int *new_target_loads = NULL;
@@ -746,40 +769,37 @@ static ssize_t store_target_loads(
if (IS_ERR(new_target_loads))
return PTR_RET(new_target_loads);

- spin_lock_irqsave(&target_loads_lock, flags);
- if (target_loads != default_target_loads)
- kfree(target_loads);
- target_loads = new_target_loads;
- ntarget_loads = ntokens;
- spin_unlock_irqrestore(&target_loads_lock, flags);
+ spin_lock_irqsave(&tunables->target_loads_lock, flags);
+ if (tunables->target_loads != default_target_loads)
+ kfree(tunables->target_loads);
+ tunables->target_loads = new_target_loads;
+ tunables->ntarget_loads = ntokens;
+ spin_unlock_irqrestore(&tunables->target_loads_lock, flags);
return count;
}

-static struct global_attr target_loads_attr =
- __ATTR(target_loads, S_IRUGO | S_IWUSR,
- show_target_loads, store_target_loads);
-
static ssize_t show_above_hispeed_delay(
- struct kobject *kobj, struct attribute *attr, char *buf)
+ struct cpufreq_interactive_tunables *tunables, char *buf)
{
int i;
ssize_t ret = 0;
unsigned long flags;

- spin_lock_irqsave(&above_hispeed_delay_lock, flags);
+ spin_lock_irqsave(&tunables->above_hispeed_delay_lock, flags);

- for (i = 0; i < nabove_hispeed_delay; i++)
- ret += sprintf(buf + ret, "%u%s", above_hispeed_delay[i],
+ for (i = 0; i < tunables->nabove_hispeed_delay; i++)
+ ret += sprintf(buf + ret, "%u%s",
+ tunables->above_hispeed_delay[i],
i & 0x1 ? ":" : " ");

ret += sprintf(buf + --ret, "\n");
- spin_unlock_irqrestore(&above_hispeed_delay_lock, flags);
+ spin_unlock_irqrestore(&tunables->above_hispeed_delay_lock, flags);
return ret;
}

static ssize_t store_above_hispeed_delay(
- struct kobject *kobj, struct attribute *attr, const char *buf,
- size_t count)
+ struct cpufreq_interactive_tunables *tunables,
+ const char *buf, size_t count)
{
int ntokens;
unsigned int *new_above_hispeed_delay = NULL;
@@ -789,29 +809,24 @@ static ssize_t store_above_hispeed_delay(
if (IS_ERR(new_above_hispeed_delay))
return PTR_RET(new_above_hispeed_delay);

- spin_lock_irqsave(&above_hispeed_delay_lock, flags);
- if (above_hispeed_delay != default_above_hispeed_delay)
- kfree(above_hispeed_delay);
- above_hispeed_delay = new_above_hispeed_delay;
- nabove_hispeed_delay = ntokens;
- spin_unlock_irqrestore(&above_hispeed_delay_lock, flags);
+ spin_lock_irqsave(&tunables->above_hispeed_delay_lock, flags);
+ if (tunables->above_hispeed_delay != default_above_hispeed_delay)
+ kfree(tunables->above_hispeed_delay);
+ tunables->above_hispeed_delay = new_above_hispeed_delay;
+ tunables->nabove_hispeed_delay = ntokens;
+ spin_unlock_irqrestore(&tunables->above_hispeed_delay_lock, flags);
return count;

}

-static struct global_attr above_hispeed_delay_attr =
- __ATTR(above_hispeed_delay, S_IRUGO | S_IWUSR,
- show_above_hispeed_delay, store_above_hispeed_delay);
-
-static ssize_t show_hispeed_freq(struct kobject *kobj,
- struct attribute *attr, char *buf)
+static ssize_t show_hispeed_freq(struct cpufreq_interactive_tunables *tunables,
+ char *buf)
{
- return sprintf(buf, "%u\n", hispeed_freq);
+ return sprintf(buf, "%u\n", tunables->hispeed_freq);
}

-static ssize_t store_hispeed_freq(struct kobject *kobj,
- struct attribute *attr, const char *buf,
- size_t count)
+static ssize_t store_hispeed_freq(struct cpufreq_interactive_tunables *tunables,
+ const char *buf, size_t count)
{
int ret;
long unsigned int val;
@@ -819,22 +834,18 @@ static ssize_t store_hispeed_freq(struct kobject *kobj,
ret = strict_strtoul(buf, 0, &val);
if (ret < 0)
return ret;
- hispeed_freq = val;
+ tunables->hispeed_freq = val;
return count;
}

-static struct global_attr hispeed_freq_attr = __ATTR(hispeed_freq, 0644,
- show_hispeed_freq, store_hispeed_freq);
-
-
-static ssize_t show_go_hispeed_load(struct kobject *kobj,
- struct attribute *attr, char *buf)
+static ssize_t show_go_hispeed_load(struct cpufreq_interactive_tunables
+ *tunables, char *buf)
{
- return sprintf(buf, "%lu\n", go_hispeed_load);
+ return sprintf(buf, "%lu\n", tunables->go_hispeed_load);
}

-static ssize_t store_go_hispeed_load(struct kobject *kobj,
- struct attribute *attr, const char *buf, size_t count)
+static ssize_t store_go_hispeed_load(struct cpufreq_interactive_tunables
+ *tunables, const char *buf, size_t count)
{
int ret;
unsigned long val;
@@ -842,21 +853,18 @@ static ssize_t store_go_hispeed_load(struct kobject *kobj,
ret = strict_strtoul(buf, 0, &val);
if (ret < 0)
return ret;
- go_hispeed_load = val;
+ tunables->go_hispeed_load = val;
return count;
}

-static struct global_attr go_hispeed_load_attr = __ATTR(go_hispeed_load, 0644,
- show_go_hispeed_load, store_go_hispeed_load);
-
-static ssize_t show_min_sample_time(struct kobject *kobj,
- struct attribute *attr, char *buf)
+static ssize_t show_min_sample_time(struct cpufreq_interactive_tunables
+ *tunables, char *buf)
{
- return sprintf(buf, "%lu\n", min_sample_time);
+ return sprintf(buf, "%lu\n", tunables->min_sample_time);
}

-static ssize_t store_min_sample_time(struct kobject *kobj,
- struct attribute *attr, const char *buf, size_t count)
+static ssize_t store_min_sample_time(struct cpufreq_interactive_tunables
+ *tunables, const char *buf, size_t count)
{
int ret;
unsigned long val;
@@ -864,21 +872,18 @@ static ssize_t store_min_sample_time(struct kobject *kobj,
ret = strict_strtoul(buf, 0, &val);
if (ret < 0)
return ret;
- min_sample_time = val;
+ tunables->min_sample_time = val;
return count;
}

-static struct global_attr min_sample_time_attr = __ATTR(min_sample_time, 0644,
- show_min_sample_time, store_min_sample_time);
-
-static ssize_t show_timer_rate(struct kobject *kobj,
- struct attribute *attr, char *buf)
+static ssize_t show_timer_rate(struct cpufreq_interactive_tunables *tunables,
+ char *buf)
{
- return sprintf(buf, "%lu\n", timer_rate);
+ return sprintf(buf, "%lu\n", tunables->timer_rate);
}

-static ssize_t store_timer_rate(struct kobject *kobj,
- struct attribute *attr, const char *buf, size_t count)
+static ssize_t store_timer_rate(struct cpufreq_interactive_tunables *tunables,
+ const char *buf, size_t count)
{
int ret;
unsigned long val;
@@ -886,22 +891,18 @@ static ssize_t store_timer_rate(struct kobject *kobj,
ret = strict_strtoul(buf, 0, &val);
if (ret < 0)
return ret;
- timer_rate = val;
+ tunables->timer_rate = val;
return count;
}

-static struct global_attr timer_rate_attr = __ATTR(timer_rate, 0644,
- show_timer_rate, store_timer_rate);
-
-static ssize_t show_timer_slack(
- struct kobject *kobj, struct attribute *attr, char *buf)
+static ssize_t show_timer_slack(struct cpufreq_interactive_tunables *tunables,
+ char *buf)
{
- return sprintf(buf, "%d\n", timer_slack_val);
+ return sprintf(buf, "%d\n", tunables->timer_slack_val);
}

-static ssize_t store_timer_slack(
- struct kobject *kobj, struct attribute *attr, const char *buf,
- size_t count)
+static ssize_t store_timer_slack(struct cpufreq_interactive_tunables *tunables,
+ const char *buf, size_t count)
{
int ret;
unsigned long val;
@@ -910,19 +911,17 @@ static ssize_t store_timer_slack(
if (ret < 0)
return ret;

- timer_slack_val = val;
+ tunables->timer_slack_val = val;
return count;
}

-define_one_global_rw(timer_slack);
-
-static ssize_t show_boost(struct kobject *kobj, struct attribute *attr,
+static ssize_t show_boost(struct cpufreq_interactive_tunables *tunables,
char *buf)
{
- return sprintf(buf, "%d\n", boost_val);
+ return sprintf(buf, "%d\n", tunables->boost_val);
}

-static ssize_t store_boost(struct kobject *kobj, struct attribute *attr,
+static ssize_t store_boost(struct cpufreq_interactive_tunables *tunables,
const char *buf, size_t count)
{
int ret;
@@ -932,9 +931,9 @@ static ssize_t store_boost(struct kobject *kobj, struct attribute *attr,
if (ret < 0)
return ret;

- boost_val = val;
+ tunables->boost_val = val;

- if (boost_val) {
+ if (tunables->boost_val) {
trace_cpufreq_interactive_boost("on");
cpufreq_interactive_boost();
} else {
@@ -944,9 +943,7 @@ static ssize_t store_boost(struct kobject *kobj, struct attribute *attr,
return count;
}

-define_one_global_rw(boost);
-
-static ssize_t store_boostpulse(struct kobject *kobj, struct attribute *attr,
+static ssize_t store_boostpulse(struct cpufreq_interactive_tunables *tunables,
const char *buf, size_t count)
{
int ret;
@@ -956,24 +953,21 @@ static ssize_t store_boostpulse(struct kobject *kobj, struct attribute *attr,
if (ret < 0)
return ret;

- boostpulse_endtime = ktime_to_us(ktime_get()) + boostpulse_duration_val;
+ tunables->boostpulse_endtime = ktime_to_us(ktime_get()) +
+ tunables->boostpulse_duration_val;
trace_cpufreq_interactive_boost("pulse");
cpufreq_interactive_boost();
return count;
}

-static struct global_attr boostpulse =
- __ATTR(boostpulse, 0200, NULL, store_boostpulse);
-
-static ssize_t show_boostpulse_duration(
- struct kobject *kobj, struct attribute *attr, char *buf)
+static ssize_t show_boostpulse_duration(struct cpufreq_interactive_tunables
+ *tunables, char *buf)
{
- return sprintf(buf, "%d\n", boostpulse_duration_val);
+ return sprintf(buf, "%d\n", tunables->boostpulse_duration_val);
}

-static ssize_t store_boostpulse_duration(
- struct kobject *kobj, struct attribute *attr, const char *buf,
- size_t count)
+static ssize_t store_boostpulse_duration(struct cpufreq_interactive_tunables
+ *tunables, const char *buf, size_t count)
{
int ret;
unsigned long val;
@@ -982,20 +976,18 @@ static ssize_t store_boostpulse_duration(
if (ret < 0)
return ret;

- boostpulse_duration_val = val;
+ tunables->boostpulse_duration_val = val;
return count;
}

-define_one_global_rw(boostpulse_duration);
-
-static ssize_t show_io_is_busy(struct kobject *kobj,
- struct attribute *attr, char *buf)
+static ssize_t show_io_is_busy(struct cpufreq_interactive_tunables *tunables,
+ char *buf)
{
- return sprintf(buf, "%u\n", io_is_busy);
+ return sprintf(buf, "%u\n", tunables->io_is_busy);
}

-static ssize_t store_io_is_busy(struct kobject *kobj,
- struct attribute *attr, const char *buf, size_t count)
+static ssize_t store_io_is_busy(struct cpufreq_interactive_tunables *tunables,
+ const char *buf, size_t count)
{
int ret;
unsigned long val;
@@ -1003,33 +995,137 @@ static ssize_t store_io_is_busy(struct kobject *kobj,
ret = kstrtoul(buf, 0, &val);
if (ret < 0)
return ret;
- io_is_busy = val;
+ tunables->io_is_busy = val;
return count;
}

-static struct global_attr io_is_busy_attr = __ATTR(io_is_busy, 0644,
- show_io_is_busy, store_io_is_busy);
-
-static struct attribute *interactive_attributes[] = {
- &target_loads_attr.attr,
- &above_hispeed_delay_attr.attr,
- &hispeed_freq_attr.attr,
- &go_hispeed_load_attr.attr,
- &min_sample_time_attr.attr,
- &timer_rate_attr.attr,
- &timer_slack.attr,
- &boost.attr,
- &boostpulse.attr,
- &boostpulse_duration.attr,
- &io_is_busy_attr.attr,
+/*
+ * Create show/store routines
+ * - sys: One governor instance for complete SYSTEM
+ * - pol: One governor instance per struct cpufreq_policy
+ */
+#define show_gov_pol_sys(file_name) \
+static ssize_t show_##file_name##_gov_sys \
+(struct kobject *kobj, struct attribute *attr, char *buf) \
+{ \
+ return show_##file_name(common_tunables, buf); \
+} \
+ \
+static ssize_t show_##file_name##_gov_pol \
+(struct cpufreq_policy *policy, char *buf) \
+{ \
+ return show_##file_name(policy->governor_data, buf); \
+}
+
+#define store_gov_pol_sys(file_name) \
+static ssize_t store_##file_name##_gov_sys \
+(struct kobject *kobj, struct attribute *attr, const char *buf, \
+ size_t count) \
+{ \
+ return store_##file_name(common_tunables, buf, count); \
+} \
+ \
+static ssize_t store_##file_name##_gov_pol \
+(struct cpufreq_policy *policy, const char *buf, size_t count) \
+{ \
+ return store_##file_name(policy->governor_data, buf, count); \
+}
+
+#define show_store_gov_pol_sys(file_name) \
+show_gov_pol_sys(file_name); \
+store_gov_pol_sys(file_name)
+
+show_store_gov_pol_sys(target_loads);
+show_store_gov_pol_sys(above_hispeed_delay);
+show_store_gov_pol_sys(hispeed_freq);
+show_store_gov_pol_sys(go_hispeed_load);
+show_store_gov_pol_sys(min_sample_time);
+show_store_gov_pol_sys(timer_rate);
+show_store_gov_pol_sys(timer_slack);
+show_store_gov_pol_sys(boost);
+store_gov_pol_sys(boostpulse);
+show_store_gov_pol_sys(boostpulse_duration);
+show_store_gov_pol_sys(io_is_busy);
+
+#define gov_sys_attr_rw(_name) \
+static struct global_attr _name##_gov_sys = \
+__ATTR(_name, 0644, show_##_name##_gov_sys, store_##_name##_gov_sys)
+
+#define gov_pol_attr_rw(_name) \
+static struct freq_attr _name##_gov_pol = \
+__ATTR(_name, 0644, show_##_name##_gov_pol, store_##_name##_gov_pol)
+
+#define gov_sys_pol_attr_rw(_name) \
+ gov_sys_attr_rw(_name); \
+ gov_pol_attr_rw(_name)
+
+gov_sys_pol_attr_rw(target_loads);
+gov_sys_pol_attr_rw(above_hispeed_delay);
+gov_sys_pol_attr_rw(hispeed_freq);
+gov_sys_pol_attr_rw(go_hispeed_load);
+gov_sys_pol_attr_rw(min_sample_time);
+gov_sys_pol_attr_rw(timer_rate);
+gov_sys_pol_attr_rw(timer_slack);
+gov_sys_pol_attr_rw(boost);
+gov_sys_pol_attr_rw(boostpulse_duration);
+gov_sys_pol_attr_rw(io_is_busy);
+
+static struct global_attr boostpulse_gov_sys =
+ __ATTR(boostpulse, 0200, NULL, store_boostpulse_gov_sys);
+
+static struct freq_attr boostpulse_gov_pol =
+ __ATTR(boostpulse, 0200, NULL, store_boostpulse_gov_pol);
+
+/* One Governor instance for entire system */
+static struct attribute *interactive_attributes_gov_sys[] = {
+ &target_loads_gov_sys.attr,
+ &above_hispeed_delay_gov_sys.attr,
+ &hispeed_freq_gov_sys.attr,
+ &go_hispeed_load_gov_sys.attr,
+ &min_sample_time_gov_sys.attr,
+ &timer_rate_gov_sys.attr,
+ &timer_slack_gov_sys.attr,
+ &boost_gov_sys.attr,
+ &boostpulse_gov_sys.attr,
+ &boostpulse_duration_gov_sys.attr,
+ &io_is_busy_gov_sys.attr,
NULL,
};

-static struct attribute_group interactive_attr_group = {
- .attrs = interactive_attributes,
+static struct attribute_group interactive_attr_group_gov_sys = {
+ .attrs = interactive_attributes_gov_sys,
.name = "interactive",
};

+/* Per policy governor instance */
+static struct attribute *interactive_attributes_gov_pol[] = {
+ &target_loads_gov_pol.attr,
+ &above_hispeed_delay_gov_pol.attr,
+ &hispeed_freq_gov_pol.attr,
+ &go_hispeed_load_gov_pol.attr,
+ &min_sample_time_gov_pol.attr,
+ &timer_rate_gov_pol.attr,
+ &timer_slack_gov_pol.attr,
+ &boost_gov_pol.attr,
+ &boostpulse_gov_pol.attr,
+ &boostpulse_duration_gov_pol.attr,
+ &io_is_busy_gov_pol.attr,
+ NULL,
+};
+
+static struct attribute_group interactive_attr_group_gov_pol = {
+ .attrs = interactive_attributes_gov_pol,
+ .name = "interactive",
+};
+
+static struct attribute_group *get_sysfs_attr(void)
+{
+ if (have_governor_per_policy())
+ return &interactive_attr_group_gov_pol;
+ else
+ return &interactive_attr_group_gov_sys;
+}
+
static int cpufreq_interactive_idle_notifier(struct notifier_block *nb,
unsigned long val,
void *data)
@@ -1057,15 +1153,88 @@ static int cpufreq_governor_interactive(struct cpufreq_policy *policy,
unsigned int j;
struct cpufreq_interactive_cpuinfo *pcpu;
struct cpufreq_frequency_table *freq_table;
+ struct cpufreq_interactive_tunables *tunables;
+
+ if (have_governor_per_policy())
+ tunables = policy->governor_data;
+ else
+ tunables = common_tunables;
+
+ WARN_ON(!tunables && (event != CPUFREQ_GOV_POLICY_INIT));

switch (event) {
+ case CPUFREQ_GOV_POLICY_INIT:
+ if (have_governor_per_policy()) {
+ WARN_ON(tunables);
+ } else if (tunables) {
+ tunables->usage_count++;
+ policy->governor_data = tunables;
+ return 0;
+ }
+
+ tunables = kzalloc(sizeof(*tunables), GFP_KERNEL);
+ if (!tunables) {
+ pr_err("%s: POLICY_INIT: kzalloc failed\n", __func__);
+ return -ENOMEM;
+ }
+
+ rc = sysfs_create_group(get_governor_parent_kobj(policy),
+ get_sysfs_attr());
+ if (rc) {
+ kfree(tunables);
+ return rc;
+ }
+
+ tunables->usage_count = 1;
+ tunables->above_hispeed_delay = default_above_hispeed_delay;
+ tunables->nabove_hispeed_delay =
+ ARRAY_SIZE(default_above_hispeed_delay);
+ tunables->go_hispeed_load = DEFAULT_GO_HISPEED_LOAD;
+ tunables->target_loads = default_target_loads;
+ tunables->ntarget_loads = ARRAY_SIZE(default_target_loads);
+ tunables->min_sample_time = DEFAULT_MIN_SAMPLE_TIME;
+ tunables->timer_rate = DEFAULT_TIMER_RATE;
+ tunables->boostpulse_duration_val = DEFAULT_MIN_SAMPLE_TIME;
+ tunables->timer_slack_val = DEFAULT_TIMER_SLACK;
+
+ spin_lock_init(&tunables->target_loads_lock);
+ spin_lock_init(&tunables->above_hispeed_delay_lock);
+
+ if (!policy->governor->initialized) {
+ idle_notifier_register(&cpufreq_interactive_idle_nb);
+ cpufreq_register_notifier(&cpufreq_notifier_block,
+ CPUFREQ_TRANSITION_NOTIFIER);
+ }
+
+ policy->governor_data = tunables;
+ if (!have_governor_per_policy())
+ common_tunables = tunables;
+
+ break;
+
+ case CPUFREQ_GOV_POLICY_EXIT:
+ if (!--tunables->usage_count) {
+ if (policy->governor->initialized == 1) {
+ cpufreq_unregister_notifier(&cpufreq_notifier_block,
+ CPUFREQ_TRANSITION_NOTIFIER);
+ idle_notifier_unregister(&cpufreq_interactive_idle_nb);
+ }
+
+ sysfs_remove_group(get_governor_parent_kobj(policy),
+ get_sysfs_attr());
+ kfree(tunables);
+ common_tunables = NULL;
+ }
+
+ policy->governor_data = NULL;
+ break;
+
case CPUFREQ_GOV_START:
mutex_lock(&gov_lock);

- freq_table =
- cpufreq_frequency_get_table(policy->cpu);
- if (!hispeed_freq)
- hispeed_freq = policy->max;
+ freq_table = cpufreq_frequency_get_table(policy->cpu);
+ if (!tunables->hispeed_freq)
+ tunables->hispeed_freq = policy->max;

for_each_cpu(j, policy->cpus) {
pcpu = &per_cpu(cpuinfo, j);
@@ -1078,30 +1247,11 @@ static int cpufreq_governor_interactive(struct cpufreq_policy *policy,
pcpu->hispeed_validate_time =
pcpu->floor_validate_time;
down_write(&pcpu->enable_sem);
- cpufreq_interactive_timer_start(j);
+ cpufreq_interactive_timer_start(tunables, j);
pcpu->governor_enabled = 1;
up_write(&pcpu->enable_sem);
}

- /*
- * Do not register the idle hook and create sysfs
- * entries if we have already done so.
- */
- if (++active_count > 1) {
- mutex_unlock(&gov_lock);
- return 0;
- }
-
- rc = sysfs_create_group(cpufreq_global_kobject,
- &interactive_attr_group);
- if (rc) {
- mutex_unlock(&gov_lock);
- return rc;
- }
-
- idle_notifier_register(&cpufreq_interactive_idle_nb);
- cpufreq_register_notifier(
- &cpufreq_notifier_block, CPUFREQ_TRANSITION_NOTIFIER);
mutex_unlock(&gov_lock);
break;

@@ -1116,18 +1266,7 @@ static int cpufreq_governor_interactive(struct cpufreq_policy *policy,
up_write(&pcpu->enable_sem);
}

- if (--active_count > 0) {
- mutex_unlock(&gov_lock);
- return 0;
- }
-
- cpufreq_unregister_notifier(
- &cpufreq_notifier_block, CPUFREQ_TRANSITION_NOTIFIER);
- idle_notifier_unregister(&cpufreq_interactive_idle_nb);
- sysfs_remove_group(cpufreq_global_kobject,
- &interactive_attr_group);
mutex_unlock(&gov_lock);
-
break;

case CPUFREQ_GOV_LIMITS:
@@ -1161,7 +1300,7 @@ static int cpufreq_governor_interactive(struct cpufreq_policy *policy,
*/
del_timer_sync(&pcpu->cpu_timer);
del_timer_sync(&pcpu->cpu_slack_timer);
- cpufreq_interactive_timer_start(j);
+ cpufreq_interactive_timer_start(tunables, j);
up_write(&pcpu->enable_sem);
}
break;
@@ -1201,9 +1340,7 @@ static int __init cpufreq_interactive_init(void)
init_rwsem(&pcpu->enable_sem);
}

- spin_lock_init(&target_loads_lock);
spin_lock_init(&speedchange_cpumask_lock);
- spin_lock_init(&above_hispeed_delay_lock);
mutex_init(&gov_lock);
speedchange_task =
kthread_create(cpufreq_interactive_speedchange_task, NULL,
--
1.7.9.5

--
To unsubscribe from this list: send the line "unsubscribe linux-kernel" in
the body of a message to majordomo@xxxxxxxxxxxxxxx
More majordomo info at http://vger.kernel.org/majordomo-info.html
Please read the FAQ at http://www.tux.org/lkml/