[PATCH 4/5] cpuidle: cache aggregated governor latency QoS constraint

[Yaxiong Tian]

cpuidle_governor_latency_req() runs on every idle-state selection and
aggregates per-CPU resume latency with the global CPU latency and
wakeup latency QoS limits.  That path repeatedly walks get_cpu_device()
and pm_qos_read_value(), which shows up hot under menu_select().

Cache the aggregated constraint per CPU and refresh it only when the
corresponding per-CPU generation changes.  The generation is already
bumped by the global and per-CPU resume latency QoS notifiers added
earlier in this series.

On a menu governor profile, cpuidle_governor_latency_req() drops from
about 19.9% of menu_select time to about 4.2%, roughly a 6x reduction
in cost per call (~1.9 us down to ~0.3 us).

Signed-off-by: Yaxiong Tian <tianyaxiong@xxxxxxxxxx>
---
 drivers/cpuidle/governor.c | 37 ++++++++++++++++++++++++++++++++-----
 1 file changed, 32 insertions(+), 5 deletions(-)

diff --git a/drivers/cpuidle/governor.c b/drivers/cpuidle/governor.c
index d286ccf19a69..67909470c14b 100644
--- a/drivers/cpuidle/governor.c
+++ b/drivers/cpuidle/governor.c
@@ -32,11 +32,17 @@ struct cpuidle_governor *cpuidle_prev_governor;
  */
 static DEFINE_PER_CPU(atomic_t, latency_req_gen);
 
+struct cpuidle_latency_req_cache {
+	unsigned int gen;
+	s64 latency_ns;
+};
+
 struct cpuidle_cpu_qos_nb {
 	struct notifier_block nb;
 	unsigned int cpu;
 };
 
+static DEFINE_PER_CPU(struct cpuidle_latency_req_cache, latency_req_cache);
 static DEFINE_PER_CPU(struct cpuidle_cpu_qos_nb, cpuidle_cpu_qos_nb);
 
 static void cpuidle_latency_req_invalidate_cpu(unsigned int cpu)
@@ -212,10 +218,22 @@ int cpuidle_register_governor(struct cpuidle_governor *gov)
  */
 s64 cpuidle_governor_latency_req(unsigned int cpu)
 {
-	struct device *device = get_cpu_device(cpu);
-	int device_req = dev_pm_qos_raw_resume_latency(device);
-	int global_req = cpu_latency_qos_limit();
-	int global_wake_req = cpu_wakeup_latency_qos_limit();
+	struct cpuidle_latency_req_cache *cache;
+	unsigned int gen;
+	struct device *device;
+	int device_req, global_req, global_wake_req;
+	s64 latency_ns;
+
+	cache = per_cpu_ptr(&latency_req_cache, cpu);
+	gen = atomic_read(per_cpu_ptr(&latency_req_gen, cpu));
+
+	if (likely(READ_ONCE(cache->gen) == gen))
+		return READ_ONCE(cache->latency_ns);
+
+	device = get_cpu_device(cpu);
+	device_req = dev_pm_qos_raw_resume_latency(device);
+	global_req = cpu_latency_qos_limit();
+	global_wake_req = cpu_wakeup_latency_qos_limit();
 
 	if (global_req > global_wake_req)
 		global_req = global_wake_req;
@@ -223,5 +241,14 @@ s64 cpuidle_governor_latency_req(unsigned int cpu)
 	if (device_req > global_req)
 		device_req = global_req;
 
-	return (s64)device_req * NSEC_PER_USEC;
+	latency_ns = (s64)device_req * NSEC_PER_USEC;
+
+	WRITE_ONCE(cache->latency_ns, latency_ns);
+	/*
+	 * Store gen last so a concurrent invalidate cannot leave a stale
+	 * latency_ns marked as current.
+	 */
+	WRITE_ONCE(cache->gen, gen);
+
+	return latency_ns;
 }
-- 
2.43.0