[PATCH v2 2/2] cpuidle: Respect latency constraints when accounting below misses
From: Christian Loehle
Date: Wed Nov 11 2026 - 07:11:06 EST
Only count a below miss when an enabled deeper state meets the latency
constraint.
Sample the limit once in cpuidle_select() and share it between governors
and accounting since a separate read before entry can already differ
from the value used for selection.
Continue checking disable flags on exit only, as they're not expected to
change as frequently.
Fixes: 04dab58a39d4 ("cpuidle: Add 'above' and 'below' idle state metrics")
Signed-off-by: Christian Loehle <christian.loehle@xxxxxxx>
---
Documentation/driver-api/pm/cpuidle.rst | 13 ++++++-------
drivers/cpuidle/cpuidle.c | 8 +++++++-
drivers/cpuidle/cpuidle.h | 1 +
drivers/cpuidle/governors/haltpoll.c | 2 +-
drivers/cpuidle/governors/ladder.c | 2 +-
drivers/cpuidle/governors/menu.c | 2 +-
drivers/cpuidle/governors/teo.c | 2 +-
include/linux/cpuidle.h | 2 +-
8 files changed, 19 insertions(+), 13 deletions(-)
diff --git a/Documentation/driver-api/pm/cpuidle.rst b/Documentation/driver-api/pm/cpuidle.rst
index d477208604b8..286061495ce7 100644
--- a/Documentation/driver-api/pm/cpuidle.rst
+++ b/Documentation/driver-api/pm/cpuidle.rst
@@ -146,13 +146,12 @@ callbacks:
In addition, ``CPUIdle`` governors are required to take power management
quality of service (PM QoS) constraints on the processor wakeup latency into
-account when selecting idle states. In order to obtain the current effective
-PM QoS wakeup latency constraint for a given CPU, a ``CPUIdle`` governor is
-expected to pass the number of the CPU to
-:c:func:`cpuidle_governor_latency_req()`. Then, the governor's ``->select()``
-callback must not return the index of an indle state whose
-:c:member:`exit_latency` value is greater than the number returned by that
-function.
+account when selecting idle states. Before invoking the governor's
+``->select()`` callback, the core saves the effective PM QoS wakeup latency
+constraint in ``dev->latency_req``, in nanoseconds. Governors use this value
+to constrain the :c:member:`exit_latency_ns` of the selected idle state.
+The core also uses the saved value for below miss accounting, so subsequent
+PM QoS updates do not change the constraint used to evaluate that selection.
CPU Idle Time Management Drivers
diff --git a/drivers/cpuidle/cpuidle.c b/drivers/cpuidle/cpuidle.c
index faa29f58738e..9af704473f6e 100644
--- a/drivers/cpuidle/cpuidle.c
+++ b/drivers/cpuidle/cpuidle.c
@@ -131,12 +131,15 @@ void cpuidle_use_deepest_state(u64 latency_limit_ns)
* @dev: cpuidle device for the given CPU.
* @latency_limit_ns: Idle state exit latency limit
*
+ * Save the limit for miss accounting on the subsequent idle entry.
+ *
* Return: the index of the deepest available idle state.
*/
int cpuidle_find_deepest_state(struct cpuidle_driver *drv,
struct cpuidle_device *dev,
u64 latency_limit_ns)
{
+ dev->latency_req = min_t(u64, latency_limit_ns, S64_MAX);
return find_deepest_state(drv, dev, latency_limit_ns, 0, false);
}
@@ -327,7 +330,8 @@ noinstr int cpuidle_enter_state(struct cpuidle_device *dev,
}
} else if (diff > delay) {
for (i = entered_state + 1; i < drv->state_count; i++) {
- if (dev->states_usage[i].disable)
+ if (dev->states_usage[i].disable ||
+ drv->states[i].exit_latency_ns > dev->latency_req)
continue;
/*
@@ -369,6 +373,8 @@ noinstr int cpuidle_enter_state(struct cpuidle_device *dev,
int cpuidle_select(struct cpuidle_driver *drv, struct cpuidle_device *dev,
bool *stop_tick)
{
+ /* Use the same constraint for state selection and miss accounting. */
+ dev->latency_req = cpuidle_governor_latency_req(dev->cpu);
return cpuidle_curr_governor->select(drv, dev, stop_tick);
}
diff --git a/drivers/cpuidle/cpuidle.h b/drivers/cpuidle/cpuidle.h
index 52701d9588f1..3a00d8db6d28 100644
--- a/drivers/cpuidle/cpuidle.h
+++ b/drivers/cpuidle/cpuidle.h
@@ -23,6 +23,7 @@ extern void cpuidle_install_idle_handler(void);
extern void cpuidle_uninstall_idle_handler(void);
/* governors */
+s64 cpuidle_governor_latency_req(unsigned int cpu);
extern struct cpuidle_governor *cpuidle_find_governor(const char *str);
extern int cpuidle_switch_governor(struct cpuidle_governor *gov);
diff --git a/drivers/cpuidle/governors/haltpoll.c b/drivers/cpuidle/governors/haltpoll.c
index b367d10279c8..0ab38f093ef1 100644
--- a/drivers/cpuidle/governors/haltpoll.c
+++ b/drivers/cpuidle/governors/haltpoll.c
@@ -50,7 +50,7 @@ static int haltpoll_select(struct cpuidle_driver *drv,
struct cpuidle_device *dev,
bool *stop_tick)
{
- if (cpuidle_governor_latency_req(dev->cpu) == 0) {
+ if (dev->latency_req == 0) {
*stop_tick = false;
return 0;
}
diff --git a/drivers/cpuidle/governors/ladder.c b/drivers/cpuidle/governors/ladder.c
index 6617eb494a11..67e0e98e699e 100644
--- a/drivers/cpuidle/governors/ladder.c
+++ b/drivers/cpuidle/governors/ladder.c
@@ -71,7 +71,7 @@ static int ladder_select_state(struct cpuidle_driver *drv,
struct ladder_device_state *last_state;
int last_idx = dev->last_state_idx;
int first_idx = drv->states[0].flags & CPUIDLE_FLAG_POLLING ? 1 : 0;
- s64 latency_req = cpuidle_governor_latency_req(dev->cpu);
+ s64 latency_req = dev->latency_req;
s64 last_residency;
/* Special case when user has set very strict latency requirement */
diff --git a/drivers/cpuidle/governors/menu.c b/drivers/cpuidle/governors/menu.c
index eb529d73d4b7..e0145417ffe7 100644
--- a/drivers/cpuidle/governors/menu.c
+++ b/drivers/cpuidle/governors/menu.c
@@ -219,7 +219,7 @@ static int menu_select(struct cpuidle_driver *drv, struct cpuidle_device *dev,
bool *stop_tick)
{
struct menu_device *data = this_cpu_ptr(&menu_devices);
- s64 latency_req = cpuidle_governor_latency_req(dev->cpu);
+ s64 latency_req = dev->latency_req;
u64 predicted_ns;
ktime_t delta, delta_tick;
int i, idx;
diff --git a/drivers/cpuidle/governors/teo.c b/drivers/cpuidle/governors/teo.c
index 906f4f89e365..735894a1a7fc 100644
--- a/drivers/cpuidle/governors/teo.c
+++ b/drivers/cpuidle/governors/teo.c
@@ -309,7 +309,7 @@ static int teo_select(struct cpuidle_driver *drv, struct cpuidle_device *dev,
bool *stop_tick)
{
struct teo_cpu *cpu_data = this_cpu_ptr(&teo_cpus);
- s64 latency_req = cpuidle_governor_latency_req(dev->cpu);
+ s64 latency_req = dev->latency_req;
ktime_t delta_tick = TICK_NSEC / 2;
unsigned int idx_intercept_sum = 0;
unsigned int intercept_sum = 0;
diff --git a/include/linux/cpuidle.h b/include/linux/cpuidle.h
index a2485348def3..14ba5e1f3cdb 100644
--- a/include/linux/cpuidle.h
+++ b/include/linux/cpuidle.h
@@ -101,6 +101,7 @@ struct cpuidle_device {
u64 last_residency_ns;
u64 poll_limit_ns;
u64 forced_idle_latency_limit_ns;
+ s64 latency_req;
struct cpuidle_state_usage states_usage[CPUIDLE_STATE_MAX];
struct cpuidle_state_kobj *kobjs[CPUIDLE_STATE_MAX];
struct cpuidle_driver_kobj *kobj_driver;
@@ -306,7 +307,6 @@ struct cpuidle_governor {
};
extern int cpuidle_register_governor(struct cpuidle_governor *gov);
-extern s64 cpuidle_governor_latency_req(unsigned int cpu);
#define __CPU_PM_CPU_IDLE_ENTER(low_level_idle_enter, \
idx, \
--
2.34.1