[tip: sched/core] sched: Replace nr_iowait_cpu() with a bool accessor
From: tip-bot2 for Christian Loehle
Date: Fri Oct 16 2026 - 04:03:23 EST
The following commit has been merged into the sched/core branch of tip:
Commit-ID: b99b7f60c5b473d374c68b53753153b4aa9e0702
Gitweb: https://git.kernel.org/tip/b99b7f60c5b473d374c68b53753153b4aa9e0702
Author: Christian Loehle <christian.loehle@xxxxxxx>
AuthorDate: Fri, 16 Oct 2026 08:33:40 +01:00
Committer: Peter Zijlstra <peterz@xxxxxxxxxxxxx>
CommitterDate: Fri, 16 Oct 2026 09:49:04 +02:00
sched: Replace nr_iowait_cpu() with a bool accessor
The menu governor was the last consumer of the per-CPU I/O-wait count
that needed its magnitude, apart from nr_iowait() summing it globally.
Commit 38f83090f515 ("cpuidle: menu: Remove iowait influence") removed
that use. All remaining per-CPU consumers only test for zero/nonzero.
Replace nr_iowait_cpu() with bool cpu_has_iowait() and update the generic
and s390 idle accounting callers. Have nr_iowait() read the counters
directly so that the global total and existing accounting are preserved.
Replace the stale menu comment with a description of the predicate.
Also convert the remaining open-coded idle accounting checks to use the
predicate, as suggested by Prateek.
The conversion was done with coccinelle:
@ predicate depends on patch @
expression cpu;
@@
- nr_iowait_cpu(cpu) > 0
+ cpu_has_iowait(cpu)
@ callers depends on patch @
expression cpu;
@@
- nr_iowait_cpu(cpu)
+ cpu_has_iowait(cpu)
Signed-off-by: Christian Loehle <christian.loehle@xxxxxxx>
Signed-off-by: Peter Zijlstra (Intel) <peterz@xxxxxxxxxxxxx>
Reviewed-by: Kayra Cizmeci <kayracizmeci@xxxxxxxxx>
Reviewed-by: K Prateek Nayak <kprateek.nayak@xxxxxxx>
Tested-by: K Prateek Nayak <kprateek.nayak@xxxxxxx>
Link: https://patch.msgid.link/20261016073341.575472-1-christian.loehle@xxxxxxx
---
arch/s390/kernel/idle.c | 4 ++--
include/linux/sched/stat.h | 2 +-
kernel/sched/core.c | 13 +++++--------
kernel/sched/cputime.c | 20 +++++++++-----------
4 files changed, 17 insertions(+), 22 deletions(-)
diff --git a/arch/s390/kernel/idle.c b/arch/s390/kernel/idle.c
index dd12efb..ff9732e 100644
--- a/arch/s390/kernel/idle.c
+++ b/arch/s390/kernel/idle.c
@@ -82,13 +82,13 @@ static u64 arch_cpu_idle_time(int cpu, enum cpu_usage_stat idx, bool compute_del
u64 arch_kcpustat_field_idle(int cpu)
{
- return arch_cpu_idle_time(cpu, CPUTIME_IDLE, !nr_iowait_cpu(cpu));
+ return arch_cpu_idle_time(cpu, CPUTIME_IDLE, !cpu_has_iowait(cpu));
}
EXPORT_SYMBOL_GPL(arch_kcpustat_field_idle);
u64 arch_kcpustat_field_iowait(int cpu)
{
- return arch_cpu_idle_time(cpu, CPUTIME_IOWAIT, nr_iowait_cpu(cpu));
+ return arch_cpu_idle_time(cpu, CPUTIME_IOWAIT, cpu_has_iowait(cpu));
}
EXPORT_SYMBOL_GPL(arch_kcpustat_field_iowait);
diff --git a/include/linux/sched/stat.h b/include/linux/sched/stat.h
index 0108a38..4810b14 100644
--- a/include/linux/sched/stat.h
+++ b/include/linux/sched/stat.h
@@ -20,7 +20,7 @@ extern int nr_processes(void);
extern unsigned int nr_running(void);
extern bool single_task_running(void);
extern unsigned int nr_iowait(void);
-extern unsigned int nr_iowait_cpu(int cpu);
+extern bool cpu_has_iowait(int cpu);
static inline int sched_info_on(void)
{
diff --git a/kernel/sched/core.c b/kernel/sched/core.c
index 047fbe8..d7c3b43 100644
--- a/kernel/sched/core.c
+++ b/kernel/sched/core.c
@@ -5638,15 +5638,12 @@ unsigned long long nr_context_switches(void)
}
/*
- * Consumers of these two interfaces, like for example the cpuidle menu
- * governor, are using nonsensical data. Preferring shallow idle state selection
- * for a CPU that has IO-wait which might not even end up running the task when
- * it does become runnable.
+ * Return whether any tasks are accounted as I/O waiters on this CPU.
+ * They may wake up on a different CPU.
*/
-
-unsigned int nr_iowait_cpu(int cpu)
+bool cpu_has_iowait(int cpu)
{
- return atomic_read(&cpu_rq(cpu)->nr_iowait);
+ return atomic_read(&cpu_rq(cpu)->nr_iowait) != 0;
}
/*
@@ -5684,7 +5681,7 @@ unsigned int nr_iowait(void)
unsigned int i, sum = 0;
for_each_possible_cpu(i)
- sum += nr_iowait_cpu(i);
+ sum += atomic_read(&cpu_rq(i)->nr_iowait);
return sum;
}
diff --git a/kernel/sched/cputime.c b/kernel/sched/cputime.c
index 19aa804..9e0fb86 100644
--- a/kernel/sched/cputime.c
+++ b/kernel/sched/cputime.c
@@ -226,9 +226,8 @@ void account_steal_time(u64 cputime)
void account_idle_time(u64 cputime)
{
u64 *cpustat = kcpustat_this_cpu->cpustat;
- struct rq *rq = this_rq();
- if (atomic_read(&rq->nr_iowait) > 0)
+ if (cpu_has_iowait(smp_processor_id()))
cpustat[CPUTIME_IOWAIT] += cputime;
else
cpustat[CPUTIME_IDLE] += cputime;
@@ -431,7 +430,7 @@ static void kcpustat_idle_stop(struct kernel_cpustat *kc, u64 now)
if (!kc->idle_elapse)
return;
- iowait = nr_iowait_cpu(smp_processor_id()) > 0;
+ iowait = cpu_has_iowait(smp_processor_id());
delta = now - kc->idle_entrytime;
steal = steal_account_process_time(delta);
@@ -565,14 +564,14 @@ static u64 kcpustat_field_dyntick(int cpu, enum cpu_usage_stat idx,
u64 kcpustat_field_idle(int cpu)
{
return kcpustat_field_dyntick(cpu, CPUTIME_IDLE,
- !nr_iowait_cpu(cpu), ktime_get());
+ !cpu_has_iowait(cpu), ktime_get());
}
EXPORT_SYMBOL_GPL(kcpustat_field_idle);
u64 kcpustat_field_iowait(int cpu)
{
return kcpustat_field_dyntick(cpu, CPUTIME_IOWAIT,
- nr_iowait_cpu(cpu), ktime_get());
+ cpu_has_iowait(cpu), ktime_get());
}
EXPORT_SYMBOL_GPL(kcpustat_field_iowait);
#else
@@ -627,7 +626,7 @@ static u64 get_cpu_sleep_time_us(int cpu, enum cpu_usage_stat idx,
u64 get_cpu_idle_time_us(int cpu, u64 *last_update_time)
{
return get_cpu_sleep_time_us(cpu, CPUTIME_IDLE,
- !nr_iowait_cpu(cpu), last_update_time);
+ !cpu_has_iowait(cpu), last_update_time);
}
EXPORT_SYMBOL_GPL(get_cpu_idle_time_us);
@@ -651,7 +650,7 @@ EXPORT_SYMBOL_GPL(get_cpu_idle_time_us);
u64 get_cpu_iowait_time_us(int cpu, u64 *last_update_time)
{
return get_cpu_sleep_time_us(cpu, CPUTIME_IOWAIT,
- nr_iowait_cpu(cpu), last_update_time);
+ cpu_has_iowait(cpu), last_update_time);
}
EXPORT_SYMBOL_GPL(get_cpu_iowait_time_us);
@@ -1136,7 +1135,6 @@ static int kcpustat_field_vtime(u64 *cpustat,
int cpu, u64 *val)
{
struct vtime *vtime = &tsk->vtime;
- struct rq *rq = cpu_rq(cpu);
unsigned int seq;
do {
@@ -1179,11 +1177,11 @@ static int kcpustat_field_vtime(u64 *cpustat,
*val += vtime->gtime + vtime_delta(vtime);
break;
case CPUTIME_IDLE:
- if (state == VTIME_IDLE && !atomic_read(&rq->nr_iowait))
+ if (state == VTIME_IDLE && !cpu_has_iowait(cpu))
*val += vtime_delta(vtime);
break;
case CPUTIME_IOWAIT:
- if (state == VTIME_IDLE && atomic_read(&rq->nr_iowait) > 0)
+ if (state == VTIME_IDLE && cpu_has_iowait(cpu))
*val += vtime_delta(vtime);
break;
default:
@@ -1278,7 +1276,7 @@ static int kcpustat_cpu_fetch_vtime(struct kernel_cpustat *dst,
}
break;
case VTIME_IDLE:
- if (atomic_read(&cpu_rq(cpu)->nr_iowait) > 0)
+ if (cpu_has_iowait(cpu))
cpustat[CPUTIME_IOWAIT] += delta;
else
cpustat[CPUTIME_IDLE] += delta;