[PATCH] sched/idle: Make forced polling state race-safe

[Hui Su]

cpu_idle_poll_ctrl() maintains a global nesting count for forced idle
polling. The count is updated with plain increments and decrements while
idle CPUs read it without synchronization.

Use an atomic counter to make the nesting count safe against concurrent
updates and to make idle-side accesses race-free.

The weak arch_cpu_idle() fallback also permanently switches the generic
idle path to polling. Keep that state separate from the temporary
request counter so a concurrent request cannot overwrite, or later
release, the fallback state.

Fixes: d16699123434 ("idle: Implement generic idle function")
Signed-off-by: Hui Su <sh_def@xxxxxxx>
---
Testing:
- On x86_64, a two-worker KCSAN harness reported the data race and ended at
  -14565 before the change; 100,000 balanced requests ended at zero with no
  target report afterward.
- The KCSAN/noinstr build completed without objtool warnings. A focused test
  confirmed that polling remained enabled after the temporary count returned
  to zero; the weak fallback itself was not exercised on x86.
---
 kernel/sched/idle.c | 24 +++++++++++++++---------
 1 file changed, 15 insertions(+), 9 deletions(-)

diff --git a/kernel/sched/idle.c b/kernel/sched/idle.c
index eb73b65ce6c4..84b02db65828 100644
--- a/kernel/sched/idle.c
+++ b/kernel/sched/idle.c
@@ -6,6 +6,7 @@
  * (NOTE: these are not related to SCHED_IDLE batch scheduled
  *        tasks which are handled in sched/fair.c )
  */
+#include <linux/atomic.h>
 #include <linux/cpuidle.h>
 #include <linux/suspend.h>
 #include <linux/livepatch.h>
@@ -24,22 +25,27 @@ void sched_idle_set_state(struct cpuidle_state *idle_state)
 	idle_set_state(this_rq(), idle_state);
 }
 
-static int __read_mostly cpu_idle_force_poll;
+static atomic_t __read_mostly cpu_idle_force_poll = ATOMIC_INIT(0);
+static bool __read_mostly cpu_idle_force_poll_sticky;
+
+static __always_inline bool cpu_idle_force_polling(void)
+{
+	return raw_atomic_read(&cpu_idle_force_poll) || READ_ONCE(cpu_idle_force_poll_sticky);
+}
 
 void cpu_idle_poll_ctrl(bool enable)
 {
 	if (enable) {
-		cpu_idle_force_poll++;
+		atomic_inc(&cpu_idle_force_poll);
 	} else {
-		cpu_idle_force_poll--;
-		WARN_ON_ONCE(cpu_idle_force_poll < 0);
+		WARN_ON_ONCE(atomic_dec_return(&cpu_idle_force_poll) < 0);
 	}
 }
 
 #ifdef CONFIG_GENERIC_IDLE_POLL_SETUP
 static int __init cpu_idle_poll_setup(char *__unused)
 {
-	cpu_idle_force_poll = 1;
+	atomic_set(&cpu_idle_force_poll, 1);
 
 	return 1;
 }
@@ -47,7 +53,7 @@ __setup("nohlt", cpu_idle_poll_setup);
 
 static int __init cpu_idle_nopoll_setup(char *__unused)
 {
-	cpu_idle_force_poll = 0;
+	atomic_set(&cpu_idle_force_poll, 0);
 
 	return 1;
 }
@@ -63,7 +69,7 @@ static noinline int __cpuidle cpu_idle_poll(void)
 
 	raw_local_irq_enable();
 	while (!tif_need_resched() &&
-	       (cpu_idle_force_poll || tick_check_broadcast_expired()))
+	       (cpu_idle_force_polling() || tick_check_broadcast_expired()))
 		cpu_relax();
 	raw_local_irq_disable();
 
@@ -83,7 +89,7 @@ void __weak arch_cpu_idle_exit(void) { }
 void __weak __noreturn arch_cpu_idle_dead(void) { while (1); }
 void __weak arch_cpu_idle(void)
 {
-	cpu_idle_force_poll = 1;
+	WRITE_ONCE(cpu_idle_force_poll_sticky, true);
 }
 
 #ifdef CONFIG_GENERIC_CLOCKEVENTS_BROADCAST_IDLE
@@ -348,7 +354,7 @@ static void do_idle(void)
 		 * broadcast device expired for us, we don't want to go deep
 		 * idle as we know that the IPI is going to arrive right away.
 		 */
-		if (cpu_idle_force_poll || tick_check_broadcast_expired()) {
+		if (cpu_idle_force_polling() || tick_check_broadcast_expired()) {
 			tick_nohz_idle_restart_tick();
 			cpu_idle_poll();
 		} else {

base-commit: 0fcb05b72344f40806cbe810de8872f1698bf053
-- 
2.55.0