[PATCH RFC 1/1] sched: Replace nr_pinned offset hack with a dedicated per-CPU counter

[Qiurong Fang]

From: fangqiurong <fangqiurong@xxxxxxxxxx>

The inlined migrate_{en,dis}able() in include/linux/sched.h cannot see
struct rq, so they bump rq->nr_pinned through a generated offset
constant, dragging along kernel/sched/rq-offsets.c, its own Kbuild rule,
a global #include <generated/rq-offsets.h> from sched.h and an
arch_raw_cpu_ptr()/PERCPU_PTR() workaround that casts away the field
type.

Replace it with a plain per-CPU counter, rq_nr_pinned: __this_cpu_inc()/
__this_cpu_dec() in the inline writers (already preempt-disabled there)
and per_cpu_ptr() + READ_ONCE() in rq_has_pinned_tasks(). Codegen and
runtime behaviour are unchanged and modules keep the exported wrappers;
the missing-syscalls check is re-anchored to the regular asm-offsets
file.

Signed-off-by: fangqiurong <fangqiurong@xxxxxxxxxx>
---
 Kbuild                    | 13 +------------
 include/linux/sched.h     | 30 +++---------------------------
 kernel/sched/core.c       |  3 ++-
 kernel/sched/rq-offsets.c | 12 ------------
 kernel/sched/sched.h      |  1 -
 5 files changed, 6 insertions(+), 53 deletions(-)
 delete mode 100644 kernel/sched/rq-offsets.c

diff --git a/Kbuild b/Kbuild
index a6a0192dea08..984765c9425f 100644
--- a/Kbuild
+++ b/Kbuild
@@ -34,17 +34,6 @@ arch/$(SRCARCH)/kernel/asm-offsets.s: $(timeconst-file) $(bounds-file)
 $(offsets-file): arch/$(SRCARCH)/kernel/asm-offsets.s FORCE
 	$(call filechk,offsets,__ASM_OFFSETS_H__)
 
-# Generate rq-offsets.h
-
-rq-offsets-file := include/generated/rq-offsets.h
-
-targets += kernel/sched/rq-offsets.s
-
-kernel/sched/rq-offsets.s: $(offsets-file)
-
-$(rq-offsets-file): kernel/sched/rq-offsets.s FORCE
-	$(call filechk,offsets,__RQ_OFFSETS_H__)
-
 # Check for missing system calls
 
 missing-syscalls-file := .tmp_missing-syscalls$(missing_syscalls_instance)
@@ -54,7 +43,7 @@ targets += $(missing-syscalls-file)
 quiet_cmd_syscalls = CALL    $< $(addprefix for ,$(missing_syscalls_instance))
       cmd_syscalls = DEPFILE=$(depfile) $(CONFIG_SHELL) $< $(CC) $(c_flags) $(missing_syscalls_flags); touch $@
 
-$(missing-syscalls-file): scripts/checksyscalls.sh $(rq-offsets-file) FORCE
+$(missing-syscalls-file): scripts/checksyscalls.sh $(offsets-file) FORCE
 	$(call if_changed_dep,syscalls)
 
 PHONY += missing-syscalls
diff --git a/include/linux/sched.h b/include/linux/sched.h
index 373bcc0598d1..413e6a29d1b1 100644
--- a/include/linux/sched.h
+++ b/include/linux/sched.h
@@ -51,9 +51,6 @@
 #include <linux/unwind_deferred_types.h>
 #include <asm/kmap_size.h>
 #include <linux/time64.h>
-#ifndef COMPILE_OFFSETS
-#include <generated/rq-offsets.h>
-#endif
 
 /* task_struct member predeclarations (sorted alphabetically): */
 struct audit_context;
@@ -2407,27 +2404,10 @@ struct sched_cache_stat { };
 #endif
 
 #ifndef MODULE
-#ifndef COMPILE_OFFSETS
 
 extern void ___migrate_enable(void);
 
-struct rq;
-DECLARE_PER_CPU_SHARED_ALIGNED(struct rq, runqueues);
-
-/*
- * The "struct rq" is not available here, so we can't access the
- * "runqueues" with this_cpu_ptr(), as the compilation will fail in
- * this_cpu_ptr() -> raw_cpu_ptr() -> __verify_pcpu_ptr():
- *   typeof((ptr) + 0)
- *
- * So use arch_raw_cpu_ptr()/PERCPU_PTR() directly here.
- */
-#ifdef CONFIG_SMP
-#define this_rq_raw() arch_raw_cpu_ptr(&runqueues)
-#else
-#define this_rq_raw() PERCPU_PTR(&runqueues)
-#endif
-#define this_rq_pinned() (*(unsigned int *)((void *)this_rq_raw() + RQ_nr_pinned))
+DECLARE_PER_CPU(unsigned int, rq_nr_pinned);
 
 static inline void __migrate_enable(void)
 {
@@ -2461,7 +2441,7 @@ static inline void __migrate_enable(void)
 	 */
 	barrier();
 	p->migration_disabled = 0;
-	this_rq_pinned()--;
+	__this_cpu_dec(rq_nr_pinned);
 }
 
 static inline void __migrate_disable(void)
@@ -2480,13 +2460,9 @@ static inline void __migrate_disable(void)
 	}
 
 	guard(preempt)();
-	this_rq_pinned()++;
+	__this_cpu_inc(rq_nr_pinned);
 	p->migration_disabled = 1;
 }
-#else /* !COMPILE_OFFSETS */
-static inline void __migrate_disable(void) { }
-static inline void __migrate_enable(void) { }
-#endif /* !COMPILE_OFFSETS */
 
 /*
  * So that it is possible to not export the runqueues variable, define and
diff --git a/kernel/sched/core.c b/kernel/sched/core.c
index 145eea2d99a1..ddfa2ee52299 100644
--- a/kernel/sched/core.c
+++ b/kernel/sched/core.c
@@ -130,6 +130,7 @@ EXPORT_TRACEPOINT_SYMBOL_GPL(sched_dl_server_stop_tp);
 
 DEFINE_PER_CPU_SHARED_ALIGNED(struct rq, runqueues);
 DEFINE_PER_CPU(struct rnd_state, sched_rnd_state);
+DEFINE_PER_CPU(unsigned int, rq_nr_pinned);
 
 #ifdef CONFIG_SCHED_PROXY_EXEC
 DEFINE_STATIC_KEY_TRUE(__sched_proxy_exec);
@@ -2502,7 +2503,7 @@ EXPORT_SYMBOL_GPL(migrate_enable);
 
 static inline bool rq_has_pinned_tasks(struct rq *rq)
 {
-	return rq->nr_pinned;
+	return READ_ONCE(*per_cpu_ptr(&rq_nr_pinned, cpu_of(rq)));
 }
 
 /*
diff --git a/kernel/sched/rq-offsets.c b/kernel/sched/rq-offsets.c
deleted file mode 100644
index a23747bbe25b..000000000000
--- a/kernel/sched/rq-offsets.c
+++ /dev/null
@@ -1,12 +0,0 @@
-// SPDX-License-Identifier: GPL-2.0
-#define COMPILE_OFFSETS
-#include <linux/kbuild.h>
-#include <linux/types.h>
-#include "sched.h"
-
-int main(void)
-{
-	DEFINE(RQ_nr_pinned, offsetof(struct rq, nr_pinned));
-
-	return 0;
-}
diff --git a/kernel/sched/sched.h b/kernel/sched/sched.h
index 7701a5a60972..b77689172793 100644
--- a/kernel/sched/sched.h
+++ b/kernel/sched/sched.h
@@ -1358,7 +1358,6 @@ struct rq {
 	struct cpuidle_state	*idle_state;
 #endif
 
-	unsigned int		nr_pinned;
 	unsigned int		push_busy;
 	struct cpu_stop_work	push_work;
 
-- 
2.43.0