[tip: sched/core] sched/fair: Add cgroup_mode: concur

[tip-bot2 for Peter Zijlstra]

The following commit has been merged into the sched/core branch of tip:

Commit-ID:     5c0b58bd1c9f09c4e9363665702c82694c6ee7a0
Gitweb:        https://git.kernel.org/tip/5c0b58bd1c9f09c4e9363665702c82694c6ee7a0
Author:        Peter Zijlstra <peterz@xxxxxxxxxxxxx>
AuthorDate:    Thu, 12 Mar 2026 16:04:18 +01:00
Committer:     Peter Zijlstra <peterz@xxxxxxxxxxxxx>
CommitterDate: Tue, 30 Jun 2026 10:56:52 +02:00

sched/fair: Add cgroup_mode: concur

Improve upon the previous scheme ("max") by no longer assuming maximal
concurrency. Instead scale by: 'min(nr_tasks, nr_cpus)'. This handles
the low concurrency cases more gracefully:

	F_g_n' = min(M, N) * F_g_n

Notably this is the first mode where:

	avg(F_g_n) = 1

In the single task case it reduces to ("smp") and then it nicely scales up
until it hits N, where it behaves like ("max").

This is no longer clipped at nice -20. Strictly speaking it isn't different
from the normal SMP scenario where all tasks are extremely unbalanced. There
are no unnatural inflations in this scheme.

The meaning of "cpu.weight" would be: weight per active CPU.

NOTE: Compute the group wide number of tasks by extending the tg->load_avg
computation with tg->runnable_avg, since cfs_rq->runnable_avg is based on
cfs_rq->h_nr_running.

Signed-off-by: Peter Zijlstra (Intel) <peterz@xxxxxxxxxxxxx>
Link: https://patch.msgid.link/20260605124051.740585993%40infradead.org
---
 kernel/sched/debug.c |  1 +-
 kernel/sched/fair.c  | 43 ++++++++++++++++++++++++++++++++++++-------
 kernel/sched/sched.h |  3 +++-
 3 files changed, 40 insertions(+), 7 deletions(-)

diff --git a/kernel/sched/debug.c b/kernel/sched/debug.c
index 84e0ac4..a5e3f8a 100644
--- a/kernel/sched/debug.c
+++ b/kernel/sched/debug.c
@@ -640,6 +640,7 @@ static int cgroup_mode = 1;
 static const char *cgroup_mode_str[] = {
 	"up",
 	"smp",
+	"concur",
 	"max",
 };
 
diff --git a/kernel/sched/fair.c b/kernel/sched/fair.c
index b556d53..64181e4 100644
--- a/kernel/sched/fair.c
+++ b/kernel/sched/fair.c
@@ -4846,6 +4846,11 @@ static int tg_cpus(struct task_group *tg)
 	return nr;
 }
 
+static inline int tg_tasks(struct task_group *tg)
+{
+	return max(1, atomic_long_read(&tg->runnable_avg) >> SCHED_CAPACITY_SHIFT);
+}
+
 /*
  * Func: min(fraction(nr_cpus * tg->shares), nice -20)
  *
@@ -4863,6 +4868,20 @@ static long calc_max_shares(struct cfs_rq *cfs_rq)
 }
 
 /*
+ * Func: fraction(nr * tg->shares); nr = min(nr_tasks, nr_cpus)
+ *
+ * Scales between "smp" and "max" in a natural way. No longer needs clipping
+ * since there are no unnatural inflations like with "max".
+ */
+static long calc_concur_shares(struct cfs_rq *cfs_rq)
+{
+	struct task_group *tg = cfs_rq->tg;
+	int nr = min(tg_tasks(tg), tg_cpus(tg));
+	long tg_shares = READ_ONCE(tg->shares);
+	return __calc_smp_shares(cfs_rq, nr * tg_shares, nr * tg_shares);
+}
+
+/*
  * Func: fraction(tg->shares)
  *
  * This infamously results in tiny shares when you have many CPUs.
@@ -4897,6 +4916,9 @@ void __sched_cgroup_mode_update(int mode)
 		func = &calc_smp_shares;
 		break;
 	case 2:
+		func = &calc_concur_shares;
+		break;
+	case 3:
 		func = &calc_max_shares;
 		break;
 	}
@@ -5043,7 +5065,7 @@ static inline bool cfs_rq_is_decayed(struct cfs_rq *cfs_rq)
  */
 static inline void update_tg_load_avg(struct cfs_rq *cfs_rq)
 {
-	long delta;
+	long dl, dr;
 	u64 now;
 
 	/*
@@ -5064,17 +5086,21 @@ static inline void update_tg_load_avg(struct cfs_rq *cfs_rq)
 	if (now - cfs_rq->last_update_tg_load_avg < NSEC_PER_MSEC)
 		return;
 
-	delta = cfs_rq->avg.load_avg - cfs_rq->tg_load_avg_contrib;
-	if (abs(delta) > cfs_rq->tg_load_avg_contrib / 64) {
-		atomic_long_add(delta, &cfs_rq->tg->load_avg);
+	dl = cfs_rq->avg.load_avg - cfs_rq->tg_load_avg_contrib;
+	dr = cfs_rq->avg.runnable_avg - cfs_rq->tg_runnable_avg_contrib;
+	if (abs(dl) > cfs_rq->tg_load_avg_contrib / 64 ||
+	    abs(dr) > cfs_rq->tg_runnable_avg_contrib / 64) {
+		atomic_long_add(dl, &cfs_rq->tg->load_avg);
+		atomic_long_add(dr, &cfs_rq->tg->runnable_avg);
 		cfs_rq->tg_load_avg_contrib = cfs_rq->avg.load_avg;
+		cfs_rq->tg_runnable_avg_contrib = cfs_rq->avg.runnable_avg;
 		cfs_rq->last_update_tg_load_avg = now;
 	}
 }
 
 static inline void clear_tg_load_avg(struct cfs_rq *cfs_rq)
 {
-	long delta;
+	long dl, dr;
 	u64 now;
 
 	/*
@@ -5084,9 +5110,12 @@ static inline void clear_tg_load_avg(struct cfs_rq *cfs_rq)
 		return;
 
 	now = rq_clock(rq_of(cfs_rq));
-	delta = 0 - cfs_rq->tg_load_avg_contrib;
-	atomic_long_add(delta, &cfs_rq->tg->load_avg);
+	dl = 0 - cfs_rq->tg_load_avg_contrib;
+	dr = 0 - cfs_rq->tg_runnable_avg_contrib;
+	atomic_long_add(dl, &cfs_rq->tg->load_avg);
+	atomic_long_add(dr, &cfs_rq->tg->runnable_avg);
 	cfs_rq->tg_load_avg_contrib = 0;
+	cfs_rq->tg_runnable_avg_contrib = 0;
 	cfs_rq->last_update_tg_load_avg = now;
 }
 
diff --git a/kernel/sched/sched.h b/kernel/sched/sched.h
index 922170b..fd267bf 100644
--- a/kernel/sched/sched.h
+++ b/kernel/sched/sched.h
@@ -493,6 +493,8 @@ struct task_group {
 	 * will also be accessed at each tick.
 	 */
 	atomic_long_t		load_avg ____cacheline_aligned;
+	atomic_long_t		runnable_avg;
+
 #endif /* CONFIG_FAIR_GROUP_SCHED */
 
 #ifdef CONFIG_RT_GROUP_SCHED
@@ -722,6 +724,7 @@ struct cfs_rq {
 #ifdef CONFIG_FAIR_GROUP_SCHED
 	u64			last_update_tg_load_avg;
 	unsigned long		tg_load_avg_contrib;
+	unsigned long		tg_runnable_avg_contrib;
 	long			propagate;
 	long			prop_runnable_sum;