[RFC PATCH v3 12/17] sched/cache: Apply multi-LLC aggregation policy to task migration

[Jianyong Wu]

The existing task migration checks only apply the cache-aware policy
when either endpoint is the task's preferred LLC. Moves between other
LLCs are left unrestricted by that policy.

Apply the multi-LLC aggregation policy to these moves as well:
if_to_prefer() decides whether the move raises the task's affinity,
and can_migrate_llcs() decides whether to block it, based on that
direction and the load of the LLCs involved. Remove the obsolete
can_migrate_llc() helper.

Active balance has the same limitation. alb_break_llc() assumes
aggregation into a single LLC: only a task on its preferred LLC counts
as aggregated, so it restricts active balance only when every runnable
task on the source runqueue is on its preferred LLC. Once aggregation
spans multiple LLCs, a task outside its preferred LLC may still be
within the aggregation range, and pulling it to an LLC later in the
affinity sequence lowers its affinity.

When the source runqueue has exactly one task, check that task with
if_to_prefer() and can_migrate_llcs() wherever its preferred LLC is.
With more than one task, the task to be moved is not known here, so
leave the decision to can_migrate_task(). Move its active-balance fast
path after the cache-aware check so that active balance cannot bypass
a cache-aware veto when NUMA locality is neutral.

Signed-off-by: Jianyong Wu <wujianyong@xxxxxxxx>
---
 kernel/sched/fair.c | 111 +++++++++++++-------------------------------
 1 file changed, 32 insertions(+), 79 deletions(-)

diff --git a/kernel/sched/fair.c b/kernel/sched/fair.c
index 03bb5c85a8b7..c3f9ef8798d6 100644
--- a/kernel/sched/fair.c
+++ b/kernel/sched/fair.c
@@ -10580,54 +10580,6 @@ enum llc_mig {
 	mig_unrestricted	/* G: Don't restrict generic load balance migration */
 };
 
-/*
- * Check if task can be moved from the source LLC to the
- * destination LLC without breaking cache aware preferrence.
- * src_cpu and dst_cpu are arbitrary CPUs within the source
- * and destination LLCs, respectively.
- */
-static enum llc_mig can_migrate_llc(int src_cpu, int dst_cpu,
-				    unsigned long tsk_util,
-				    bool to_pref)
-{
-	unsigned long src_util, dst_util, src_cap, dst_cap;
-
-	if (!get_llc_stats(src_cpu, &src_util, &src_cap) ||
-	    !get_llc_stats(dst_cpu, &dst_util, &dst_cap))
-		return mig_unrestricted;
-
-	src_util = src_util < tsk_util ? 0 : src_util - tsk_util;
-	dst_util = dst_util + tsk_util;
-
-	if (!fits_llc_capacity(dst_util, dst_cap, false) &&
-	    !fits_llc_capacity(src_util, src_cap, false))
-		return mig_unrestricted;
-
-	if (to_pref) {
-		/*
-		 * Don't migrate if we will get preferred LLC too
-		 * heavily loaded and if the dest is much busier
-		 * than the src, in which case migration will
-		 * increase the imbalance too much.
-		 */
-		if (!fits_llc_capacity(dst_util, dst_cap, false) &&
-		    util_greater(dst_util, src_util))
-			return mig_forbid;
-	} else {
-		/*
-		 * Don't migrate if we will leave preferred LLC
-		 * too idle, or if this migration leads to the
-		 * non-preferred LLC falls within sysctl_aggr_imb percent
-		 * of preferred LLC, leading to migration again
-		 * back to preferred LLC.
-		 */
-		if (fits_llc_capacity(src_util, src_cap, false) ||
-		    !util_greater(src_util, dst_util))
-			return mig_forbid;
-	}
-	return mig_llc;
-}
-
 /* Accumulate a NUMA node's utilization and capacity at LLC granularity. */
 static bool get_node_stats(int node, unsigned long *util_out,
 			   unsigned long *cap_out, int *nr_llcs)
@@ -10768,7 +10720,7 @@ static enum llc_mig can_migrate_llcs(int src_cpu, int dst_cpu,
 }
 
 /* Decide whether the destination precedes the source in the affinity order. */
-static bool __maybe_unused if_to_prefer(int src_cpu, int dst_cpu, int pref_llc)
+static bool if_to_prefer(int src_cpu, int dst_cpu, int pref_llc)
 {
 	int src_rank, dst_rank, src_node, dst_node, pref_node;
 
@@ -10827,55 +10779,52 @@ static enum llc_mig can_migrate_llc_task(int src_cpu, int dst_cpu,
 		return mig_unrestricted;
 	}
 
-	if (cpus_share_cache(dst_cpu, cpu))
-		to_pref = true;
-	else if (cpus_share_cache(src_cpu, cpu))
-		to_pref = false;
-	else
-		return mig_unrestricted;
+	to_pref = if_to_prefer(src_cpu, dst_cpu, llc_id(cpu));
 
-	return can_migrate_llc(src_cpu, dst_cpu,
-			       task_util(p), to_pref);
+	return can_migrate_llcs(src_cpu, dst_cpu,
+			       p, to_pref);
 }
 
 /*
  * Check if active load balance breaks LLC locality in
  * terms of cache aware load balance. The load level and
  * imbalance do not warrant breaking LLC preference per
- * the can_migrate_llc() policy. Here, the benefit of
+ * the can_migrate_llcs() policy. Here, the benefit of
  * LLC locality outweighs the power efficiency gained from
  * migrating the only runnable task away.
  */
 static inline bool
 alb_break_llc(struct lb_env *env)
 {
+	int pref_llc = -1;
+	bool to_pref = false;
+
 	if (!sched_cache_enabled())
 		return false;
 
 	if (cpus_share_cache(env->src_cpu, env->dst_cpu))
 		return false;
 	/*
-	 * All tasks prefer to stay on their current CPU.
-	 * Do not pull a task from its preferred CPU if:
-	 * 1. It is the only task running and does not exceed
-	 *    imbalance allowance; OR
-	 * 2. Migrating it away from its preferred LLC would violate
-	 *    the cache-aware scheduling policy.
+	 * We need the preferred LLC to decide whether we can perform migration.
+	 * Therefore, we need to obtain task_struct, which is only meaningful
+	 * in the case that only one task on the rq.
+	 * For cases with more than one task on the rq, we need to check
+	 * this in can_migrate_task().
 	 */
-	if (env->src_rq->nr_pref_llc_running &&
-	    env->src_rq->nr_pref_llc_running == env->src_rq->cfs.h_nr_runnable) {
-		unsigned long util = 0;
+	if (env->src_rq->nr_running == 1) {
 		struct task_struct *cur;
 
-		if (env->src_rq->nr_running <= 1)
-			return true;
-
 		cur = rcu_dereference_all(env->src_rq->curr);
-		if (cur && cur->sched_class == &fair_sched_class)
-			util = task_util(cur);
+		if (!cur || !cur->mm)
+			return false;
+
+		if (cur->sched_class == &fair_sched_class)
+			pref_llc = llc_id(cur->mm->sc_stat.cpu);
 
-		if (can_migrate_llc(env->src_cpu, env->dst_cpu,
-				    util, false) == mig_forbid)
+		to_pref = if_to_prefer(env->src_cpu, env->dst_cpu, pref_llc);
+
+		if (can_migrate_llcs(env->src_cpu, env->dst_cpu,
+				    cur, to_pref) == mig_forbid)
 			return true;
 	}
 
@@ -11027,14 +10976,11 @@ int can_migrate_task(struct task_struct *p, struct lb_env *env)
 
 	/*
 	 * Aggressive migration if:
-	 * 1) active balance
-	 * 2) destination numa is preferred
+	 * 1) destination numa is preferred
+	 * 2) active balance
 	 * 3) task is cache cold, or
 	 * 4) too many balance attempts have failed.
 	 */
-	if (env->flags & LBF_ACTIVE_LB)
-		return 1;
-
 	degrades = migrate_degrades_locality(p, env);
 	if (!degrades) {
 		/*
@@ -11060,6 +11006,13 @@ int can_migrate_task(struct task_struct *p, struct lb_env *env)
 		hot = degrades > 0;
 	}
 
+	/*
+	 * Keep this after the cache-aware check so active balance respects its
+	 * migration veto when NUMA locality is neutral.
+	 */
+	if (env->flags & LBF_ACTIVE_LB)
+		return 1;
+
 	if (!hot || env->sd->nr_balance_failed > env->sd->cache_nice_tries) {
 		if (hot)
 			p->sched_task_hot = 1;
-- 
2.34.1