[PATCH 10/10] sched/fair: Strive to find a task to migrate if newly idle when LB_PROMOTE

[Xin Zhao]

This patch is a core modification within this patch set. The simple_find
label in this patch implements a straightforward logic for finding a
migration task. It allows any instances in sched_balance_rq() that are
unable to find a migration task for various reasons to fallback to logic
executing the simple_find label to find one before exiting newly idle
process.

This patch addresses the newly idle scenario by preventing early exit in
cases when !ld_moved && !active_balance, effectively executing the logic
of simple_find label. Testing has shown that the situations listed below,
account for a significant proportion of early exits:
1. Failure in sched_balance_find_src_group
2. Failure in sched_balance_find_src_rq
3. ld_moved is 0 and active_balance has not been triggered

Of course, even with this change, it is still possible that no migration
task can be found. However, it at least ensures that all selectable CPUs
within the sched_domain have been thoroughly traversed.

Signed-off-by: Xin Zhao <jackzxcui1989@xxxxxxx>
---
 kernel/sched/fair.c | 38 +++++++++++++++++++++++++++++++++++++-
 1 file changed, 37 insertions(+), 1 deletion(-)

diff --git a/kernel/sched/fair.c b/kernel/sched/fair.c
index 1ae351ea5949..10ec7bb9e18c 100644
--- a/kernel/sched/fair.c
+++ b/kernel/sched/fair.c
@@ -13470,6 +13470,8 @@ static int sched_balance_rq(int this_cpu, struct rq *this_rq,
 	struct rq *busiest;
 	struct rq_flags rf;
 	struct cpumask *cpus = this_cpu_cpumask_var_ptr(load_balance_mask);
+	int cpu;
+	bool sfind = false;
 	struct lb_env env = {
 		.sd		= sd,
 		.dst_cpu	= this_cpu,
@@ -13504,15 +13506,41 @@ static int sched_balance_rq(int this_cpu, struct rq *this_rq,
 	group = sched_balance_find_src_group(&env);
 	if (!group) {
 		schedstat_inc(sd->lb_nobusyg[idle]);
+		if (sched_feat(LB_PROMOTE))
+			goto simple_find;
 		goto out_balanced;
 	}
 
 	busiest = sched_balance_find_src_rq(&env, group);
 	if (!busiest) {
 		schedstat_inc(sd->lb_nobusyq[idle]);
+		if (sched_feat(LB_PROMOTE))
+			goto simple_find;
 		goto out_balanced;
 	}
+	goto begin_balance;
+
+simple_find:
+	if (env.idle != CPU_NEWLY_IDLE ||
+	    (env.dst_rq->nr_running > 0 || env.dst_rq->ttwu_pending))
+		goto out_balanced;
 
+	sfind = true;
+	env.migration_type = migrate_task;
+	env.imbalance = 1;
+
+	for_each_cpu_andnot(cpu, env.cpus, env.dst_grpmask) {
+		busiest = cpu_rq(cpu);
+		if (busiest->nr_running <= 1) {
+			__cpumask_clear_cpu(cpu, cpus);
+			continue;
+		}
+		break;
+	}
+	if (cpu >= nr_cpu_ids)
+		goto out_balanced;
+
+begin_balance:
 	WARN_ON_ONCE(busiest == env.dst_rq);
 
 	update_lb_imbalance_stat(&env, sd, idle);
@@ -13615,6 +13643,7 @@ static int sched_balance_rq(int this_cpu, struct rq *this_rq,
 
 		/* All tasks on this runqueue were pinned by CPU affinity */
 		if (unlikely(env.flags & LBF_ALL_PINNED)) {
+check_redo:
 			__cpumask_clear_cpu(cpu_of(busiest), cpus);
 			/*
 			 * Attempting to continue load balancing at the current
@@ -13627,6 +13656,8 @@ static int sched_balance_rq(int this_cpu, struct rq *this_rq,
 			if (!cpumask_subset(cpus, env.dst_grpmask)) {
 				env.loop = 0;
 				env.loop_break = SCHED_NR_MIGRATE_BREAK;
+				if (sfind)
+					goto simple_find;
 				goto redo;
 			}
 			goto out_all_pinned;
@@ -13668,8 +13699,11 @@ static int sched_balance_rq(int this_cpu, struct rq *this_rq,
 		 * if the curr task on busiest CPU can't be
 		 * moved to this_cpu:
 		 */
-		if (!cpumask_test_cpu(this_cpu, busiest->curr->cpus_ptr))
+		if (!cpumask_test_cpu(this_cpu, busiest->curr->cpus_ptr)) {
+			if (sched_feat(LB_PROMOTE) && env.idle == CPU_NEWLY_IDLE)
+				goto check_redo;
 			goto out_one_pinned;
+		}
 
 		/* Record that we found at least one task that could run on this_cpu */
 		env.flags &= ~LBF_ALL_PINNED;
@@ -13703,6 +13737,8 @@ static int sched_balance_rq(int this_cpu, struct rq *this_rq,
 	preempt_enable();
 
 out_unbalanced:
+	if (sched_feat(LB_PROMOTE) && !active_balance && env.idle == CPU_NEWLY_IDLE)
+		goto check_redo;
 	/* We were unbalanced, so reset the balancing interval */
 	sd->balance_interval = sd->min_interval;
 	goto out;
-- 
2.34.1