[RFC PATCH 1/6] sched/fair: Add SMT sibling push in periodic load balance

[Li kunyu]

[Purpose]

An attempt is made to migrate the task to an SMT sibling CPU. If the
migration fails (i.e., the conditions are not met), it generally incurs
no additional overhead.

Migration fails in the following scenarios: (1) the current CPU is in
the CPU_IDLE state; (2) the current CPU load is relatively low; (3) the
overall CPU load is evenly distributed. All three cases are expected, so
no intervention is necessary.

Migration succeeds in the following scenarios: (1) the current CPU load
is moderately high; (2) the current CPU spawns a large number of
processes or threads within a short period (i.e., bursty tasks);  (3)
the SMT sibling CPUs are idle while the current CPU meets the minimum
migration requirements. All three cases are expected and can achieve
rapid task distribution.

[Implementation]

When conceptualizing this plan, we attempted to add push functions in
multiple locations, and finally tested and analyzed that adding this
function in periodic load balancing is the most appropriate:

1. There is no need to consider complex locking logic, as the CPU is
already idle and the task is no longer running.

2. At this point, it is only necessary for the CFS load of the SMT
sibling CPU to be lower than a certain proportion of the current CPU,
which is more efficient.

3. After the current CPU load drops (task migration successful), there
is an opportunity to pull CFS tasks from other CPUs, which is meaningful
for the overall situation.

Tested-by: Li Wang (Hieco) <li.wang@xxxxxxxxx>
Signed-off-by: Li kunyu <likunyu@xxxxxxxxxxxxxx>
---
 kernel/sched/fair.c     | 108 ++++++++++++++++++++++++++++++++++++++++
 kernel/sched/features.h |   9 ++++
 2 files changed, 117 insertions(+)

diff --git a/kernel/sched/fair.c b/kernel/sched/fair.c
index 6e005f1e8dc9..a43a168f23ba 100644
--- a/kernel/sched/fair.c
+++ b/kernel/sched/fair.c
@@ -136,6 +136,15 @@ static unsigned int sysctl_sched_cfs_bandwidth_slice		= 5000UL;
 static unsigned int sysctl_numa_balancing_promote_rate_limit = 65536;
 #endif
 
+/*
+ * The imbalance percentage condition for pushing tasks to SMT sibling CPUs.
+ *
+ * (default: ~25%)
+ */
+static unsigned int sysctl_smt_buddy_imbalance_pct = 125;
+static unsigned int sysctl_smt_buddy_imbalance_pct_min = 105;
+static unsigned int sysctl_smt_buddy_imbalance_pct_max = 200;
+
 #ifdef CONFIG_SYSCTL
 static const struct ctl_table sched_fair_sysctls[] = {
 #ifdef CONFIG_CFS_BANDWIDTH
@@ -158,6 +167,15 @@ static const struct ctl_table sched_fair_sysctls[] = {
 		.extra1		= SYSCTL_ZERO,
 	},
 #endif /* CONFIG_NUMA_BALANCING */
+	{
+		.procname	= "smt_buddy_imbalance_pct",
+		.data		= &sysctl_smt_buddy_imbalance_pct,
+		.maxlen		= sizeof(unsigned int),
+		.mode		= 0644,
+		.proc_handler	= proc_dointvec_minmax,
+		.extra1		= &sysctl_smt_buddy_imbalance_pct_min,
+		.extra2		= &sysctl_smt_buddy_imbalance_pct_max,
+	},
 };
 
 static int __init sched_fair_sysctl_init(void)
@@ -14226,6 +14244,94 @@ update_newidle_cost(struct sched_domain *sd, u64 cost, unsigned int success)
 	return false;
 }
 
+/*
+ * An exciting thing is that we have implemented an efficient push
+ * function process.
+ *
+ * It is applied in cycle load balancing and nohz_kick process.
+ * It can help us solve the problem of high workload of CFS tasks in
+ * the RQ queue.
+ */
+static bool smt_buddy_move_task(struct rq *src_rq)
+{
+	bool is_overload, is_move = false;
+	int src_cpu = cpu_of(src_rq), smt_cpu;
+	unsigned long src_cpu_runnable, smt_cpu_runnable;
+	struct sched_domain *smt_sd;
+	struct task_struct *p;
+	struct rq *smt_rq;
+	struct rq_flags rf;
+
+	if (!sched_feat(SMT_BUDDY_ASSIST) || !sched_smt_active())
+		goto exit;
+
+restart:
+	if (src_rq->nr_running < 2)
+		goto exit;
+
+	rcu_read_lock();
+
+	is_overload = false;
+	src_cpu_runnable = cpu_runnable(src_rq);
+	for_each_cpu(smt_cpu, cpu_smt_mask(src_cpu)) {
+		if (smt_cpu == src_cpu)
+			continue;
+
+		smt_cpu_runnable = cpu_runnable(cpu_rq(smt_cpu));
+		if ((src_cpu_runnable * 100 > smt_cpu_runnable * sysctl_smt_buddy_imbalance_pct)
+			&& cpu_active(smt_cpu)) {
+			is_overload = true;
+			break;
+		}
+	}
+
+	if (!is_overload) {
+		rcu_read_unlock();
+		goto exit;
+	}
+
+	smt_sd = lowest_flag_domain(src_cpu, SD_SHARE_CPUCAPACITY);
+	if (!smt_sd) {
+		rcu_read_unlock();
+		goto exit;
+	}
+
+	rcu_read_unlock();
+
+	rq_lock_irqsave(src_rq, &rf);
+	smt_rq = cpu_rq(smt_cpu);
+
+	struct lb_env env = {
+		.sd             = smt_sd,
+		.dst_cpu        = smt_cpu,
+		.dst_rq         = smt_rq,
+		.src_cpu        = src_cpu,
+		.src_rq         = src_rq,
+		.idle           = 0,		/* __CPU_NOT_IDLE */
+		.flags          = LBF_ACTIVE_LB,
+	};
+
+	/* Update the src_rq clock only once when migrating tasks. */
+	if (!is_move)
+		update_rq_clock(src_rq);
+
+	p = detach_one_task(&env);
+	if (!p) {
+		rq_unlock_irqrestore(src_rq, &rf);
+		goto exit;
+	}
+
+	rq_unlock(src_rq, &rf);
+	attach_one_task(smt_rq, p);
+	local_irq_restore(rf.flags);
+	is_move = true;
+
+	goto restart;
+
+exit:
+	return is_move;
+}
+
 /*
  * It checks each scheduling domain to see if it is due to be balanced,
  * and initiates a balancing operation if so.
@@ -14245,6 +14351,8 @@ static void sched_balance_domains(struct rq *rq, enum cpu_idle_type idle)
 	int need_decay = 0;
 	u64 max_cost = 0;
 
+	smt_buddy_move_task(rq);
+
 	rcu_read_lock();
 	for_each_domain(cpu, sd) {
 		/*
diff --git a/kernel/sched/features.h b/kernel/sched/features.h
index 8f0dee8fc475..173e39bb950a 100644
--- a/kernel/sched/features.h
+++ b/kernel/sched/features.h
@@ -142,3 +142,12 @@ SCHED_FEAT(LATENCY_WARN, false)
  */
 SCHED_FEAT(NI_RANDOM, true)
 SCHED_FEAT(NI_RATE, true)
+
+/*
+ * Based on the scalability feature added on the basis of SMT, by pushing
+ * CFS tasks to SMT sibling CPUs, the load can be rapidly reduced.
+ * When the load of SMT sibling CPUs is relatively high compared to
+ * the scheduling domain (above SMT), CFS tasks can be migrated to other
+ * CPUs in the scheduling domain, achieving fast and efficient migration of CFS tasks.
+ */
+SCHED_FEAT(SMT_BUDDY_ASSIST, false)
-- 
2.47.3