[RFC PATCH 5/6] sched/fair: Add task burst threshold for large-scale bursts

[Li kunyu]

In practice, bursty tasks can lead to uneven task distribution. For
example, some CPUs may have kernel-mode utilization above 50%, while
more than 10% of CPUs in the system remain idle. To address this, a task
burst threshold is introduced:

1. When a task is woken up, the scheduler first follows the original
wake-up logic (e.g., wake-up on the current CPU or affinity wake-up)
to select a CPU. If no CPU is selected, the task burst threshold of
new_cpu is then checked.

2. For newly created child processes or threads, only the task burst
threshold needs to be checked.

Benefits:

1. The performance degradation observed in the hackbench scenario is
resolved, allowing tasks to be distributed quickly.

2. On a Hygon 7447v CPU (Model 7) server with 384 logical CPUs, UnixBench
shows approximately 3% to 5% performance improvement for this patch, with
a cumulative improvement of 8% to 10% across the patch series.

Tested-by: Li Wang (Hieco) <li.wang@xxxxxxxxx>
Signed-off-by: Li kunyu <likunyu@xxxxxxxxxxxxxx>
---
 kernel/sched/fair.c | 62 +++++++++++++++++++++++++++++++++++++++++----
 1 file changed, 57 insertions(+), 5 deletions(-)

diff --git a/kernel/sched/fair.c b/kernel/sched/fair.c
index f2bf4b53d8a9..0cd0c5a5f158 100644
--- a/kernel/sched/fair.c
+++ b/kernel/sched/fair.c
@@ -145,6 +145,15 @@ static unsigned int sysctl_smt_buddy_imbalance_pct = 125;
 static unsigned int sysctl_smt_buddy_imbalance_pct_min = 105;
 static unsigned int sysctl_smt_buddy_imbalance_pct_max = 200;
 
+/*
+ * In certain scenarios, the task burst threshold helps distribute tasks to
+ * idle CPUs quickly.
+ *
+ * (default: 4)
+ */
+static unsigned int sysctl_smt_buddy_balance_task = 4;
+static unsigned int sysctl_smt_buddy_balance_task_min = 2;
+
 #ifdef CONFIG_SYSCTL
 static const struct ctl_table sched_fair_sysctls[] = {
 #ifdef CONFIG_CFS_BANDWIDTH
@@ -176,6 +185,14 @@ static const struct ctl_table sched_fair_sysctls[] = {
 		.extra1		= &sysctl_smt_buddy_imbalance_pct_min,
 		.extra2		= &sysctl_smt_buddy_imbalance_pct_max,
 	},
+	{
+		.procname       = "smt_buddy_balance_task",
+		.data           = &sysctl_smt_buddy_balance_task,
+		.maxlen         = sizeof(unsigned int),
+		.mode           = 0644,
+		.proc_handler   = proc_dointvec_minmax,
+		.extra1         = &sysctl_smt_buddy_balance_task_min,
+	},
 };
 
 static int __init sched_fair_sysctl_init(void)
@@ -9937,7 +9954,7 @@ select_task_rq_fair(struct task_struct *p, int prev_cpu, int wake_flags)
 	int sync = (wake_flags & WF_SYNC) && !(current->flags & PF_EXITING);
 	struct sched_domain *tmp, *sd = NULL;
 	int cpu = smp_processor_id();
-	int new_cpu = prev_cpu;
+	int new_cpu = prev_cpu, nohz_idle_cpu = 0;
 	int want_affine = 0;
 	/* SD_flags and WF_flags share the first nibble */
 	int sd_flag = wake_flags & 0xF;
@@ -9990,11 +10007,46 @@ select_task_rq_fair(struct task_struct *p, int prev_cpu, int wake_flags)
 
 	/* Slow path */
 	if (unlikely(sd))
-		return sched_balance_find_dst_cpu(sd, p, cpu, prev_cpu, sd_flag);
-
+		new_cpu = sched_balance_find_dst_cpu(sd, p, cpu, prev_cpu, sd_flag);
 	/* Fast path */
-	if (wake_flags & WF_TTWU)
-		return select_idle_sibling(p, prev_cpu, new_cpu);
+	else if (wake_flags & WF_TTWU)
+		new_cpu = select_idle_sibling(p, prev_cpu, new_cpu);
+
+#ifdef CONFIG_NO_HZ_COMMON
+	/*
+	 * In practice, bursty tasks can lead to uneven task distribution.
+	 * For example, some CPUs may have kernel-mode utilization above 50%,
+	 * while more than 10% of CPUs in the system remain idle.
+	 * Therefore, we define a task burst threshold:
+	 *
+	 * 1. For a task to be woken up, the scheduler first follows the
+	 * original wake-up logic (such as wake-up on the current CPU or
+	 * affinity wake-up) to return a CPU. If no CPU is returned, it
+	 * then checks the task burst threshold of new_cpu.
+	 * 2. Newly created child processes or threads have fewer
+	 * considerations; only the task burst threshold needs to be checked.
+	 */
+	if (sched_feat(SMT_BUDDY_ASSIST) && cpu_rq(new_cpu)->nr_running >=
+			sysctl_smt_buddy_balance_task)
+		for_each_cpu_wrap(nohz_idle_cpu,  nohz.idle_cpus_mask, new_cpu + 1) {
+			if (!idle_cpu(nohz_idle_cpu))
+				continue;
+
+			/*
+			 * In general-purpose scenarios, the CPU can
+			 * typically meet task requirements.
+			 */
+			if (!cpumask_test_cpu(nohz_idle_cpu, p->cpus_ptr))
+				continue;
+
+		/*
+		 * Simply returning nohz_idle_cpu is sufficient, since the
+		 * subsequent NOHZ kick will clear the corresponding bit in
+		 * nohz.idle_cpus_mask.
+		 */
+		return nohz_idle_cpu;
+	}
+#endif
 
 	return new_cpu;
 }
-- 
2.47.3