[PATCH 07/10] sched/fair: Trigger active balance if a CFS task is preempted when LB_PROMOTE

[Xin Zhao]

When LB_PROMOTE is on, in addition to the scenarios modified in the
previous patches regarding .select_task_rq implement, preemption of CFS
tasks also needs to be addressed. Once a CFS task is preempted by a
real-time task, if that task is executing logic in a critical section that
does not support priority inheritance, such as read-write locks or
read-write semaphores, it may lead to performance issues.

By adding this checkpoint for when a CFS task is preempted, we can migrate
the task to an idle CPU, thus alleviating such performance problems.
Additionally, the patch set for the LB_PROMOTE feature does not alter the
logic of task_hot(), so this patch will not significantly increase the
frequency of load balance migration.

The newly introduced function preempt_active_balance() will be triggered
in __schedule() when it detects that the previous task has been preempted
while the LB_PROMOTE feature is enabled. preempt_active_balance() will
first check whether the currently preempted task has other CPUs available
to run on. If there are idle CPUs available, the preempted task will be
migrated to one of those idle CPUs.

The implementation of preempt_active_balance() considers the desire not to
introduce additional "holes" of rq locks, so the migration triggering
action is deferred to the balance_callback which is trigger_preempt_alb().
trigger_preempt_alb() will issue a stop work to allow the idle target CPU
to perform the migration.

After the modifications made by the previous patches, we can reuse
active_load_balance_cpu_stop() to assist in this migration action. In
preempt_active_balance(), we use list_move_tail to move the currently
preempted task to position where detach_one_task() will first traverse,
allowing active_load_balance_cpu_stop() to prioritize finding the task.

Signed-off-by: Xin Zhao <jackzxcui1989@xxxxxxx>
---
 kernel/sched/core.c  |  3 ++
 kernel/sched/fair.c  | 80 ++++++++++++++++++++++++++++++++++++++++++++
 kernel/sched/sched.h |  2 ++
 3 files changed, 85 insertions(+)

diff --git a/kernel/sched/core.c b/kernel/sched/core.c
index 2e7cde033a31..757ea5ef303d 100644
--- a/kernel/sched/core.c
+++ b/kernel/sched/core.c
@@ -7227,6 +7227,9 @@ static void __sched notrace __schedule(int sched_mode)
 
 		trace_sched_switch(preempt, prev, next, prev_state);
 
+		if (sched_feat(LB_PROMOTE) && preempt)
+			preempt_active_balance(prev);
+
 		/* Also unlocks the rq: */
 		rq = context_switch(rq, prev, next, &rf);
 	} else {
diff --git a/kernel/sched/fair.c b/kernel/sched/fair.c
index 9ffd01717599..d2b538abdf3e 100644
--- a/kernel/sched/fair.c
+++ b/kernel/sched/fair.c
@@ -13875,6 +13875,86 @@ static int active_load_balance_cpu_stop(void *data)
 	return 0;
 }
 
+static DEFINE_PER_CPU(struct balance_callback, preempt_push_head);
+static DEFINE_PER_CPU(int, push_cpu);
+
+static void trigger_preempt_alb(struct rq *rq)
+{
+	unsigned long flags;
+	int active_balance = 0;
+	int src_cpu = rq->cpu;
+	int idle_cpu = per_cpu(push_cpu, src_cpu);
+	struct rq *dst_rq = cpu_rq(idle_cpu);
+
+	preempt_disable();
+	raw_spin_rq_unlock(rq);
+	raw_spin_rq_lock_irqsave(dst_rq, flags);
+	if (dst_rq->active_balance)
+		goto unlock_rq;
+	dst_rq->active_balance = 1;
+	dst_rq->push_cpu = idle_cpu;
+	active_balance = 1;
+
+unlock_rq:
+	raw_spin_rq_unlock_irqrestore(dst_rq, flags);
+	if (active_balance) {
+		stop_one_cpu_nowait(idle_cpu, active_load_balance_cpu_stop, rq,
+				    &dst_rq->active_balance_work);
+	}
+	preempt_enable();
+	raw_spin_rq_lock(rq);
+}
+
+static void queue_preempt_alb_callback(struct rq *rq, int dst_cpu)
+{
+	per_cpu(push_cpu, rq->cpu) = dst_cpu;
+	queue_balance_callback(rq, &per_cpu(preempt_push_head, rq->cpu), trigger_preempt_alb);
+}
+
+void preempt_active_balance(struct task_struct *prev)
+{
+	struct rq *this_rq = this_rq();
+	int this_cpu = smp_processor_id(), cpu;
+	int start = nr_cpu_ids, idle_cpu = nr_cpu_ids;
+	struct sched_domain *sd;
+
+	if (unlikely(prev->sched_class != &fair_sched_class))
+		return;
+
+	if (unlikely(prev->migration_disabled))
+		return;
+
+	cpu = prev->recent_used_cpu;
+	if (cpu != this_cpu && available_idle_cpu(cpu) && !cpu_rq(cpu)->active_balance) {
+		idle_cpu = cpu;
+		goto active_balance;
+	}
+
+	rcu_read_lock();
+
+	sd = rcu_dereference(per_cpu(sd_llc, this_cpu));
+	if (sd)
+		start = cpumask_first_and(sched_domain_span(sd), prev->cpus_ptr);
+	if (start >= nr_cpu_ids)
+		start = cpumask_first(prev->cpus_ptr);
+
+	for_each_cpu_wrap(cpu, prev->cpus_ptr, start) {
+		if (cpu != this_cpu && available_idle_cpu(cpu) && !cpu_rq(cpu)->active_balance) {
+			idle_cpu = cpu;
+			goto unlock;
+		}
+	}
+
+unlock:
+	rcu_read_unlock();
+	if (idle_cpu == nr_cpu_ids)
+		return;
+
+active_balance:
+	list_move_tail(&prev->se.group_node, &this_rq->cfs_tasks);
+	queue_preempt_alb_callback(this_rq, idle_cpu);
+}
+
 /*
  * Scale the max sched_balance_rq interval with the number of CPUs in the system.
  * This trades load-balance latency on larger machines for less cross talk.
diff --git a/kernel/sched/sched.h b/kernel/sched/sched.h
index 26ae13c86b69..11848708e5ce 100644
--- a/kernel/sched/sched.h
+++ b/kernel/sched/sched.h
@@ -4193,6 +4193,8 @@ extern struct balance_callback *splice_balance_callbacks(struct rq *rq);
 extern void __balance_callbacks(struct rq *rq, struct rq_flags *rf);
 extern void balance_callbacks(struct rq *rq, struct balance_callback *head);
 
+extern void preempt_active_balance(struct task_struct *prev);
+
 /*
  * The 'sched_change' pattern is the safe, easy and slow way of changing a
  * task's scheduling properties. It dequeues a task, such that the scheduler
-- 
2.34.1