Re: [PATCH v2] sched/fair: Prefer waker CPU for reciprocal sync wakeups
From: Madadi Vineeth Reddy
Date: Fri Jul 24 2026 - 08:53:49 EST
On 23/07/26 04:20, Shubhang Kaushik (Ampere) wrote:
> Pipe-style ping-pong workloads can be dominated by handoff cost. In
> such cases, placing the wakee on an idle CPU can be slower than keeping
> the pair on the same runqueue.
>
> Use the existing last_wakee and wake_wide() state to identify narrow
> reciprocal WF_SYNC wakeups:
>
> A wakes B
> B wakes A
> A wakes B
> ...
>
I've been looking at the same underlying problem from the POWER side,
where the sync hint is consumed by wake_affine() and then discarded
by select_idle_sibling().
> When the wake-affine domain allows SD_WAKE_AFFINE, prefer the waker CPU
> for these narrow reciprocal handoffs. Do so only when the waker CPU has no
> other runnable fair task, the wakee is allowed on that CPU, and the wakee
> fits there on asymmetric capacity systems.
>
> Wakeups that do not match this pattern continue through the existing
> wake_affine() and select_idle_sibling() path.
>
> Signed-off-by: Shubhang Kaushik (Ampere) <sh@xxxxxxxxxx>
> ---
> Tested on 80-core Ampere Altra: perf bench sched pipe -l 1000000 improved
> by about 30%, averaged over 20 runs. Hackbench, schbench and SPECjBB
> showed no material regression.
>
> Baseline ~ v7.2-rc4 (mainline origin/master at 248951ddc14d)
> ---
> Changes in v2:
> - Move the reciprocal handoff preference under the existing
> SD_WAKE_AFFINE domain check.
> - Drop futex from the changelog motivation.
> - Refresh perf bench sched pipe results after rebasing.
>
> Link to v1: https://lore.kernel.org/r/20260721-b4-sched-sync-wakeup-v1-1-dc94f184e27f@xxxxxxxxxx
> ---
> kernel/sched/fair.c | 28 ++++++++++++++++++++++++++++
> 1 file changed, 28 insertions(+)
>
> diff --git a/kernel/sched/fair.c b/kernel/sched/fair.c
> index d78467ec6ee1343050fcc2794dafb38ade3599e5..d188e91b85d74dd3ed3c8b99f171bd142e72da5e 100644
> --- a/kernel/sched/fair.c
> +++ b/kernel/sched/fair.c
> @@ -8794,6 +8794,29 @@ static inline bool asym_fits_cpu(unsigned long util,
> return true;
> }
>
> +/*
> + * For reciprocal WF_SYNC handoffs, prefer the waker CPU when it has no
> + * other runnable fair task.
> + */
> +static bool prefer_sync_pair_cpu(struct task_struct *p, int cpu)
> +{
> + struct rq *rq = cpu_rq(cpu);
> +
> + if ((rq->nr_running - cfs_h_nr_delayed(rq)) != 1)
> + return false;
> +
> + if (!cpumask_test_cpu(cpu, p->cpus_ptr))
> + return false;
> +
> + if (sched_asym_cpucap_active()) {
> + sync_entity_load_avg(&p->se);
> + if (!task_fits_cpu(p, cpu))
> + return false;
> + }
> +
> + return true;
> +}
> +
> /*
> * Try and locate an idle core/thread in the LLC cache domain.
> */
> @@ -9579,6 +9602,11 @@ select_task_rq_fair(struct task_struct *p, int prev_cpu, int wake_flags)
> */
> if (want_affine && (tmp->flags & SD_WAKE_AFFINE) &&
> cpumask_test_cpu(prev_cpu, sched_domain_span(tmp))) {
> + if (sync &&
> + READ_ONCE(p->last_wakee) == current &&
> + prefer_sync_pair_cpu(p, cpu))
> + return cpu;
The difference is SMT. POWER10/11 are SMT8: stacking there puts the pair on one thread while up to
seven siblings on the same core sit idle, sharing the same LLC.
So instead of returning the waker's CPU, I'm looking at letting the waker's *core* count as idle
when the waker's runqueue has a single runnable task, so the wakee lands on a sibling thread.
Like below
diff --git a/kernel/sched/fair.c b/kernel/sched/fair.c
index 728965851842..8d1a2f62431b 100644
--- a/kernel/sched/fair.c
+++ b/kernel/sched/fair.c
@@ -1233,7 +1233,7 @@ static bool update_deadline(struct cfs_rq *cfs_rq, struct sched_entity *se)
#include "pelt.h"
-static int select_idle_sibling(struct task_struct *p, int prev_cpu, int cpu);
+static int select_idle_sibling(struct task_struct *p, int prev_cpu, int cpu, int want_affine);
static unsigned long task_h_load(struct task_struct *p);
static unsigned long capacity_of(int cpu);
@@ -7840,13 +7840,18 @@ void __update_idle_core(struct rq *rq)
* there are no idle cores left in the system; tracked through
* sd_llc->shared->has_idle_cores and enabled through update_idle_core() above.
*/
-static int select_idle_core(struct task_struct *p, int core, struct cpumask *cpus, int *idle_cpu)
+static int select_idle_core(struct task_struct *p, int core, struct cpumask *cpus, int *idle_cpu, bool want_affine)
{
bool idle = true;
int cpu;
+ int this_cpu = smp_processor_id();
for_each_cpu(cpu, cpu_smt_mask(core)) {
- if (!available_idle_cpu(cpu)) {
+ struct rq *rq = cpu_rq(cpu);
+ bool sync_waker = want_affine && cpu == this_cpu &&
+ ((rq->nr_running - cfs_h_nr_delayed(rq)) == 1);
+
+ if (!available_idle_cpu(cpu) && !sync_waker) {
idle = false;
if (*idle_cpu == -1) {
if (choose_sched_idle_rq(cpu_rq(cpu), p) &&
@@ -7858,6 +7863,7 @@ static int select_idle_core(struct task_struct *p, int core, struct cpumask *cpu
}
break;
}
+
if (*idle_cpu == -1 && cpumask_test_cpu(cpu, cpus))
*idle_cpu = cpu;
}
@@ -7920,7 +7926,7 @@ static inline int select_idle_smt(struct task_struct *p, struct sched_domain *sd
* comparing the average scan cost (tracked in sd->avg_scan_cost) against the
* average idle time for this rq (as found in rq->avg_idle).
*/
-static int select_idle_cpu(struct task_struct *p, struct sched_domain *sd, bool has_idle_core, int target)
+static int select_idle_cpu(struct task_struct *p, struct sched_domain *sd, bool has_idle_core, int target, int want_affine)
{
struct cpumask *cpus = this_cpu_cpumask_var_ptr(select_rq_mask);
int i, cpu, idle_cpu = -1, nr = INT_MAX;
@@ -7953,7 +7959,7 @@ static int select_idle_cpu(struct task_struct *p, struct sched_domain *sd, bool
continue;
if (has_idle_core) {
- i = select_idle_core(p, cpu, cpus, &idle_cpu);
+ i = select_idle_core(p, cpu, cpus, &idle_cpu, want_affine);
if ((unsigned int)i < nr_cpumask_bits)
return i;
} else {
@@ -7970,7 +7976,7 @@ static int select_idle_cpu(struct task_struct *p, struct sched_domain *sd, bool
for_each_cpu_wrap(cpu, cpus, target + 1) {
if (has_idle_core) {
- i = select_idle_core(p, cpu, cpus, &idle_cpu);
+ i = select_idle_core(p, cpu, cpus, &idle_cpu, want_affine);
if ((unsigned int)i < nr_cpumask_bits)
return i;
@@ -8060,7 +8066,7 @@ static inline bool asym_fits_cpu(unsigned long util,
/*
* Try and locate an idle core/thread in the LLC cache domain.
*/
-static int select_idle_sibling(struct task_struct *p, int prev, int target)
+static int select_idle_sibling(struct task_struct *p, int prev, int target, int want_affine)
{
bool has_idle_core = false;
struct sched_domain *sd;
@@ -8169,7 +8175,7 @@ static int select_idle_sibling(struct task_struct *p, int prev, int target)
}
}
- i = select_idle_cpu(p, sd, has_idle_core, target);
+ i = select_idle_cpu(p, sd, has_idle_core, target, want_affine);
if ((unsigned)i < nr_cpumask_bits)
return i;
@@ -8858,8 +8864,10 @@ select_task_rq_fair(struct task_struct *p, int prev_cpu, int wake_flags)
return sched_balance_find_dst_cpu(sd, p, cpu, prev_cpu, sd_flag);
/* Fast path */
- if (wake_flags & WF_TTWU)
- return select_idle_sibling(p, prev_cpu, new_cpu);
+ if (wake_flags & WF_TTWU) {
+ want_affine = want_affine && sync && (new_cpu != prev_cpu);
+ return select_idle_sibling(p, prev_cpu, new_cpu, want_affine);
+ }
return new_cpu;
}
The cache locality is the same as stacking, but the waker's remaining work
after the wakeup still runs in parallel rather than being serialized behind
the wakee.
I plan to post this patch as part of powerpc sched-domain series.
Thanks,
Vineeth
> +
> if (cpu != prev_cpu)
> new_cpu = wake_affine(tmp, p, cpu, prev_cpu, sync);
>
>
> ---
> base-commit: 248951ddc14de84de3910f9b13f51491a8cd91df
> change-id: 20260721-b4-sched-sync-wakeup-04d40cbeb1da
>
> Best regards,