[PATCH 11/12] sched_ext: scx_qmap: Add proxy execution support
From: Andrea Righi
Date: Tue Jul 21 2026 - 02:46:08 EST
Add a -B option to opt scx_qmap into queueing mutex-blocked tasks for
proxy execution. Without the option, SCX_OPS_ENQ_BLOCKED remains clear
and mutex waiters block normally. With -B, blocked donors are passed to
qmap_enqueue() with SCX_ENQ_BLOCKED.
When scx_qmap receives a blocked donor, dispatch it directly to the
local DSQ of its current cid with a fresh slice and SCX_ENQ_PREEMPT.
This places the donor at the head of the DSQ and requests an immediate
reschedule, allowing the core proxy-exec path to run the mutex owner
using the donor's scheduling context as soon as the donor is selected.
The blocked policy is intentionally unfair and can strongly prioritize
tasks using contended mutexes, but scx_qmap is a demo scheduler and such
aggressive behavior makes proxy-exec support easy to observe. Count
these dispatch attempts in nr_enq_blocked and report their per-interval
delta.
Signed-off-by: Andrea Righi <arighi@xxxxxxxxxx>
---
tools/sched_ext/scx_qmap.bpf.c | 34 ++++++++++++++++++++++++++++++++++
tools/sched_ext/scx_qmap.c | 13 ++++++++++---
tools/sched_ext/scx_qmap.h | 1 +
3 files changed, 45 insertions(+), 3 deletions(-)
diff --git a/tools/sched_ext/scx_qmap.bpf.c b/tools/sched_ext/scx_qmap.bpf.c
index aead17658573b..5ca1467c1325a 100644
--- a/tools/sched_ext/scx_qmap.bpf.c
+++ b/tools/sched_ext/scx_qmap.bpf.c
@@ -451,6 +451,40 @@ void BPF_STRUCT_OPS(qmap_enqueue, struct task_struct *p, u64 enq_flags)
*/
taskc->core_sched_seq = qa.core_sched_tail_seqs[idx]++;
+ /*
+ * A blocked donor rejected by a local DSQ must not be dispatched back
+ * to the same DSQ immediately. Move it to the global DSQ so another CPU
+ * can pick it up and kick an idle cid to accelerate its consumption.
+ */
+ if ((enq_flags & (SCX_ENQ_BLOCKED | SCX_ENQ_REENQ)) ==
+ (SCX_ENQ_BLOCKED | SCX_ENQ_REENQ)) {
+ taskc->force_local = false;
+ scx_bpf_dsq_insert(p, SHARED_DSQ, 0, enq_flags);
+ cid = cmask_next_and2_set_wrap(&taskc->cpus_allowed,
+ &qa.idle_cids.mask,
+ &qa.self_cids.mask, 0);
+ if (cid < scx_bpf_nr_cids())
+ scx_bpf_kick_cid(cid, SCX_KICK_IDLE);
+ return;
+ }
+
+ /*
+ * Insert a blocked mutex donor at the head of its current cid's local
+ * DSQ with a fresh slice and %SCX_ENQ_PREEMPT, requesting an immediate
+ * reschedule. Once selected, the core proxy-exec path can immediately
+ * run the mutex owner using the donor's scheduling context.
+ *
+ * This policy is intentionally unfair and can strongly prioritize tasks
+ * using contended mutexes; scx_qmap is a demonstration scheduler and
+ * this behavior makes proxy-exec support easy to observe.
+ */
+ if (enq_flags & SCX_ENQ_BLOCKED) {
+ __sync_fetch_and_add(&qa.nr_enq_blocked, 1);
+ scx_bpf_dsq_insert(p, SCX_DSQ_LOCAL_ON | scx_bpf_task_cid(p),
+ slice_ns, enq_flags | SCX_ENQ_PREEMPT);
+ return;
+ }
+
/*
* A node with children delegates most cids. A task of ours that can run
* on none of our self cids (e.g. a per-NUMA kthread pinned to delegated
diff --git a/tools/sched_ext/scx_qmap.c b/tools/sched_ext/scx_qmap.c
index 27ffda1c519e8..8e943e8cb202d 100644
--- a/tools/sched_ext/scx_qmap.c
+++ b/tools/sched_ext/scx_qmap.c
@@ -46,7 +46,7 @@ const char help_fmt[] =
"See the top-of-file comment in .bpf.c for the design.\n"
"\n"
"Usage: %s [-s SLICE_US] [-e COUNT] [-t COUNT] [-T COUNT] [-l COUNT] [-b COUNT]\n"
-" [-N COUNT] [-P] [-M] [-H] [-c CG_PATH] [-d PID] [-D LEN] [-S] [-p] [-I]\n"
+" [-N COUNT] [-P] [-M] [-H] [-c CG_PATH] [-d PID] [-D LEN] [-S] [-p] [-I] [-B]\n"
" [-F COUNT] [-i SEC] [-R MS] [-J MODE] [-v]\n"
"\n"
" -s SLICE_US Override slice duration\n"
@@ -65,6 +65,7 @@ const char help_fmt[] =
" -S Suppress qmap-specific debug dump\n"
" -p Switch only tasks on SCHED_EXT policy instead of all\n"
" -I Turn on SCX_OPS_ALWAYS_ENQ_IMMED\n"
+" -B Turn on SCX_OPS_ENQ_BLOCKED\n"
" -F COUNT IMMED stress: force every COUNT'th enqueue to a busy local DSQ (use with -I)\n"
" -C MODE cid-override test (shuffle|bad-dup|bad-range|bad-mono)\n"
" -i SEC Stats interval, seconds (default 5)\n"
@@ -105,6 +106,7 @@ struct hier_prev {
u64 nr_dsps[MAX_SUB_SCHEDS];
u64 nr_reenq_cap;
u64 nr_reenq_immed;
+ u64 nr_enq_blocked;
u64 nr_inject_attempts;
};
@@ -187,13 +189,15 @@ static void print_hier(struct qmap_arena *qa, struct hier_prev *prev, u64 own_cg
}
format_cid_ranges(qa, CID_SHARED, ranges, sizeof(ranges));
- printf("hier : nsub=%llu excl=%u shared=%s rr=%s reenq cap/immed +%llu/+%llu inj=+%llu\n",
+ printf("hier : nsub=%llu excl=%u shared=%s rr=%s reenq cap/immed +%llu/+%llu blocked=+%llu inj=+%llu\n",
(unsigned long long)qa->nr_sub_scheds, qa->part.nr_excl, ranges, rr,
(unsigned long long)(qa->nr_reenq_cap - prev->nr_reenq_cap),
(unsigned long long)(qa->nr_reenq_immed - prev->nr_reenq_immed),
+ (unsigned long long)(qa->nr_enq_blocked - prev->nr_enq_blocked),
(unsigned long long)(qa->nr_inject_attempts - prev->nr_inject_attempts));
prev->nr_reenq_cap = qa->nr_reenq_cap;
prev->nr_reenq_immed = qa->nr_reenq_immed;
+ prev->nr_enq_blocked = qa->nr_enq_blocked;
prev->nr_inject_attempts = qa->nr_inject_attempts;
printf("hier : %-4s %10s %4s %6s %8s %s\n",
@@ -256,7 +260,7 @@ int main(int argc, char **argv)
skel->rodata->slice_ns = __COMPAT_ENUM_OR_ZERO("scx_public_consts", "SCX_SLICE_DFL");
skel->rodata->max_tasks = 16384;
- while ((opt = getopt(argc, argv, "s:e:t:T:l:b:N:PMHc:d:D:SpIF:C:i:R:J:vh")) != -1) {
+ while ((opt = getopt(argc, argv, "s:e:t:T:l:b:N:PMHc:d:D:SpIBF:C:i:R:J:vh")) != -1) {
switch (opt) {
case 's':
skel->rodata->slice_ns = strtoull(optarg, NULL, 0) * 1000;
@@ -318,6 +322,9 @@ int main(int argc, char **argv)
skel->rodata->always_enq_immed = true;
skel->struct_ops.qmap_ops->flags |= SCX_OPS_ALWAYS_ENQ_IMMED;
break;
+ case 'B':
+ skel->struct_ops.qmap_ops->flags |= SCX_OPS_ENQ_BLOCKED;
+ break;
case 'F':
skel->rodata->immed_stress_nth = strtoul(optarg, NULL, 0);
break;
diff --git a/tools/sched_ext/scx_qmap.h b/tools/sched_ext/scx_qmap.h
index c42f7ef74b894..7f4d3b33e6133 100644
--- a/tools/sched_ext/scx_qmap.h
+++ b/tools/sched_ext/scx_qmap.h
@@ -174,6 +174,7 @@ struct qmap_arena {
/* bpf -> userspace: stats */
u64 nr_reenq_cap; /* SCX_TASK_REENQ_CAP bounces */
u64 nr_reenq_immed; /* SCX_TASK_REENQ_IMMED bounces */
+ u64 nr_enq_blocked; /* SCX_ENQ_BLOCKED dispatches */
u64 nr_inject_attempts; /* fault-injection: dispatches to an unheld cid */
u32 inject_mode; /* fault-injection mode (QMAP_INJ_*) */
};
--
2.55.0