[PATCH 2/4] sched_ext: Collect CID construction state in a private builder
From: Zhang Qiao
Date: Thu Nov 05 2026 - 01:17:06 EST
The running counters and scratch masks are about to cross function
boundaries in both directions, which would grow the parameter lists of
every subsequent extraction step. Collect the build-only state instead
- the under-construction tables, the seven scratch masks borrowed from
scx_cid_init() and the running cid and index counters - in a
file-private scx_cid_builder and thread it through the stages
extracted so far.
No behavior change.
Signed-off-by: Zhang Qiao <zhangqiao22@xxxxxxxxxx>
---
kernel/sched/ext/cid.c | 188 ++++++++++++++++++++++-------------------
1 file changed, 103 insertions(+), 85 deletions(-)
diff --git a/kernel/sched/ext/cid.c b/kernel/sched/ext/cid.c
index b6be8806a7d5..ee55f101e5c5 100644
--- a/kernel/sched/ext/cid.c
+++ b/kernel/sched/ext/cid.c
@@ -165,9 +165,22 @@ void scx_cid_retire_tables(void)
call_rcu(&tbls->rcu, scx_cid_tables_free_rcufn);
}
-static void scx_cid_build_notopo(struct scx_cid_tables *tbls, u32 shard_size,
- u32 *next_cid, s32 *next_shard_idx,
- struct cpumask *online_no_topo)
+struct scx_cid_builder {
+ struct scx_cid_tables *tbls;
+ struct cpumask *to_walk;
+ struct cpumask *node_scratch;
+ struct cpumask *llc_scratch;
+ struct cpumask *cluster_scratch;
+ struct cpumask *core_scratch;
+ struct cpumask *llc_fallback;
+ struct cpumask *online_no_topo;
+ u32 next_cid;
+ s32 next_node_idx, next_llc_idx, next_core_idx;
+ s32 next_cluster_idx, next_shard_idx;
+ u32 shard_size, max_cids;
+};
+
+static void scx_cid_build_notopo(struct scx_cid_builder *b)
{
u32 notopo_in_shard;
s32 notopo_shard_cid, notopo_shard_idx;
@@ -175,42 +188,41 @@ static void scx_cid_build_notopo(struct scx_cid_tables *tbls, u32 shard_size,
/*
* No-topo section: any possible cpu without a cid - normally just the
- * not-online ones. Pack into shards of up to min(@shard_size,
+ * not-online ones. Pack into shards of up to min(@b->shard_size,
* SCX_CID_SHARD_MAX_CPUS) cids so that every cid has a valid shard
- * assignment and the hard cap holds even with a large @shard_size.
- * Collect any currently-online cpus that land here in @online_no_topo
+ * assignment and the hard cap holds even with a large @b->shard_size.
+ * Collect any currently-online cpus that land here in @b->online_no_topo
* so we can warn about them at the end.
*/
- notopo_in_shard = min_t(u32, shard_size, SCX_CID_SHARD_MAX_CPUS);
+ notopo_in_shard = min_t(u32, b->shard_size, SCX_CID_SHARD_MAX_CPUS);
notopo_shard_cid = -1;
notopo_shard_idx = -1;
for_each_cpu(cpu, cpu_possible_mask) {
- if (tbls->cpu_to_cid[cpu] != -1)
+ if (b->tbls->cpu_to_cid[cpu] != -1)
continue;
if (cpu_online(cpu))
- cpumask_set_cpu(cpu, online_no_topo);
+ cpumask_set_cpu(cpu, b->online_no_topo);
- cid = (*next_cid)++;
- tbls->cid_to_cpu[cid] = cpu;
- tbls->cpu_to_cid[cpu] = cid;
+ cid = b->next_cid++;
+ b->tbls->cid_to_cpu[cid] = cpu;
+ b->tbls->cpu_to_cid[cpu] = cid;
- if (notopo_in_shard >= min_t(u32, shard_size, SCX_CID_SHARD_MAX_CPUS)) {
+ if (notopo_in_shard >= min_t(u32, b->shard_size, SCX_CID_SHARD_MAX_CPUS)) {
notopo_shard_cid = cid;
- notopo_shard_idx = (*next_shard_idx)++;
+ notopo_shard_idx = b->next_shard_idx++;
notopo_in_shard = 0;
}
notopo_in_shard++;
- tbls->cid_to_shard[cid] = notopo_shard_idx;
- tbls->topo[cid] = SCX_CID_TOPO_NEG;
- tbls->topo[cid].shard_cid = notopo_shard_cid;
- tbls->topo[cid].shard_idx = notopo_shard_idx;
+ b->tbls->cid_to_shard[cid] = notopo_shard_idx;
+ b->tbls->topo[cid] = SCX_CID_TOPO_NEG;
+ b->tbls->topo[cid].shard_cid = notopo_shard_cid;
+ b->tbls->topo[cid].shard_idx = notopo_shard_idx;
}
}
-static void scx_cid_finalize_tables(struct scx_cid_tables *tbls, u32 next_cid,
- s32 next_shard_idx)
+static void scx_cid_finalize_tables(struct scx_cid_builder *b)
{
s32 cid;
@@ -219,15 +231,15 @@ static void scx_cid_finalize_tables(struct scx_cid_tables *tbls, u32 next_cid,
* cid ranges by construction: base_cid is the first cid landing in a
* shard, nr_cids is the count.
*/
- for (cid = 0; cid < next_cid; cid++) {
- s32 sidx = tbls->cid_to_shard[cid];
+ for (cid = 0; cid < b->next_cid; cid++) {
+ s32 sidx = b->tbls->cid_to_shard[cid];
- if (tbls->shard_ranges[sidx].nr_cids == 0)
- tbls->shard_ranges[sidx].base_cid = cid;
- tbls->shard_ranges[sidx].nr_cids++;
+ if (b->tbls->shard_ranges[sidx].nr_cids == 0)
+ b->tbls->shard_ranges[sidx].base_cid = cid;
+ b->tbls->shard_ranges[sidx].nr_cids++;
}
- tbls->nr_shards = next_shard_idx;
+ b->tbls->nr_shards = b->next_shard_idx;
}
/**
@@ -253,11 +265,7 @@ s32 scx_cid_init(struct scx_sched *sch)
cpumask_var_t llc_fallback __free(free_cpumask_var) = CPUMASK_VAR_NULL;
cpumask_var_t online_no_topo __free(free_cpumask_var) = CPUMASK_VAR_NULL;
struct scx_cid_tables *tbls;
- u32 next_cid = 0;
- s32 next_node_idx = 0, next_llc_idx = 0, next_core_idx = 0;
- s32 next_cluster_idx = 0;
- s32 next_shard_idx = 0;
- u32 shard_size, max_cids;
+ struct scx_cid_builder b = {};
s32 cpu, si;
/* CMASK_MAX_WORDS in cid.bpf.h covers NR_CPUS up to 8192 */
@@ -266,8 +274,8 @@ s32 scx_cid_init(struct scx_sched *sch)
lockdep_assert_cpus_held();
lockdep_assert_held(&scx_enable_mutex);
- shard_size = sch->ops.cid_shard_size ?: SCX_CID_SHARD_SIZE_DFL;
- max_cids = min_t(u32, shard_size, SCX_CID_SHARD_MAX_CPUS);
+ b.shard_size = sch->ops.cid_shard_size ?: SCX_CID_SHARD_SIZE_DFL;
+ b.max_cids = min_t(u32, b.shard_size, SCX_CID_SHARD_MAX_CPUS);
tbls = scx_cid_alloc_tables();
if (!tbls)
@@ -287,16 +295,25 @@ s32 scx_cid_init(struct scx_sched *sch)
!zalloc_cpumask_var(&online_no_topo, GFP_KERNEL))
return -ENOMEM;
+ b.tbls = tbls;
+ b.to_walk = to_walk;
+ b.node_scratch = node_scratch;
+ b.llc_scratch = llc_scratch;
+ b.cluster_scratch = cluster_scratch;
+ b.core_scratch = core_scratch;
+ b.llc_fallback = llc_fallback;
+ b.online_no_topo = online_no_topo;
+
/* -1 sentinels for sparse-possible cpu id holes (0 is a valid cid) */
for (cpu = 0; cpu < nr_cpu_ids; cpu++)
tbls->cpu_to_cid[cpu] = -1;
- cpumask_copy(to_walk, cpu_online_mask);
+ cpumask_copy(b.to_walk, cpu_online_mask);
- while (!cpumask_empty(to_walk)) {
- s32 next_cpu = cpumask_first(to_walk);
+ while (!cpumask_empty(b.to_walk)) {
+ s32 next_cpu = cpumask_first(b.to_walk);
s32 nid = cpu_to_node(next_cpu);
- s32 node_cid = next_cid;
+ s32 node_cid = b.next_cid;
s32 node_idx;
/*
@@ -304,38 +321,38 @@ s32 scx_cid_init(struct scx_sched *sch)
* cid. cpumask_of_node(-1) is undefined.
*/
if (nid < 0) {
- cpumask_clear_cpu(next_cpu, to_walk);
+ cpumask_clear_cpu(next_cpu, b.to_walk);
continue;
}
- node_idx = next_node_idx++;
+ node_idx = b.next_node_idx++;
- /* node_scratch = to_walk & this node */
- cpumask_and(node_scratch, to_walk, cpumask_of_node(nid));
- if (WARN_ON_ONCE(!cpumask_test_cpu(next_cpu, node_scratch)))
+ /* b.node_scratch = b.to_walk & this node */
+ cpumask_and(b.node_scratch, b.to_walk, cpumask_of_node(nid));
+ if (WARN_ON_ONCE(!cpumask_test_cpu(next_cpu, b.node_scratch)))
return -EINVAL;
- while (!cpumask_empty(node_scratch)) {
- s32 ncpu = cpumask_first(node_scratch);
- const struct cpumask *llc_mask = cpu_llc_mask(ncpu, llc_fallback);
- s32 llc_cid = next_cid;
- s32 llc_idx = next_llc_idx++;
+ while (!cpumask_empty(b.node_scratch)) {
+ s32 ncpu = cpumask_first(b.node_scratch);
+ const struct cpumask *llc_mask = cpu_llc_mask(ncpu, b.llc_fallback);
+ s32 llc_cid = b.next_cid;
+ s32 llc_idx = b.next_llc_idx++;
u32 cores_per_shard, nr_large;
u32 shard_local = 0, cores_in_shard = 0, cids_in_shard = 0;
s32 shard_cid, shard_idx;
s32 cluster_cid = -1, cluster_idx = -1;
- /* llc_scratch = node_scratch & this llc */
- cpumask_and(llc_scratch, node_scratch, llc_mask);
- if (WARN_ON_ONCE(!cpumask_test_cpu(ncpu, llc_scratch)))
+ /* b.llc_scratch = b.node_scratch & this llc */
+ cpumask_and(b.llc_scratch, b.node_scratch, llc_mask);
+ if (WARN_ON_ONCE(!cpumask_test_cpu(ncpu, b.llc_scratch)))
return -EINVAL;
- calc_shard_layout(llc_scratch, shard_size, &cores_per_shard, &nr_large);
- shard_cid = next_cid;
- shard_idx = next_shard_idx++;
- tbls->shard_node[shard_idx] = nid;
+ calc_shard_layout(b.llc_scratch, b.shard_size, &cores_per_shard, &nr_large);
+ shard_cid = b.next_cid;
+ shard_idx = b.next_shard_idx++;
+ b.tbls->shard_node[shard_idx] = nid;
- while (!cpumask_empty(llc_scratch)) {
+ while (!cpumask_empty(b.llc_scratch)) {
const struct cpumask *sib;
s32 core_cid, core_idx, lcpu, ccpu;
u32 max_cores, cids_in_core;
@@ -345,24 +362,26 @@ s32 scx_cid_init(struct scx_sched *sch)
* on, so that a cluster is a contiguous cid
* range like the core, LLC and node levels.
*/
- if (cpumask_empty(cluster_scratch)) {
- s32 xcpu = cpumask_first(llc_scratch);
+ if (cpumask_empty(b.cluster_scratch)) {
+ s32 xcpu = cpumask_first(b.llc_scratch);
- cpumask_or(cluster_scratch, topology_cluster_cpumask(xcpu),
+ cpumask_or(b.cluster_scratch,
+ topology_cluster_cpumask(xcpu),
topology_sibling_cpumask(xcpu));
- cpumask_and(cluster_scratch, cluster_scratch, llc_scratch);
- cluster_cid = next_cid;
- cluster_idx = next_cluster_idx++;
+ cpumask_and(b.cluster_scratch, b.cluster_scratch,
+ b.llc_scratch);
+ cluster_cid = b.next_cid;
+ cluster_idx = b.next_cluster_idx++;
}
- lcpu = cpumask_first(cluster_scratch);
+ lcpu = cpumask_first(b.cluster_scratch);
sib = topology_sibling_cpumask(lcpu);
- core_cid = next_cid;
- core_idx = next_core_idx++;
+ core_cid = b.next_cid;
+ core_idx = b.next_core_idx++;
- /* core_scratch = cluster_scratch & this core */
- cpumask_and(core_scratch, cluster_scratch, sib);
- if (WARN_ON_ONCE(!cpumask_test_cpu(lcpu, core_scratch)))
+ /* b.core_scratch = b.cluster_scratch & this core */
+ cpumask_and(b.core_scratch, b.cluster_scratch, sib);
+ if (WARN_ON_ONCE(!cpumask_test_cpu(lcpu, b.core_scratch)))
return -EINVAL;
/*
@@ -371,28 +390,28 @@ s32 scx_cid_init(struct scx_sched *sch)
* shard sizes under uneven SMT. Never start an
* empty shard.
*/
- cids_in_core = cpumask_weight(core_scratch);
+ cids_in_core = cpumask_weight(b.core_scratch);
max_cores = cores_per_shard + (shard_local < nr_large ? 1 : 0);
if (cores_in_shard &&
(cores_in_shard >= max_cores ||
- cids_in_shard + cids_in_core > max_cids)) {
+ cids_in_shard + cids_in_core > b.max_cids)) {
shard_local++;
cores_in_shard = 0;
cids_in_shard = 0;
- shard_cid = next_cid;
- shard_idx = next_shard_idx++;
- tbls->shard_node[shard_idx] = nid;
+ shard_cid = b.next_cid;
+ shard_idx = b.next_shard_idx++;
+ b.tbls->shard_node[shard_idx] = nid;
}
cores_in_shard++;
cids_in_shard += cids_in_core;
- for_each_cpu(ccpu, core_scratch) {
- s32 cid = next_cid++;
+ for_each_cpu(ccpu, b.core_scratch) {
+ s32 cid = b.next_cid++;
- tbls->cid_to_cpu[cid] = ccpu;
- tbls->cpu_to_cid[ccpu] = cid;
- tbls->cid_to_shard[cid] = shard_idx;
- tbls->topo[cid] = (struct scx_cid_topo){
+ b.tbls->cid_to_cpu[cid] = ccpu;
+ b.tbls->cpu_to_cid[ccpu] = cid;
+ b.tbls->cid_to_shard[cid] = shard_idx;
+ b.tbls->topo[cid] = (struct scx_cid_topo){
.core_cid = core_cid,
.core_idx = core_idx,
.llc_cid = llc_cid,
@@ -405,17 +424,16 @@ s32 scx_cid_init(struct scx_sched *sch)
.cluster_idx = cluster_idx,
};
- cpumask_clear_cpu(ccpu, cluster_scratch);
- cpumask_clear_cpu(ccpu, llc_scratch);
- cpumask_clear_cpu(ccpu, node_scratch);
- cpumask_clear_cpu(ccpu, to_walk);
+ cpumask_clear_cpu(ccpu, b.cluster_scratch);
+ cpumask_clear_cpu(ccpu, b.llc_scratch);
+ cpumask_clear_cpu(ccpu, b.node_scratch);
+ cpumask_clear_cpu(ccpu, b.to_walk);
}
}
}
}
- scx_cid_build_notopo(tbls, shard_size, &next_cid, &next_shard_idx,
- online_no_topo);
+ scx_cid_build_notopo(&b);
if (!cpumask_empty(llc_fallback))
pr_warn("scx_cid: cpus without cacheinfo, using node mask as llc: %*pbl\n",
@@ -424,7 +442,7 @@ s32 scx_cid_init(struct scx_sched *sch)
pr_warn("scx_cid: online cpus with no usable topology: %*pbl\n",
cpumask_pr_args(online_no_topo));
- scx_cid_finalize_tables(tbls, next_cid, next_shard_idx);
+ scx_cid_finalize_tables(&b);
return 0;
}
--
2.43.0