[PATCH 05/17] sched/mmcid: Take the allowed CPUs of the first user when it joins
From: Mathieu Desnoyers
Date: Tue Nov 17 2026 - 11:18:55 EST
The CIDs of an mm are sized from the CPUs its threads are allowed on.
mm_init_cid() copies the allowed CPUs of the task into a new mm. Later
changes reach the mm through mm_update_cpus_allowed(). It uses the mm
the task points to, and ignores an mm with no user.
An affinity change between mm_init_cid() and the join of the first user
is lost. In this example, P is a task allowed on one CPU, and H is
another task:
P H
execve()
exec_mmap()
mm_init_cid()
copies the allowed CPUs of P
sched_setaffinity(P, all CPUs)
mm_update_cpus_allowed()
the new mm has no user:
ignored
sched_mm_cid_after_execve()
P joins the new mm, which
counts one allowed CPU
>From reading the code, a fork() child has the same window between
dup_mm() and sched_mm_cid_fork().
The new mm then stays sized for the CPUs allowed at that time. A
process that starts on one CPU and is widened to 64 CPUs during its
execve() switches to per CPU mode with its second thread. In testing,
two threads of such a process read mm_cid 0 and 63. The IDs stay below
the number of possible CPUs. They are no longer compact.
Fix this by adding the allowed CPUs of the first user to the mm when it
joins. Hold t->pi_lock for that, as an affinity change holds it. A
concurrent change is then in t->cpus_mask already, or it finds the user
and updates the mm itself.
Fixes: b0c3d51b54f8 ("sched/mmcid: Provide precomputed maximal value")
Signed-off-by: Mathieu Desnoyers <mathieu.desnoyers@xxxxxxxxxxxx>
Cc: stable@xxxxxxxxxxxxxxx
---
kernel/sched/core.c | 65 +++++++++++++++++++++++++++++++--------------
1 file changed, 45 insertions(+), 20 deletions(-)
diff --git a/kernel/sched/core.c b/kernel/sched/core.c
index 510cd775723b..456b76e823b3 100644
--- a/kernel/sched/core.c
+++ b/kernel/sched/core.c
@@ -10833,24 +10833,17 @@ static bool mm_update_max_cids(struct mm_struct *mm)
return true;
}
-static inline void mm_update_cpus_allowed(struct mm_struct *mm, const struct cpumask *affmsk)
+static void __mm_update_cpus_allowed(struct mm_struct *mm, const struct cpumask *affmsk)
{
+ struct mm_mm_cid *mc = &mm->mm_cid;
struct cpumask *mm_allowed;
- struct mm_mm_cid *mc;
unsigned int weight;
- if (!mm || !READ_ONCE(mm->mm_cid.users))
- return;
+ lockdep_assert_held(&mc->lock);
/*
* mm::mm_cid::mm_cpus_allowed is the superset of each threads
* allowed CPUs mask which means it can only grow.
*/
- mc = &mm->mm_cid;
- guard(raw_spinlock)(&mc->lock);
- /* Might have raced with exit() */
- if (!mm->mm_cid.users)
- return;
-
mm_allowed = mm_cpus_allowed(mm);
weight = cpumask_weighted_or(mm_allowed, mm_allowed, affmsk);
if (weight == mc->nr_cpus_allowed)
@@ -10876,6 +10869,20 @@ static inline void mm_update_cpus_allowed(struct mm_struct *mm, const struct cpu
irq_work_queue(&mc->irq_work);
}
+static inline void mm_update_cpus_allowed(struct mm_struct *mm, const struct cpumask *affmsk)
+{
+ /* The first user takes the allowed CPUs of its task when it joins */
+ if (!mm || !READ_ONCE(mm->mm_cid.users))
+ return;
+
+ guard(raw_spinlock)(&mm->mm_cid.lock);
+ /* Might have raced with exit() */
+ if (!mm->mm_cid.users)
+ return;
+
+ __mm_update_cpus_allowed(mm, affmsk);
+}
+
static inline void mm_cid_complete_transit(struct mm_struct *mm, unsigned int mode)
{
/*
@@ -11017,6 +11024,28 @@ static void sched_mm_cid_cleanup_execve(struct task_struct *t, struct mm_struct
cpumask_copy(mm_cpus_allowed(mm), &t->cpus_mask);
}
+/*
+ * Make @t the first user of the CIDs of its mm.
+ *
+ * mm_init_cid() copied the allowed CPUs of @t before @t was a user. An
+ * affinity change since then did not reach the mm. Add the allowed CPUs
+ * of @t again. Hold t->pi_lock, which every affinity change takes. A
+ * concurrent change is then in t->cpus_mask already, or it finds the
+ * user.
+ */
+static void sched_mm_cid_add_first_user(struct task_struct *t, struct mm_struct *mm)
+{
+ guard(raw_spinlock_irq)(&t->pi_lock);
+ guard(raw_spinlock)(&mm->mm_cid.lock);
+
+ sched_mm_cid_cleanup_execve(t, mm);
+ sched_mm_cid_add_user(t, mm);
+ __mm_update_cpus_allowed(mm, &t->cpus_mask);
+ t->mm_cid.cid = mm_get_cid(mm);
+ /* Required for execve() */
+ __this_cpu_write(mm->mm_cid.pcpu->cid, t->mm_cid.cid);
+}
+
/*
* Make @t a user of the CIDs of its mm. Called on fork(2) and on
* execve(2). A failed execve(2) rejoins the mm it left.
@@ -11041,19 +11070,15 @@ static void sched_mm_cid_fork(struct task_struct *t)
WARN_ON_ONCE(t->mm_cid.cid != MM_CID_UNSET);
guard(mutex)(&mm->mm_cid.mutex);
+ /* The mutex keeps other users from joining or leaving */
+ if (!mm->mm_cid.users) {
+ sched_mm_cid_add_first_user(t, mm);
+ return;
+ }
+
scoped_guard(raw_spinlock_irq, &mm->mm_cid.lock) {
struct mm_cid_pcpu *pcp = this_cpu_ptr(mm->mm_cid.pcpu);
- /* First user ? */
- if (!mm->mm_cid.users) {
- sched_mm_cid_cleanup_execve(t, mm);
- sched_mm_cid_add_user(t, mm);
- t->mm_cid.cid = mm_get_cid(mm);
- /* Required for execve() */
- pcp->cid = t->mm_cid.cid;
- return;
- }
-
flip = sched_mm_cid_add_user(t, mm);
/*
* A running joiner takes no CID at a schedule in before the
--
2.47.3