[PATCH v4 05/40] sched: add cpumask_find_and_set() and use it in __mm_cid_get()

[Yury Norov]

__mm_cid_get() uses __mm_cid_try_get() helper to atomically acquire a
bit in mm cid mask. Now that we have atomic find_and_set_bit(), we can
easily extend it to cpumasks and use in the scheduler code.

cpumask_find_and_set() considers cid mask as a volatile region of memory,
as it actually is in this case. So, if it's changed while search is in
progress, KCSAN wouldn't fire warning on it.

CC: Peter Zijlstra <peterz@xxxxxxxxxxxxx>
Signed-off-by: Yury Norov <yury.norov@xxxxxxxxx>
Reviewed-by: Mathieu Desnoyers <mathieu.desnoyers@xxxxxxxxxxxx>
---
 MAINTAINERS                    |  1 +
 include/linux/cpumask_atomic.h | 20 ++++++++++++++++++++
 kernel/sched/sched.h           | 15 ++++++---------
 3 files changed, 27 insertions(+), 9 deletions(-)
 create mode 100644 include/linux/cpumask_atomic.h

diff --git a/MAINTAINERS b/MAINTAINERS
index 54f37d4f33dd..7173c74896d8 100644
--- a/MAINTAINERS
+++ b/MAINTAINERS
@@ -3730,6 +3730,7 @@ F:	include/linux/bitmap-str.h
 F:	include/linux/bitmap.h
 F:	include/linux/bits.h
 F:	include/linux/cpumask.h
+F:	include/linux/cpumask_atomic.h
 F:	include/linux/find_atomic.h
 F:	include/linux/find.h
 F:	include/linux/nodemask.h
diff --git a/include/linux/cpumask_atomic.h b/include/linux/cpumask_atomic.h
new file mode 100644
index 000000000000..1aaf9a63cbe6
--- /dev/null
+++ b/include/linux/cpumask_atomic.h
@@ -0,0 +1,20 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+#ifndef __LINUX_CPUMASK_ATOMIC_H_
+#define __LINUX_CPUMASK_ATOMIC_H_
+
+#include <linux/cpumask.h>
+#include <linux/find_atomic.h>
+
+/*
+ * cpumask_find_and_set - find the first unset cpu in a cpumask and
+ *			  set it atomically
+ * @srcp: the cpumask pointer
+ *
+ * Return: >= nr_cpu_ids if nothing is found.
+ */
+static inline unsigned int cpumask_find_and_set(volatile struct cpumask *srcp)
+{
+	return find_and_set_bit(cpumask_bits(srcp), small_cpumask_bits);
+}
+
+#endif /* __LINUX_CPUMASK_ATOMIC_H_ */
diff --git a/kernel/sched/sched.h b/kernel/sched/sched.h
index a831af102070..557896f8ccd7 100644
--- a/kernel/sched/sched.h
+++ b/kernel/sched/sched.h
@@ -30,6 +30,7 @@
 #include <linux/context_tracking.h>
 #include <linux/cpufreq.h>
 #include <linux/cpumask_api.h>
+#include <linux/cpumask_atomic.h>
 #include <linux/ctype.h>
 #include <linux/file.h>
 #include <linux/fs_api.h>
@@ -3312,23 +3313,19 @@ static inline void mm_cid_put(struct mm_struct *mm)
 
 static inline int __mm_cid_try_get(struct mm_struct *mm)
 {
-	struct cpumask *cpumask;
-	int cid;
+	struct cpumask *cpumask = mm_cidmask(mm);
+	int cid = nr_cpu_ids;
 
-	cpumask = mm_cidmask(mm);
 	/*
 	 * Retry finding first zero bit if the mask is temporarily
 	 * filled. This only happens during concurrent remote-clear
 	 * which owns a cid without holding a rq lock.
 	 */
-	for (;;) {
-		cid = cpumask_first_zero(cpumask);
-		if (cid < nr_cpu_ids)
-			break;
+	while (cid >= nr_cpu_ids) {
+		cid = cpumask_find_and_set(cpumask);
 		cpu_relax();
 	}
-	if (cpumask_test_and_set_cpu(cid, cpumask))
-		return -1;
+
 	return cid;
 }
 
-- 
2.43.0