[tip: locking/core] locking/osq: Set prev_cpu=0 instead of locked=1

[tip-bot2 for David Laight]

The following commit has been merged into the locking/core branch of tip:

Commit-ID:     ee2e144a4e8538c4d2de0301792b03608b79e71e
Gitweb:        https://git.kernel.org/tip/ee2e144a4e8538c4d2de0301792b03608b79e71e
Author:        David Laight <david.laight.linux@xxxxxxxxx>
AuthorDate:    Mon, 07 Sep 2026 09:41:27 +01:00
Committer:     Ingo Molnar <mingo@xxxxxxxxxx>
CommitterDate: Fri, 06 Nov 2026 11:16:24 +01:00

locking/osq: Set prev_cpu=0 instead of locked=1

There is no need for separate prev_cpu and locked members of struct
optimistic_spin_node. Using a single field simplifies the code slightly. It
also removes any possibility of the two values being out of sync.

[ peterz: Removed the random prev_node store ]

Signed-off-by: David Laight <david.laight.linux@xxxxxxxxx>
Signed-off-by: Peter Zijlstra (Intel) <peterz@xxxxxxxxxxxxx>
Signed-off-by: Ingo Molnar <mingo@xxxxxxxxxx>
Cc: Linus Torvalds <torvalds@xxxxxxxxxxxxxxxxxxxx>
Cc: Waiman Long <longman@xxxxxxxxxx>
Link: https://patch.msgid.link/20260907084133.3696-4-david.laight.linux@xxxxxxxxx
---
 kernel/locking/osq_lock.c | 45 ++++++++++++++++----------------------
 1 file changed, 19 insertions(+), 26 deletions(-)

diff --git a/kernel/locking/osq_lock.c b/kernel/locking/osq_lock.c
index ee1f69b..9daf197 100644
--- a/kernel/locking/osq_lock.c
+++ b/kernel/locking/osq_lock.c
@@ -14,7 +14,6 @@
 
 struct optimistic_spin_node {
 	struct optimistic_spin_node *next;
-	int locked; /* 1 if lock acquired */
 	int prev_cpu; /* CPU number offset by 1 */
 };
 
@@ -134,7 +133,6 @@ bool osq_lock(struct optimistic_spin_queue *lock)
 	int curr = encode_cpu(smp_processor_id());
 	int prev_cpu;
 
-	node->locked = 0;
 	node->next = NULL;
 
 	/*
@@ -166,44 +164,39 @@ bool osq_lock(struct optimistic_spin_queue *lock)
 	 * is implemented with a monitor-wait. vcpu_is_preempted() relies on
 	 * polling, be careful.
 	 */
-	if (smp_cond_load_relaxed(&node->locked, VAL || need_resched() ||
-				  vcpu_is_preempted(node->prev_cpu - 1)))
-		return true;
+	prev_cpu = smp_cond_load_relaxed(&node->prev_cpu, !VAL || need_resched() ||
+					 vcpu_is_preempted(VAL - 1));
 
 	/* unqueue */
 	/*
 	 * Step - A  -- stabilize @prev
 	 *
-	 * Undo our @prev->next assignment; this will make @prev's
-	 * unlock()/unqueue() wait for a next pointer since @lock points to us
-	 * (or later).
+	 * Loop until either node->prev_cpu is zero (lock acquired) or we
+	 * atomically change prev->next from node to NULL (stopping prev
+	 * handing on the lock).
+	 * Note that 'prev' can unlink itself concurrently with this
+	 * test so that prev/prev_ptr can be stale, but since it
+	 * is per-cpu data the memory can always be read.
 	 */
 
 	for (;;) {
-		/*
-		 * cpu_relax() below implies a compiler barrier which would
-		 * prevent this comparison being optimized away.
-		 */
+		if (!prev_cpu) {
+			smp_acquire__after_ctrl_dep();
+			return true;
+		}
+
+		prev = decode_cpu(prev_cpu);
+
 		if (data_race(prev->next) == node &&
 		    cmpxchg(&prev->next, node, NULL) == node)
 			break;
 
 		/*
-		 * We can only fail the cmpxchg() racing against an unlock(),
-		 * in which case we should observe @node->locked becoming
-		 * true.
+		 * 'prev' must have unlinked (or be in the process of unlinking)
+		 * itself from the list.
 		 */
-		if (smp_load_acquire(&node->locked))
-			return true;
-
 		cpu_relax();
-
-		/*
-		 * Or we race against a concurrent unqueue()'s step-B, in which
-		 * case its step-C will write us a new @node->prev pointer.
-		 */
 		prev_cpu = READ_ONCE(node->prev_cpu);
-		prev = decode_cpu(prev_cpu);
 	}
 
 	/*
@@ -248,11 +241,11 @@ void osq_unlock(struct optimistic_spin_queue *lock)
 	node = this_cpu_ptr(&osq_node);
 	next = xchg(&node->next, NULL);
 	if (next) {
-		WRITE_ONCE(next->locked, 1);
+		WRITE_ONCE(next->prev_cpu, 0);
 		return;
 	}
 
 	next = osq_wait_next(lock, node, OSQ_UNLOCKED_VAL);
 	if (next)
-		WRITE_ONCE(next->locked, 1);
+		WRITE_ONCE(next->prev_cpu, 0);
 }