[tip: locking/core] locking/osq: Use CPU number for 'next' pointer
From: tip-bot2 for David Laight
Date: Fri Nov 06 2026 - 05:37:47 EST
The following commit has been merged into the locking/core branch of tip:
Commit-ID: 8e7c6217be7f8c016975f5b0954f6736b9ac42cf
Gitweb: https://git.kernel.org/tip/8e7c6217be7f8c016975f5b0954f6736b9ac42cf
Author: David Laight <david.laight.linux@xxxxxxxxx>
AuthorDate: Mon, 07 Sep 2026 09:41:30 +01:00
Committer: Ingo Molnar <mingo@xxxxxxxxxx>
CommitterDate: Fri, 06 Nov 2026 11:16:24 +01:00
locking/osq: Use CPU number for 'next' pointer
There is only one write done through node->next (setting prev) when
'node' is being removed. So the code can consistently use the CPU
numbers pretty much throughout (this was suggested by Linus a while
back).
This reduces struct optimistic_spin_node to 8 bytes. This is currently
padded out to a cache line which is silly. Change to be __aligned(8) so
that it isn't split between cache lines.
Accesses to 'other CPU' data are very limited and only happen during
the enqueue and dequeue operation.
[ peterz: Rewrote it completely ]
Signed-off-by: David Laight <david.laight.linux@xxxxxxxxx>
Signed-off-by: Peter Zijlstra (Intel) <peterz@xxxxxxxxxxxxx>
Signed-off-by: Ingo Molnar <mingo@xxxxxxxxxx>
Cc: Linus Torvalds <torvalds@xxxxxxxxxxxxxxxxxxxx>
Cc: Waiman Long <longman@xxxxxxxxxx>
Link: https://patch.msgid.link/20260907084133.3696-7-david.laight.linux@xxxxxxxxx
---
kernel/locking/osq_lock.c | 49 ++++++++++++++++++--------------------
1 file changed, 24 insertions(+), 25 deletions(-)
diff --git a/kernel/locking/osq_lock.c b/kernel/locking/osq_lock.c
index a6c89e5..735bf62 100644
--- a/kernel/locking/osq_lock.c
+++ b/kernel/locking/osq_lock.c
@@ -13,11 +13,11 @@
*/
struct optimistic_spin_node {
- struct optimistic_spin_node *next;
+ int next_cpu; /* CPU number offset by 1 */
int prev_cpu; /* CPU number offset by 1 */
-};
+} __aligned(8);
-static DEFINE_PER_CPU_SHARED_ALIGNED(struct optimistic_spin_node, osq_node);
+static DEFINE_PER_CPU(struct optimistic_spin_node, osq_node);
/*
* We use the value 0 to represent "no CPU", thus the encoded value
@@ -44,22 +44,21 @@ static inline struct optimistic_spin_node *decode_cpu(int encoded_cpu_val)
* For osq_unlock() there is never a previous node and old_cpu is
* set to OSQ_UNLOCKED_VAL.
*/
-static inline struct optimistic_spin_node *
-osq_wait_unlink_next(struct optimistic_spin_queue *lock,
- int old_cpu)
+static inline int
+osq_wait_unlink_next(struct optimistic_spin_queue *lock, int old_cpu)
{
struct optimistic_spin_node *next, *node = this_cpu_ptr(&osq_node);
- int curr = encode_cpu(smp_processor_id());
+ int next_cpu, node_cpu = encode_cpu(smp_processor_id());
for (;;) {
- if (atomic_read(&lock->tail) == curr &&
- atomic_cmpxchg_release(&lock->tail, curr, old_cpu) == curr) {
+ if (atomic_read(&lock->tail) == node_cpu &&
+ atomic_cmpxchg_release(&lock->tail, node_cpu, old_cpu) == node_cpu) {
/*
* We were the last queued, we moved @lock back. @prev
* will now observe @lock and will complete its
* unlock()/unqueue().
*/
- return NULL;
+ return 0;
}
/*
@@ -72,9 +71,9 @@ osq_wait_unlink_next(struct optimistic_spin_queue *lock,
* wait for either @lock to point to us, through its Step-B, or
* wait for a new @node->next from its Step-C.
*/
- if (node->next) {
- next = xchg(&node->next, NULL);
- if (next)
+ if (node->next_cpu) {
+ next_cpu = xchg(&node->next_cpu, 0);
+ if (next_cpu)
break;
}
@@ -84,16 +83,16 @@ osq_wait_unlink_next(struct optimistic_spin_queue *lock,
/*
* Unlink @node from @next.
*/
+ next = decode_cpu(next_cpu);
WRITE_ONCE(next->prev_cpu, old_cpu);
- return next;
+ return next_cpu;
}
/*
* Set prev->next; this is the counterpart of osq_wait_unlink_next() in that
* by setting ->next the wait is terminated and progress is resumed.
*/
-static inline void osq_link_next(struct optimistic_spin_node *prev,
- struct optimistic_spin_node *next)
+static inline void osq_link_next(struct optimistic_spin_node *prev, int next_cpu)
{
/*
* Suppose:
@@ -128,17 +127,17 @@ static inline void osq_link_next(struct optimistic_spin_node *prev,
*
* Which would result in list corruption.
*/
- smp_store_release(&prev->next, next);
+ smp_store_release(&prev->next_cpu, next_cpu);
}
bool osq_lock(struct optimistic_spin_queue *lock)
{
struct optimistic_spin_node *node = this_cpu_ptr(&osq_node);
- struct optimistic_spin_node *prev, *next;
- int curr = encode_cpu(smp_processor_id());
- int prev_cpu;
+ int node_cpu = encode_cpu(smp_processor_id());
+ struct optimistic_spin_node *prev;
+ int prev_cpu, next_cpu;
- node->next = NULL;
+ node->next_cpu = 0;
/*
* We need both ACQUIRE (pairs with corresponding RELEASE in
@@ -146,13 +145,13 @@ bool osq_lock(struct optimistic_spin_queue *lock)
* the node fields we just initialised) semantics when updating
* the lock tail.
*/
- prev_cpu = atomic_xchg(&lock->tail, curr);
+ prev_cpu = atomic_xchg(&lock->tail, node_cpu);
if (prev_cpu == OSQ_UNLOCKED_VAL)
return true;
prev = decode_cpu(prev_cpu);
node->prev_cpu = prev_cpu;
- osq_link_next(prev, node);
+ osq_link_next(prev, node_cpu);
/*
* Normally @prev is untouchable after the above store; because at that
@@ -191,8 +190,8 @@ bool osq_lock(struct optimistic_spin_queue *lock)
prev = decode_cpu(prev_cpu);
- if (data_race(prev->next) == node &&
- cmpxchg(&prev->next, node, NULL) == node)
+ if (data_race(prev->next_cpu) == node_cpu &&
+ cmpxchg(&prev->next_cpu, node_cpu, 0) == node_cpu)
break;
/*