[PATCH] lib/percpu_counter: allow concurrent sums with an rwlock

From: Jiebin Sun

Date: Sun Nov 15 2026 - 20:52:32 EST


__percpu_counter_sum() holds a raw spinlock while reading the global
count and per-CPU deltas, serializing concurrent sums of the same
counter. Since a sum changes neither the global count nor the per-CPU
deltas, use rwlock_t on !PREEMPT_RT to allow concurrent sums with a
read lock. Global updates take the write lock. The per-CPU fast path
remains lockless, so sums keep their existing snapshot semantics.

stress-ng dentry reaches this path through ext4_statfs():

statfs -> ext4_statfs -> __percpu_counter_sum
-> native_queued_spin_lock_slowpath

Keep the raw spinlock on PREEMPT_RT, where rwlock_t can sleep.
Global counter updates can run in atomic context and must not sleep.

stress-ng dentry test average throughput change:

+------------------------+------------+--------------------+
| Case | Unit | Average change (%) |
+------------------------+------------+--------------------+
| Dentry, 64 workers | bogo ops/s | +47.34 |
+------------------------+------------+--------------------+

Dentry ran on ext4 with 64 workers pinned to physical CPUs:

stress-ng --dentry 64 --dentries 4096 --timeout 20s

Suggested-by: Tim Chen <tim.c.chen@xxxxxxxxxxxxxxx>
Reviewed-by: Tianyou Li <tianyou.li@xxxxxxxxx>
Reviewed-by: Wangyang Guo <wangyang.guo@xxxxxxxxx>
Signed-off-by: Jiebin Sun <jiebin.sun@xxxxxxxxx>
---
include/linux/percpu_counter.h | 4 ++
lib/percpu_counter.c | 71 ++++++++++++++++++++++++++--------
2 files changed, 59 insertions(+), 16 deletions(-)

diff --git a/include/linux/percpu_counter.h b/include/linux/percpu_counter.h
index 3a44dd1e33d2..7b28b0e22207 100644
--- a/include/linux/percpu_counter.h
+++ b/include/linux/percpu_counter.h
@@ -20,7 +20,11 @@
#ifdef CONFIG_SMP

struct percpu_counter {
+#ifdef CONFIG_PREEMPT_RT
raw_spinlock_t lock;
+#else
+ rwlock_t lock; /* shared for sums, exclusive for updates */
+#endif
s64 count;
#ifdef CONFIG_HOTPLUG_CPU
struct list_head list; /* All percpu_counters are on a list */
diff --git a/lib/percpu_counter.c b/lib/percpu_counter.c
index 87b261b87b0b..ce0284bacf07 100644
--- a/lib/percpu_counter.c
+++ b/lib/percpu_counter.c
@@ -15,6 +15,46 @@ static LIST_HEAD(percpu_counters);
static DEFINE_SPINLOCK(percpu_counters_lock);
#endif

+/*
+ * Non-RT sums share the lock; global updates take it exclusively.
+ * Keep raw spinlocks on RT: rwlocks may sleep and their irqsave variants
+ * do not disable interrupts. Updates run in atomic contexts and need
+ * IRQ exclusion for CPU-local read-modify-write sequences.
+ */
+#ifdef CONFIG_PREEMPT_RT
+#define fbc_lock(fbc) raw_spin_lock(&(fbc)->lock)
+#define fbc_unlock(fbc) raw_spin_unlock(&(fbc)->lock)
+#define fbc_lock_irqsave(fbc, f) raw_spin_lock_irqsave(&(fbc)->lock, f)
+#define fbc_unlock_irqrestore(fbc, f) raw_spin_unlock_irqrestore(&(fbc)->lock, f)
+#define fbc_read_lock_irqsave fbc_lock_irqsave
+#define fbc_read_unlock_irqrestore fbc_unlock_irqrestore
+#else
+#define fbc_lock(fbc) write_lock(&(fbc)->lock)
+#define fbc_unlock(fbc) write_unlock(&(fbc)->lock)
+#define fbc_lock_irqsave(fbc, f) write_lock_irqsave(&(fbc)->lock, f)
+#define fbc_unlock_irqrestore(fbc, f) write_unlock_irqrestore(&(fbc)->lock, f)
+#define fbc_read_lock_irqsave(fbc, f) read_lock_irqsave(&(fbc)->lock, f)
+#define fbc_read_unlock_irqrestore(fbc, f) read_unlock_irqrestore(&(fbc)->lock, f)
+#endif
+
+static void fbc_lock_init(struct percpu_counter *fbc,
+ struct lock_class_key *key)
+{
+#ifdef CONFIG_PREEMPT_RT
+ raw_spin_lock_init(&fbc->lock);
+ lockdep_set_class(&fbc->lock, key);
+#else
+ rwlock_init(&fbc->lock);
+ /*
+ * rwlock_t normally uses LD_WAIT_CONFIG because it sleeps on RT.
+ * This counter uses a raw spinlock on RT, so it can be taken under
+ * raw spinlocks or with IRQs off on every configuration.
+ */
+ lockdep_init_map_wait(&fbc->lock.dep_map, "&fbc->lock", key, 0,
+ LD_WAIT_SPIN);
+#endif
+}
+
#ifdef CONFIG_DEBUG_OBJECTS_PERCPU_COUNTER

static const struct debug_obj_descr percpu_counter_debug_descr;
@@ -62,13 +102,13 @@ void percpu_counter_set(struct percpu_counter *fbc, s64 amount)
int cpu;
unsigned long flags;

- raw_spin_lock_irqsave(&fbc->lock, flags);
+ fbc_lock_irqsave(fbc, flags);
for_each_possible_cpu(cpu) {
s32 *pcount = per_cpu_ptr(fbc->counters, cpu);
*pcount = 0;
}
fbc->count = amount;
- raw_spin_unlock_irqrestore(&fbc->lock, flags);
+ fbc_unlock_irqrestore(fbc, flags);
}
EXPORT_SYMBOL(percpu_counter_set);

@@ -98,7 +138,7 @@ void percpu_counter_add_batch(struct percpu_counter *fbc, s64 amount, s32 batch)
count = this_cpu_read(*fbc->counters);
do {
if (unlikely(abs(count + amount) >= batch)) {
- raw_spin_lock_irqsave(&fbc->lock, flags);
+ fbc_lock_irqsave(fbc, flags);
/*
* Note: by now we might have migrated to another CPU
* or the value might have changed.
@@ -106,7 +146,7 @@ void percpu_counter_add_batch(struct percpu_counter *fbc, s64 amount, s32 batch)
count = __this_cpu_read(*fbc->counters);
fbc->count += count + amount;
__this_cpu_sub(*fbc->counters, count);
- raw_spin_unlock_irqrestore(&fbc->lock, flags);
+ fbc_unlock_irqrestore(fbc, flags);
return;
}
} while (!this_cpu_try_cmpxchg(*fbc->counters, &count, count + amount));
@@ -125,10 +165,10 @@ void percpu_counter_add_batch(struct percpu_counter *fbc, s64 amount, s32 batch)
local_irq_save(flags);
count = __this_cpu_read(*fbc->counters) + amount;
if (abs(count) >= batch) {
- raw_spin_lock(&fbc->lock);
+ fbc_lock(fbc);
fbc->count += count;
__this_cpu_sub(*fbc->counters, count - amount);
- raw_spin_unlock(&fbc->lock);
+ fbc_unlock(fbc);
} else {
this_cpu_add(*fbc->counters, amount);
}
@@ -148,11 +188,11 @@ void percpu_counter_sync(struct percpu_counter *fbc)
unsigned long flags;
s64 count;

- raw_spin_lock_irqsave(&fbc->lock, flags);
+ fbc_lock_irqsave(fbc, flags);
count = __this_cpu_read(*fbc->counters);
fbc->count += count;
__this_cpu_sub(*fbc->counters, count);
- raw_spin_unlock_irqrestore(&fbc->lock, flags);
+ fbc_unlock_irqrestore(fbc, flags);
}
EXPORT_SYMBOL(percpu_counter_sync);

@@ -174,13 +214,13 @@ s64 __percpu_counter_sum(struct percpu_counter *fbc)
int cpu;
unsigned long flags;

- raw_spin_lock_irqsave(&fbc->lock, flags);
+ fbc_read_lock_irqsave(fbc, flags);
ret = fbc->count;
for_each_cpu_or(cpu, cpu_online_mask, cpu_dying_mask) {
s32 *pcount = per_cpu_ptr(fbc->counters, cpu);
ret += *pcount;
}
- raw_spin_unlock_irqrestore(&fbc->lock, flags);
+ fbc_read_unlock_irqrestore(fbc, flags);
return ret;
}
EXPORT_SYMBOL(__percpu_counter_sum);
@@ -203,8 +243,7 @@ int __percpu_counter_init_many(struct percpu_counter *fbc, s64 amount,
}

for (i = 0; i < nr_counters; i++) {
- raw_spin_lock_init(&fbc[i].lock);
- lockdep_set_class(&fbc[i].lock, key);
+ fbc_lock_init(&fbc[i], key);
#ifdef CONFIG_HOTPLUG_CPU
INIT_LIST_HEAD(&fbc[i].list);
#endif
@@ -274,11 +313,11 @@ static int percpu_counter_cpu_dead(unsigned int cpu)
list_for_each_entry(fbc, &percpu_counters, list) {
s32 *pcount;

- raw_spin_lock(&fbc->lock);
+ fbc_lock(fbc);
pcount = per_cpu_ptr(fbc->counters, cpu);
fbc->count += *pcount;
*pcount = 0;
- raw_spin_unlock(&fbc->lock);
+ fbc_unlock(fbc);
}
spin_unlock_irq(&percpu_counters_lock);
#endif
@@ -354,7 +393,7 @@ bool __percpu_counter_limited_add(struct percpu_counter *fbc,
return true;
}

- raw_spin_lock(&fbc->lock);
+ fbc_lock(fbc);
count = fbc->count + amount;

/* Skip percpu_counter_sum() when safe */
@@ -392,7 +431,7 @@ bool __percpu_counter_limited_add(struct percpu_counter *fbc,
fbc->count += count + amount;
__this_cpu_sub(*fbc->counters, count);
out:
- raw_spin_unlock(&fbc->lock);
+ fbc_unlock(fbc);
local_irq_restore(flags);
return good;
}
--
2.52.0