[RFC PATCH v25 11/14] mm: shmem: use a limit counter for the tmpfs size limit

From: Mathieu Desnoyers

Date: Wed Nov 11 2026 - 15:42:18 EST


The size limit of tmpfs (the size= mount option) is a struct
percpu_counter. Each block allocation checks it with
percpu_counter_limited_add(). That check takes no lock only while the
mount is more than batch * nr_cpus pages below its size. With the
default batch size of 768 at 384 CPUs, this is 294912 pages, 1.1 GB.
A smaller mount, such as a 64 MB /dev/shm, takes the counter spinlock
and sums all CPUs on every page. On a 384-CPU host with 384 processes
allocating, 99.5% of the time goes to that spinlock.

Use a hierarchical limit counter instead. Its batch size is chosen from
the size. When the size allows, its accuracy is within 1/4 of the size.
Block allocations then sum the CPUs only in the last 1/4 of the size.
They take no lock. The limit stays strict.

Near a full mount, an allocation can fail although it fits. This
happens while a concurrent allocation of a few pages, which does not
fit, is being undone. A large folio is checked before it is added, so a
large folio which does not fit has no such effect.

A mount without a size limit never counts its blocks. A remount cannot
add a limit. So such a mount no longer allocates the counter.

A remount keeps the batch size. A mount grown by remount thus keeps the
smaller batch size of its first size. Under concurrent allocations, a
remount to a smaller size can be refused although the use fits.

Test: fallocate() of one page at a time into one file per process, all
on one mount. This is a will-it-scale variant. It ran in a 384-vCPU
guest pinned to a 2-socket AMD EPYC 9654 host, on v7.4-rc2. In
thousands of calls per second:

size= processes before after
64 MB 1 208 483
64 MB 64 256 17771
64 MB 384 126 54275
1 GB 384 125 93879
2 GB 384 93851 93476
16 GB 384 93737 94806

Below 1.1 GB, the mount scales again with the number of processes. At
2 GB and above, the results are within 1% of before. Within the last
1/4 of the size, each allocation sums both counters over all CPUs. With
one process, this is 37% slower than before. With 384 processes, it is
about 100 times faster.

Signed-off-by: Mathieu Desnoyers <mathieu.desnoyers@xxxxxxxxxxxx>
---
include/linux/shmem_fs.h | 3 ++-
mm/shmem.c | 33 +++++++++++++++++++++++----------
2 files changed, 25 insertions(+), 11 deletions(-)

diff --git a/include/linux/shmem_fs.h b/include/linux/shmem_fs.h
index f89b4b151133..b37adf32e4ef 100644
--- a/include/linux/shmem_fs.h
+++ b/include/linux/shmem_fs.h
@@ -7,6 +7,7 @@
#include <linux/mempolicy.h>
#include <linux/pagemap.h>
#include <linux/percpu_counter.h>
+#include <linux/percpu_counter_tree.h>
#include <linux/xattr.h>
#include <linux/fs_parser.h>
#include <linux/userfaultfd_k.h>
@@ -76,7 +77,7 @@ struct shmem_quota_limits {

struct shmem_sb_info {
unsigned long max_blocks; /* How many blocks are allowed */
- struct percpu_counter used_blocks; /* How many are allocated */
+ struct percpu_counter_tree_limit used_blocks; /* How many are allocated */
unsigned long max_inodes; /* How many inodes are allowed */
unsigned long free_ispace; /* How much ispace left for allocation */
raw_spinlock_t stat_lock; /* Serialize shmem_sb_info changes */
diff --git a/mm/shmem.c b/mm/shmem.c
index b8399bb71997..2e368d7b0e41 100644
--- a/mm/shmem.c
+++ b/mm/shmem.c
@@ -231,13 +231,13 @@ int shmem_inode_acct_blocks(struct inode *inode, long pages)

might_sleep(); /* when quotas */
if (sbinfo->max_blocks) {
- if (!percpu_counter_limited_add(&sbinfo->used_blocks,
- sbinfo->max_blocks, pages))
+ if (!percpu_counter_tree_limit_add(&sbinfo->used_blocks,
+ sbinfo->max_blocks, pages))
goto unacct;

err = dquot_alloc_block_nodirty(inode, pages);
if (err) {
- percpu_counter_sub(&sbinfo->used_blocks, pages);
+ percpu_counter_tree_limit_sub(&sbinfo->used_blocks, pages);
goto unacct;
}
} else {
@@ -262,7 +262,7 @@ static void shmem_inode_unacct_blocks(struct inode *inode, long pages)
dquot_free_block_nodirty(inode, pages);

if (sbinfo->max_blocks)
- percpu_counter_sub(&sbinfo->used_blocks, pages);
+ percpu_counter_tree_limit_sub(&sbinfo->used_blocks, pages);
shmem_unacct_blocks(info->flags, pages);
}

@@ -4048,9 +4048,11 @@ static int shmem_statfs(struct dentry *dentry, struct kstatfs *buf)
buf->f_namelen = NAME_MAX;
if (sbinfo->max_blocks) {
buf->f_blocks = sbinfo->max_blocks;
+ /* Under concurrent allocations, the sum can be above the size. */
buf->f_bavail =
buf->f_bfree = sbinfo->max_blocks -
- percpu_counter_sum(&sbinfo->used_blocks);
+ min(percpu_counter_tree_limit_precise_sum(&sbinfo->used_blocks),
+ sbinfo->max_blocks);
}
if (sbinfo->max_inodes) {
buf->f_files = sbinfo->max_inodes;
@@ -5020,8 +5022,8 @@ static int shmem_reconfigure(struct fs_context *fc)
err = "Cannot retroactively limit size";
goto out;
}
- if (percpu_counter_compare(&sbinfo->used_blocks,
- ctx->blocks) > 0) {
+ if (percpu_counter_tree_limit_precise_sum(&sbinfo->used_blocks) >
+ ctx->blocks) {
err = "Too small a size for current use";
goto out;
}
@@ -5187,7 +5189,7 @@ static void shmem_put_super(struct super_block *sb)
shmem_disable_quotas(sb);
#endif
free_percpu(sbinfo->ino_batch);
- percpu_counter_destroy(&sbinfo->used_blocks);
+ percpu_counter_tree_limit_destroy(&sbinfo->used_blocks);
#ifdef CONFIG_TRANSPARENT_HUGEPAGE
list_lru_destroy(&sbinfo->shrinklist);
#endif
@@ -5212,6 +5214,7 @@ static int shmem_fill_super(struct super_block *sb, struct fs_context *fc)
static struct lock_class_key shmem_unused_huge_lock_key;
#endif
struct shmem_options *ctx = fc->fs_private;
+ unsigned long batch;
struct inode *inode;
struct shmem_sb_info *sbinfo;
int error = -ENOMEM;
@@ -5286,8 +5289,18 @@ static int shmem_fill_super(struct super_block *sb, struct fs_context *fc)
ctx->mpol = NULL;

raw_spin_lock_init(&sbinfo->stat_lock);
- if (percpu_counter_init(&sbinfo->used_blocks, 0, GFP_KERNEL))
- goto failed;
+ /*
+ * A mount without a size limit never counts its blocks. A remount
+ * cannot add a limit. Otherwise, pick the batch size from the size.
+ * When the size allows, its accuracy is within 1/4 of the size. A
+ * block allocation then sums all CPUs only within the last 1/4 of the
+ * size. A remount keeps the batch size.
+ */
+ if (sbinfo->max_blocks) {
+ batch = percpu_counter_tree_batch_for_accuracy(sbinfo->max_blocks / 4);
+ if (percpu_counter_tree_limit_init(&sbinfo->used_blocks, batch, GFP_KERNEL))
+ goto failed;
+ }

#ifdef CONFIG_TRANSPARENT_HUGEPAGE
if (list_lru_init_memcg_key(&sbinfo->shrinklist, sb->s_shrink,
--
2.47.3