[RFC PATCH 4/7] mm/shmem: handle memory.high overage in fallocate and reads
From: Shakeel Butt
Date: Sat Nov 21 2026 - 21:48:28 EST
fallocate() on tmpfs or memfd allocates the whole range in one call,
and a read can copy almost 2GiB, faulting in the user buffer and
swapping the file back in. Without swap, this memory can't be
reclaimed, and the internal mount behind memfd_create(),
MAP_SHARED|MAP_ANONYMOUS and SysV shm has no size limit. Today the
charge path throttles both for memory.high, except huge page charges,
which can't block under the default defrag setting. Writes go through
generic_perform_write(), which the previous patch covers.
A later patch stops enforcing memory.high in the charge path, where
the task may hold locks other tasks need. Handle the overage after each
fallocate iteration, where only inode_lock is held, and after each
folio a read copies, where no locks are held. This covers huge pages
too. Reclaim doesn't need inode_lock: shmem_writeout() reads i_private
under i_lock. As in filemap_read(), reads into kernel buffers and
IOCB_NOWAIT or IOCB_WAITQ reads skip it. Reclaim uses the mapping's gfp
constraint.
With swap, this reclaim finds the folios the fallocate added.
shmem_writeout() won't swap them out and counts each try in
nr_unswapped, and once there are more tries than new folios, the
fallocate fails with -ENOMEM. memory.high should throttle the call, not
fail it, so tries made by the fallocating task's own memory.high
reclaim don't count. Tries by other reclaim still count. In a VM with
swap, a 512M tmpfs fallocate under memory.high=64M fails with -ENOMEM 3
times out of 3 today with small pages, and would with huge pages too if
this reclaim's tries counted. With this patch it is throttled instead.
Signed-off-by: Shakeel Butt <shakeel.butt@xxxxxxxxx>
---
mm/shmem.c | 37 ++++++++++++++++++++++++++++++++++---
1 file changed, 34 insertions(+), 3 deletions(-)
diff --git a/mm/shmem.c b/mm/shmem.c
index 951bb3c8f837..fad86c96170f 100644
--- a/mm/shmem.c
+++ b/mm/shmem.c
@@ -110,6 +110,8 @@ struct shmem_falloc {
pgoff_t next; /* the next page offset to be fallocated */
pgoff_t nr_falloced; /* how many new pages have been fallocated */
pgoff_t nr_unswapped; /* how often writeout refused to swap out */
+ /* set while the fallocating task handles memory.high overage */
+ struct task_struct *high_reclaimer;
};
struct shmem_options {
@@ -1918,10 +1920,13 @@ int shmem_writeout(struct swap_io_ctx *ctx, struct folio *folio,
if (shmem_falloc &&
!shmem_falloc->waitq &&
index >= shmem_falloc->start &&
- index < shmem_falloc->next)
- shmem_falloc->nr_unswapped += nr_pages;
- else
+ index < shmem_falloc->next) {
+ /* See the memory.high comment in shmem_fallocate() */
+ if (READ_ONCE(shmem_falloc->high_reclaimer) != current)
+ shmem_falloc->nr_unswapped += nr_pages;
+ } else {
shmem_falloc = NULL;
+ }
spin_unlock(&inode->i_lock);
if (shmem_falloc)
goto redirty;
@@ -3567,6 +3572,7 @@ static ssize_t shmem_file_read_iter(struct kiocb *iocb, struct iov_iter *to)
struct file *file = iocb->ki_filp;
struct inode *inode = file_inode(file);
struct address_space *mapping = inode->i_mapping;
+ gfp_t high_gfp = mapping_gfp_constraint(mapping, GFP_KERNEL);
pgoff_t index;
unsigned long offset;
int error = 0;
@@ -3675,6 +3681,15 @@ static ssize_t shmem_file_read_iter(struct kiocb *iocb, struct iov_iter *to)
error = -EFAULT;
break;
}
+ /*
+ * A large read can charge a lot, for faulting in the buffer
+ * and swapping in the file. Handle memory.high overage here,
+ * where no locks are held. As in filemap_read(), reads into
+ * kernel buffers and reads that must not block skip this.
+ */
+ if (user_backed_iter(to) &&
+ !(iocb->ki_flags & (IOCB_NOWAIT | IOCB_WAITQ)))
+ mem_cgroup_over_high_throttle(high_gfp);
cond_resched();
}
@@ -3883,6 +3898,7 @@ static long shmem_fallocate(struct file *file, int mode, loff_t offset,
struct inode *inode = file_inode(file);
struct shmem_sb_info *sbinfo = SHMEM_SB(inode->i_sb);
struct shmem_inode_info *info = SHMEM_I(inode);
+ gfp_t high_gfp = mapping_gfp_constraint(inode->i_mapping, GFP_KERNEL);
struct shmem_falloc shmem_falloc;
pgoff_t start, index, end, undo_fallocend;
loff_t aligned_end;
@@ -3962,6 +3978,7 @@ static long shmem_fallocate(struct file *file, int mode, loff_t offset,
shmem_falloc.next = start;
shmem_falloc.nr_falloced = 0;
shmem_falloc.nr_unswapped = 0;
+ shmem_falloc.high_reclaimer = NULL;
spin_lock(&inode->i_lock);
WRITE_ONCE(inode->i_private, &shmem_falloc);
spin_unlock(&inode->i_lock);
@@ -4031,6 +4048,20 @@ static long shmem_fallocate(struct file *file, int mode, loff_t offset,
folio_mark_dirty(folio);
folio_unlock(folio);
folio_put(folio);
+
+ /*
+ * Handle memory.high overage here, where only inode_lock is
+ * held; reclaim doesn't take it. shmem_writeout() won't swap
+ * out the folios added so far, and counts each try toward the
+ * -ENOMEM check above. memory.high should only throttle the
+ * fallocate, so tell shmem_writeout() not to count tries made
+ * by this reclaim.
+ */
+ if (mem_cgroup_over_high_pending()) {
+ WRITE_ONCE(shmem_falloc.high_reclaimer, current);
+ mem_cgroup_handle_over_high(high_gfp);
+ WRITE_ONCE(shmem_falloc.high_reclaimer, NULL);
+ }
cond_resched();
}
--
2.53.1-Meta