[PATCH] mm: prevent VMA generation reuse from skipping readers

[Daehyeon Ko]

Per-VMA write locking treats equality between vm_lock_seq and the current
mm_lock_seq as proof that the active mmap writer already excluded readers.

Both values are 32-bit. After 2^31 complete mmap write-lock acquisitions,
an old VMA stamp can alias the live odd generation. vma_start_write() then
returns without waiting for an existing reader.

A capability-free reproducer uses brk(0) to recycle the generation, then
races an anonymous fault with MREMAP_DONTUNMAP. The stale equality lets
mremap move and tear down a PMD table while the fault retains vmf->pmd.
Unmodified kernels panic in folio_add_new_anon_rmap(), while diagnostic
runs show a freed PMD access and a writable mapping created in another
PROT_NONE VMA.

Add an epoch that advances whenever mm_lock_seq wraps. Stamp it with the
sequence after excluding readers, and require both fields to match before a
writer takes the already-locked fast path. Reader checks remain unchanged;
reused sequence values only cause a pessimistic fallback there.

The fixed KASAN kernel completed the real-wrap THP schedule on three fresh
boots without KASAN, warnings, oopses or panics. Kernels before v6.15 use
the older rwsem-based per-VMA locking and need an adapted backport.

Fixes: 5e31275cc997 ("mm: add per-VMA lock and helper functions to control it")
Link: https://lore.kernel.org/r/20241010205644.3831427-3-andrii@xxxxxxxxxx
Cc: stable@xxxxxxxxxxxxxxx
Assisted-by: LLM
Signed-off-by: Daehyeon Ko <4ncienth@xxxxxxxxx>
---
 include/linux/mm_types.h        | 10 +++++++---
 include/linux/mmap_lock.h       | 14 +++++++++++---
 mm/mmap_lock.c                  |  2 ++
 tools/testing/vma/include/dup.h | 10 ++++++----
 4 files changed, 26 insertions(+), 10 deletions(-)

diff --git a/include/linux/mm_types.h b/include/linux/mm_types.h
index f3e5a2fadbe5b7..309a278e552451 100644
--- a/include/linux/mm_types.h
+++ b/include/linux/mm_types.h
@@ -961,9 +961,9 @@ struct vm_area_struct {
 	 * Can be read unreliably (using READ_ONCE()) for pessimistic bailout
 	 * while holding nothing (except RCU to keep the VMA struct allocated).
 	 *
-	 * This sequence counter is explicitly allowed to overflow; sequence
-	 * counter reuse can only lead to occasional unnecessary use of the
-	 * slowpath.
+	 * This sequence counter is explicitly allowed to overflow. Readers can
+	 * treat a reused value as a pessimistic false positive. Writers also
+	 * compare vm_lock_seq_epoch so reuse cannot skip reader exclusion.
 	 */
 	unsigned int vm_lock_seq;
 #endif
@@ -1004,6 +1004,8 @@ struct vm_area_struct {
 	struct vma_numab_state *numab_state;	/* NUMA Balancing state */
 #endif
 #ifdef CONFIG_PER_VMA_LOCK
+	/* Extends vm_lock_seq without moving the hot VMA header fields. */
+	unsigned int vm_lock_seq_epoch;
 	/*
 	 * Used to keep track of firstly, whether the VMA is attached, secondly,
 	 * if attached, how many read locks are taken, and thirdly, if the
@@ -1274,6 +1276,8 @@ struct mm_struct {
 		 * mmap_lock.
 		 */
 		seqcount_t mm_lock_seq;
+		/* Extends the VMA writer generation across sequence wrap. */
+		unsigned int mm_lock_seq_epoch;
 #endif
 		struct futex_mm_data	futex;
 
diff --git a/include/linux/mmap_lock.h b/include/linux/mmap_lock.h
index b8a13b8d36a45b..27caf8818e0b75 100644
--- a/include/linux/mmap_lock.h
+++ b/include/linux/mmap_lock.h
@@ -118,6 +118,7 @@ static inline void mmap_assert_write_locked(const struct mm_struct *mm)
 static inline void mm_lock_seqcount_init(struct mm_struct *mm)
 {
 	seqcount_init(&mm->mm_lock_seq);
+	mm->mm_lock_seq_epoch = 0;
 }
 
 static inline void mm_lock_seqcount_begin(struct mm_struct *mm)
@@ -128,6 +129,8 @@ static inline void mm_lock_seqcount_begin(struct mm_struct *mm)
 static inline void mm_lock_seqcount_end(struct mm_struct *mm)
 {
 	ASSERT_EXCLUSIVE_WRITER(mm->mm_lock_seq);
+	if (unlikely(mm->mm_lock_seq.sequence == UINT_MAX))
+		mm->mm_lock_seq_epoch++;
 	do_raw_write_seqcount_end(&mm->mm_lock_seq);
 }
 
@@ -157,6 +160,7 @@ static inline void vma_lock_init(struct vm_area_struct *vma, bool reset_refcnt)
 	if (reset_refcnt)
 		refcount_set(&vma->vm_refcnt, 0);
 	vma->vm_lock_seq = UINT_MAX;
+	vma->vm_lock_seq_epoch = UINT_MAX;
 }
 
 /*
@@ -281,11 +285,15 @@ static inline unsigned int __vma_raw_mm_seqnum(struct vm_area_struct *vma)
  */
 static inline bool __is_vma_write_locked(struct vm_area_struct *vma)
 {
+	const struct mm_struct *mm = vma->vm_mm;
+
 	/*
-	 * current task is holding mmap_write_lock, both vma->vm_lock_seq and
-	 * mm->mm_lock_seq can't be concurrently modified.
+	 * The current task holds mmap_write_lock, so neither generation can be
+	 * concurrently modified. The epoch prevents a wrapped sequence from
+	 * making an old VMA stamp look current to a writer.
 	 */
-	return vma->vm_lock_seq == __vma_raw_mm_seqnum(vma);
+	return vma->vm_lock_seq == __vma_raw_mm_seqnum(vma) &&
+		vma->vm_lock_seq_epoch == mm->mm_lock_seq_epoch;
 }
 
 int __vma_start_write(struct vm_area_struct *vma, int state);
diff --git a/mm/mmap_lock.c b/mm/mmap_lock.c
index 898c2ef1e95803..281659225b1dbb 100644
--- a/mm/mmap_lock.c
+++ b/mm/mmap_lock.c
@@ -139,6 +139,7 @@ static int __vma_start_exclude_readers(struct vma_exclude_readers_state *ves)
 int __vma_start_write(struct vm_area_struct *vma, int state)
 {
 	const unsigned int mm_lock_seq = __vma_raw_mm_seqnum(vma);
+	const unsigned int mm_lock_seq_epoch = vma->vm_mm->mm_lock_seq_epoch;
 	struct vma_exclude_readers_state ves = {
 		.vma = vma,
 		.state = state,
@@ -158,6 +159,7 @@ int __vma_start_write(struct vm_area_struct *vma, int state)
 	 * we should use WRITE_ONCE() for cleanliness and to keep KCSAN happy.
 	 */
 	WRITE_ONCE(vma->vm_lock_seq, mm_lock_seq);
+	WRITE_ONCE(vma->vm_lock_seq_epoch, mm_lock_seq_epoch);
 
 	if (ves.exclusive) {
 		__vma_end_exclude_readers(&ves);
diff --git a/tools/testing/vma/include/dup.h b/tools/testing/vma/include/dup.h
index 4c58487b764e9d..211ffbf5850057 100644
--- a/tools/testing/vma/include/dup.h
+++ b/tools/testing/vma/include/dup.h
@@ -571,9 +571,9 @@ struct vm_area_struct {
 	 * Can be read unreliably (using READ_ONCE()) for pessimistic bailout
 	 * while holding nothing (except RCU to keep the VMA struct allocated).
 	 *
-	 * This sequence counter is explicitly allowed to overflow; sequence
-	 * counter reuse can only lead to occasional unnecessary use of the
-	 * slowpath.
+	 * This sequence counter is explicitly allowed to overflow. Readers can
+	 * treat a reused value as a pessimistic false positive. Writers also
+	 * compare vm_lock_seq_epoch so reuse cannot skip reader exclusion.
 	 */
 	unsigned int vm_lock_seq;
 #endif
@@ -611,7 +611,8 @@ struct vm_area_struct {
 	struct vma_numab_state *numab_state;	/* NUMA Balancing state */
 #endif
 #ifdef CONFIG_PER_VMA_LOCK
-	/* Unstable RCU readers are allowed to read this. */
+	/* Extends vm_lock_seq without moving the hot VMA header fields. */
+	unsigned int vm_lock_seq_epoch;
 	refcount_t vm_refcnt;
 #endif
 #ifdef CONFIG_64BIT
@@ -1230,6 +1231,7 @@ static inline void vma_init(struct vm_area_struct *vma, struct mm_struct *mm)
 	vma->vm_ops = &vma_dummy_vm_ops;
 	INIT_LIST_HEAD(&vma->anon_vma_chain);
 	vma->vm_lock_seq = UINT_MAX;
+	vma->vm_lock_seq_epoch = UINT_MAX;
 }
 
 /*

base-commit: 0fcb05b72344f40806cbe810de8872f1698bf053
-- 
2.55.0