[PATCH] mm: shmem: fix incorrect vm_flags usage when checking allowable orders

[Baolin Wang]

Lance reported that when MADV_HUGEPAGE is set on a tmpfs file mounted with
huge=advise option, khugepaged fails the allowable order check and does not
scan the tmpfs file for collapse.

After commit 6beeab870e70 ("mm: shmem: move shmem_huge_global_enabled() into
shmem_allowable_huge_orders()"), the shmem/tmpfs allowable order check reads
vma->vm_flags directly.  However, when MADV_HUGEPAGE is handled,
khugepaged_enter_vma() is called before the VMA's vm_flags have been updated,
so the check uses stale flags and incorrectly rejects the VMA for collapse.
As a result, khugepaged does not collapse the tmpfs file into PMD order in time.

Fix this by passing vm_flags as a parameter to shmem_allowable_huge_orders()
instead of reading it from the vm_area_struct.

Reported-by: Lance Yang <lance.yang@xxxxxxxxx>
Closes: https://lore.kernel.org/all/20260815181632.21453-1-lance.yang@xxxxxxxxx/
Fixes: 6beeab870e70 ("mm: shmem: move shmem_huge_global_enabled() into shmem_allowable_huge_orders()")
Cc: stable@xxxxxxxxxxxxxxx
Signed-off-by: Baolin Wang <baolin.wang@xxxxxxxxxxxxxxxxx>
---
 include/linux/shmem_fs.h | 8 ++++----
 mm/huge_memory.c         | 2 +-
 mm/shmem.c               | 9 +++++----
 3 files changed, 10 insertions(+), 9 deletions(-)

diff --git a/include/linux/shmem_fs.h b/include/linux/shmem_fs.h
index 5663dff53186..321017e0fd63 100644
--- a/include/linux/shmem_fs.h
+++ b/include/linux/shmem_fs.h
@@ -127,13 +127,13 @@ int shmem_unuse(unsigned int type);
 
 #if defined(CONFIG_TRANSPARENT_HUGEPAGE) && defined(CONFIG_SHMEM)
 unsigned long shmem_allowable_huge_orders(struct inode *inode,
-				struct vm_area_struct *vma, pgoff_t index,
-				loff_t write_end, bool shmem_huge_force);
+		struct vm_area_struct *vma, vm_flags_t vm_flags,
+		pgoff_t index, loff_t write_end, bool shmem_huge_force);
 bool shmem_hpage_pmd_enabled(void);
 #else
 static inline unsigned long shmem_allowable_huge_orders(struct inode *inode,
-				struct vm_area_struct *vma, pgoff_t index,
-				loff_t write_end, bool shmem_huge_force)
+		struct vm_area_struct *vma, vm_flags_t vm_flags, pgoff_t index,
+		loff_t write_end, bool shmem_huge_force)
 {
 	return 0;
 }
diff --git a/mm/huge_memory.c b/mm/huge_memory.c
index ced400f72d43..70f57d700739 100644
--- a/mm/huge_memory.c
+++ b/mm/huge_memory.c
@@ -187,7 +187,7 @@ unsigned long __thp_vma_allowable_orders(struct vm_area_struct *vma,
 	 */
 	if (!in_pf && shmem_file(vma->vm_file))
 		return orders & shmem_allowable_huge_orders(file_inode(vma->vm_file),
-						   vma, vma_start_pgoff(vma), 0,
+						   vma, vm_flags, vma_start_pgoff(vma), 0,
 						   forced_collapse);
 
 	if (!vma_is_anonymous(vma)) {
diff --git a/mm/shmem.c b/mm/shmem.c
index 599665a3d6e7..6f1ad5456aca 100644
--- a/mm/shmem.c
+++ b/mm/shmem.c
@@ -1834,12 +1834,11 @@ bool shmem_hpage_pmd_enabled(void)
 }
 
 unsigned long shmem_allowable_huge_orders(struct inode *inode,
-				struct vm_area_struct *vma, pgoff_t index,
-				loff_t write_end, bool shmem_huge_force)
+		struct vm_area_struct *vma, vm_flags_t vm_flags,
+		pgoff_t index, loff_t write_end, bool shmem_huge_force)
 {
 	unsigned long mask = READ_ONCE(huge_shmem_orders_always);
 	unsigned long within_size_orders = READ_ONCE(huge_shmem_orders_within_size);
-	vm_flags_t vm_flags = vma ? vma->vm_flags : 0;
 	unsigned int global_orders;
 
 	if (thp_disabled_by_hw() || (vma && vma_thp_disabled(vma, vm_flags, shmem_huge_force)))
@@ -2430,6 +2429,7 @@ static int shmem_get_folio_gfp(struct inode *inode, pgoff_t index,
 		gfp_t gfp, struct vm_fault *vmf, vm_fault_t *fault_type)
 {
 	struct vm_area_struct *vma = vmf ? vmf->vma : NULL;
+	vm_flags_t vm_flags = vma ? vma->vm_flags : 0;
 	struct mm_struct *fault_mm;
 	struct folio *folio;
 	int error;
@@ -2507,7 +2507,8 @@ static int shmem_get_folio_gfp(struct inode *inode, pgoff_t index,
 	}
 
 	/* Find hugepage orders that are allowed for anonymous shmem and tmpfs. */
-	orders = shmem_allowable_huge_orders(inode, vma, index, write_end, false);
+	orders = shmem_allowable_huge_orders(inode, vma, vm_flags, index,
+					     write_end, false);
 	if (orders > 0) {
 		gfp_t huge_gfp;
 
-- 
2.47.3