[PATCH] mm/hugetlb_vmemmap: fix PMD split TLB flush race
From: Muchun Song
Date: Tue Nov 10 2026 - 06:40:56 EST
Bulk HVO optimization converts vmemmap PMDs for a list of folios into
PTE tables before issuing a single global TLB flush. After publishing
a PTE table, a concurrent single-folio optimization can observe it and
remap its PTEs before the deferred flush:
CPU0 CPU1
hugetlb_vmemmap_optimize_folios()
hugetlb_vmemmap_pteify_folio()
vmemmap_split_pmd()
pmd_populate_kernel()
/* TLB flush deferred */
hugetlb_vmemmap_optimize_folio()
vmemmap_pmd_entry()
/* sees a PTE table */
vmemmap_optimize_pte()
set_pte_at()
flush_tlb_all()
This can create overlapping TLB entries at different granularities
that map the same virtual address to different physical pages.
Serialize deferred PMD splitting against PTE remapping with init_mm's
mmap lock. Hold it for write across the entire pteify batch and the
final TLB flush, while optimize and restore remaps hold it for read.
A remapper can then observe a split PMD only after stale PMD
translations have been invalidated.
Fixes: f4b7e3efaddb ("hugetlb: batch PMD split for bulk vmemmap dedup")
Cc: stable@xxxxxxxxxxxxxxx
Reported-by: Sashiko <sashiko-bot@xxxxxxxxxx>
Closes: https://sashiko.dev/#/bug/linux-fbdc3410-a4cf-457e-88e6-684380fbadfe
Signed-off-by: Muchun Song <songmuchun@xxxxxxxxxxxxx>
---
mm/hugetlb_vmemmap.c | 17 +++++++++++++----
1 file changed, 13 insertions(+), 4 deletions(-)
diff --git a/mm/hugetlb_vmemmap.c b/mm/hugetlb_vmemmap.c
index ba8e11e399b2..ba6559bef742 100644
--- a/mm/hugetlb_vmemmap.c
+++ b/mm/hugetlb_vmemmap.c
@@ -158,10 +158,8 @@ static int vmemmap_remap_range(unsigned long start, unsigned long end,
VM_BUG_ON(!PAGE_ALIGNED(start | end));
- mmap_read_lock(&init_mm);
ret = walk_kernel_page_table_range(start, end, &vmemmap_remap_ops,
NULL, walk);
- mmap_read_unlock(&init_mm);
if (ret)
return ret;
@@ -308,6 +306,9 @@ static inline int vmemmap_restore_range(unsigned long start, unsigned long end,
* is skipped, the caller must flush TLBs after all pteify operations are complete
* before remapping the folio's vmemmap pages.
*
+ * Note: The caller must hold init_mm's mmap write lock for all pteify operations
+ * and the final TLB flush.
+ *
* Return: %0 on success, negative error code otherwise.
*/
static int vmemmap_pteify_folio(const struct folio *folio, unsigned long flags)
@@ -318,6 +319,8 @@ static int vmemmap_pteify_folio(const struct folio *folio, unsigned long flags)
.flags = flags,
};
+ mmap_assert_write_locked(&init_mm);
+
return vmemmap_remap_range(start, end, &walk);
}
@@ -358,6 +361,7 @@ static int vmemmap_optimize_folio(const struct folio *folio,
if (ret)
return ret;
+ guard(mmap_read_lock)(&init_mm);
ret = vmemmap_remap_range(start, end, &walk);
if (!ret || !walk.nr_walked)
return ret;
@@ -392,6 +396,8 @@ static int vmemmap_restore_folio(const struct folio *folio, unsigned long flags)
if (ret)
return ret;
+ guard(mmap_read_lock)(&init_mm);
+
return vmemmap_restore_range(start, end, &vmemmap_pages, flags);
}
@@ -559,6 +565,7 @@ void hugetlb_vmemmap_optimize_folios(struct hstate *h, struct list_head *folio_l
if (list_empty(folio_list))
return;
+ mmap_write_lock(&init_mm);
list_for_each_entry(folio, folio_list, lru) {
int ret;
@@ -583,11 +590,13 @@ void hugetlb_vmemmap_optimize_folios(struct hstate *h, struct list_head *folio_l
break;
}
+ if (nr_to_optimize)
+ flush_tlb_all();
+ mmap_write_unlock(&init_mm);
+
if (!nr_to_optimize)
return;
- flush_tlb_all();
-
list_for_each_entry(folio, folio_list, lru) {
int ret;
base-commit: b95f8bb86713c860ce8536d22a988b949b884d5f
--
2.54.0