[PATCH 04/18] hugetlb_vmemmap: Use try_update_vmemmap_pte to update in-use PTEs
From: James Houghton
Date: Tue Jul 07 2026 - 23:13:33 EST
set_pte_at() cannot be used to replace in-use vmemmap PTEs on arm64, so
replace it with a more specific routine, try_update_vmemmap_pte().
try_update_vmemmap_pte() is used for modifying page table entries such
that there is a guarantee that no fault will be taken.
For the x86, riscv, and loongarch implementations, please note one
difference: set_pte() does not invoke page_table_check_ptes_set(), but
set_pte_at(), the call we are about to replace, does. However, this is a
functional no-op because page_table_check_ptes_set() does nothing for
init_mm PTEs, which vmemmap PTEs are.
Signed-off-by: James Houghton <jthoughton@xxxxxxxxxx>
---
arch/loongarch/include/asm/pgtable.h | 8 ++++
arch/riscv/include/asm/pgtable.h | 8 ++++
arch/x86/include/asm/pgtable.h | 8 ++++
include/linux/pgtable.h | 21 +++++++++++
mm/hugetlb_vmemmap.c | 56 +++++++++++++++++++---------
5 files changed, 84 insertions(+), 17 deletions(-)
diff --git a/arch/loongarch/include/asm/pgtable.h b/arch/loongarch/include/asm/pgtable.h
index 223528c04d73..e7b65056ef73 100644
--- a/arch/loongarch/include/asm/pgtable.h
+++ b/arch/loongarch/include/asm/pgtable.h
@@ -638,6 +638,14 @@ static inline long pmd_protnone(pmd_t pmd)
#define pmd_leaf(pmd) ((pmd_val(pmd) & _PAGE_HUGE) != 0)
#define pud_leaf(pud) ((pud_val(pud) & _PAGE_HUGE) != 0)
+#define __HAVE_ARCH_TRY_UPDATE_VMEMMAP_PTE
+static inline int try_update_vmemmap_pte(unsigned long addr, pte_t *ptep,
+ pte_t pte)
+{
+ set_pte(ptep, pte);
+ return 0;
+}
+
/*
* We provide our own get_unmapped area to cope with the virtual aliasing
* constraints placed on us by the cache architecture.
diff --git a/arch/riscv/include/asm/pgtable.h b/arch/riscv/include/asm/pgtable.h
index 5d5756bda82e..a11c569ee2f0 100644
--- a/arch/riscv/include/asm/pgtable.h
+++ b/arch/riscv/include/asm/pgtable.h
@@ -1162,6 +1162,14 @@ static inline pud_t pud_modify(pud_t pud, pgprot_t newprot)
#endif /* CONFIG_TRANSPARENT_HUGEPAGE */
+#define __HAVE_ARCH_TRY_UPDATE_VMEMMAP_PTE
+static inline int try_update_vmemmap_pte(unsigned long addr, pte_t *ptep,
+ pte_t pte)
+{
+ set_pte(ptep, pte);
+ return 0;
+}
+
/*
* Encode/decode swap entries and swap PTEs. Swap PTEs are all PTEs that
* are !pte_none() && !pte_present().
diff --git a/arch/x86/include/asm/pgtable.h b/arch/x86/include/asm/pgtable.h
index ac295ca6c92f..974151564071 100644
--- a/arch/x86/include/asm/pgtable.h
+++ b/arch/x86/include/asm/pgtable.h
@@ -1372,6 +1372,14 @@ static inline pmd_t pmdp_establish(struct vm_area_struct *vma,
}
#endif
+#define __HAVE_ARCH_TRY_UPDATE_VMEMMAP_PTE
+static inline int try_update_vmemmap_pte(unsigned long addr, pte_t *ptep,
+ pte_t pte)
+{
+ set_pte(ptep, pte);
+ return 0;
+}
+
#ifdef CONFIG_HAVE_ARCH_TRANSPARENT_HUGEPAGE_PUD
static inline pud_t pudp_establish(struct vm_area_struct *vma,
unsigned long address, pud_t *pudp, pud_t pud)
diff --git a/include/linux/pgtable.h b/include/linux/pgtable.h
index 2981e386da7b..e1abccf76bb4 100644
--- a/include/linux/pgtable.h
+++ b/include/linux/pgtable.h
@@ -457,6 +457,27 @@ static inline void set_ptes(struct mm_struct *mm, unsigned long addr,
#endif
#define set_pte_at(mm, addr, ptep, pte) set_ptes(mm, addr, ptep, pte, 1)
+#ifndef __HAVE_ARCH_TRY_UPDATE_VMEMMAP_PTE
+/*
+ * try_update_vmemmap_pte - Remap PTEs used by the vmemmap.
+ * @addr: Base address of the remapped PTE.
+ * @ptep: Page table pointer to be overwritten.
+ * @pte: Page table entry to write.
+ *
+ * This function is only to be used to update PTEs that map the vmemmap. The
+ * only valid transitions supported by this function are: leaf-level
+ * (PAGE_SIZE), valid-to-valid. The pfn and prot bits may be changed.
+ *
+ * Implementations of this function must ensure that, while the update is taking
+ * place, CPUs will not fault on the remapped virtual address.
+ */
+static inline int try_update_vmemmap_pte(unsigned long addr, pte_t *ptep,
+ pte_t pte)
+{
+ return -EOPNOTSUPP;
+}
+#endif
+
#ifndef __HAVE_ARCH_PTEP_SET_ACCESS_FLAGS
extern int ptep_set_access_flags(struct vm_area_struct *vma,
unsigned long address, pte_t *ptep,
diff --git a/mm/hugetlb_vmemmap.c b/mm/hugetlb_vmemmap.c
index 5fee01bf5b34..3951c043164a 100644
--- a/mm/hugetlb_vmemmap.c
+++ b/mm/hugetlb_vmemmap.c
@@ -34,7 +34,7 @@
* operations.
*/
struct vmemmap_remap_walk {
- void (*remap_pte)(pte_t *pte, unsigned long addr,
+ int (*remap_pte)(pte_t *pte, unsigned long addr,
struct vmemmap_remap_walk *walk);
unsigned long nr_walked;
@@ -141,11 +141,13 @@ static int vmemmap_pte_entry(pte_t *pte, unsigned long addr,
unsigned long next, struct mm_walk *walk)
{
struct vmemmap_remap_walk *vmemmap_walk = walk->private;
+ int ret = 0;
- vmemmap_walk->remap_pte(pte, addr, vmemmap_walk);
- vmemmap_walk->nr_walked++;
+ ret = vmemmap_walk->remap_pte(pte, addr, vmemmap_walk);
+ if (!ret)
+ vmemmap_walk->nr_walked++;
- return 0;
+ return ret;
}
static const struct mm_walk_ops vmemmap_remap_ops = {
@@ -197,18 +199,20 @@ static void free_vmemmap_page_list(struct list_head *list)
free_vmemmap_page(page);
}
-static void vmemmap_remap_pte(pte_t *pte, unsigned long addr,
- struct vmemmap_remap_walk *walk)
+static int vmemmap_remap_pte(pte_t *pte, unsigned long addr,
+ struct vmemmap_remap_walk *walk)
{
struct page *page = pte_page(ptep_get(pte));
pte_t entry;
+ bool head;
+ int ret;
+
+ head = walk->nr_walked == 0 && walk->vmemmap_head;
/* Remapping the head page requires r/w */
- if (unlikely(walk->nr_walked == 0 && walk->vmemmap_head)) {
+ if (unlikely(head)) {
VM_WARN_ON_ONCE(!PageHead((const struct page *)addr));
- list_del(&walk->vmemmap_head->lru);
-
/*
* Makes sure that preceding stores to the page contents from
* vmemmap_remap_free() become visible before the set_pte_at()
@@ -227,17 +231,30 @@ static void vmemmap_remap_pte(pte_t *pte, unsigned long addr,
entry = mk_pte(walk->vmemmap_tail, PAGE_KERNEL_RO);
}
+ ret = try_update_vmemmap_pte(addr, pte, entry);
+ if (ret)
+ return ret;
+
+ /* We successfully overwrote the vmemmap PTE, so we can free
+ * the vmemmap page that was just unmapped, and if we mapped
+ * the new head page, remove it from the list so that it
+ * doesn't get freed later.
+ */
list_add(&page->lru, walk->vmemmap_pages);
- set_pte_at(&init_mm, addr, pte, entry);
+ if (head)
+ list_del(&walk->vmemmap_head->lru);
+
+ return 0;
}
-static void vmemmap_restore_pte(pte_t *pte, unsigned long addr,
- struct vmemmap_remap_walk *walk)
+static int vmemmap_restore_pte(pte_t *pte, unsigned long addr,
+ struct vmemmap_remap_walk *walk)
{
struct page *src = pte_page(ptep_get(pte)), *dst;
+ int ret;
if (WARN_ON_ONCE(!walk->vmemmap_tail))
- return;
+ return -EINVAL;
/*
* When restoring a partially-HVOed page, keep the copied head page
@@ -245,20 +262,25 @@ static void vmemmap_restore_pte(pte_t *pte, unsigned long addr,
* page.
*/
if (walk->vmemmap_tail != src)
- return;
+ return 0;
VM_WARN_ON_ONCE(PageHead((const struct page *)addr));
dst = list_first_entry(walk->vmemmap_pages, struct page, lru);
- list_del(&dst->lru);
copy_page(page_to_virt(dst), page_to_virt(src));
/*
* Makes sure that preceding stores to the page contents become visible
- * before the set_pte_at() write.
+ * before the try_update_vmemmap_pte() write.
*/
smp_wmb();
- set_pte_at(&init_mm, addr, pte, mk_pte(dst, PAGE_KERNEL));
+
+ ret = try_update_vmemmap_pte(addr, pte, mk_pte(dst, PAGE_KERNEL));
+ if (ret)
+ return ret;
+
+ list_del(&dst->lru);
+ return 0;
}
/**
--
2.55.0.795.g602f6c329a-goog