[PATCH v4 2/4] mm/huge_memory: copy file THPs at fork like copy_present_ptes() does
From: Konstantin Khorenko
Date: Wed Oct 21 2026 - 16:37:54 EST
This fixes no bug: it makes fork copy a PMD the way copy_present_ptes()
copies a PTE. Until now copy_huge_pmd() skipped every folio PMD of a
non-anonymous VMA and left the child to fault them in again, while the
PTEs of the same VMA are copied.
Treat file folios as the PTE path does: take a reference, duplicate the
rmap and account them. fork then pays a reference and an rmap update
per PMD, and saves the child a fault per PMD it touches. The next patch
sends PFN mappings through the same tail.
That covers the VMAs vma_needs_copy() sends here:
- private file mappings with an anon_vma;
- VM_COPY_ON_FORK ones: uffd-wp and uffd-rwp, VM_MIXEDMAP (fuse DAX),
and VM_MAYBE_GUARD, which shared shmem, file and DAX mappings get
from a guard region.
The DAX huge zero PMD is copied too, with a reference taken by
mm_get_huge_zero_folio() as for anonymous VMAs. A migration entry of a
file folio is copied and accounted like copy_nonpresent_pte() does, and
made read-only in both processes only in a CoW mapping.
The copy then gets the same tail as __copy_present_ptes(): write
protect both sides in a CoW mapping, mark the child's copy clean in a
shared one, and drop the uffd-wp bit where the child is not armed. An
anonymous THP is now write protected only when its PMD is writable, as
a PTE is.
A page table is deposited under the copy where the fault paths deposit
one: in anonymous VMAs, and wherever the architecture needs one.
Suggested-by: David Hildenbrand <david@xxxxxxxxxx>
Signed-off-by: Konstantin Khorenko <khorenko@xxxxxxxxxxxxx>
---
mm/huge_memory.c | 60 ++++++++++++++++++++++++++++++++++----------------------
1 file changed, 37 insertions(+), 23 deletions(-)
diff --git a/mm/huge_memory.c b/mm/huge_memory.c
index 1e5d68acf62a5..13e3674fb6115 100644
--- a/mm/huge_memory.c
+++ b/mm/huge_memory.c
@@ -1898,12 +1898,13 @@ static void copy_huge_non_present_pmd(
pmd_t pmd, pgtable_t pgtable)
{
softleaf_t entry = softleaf_from_pmd(pmd);
- struct folio *src_folio;
+ struct folio *src_folio = softleaf_to_folio(entry);
VM_WARN_ON_ONCE(!pmd_is_valid_softleaf(pmd));
- if (softleaf_is_migration_write(entry) ||
- softleaf_is_migration_read_exclusive(entry)) {
+ if ((softleaf_is_migration_write(entry) ||
+ softleaf_is_migration_read_exclusive(entry)) &&
+ vma_is_cow_mapping(src_vma)) {
entry = make_readable_migration_entry(swp_offset(entry));
pmd = softleaf_to_pmd(entry);
if (pmd_swp_soft_dirty(*src_pmd))
@@ -1927,7 +1928,6 @@ static void copy_huge_non_present_pmd(
set_pmd_at(src_mm, addr, src_pmd, pmd);
}
- src_folio = softleaf_to_folio(entry);
VM_WARN_ON(!folio_test_large(src_folio));
folio_get(src_folio);
@@ -1939,9 +1939,11 @@ static void copy_huge_non_present_pmd(
dst_vma, src_vma);
}
- add_mm_counter(dst_mm, MM_ANONPAGES, HPAGE_PMD_NR);
- mm_inc_nr_ptes(dst_mm);
- pgtable_trans_huge_deposit(dst_mm, dst_pmd, pgtable);
+ add_mm_counter(dst_mm, mm_counter(src_folio), HPAGE_PMD_NR);
+ if (pgtable) {
+ mm_inc_nr_ptes(dst_mm);
+ pgtable_trans_huge_deposit(dst_mm, dst_pmd, pgtable);
+ }
if (!userfaultfd_protected(dst_vma))
pmd = pmd_swp_clear_uffd(pmd);
set_pmd_at(dst_mm, addr, dst_pmd, pmd);
@@ -1957,6 +1959,7 @@ int copy_huge_pmd(struct mm_struct *dst_mm, struct mm_struct *src_mm,
pmd_t pmd;
pgtable_t pgtable = NULL;
int ret = -ENOMEM;
+ bool writable;
pmd = pmdp_get_lockless(src_pmd);
if (unlikely(pmd_present(pmd) && pmd_special(pmd) &&
@@ -1977,13 +1980,11 @@ int copy_huge_pmd(struct mm_struct *dst_mm, struct mm_struct *src_mm,
goto set_pmd;
}
- /* Skip if can be re-fill on fault */
- if (!vma_is_anonymous(dst_vma))
- return 0;
-
- pgtable = pte_alloc_one(dst_mm);
- if (unlikely(!pgtable))
- goto out;
+ if (vma_is_anonymous(dst_vma) || arch_needs_pgtable_deposit()) {
+ pgtable = pte_alloc_one(dst_mm);
+ if (unlikely(!pgtable))
+ goto out;
+ }
dst_ptl = pmd_lock(dst_mm, dst_pmd);
src_ptl = pmd_lockptr(src_mm, src_pmd);
@@ -2001,7 +2002,8 @@ int copy_huge_pmd(struct mm_struct *dst_mm, struct mm_struct *src_mm,
}
if (unlikely(!pmd_trans_huge(pmd))) {
- pte_free(dst_mm, pgtable);
+ if (pgtable)
+ pte_free(dst_mm, pgtable);
goto out_unlock;
}
/*
@@ -2016,7 +2018,7 @@ int copy_huge_pmd(struct mm_struct *dst_mm, struct mm_struct *src_mm,
* copy. It just takes a reference.
*/
mm_get_huge_zero_folio(dst_mm);
- goto out_zero_page;
+ goto copy_pmd;
}
src_page = pmd_page(pmd);
@@ -2024,6 +2026,12 @@ int copy_huge_pmd(struct mm_struct *dst_mm, struct mm_struct *src_mm,
src_folio = page_folio(src_page);
folio_get(src_folio);
+ if (!folio_test_anon(src_folio)) {
+ folio_dup_file_rmap_pmd(src_folio, src_page, dst_vma);
+ add_mm_counter(dst_mm, mm_counter_file(src_folio),
+ HPAGE_PMD_NR);
+ goto copy_pmd;
+ }
if (unlikely(folio_try_dup_anon_rmap_pmd(src_folio, src_page, dst_vma, src_vma))) {
/* Page maybe pinned: split and retry the fault on PTEs. */
folio_put(src_folio);
@@ -2034,19 +2042,25 @@ int copy_huge_pmd(struct mm_struct *dst_mm, struct mm_struct *src_mm,
return -EAGAIN;
}
add_mm_counter(dst_mm, MM_ANONPAGES, HPAGE_PMD_NR);
-out_zero_page:
- mm_inc_nr_ptes(dst_mm);
- pgtable_trans_huge_deposit(dst_mm, dst_pmd, pgtable);
+copy_pmd:
+ if (pgtable) {
+ mm_inc_nr_ptes(dst_mm);
+ pgtable_trans_huge_deposit(dst_mm, dst_pmd, pgtable);
+ }
- /* See __copy_present_ptes(): restore accessible protection. */
+ /* See __copy_present_ptes(). */
+ writable = pmd_write(pmd);
if (!userfaultfd_protected(dst_vma)) {
if (userfaultfd_rwp(src_vma) && pmd_uffd(pmd))
pmd = pmd_modify(pmd, dst_vma->vm_page_prot);
pmd = pmd_clear_uffd(pmd);
}
-
- pmdp_set_wrprotect(src_mm, addr, src_pmd);
- pmd = pmd_wrprotect(pmd);
+ if (vma_is_cow_mapping(src_vma) && writable) {
+ pmdp_set_wrprotect(src_mm, addr, src_pmd);
+ pmd = pmd_wrprotect(pmd);
+ }
+ if (src_vma->vm_flags & VM_SHARED)
+ pmd = pmd_mkclean(pmd);
set_pmd:
pmd = pmd_mkold(pmd);
set_pmd_at(dst_mm, addr, dst_pmd, pmd);
--
2.43.0