[PATCH v6 2/4] mm/huge_memory: copy file THPs at fork like copy_present_ptes() does
From: Konstantin Khorenko
Date: Thu Nov 19 2026 - 08:45:19 EST
This fixes no bug: it makes fork copy a PMD the way copy_present_ptes()
copies a PTE. Until now copy_huge_pmd() skipped every folio PMD of a
non-anonymous VMA and left the child to fault them in again, while the
PTEs of the same VMA are copied.
Treat file folios as the PTE path does: take a reference, duplicate the
rmap and account them. fork then pays a reference and an rmap update
per PMD, and saves the child a fault per PMD it touches. The next patch
sends PFN mappings through the same tail.
That covers the VMAs vma_needs_copy() sends here:
- private file mappings with an anon_vma;
- VM_COPY_ON_FORK ones: uffd-wp and uffd-rwp, VM_MIXEDMAP, and
VM_MAYBE_GUARD, which shared shmem, file and DAX mappings get from a
guard region.
A folio over the page cache refcount limit is not copied, as in
copy_present_ptes(): the child faults it in, and fork() fails with
-ENOMEM only in a VM_MIXEDMAP VMA, which may have no fault handler.
Unlike a skipped PTE, the PMD needs no uffd marker: a file PMD never
carries the uffd bit, as protecting it splits it.
The DAX huge zero PMD is copied too, with a reference taken by
mm_get_huge_zero_folio() as for anonymous VMAs. A migration entry of a
file folio is copied and accounted like copy_nonpresent_pte() does, and
made read-only in both processes only in a CoW mapping. A device
private entry gets the same CoW check, as in copy_nonpresent_pte().
The copy then gets the same tail as __copy_present_ptes(): write
protect both sides in a CoW mapping, mark the child's copy clean in a
shared one, and drop the uffd-wp bit where the child is not armed. An
anonymous THP is now write protected only when its PMD is writable, as
a PTE is.
A page table is deposited under the copy where the fault paths deposit
one: in anonymous VMAs, and wherever the architecture needs one.
Cc: stable@xxxxxxxxxxxxxxx # needed by the next patch
Suggested-by: David Hildenbrand (Arm) <david@xxxxxxxxxx>
Reviewed-by: Kiryl Shutsemau (Meta) <kas@xxxxxxxxxx>
Reviewed-by: Zi Yan <ziy@xxxxxxxxxx>
Reviewed-by: Jose A. Perez de Azpillaga <azpijr@xxxxxxxxx>
Signed-off-by: Konstantin Khorenko <khorenko@xxxxxxxxxxxxx>
---
mm/huge_memory.c | 74 ++++++++++++++++++++++++++++++++++++++------------------
1 file changed, 50 insertions(+), 24 deletions(-)
diff --git a/mm/huge_memory.c b/mm/huge_memory.c
index 641e5a3822e3a..277d27a8a4364 100644
--- a/mm/huge_memory.c
+++ b/mm/huge_memory.c
@@ -1913,12 +1913,13 @@ static void copy_huge_non_present_pmd(
pmd_t pmd, pgtable_t pgtable)
{
softleaf_t entry = softleaf_from_pmd(pmd);
- struct folio *src_folio;
+ struct folio *src_folio = softleaf_to_folio(entry);
VM_WARN_ON_ONCE(!pmd_is_valid_softleaf(pmd));
- if (softleaf_is_migration_write(entry) ||
- softleaf_is_migration_read_exclusive(entry)) {
+ if ((softleaf_is_migration_write(entry) ||
+ softleaf_is_migration_read_exclusive(entry)) &&
+ vma_is_cow_mapping(src_vma)) {
entry = make_readable_migration_entry(swp_offset(entry));
pmd = softleaf_to_pmd(entry);
if (pmd_swp_soft_dirty(*src_pmd))
@@ -1931,7 +1932,8 @@ static void copy_huge_non_present_pmd(
* For device private entries, since there are no
* read exclusive entries, writable = !readable
*/
- if (softleaf_is_device_private_write(entry)) {
+ if (softleaf_is_device_private_write(entry) &&
+ vma_is_cow_mapping(src_vma)) {
entry = make_readable_device_private_entry(swp_offset(entry));
pmd = softleaf_to_pmd(entry);
@@ -1942,7 +1944,6 @@ static void copy_huge_non_present_pmd(
set_pmd_at(src_mm, addr, src_pmd, pmd);
}
- src_folio = softleaf_to_folio(entry);
VM_WARN_ON(!folio_test_large(src_folio));
folio_get(src_folio);
@@ -1954,9 +1955,11 @@ static void copy_huge_non_present_pmd(
dst_vma, src_vma);
}
- add_mm_counter(dst_mm, MM_ANONPAGES, HPAGE_PMD_NR);
- mm_inc_nr_ptes(dst_mm);
- pgtable_trans_huge_deposit(dst_mm, dst_pmd, pgtable);
+ add_mm_counter(dst_mm, mm_counter(src_folio), HPAGE_PMD_NR);
+ if (pgtable) {
+ mm_inc_nr_ptes(dst_mm);
+ pgtable_trans_huge_deposit(dst_mm, dst_pmd, pgtable);
+ }
if (!userfaultfd_protected(dst_vma))
pmd = pmd_swp_clear_uffd(pmd);
set_pmd_at(dst_mm, addr, dst_pmd, pmd);
@@ -1972,6 +1975,7 @@ int copy_huge_pmd(struct mm_struct *dst_mm, struct mm_struct *src_mm,
pmd_t pmd;
pgtable_t pgtable = NULL;
int ret = -ENOMEM;
+ bool writable;
pmd = pmdp_get_lockless(src_pmd);
if (unlikely(pmd_present(pmd) && pmd_special(pmd) &&
@@ -1992,13 +1996,11 @@ int copy_huge_pmd(struct mm_struct *dst_mm, struct mm_struct *src_mm,
goto set_pmd;
}
- /* Skip if can be re-fill on fault */
- if (!vma_is_anonymous(dst_vma))
- return 0;
-
- pgtable = pte_alloc_one(dst_mm);
- if (unlikely(!pgtable))
- goto out;
+ if (vma_is_anonymous(dst_vma) || arch_needs_pgtable_deposit()) {
+ pgtable = pte_alloc_one(dst_mm);
+ if (unlikely(!pgtable))
+ goto out;
+ }
dst_ptl = pmd_lock(dst_mm, dst_pmd);
src_ptl = pmd_lockptr(src_mm, src_pmd);
@@ -2016,7 +2018,8 @@ int copy_huge_pmd(struct mm_struct *dst_mm, struct mm_struct *src_mm,
}
if (unlikely(!pmd_trans_huge(pmd))) {
- pte_free(dst_mm, pgtable);
+ if (pgtable)
+ pte_free(dst_mm, pgtable);
goto out_unlock;
}
/*
@@ -2031,13 +2034,30 @@ int copy_huge_pmd(struct mm_struct *dst_mm, struct mm_struct *src_mm,
* copy. It just takes a reference.
*/
mm_get_huge_zero_folio(dst_mm);
- goto out_zero_page;
+ goto copy_pmd;
}
src_page = pmd_page(pmd);
VM_BUG_ON_PAGE(!PageHead(src_page), src_page);
src_folio = page_folio(src_page);
+ if (!folio_test_anon(src_folio)) {
+ if (unlikely(folio_ref_count_over_limit(src_folio))) {
+ /*
+ * VM_MIXEDMAP may contain driver-inserted pages with no
+ * fault handler to reconstruct the mapping.
+ */
+ ret = (dst_vma->vm_flags & VM_MIXEDMAP) ? -ENOMEM : 0;
+ if (pgtable)
+ pte_free(dst_mm, pgtable);
+ goto out_unlock;
+ }
+ folio_get(src_folio);
+ folio_dup_file_rmap_pmd(src_folio, src_page, dst_vma);
+ add_mm_counter(dst_mm, mm_counter_file(src_folio),
+ HPAGE_PMD_NR);
+ goto copy_pmd;
+ }
folio_get(src_folio);
if (unlikely(folio_try_dup_anon_rmap_pmd(src_folio, src_page, dst_vma, src_vma))) {
/* Page maybe pinned: split and retry the fault on PTEs. */
@@ -2049,19 +2069,25 @@ int copy_huge_pmd(struct mm_struct *dst_mm, struct mm_struct *src_mm,
return -EAGAIN;
}
add_mm_counter(dst_mm, MM_ANONPAGES, HPAGE_PMD_NR);
-out_zero_page:
- mm_inc_nr_ptes(dst_mm);
- pgtable_trans_huge_deposit(dst_mm, dst_pmd, pgtable);
+copy_pmd:
+ if (pgtable) {
+ mm_inc_nr_ptes(dst_mm);
+ pgtable_trans_huge_deposit(dst_mm, dst_pmd, pgtable);
+ }
- /* See __copy_present_ptes(): restore accessible protection. */
+ /* See __copy_present_ptes(). */
+ writable = pmd_write(pmd);
if (!userfaultfd_protected(dst_vma)) {
if (userfaultfd_rwp(src_vma) && pmd_uffd(pmd))
pmd = pmd_modify(pmd, dst_vma->vm_page_prot);
pmd = pmd_clear_uffd(pmd);
}
-
- pmdp_set_wrprotect(src_mm, addr, src_pmd);
- pmd = pmd_wrprotect(pmd);
+ if (vma_is_cow_mapping(src_vma) && writable) {
+ pmdp_set_wrprotect(src_mm, addr, src_pmd);
+ pmd = pmd_wrprotect(pmd);
+ }
+ if (src_vma->vm_flags & VM_SHARED)
+ pmd = pmd_mkclean(pmd);
set_pmd:
pmd = pmd_mkold(pmd);
set_pmd_at(dst_mm, addr, dst_pmd, pmd);
--
2.43.0