[PATCH v4 8/9] mm/rmap: batch unmap anonymous swap-backed large folios
From: Dev Jain
Date: Sun Nov 15 2026 - 09:40:30 EST
Enable batch clearing of PTEs and batch swap setting of PTEs for anon
swap-backed folio unmapping.
Processing all PTEs of a large folio in one go helps us batch across
atomics (add_mm_counter() etc), barriers in __folio_try_share_anon_rmap(),
and repeated calls to page_vma_mapped_walk(). In general, batching helps
execute similar code together, making the path more memory and CPU
friendly.
On arm64-contpte, batching also helps avoid redundant ptep_get() calls
and TLB flushes while breaking the contpte mapping.
The handling of anon-exclusivity is very similar to commit cac1db8c3aad
("mm: optimize mprotect() by PTE batching"). Since
folio_unmap_pte_batch() does not look at the bits of the underlying page,
process sub-batches of PTEs pointing to pages with the same exclusivity
state, and batch set only those PTEs to swap PTEs in one go.
Disable batching for sparc (because of arch_unmap_one), we will batch this
later.
Rmap accounting and reference accounting must happen when anon folio unmap
succeeds. If a large folio is only partially batched or a later sub-batch
fails, account only the pages that were actually unmapped.
If the batch length is less than the number of pages in the folio, skip
over this batch. page_vma_mapped_walk() handles this: check_pte() returns
true only if any of [pvmw->pfn, pvmw->pfn + nr_pages) is mapped by the
PTE. Swap PTEs have no underlying PFN, so check_pte() returns false until
the walk reaches the next present PTE to unmap.
Remove the finish_unmap label since no goto callers are left now.
Signed-off-by: Dev Jain <dev.jain@xxxxxxx>
---
mm/rmap.c | 118 +++++++++++++++++++++++++++++++++++++++---------------
1 file changed, 86 insertions(+), 32 deletions(-)
diff --git a/mm/rmap.c b/mm/rmap.c
index 6a6fb9b56f2cd..a2481323b778a 100644
--- a/mm/rmap.c
+++ b/mm/rmap.c
@@ -1981,12 +1981,13 @@ static inline unsigned int folio_unmap_pte_batch(struct folio *folio,
end_addr = pmd_addr_end(addr, vma->vm_end);
max_nr = (end_addr - addr) >> PAGE_SHIFT;
- /* We only support lazyfree or file folios batching for now ... */
- if (folio_test_anon(folio) && folio_test_swapbacked(folio))
+ if (pte_unused(pte))
return 1;
- if (pte_unused(pte))
+#ifdef __HAVE_ARCH_UNMAP_ONE
+ if (folio_test_anon(folio) && folio_test_swapbacked(folio))
return 1;
+#endif
/*
* If unmap fails, we need to restore the ptes. To avoid accidentally
@@ -2156,16 +2157,25 @@ static pte_t swp_pte_prepare(swp_entry_t entry, pte_t old_pte,
return swp_pte;
}
-static bool ttu_anon_swapbacked_folio(struct vm_area_struct *vma,
+static void finish_folio_unmap_batch(struct vm_area_struct *vma,
+ struct folio *folio, struct page *page, unsigned long nr_pages)
+{
+ folio_remove_rmap_ptes(folio, page, nr_pages, vma);
+ if (vma_test_any_mask(vma, VMA_LOCKED_MASK))
+ mlock_drain_local();
+ folio_put_refs(folio, nr_pages);
+}
+
+static bool __ttu_anon_swapbacked_folio(struct vm_area_struct *vma,
struct folio *folio, struct page *page, unsigned long address,
- pte_t *ptep, pte_t pteval)
+ pte_t *ptep, pte_t pteval, unsigned long nr_pages,
+ bool anon_exclusive)
{
- const bool anon_exclusive = folio_test_anon(folio) &&
- PageAnonExclusive(page);
swp_entry_t entry = folio_page_swap_entry(folio, page);
struct mm_struct *mm = vma->vm_mm;
+ pte_t swp_pte;
- if (folio_dup_swap_pages(folio, page, 1) < 0)
+ if (folio_dup_swap_pages(folio, page, nr_pages) < 0)
return false;
/*
@@ -2174,25 +2184,61 @@ static bool ttu_anon_swapbacked_folio(struct vm_area_struct *vma,
* so we'll not check/care.
*/
if (arch_unmap_one(mm, vma, address, pteval) < 0) {
- folio_put_swap_pages(folio, page, 1);
+ VM_WARN_ON(nr_pages != 1);
+ folio_put_swap_pages(folio, page, nr_pages);
return false;
}
/* See folio_try_share_anon_rmap(): clear PTE first. */
- if (anon_exclusive && folio_try_share_anon_rmap_pte(folio, page)) {
- folio_put_swap_pages(folio, page, 1);
+ if (anon_exclusive &&
+ folio_try_share_anon_rmap_ptes(folio, page, nr_pages)) {
+ folio_put_swap_pages(folio, page, nr_pages);
return false;
}
mm_prepare_for_swap_entries(mm);
- dec_mm_counter(mm, MM_ANONPAGES);
- inc_mm_counter(mm, MM_SWAPENTS);
- set_pte_at(mm, address, ptep,
- swp_pte_prepare(entry, pteval, anon_exclusive));
+ add_mm_counter(mm, MM_ANONPAGES, -nr_pages);
+ add_mm_counter(mm, MM_SWAPENTS, nr_pages);
+ swp_pte = swp_pte_prepare(entry, pteval, anon_exclusive);
+ set_softleaf_ptes(mm, address, ptep, swp_pte, nr_pages);
return true;
}
-static bool ttu_anon_folio(struct vm_area_struct *vma, struct folio *folio,
+static unsigned long ttu_anon_swapbacked_folio(struct vm_area_struct *vma,
+ struct folio *folio, struct page *first_page,
+ unsigned long address, pte_t *ptep, pte_t pteval,
+ unsigned long nr_pages)
+{
+ unsigned long batch_idx = 0;
+ unsigned long nr_to_unmap = nr_pages;
+
+ while (nr_pages) {
+ bool exclusive;
+ unsigned long len = page_anon_exclusive_batch(batch_idx,
+ nr_pages, first_page, &exclusive);
+
+ if (!__ttu_anon_swapbacked_folio(vma, folio,
+ first_page + batch_idx, address, ptep, pteval,
+ len, exclusive)) {
+ /* Restore the remaining PTEs that were cleared. */
+ set_ptes(vma->vm_mm, address, ptep, pteval, nr_pages);
+ return nr_to_unmap - nr_pages;
+ }
+
+ nr_pages -= len;
+ if (!nr_pages)
+ break;
+
+ pteval = pte_advance_pfn(pteval, len);
+ address += len * PAGE_SIZE;
+ batch_idx += len;
+ ptep += len;
+ }
+
+ return nr_to_unmap;
+}
+
+static unsigned long ttu_anon_folio(struct vm_area_struct *vma, struct folio *folio,
struct page *page, unsigned long address, pte_t *ptep,
pte_t pteval, unsigned long nr_pages)
{
@@ -2201,15 +2247,21 @@ static bool ttu_anon_folio(struct vm_area_struct *vma, struct folio *folio,
* See handle_pte_fault() ...
*/
if (WARN_ON_ONCE(folio_test_swapbacked(folio) !=
- folio_test_swapcache(folio)))
- return false;
+ folio_test_swapcache(folio))) {
+ set_ptes(vma->vm_mm, address, ptep, pteval, nr_pages);
+ return 0;
+ }
- if (!folio_test_swapbacked(folio))
- return ttu_anon_lazyfree_folio(vma, folio, nr_pages);
+ if (!folio_test_swapbacked(folio)) {
+ if (!ttu_anon_lazyfree_folio(vma, folio, nr_pages)) {
+ set_ptes(vma->vm_mm, address, ptep, pteval, nr_pages);
+ return 0;
+ }
+ return nr_pages;
+ }
- /* nr_pages > 1 not supported yet */
return ttu_anon_swapbacked_folio(vma, folio, page, address, ptep,
- pteval);
+ pteval, nr_pages);
}
/*
@@ -2378,13 +2430,18 @@ static bool try_to_unmap_one(struct folio *folio, struct vm_area_struct *vma,
*/
dec_mm_counter(mm, mm_counter(folio));
} else if (folio_test_anon(folio)) {
- if (!ttu_anon_folio(vma, folio, page, address,
- pvmw.pte, pteval, nr_pages)) {
- set_ptes(mm, address, pvmw.pte, pteval, nr_pages);
+ unsigned long nr_unmapped = ttu_anon_folio(vma, folio,
+ page, address, pvmw.pte,
+ pteval, nr_pages);
+
+ if (!nr_unmapped)
goto walk_abort;
- }
- goto finish_unmap;
+ /* partial success: finish for nr_unmapped and abort. */
+ if (nr_unmapped < nr_pages) {
+ finish_folio_unmap_batch(vma, folio, page, nr_unmapped);
+ goto walk_abort;
+ }
} else {
/*
* This is a locked file-backed folio,
@@ -2399,11 +2456,8 @@ static bool try_to_unmap_one(struct folio *folio, struct vm_area_struct *vma,
*/
add_mm_counter(mm, mm_counter_file(folio), -nr_pages);
}
-finish_unmap:
- folio_remove_rmap_ptes(folio, page, nr_pages, vma);
- if (vma_test_any_mask(vma, VMA_LOCKED_MASK))
- mlock_drain_local();
- folio_put_refs(folio, nr_pages);
+
+ finish_folio_unmap_batch(vma, folio, page, nr_pages);
/*
* If we are sure that we batched the entire folio and cleared
--
2.43.0