[PATCH v4 04/17] mm/huge_memory: split the routine for splitting anon and file folio
From: Kairui Song via B4 Relay
Date: Mon Sep 07 2026 - 14:12:27 EST
From: Kairui Song <kasong@xxxxxxxxxxx>
No functional change intended. Before adding more logic, split
__folio_freeze_and_split_unmapped() into an anon and a file variant so
each path can evolve independently. The two paths shared little beyond
the folio freeze call, the LRU locking, and the unfreeze skeleton, but
differed in all other per-folio bookkeeping and routines.
While splitting, some cleanups become easy to apply, and helped drop a
few now-redundant checks.
The zone_device_private_split_cb() calls are only kept in the anon
variant, as device private folios can only back anonymous memory, and
add a VM_WARN_ON_ONCE_FOLIO() at the entry of the file variant.
Acked-by: David Hildenbrand (Arm) <david@xxxxxxxxxx>
Reviewed-by: Zi Yan <ziy@xxxxxxxxxx>
Reviewed-by: Yeoreum Yun <yeoreum.yun@xxxxxxx>
Signed-off-by: Kairui Song <kasong@xxxxxxxxxxx>
---
mm/huge_memory.c | 123 +++++++++++++++++++++++++++++++++++--------------------
1 file changed, 78 insertions(+), 45 deletions(-)
diff --git a/mm/huge_memory.c b/mm/huge_memory.c
index ad45b86819fd..0e28a3e66b67 100644
--- a/mm/huge_memory.c
+++ b/mm/huge_memory.c
@@ -3984,11 +3984,9 @@ static unsigned int folio_cache_ref_count(const struct folio *folio)
return folio_nr_pages(folio);
}
-static int __folio_freeze_and_split_unmapped(struct folio *folio, unsigned int new_order,
- struct page *split_at, struct xa_state *xas,
- struct address_space *mapping, bool do_lru,
- struct list_head *list, enum split_type split_type,
- pgoff_t end, int *nr_shmem_dropped)
+static int __folio_freeze_split_anon(struct folio *folio,
+ unsigned int new_order, struct page *split_at, bool do_lru,
+ struct list_head *list, enum split_type split_type)
{
struct folio *end_folio = folio_next(folio);
struct swap_cluster_info *ci = NULL;
@@ -3997,8 +3995,6 @@ static int __folio_freeze_and_split_unmapped(struct folio *folio, unsigned int n
struct lruvec *lruvec;
int ret = 0;
- VM_WARN_ON_ONCE(!mapping && end);
-
if (!folio_ref_freeze(folio, folio_cache_ref_count(folio) + 1))
return -EAGAIN;
@@ -4017,24 +4013,75 @@ static int __folio_freeze_and_split_unmapped(struct folio *folio, unsigned int n
MTHP_STAT_NR_ANON_PARTIALLY_MAPPED, -1);
}
- if (mapping) {
+ if (folio_test_swapcache(folio))
+ ci = swap_cluster_get_and_lock(folio);
+
+ if (do_lru)
+ lruvec = folio_lruvec_lock(folio);
+
+ ret = __split_unmapped_folio(folio, new_order, split_at, NULL,
+ NULL, split_type);
+
+ /*
+ * Unfreeze the after-split folios and put them back to the right
+ * place. Keep the head @folio frozen until the end: sub entries
+ * in swap cache must be updated first, so a concurrent
+ * swap_cache_get_folio() cannot return the head folio for a sub
+ * entry (folio_try_get() will fail on the head @folio until unfreeze).
+ */
+ for (new_folio = folio_next(folio); new_folio != end_folio;
+ new_folio = next) {
+ next = folio_next(new_folio);
+ zone_device_private_split_cb(folio, new_folio);
+ folio_ref_unfreeze(new_folio,
+ folio_cache_ref_count(new_folio) + 1);
+ if (do_lru)
+ lru_add_split_folio(folio, new_folio, lruvec, list);
+ if (ci)
+ __swap_cache_replace_folio(ci, folio, new_folio);
+ }
+
+ zone_device_private_split_cb(folio, NULL);
+ folio_ref_unfreeze(folio, folio_cache_ref_count(folio) + 1);
+
+ if (do_lru)
+ lruvec_unlock(lruvec);
+ if (ci)
+ swap_cluster_unlock(ci);
+
+ return ret;
+}
+
+static int __folio_freeze_split_file(struct folio *folio,
+ unsigned int new_order, struct page *split_at,
+ struct xa_state *xas, struct address_space *mapping,
+ bool do_lru, struct list_head *list,
+ enum split_type split_type, pgoff_t end, int *nr_shmem_dropped)
+{
+ struct folio *end_folio = folio_next(folio);
+ struct folio *new_folio, *next;
+ struct lruvec *lruvec;
+ int ret;
+
+ /* Currently device private folios can only back anonymous memory. */
+ VM_WARN_ON_ONCE_FOLIO(folio_is_device_private(folio), folio);
+
+ if (!folio_ref_freeze(folio, folio_cache_ref_count(folio) + 1))
+ return -EAGAIN;
+
+ if (folio_test_pmd_mappable(folio) &&
+ new_order < HPAGE_PMD_ORDER) {
int nr = folio_nr_pages(folio);
- if (folio_test_pmd_mappable(folio) &&
- new_order < HPAGE_PMD_ORDER) {
- if (folio_test_swapbacked(folio)) {
- lruvec_stat_mod_folio(folio,
- NR_SHMEM_THPS, -nr);
- } else {
- lruvec_stat_mod_folio(folio,
- NR_FILE_THPS, -nr);
- }
+ if (folio_test_swapbacked(folio)) {
+ lruvec_stat_mod_folio(folio,
+ NR_SHMEM_THPS, -nr);
+ } else {
+ lruvec_stat_mod_folio(folio,
+ NR_FILE_THPS, -nr);
}
}
- if (folio_test_swapcache(folio))
- ci = swap_cluster_get_and_lock(folio);
-
/* lock lru list/PageCompound, ref frozen by page_ref_freeze */
if (do_lru)
lruvec = folio_lruvec_lock(folio);
@@ -4044,7 +4091,7 @@ static int __folio_freeze_and_split_unmapped(struct folio *folio, unsigned int n
/*
* Unfreeze after-split folios and put them back to the right
- * list. @folio should be kept frozon until page cache
+ * list. @folio should be kept frozen until page cache
* entries are updated with all the other after-split folios
* to prevent others seeing stale page cache entries.
* As a result, new_folio starts from the next folio of
@@ -4054,29 +4101,15 @@ static int __folio_freeze_and_split_unmapped(struct folio *folio, unsigned int n
new_folio = next) {
unsigned long nr_pages = folio_nr_pages(new_folio);
+ /* compute next before the folio can be freed below */
next = folio_next(new_folio);
- zone_device_private_split_cb(folio, new_folio);
-
folio_ref_unfreeze(new_folio,
folio_cache_ref_count(new_folio) + 1);
if (do_lru)
lru_add_split_folio(folio, new_folio, lruvec, list);
- /*
- * Anonymous folio with swap cache.
- * NOTE: shmem in swap cache is not supported yet.
- */
- if (ci) {
- __swap_cache_replace_folio(ci, folio, new_folio);
- continue;
- }
-
- /* Anonymous folio without swap cache */
- if (!mapping)
- continue;
-
/* Add the new folio to the page cache. */
if (new_folio->index < end) {
__xa_store(&mapping->i_pages, new_folio->index,
@@ -4095,7 +4128,6 @@ static int __folio_freeze_and_split_unmapped(struct folio *folio, unsigned int n
folio_put_refs(new_folio, nr_pages);
}
- zone_device_private_split_cb(folio, NULL);
/*
* Unfreeze @folio only after all page cache entries, which
* used to point to it, have been updated with new folios.
@@ -4106,8 +4138,6 @@ static int __folio_freeze_and_split_unmapped(struct folio *folio, unsigned int n
if (do_lru)
lruvec_unlock(lruvec);
- if (ci)
- swap_cluster_unlock(ci);
return ret;
}
@@ -4250,7 +4280,10 @@ static int __folio_split(struct folio *folio, unsigned int new_order,
/* block interrupt reentry in xa_lock and spinlock */
local_irq_disable();
- if (mapping) {
+ if (is_anon) {
+ ret = __folio_freeze_split_anon(folio, new_order, split_at,
+ true, list, split_type);
+ } else {
/*
* Check if the folio is present in page cache.
* We assume all tail are present too, if folio is there.
@@ -4261,10 +4294,11 @@ static int __folio_split(struct folio *folio, unsigned int new_order,
ret = -EAGAIN;
goto fail;
}
+ ret = __folio_freeze_split_file(folio, new_order, split_at, &xas, mapping,
+ true, list, split_type, end,
+ &nr_shmem_dropped);
}
- ret = __folio_freeze_and_split_unmapped(folio, new_order, split_at, &xas, mapping,
- true, list, split_type, end, &nr_shmem_dropped);
fail:
if (mapping)
xas_unlock(&xas);
@@ -4364,9 +4398,8 @@ int folio_split_unmapped(struct folio *folio, unsigned int new_order)
return -EAGAIN;
local_irq_disable();
- ret = __folio_freeze_and_split_unmapped(folio, new_order, &folio->page, NULL,
- NULL, false, NULL, SPLIT_TYPE_UNIFORM,
- 0, NULL);
+ ret = __folio_freeze_split_anon(folio, new_order, &folio->page,
+ false, NULL, SPLIT_TYPE_UNIFORM);
local_irq_enable();
return ret;
}
--
2.55.0