[PATCH v2 06/14] mm/mm_init: factor out frozen compound page initialization

From: Muchun Song

Date: Fri Nov 13 2026 - 02:31:50 EST


ZONE_DEVICE initializes compound struct pages while their refcounts remain
frozen. The setup is open-coded in memmap_init_compound(), including the
per-page initialization and compound tail preparation.

Factor the common setup into memmap_init_compound_page_frozen(). Keep the
pgmap association and MEMORY_DEVICE_GENERIC refcount handling in the
ZONE_DEVICE-specific path.

This provides an MM-internal helper for other early memmap users to
share.

No functional change is intended.

Signed-off-by: Muchun Song <songmuchun@xxxxxxxxxxxxx>
---
v2:
- Rebase on memmap_init_page_from_template().
- Rename memmap_init_zone_device_head_page to memmap_init_zone_device_head_fields.
---
mm/mm_init.c | 114 +++++++++++++++++++++++++--------------------------
mm/mm_init.h | 2 +
2 files changed, 59 insertions(+), 57 deletions(-)

diff --git a/mm/mm_init.c b/mm/mm_init.c
index 69bbaf75f935..ea9ff690d4e0 100644
--- a/mm/mm_init.c
+++ b/mm/mm_init.c
@@ -739,6 +739,28 @@ void __meminit memmap_init_page_from_template(struct page *page,
vmemmap_optimization_verify_zone(page);
}

+void __meminit memmap_init_compound_page_frozen(struct page *head,
+ unsigned long pfn, const struct zone *zone, unsigned int order)
+{
+ unsigned long nr_pages = 1UL << order;
+ struct page template;
+
+ memmap_init_page_common(&template, zone_idx(zone), zone_to_nid(zone));
+ prep_compound_tail(&template, head, order);
+
+ /* Initialize the head page. */
+ __init_single_page(head, pfn, zone_idx(zone), zone_to_nid(zone));
+ set_page_count(head, 0);
+ __SetPageHead(head);
+
+ if (vmemmap_optimizable_order(pfn_to_section_compound_order(pfn)))
+ nr_pages = VMEMMAP_OPTIMIZATION_NR_STRUCT_PAGES;
+ /* Initialize the tail pages from PFN-independent template page. */
+ for (unsigned long i = 1; i < nr_pages; i++)
+ memmap_init_page_from_template(head + i, pfn + i, zone, &template);
+ prep_compound_head(head, order);
+}
+
struct zone __meminit *pfn_to_zone(unsigned long pfn, int nid)
{
pg_data_t *pgdat = NODE_DATA(nid);
@@ -1064,44 +1086,39 @@ static void __init memmap_init(void)
}

#ifdef CONFIG_ZONE_DEVICE
-/*
- * With compound page geometry and when struct pages are stored in ram most
- * tail pages are reused. Consequently, the amount of unique struct pages to
- * initialize is a lot smaller that the total amount of struct pages being
- * mapped. This is a paired / mild layering violation with explicit knowledge
- * of how the sparse_vmemmap internals handle compound pages in the lack
- * of an altmap.
- */
-static inline unsigned long compound_nr_pages(unsigned long pfn,
- struct dev_pagemap *pgmap)
+static void memmap_init_zone_device_head_fields(struct page *head,
+ struct dev_pagemap *pgmap)
{
- const struct mem_section *ms = __pfn_to_section(pfn);
-
- if (!section_vmemmap_optimizable(ms))
- return pgmap_vmemmap_nr(pgmap);
-
- return VMEMMAP_OPTIMIZATION_PAGES * (PAGE_SIZE / sizeof(struct page));
+ /*
+ * MEMORY_DEVICE_GENERIC pages start with a refcount of one; other
+ * ZONE_DEVICE pages raise it when handed out. Use atomic_set() because
+ * @head may be a stack template, while init_page_count() expects a
+ * memmap-backed page.
+ */
+ if (pgmap->type == MEMORY_DEVICE_GENERIC)
+ atomic_set(&head->_refcount, 1);
+ /*
+ * ZONE_DEVICE pages union ->lru with a ->pgmap back pointer and
+ * zone_device_data. It is a bug if a ZONE_DEVICE page is ever freed
+ * or placed on a driver-private list.
+ */
+ ((struct folio *)head)->pgmap = pgmap;
+ head->zone_device_data = NULL;
}

-static void __ref memmap_init_compound(struct page *head,
- unsigned long head_pfn,
- unsigned long zone_idx, int nid,
- struct dev_pagemap *pgmap,
- unsigned long nr_pages)
+static void memmap_init_zone_device_basepages(unsigned long start, unsigned long end,
+ const struct zone *zone, struct dev_pagemap *pgmap)
{
- unsigned long pfn, end_pfn = head_pfn + nr_pages;
- unsigned int order = pgmap->vmemmap_shift;
struct page template;

- memmap_init_page_common(&template, zone_idx, nid);
- prep_compound_tail(&template, head, order);
+ memmap_init_page_common(&template, zone_idx(zone), zone_to_nid(zone));
+ memmap_init_zone_device_head_fields(&template, pgmap);

- __SetPageHead(head);
- /* Initialize the tail pages from template. */
- for (pfn = head_pfn + 1; pfn < end_pfn; pfn++)
- memmap_init_page_from_template(pfn_to_page(pfn), pfn,
- device_zone(nid), &template);
- prep_compound_head(head, order);
+ for (unsigned long pfn = start; pfn < end; pfn++) {
+ memmap_init_page_from_template(pfn_to_page(pfn), pfn, zone, &template);
+ if (IS_ALIGNED(pfn, PAGES_PER_SECTION))
+ cond_resched();
+ }
}

void __ref memmap_init_zone_device(struct zone *zone,
@@ -1110,16 +1127,11 @@ void __ref memmap_init_zone_device(struct zone *zone,
struct dev_pagemap *pgmap)
{
unsigned long pfn, end_pfn = start_pfn + nr_pages;
- struct pglist_data *pgdat = zone->zone_pgdat;
struct vmem_altmap *altmap = pgmap_altmap(pgmap);
unsigned int pfns_per_compound = pgmap_vmemmap_nr(pgmap);
- unsigned long zone_idx = zone_idx(zone);
unsigned long start = jiffies;
- int nid = pgdat->node_id;
- struct page template;
- struct page *page;

- if (WARN_ON_ONCE(!pgmap || zone_idx != ZONE_DEVICE))
+ if (WARN_ON_ONCE(!pgmap || !zone_is_zone_device(zone)))
return;

/*
@@ -1134,32 +1146,20 @@ void __ref memmap_init_zone_device(struct zone *zone,

VM_WARN_ON(!IS_ALIGNED(start_pfn | nr_pages, pfns_per_compound));

- memmap_init_page_common(&template, zone_idx, nid);
- page_folio(&template)->pgmap = pgmap;
- /*
- * MEMORY_DEVICE_GENERIC pages regain a refcount of one in the free
- * path. The remaining ZONE_DEVICE types start from zero here and raise
- * the count again when the allocator or driver hands the page out.
- */
- if (pgmap->type == MEMORY_DEVICE_GENERIC)
- atomic_set(&template._refcount, 1);
+ if (pfns_per_compound == 1) {
+ memmap_init_zone_device_basepages(start_pfn, end_pfn, zone, pgmap);
+ goto out;
+ }

- /* Initialize the head pages from template. */
for (pfn = start_pfn; pfn < end_pfn; pfn += pfns_per_compound) {
- page = pfn_to_page(pfn);
-
- memmap_init_page_from_template(page, pfn, zone, &template);
+ struct page *head = pfn_to_page(pfn);

+ memmap_init_compound_page_frozen(head, pfn, zone, pgmap->vmemmap_shift);
+ memmap_init_zone_device_head_fields(head, pgmap);
if (IS_ALIGNED(pfn, PAGES_PER_SECTION))
cond_resched();
-
- if (pfns_per_compound == 1)
- continue;
-
- memmap_init_compound(page, pfn, zone_idx, nid, pgmap,
- compound_nr_pages(pfn, pgmap));
}
-
+out:
pageblock_migratetype_init_range(start_pfn, nr_pages, MIGRATE_MOVABLE,
/* isolate */ false, /* atomic */ false);

diff --git a/mm/mm_init.h b/mm/mm_init.h
index 9de9db31320d..721221d2e4c0 100644
--- a/mm/mm_init.h
+++ b/mm/mm_init.h
@@ -114,5 +114,7 @@ void __meminit __init_single_page(struct page *page, unsigned long pfn,
unsigned long zone, int nid);
void __meminit memmap_init_page_from_template(struct page *page,
unsigned long pfn, const struct zone *zone, struct page *template);
+void __meminit memmap_init_compound_page_frozen(struct page *head,
+ unsigned long pfn, const struct zone *zone, unsigned int order);

#endif /* __MM_MM_INIT_H */
--
2.54.0