[PATCH 07/13] mm/page_alloc: allow lower-order unaccepted_pages
From: Kevin Loughlin
Date: Mon Oct 26 2026 - 23:04:06 EST
Currently, zone->unaccepted_pages assumes all unaccepted blocks are
MAX_PAGE_ORDER. However, dynamic memory unacceptance operates at
unaccepted_memory_order() granularity (typically 2 MiB / PMD_ORDER on
x86).
Introduce has_unaccepted_memory() to report whether lazy unaccepted
memory and dynamic unacceptance are active (returning false when booted
with accept_memory=eager), and generalize zone->unaccepted_pages from
MAX_PAGE_ORDER to orders >= unaccepted_memory_order() stored in
page_private(page):
- Update __free_unaccepted_locked() to verify that pageblocks are
mergeable (moving blocks >= pageblock_order to MIGRATE_MOVABLE) and
record the block order in page_private(page).
- Update __accept_page(), try_to_accept_memory_one(), and
cond_accept_memory() to accept blocks of order page_private(page),
and ensure cond_accept_memory() accepts at least (1 << order) pages
when high-order watermark or rmqueue checks fail.
- Update prep_move_freepages_block(), set_migratetype_isolate(),
isolate_single_pageblock(), and skip_buddy_pages() to handle
variable-order PageUnaccepted blocks.
Signed-off-by: Kevin Loughlin <kevinloughlin@xxxxxxxxxx>
---
Documentation/mm/physical_memory.rst | 3 +-
drivers/firmware/efi/unaccepted_memory.c | 2 +-
include/linux/mm.h | 6 ++
include/linux/mmzone.h | 2 +-
mm/page_alloc.c | 127 ++++++++++++++++++-----
mm/page_isolation.c | 44 +++++++-
mm/page_owner.c | 7 +-
7 files changed, 155 insertions(+), 36 deletions(-)
diff --git a/Documentation/mm/physical_memory.rst b/Documentation/mm/physical_memory.rst
index 2e67e8b23a99..e9fb554932d5 100644
--- a/Documentation/mm/physical_memory.rst
+++ b/Documentation/mm/physical_memory.rst
@@ -526,7 +526,8 @@ General
``zone_init_free_lists()``.
``unaccepted_pages``
- The list of pages to be accepted. All pages on the list are ``MAX_PAGE_ORDER``.
+ The list of pages to be accepted. All pages on the list are at least
+ ``unaccepted_memory_order()`` and record their order in ``page_private()``.
Defined only when ``CONFIG_UNACCEPTED_MEMORY`` is enabled.
``flags``
diff --git a/drivers/firmware/efi/unaccepted_memory.c b/drivers/firmware/efi/unaccepted_memory.c
index 128bc1b113cd..769faf430690 100644
--- a/drivers/firmware/efi/unaccepted_memory.c
+++ b/drivers/firmware/efi/unaccepted_memory.c
@@ -246,7 +246,7 @@ int mark_memory_unaccepted(phys_addr_t start, unsigned long size)
unsigned long flags;
u64 unit_size;
- if (!arch_has_unaccept_memory())
+ if (!has_unaccepted_memory())
return -EOPNOTSUPP;
unaccepted = efi_get_unaccepted_table();
diff --git a/include/linux/mm.h b/include/linux/mm.h
index 27bb40651c85..619c71565dd6 100644
--- a/include/linux/mm.h
+++ b/include/linux/mm.h
@@ -5362,6 +5362,7 @@ int set_anon_vma_name(unsigned long addr, unsigned long size,
bool range_contains_unaccepted_memory(phys_addr_t start, unsigned long size);
void accept_memory(phys_addr_t start, unsigned long size);
+bool has_unaccepted_memory(void);
unsigned int unaccepted_memory_order(void);
int mark_memory_unaccepted(phys_addr_t start, unsigned long size);
@@ -5377,6 +5378,11 @@ static inline void accept_memory(phys_addr_t start, unsigned long size)
{
}
+static inline bool has_unaccepted_memory(void)
+{
+ return false;
+}
+
static inline unsigned int unaccepted_memory_order(void)
{
return 0;
diff --git a/include/linux/mmzone.h b/include/linux/mmzone.h
index d44b04589f48..ae4657dbb5a8 100644
--- a/include/linux/mmzone.h
+++ b/include/linux/mmzone.h
@@ -1112,7 +1112,7 @@ struct zone {
struct free_area free_area[NR_PAGE_ORDERS];
#ifdef CONFIG_UNACCEPTED_MEMORY
- /* Pages to be accepted. All pages on the list are MAX_PAGE_ORDER */
+ /* Pages to be accepted. All pages are >= unaccepted_memory_order() */
struct list_head unaccepted_pages;
/* To be called once the last page in the zone is accepted */
diff --git a/mm/page_alloc.c b/mm/page_alloc.c
index 3322f9e75616..46ae8c6f3814 100644
--- a/mm/page_alloc.c
+++ b/mm/page_alloc.c
@@ -300,8 +300,10 @@ static unsigned long alloc_stall_warn_jiffies = INITIAL_JIFFIES;
static bool page_contains_unaccepted(struct page *page, unsigned int order);
static bool cond_accept_memory(struct zone *zone, unsigned int order,
- int alloc_flags);
-static bool __free_unaccepted(struct page *page);
+ int alloc_flags, bool force);
+static bool __free_unaccepted(struct page *page, unsigned int order);
+static bool __free_unaccepted_locked(struct page *page, struct zone *zone,
+ unsigned int order);
int page_group_by_mobility_disabled __read_mostly;
@@ -1614,8 +1616,10 @@ void __meminit __free_pages_core(struct page *page, unsigned int order,
atomic_long_add(nr_pages, &page_zone(page)->managed_pages);
}
- if (page_contains_unaccepted(page, order)) {
- if (order == MAX_PAGE_ORDER && __free_unaccepted(page))
+ if (range_contains_unaccepted_memory(page_to_phys(page),
+ PAGE_SIZE << order)) {
+ if (order >= unaccepted_memory_order() &&
+ __free_unaccepted(page, order))
return;
accept_memory(page_to_phys(page), PAGE_SIZE << order);
@@ -2003,6 +2007,13 @@ static bool prep_move_freepages_block(struct zone *zone, struct page *page,
pfn += nr;
continue;
}
+ if (PageUnaccepted(page)) {
+ int nr = 1 << page_private(page);
+
+ *num_free += nr;
+ pfn += nr;
+ continue;
+ }
/*
* We assume that pages that could be isolated for
* migration are movable. But we don't actually try
@@ -3872,7 +3883,7 @@ get_page_from_freelist(gfp_t gfp_mask, unsigned int order, int alloc_flags,
continue;
}
- cond_accept_memory(zone, order, alloc_flags);
+ cond_accept_memory(zone, order, alloc_flags, false);
/*
* Detect whether the number of free pages is below high
@@ -3897,7 +3908,7 @@ get_page_from_freelist(gfp_t gfp_mask, unsigned int order, int alloc_flags,
if (!zone_watermark_fast(zone, order, mark,
ac->highest_zoneidx, alloc_flags,
gfp_mask)) {
- if (cond_accept_memory(zone, order, alloc_flags))
+ if (cond_accept_memory(zone, order, alloc_flags, true))
goto try_this_zone;
/*
@@ -3934,7 +3945,7 @@ get_page_from_freelist(gfp_t gfp_mask, unsigned int order, int alloc_flags,
return page;
} else {
- if (cond_accept_memory(zone, order, alloc_flags))
+ if (cond_accept_memory(zone, order, alloc_flags, true))
goto try_this_zone;
/* Try again if zone has deferred pages */
@@ -5216,7 +5227,7 @@ unsigned long alloc_pages_bulk_noprof(gfp_t gfp, int preferred_nid,
goto failed;
}
- cond_accept_memory(zone, 0, alloc_flags);
+ cond_accept_memory(zone, 0, alloc_flags, false);
retry_this_zone:
mark = wmark_pages(zone, alloc_flags & ALLOC_WMARK_MASK) + nr_pages - nr_populated;
if (zone_watermark_fast(zone, 0, mark,
@@ -5225,7 +5236,7 @@ unsigned long alloc_pages_bulk_noprof(gfp_t gfp, int preferred_nid,
break;
}
- if (cond_accept_memory(zone, 0, alloc_flags))
+ if (cond_accept_memory(zone, 0, alloc_flags, true))
goto retry_this_zone;
/* Try again if zone has deferred pages */
@@ -7867,6 +7878,7 @@ bool has_managed_zone(enum zone_type zone)
}
#ifdef CONFIG_UNACCEPTED_MEMORY
+#include <asm/unaccepted_memory.h>
static bool lazy_accept = true;
@@ -7884,25 +7896,45 @@ static int __init accept_memory_parse(char *p)
}
early_param("accept_memory", accept_memory_parse);
+bool has_unaccepted_memory(void)
+{
+ return lazy_accept && efi_get_unaccepted_table() &&
+ arch_has_unaccept_memory() &&
+ unaccepted_memory_order() <= MAX_PAGE_ORDER;
+}
+EXPORT_SYMBOL_GPL(has_unaccepted_memory);
+
static bool page_contains_unaccepted(struct page *page, unsigned int order)
{
phys_addr_t start = page_to_phys(page);
+ /*
+ * Unaccepted memory is tracked and freed in blocks of at least
+ * unaccepted_memory_order(). Skipping lower-order frees avoids
+ * acquiring unaccepted_memory_lock on normal page frees.
+ */
+ if (!has_unaccepted_memory() || order < unaccepted_memory_order())
+ return false;
+
return range_contains_unaccepted_memory(start, PAGE_SIZE << order);
}
static void __accept_page(struct zone *zone, unsigned long *flags,
struct page *page)
{
+ unsigned int order = page_private(page);
+ unsigned int nr_pages = 1 << order;
+
list_del(&page->lru);
- account_freepages(zone, -MAX_ORDER_NR_PAGES, MIGRATE_MOVABLE);
- __mod_zone_page_state(zone, NR_UNACCEPTED, -MAX_ORDER_NR_PAGES);
+ account_freepages(zone, -nr_pages, MIGRATE_MOVABLE);
+ __mod_zone_page_state(zone, NR_UNACCEPTED, -nr_pages);
__ClearPageUnaccepted(page);
+ set_page_private(page, 0);
spin_unlock_irqrestore(&zone->lock, *flags);
- accept_memory(page_to_phys(page), PAGE_SIZE << MAX_PAGE_ORDER);
+ accept_memory(page_to_phys(page), PAGE_SIZE << order);
- __free_pages_ok(page, MAX_PAGE_ORDER, FPI_TO_TAIL);
+ __free_pages_ok(page, order, FPI_TO_TAIL);
}
void accept_page(struct page *page)
@@ -7920,27 +7952,29 @@ void accept_page(struct page *page)
__accept_page(zone, &flags, page);
}
-static bool try_to_accept_memory_one(struct zone *zone)
+static unsigned int try_to_accept_memory_one(struct zone *zone)
{
unsigned long flags;
struct page *page;
+ unsigned int nr_pages;
spin_lock_irqsave(&zone->lock, flags);
page = list_first_entry_or_null(&zone->unaccepted_pages,
struct page, lru);
if (!page) {
spin_unlock_irqrestore(&zone->lock, flags);
- return false;
+ return 0;
}
+ nr_pages = 1 << page_private(page);
/* Unlocks zone->lock */
__accept_page(zone, &flags, page);
- return true;
+ return nr_pages;
}
static bool cond_accept_memory(struct zone *zone, unsigned int order,
- int alloc_flags)
+ int alloc_flags, bool force)
{
long to_accept, wmark;
bool ret = false;
@@ -7957,7 +7991,7 @@ static bool cond_accept_memory(struct zone *zone, unsigned int order,
/*
* Watermarks have not been initialized yet.
*
- * Accepting one MAX_PAGE_ORDER page to ensure progress.
+ * Accepting one block to ensure progress.
*/
if (!wmark)
return try_to_accept_memory_one(zone);
@@ -7968,32 +8002,62 @@ static bool cond_accept_memory(struct zone *zone, unsigned int order,
__zone_watermark_unusable_free(zone, order, 0) -
zone_page_state(zone, NR_UNACCEPTED));
+ if (force && to_accept < (1L << order))
+ to_accept = 1L << order;
+
while (to_accept > 0) {
- if (!try_to_accept_memory_one(zone))
+ unsigned int accepted = try_to_accept_memory_one(zone);
+
+ if (!accepted)
break;
ret = true;
- to_accept -= MAX_ORDER_NR_PAGES;
+ to_accept -= accepted;
}
return ret;
}
-static bool __free_unaccepted(struct page *page)
+static bool __free_unaccepted_locked(struct page *page, struct zone *zone,
+ unsigned int order)
+{
+ unsigned int nr_pages = 1 << order;
+ unsigned int step = max_t(unsigned int, 1, pageblock_nr_pages);
+ unsigned int i;
+
+ lockdep_assert_held(&zone->lock);
+
+ for (i = 0; i < nr_pages; i += step) {
+ int mt = get_pageblock_migratetype(page + i);
+
+ if (!migratetype_is_mergeable(mt))
+ return false;
+ }
+
+ if (order >= pageblock_order)
+ change_pageblock_range(page, order, MIGRATE_MOVABLE);
+
+ set_page_private(page, order);
+ list_add_tail(&page->lru, &zone->unaccepted_pages);
+ account_freepages(zone, nr_pages, MIGRATE_MOVABLE);
+ __mod_zone_page_state(zone, NR_UNACCEPTED, nr_pages);
+ __SetPageUnaccepted(page);
+ return true;
+}
+
+static bool __free_unaccepted(struct page *page, unsigned int order)
{
struct zone *zone = page_zone(page);
unsigned long flags;
+ bool ret;
if (!lazy_accept)
return false;
spin_lock_irqsave(&zone->lock, flags);
- list_add_tail(&page->lru, &zone->unaccepted_pages);
- account_freepages(zone, MAX_ORDER_NR_PAGES, MIGRATE_MOVABLE);
- __mod_zone_page_state(zone, NR_UNACCEPTED, MAX_ORDER_NR_PAGES);
- __SetPageUnaccepted(page);
+ ret = __free_unaccepted_locked(page, zone, order);
spin_unlock_irqrestore(&zone->lock, flags);
- return true;
+ return ret;
}
#else
@@ -8004,12 +8068,19 @@ static bool page_contains_unaccepted(struct page *page, unsigned int order)
}
static bool cond_accept_memory(struct zone *zone, unsigned int order,
- int alloc_flags)
+ int alloc_flags, bool force)
{
return false;
}
-static bool __free_unaccepted(struct page *page)
+static bool __free_unaccepted_locked(struct page *page, struct zone *zone,
+ unsigned int order)
+{
+ BUILD_BUG();
+ return false;
+}
+
+static bool __free_unaccepted(struct page *page, unsigned int order)
{
BUILD_BUG();
return false;
diff --git a/mm/page_isolation.c b/mm/page_isolation.c
index 5aaad037384e..c85ce158a646 100644
--- a/mm/page_isolation.c
+++ b/mm/page_isolation.c
@@ -165,6 +165,36 @@ static struct page *has_unmovable_pages(unsigned long start_pfn, unsigned long e
return NULL;
}
+static struct page *find_unaccepted_page_in_pageblock(struct zone *zone,
+ struct page *page)
+{
+#ifdef CONFIG_UNACCEPTED_MEMORY
+ unsigned long pb_start = pageblock_start_pfn(page_to_pfn(page));
+ unsigned long pb_end = pageblock_end_pfn(page_to_pfn(page));
+ unsigned long start_pfn, pfn, step;
+
+ lockdep_assert_held(&zone->lock);
+
+ if (list_empty(&zone->unaccepted_pages) ||
+ unaccepted_memory_order() > MAX_PAGE_ORDER)
+ return NULL;
+
+ start_pfn = ALIGN_DOWN(pb_start, MAX_ORDER_NR_PAGES);
+ step = 1UL << unaccepted_memory_order();
+
+ for (pfn = start_pfn; pfn < pb_end; pfn += step) {
+ struct page *cand = pfn_to_online_page(pfn);
+
+ if (!cand || page_zone(cand) != zone || !PageUnaccepted(cand))
+ continue;
+
+ if (pfn + (1UL << page_private(cand)) > pb_start)
+ return cand;
+ }
+#endif
+ return NULL;
+}
+
/*
* This function set pageblock migratetype to isolate if no unmovable page is
* present in [start_pfn, end_pfn). The pageblock must intersect with
@@ -175,10 +205,14 @@ static int set_migratetype_isolate(struct page *page, enum pb_isolate_mode mode,
{
struct zone *zone = page_zone(page);
struct page *unmovable;
+ struct page *unaccepted = NULL;
unsigned long check_unmovable_start, check_unmovable_end;
- if (PageUnaccepted(page))
- accept_page(page);
+retry:
+ if (unaccepted) {
+ accept_page(unaccepted);
+ unaccepted = NULL;
+ }
scoped_guard(spinlock_irqsave, &zone->lock) {
/*
@@ -189,6 +223,10 @@ static int set_migratetype_isolate(struct page *page, enum pb_isolate_mode mode,
if (is_migrate_isolate_page(page))
return -EBUSY;
+ unaccepted = find_unaccepted_page_in_pageblock(zone, page);
+ if (unaccepted)
+ goto retry;
+
/*
* FIXME: Now, memory hotplug doesn't call shrink_slab() by
* itself. We just check MOVABLE pages.
@@ -383,7 +421,7 @@ static int isolate_single_pageblock(unsigned long boundary_pfn,
pfn = page_to_pfn(page);
if (PageUnaccepted(page)) {
- pfn += MAX_ORDER_NR_PAGES;
+ accept_page(page);
continue;
}
diff --git a/mm/page_owner.c b/mm/page_owner.c
index cfc31c92d765..e3f3a019b0bc 100644
--- a/mm/page_owner.c
+++ b/mm/page_owner.c
@@ -459,10 +459,13 @@ static inline bool skip_buddy_pages(unsigned long *pfn, struct page *page)
{
unsigned long order;
- if (!PageBuddy(page))
+ if (PageBuddy(page))
+ order = buddy_order_unsafe(page);
+ else if (PageUnaccepted(page))
+ order = READ_ONCE(page->private);
+ else
return false;
- order = buddy_order_unsafe(page);
if (order <= MAX_PAGE_ORDER) {
unsigned long new_pfn = *pfn + (1UL << order);
unsigned long boundary = ALIGN(*pfn + 1, MAX_ORDER_NR_PAGES);
--
2.56.0.307.g5fe7bd680c-goog