[PATCH v2 6/7] mm/page_ext: inline unchecked page extension lookup

From: Muchun Song

Date: Tue Nov 17 2026 - 07:07:51 EST


The KMSAN runtime needs an unchecked page extension lookup that can be
inlined into its uninstrumented compilation unit while page_ext.c remains
instrumented.

Provide an inline lookup for this use case while keeping the existing RCU
checks for other callers.

Signed-off-by: Muchun Song <songmuchun@xxxxxxxxxxxxx>
---
v2:
- Add a preparatory patch that inlines the unchecked lookup (suggested
by Chris Down)
- Use a PFN interface to avoid architecture header-order dependencies
---
include/linux/page_ext.h | 46 ++++++++++++++++++++++++++++++
mm/page_ext.c | 60 +++-------------------------------------
2 files changed, 50 insertions(+), 56 deletions(-)

diff --git a/include/linux/page_ext.h b/include/linux/page_ext.h
index d00320106a58..166f8be11564 100644
--- a/include/linux/page_ext.h
+++ b/include/linux/page_ext.h
@@ -64,7 +64,14 @@ static inline bool early_page_ext_enabled(void)
void page_ext_alloc(void);
void page_ext_init(void);

+static __always_inline struct page_ext *page_ext_entry(void *base, unsigned long index)
+{
+ return base + page_ext_size * index;
+}
+
#ifdef CONFIG_SPARSEMEM
+#define PAGE_EXT_INVALID 0x1
+
static inline bool page_ext_iter_next_fast_possible(unsigned long next_pfn)
{
/*
@@ -73,11 +80,50 @@ static inline bool page_ext_iter_next_fast_possible(unsigned long next_pfn)
*/
return next_pfn % PAGES_PER_SECTION;
}
+
+static __always_inline bool page_ext_invalid(struct page_ext *page_ext)
+{
+ return !page_ext || (((unsigned long)page_ext & PAGE_EXT_INVALID) == PAGE_EXT_INVALID);
+}
+
+static __always_inline struct page_ext *__page_ext_lookup(unsigned long pfn)
+{
+ struct mem_section *section = __pfn_to_section(pfn);
+ struct page_ext *page_ext = READ_ONCE(section->page_ext);
+
+ /*
+ * The sanity checks the page allocator does upon freeing a page can
+ * reach here before the page_ext arrays are allocated when feeding a
+ * range of pages to the allocator for the first time during bootup or
+ * memory hotplug.
+ */
+ if (page_ext_invalid(page_ext))
+ return NULL;
+ return page_ext_entry(page_ext, pfn);
+}
#else
static inline bool page_ext_iter_next_fast_possible(unsigned long next_pfn)
{
return true;
}
+
+static __always_inline struct page_ext *__page_ext_lookup(unsigned long pfn)
+{
+ int nid = pfn_to_nid(pfn);
+ struct page_ext *base = NODE_DATA(nid)->node_page_ext;
+ unsigned long index;
+
+ /*
+ * The sanity checks the page allocator does upon freeing a page can
+ * reach here before the page_ext arrays are allocated when feeding a
+ * range of pages to the allocator for the first time during bootup or
+ * memory hotplug.
+ */
+ if (unlikely(!base))
+ return NULL;
+ index = pfn - round_down(node_start_pfn(nid), MAX_ORDER_NR_PAGES);
+ return page_ext_entry(base, index);
+}
#endif

extern struct page_ext *page_ext_get(const struct page *page);
diff --git a/mm/page_ext.c b/mm/page_ext.c
index 2910727e99e2..0f31e2da2611 100644
--- a/mm/page_ext.c
+++ b/mm/page_ext.c
@@ -63,10 +63,6 @@
* can utilize this callback to initialize the state of it correctly.
*/

-#ifdef CONFIG_SPARSEMEM
-#define PAGE_EXT_INVALID (0x1)
-#endif
-
#if defined(CONFIG_PAGE_IDLE_FLAG) && !defined(CONFIG_64BIT)
static bool need_page_idle(void)
{
@@ -175,33 +171,7 @@ static void __init invoke_init_callbacks(void)
}
}

-static inline struct page_ext *get_entry(void *base, unsigned long index)
-{
- return base + page_ext_size * index;
-}
-
#ifndef CONFIG_SPARSEMEM
-static struct page_ext *lookup_page_ext(const struct page *page)
-{
- unsigned long pfn = page_to_pfn(page);
- unsigned long index;
- struct page_ext *base;
-
- WARN_ON_ONCE(!rcu_read_lock_held());
- base = NODE_DATA(page_to_nid(page))->node_page_ext;
- /*
- * The sanity checks the page allocator does upon freeing a
- * page can reach here before the page_ext arrays are
- * allocated when feeding a range of pages to the allocator
- * for the first time during bootup or memory hotplug.
- */
- if (unlikely(!base))
- return NULL;
- index = pfn - round_down(node_start_pfn(page_to_nid(page)),
- MAX_ORDER_NR_PAGES);
- return get_entry(base, index);
-}
-
static int __init alloc_node_page_ext(int nid)
{
struct page_ext *base;
@@ -247,29 +217,6 @@ static int __init __page_ext_alloc(void)
}

#else /* CONFIG_SPARSEMEM */
-static bool page_ext_invalid(struct page_ext *page_ext)
-{
- return !page_ext || (((unsigned long)page_ext & PAGE_EXT_INVALID) == PAGE_EXT_INVALID);
-}
-
-static struct page_ext *lookup_page_ext(const struct page *page)
-{
- unsigned long pfn = page_to_pfn(page);
- struct mem_section *section = __pfn_to_section(pfn);
- struct page_ext *page_ext = READ_ONCE(section->page_ext);
-
- WARN_ON_ONCE(!rcu_read_lock_held());
- /*
- * The sanity checks the page allocator does upon freeing a
- * page can reach here before the page_ext arrays are
- * allocated when feeding a range of pages to the allocator
- * for the first time during bootup or memory hotplug.
- */
- if (page_ext_invalid(page_ext))
- return NULL;
- return get_entry(page_ext, pfn);
-}
-
static __ref void *alloc_page_ext(size_t size, int nid)
{
void *addr;
@@ -360,7 +307,7 @@ static void __free_page_ext(unsigned long pfn)
base = (void *)base - PAGE_EXT_INVALID;
WRITE_ONCE(ms->page_ext, NULL);

- base = get_entry(base, pfn);
+ base = page_ext_entry(base, pfn);
free_page_ext(base);
}

@@ -503,7 +450,8 @@ void __init page_ext_init(void)
*/
struct page_ext *page_ext_lookup(unsigned long pfn)
{
- return lookup_page_ext(pfn_to_page(pfn));
+ WARN_ON_ONCE(!rcu_read_lock_held());
+ return __page_ext_lookup(pfn);
}

/**
@@ -522,7 +470,7 @@ struct page_ext *page_ext_get(const struct page *page)
struct page_ext *page_ext;

rcu_read_lock();
- page_ext = lookup_page_ext(page);
+ page_ext = page_ext_lookup(page_to_pfn(page));
if (!page_ext) {
rcu_read_unlock();
return NULL;
--
2.54.0