[PATCH v2 1/7] kmsan: encapsulate per-page metadata access

From: Muchun Song

Date: Tue Nov 17 2026 - 07:07:27 EST


KMSAN directly accesses page->kmsan_shadow and page->kmsan_origin
throughout the shadow implementation. This couples these users to
storing the metadata pointers in struct page, requiring every access
site to change when the storage moves elsewhere.

Introduce helpers that get and set both metadata page pointers as a pair
and convert the existing users to them.

kmsan_get_shadow_origin_ptr() is the main metadata lookup path for
compiler-instrumented memory accesses and requires both pointers. For
page-backed addresses, resolve the backing page once and obtain the
metadata pair through one lookup instead of calling the single-metadata
path twice. This also avoids duplicate lookups after moving the metadata
storage out of struct page.

No functional change is intended.

Signed-off-by: Muchun Song <songmuchun@xxxxxxxxxxxxx>
---
v2:
- Use one page metadata lookup for shadow and origin pointers (suggested
by Chris Down)
---
mm/kmsan/shadow.c | 166 +++++++++++++++++++++++++++++++---------------
1 file changed, 111 insertions(+), 55 deletions(-)

diff --git a/mm/kmsan/shadow.c b/mm/kmsan/shadow.c
index 0c88d89bf0d6..300b48818af1 100644
--- a/mm/kmsan/shadow.c
+++ b/mm/kmsan/shadow.c
@@ -9,6 +9,7 @@

#include <asm/kmsan.h>
#include <asm/tlbflush.h>
+#include <linux/bits.h>
#include <linux/cacheflush.h>
#include <linux/memblock.h>
#include <linux/mm_types.h>
@@ -20,29 +21,46 @@
#include "../vmalloc.h"
#include "kmsan.h"

-#define shadow_page_for(page) ((page)->kmsan_shadow)
+#define KMSAN_META_MASK_SHADOW BIT(0)
+#define KMSAN_META_MASK_ORIGIN BIT(1)

-#define origin_page_for(page) ((page)->kmsan_origin)
+struct page_metadata {
+ struct page *shadow;
+ struct page *origin;
+};

-static void *shadow_ptr_for(struct page *page)
+static inline struct page_metadata get_page_metadata(struct page *page)
{
- return page_address(shadow_page_for(page));
+ return (struct page_metadata) {
+ .shadow = page->kmsan_shadow,
+ .origin = page->kmsan_origin,
+ };
}

-static void *origin_ptr_for(struct page *page)
+static inline void set_page_metadata(struct page *page, struct page_metadata meta)
{
- return page_address(origin_page_for(page));
+ page->kmsan_shadow = meta.shadow;
+ page->kmsan_origin = meta.origin;
}

-static bool page_has_metadata(struct page *page)
+static inline void *shadow_ptr_for(struct page_metadata meta)
{
- return shadow_page_for(page) && origin_page_for(page);
+ return page_address(meta.shadow);
+}
+
+static inline void *origin_ptr_for(struct page_metadata meta)
+{
+ return page_address(meta.origin);
+}
+
+static inline bool page_has_metadata(struct page_metadata meta)
+{
+ return meta.shadow && meta.origin;
}

static void set_no_shadow_origin_page(struct page *page)
{
- shadow_page_for(page) = NULL;
- origin_page_for(page) = NULL;
+ set_page_metadata(page, (struct page_metadata) {});
}

/*
@@ -79,11 +97,56 @@ static struct page *virt_to_page_or_null(void *vaddr)
return NULL;
}

+static __always_inline
+struct shadow_origin_ptr __kmsan_get_metadata(void *address, unsigned int mask)
+{
+ struct shadow_origin_ptr ret = {};
+ struct page_metadata meta;
+ struct page *page;
+ u64 addr = (u64)address;
+ u64 origin_addr = ALIGN_DOWN(addr, KMSAN_ORIGIN_SIZE);
+
+ if (kmsan_internal_is_vmalloc_addr(address) ||
+ kmsan_internal_is_module_addr(address)) {
+ if (mask & KMSAN_META_MASK_SHADOW)
+ ret.shadow = (void *)vmalloc_meta(address, KMSAN_META_SHADOW);
+ if (mask & KMSAN_META_MASK_ORIGIN)
+ ret.origin = (void *)vmalloc_meta((void *)origin_addr,
+ KMSAN_META_ORIGIN);
+ return ret;
+ }
+
+ if (mask & KMSAN_META_MASK_SHADOW) {
+ ret.shadow = arch_kmsan_get_meta_or_null(address, KMSAN_META_SHADOW);
+ if (!ret.shadow)
+ goto lookup_page;
+ }
+ if (mask & KMSAN_META_MASK_ORIGIN)
+ ret.origin = arch_kmsan_get_meta_or_null((void *)origin_addr,
+ KMSAN_META_ORIGIN);
+ if (ret.shadow || ret.origin)
+ return ret;
+
+lookup_page:
+ page = virt_to_page_or_null(address);
+ if (!page)
+ return ret;
+ meta = get_page_metadata(page);
+ if (!page_has_metadata(meta))
+ return ret;
+
+ if (mask & KMSAN_META_MASK_SHADOW)
+ ret.shadow = shadow_ptr_for(meta) + offset_in_page(addr);
+ if (mask & KMSAN_META_MASK_ORIGIN)
+ ret.origin = origin_ptr_for(meta) + offset_in_page(origin_addr);
+
+ return ret;
+}
+
struct shadow_origin_ptr kmsan_get_shadow_origin_ptr(void *address, u64 size,
bool store)
{
struct shadow_origin_ptr ret;
- void *shadow;

/*
* Even if we redirect this memory access to the dummy page, it will
@@ -95,12 +158,10 @@ struct shadow_origin_ptr kmsan_get_shadow_origin_ptr(void *address, u64 size,
goto return_dummy;

KMSAN_WARN_ON(!kmsan_metadata_is_contiguous(address, size));
- shadow = kmsan_get_metadata(address, KMSAN_META_SHADOW);
- if (!shadow)
+ ret = __kmsan_get_metadata(address, KMSAN_META_MASK_SHADOW |
+ KMSAN_META_MASK_ORIGIN);
+ if (!ret.shadow)
goto return_dummy;
-
- ret.shadow = shadow;
- ret.origin = kmsan_get_metadata(address, KMSAN_META_ORIGIN);
return ret;

return_dummy:
@@ -124,46 +185,36 @@ struct shadow_origin_ptr kmsan_get_shadow_origin_ptr(void *address, u64 size,
*/
void *kmsan_get_metadata(void *address, bool is_origin)
{
- u64 addr = (u64)address, off;
- struct page *page;
- void *ret;
-
- if (is_origin)
- addr = ALIGN_DOWN(addr, KMSAN_ORIGIN_SIZE);
- address = (void *)addr;
- if (kmsan_internal_is_vmalloc_addr(address) ||
- kmsan_internal_is_module_addr(address))
- return (void *)vmalloc_meta(address, is_origin);
-
- ret = arch_kmsan_get_meta_or_null(address, is_origin);
- if (ret)
- return ret;
-
- page = virt_to_page_or_null(address);
- if (!page)
- return NULL;
- if (!page_has_metadata(page))
- return NULL;
- off = offset_in_page(addr);
+ struct shadow_origin_ptr ret;
+ unsigned int mask;

- return (is_origin ? origin_ptr_for(page) : shadow_ptr_for(page)) + off;
+ mask = is_origin ? KMSAN_META_MASK_ORIGIN : KMSAN_META_MASK_SHADOW;
+ ret = __kmsan_get_metadata(address, mask);
+ return is_origin ? ret.origin : ret.shadow;
}

void kmsan_copy_page_meta(struct page *dst, struct page *src)
{
+ struct page_metadata dst_meta, src_meta;
+
if (!kmsan_enabled || kmsan_in_runtime())
return;
- if (!dst || !page_has_metadata(dst))
+ if (!dst)
return;
- if (!src || !page_has_metadata(src)) {
+ dst_meta = get_page_metadata(dst);
+ if (!page_has_metadata(dst_meta))
+ return;
+ if (src)
+ src_meta = get_page_metadata(src);
+ if (!src || !page_has_metadata(src_meta)) {
kmsan_internal_unpoison_memory(page_address(dst), PAGE_SIZE,
/*checked*/ false);
return;
}

kmsan_enter_runtime();
- __memcpy(shadow_ptr_for(dst), shadow_ptr_for(src), PAGE_SIZE);
- __memcpy(origin_ptr_for(dst), origin_ptr_for(src), PAGE_SIZE);
+ __memcpy(shadow_ptr_for(dst_meta), shadow_ptr_for(src_meta), PAGE_SIZE);
+ __memcpy(origin_ptr_for(dst_meta), origin_ptr_for(src_meta), PAGE_SIZE);
kmsan_leave_runtime();
}
EXPORT_SYMBOL(kmsan_copy_page_meta);
@@ -171,19 +222,18 @@ EXPORT_SYMBOL(kmsan_copy_page_meta);
void kmsan_alloc_page(struct page *page, unsigned int order, gfp_t flags)
{
bool initialized = (flags & __GFP_ZERO) || !kmsan_enabled;
- struct page *shadow, *origin;
+ struct page_metadata meta;
depot_stack_handle_t handle;
int pages = 1 << order;

if (!page)
return;

- shadow = shadow_page_for(page);
- origin = origin_page_for(page);
+ meta = get_page_metadata(page);

if (initialized) {
- __memset(page_address(shadow), 0, PAGE_SIZE * pages);
- __memset(page_address(origin), 0, PAGE_SIZE * pages);
+ __memset(shadow_ptr_for(meta), 0, PAGE_SIZE * pages);
+ __memset(origin_ptr_for(meta), 0, PAGE_SIZE * pages);
return;
}

@@ -191,7 +241,7 @@ void kmsan_alloc_page(struct page *page, unsigned int order, gfp_t flags)
if (kmsan_in_runtime())
return;

- __memset(page_address(shadow), -1, PAGE_SIZE * pages);
+ __memset(shadow_ptr_for(meta), -1, PAGE_SIZE * pages);
kmsan_enter_runtime();
handle = kmsan_save_stack_with_flags(flags, /*extra_bits*/ 0);
kmsan_leave_runtime();
@@ -200,7 +250,7 @@ void kmsan_alloc_page(struct page *page, unsigned int order, gfp_t flags)
* to just fill the origin pages with @handle.
*/
for (int i = 0; i < PAGE_SIZE * pages / sizeof(handle); i++)
- ((depot_stack_handle_t *)page_address(origin))[i] = handle;
+ ((depot_stack_handle_t *)origin_ptr_for(meta))[i] = handle;
}

void kmsan_free_page(struct page *page, unsigned int order)
@@ -238,8 +288,10 @@ int kmsan_vmap_pages_range_noflush(unsigned long start, unsigned long end,
goto ret;
}
for (int i = 0; i < nr; i++) {
- s_pages[i] = shadow_page_for(pages[i]);
- o_pages[i] = origin_page_for(pages[i]);
+ struct page_metadata meta = get_page_metadata(pages[i]);
+
+ s_pages[i] = meta.shadow;
+ o_pages[i] = meta.origin;
}
prot = PAGE_KERNEL;

@@ -289,10 +341,12 @@ void __init kmsan_init_alloc_meta_for_range(void *start, void *end)
page = virt_to_page_or_null((char *)start + addr);
shadow_p = virt_to_page((char *)shadow + addr);
set_no_shadow_origin_page(shadow_p);
- shadow_page_for(page) = shadow_p;
origin_p = virt_to_page((char *)origin + addr);
set_no_shadow_origin_page(origin_p);
- origin_page_for(page) = origin_p;
+ set_page_metadata(page, (struct page_metadata) {
+ .shadow = shadow_p,
+ .origin = origin_p,
+ });
}
}

@@ -302,7 +356,9 @@ void kmsan_setup_meta(struct page *page, struct page *shadow,
for (int i = 0; i < (1 << order); i++) {
set_no_shadow_origin_page(&shadow[i]);
set_no_shadow_origin_page(&origin[i]);
- shadow_page_for(&page[i]) = &shadow[i];
- origin_page_for(&page[i]) = &origin[i];
+ set_page_metadata(&page[i], (struct page_metadata) {
+ .shadow = &shadow[i],
+ .origin = &origin[i],
+ });
}
}
--
2.54.0