Re: [PATCH 1/1] mm: Fix struct page layout on 32-bit systems

From: Matthew Wilcox
Date: Mon Apr 12 2021 - 14:24:21 EST

On Sun, Apr 11, 2021 at 11:43:07AM +0200, Jesper Dangaard Brouer wrote:
> Could you explain your intent here?
> I worry about @index.
>
> As I mentioned in other thread[1] netstack use page_is_pfmemalloc()
> (code copy-pasted below signature) which imply that the member @index
> have to be kept intact. In above, I'm unsure @index is untouched.

Well, I tried three different approaches. Here's the one I hated the least.

From: "Matthew Wilcox (Oracle)" <willy@xxxxxxxxxxxxx>
Date: Sat, 10 Apr 2021 16:12:06 -0400
Subject: [PATCH] mm: Fix struct page layout on 32-bit systems

32-bit architectures which expect 8-byte alignment for 8-byte integers
and need 64-bit DMA addresses (arc, arm, mips, ppc) had their struct
page inadvertently expanded in 2019. When the dma_addr_t was added,
it forced the alignment of the union to 8 bytes, which inserted a 4 byte
gap between 'flags' and the union.

We could fix this by telling the compiler to use a smaller alignment
for the dma_addr, but that seems a little fragile. Instead, move the
'flags' into the union. That causes dma_addr to shift into the same
bits as 'mapping', which causes problems with page_mapping() called from
set_page_dirty() in the munmap path. To avoid this, insert three words
of padding and use the same bits as ->index and ->private, neither of
which have to be cleared on free.

However, page->index is currently used to indicate page_is_pfmemalloc.
Move that information to bit 1 of page->lru (aka compound_head). This
has the same properties; it will be overwritten by callers who do
not care about pfmemalloc (as opposed to using a bit in page->flags).

Fixes: c25fff7171be ("mm: add dma_addr_t to struct page")
Signed-off-by: Matthew Wilcox (Oracle) <willy@xxxxxxxxxxxxx>
---
include/linux/mm.h | 12 +++++++-----
include/linux/mm_types.h | 38 ++++++++++++++++++++++++++------------
2 files changed, 33 insertions(+), 17 deletions(-)

diff --git a/include/linux/mm.h b/include/linux/mm.h
index b58c73e50da0..23cca0eaa9da 100644
--- a/include/linux/mm.h
+++ b/include/linux/mm.h
@@ -1668,10 +1668,12 @@ struct address_space *page_mapping(struct page *page);
static inline bool page_is_pfmemalloc(const struct page *page)
{
/*
- * Page index cannot be this large so this must be
- * a pfmemalloc page.
+ * This is not a tail page; compound_head of a head page is unused
+ * at return from the page allocator, and will be overwritten
+ * by callers who do not care whether the page came from the
+ * reserves.
*/
- return page->index == -1UL;
+ return page->compound_head & 2;
}

/*
@@ -1680,12 +1682,12 @@ static inline bool page_is_pfmemalloc(const struct page *page)
*/
static inline void set_page_pfmemalloc(struct page *page)
{
- page->index = -1UL;
+ page->compound_head = 2;
}

static inline void clear_page_pfmemalloc(struct page *page)
{
- page->index = 0;
+ page->compound_head = 0;
}

/*
diff --git a/include/linux/mm_types.h b/include/linux/mm_types.h
index 6613b26a8894..45c563e9b50e 100644
--- a/include/linux/mm_types.h
+++ b/include/linux/mm_types.h
@@ -68,16 +68,22 @@ struct mem_cgroup;
#endif

struct page {
- unsigned long flags; /* Atomic flags, some possibly
- * updated asynchronously */
/*
- * Five words (20/40 bytes) are available in this union.
- * WARNING: bit 0 of the first word is used for PageTail(). That
- * means the other users of this union MUST NOT use the bit to
+ * This union is six words (24 / 48 bytes) in size.
+ * The first word is reserved for atomic flags, often updated
+ * asynchronously. Use the PageFoo() macros to access it. Some
+ * of the flags can be reused for your own purposes, but the
+ * word as a whole often contains other information and overwriting
+ * it will cause functions like page_zone() and page_node() to stop
+ * working correctly.
+ *
+ * Bit 0 of the second word is used for PageTail(). That
+ * means the other users of this union MUST leave the bit zero to
* avoid collision and false-positive PageTail().
*/
union {
struct { /* Page cache and anonymous pages */
+ unsigned long flags;
/**
* @lru: Pageout list, eg. active_list protected by
* lruvec->lru_lock. Sometimes used as a generic list
@@ -96,13 +102,14 @@ struct page {
unsigned long private;
};
struct { /* page_pool used by netstack */
- /**
- * @dma_addr: might require a 64-bit value even on
- * 32-bit architectures.
- */
- dma_addr_t dma_addr;
+ unsigned long _pp_flags;
+ unsigned long pp_magic;
+ unsigned long xmi;
+ unsigned long _pp_mapping_pad;
+ dma_addr_t dma_addr; /* might be one or two words */
};
struct { /* slab, slob and slub */
+ unsigned long _slab_flags;
union {
struct list_head slab_list;
struct { /* Partial pages */
@@ -130,6 +137,7 @@ struct page {
};
};
struct { /* Tail pages of compound page */
+ unsigned long _t1_flags;
unsigned long compound_head; /* Bit zero is set */

/* First tail page only */
@@ -139,12 +147,14 @@ struct page {
unsigned int compound_nr; /* 1 << compound_order */
};
struct { /* Second tail page of compound page */
+ unsigned long _t2_flags;
unsigned long _compound_pad_1; /* compound_head */
atomic_t hpage_pinned_refcount;
/* For both global and memcg */
struct list_head deferred_list;
};
struct { /* Page table pages */
+ unsigned long _pt_flags;
unsigned long _pt_pad_1; /* compound_head */
pgtable_t pmd_huge_pte; /* protected by page->ptl */
unsigned long _pt_pad_2; /* mapping */
@@ -159,6 +169,7 @@ struct page {
#endif
};
struct { /* ZONE_DEVICE pages */
+ unsigned long _zd_flags;
/** @pgmap: Points to the hosting device page map. */
struct dev_pagemap *pgmap;
void *zone_device_data;
@@ -174,8 +185,11 @@ struct page {
*/
};

- /** @rcu_head: You can use this to free a page by RCU. */
- struct rcu_head rcu_head;
+ struct {
+ unsigned long _rcu_flags;
+ /** @rcu_head: You can use this to free a page by RCU. */
+ struct rcu_head rcu_head;
+ };
};

union { /* This union is 4 bytes in size. */
--
2.30.2