[PATCH 8/9] x86/mm/pat: move the collapse of large pages out of cpa_flush()
From: Mike Rapoport (Microsoft)
Date: Thu Nov 12 2026 - 11:42:33 EST
Collapsing of large pages was first written to reduce the fragmentation
of the direct map, and cpa_flush() was the best place to hook it
into [1].
When the collapse was reposted for set_memory_rox() only, it stayed in
cpa_flush(), although it has nothing to do with the TLB flush.
Every early exit in cpa_flush() has to jump to the collapse.
Collapse the large pages in change_page_attr_set_clr() after the flush.
[1] https://lore.kernel.org/all/20200416213229.19174-1-kirill.shutemov@xxxxxxxxxxxxxxx
Signed-off-by: Mike Rapoport (Microsoft) <rppt@xxxxxxxxxx>
---
arch/x86/mm/pat/set_memory.c | 38 +++++++++++++++++---------------------
1 file changed, 17 insertions(+), 21 deletions(-)
diff --git a/arch/x86/mm/pat/set_memory.c b/arch/x86/mm/pat/set_memory.c
index 94b661c5f7885..80b5a2faefd7c 100644
--- a/arch/x86/mm/pat/set_memory.c
+++ b/arch/x86/mm/pat/set_memory.c
@@ -407,7 +407,7 @@ static void __cpa_flush_tlb(void *data)
static int collapse_large_pages(unsigned long addr, struct list_head *pgtables);
-static void __cpa_collapse_large_pages(struct cpa_data *cpa)
+static void cpa_collapse_large_pages(struct cpa_data *cpa)
{
unsigned long start, addr, end;
struct ptdesc *ptdesc, *tmp;
@@ -415,8 +415,6 @@ static void __cpa_collapse_large_pages(struct cpa_data *cpa)
int collapsed = 0;
int i;
- guard(spinlock)(&cpa_lock);
-
if (cpa->flags & CPA_PAGES_ARRAY) {
for (i = 0; i < cpa->numpages; i++)
collapsed += collapse_large_pages(__cpa_addr(cpa, i),
@@ -449,18 +447,6 @@ static void __cpa_collapse_large_pages(struct cpa_data *cpa)
}
}
-static void cpa_collapse_large_pages(struct cpa_data *cpa)
-{
- /*
- * Take the mmap write lock on init_mm to:
- * - Avoid a use-after-free if raced by ptdump (which takes its own
- * write lock on init_mm).
- * - Serialise concurrent CPA walkers.
- */
- scoped_guard(mmap_write_lock, &init_mm)
- __cpa_collapse_large_pages(cpa);
-}
-
static void cpa_flush(struct cpa_data *cpa, int cache)
{
unsigned int i;
@@ -469,7 +455,7 @@ static void cpa_flush(struct cpa_data *cpa, int cache)
if (cache && !cpu_feature_enabled(X86_FEATURE_CLFLUSH)) {
cpa_flush_all(cache);
- goto collapse_large_pages;
+ return;
}
if (cpa->force_flush_all || cpa->numpages > tlb_single_page_flush_ceiling)
@@ -478,7 +464,7 @@ static void cpa_flush(struct cpa_data *cpa, int cache)
on_each_cpu(__cpa_flush_tlb, cpa, 1);
if (!cache)
- goto collapse_large_pages;
+ return;
mb();
for (i = 0; i < cpa->numpages; i++) {
@@ -494,10 +480,6 @@ static void cpa_flush(struct cpa_data *cpa, int cache)
clflush_cache_range_opt((void *)fix_addr(addr), PAGE_SIZE);
}
mb();
-
-collapse_large_pages:
- if (cpa->flags & CPA_COLLAPSE)
- cpa_collapse_large_pages(cpa);
}
static bool overlaps(unsigned long r1_start, unsigned long r1_end,
@@ -2176,6 +2158,20 @@ static int change_page_attr_set_clr(unsigned long addr, int numpages,
}
cpa_flush(&cpa, cache);
+
+ if (!(cpa.flags & CPA_COLLAPSE))
+ goto out;
+
+ /*
+ * Take the mmap write lock on init_mm to:
+ * - Avoid a use-after-free if raced by ptdump (which takes its own
+ * write lock on init_mm).
+ * - Serialise concurrent CPA walkers.
+ */
+ scoped_guard(mmap_write_lock, &init_mm) {
+ guard(spinlock)(&cpa_lock);
+ cpa_collapse_large_pages(&cpa);
+ }
out:
return ret;
}
--
2.53.0