[PATCH] mm: Skip the reserved bootmem for compaction

[Rong Qianfeng]

Reserved pages are basically non-lru pages. This kind of memory can't be
used as migration sources and targets, skip it can bring some performance
benefits.

Because some drivers may also use PG_reserved, we just set PB_migrate_skip
for those clustered reserved bootmem during memory initialization.

Signed-off-by: Rong Qianfeng <rongqianfeng@xxxxxxxx>
---
 include/linux/pageblock-flags.h | 13 +++++++++++
 mm/compaction.c                 | 40 +++++++++++++++++++++++++++++++++
 mm/mm_init.c                    | 14 ++++++++++++
 mm/page_alloc.c                 |  7 ++++++
 4 files changed, 74 insertions(+)

diff --git a/include/linux/pageblock-flags.h b/include/linux/pageblock-flags.h
index fc6b9c87cb0a..63c5b0c69c1a 100644
--- a/include/linux/pageblock-flags.h
+++ b/include/linux/pageblock-flags.h
@@ -86,6 +86,11 @@ void set_pfnblock_flags_mask(struct page *page,
 	set_pfnblock_flags_mask(page, (1 << PB_migrate_skip),	\
 			page_to_pfn(page),			\
 			(1 << PB_migrate_skip))
+
+extern void set_pageblock_skip_range(unsigned long start_pfn,
+				     unsigned long end_pfn);
+extern void clear_pageblock_skip_range(unsigned long start_pfn,
+				       unsigned long end_pfn);
 #else
 static inline bool get_pageblock_skip(struct page *page)
 {
@@ -97,6 +102,14 @@ static inline void clear_pageblock_skip(struct page *page)
 static inline void set_pageblock_skip(struct page *page)
 {
 }
+static inline void set_pageblock_skip_range(unsigned long start_pfn,
+					    unsigned long end_pfn)
+{
+}
+static inline void clear_pageblock_skip_range(unsigned long start_pfn,
+					      unsigned long end_pfn)
+{
+}
 #endif /* CONFIG_COMPACTION */
 
 #endif	/* PAGEBLOCK_FLAGS_H */
diff --git a/mm/compaction.c b/mm/compaction.c
index f2af4493a878..7861588b34f3 100644
--- a/mm/compaction.c
+++ b/mm/compaction.c
@@ -286,6 +286,46 @@ static unsigned long skip_offline_sections_reverse(unsigned long start_pfn)
 }
 #endif
 
+/*
+ * This function is currently used to set PB_migrate_skip for the reserved
+ * bootmem which can't be used as migration sources and targets(except CMA).
+ */
+void set_pageblock_skip_range(unsigned long start_pfn,
+			      unsigned long end_pfn)
+{
+	unsigned long pfn;
+
+	start_pfn = ALIGN(start_pfn, pageblock_nr_pages);
+	end_pfn = ALIGN_DOWN(end_pfn, pageblock_nr_pages);
+
+	for (pfn = start_pfn; pfn < end_pfn;
+				pfn += pageblock_nr_pages) {
+		if (pfn_valid(pfn)) {
+			struct page *page = pfn_to_page(pfn);
+
+			set_pageblock_skip(page);
+		}
+	}
+}
+
+void clear_pageblock_skip_range(unsigned long start_pfn,
+				unsigned long end_pfn)
+{
+	unsigned long pfn;
+
+	start_pfn = ALIGN_DOWN(start_pfn, pageblock_nr_pages);
+	end_pfn = ALIGN(end_pfn, pageblock_nr_pages);
+
+	for (pfn = start_pfn; pfn < end_pfn;
+				pfn += pageblock_nr_pages) {
+		if (pfn_valid(pfn)) {
+			struct page *page = pfn_to_page(pfn);
+
+			clear_pageblock_skip(page);
+		}
+	}
+}
+
 /*
  * Compound pages of >= pageblock_order should consistently be skipped until
  * released. It is always pointless to compact pages of such order (if they are
diff --git a/mm/mm_init.c b/mm/mm_init.c
index 4ba5607aaf19..8b7dc8e00bf1 100644
--- a/mm/mm_init.c
+++ b/mm/mm_init.c
@@ -768,6 +768,13 @@ void __meminit reserve_bootmem_region(phys_addr_t start,
 			__SetPageReserved(page);
 		}
 	}
+
+	/*
+	 * Set PB_migrate_skip for reserved region. for cma memory
+	 * and the memory released by free_reserved_area(), we will
+	 * clear PB_migrate_skip when they are initialized.
+	 */
+	set_pageblock_skip_range(start_pfn, end_pfn);
 }
 
 /* If zone is ZONE_MOVABLE but memory is mirrored, it is an overlapped init */
@@ -2236,6 +2243,13 @@ void __init init_cma_reserved_pageblock(struct page *page)
 		set_page_count(p, 0);
 	} while (++p, --i);
 
+	/*
+	 * We set the PB_migrate_skip in
+	 * reserve_bootmem_region() for cma
+	 * memory, clear it now.
+	 */
+	clear_pageblock_skip(page);
+
 	set_pageblock_migratetype(page, MIGRATE_CMA);
 	set_page_refcounted(page);
 	/* pages were reserved and not allocated */
diff --git a/mm/page_alloc.c b/mm/page_alloc.c
index b98f9bb28234..a7729dac0198 100644
--- a/mm/page_alloc.c
+++ b/mm/page_alloc.c
@@ -5887,6 +5887,13 @@ unsigned long free_reserved_area(void *start, void *end, int poison, const char
 	if (pages && s)
 		pr_info("Freeing %s memory: %ldK\n", s, K(pages));
 
+	/*
+	 * Clear PB_migrate_skip if the memory have released
+	 * to the buddy system.
+	 */
+	clear_pageblock_skip_range(page_to_pfn(virt_to_page(start)),
+				   page_to_pfn(virt_to_page(end)));
+
 	return pages;
 }
 
-- 
2.39.0