[PATCH v2] mm/vmstat: add per-order allocation slow path statistics

[Daniil Tatianin]

Production incidents caused by bursts of high-order allocations all
entering direct compaction are currently hard to attribute from
/proc/vmstat: pgalloc_* has no order breakdown, and compact_stall does
not say which order stalled. Tracepoints can recover this on a single
machine, but they are impractical as an always-on fleet-wide monitoring
source, which is what is needed to correlate latency regressions with
allocation behavior after the fact.

Add per-order event counters to /proc/vmstat, covering only the
allocation slow path, so the page allocator fast path is not touched
at all:

 - pgalloc_slowpath_orderN: entries into __alloc_pages_slowpath(),
   counted once per allocation, before the restart loop
 - pgalloc_fail_orderN: allocations that returned NULL to the caller
   (including a successful allocation freed by memcg charge failure)
 - compact_stall_orderN / compact_success_orderN: per-order split of
   the existing direct compaction counters, order 0 is omitted since
   direct compaction is never entered for it

All new counters are purely additive: the existing keys are untouched
and compact_stall == sum of compact_stall_orderN.

alloc_pages_nolock() is deliberately not counted: it is opportunistic,
never enters the slow path, and its NULL returns are expected rather
than failures.

Counter names are generated for all MAX_PAGE_ORDER configurations that
are currently possible via ARCH_FORCE_MAX_ORDER. Orders 1..3 are
generated unconditionally, 4..13 are MAX_PAGE_ORDER dependent (e.g.
ppc32 with 256k pages forces 4 by default).

Sample output, taken on a small x86-64 test VM after a burst of THP
sized (order-9) allocations against deliberately fragmented physical
memory (only non-zero counters shown):

  $ grep -E "order[0-9]+" /proc/vmstat
  pgalloc_slowpath_order0 9104
  pgalloc_slowpath_order9 26
  pgalloc_slowpath_order10 5
  pgalloc_fail_order0 1
  pgalloc_fail_order9 25
  pgalloc_fail_order10 5
  compact_stall_order9 51

inspecting these we can see exactly what happened on the system in that
moment: the order-9 burst missed the fast path 26 times, stalled in
direct compaction 51 times and 25 of the requests failed outright,
against a background of ordinary order-0 slow path traffic (9104).

A per-order split of PGALLOC itself was proposed in 2017 but stalled
over fast path overhead concerns, restricting the counters to the slow
path avoids that overhead entirely while still capturing the
allocations that cause latency.

Link: https://lore.kernel.org/all/1499346271-15653-1-git-send-email-guro@xxxxxx/

Signed-off-by: Daniil Tatianin <d-tatianin@xxxxxxxxxxxxxx>
---
v1 -> v2:
 - handle MAX_PAGE_ORDER below 10: orders 4..10 now track MAX_PAGE_ORDER
   like 11..13 already did, so configs such as arm pxa_defconfig
   (MAX_PAGE_ORDER = 8) or ppc32 with 256K pages (4, the smallest any
   arch Kconfig currently sets) no longer overrun the enum ranges and
   trip the vmstat_text size assertion
 - quote sample /proc/vmstat output in the changelog

 include/linux/vm_event_item.h | 19 +++++++
 mm/page_alloc.c               |  7 +++
 mm/vmstat.c                   | 96 +++++++++++++++++++++++++++++++++++
 3 files changed, 122 insertions(+)

diff --git a/include/linux/vm_event_item.h b/include/linux/vm_event_item.h
index 03fe95f5a020..724561b7e800 100644
--- a/include/linux/vm_event_item.h
+++ b/include/linux/vm_event_item.h
@@ -175,6 +175,25 @@ enum vm_event_item { PGPGIN, PGPGOUT, PSWPIN, PSWPOUT,
 		KSTACK_REST,
 #endif
 #endif /* CONFIG_DEBUG_STACK_USAGE */
+		/*
+		 * Per-order allocation statistics: each *_FIRST..*_LAST range
+		 * is indexed by allocation order.
+		 */
+		PGALLOC_SLOWPATH_ORDER_FIRST,
+		PGALLOC_SLOWPATH_ORDER_LAST =
+			PGALLOC_SLOWPATH_ORDER_FIRST + MAX_PAGE_ORDER,
+		PGALLOC_FAIL_ORDER_FIRST,
+		PGALLOC_FAIL_ORDER_LAST =
+			PGALLOC_FAIL_ORDER_FIRST + MAX_PAGE_ORDER,
+#ifdef CONFIG_COMPACTION
+		/* Direct compaction is never entered for order 0 */
+		COMPACTSTALL_ORDER_FIRST,
+		COMPACTSTALL_ORDER_LAST =
+			COMPACTSTALL_ORDER_FIRST + MAX_PAGE_ORDER - 1,
+		COMPACTSUCCESS_ORDER_FIRST,
+		COMPACTSUCCESS_ORDER_LAST =
+			COMPACTSUCCESS_ORDER_FIRST + MAX_PAGE_ORDER - 1,
+#endif
 		NR_VM_EVENT_ITEMS
 };
 
diff --git a/mm/page_alloc.c b/mm/page_alloc.c
index ee902a468c2f..2a2b14f3b516 100644
--- a/mm/page_alloc.c
+++ b/mm/page_alloc.c
@@ -4169,6 +4169,7 @@ __alloc_pages_direct_compact(gfp_t gfp_mask, unsigned int order,
 	 * count a compaction stall
 	 */
 	count_vm_event(COMPACTSTALL);
+	count_vm_event(COMPACTSTALL_ORDER_FIRST + order - 1);
 
 	/* Prep a captured page if available */
 	if (page)
@@ -4184,6 +4185,7 @@ __alloc_pages_direct_compact(gfp_t gfp_mask, unsigned int order,
 		zone->compact_blockskip_flush = false;
 		compaction_defer_reset(zone, order, true);
 		count_vm_event(COMPACTSUCCESS);
+		count_vm_event(COMPACTSUCCESS_ORDER_FIRST + order - 1);
 		return page;
 	}
 
@@ -4757,6 +4759,8 @@ __alloc_pages_slowpath(gfp_t gfp_mask, unsigned int order,
 		WARN_ON_ONCE(current->flags & PF_MEMALLOC);
 	}
 
+	count_vm_event(PGALLOC_SLOWPATH_ORDER_FIRST + order);
+
 restart:
 	compaction_retries = 0;
 	no_progress_loops = 0;
@@ -5323,6 +5327,9 @@ struct page *__alloc_frozen_pages_noprof(gfp_t gfp, unsigned int order,
 		page = NULL;
 	}
 
+	if (unlikely(!page))
+		count_vm_event(PGALLOC_FAIL_ORDER_FIRST + order);
+
 	trace_mm_page_alloc(page, order, alloc_gfp, ac.migratetype);
 	kmsan_alloc_page(page, order, alloc_gfp);
 
diff --git a/mm/vmstat.c b/mm/vmstat.c
index f534972f517d..08f0ebc7ff43 100644
--- a/mm/vmstat.c
+++ b/mm/vmstat.c
@@ -1184,6 +1184,94 @@ int fragmentation_index(struct zone *zone, unsigned int order)
 	[xx##_MOVABLE] = yy "_movable",		\
 	TEXT_FOR_DEVICE(xx, yy)
 
+#if MAX_PAGE_ORDER >= 4
+#define TEXT_FOR_ORDER_4(xx, yy) [I((xx) + 4)] = yy "4",
+#else
+#define TEXT_FOR_ORDER_4(xx, yy)
+#endif
+
+#if MAX_PAGE_ORDER >= 5
+#define TEXT_FOR_ORDER_5(xx, yy) [I((xx) + 5)] = yy "5",
+#else
+#define TEXT_FOR_ORDER_5(xx, yy)
+#endif
+
+#if MAX_PAGE_ORDER >= 6
+#define TEXT_FOR_ORDER_6(xx, yy) [I((xx) + 6)] = yy "6",
+#else
+#define TEXT_FOR_ORDER_6(xx, yy)
+#endif
+
+#if MAX_PAGE_ORDER >= 7
+#define TEXT_FOR_ORDER_7(xx, yy) [I((xx) + 7)] = yy "7",
+#else
+#define TEXT_FOR_ORDER_7(xx, yy)
+#endif
+
+#if MAX_PAGE_ORDER >= 8
+#define TEXT_FOR_ORDER_8(xx, yy) [I((xx) + 8)] = yy "8",
+#else
+#define TEXT_FOR_ORDER_8(xx, yy)
+#endif
+
+#if MAX_PAGE_ORDER >= 9
+#define TEXT_FOR_ORDER_9(xx, yy) [I((xx) + 9)] = yy "9",
+#else
+#define TEXT_FOR_ORDER_9(xx, yy)
+#endif
+
+#if MAX_PAGE_ORDER >= 10
+#define TEXT_FOR_ORDER_10(xx, yy) [I((xx) + 10)] = yy "10",
+#else
+#define TEXT_FOR_ORDER_10(xx, yy)
+#endif
+
+#if MAX_PAGE_ORDER >= 11
+#define TEXT_FOR_ORDER_11(xx, yy) [I((xx) + 11)] = yy "11",
+#else
+#define TEXT_FOR_ORDER_11(xx, yy)
+#endif
+
+#if MAX_PAGE_ORDER >= 12
+#define TEXT_FOR_ORDER_12(xx, yy) [I((xx) + 12)] = yy "12",
+#else
+#define TEXT_FOR_ORDER_12(xx, yy)
+#endif
+
+#if MAX_PAGE_ORDER >= 13
+#define TEXT_FOR_ORDER_13(xx, yy) [I((xx) + 13)] = yy "13",
+#else
+#define TEXT_FOR_ORDER_13(xx, yy)
+#endif
+
+static_assert(MAX_PAGE_ORDER >= 4 && MAX_PAGE_ORDER <= 13,
+	      "the per-order counter names need updating for this MAX_PAGE_ORDER");
+
+/*
+ * (xx) + n must resolve to the vm_event_item for order n, so ranges that
+ * start at order 1 pass their *_ORDER_FIRST item minus one. Orders 1..3
+ * are generated unconditionally, 4..13 are MAX_PAGE_ORDER dependent (e.g.
+ * ppc32 with 256k pages forces 4 by default).
+ */
+#define TEXTS_FOR_NONZERO_ORDERS(xx, yy)	\
+	[I((xx) + 1)] = yy "1",			\
+	[I((xx) + 2)] = yy "2",			\
+	[I((xx) + 3)] = yy "3",			\
+	TEXT_FOR_ORDER_4(xx, yy)		\
+	TEXT_FOR_ORDER_5(xx, yy)		\
+	TEXT_FOR_ORDER_6(xx, yy)		\
+	TEXT_FOR_ORDER_7(xx, yy)		\
+	TEXT_FOR_ORDER_8(xx, yy)		\
+	TEXT_FOR_ORDER_9(xx, yy)		\
+	TEXT_FOR_ORDER_10(xx, yy)		\
+	TEXT_FOR_ORDER_11(xx, yy)		\
+	TEXT_FOR_ORDER_12(xx, yy)		\
+	TEXT_FOR_ORDER_13(xx, yy)
+
+#define TEXTS_FOR_ORDERS(xx, yy)		\
+	[I(xx)] = yy "0",			\
+	TEXTS_FOR_NONZERO_ORDERS(xx, yy)
+
 const char * const vmstat_text[] = {
 	/* enum zone_stat_item counters */
 #define I(x) (x)
@@ -1488,6 +1576,14 @@ const char * const vmstat_text[] = {
 #if THREAD_SIZE > 65536
 	[I(KSTACK_REST)]			= "kstack_rest",
 #endif
+#endif
+	TEXTS_FOR_ORDERS(PGALLOC_SLOWPATH_ORDER_FIRST, "pgalloc_slowpath_order")
+	TEXTS_FOR_ORDERS(PGALLOC_FAIL_ORDER_FIRST, "pgalloc_fail_order")
+#ifdef CONFIG_COMPACTION
+	TEXTS_FOR_NONZERO_ORDERS(COMPACTSTALL_ORDER_FIRST - 1,
+				 "compact_stall_order")
+	TEXTS_FOR_NONZERO_ORDERS(COMPACTSUCCESS_ORDER_FIRST - 1,
+				 "compact_success_order")
 #endif
 #undef I
 #endif /* CONFIG_VM_EVENT_COUNTERS */