[RFC v2 1/3] mm/lru_gen: add AGING counter and proactive aging helper

[Zicheng Wang]

Add the two pieces the rest of the series builds on: an AGING node
stat counter (the observation an aging policy pairs with
workingset_refault to spot over-aging) and a reusable helper,
lru_gen_age_memcg(), that ages a memcg by advancing max_seq without
reclaim (the control primitive memory.aging wraps).

The counter counts every pass in inc_max_seq(), regardless of caller.
The helper mirrors lru_gen_seq_write()'s reclaim_state /
memalloc_noreclaim / mm_walk setup; swappiness comes from
get_swappiness() so anon is not rotated without swap, and a racing
lruvec is skipped (-EAGAIN) without blocking other nodes.

Gated by CONFIG_LRU_GEN.  No user-visible change yet.

Signed-off-by: Zicheng Wang <wangzicheng@xxxxxxxxx>
---
 include/linux/mmzone.h |  3 ++
 mm/internal.h          |  4 +++
 mm/memcontrol.c        |  9 +++++
 mm/vmscan.c            | 80 ++++++++++++++++++++++++++++++++++++++++++
 mm/vmstat.c            |  3 ++
 5 files changed, 99 insertions(+)

diff --git a/include/linux/mmzone.h b/include/linux/mmzone.h
index 9adb2ad21da5..ab5137c272ca 100644
--- a/include/linux/mmzone.h
+++ b/include/linux/mmzone.h
@@ -323,6 +323,9 @@ enum node_stat_item {
 	PGSCAN_ANON,
 	PGSCAN_FILE,
 	PGREFILL,
+#ifdef CONFIG_LRU_GEN
+	AGING,
+#endif
 #ifdef CONFIG_HUGETLB_PAGE
 	NR_HUGETLB,
 #endif
diff --git a/mm/internal.h b/mm/internal.h
index 5a2ddcf68e0b..96add6cd4627 100644
--- a/mm/internal.h
+++ b/mm/internal.h
@@ -634,6 +634,10 @@ extern void reclaim_throttle(pg_data_t *pgdat, enum vmscan_throttle_state reason
 int user_proactive_reclaim(char *buf,
 			   struct mem_cgroup *memcg, pg_data_t *pgdat);
 
+#ifdef CONFIG_LRU_GEN
+int lru_gen_age_memcg(struct mem_cgroup *memcg, unsigned long nr_gens);
+#endif
+
 /*
  * in mm/rmap.c:
  */
diff --git a/mm/memcontrol.c b/mm/memcontrol.c
index d978e18b9b2d..e42f97e004b6 100644
--- a/mm/memcontrol.c
+++ b/mm/memcontrol.c
@@ -420,6 +420,9 @@ static const unsigned int memcg_node_stat_items[] = {
 	PGSCAN_ANON,
 	PGSCAN_FILE,
 	PGREFILL,
+#ifdef CONFIG_LRU_GEN
+	AGING,
+#endif
 #ifdef CONFIG_HUGETLB_PAGE
 	NR_HUGETLB,
 #endif
@@ -1606,6 +1609,9 @@ static const struct memory_stat memory_stats[] = {
 #ifdef CONFIG_NUMA_BALANCING
 	{ "pgpromote_success",		PGPROMOTE_SUCCESS	},
 #endif
+#ifdef CONFIG_LRU_GEN
+	{ "aging",		AGING	},
+#endif
 };
 
 /* The actual unit of the state item, not the same as the output unit */
@@ -1655,6 +1661,9 @@ static int memcg_page_state_output_unit(int item)
 	case PGSCAN_KHUGEPAGED:
 	case PGSCAN_PROACTIVE:
 	case PGREFILL:
+#ifdef CONFIG_LRU_GEN
+	case AGING:
+#endif
 #ifdef CONFIG_NUMA_BALANCING
 	case PGPROMOTE_SUCCESS:
 #endif
diff --git a/mm/vmscan.c b/mm/vmscan.c
index b3e555561417..805e29c499c8 100644
--- a/mm/vmscan.c
+++ b/mm/vmscan.c
@@ -3987,6 +3987,9 @@ static bool inc_max_seq(struct lruvec *lruvec, unsigned long seq, int swappiness
 	WRITE_ONCE(lrugen->timestamps[next], jiffies);
 	/* make sure preceding modifications appear */
 	smp_store_release(&lrugen->max_seq, lrugen->max_seq + 1);
+
+	/* Count every aging pass for over-aging diagnosis. */
+	mod_lruvec_state(lruvec, AGING, 1);
 unlock:
 	lruvec_unlock_irq(lruvec);
 
@@ -5855,6 +5858,83 @@ static int __init init_lru_gen(void)
 };
 late_initcall(init_lru_gen);
 
+/**
+ * lru_gen_age_memcg - Proactively age a memcg by N generations
+ * @memcg: target memcg
+ * @nr_gens: number of generations to age (1..MAX_NR_GENS)
+ *
+ * Advances max_seq without page reclaim on all nodes of @memcg, using
+ * the same lock/walk protocol as lru_gen_seq_write().  Swappiness comes
+ * from get_swappiness() so anon rotation is skipped without swap.
+ * to the next node.
+ *
+ * Return: 0 on success, -EINVAL on bad input, -EAGAIN if a lruvec did
+ * not advance, -ENOMEM on mm_walk failure, -EINTR on signal.
+ */
+int lru_gen_age_memcg(struct mem_cgroup *memcg, unsigned long nr_gens)
+{
+	struct scan_control sc = {
+		.may_writepage = true,
+		.may_unmap = true,
+		.may_swap = true,
+		.reclaim_idx = MAX_NR_ZONES - 1,
+		.gfp_mask = GFP_KERNEL,
+		.proactive = true,
+	};
+	int nid, swappiness;
+	unsigned long i;
+	int ret = 0;
+	unsigned int flags;
+	struct blk_plug plug;
+	struct lruvec *lruvec;
+
+	if (!memcg)
+		return -EINVAL;
+
+	if (nr_gens == 0 || nr_gens > MAX_NR_GENS)
+		return -EINVAL;
+
+	set_task_reclaim_state(current, &sc.reclaim_state);
+	flags = memalloc_noreclaim_save();
+	blk_start_plug(&plug);
+	if (!set_mm_walk(NULL, true)) {
+		ret = -ENOMEM;
+		goto done;
+	}
+
+	for_each_node_state(nid, N_MEMORY) {
+		if (signal_pending(current)) {
+			ret = -EINTR;
+			goto done;
+		}
+
+		lruvec = get_lruvec(memcg, nid);
+		if (!lruvec)
+			continue;
+
+		swappiness = get_swappiness(lruvec, &sc);
+
+		for (i = 0; i < nr_gens; i++) {
+			DEFINE_MAX_SEQ(lruvec);
+
+			if (!try_to_inc_max_seq(lruvec, max_seq,
+						swappiness, false)) {
+				ret = -EAGAIN;
+				break;
+			}
+		}
+
+		cond_resched();
+	}
+
+done:
+	clear_mm_walk();
+	blk_finish_plug(&plug);
+	memalloc_noreclaim_restore(flags);
+	set_task_reclaim_state(current, NULL);
+	return ret;
+}
+
 #else /* !CONFIG_LRU_GEN */
 
 static void lru_gen_age_node(struct pglist_data *pgdat, struct scan_control *sc)
diff --git a/mm/vmstat.c b/mm/vmstat.c
index f534972f517d..78ea093428c7 100644
--- a/mm/vmstat.c
+++ b/mm/vmstat.c
@@ -1290,6 +1290,9 @@ const char * const vmstat_text[] = {
 	[I(PGSCAN_ANON)]			= "pgscan_anon",
 	[I(PGSCAN_FILE)]			= "pgscan_file",
 	[I(PGREFILL)]				= "pgrefill",
+#ifdef CONFIG_LRU_GEN
+	[I(AGING)]			= "aging",
+#endif
 #ifdef CONFIG_HUGETLB_PAGE
 	[I(NR_HUGETLB)]				= "nr_hugetlb",
 #endif
-- 
2.25.1