[RFC PATCH v4 14/16] mm/mglru: run aging when pages are severely imbalanced across gens

[Barry Song (Xiaomi)]

From: lyugaofei <lyugaofei@xxxxxxxxxx>

This partially restores the reclaim behavior introduced in Yu
Zhao's initial MGLRU commit, ac35a4902370 ("mm: multi-gen LRU:
minimal implementation"):
	/*
	 * It's also ideal to spread pages out evenly, i.e., 1/(MIN_NR_GENS+1)
	 * of the total number of pages for each generation. A reasonable range
	 * for this average portion is [1/MIN_NR_GENS, 1/(MIN_NR_GENS+2)]. The
	 * aging cares about the upper bound of hot pages, while the eviction
	 * cares about the lower bound of cold pages.
	 */
	if (young * MIN_NR_GENS > total)
		return true;
	if (old * (MIN_NR_GENS + 2) < total)
		return true;

But with a stricter condition: the younger generations must
contain at least MAX_NR_GENS times as many folios as the older
generations.
We also consider the cost of inc_min_seq(). If the oldest generation
of the other type has fallen significantly behind, pulling those
folios from the oldest generation to the second oldest generation
can be very expensive. In this case, skip imbalance aging unless
extreme swappiness is in use.

Signed-off-by: lyugaofei <lyugaofei@xxxxxxxxxx>
Co-developed-by: Barry Song (Xiaomi) <baohua@xxxxxxxxxx>
Signed-off-by: Barry Song (Xiaomi) <baohua@xxxxxxxxxx>
---
 mm/vmscan.c | 57 ++++++++++++++++++++++++++++++++++++++++++++++-------
 1 file changed, 50 insertions(+), 7 deletions(-)

diff --git a/mm/vmscan.c b/mm/vmscan.c
index a276560bc66b..d6fac5b91ac1 100644
--- a/mm/vmscan.c
+++ b/mm/vmscan.c
@@ -4219,20 +4219,28 @@ static void set_initial_priority(struct pglist_data *pgdat, struct scan_control
 	sc->priority = clamp(priority, DEF_PRIORITY / 2, DEF_PRIORITY);
 }
 
+static inline unsigned long lruvec_gen_size(struct lru_gen_folio *lrugen,
+		int type, unsigned long seq)
+{
+	int gen = lru_gen_from_seq(seq);
+	unsigned long size = 0;
+
+	for (int zone = 0; zone < MAX_NR_ZONES; zone++)
+		size += max(READ_ONCE(lrugen->nr_pages[gen][type][zone]), 0L);
+	return size;
+}
+
 static unsigned long lruvec_evictable_size(struct lruvec *lruvec, int swappiness)
 {
-	int gen, type, zone;
+	int type;
 	unsigned long seq, total = 0;
 	struct lru_gen_folio *lrugen = &lruvec->lrugen;
 	DEFINE_MAX_SEQ(lruvec);
 	DEFINE_MIN_SEQ(lruvec);
 
 	for_each_evictable_type(type, swappiness) {
-		for (seq = min_seq[type]; seq <= max_seq; seq++) {
-			gen = lru_gen_from_seq(seq);
-			for (zone = 0; zone < MAX_NR_ZONES; zone++)
-				total += max(READ_ONCE(lrugen->nr_pages[gen][type][zone]), 0L);
-		}
+		for (seq = min_seq[type]; seq <= max_seq; seq++)
+			total += lruvec_gen_size(lrugen, type, seq);
 	}
 
 	return total;
@@ -5092,6 +5100,37 @@ static int evict_folios(unsigned long nr_to_scan, struct lruvec *lruvec,
 	return scanned;
 }
 
+static bool lru_gen_imbalanced(struct lruvec *lruvec, unsigned long max_seq,
+		struct scan_control *sc, int type, int swappiness)
+{
+	struct lru_gen_folio *lrugen = &lruvec->lrugen;
+	unsigned long young = 0, old = 0, lag = 0;
+	DEFINE_MIN_SEQ(lruvec);
+
+	/* we still have enough generations to reclaim */
+	if (min_seq[type] + MIN_NR_GENS < max_seq)
+		return false;
+
+	/*
+	 * Trigger aging if the preferred type is running low on reclaimable
+	 * folios, provided the generation lag of the other type remains small
+	 * enough that inc_min_seq() introduces negligible overhead
+	 */
+	for (unsigned long seq = min_seq[type]; seq <= max_seq; seq++) {
+		unsigned long size = lruvec_gen_size(lrugen, type, seq);
+
+		if (seq + MIN_NR_GENS > max_seq)
+			young += size;
+		else
+			old += size;
+	}
+	if (min_seq[!type] + MAX_NR_GENS == max_seq + 1)
+		lag += lruvec_gen_size(lrugen, !type, min_seq[!type]);
+
+	return young > old * MAX_NR_GENS && (lag < MAX_LRU_BATCH ||
+	       (is_extreme_swappiness(swappiness) && sc->priority > 2));
+}
+
 static bool should_run_aging(struct lruvec *lruvec, unsigned long max_seq,
 			     struct scan_control *sc, int swappiness)
 {
@@ -5110,7 +5149,11 @@ static bool should_run_aging(struct lruvec *lruvec, unsigned long max_seq,
 		return false;
 
 	/* better to run aging even though eviction is still possible */
-	return evictable_min_seq(min_seq, swappiness) + MIN_NR_GENS == max_seq;
+	if (evictable_min_seq(min_seq, swappiness) + MIN_NR_GENS == max_seq)
+		return true;
+
+	/* Run aging if the preferred type is severely imbalanced across gens */
+	return lru_gen_imbalanced(lruvec, max_seq, sc, type, swappiness);
 }
 
 static long get_nr_to_scan(struct lruvec *lruvec, struct scan_control *sc,
-- 
2.34.1