[PATCH v9 11/16] mm/mglru: Run aging when one type is drained

From: Barry Song (Xiaomi)

Date: Sat Nov 14 2026 - 15:09:46 EST


We may have remaining scan budget for anon or file but no folios
in populatable generations to scan. In this case, `scan_folios()`
may repeatedly scan that type without making progress, even when
the other type still has populatable folios.

Run aging when a type is drained to make more folios available for
reclaim. This also helps honor the scan budget distribution
determined by swappiness.

Move aging after eviction to detect a lack of scan progress and
avoid unnecessary aging when eviction can still make progress.

To avoid excessive aging, allow drained aging only once per
`try_to_shrink_lruvec()` invocation. Also skip drained aging if
another reclaimer has advanced `max_seq` during eviction.

At extreme swappiness values such as 1 and 200, reclaim initially
focuses on the preferred type. Rely on imbalanced aging once
`sc->priority` reaches `DEF_PRIORITY / 2`, rather than triggering
drained aging for the unpreferred type. This avoids unnecessary
aging and has shown better performance in testing.

If `evict_folio_lists()` returns zero, retry eviction once if
`max_seq` has advanced since the aging decision.

Signed-off-by: Barry Song (Xiaomi) <baohua@xxxxxxxxxx>
---
mm/vmscan.c | 49 +++++++++++++++++++++++++++++++++++++------------
1 file changed, 37 insertions(+), 12 deletions(-)

diff --git a/mm/vmscan.c b/mm/vmscan.c
index 1450542d709c..6d942f192e63 100644
--- a/mm/vmscan.c
+++ b/mm/vmscan.c
@@ -5175,7 +5175,8 @@ static bool should_abort_scan(struct lruvec *lruvec, struct scan_control *sc)
}

static long evict_folio_lists(unsigned long nr_to_scan[], struct lruvec *lruvec,
- struct scan_control *sc, int swappiness, unsigned long max_batch)
+ struct scan_control *sc, int swappiness,
+ unsigned long max_batch, bool *drained)
{
int type;
unsigned long batch, scanned, total_scanned = 0;
@@ -5191,6 +5192,8 @@ static long evict_folio_lists(unsigned long nr_to_scan[], struct lruvec *lruvec,
batch = min(nr_to_scan[type], max_batch);
scanned = evict_folios(batch, lruvec, sc, type, swappiness);

+ if (drained && !scanned)
+ *drained = true;
nr_to_scan[type] -= min(nr_to_scan[type], scanned);
total_scanned += scanned;
}
@@ -5206,9 +5209,15 @@ static long evict_folio_lists(unsigned long nr_to_scan[], struct lruvec *lruvec,
static bool try_to_shrink_lruvec(struct lruvec *lruvec, struct scan_control *sc)
{
unsigned long nr[ANON_AND_FILE];
- bool need_rotate = false, should_age = false;
+ bool need_rotate = false, should_age = false, retried = false;
int swappiness = get_swappiness(lruvec, sc);
struct mem_cgroup *memcg = lruvec_memcg(lruvec);
+ /*
+ * Avoid aging the lagging, unpreferred type at extreme swappiness;
+ * rely on imbalanced aging once priority reaches AGING_PRIORITY
+ */
+ bool allow_drained_age = sc->priority < DEF_PRIORITY &&
+ swappiness > MIN_SWAPPINESS + 1 && swappiness < MAX_SWAPPINESS;

/*
* Proactive reclaim initiated by userspace for anonymous memory only.
@@ -5226,22 +5235,17 @@ static bool try_to_shrink_lruvec(struct lruvec *lruvec, struct scan_control *sc)
lru_gen_prepare_scan(lruvec, sc, memcg, swappiness, nr);

while (nr[LRU_GEN_ANON] > 0 || nr[LRU_GEN_FILE] > 0) {
+ bool drained = false, do_drained_age;
DEFINE_MAX_SEQ(lruvec);
+ unsigned long new_seq;
+ long scanned;

if (mem_cgroup_below_min(sc->target_mem_cgroup, memcg)) {
need_rotate = true;
break;
}

- if (should_run_aging(lruvec, max_seq, sc, swappiness)) {
- trace_mm_mglru_aging(lruvec, sc, swappiness);
- if (try_to_inc_max_seq(lruvec, max_seq, swappiness, false))
- need_rotate = true;
- should_age = true;
- }
-
- if (!evict_folio_lists(nr, lruvec, sc, swappiness, MIN_LRU_BATCH))
- break;
+ scanned = evict_folio_lists(nr, lruvec, sc, swappiness, MIN_LRU_BATCH, &drained);

if (should_abort_scan(lruvec, sc))
break;
@@ -5253,6 +5257,27 @@ static bool try_to_shrink_lruvec(struct lruvec *lruvec, struct scan_control *sc)
if (root_reclaim(sc) && should_age)
break;

+ /*
+ * Allow drained aging only once per invocation and skip it if
+ * max_seq advanced concurrently during eviction
+ */
+ new_seq = READ_ONCE(lruvec->lrugen.max_seq);
+ do_drained_age = drained && allow_drained_age && max_seq == new_seq;
+ allow_drained_age &= !do_drained_age;
+ if (do_drained_age || should_run_aging(lruvec, new_seq, sc, swappiness)) {
+ trace_mm_mglru_aging(lruvec, sc, swappiness);
+ if (try_to_inc_max_seq(lruvec, new_seq, swappiness, false))
+ need_rotate = true;
+ should_age = true;
+ }
+
+ if (!scanned) {
+ /* Retry once if aging advanced max_seq */
+ if (retried || new_seq == READ_ONCE(lruvec->lrugen.max_seq))
+ break;
+ retried = true;
+ }
+
cond_resched();
}

@@ -5885,7 +5910,7 @@ static int run_eviction(struct lruvec *lruvec, unsigned long seq, struct scan_co
lru_gen_balance_scan(lruvec, swappiness,
nr_to_reclaim - sc->nr_reclaimed, nr_to_scan);

- if (!evict_folio_lists(nr_to_scan, lruvec, sc, swappiness, MAX_LRU_BATCH))
+ if (!evict_folio_lists(nr_to_scan, lruvec, sc, swappiness, MAX_LRU_BATCH, NULL))
return 0;

cond_resched();
--
2.39.3 (Apple Git-146)