[PATCH v3 4/4] mm/mglru: fix anon-only reclaim evicting file pages when swappiness=max

[Ridong]

From: Ridong Chen <chenridong@xxxxxxxxxx>

The previous patch fixed this issue for the traditional LRU. The same
problem exists in MGLRU [1]: when swappiness=max (SWAPPINESS_ANON_ONLY)
is set, reclaim is expected to evict anonymous pages exclusively, but
file pages can still be reclaimed when anonymous pages cannot be
reclaimed (e.g. no swap and no demotion target).

With SWAPPINESS_ANON_ONLY, get_type_to_scan() always returns
LRU_GEN_ANON and for_each_evictable_type() only iterates the anon type.
But if get_nr_to_scan() does not bail out, evict_folios() still runs and
isolate_folios() falls back to scanning file pages in the !scanned case:

  shrink_one
    try_to_shrink_lruvec
      get_swappiness   // returns SWAPPINESS_ANON_ONLY for swappiness=max
      get_nr_to_scan
      evict_folios
        isolate_folios // falls back to file type when !scanned

This wrongly evicts file pages, violating the anon-only semantics.

Fix it the same way as the traditional LRU: keep returning
SWAPPINESS_ANON_ONLY in get_swappiness(), and return 0 from
get_nr_to_scan() when SWAPPINESS_ANON_ONLY is set but anon pages cannot
be reclaimed. This protects file pages from being reclaimed, and as a
side effect avoids the useless scan work.

The test result:
Before fix:

  # cat /sys/kernel/mm/lru_gen/enabled
  0x0007
  # cat memory.stat
  anon 204800
  file 67108864
  ...
  pgsteal_proactive 0
  pgscan_proactive 0

  # echo "64M swappiness=max" > memory.reclaim
  # cat memory.stat
  anon 208896
  file 0
  ...
  pgsteal_proactive 16384
  pgscan_proactive 16384

After fix:

  # cat memory.stat
  anon 188416
  file 67215360
  kernel 1970176
  ...
  pgsteal_proactive 0
  pgscan_proactive 0

  # echo "64M swappiness=max" > memory.reclaim
  -bash: echo: write error: Resource temporarily unavailable
  # cat memory.stat
  anon 204800
  file 67215360
  ...
  pgsteal_proactive 0
  pgscan_proactive 0

[1] https://sashiko.dev/#/patchset/20260717113300.214717-1-ridong.chen@xxxxxxxxx

Fixes: 68a1436bde00 ("mm: add swappiness=max arg to memory.reclaim for only anon reclaim")
Acked-by: Qi Zheng <qi.zheng@xxxxxxxxx>
Signed-off-by: Ridong Chen <chenridong@xxxxxxxxxx>
---
 mm/vmscan.c | 19 ++++++++++++++++++-
 1 file changed, 18 insertions(+), 1 deletion(-)

diff --git a/mm/vmscan.c b/mm/vmscan.c
index 859655fcf60d..20cb93fd3da8 100644
--- a/mm/vmscan.c
+++ b/mm/vmscan.c
@@ -2708,6 +2708,10 @@ static int get_swappiness(struct lruvec *lruvec, struct scan_control *sc)
 {
 	struct mem_cgroup *memcg = lruvec_memcg(lruvec);
 	struct pglist_data *pgdat = lruvec_pgdat(lruvec);
+	int swappiness = sc_swappiness(sc, memcg);
+
+	if (swappiness == SWAPPINESS_ANON_ONLY)
+		return swappiness;
 
 	if (!sc->may_swap)
 		return 0;
@@ -2716,7 +2720,7 @@ static int get_swappiness(struct lruvec *lruvec, struct scan_control *sc)
 	    mem_cgroup_get_nr_swap_pages(memcg) < MIN_LRU_BATCH)
 		return 0;
 
-	return sc_swappiness(sc, memcg);
+	return swappiness;
 }
 
 static int get_nr_gens(struct lruvec *lruvec, int type)
@@ -4921,6 +4925,19 @@ static long get_nr_to_scan(struct lruvec *lruvec, struct scan_control *sc,
 			   struct mem_cgroup *memcg, int swappiness)
 {
 	unsigned long nr_to_scan, evictable;
+	struct pglist_data *pgdat = lruvec_pgdat(lruvec);
+	/*
+	 * Proactive reclaim initiated by userspace for anonymous memory only.
+	 * SWAPPINESS_ANON_ONLY is set only on the proactive reclaim path, so
+	 * warn if it shows up elsewhere. When anon cannot be reclaimed (e.g.
+	 * no swap), return 0 instead of falling through and evicting file
+	 * pages, which would violate the anon-only semantics.
+	 */
+	if (swappiness == SWAPPINESS_ANON_ONLY &&
+	    !can_reclaim_anon_pages(memcg, pgdat->node_id, sc)) {
+		WARN_ON_ONCE(!sc->proactive);
+		return 0;
+	}
 
 	evictable = lruvec_evictable_size(lruvec, swappiness);
 
-- 
2.34.1