[PATCH mm-unstable v4 2/2] mm/vmscan: apply too_many_isolated() throttling to MGLRU eviction

[Hui Zhu]

From: Hui Zhu <zhuhui@xxxxxxxxxx>

The legacy path throttles direct reclaim in shrink_inactive_list() when
too many isolated folios pile up, but MGLRU's evict_folios() isolates
folios without this check, which can lead to unnecessary swapping,
thrashing and OOM.

With the NR_ISOLATED counters now updated in evict_folios(), add
throttle_evictable_types() and call it from evict_folios(), before the
lruvec lock is taken since throttling sleeps. The legacy path is left
untouched.

Unlike the legacy path, where the LRU list to isolate from is known
before isolation, isolate_folios() picks the type to scan from the
refault feedback and may fall back to the other one. Therefore, instead
of throttling on a single type, throttle_evictable_types() collects the
evictable types that do not have too many isolated folios, and only
sleeps when all of them do: like the legacy path, it waits once for
concurrent reclaimers to put their isolated folios back, and gives up
if that makes no progress.

The resulting mask of the types that are not over-isolated is passed to
isolate_folios(), which restricts both its initial choice and its
fallback to it. This way a type that is merely over-isolated never
blocks the reclaim of the other type, and isolation never lands on a
throttled type.

If a fatal signal is pending, fake reclaim progress the same way the
legacy path does, so the dying task exits reclaim quickly instead of
being held in the throttle.

Signed-off-by: Hui Zhu <zhuhui@xxxxxxxxxx>
---
 mm/vmscan.c | 85 +++++++++++++++++++++++++++++++++++++++++++++++++----
 1 file changed, 80 insertions(+), 5 deletions(-)

diff --git a/mm/vmscan.c b/mm/vmscan.c
index 98226bb021f3..693dc91a1695 100644
--- a/mm/vmscan.c
+++ b/mm/vmscan.c
@@ -4833,15 +4833,73 @@ static int get_type_to_scan(struct lruvec *lruvec, int swappiness)
 	return positive_ctrl_err(&sp, &pv);
 }
 
+/*
+ * Unlike the legacy path, where the LRU list to isolate from is known
+ * before isolation, isolate_folios() picks the type from the refault
+ * feedback and may fall back to the other one. Therefore, instead of
+ * throttling on a single type, collect the evictable types that do not
+ * have too many isolated folios, and only sleep when all of them do.
+ *
+ * Returns the mask of the types isolate_folios() may isolate from, or
+ * 0 if reclaim should stop. Also sets @fatal to tell the caller that
+ * the task received a fatal signal while waiting.
+ */
+static unsigned int throttle_evictable_types(struct pglist_data *pgdat,
+					     int swappiness,
+					     struct scan_control *sc,
+					     bool *fatal)
+{
+	unsigned int allowed;
+	bool stalled = false;
+	int i;
+
+	*fatal = false;
+
+	for (;;) {
+		allowed = 0;
+		for_each_evictable_type(i, swappiness) {
+			if (!too_many_isolated(pgdat, i, sc))
+				allowed |= BIT(i);
+		}
+
+		if (allowed)
+			return allowed;
+
+		/*
+		 * All evictable types are over-isolated. Like the legacy
+		 * path, wait once for concurrent reclaimers to put their
+		 * isolated folios back; give up if that makes no progress.
+		 */
+		if (stalled)
+			return 0;
+
+		stalled = true;
+		reclaim_throttle(pgdat, VMSCAN_THROTTLE_ISOLATED);
+
+		/* We are about to die and free our memory. Return now. */
+		if (fatal_signal_pending(current)) {
+			*fatal = true;
+			return 0;
+		}
+	}
+}
+
 static int isolate_folios(unsigned long nr_to_scan, struct lruvec *lruvec,
 			  struct scan_control *sc, int swappiness,
-			  struct list_head *list, int *isolated,
-			  int *isolate_type, int *isolate_scanned)
+			  unsigned int allowed, struct list_head *list,
+			  int *isolated, int *isolate_type, int *isolate_scanned)
 {
 	int i;
 	int total_scanned = 0;
 	int type = get_type_to_scan(lruvec, swappiness);
 
+	/*
+	 * The preferred type may have been excluded by
+	 * throttle_evictable_types(); start from the other one.
+	 */
+	if (!(allowed & BIT(type)))
+		type = !type;
+
 	for_each_evictable_type(i, swappiness) {
 		int scanned;
 		int tier = get_tier_idx(lruvec, type);
@@ -4858,9 +4916,10 @@ static int isolate_folios(unsigned long nr_to_scan, struct lruvec *lruvec,
 		/*
 		 * If scanned > 0 and isolated == 0, avoid falling back to the
 		 * other type, as this type remains sufficient. Falling back
-		 * too readily can disrupt the positive_ctrl_err() bias.
+		 * too readily can disrupt the positive_ctrl_err() bias. Only
+		 * fall back to a type that is not throttled.
 		 */
-		if (!scanned)
+		if (!scanned && (allowed & BIT(!type)))
 			type = !type;
 	}
 
@@ -4883,13 +4942,29 @@ static int evict_folios(unsigned long nr_to_scan, struct lruvec *lruvec,
 	bool skip_retry = false;
 	struct mem_cgroup *memcg = lruvec_memcg(lruvec);
 	struct pglist_data *pgdat = lruvec_pgdat(lruvec);
+	unsigned int allowed;
+	bool fatal;
+
+	allowed = throttle_evictable_types(pgdat, swappiness, sc, &fatal);
+	if (!allowed) {
+		/*
+		 * We are about to die and free our memory. Like the legacy
+		 * path, pretend some pages were reclaimed so reclaim
+		 * unwinds quickly instead of looping back into the
+		 * throttle.
+		 */
+		if (fatal)
+			sc->nr_reclaimed += SWAP_CLUSTER_MAX;
+
+		return 0;
+	}
 
 	lruvec_lock_irq(lruvec);
 
 	/* In case folio deletion left empty old gens, flush them */
 	try_to_inc_min_seq(lruvec, swappiness);
 
-	scanned = isolate_folios(nr_to_scan, lruvec, sc, swappiness,
+	scanned = isolate_folios(nr_to_scan, lruvec, sc, swappiness, allowed,
 				 &list, &isolated, &type, &type_scanned);
 	nr_isolated = isolated;
 	if (nr_isolated)
-- 
2.53.0