[PATCH v5 13/19] mm, swap: add xswap grow trigger on cluster allocation

From: Baoquan He

Date: Mon Nov 09 2026 - 01:00:52 EST


From: Baoquan He <hebaoquan@xxxxxxxxxx>

Grow the mapped xswap range before it runs out. When usage reaches 85%
of the mapped range, map more clusters from cluster_alloc_swap_entry().

Grow to about 65% usage. This leaves free space in the mapped range
before the next grow at 85%, so one grow is not immediately followed by
another.

Mapping pages can sleep. Grow before the allocation that needs them,
not after it fails. xswap_grow() drops the per-CPU cluster lock around
xswap_map_clusters(), because mapping can sleep and the lock only
protects the per-CPU cache. xswap is always SWP_SOLIDSTATE, so this path
does not use global_cluster_lock.

Link the new clusters even if swapoff cleared SWP_WRITEOK while mapping.
The caller keeps the device alive, and the allocation path checks the
flag before taking a cluster. If try_to_unuse() then fails, the
reinserted device needs these clusters; otherwise they go away with the
device.

After a failed scan, retry the free list once. A concurrent free or
grow may have added clusters.

Signed-off-by: Baoquan He <hebaoquan@xxxxxxxxxx>
---
mm/swapfile.c | 100 ++++++++++++++++++++++++++++++++++++++++++++++++++
1 file changed, 100 insertions(+)

diff --git a/mm/swapfile.c b/mm/swapfile.c
index 7b1d16226b7b..ba730fa19194 100644
--- a/mm/swapfile.c
+++ b/mm/swapfile.c
@@ -80,7 +80,18 @@
max_t(unsigned long, \
((XSWAP_UNMAP_BATCH_PAGES * PAGE_SIZE) / \
sizeof(struct swap_cluster_info)), 16)
+/*
+ * Grow thresholds, as a percentage of the mapped range in use. The target
+ * sits above the point where the shrink starts looking (50%) and below the
+ * next grow trigger (85%), so a grow cannot leave the range in a state the
+ * shrink would undo.
+ */
+#define XSWAP_GROW_UNTIL 65
+#define XSWAP_GROW_WHEN 85
+#define XSWAP_GROW_CHUNKS 4

+static bool xswap_should_grow(struct swap_info_struct *si);
+static unsigned long xswap_grow(struct swap_info_struct *si);
static int xswap_map_clusters(struct swap_info_struct *si,
unsigned long start_idx, unsigned long nr);
static void xswap_unmap_clusters(struct swap_info_struct *si,
@@ -1359,6 +1370,18 @@ static unsigned long cluster_alloc_swap_entry(struct swap_info_struct *si,
if (order && !(si->flags & SWP_BLKDEV))
return 0;

+#ifdef CONFIG_XSWAP
+ /*
+ * Top the range up early. Every path below leaves through `done`,
+ * so this has to come first; mapping pages can sleep, and doing it
+ * from the allocation that would otherwise fail leaves nothing to
+ * fall back on.
+ */
+ if ((si->flags & SWP_XSWAP) && (si->flags & SWP_WRITEOK) &&
+ xswap_should_grow(si))
+ xswap_grow(si);
+#endif
+
if (!(si->flags & SWP_SOLIDSTATE)) {
/* Serialize HDD SWAP allocation for each device. */
spin_lock(&si->global_cluster_lock);
@@ -1435,6 +1458,12 @@ static unsigned long cluster_alloc_swap_entry(struct swap_info_struct *si,
if (found)
goto done;
}
+
+#ifdef CONFIG_XSWAP
+ /* A concurrent free or grow may have added clusters; retry once. */
+ if (!found && (si->flags & SWP_XSWAP))
+ found = alloc_swap_scan_list(si, &si->free_clusters, folio, false);
+#endif
done:
if (!(si->flags & SWP_SOLIDSTATE))
spin_unlock(&si->global_cluster_lock);
@@ -4168,6 +4197,77 @@ static void xswap_unmap_range(struct swap_info_struct *si,
__free_page(pages[i]);
}

+static bool xswap_should_grow(struct swap_info_struct *si)
+{
+ unsigned long mapped = READ_ONCE(si->nr_clusters_mapped);
+
+ if (mapped >= READ_ONCE(si->nr_clusters_max))
+ return false;
+
+ return swap_usage_in_pages(si) * 100 >=
+ mapped * SWAPFILE_CLUSTER * XSWAP_GROW_WHEN;
+}
+
+/*
+ * Map more of the cluster_info array, up to GROW_UNTIL in use. The caller
+ * holds percpu_swap_cluster.lock, which this drops while it sleeps.
+ */
+static unsigned long xswap_grow(struct swap_info_struct *si)
+{
+ unsigned long mapped = READ_ONCE(si->nr_clusters_mapped);
+ unsigned long nr_new, want, i;
+ int ret;
+
+ /* The unmap below is paired with the caller's lock. */
+ lockdep_assert_held(&this_cpu_ptr(&percpu_swap_cluster)->lock);
+
+ want = DIV_ROUND_UP(swap_usage_in_pages(si) * 100,
+ XSWAP_GROW_UNTIL * SWAPFILE_CLUSTER);
+ if (want <= mapped)
+ return 0;
+
+ nr_new = rounddown(want - mapped, XSWAP_GROW_CLUSTERS);
+ if (!nr_new)
+ nr_new = XSWAP_GROW_CLUSTERS;
+ if (nr_new > XSWAP_GROW_CHUNKS * XSWAP_GROW_CLUSTERS)
+ nr_new = XSWAP_GROW_CHUNKS * XSWAP_GROW_CLUSTERS;
+ nr_new = min(nr_new, READ_ONCE(si->nr_clusters_max) - mapped);
+ if (!nr_new)
+ return 0;
+
+ /*
+ * Mapping pages can sleep. The lock only guards the per-cpu
+ * cluster cache, which this path does not touch.
+ */
+ local_unlock(&percpu_swap_cluster.lock);
+ ret = xswap_map_clusters(si, mapped, nr_new);
+ local_lock(&percpu_swap_cluster.lock);
+ if (ret)
+ return 0;
+
+ /*
+ * Link the clusters even if swapoff cleared SWP_WRITEOK while we
+ * mapped. The caller keeps the device alive, and nothing takes a
+ * cluster while WRITEOK is clear. If try_to_unuse() fails, the
+ * reinserted device needs them.
+ */
+ for (i = mapped; i < mapped + nr_new; i++) {
+ struct swap_cluster_info *ci = &si->cluster_info[i];
+
+ /*
+ * A concurrent grower may have taken these already;
+ * only add the off-list ones.
+ */
+ spin_lock(&ci->lock);
+ if (ci->flags == CLUSTER_FLAG_NONE)
+ move_cluster(si, ci, &si->free_clusters,
+ CLUSTER_FLAG_FREE);
+ spin_unlock(&ci->lock);
+ }
+
+ return nr_new;
+}
+
/* Caller must hold si->xswap_lock. Cannot fail. */
static void xswap_unmap_clusters(struct swap_info_struct *si,
unsigned long start_idx, unsigned long nr)
--
2.54.0