Re: [RFC PATCH v4 12/18] mm/swap: bound synchronous discard outside the allocation reader
From: Kunwu Chan
Date: Thu Nov 19 2026 - 23:37:06 EST
Reviewed-by: Kunwu Chan <kunwu.chan@xxxxxxxxx>
Thanks,
Kunwu
>
> From: Kairui Song <kasong@xxxxxxxxxxx>
>
> The synchronous discard fallback drains a list that freeing paths can
> keep replenishing. Limit each foreground step to one cluster, retaining
> CLUSTER_FLAG_DISCARD while I/O is in flight and returning the cluster to
> the free list only after completion. The background worker continues to
> process the remaining list.
>
> Keep synchronous discard outside the topology reader. Enter this fallback
> only after a complete normal order-0 allocation walk fails. A higher-
> priority device with only pending discard work does not force synchronous
> I/O if a lower-priority device satisfies that walk.
>
> Find a writable discard-capable device under the reader and take its live
> reference, then release the reader before issuing I/O. Drop the reference
> after completion and restart allocation with a fresh cursor, so a
> concurrent topology change cannot leave stale ring state. This also sees
> discard work produced by the preceding reclaim scans.
>
> Track the swap types already processed during this folio allocation and
> check each candidate at most once per fallback search. An empty check
> does not consume the I/O budget. Each swap type can complete at most one
> synchronous cluster discard, bounding successful retries by MAX_SWAPFILES
> even if membership changes. Do not keep discarding until this task wins
> a free cluster from a peer.
>
> Large folios retain the existing same-priority retry and split boundary;
> their reader-held cluster walk does not issue synchronous discard. The
> budget bounds work count, not device I/O completion latency.
>
> Link: https://lore.kernel.org/all/CAMgjq7CsYhEjvtN85XGkrONYAJxve7gG593TFeOGV-oax++kWA@xxxxxxxxxxxxxx/
> Signed-off-by: Kairui Song <kasong@xxxxxxxxxxx>
> Co-developed-by: Lian Wang <lianux.mm@xxxxxxxxx>
> Signed-off-by: Lian Wang <lianux.mm@xxxxxxxxx>
> ---
> mm/swapfile.c | 175 ++++++++++++++++++++++++++++----------------------
> 1 file changed, 100 insertions(+), 75 deletions(-)
>
> diff --git a/mm/swapfile.c b/mm/swapfile.c
> index 5c5e449e9da3..6b2dbaf33cb6 100644
> --- a/mm/swapfile.c
> +++ b/mm/swapfile.c
> @@ -6,6 +6,7 @@
> * Swap reorganised 29.12.95, Stephen Tweedie
> */
>
> +#include <linux/bitmap.h>
> #include <linux/blkdev.h>
> #include <linux/mm.h>
> #include <linux/sched/mm.h>
> @@ -585,6 +586,26 @@ static long swap_usage_in_pages(struct swap_info_struct *si)
> return atomic_long_read(&si->inuse_pages) & SWAP_USAGE_COUNTER_MASK;
> }
>
> +/*
> + * Serialize the allocation on single CPU or globally to avoid
> + * fragmentation and make the workflow easier to follow.
> + */
> +static void swap_alloc_lock_device(struct swap_info_struct *si)
> +{
> + if (si->flags & SWP_SOLIDSTATE)
> + local_lock(&si->percpu_cluster->lock);
> + else
> + spin_lock(&si->global_cluster->lock);
> +}
> +
> +static void swap_alloc_unlock_device(struct swap_info_struct *si)
> +{
> + if (si->flags & SWP_SOLIDSTATE)
> + local_unlock(&si->percpu_cluster->lock);
> + else
> + spin_unlock(&si->global_cluster->lock);
> +}
> +
> /* Reclaim the swap entry anyway if possible */
> #define TTRS_ANYWAY 0x1
> /*
> @@ -916,12 +937,8 @@ static int swap_cluster_alloc_table(struct swap_info_struct *si,
>
> lock:
> if (!locked) {
> - if (si) {
> - if (si->flags & SWP_SOLIDSTATE)
> - local_lock(&si->percpu_cluster->lock);
> - else
> - spin_lock(&si->global_cluster->lock);
> - }
> + if (si)
> + swap_alloc_lock_device(si);
> spin_lock(&ci->lock);
> }
> lockdep_assert_held(&ci->lock);
> @@ -1024,10 +1041,7 @@ swap_cluster_populate(struct swap_info_struct *si,
> * which could give allocator hooks and failure cleanup reserve access.
> */
> spin_unlock(&ci->lock);
> - if (si->flags & SWP_SOLIDSTATE)
> - local_unlock(&si->percpu_cluster->lock);
> - else
> - spin_unlock(&si->global_cluster->lock);
> + swap_alloc_unlock_device(si);
>
> gfp = (GFP_KERNEL | __GFP_HIGH | __GFP_NOMEMALLOC) &
> ~__GFP_DIRECT_RECLAIM;
> @@ -1134,44 +1148,40 @@ static struct swap_cluster_info *isolate_lock_cluster(
> }
>
> /*
> - * Doing discard actually. After a cluster discard is finished, the cluster
> - * will be added to free cluster list. Discard cluster is a bit special as
> - * they don't participate in allocation or reclaim, so clusters marked as
> - * CLUSTER_FLAG_DISCARD must remain off-list or on discard list.
> + * Discard one cluster. After the discard is finished, the cluster will be
> + * added to the free cluster list. Discard clusters are special because they
> + * don't participate in allocation or reclaim, so CLUSTER_FLAG_DISCARD must
> + * remain set while a cluster is either queued or being discarded.
> */
> -static bool swap_do_scheduled_discard(struct swap_info_struct *si)
> +static bool swap_discard_one_cluster(struct swap_info_struct *si)
> {
> struct swap_cluster_info *ci;
> - bool ret = false;
> unsigned int idx;
>
> spin_lock(&si->lock);
> - while (!list_empty(&si->discard_clusters)) {
> - ci = list_first_entry(&si->discard_clusters, struct swap_cluster_info, list);
> - /*
> - * Delete the cluster from list to prepare for discard, but keep
> - * the CLUSTER_FLAG_DISCARD flag, there could be percpu_cluster
> - * pointing to it, or ran into by relocate_cluster.
> - */
> - list_del(&ci->list);
> - idx = cluster_index(si, ci);
> + if (list_empty(&si->discard_clusters)) {
> spin_unlock(&si->lock);
> - discard_swap_cluster(si, idx * SWAPFILE_CLUSTER,
> - SWAPFILE_CLUSTER);
> -
> - spin_lock(&ci->lock);
> - /*
> - * Discard is done, clear its flags as it's off-list, then
> - * return the cluster to allocation list.
> - */
> - ci->flags = CLUSTER_FLAG_NONE;
> - __free_cluster(si, ci);
> - spin_unlock(&ci->lock);
> - ret = true;
> - spin_lock(&si->lock);
> + return false;
> }
> +
> + ci = list_first_entry(&si->discard_clusters,
> + struct swap_cluster_info, list);
> + /*
> + * Delete the cluster from the list, but keep CLUSTER_FLAG_DISCARD
> + * set while the discard is in flight. A percpu cluster may still
> + * point to it, or relocate_cluster() may encounter it.
> + */
> + list_del(&ci->list);
> + idx = cluster_index(si, ci);
> spin_unlock(&si->lock);
> - return ret;
> +
> + discard_swap_cluster(si, idx * SWAPFILE_CLUSTER, SWAPFILE_CLUSTER);
> +
> + spin_lock(&ci->lock);
> + ci->flags = CLUSTER_FLAG_NONE;
> + __free_cluster(si, ci);
> + spin_unlock(&ci->lock);
> + return true;
> }
>
> static void swap_discard_work(struct work_struct *work)
> @@ -1180,7 +1190,8 @@ static void swap_discard_work(struct work_struct *work)
>
> si = container_of(work, struct swap_info_struct, discard_work);
>
> - swap_do_scheduled_discard(si);
> + while (swap_discard_one_cluster(si))
> + cond_resched();
> }
>
> static void swap_users_ref_free(struct percpu_ref *ref)
> @@ -1609,15 +1620,11 @@ static unsigned long cluster_alloc_swap_entry(struct swap_info_struct *si,
> if (order && !(si->flags & SWP_BLKDEV))
> return 0;
>
> - if (si->flags & SWP_SOLIDSTATE) {
> - /* Fast path using per CPU cluster */
> - local_lock(&si->percpu_cluster->lock);
> + swap_alloc_lock_device(si);
> + if (si->flags & SWP_SOLIDSTATE)
> offset = __this_cpu_read(si->percpu_cluster->next[order]);
> - } else {
> - /* Serialize HDD SWAP allocation for each device. */
> - spin_lock(&si->global_cluster->lock);
> + else
> offset = si->global_cluster->next[order];
> - }
>
> if (offset != SWAP_ENTRY_INVALID) {
> ci = swap_cluster_lock(si, offset);
> @@ -1685,10 +1692,7 @@ static unsigned long cluster_alloc_swap_entry(struct swap_info_struct *si,
> goto done;
> }
> done:
> - if (si->flags & SWP_SOLIDSTATE)
> - local_unlock(&si->percpu_cluster->lock);
> - else
> - spin_unlock(&si->global_cluster->lock);
> + swap_alloc_unlock_device(si);
>
> return found;
> }
> @@ -1883,33 +1887,50 @@ static int swap_alloc_entry(struct folio *folio)
> }
>
> /*
> - * Discard pending clusters in a synchronized way when under high pressure.
> - * Return: true if any cluster is discarded.
> + * Discard outside the topology reader, holding only the selected device's
> + * live reference across I/O. Each successful retry starts a fresh queue walk.
> */
> -static bool swap_sync_discard(void)
> +static bool swap_discard_for_alloc(unsigned long *discarded)
> {
> - bool ret = false;
> - struct swap_info_struct *si, *next;
> + DECLARE_BITMAP(checked, MAX_SWAPFILES) = {};
> + struct swap_info_struct *si, *candidate;
> + unsigned int i, j;
> + bool done;
> + int type;
>
> - percpu_down_read(&swapon_rwsem);
> -start_over:
> - plist_for_each_entry_safe(si, next, &swap_active_head, list) {
> - percpu_up_read(&swapon_rwsem);
> - if (get_swap_device_info(si)) {
> - if (si->flags & SWP_PAGE_DISCARD)
> - ret = swap_do_scheduled_discard(si);
> - put_swap_device(si);
> + for (;;) {
> + candidate = NULL;
> + percpu_down_read(&swapon_rwsem);
> + for (i = 0; i < swap_queue_len && !candidate; i++) {
> + struct swap_prio_ring *ring = swap_queue[i];
> +
> + for (j = 0; j < ring->size; j++) {
> + si = swap_device_unmask_ptr(READ_ONCE(ring->dev[j]));
> + type = si->type;
> + if (test_bit(type, discarded) || test_bit(type, checked))
> + continue;
> + __set_bit(type, checked);
> + if ((READ_ONCE(si->flags) & (SWP_WRITEOK | SWP_PAGE_DISCARD)) !=
> + (SWP_WRITEOK | SWP_PAGE_DISCARD))
> + continue;
> + if (get_swap_device_info(si)) {
> + candidate = si;
> + break;
> + }
> + }
> }
> - if (ret)
> - return true;
> + percpu_up_read(&swapon_rwsem);
> + if (!candidate)
> + return false;
>
> - percpu_down_read(&swapon_rwsem);
> - if (plist_node_empty(&next->list))
> - goto start_over;
> + type = candidate->type;
> + done = swap_discard_one_cluster(candidate);
> + if (done)
> + __set_bit(type, discarded);
> + put_swap_device(candidate);
> + if (done)
> + return true;
> }
> - percpu_up_read(&swapon_rwsem);
> -
> - return false;
> }
>
> static int swap_extend_table_alloc(struct swap_info_struct *si,
> @@ -2213,12 +2234,16 @@ int folio_alloc_swap(struct folio *folio)
> }
> }
>
> -again:
> swap_alloc_entry(folio);
>
> if (!order && unlikely(!folio_test_swapcache(folio))) {
> - if (swap_sync_discard())
> - goto again;
> + DECLARE_BITMAP(discarded, MAX_SWAPFILES) = {};
> +
> + while (swap_discard_for_alloc(discarded)) {
> + swap_alloc_entry(folio);
> + if (folio_test_swapcache(folio))
> + break;
> + }
> }
>
> /* Need to call this even if allocation failed, for MEMCG_SWAP_FAIL. */
>