[PATCH 1/2] mm: zswap: free synchronous-IO writeback folios directly

[Alexandre Ghiti]

When zswap writes an entry back, it allocates a swap cache folio,
decompresses the entry into it and writes it out. That folio is cold by
construction, but it is currently left on the LRU for page reclaim to find
and free later. This wastes a reclaim scan and keeps cold memory resident
longer than necessary.

For synchronous-IO swap devices writeback completes in the calling context,
so the folio can be freed right after the write rather than left behind; do
that. Because it is freed directly rather than through reclaim, it is
allocated off the LRU: dropping the last reference on a folio still on the
LRU would trip the free-time page-flag checks. A folio that a concurrent
swapin has meanwhile claimed is left in place and reclaimed as usual.

Asynchronous and filesystem-backed swap complete writeback in interrupt
context, where the folio cannot be freed; they are handled in a later
change.

Signed-off-by: Alexandre Ghiti <alex@xxxxxxxx>
---
 mm/swap.h       |  3 ++-
 mm/swap_state.c | 17 +++++++++++------
 mm/zswap.c      | 36 ++++++++++++++++++++++++++++++++++--
 3 files changed, 47 insertions(+), 9 deletions(-)

diff --git a/mm/swap.h b/mm/swap.h
index 77d2d14eda42..c617e5a0257f 100644
--- a/mm/swap.h
+++ b/mm/swap.h
@@ -306,7 +306,8 @@ void *swap_cache_get_shadow(swp_entry_t entry);
 void swap_cache_del_folio(struct folio *folio);
 struct folio *swap_cache_alloc_folio(swp_entry_t target_entry, gfp_t gfp_mask,
 				     unsigned long orders, struct vm_fault *vmf,
-				     struct mempolicy *mpol, pgoff_t ilx);
+				     struct mempolicy *mpol, pgoff_t ilx,
+				     bool skip_lru);
 /* Below helpers require the caller to lock and pass in the swap cluster. */
 void __swap_cache_add_folio(struct swap_cluster_info *ci,
 			    struct folio *folio, swp_entry_t entry);
diff --git a/mm/swap_state.c b/mm/swap_state.c
index 9c3a5cf99778..048efc7ca612 100644
--- a/mm/swap_state.c
+++ b/mm/swap_state.c
@@ -403,7 +403,8 @@ void __swap_cache_replace_folio(struct swap_cluster_info *ci,
 static struct folio *__swap_cache_alloc(struct swap_cluster_info *ci,
 					swp_entry_t targ_entry, gfp_t gfp,
 					unsigned int order, struct vm_fault *vmf,
-					struct mempolicy *mpol, pgoff_t ilx)
+					struct mempolicy *mpol, pgoff_t ilx,
+					bool skip_lru)
 {
 	int err;
 	swp_entry_t entry;
@@ -484,7 +485,8 @@ static struct folio *__swap_cache_alloc(struct swap_cluster_info *ci,
 	lruvec_stat_mod_folio(folio, NR_SWAPCACHE, nr_pages);
 
 	/* Caller will initiate read into locked new_folio */
-	folio_add_lru(folio);
+	if (!skip_lru)
+		folio_add_lru(folio);
 	return folio;
 }
 
@@ -507,7 +509,8 @@ static struct folio *__swap_cache_alloc(struct swap_cluster_info *ci,
  */
 struct folio *swap_cache_alloc_folio(swp_entry_t targ_entry, gfp_t gfp,
 				     unsigned long orders, struct vm_fault *vmf,
-				     struct mempolicy *mpol, pgoff_t ilx)
+				     struct mempolicy *mpol, pgoff_t ilx,
+				     bool skip_lru)
 {
 	int order, err;
 	struct folio *ret;
@@ -522,7 +525,7 @@ struct folio *swap_cache_alloc_folio(swp_entry_t targ_entry, gfp_t gfp,
 
 	do {
 		ret = __swap_cache_alloc(ci, targ_entry, gfp, order,
-					 vmf, mpol, ilx);
+					 vmf, mpol, ilx, skip_lru);
 		if (!IS_ERR(ret))
 			break;
 		err = PTR_ERR(ret);
@@ -643,7 +646,8 @@ static struct folio *swap_cache_read_folio(swp_entry_t entry, gfp_t gfp,
 		folio = swap_cache_get_folio(entry);
 		if (folio)
 			return folio;
-		folio = swap_cache_alloc_folio(entry, gfp, BIT(0), NULL, mpol, ilx);
+		folio = swap_cache_alloc_folio(entry, gfp, BIT(0), NULL, mpol, ilx,
+					       false);
 	} while (PTR_ERR(folio) == -EEXIST);
 
 	if (IS_ERR_OR_NULL(folio))
@@ -683,7 +687,8 @@ struct folio *swapin_sync(swp_entry_t entry, gfp_t gfp, unsigned long orders,
 		folio = swap_cache_get_folio(entry);
 		if (folio)
 			return folio;
-		folio = swap_cache_alloc_folio(entry, gfp, orders, vmf, mpol, ilx);
+		folio = swap_cache_alloc_folio(entry, gfp, orders, vmf, mpol, ilx,
+					       false);
 	} while (PTR_ERR(folio) == -EEXIST);
 
 	if (IS_ERR(folio))
diff --git a/mm/zswap.c b/mm/zswap.c
index 761cd699e0a3..3c494c5a671c 100644
--- a/mm/zswap.c
+++ b/mm/zswap.c
@@ -972,6 +972,30 @@ static bool zswap_decompress(struct zswap_entry *entry, struct folio *folio)
 /*********************************
 * writeback code
 **********************************/
+static void zswap_writeback_free_folio(struct folio *folio)
+{
+	folio_lock(folio);
+
+	/* The folio was allocated off the LRU and nothing re-adds it here. */
+	VM_WARN_ON_ONCE_FOLIO(folio_test_lru(folio), folio);
+
+	/*
+	 * Gate remove_mapping() on folio_test_swapcache(): a racing swapin may
+	 * have freed the swap slot (folio_free_swap()) and dropped the folio from
+	 * the cache, and remove_mapping() must not run on a non-swapcache folio
+	 * (it would trip __remove_mapping()'s mapping == folio_mapping() check).
+	 */
+	if (folio_test_swapcache(folio) &&
+	    remove_mapping(swap_address_space(folio->swap), folio))
+		goto out;
+
+	/* Raced: the folio is now owned by the swapin; put it back on the LRU. */
+	folio_add_lru(folio);
+out:
+	folio_unlock(folio);
+	folio_put(folio);
+}
+
 /*
  * Attempts to free an entry by adding a folio to the swap cache,
  * decompressing the entry data into the folio, and issuing a
@@ -992,16 +1016,18 @@ static int zswap_writeback_entry(struct zswap_entry *entry,
 	struct folio *folio;
 	struct mempolicy *mpol;
 	struct swap_info_struct *si;
+	bool sync;
 	int ret = 0;
 
 	/* try to allocate swap cache folio */
 	si = get_swap_device(swpentry);
 	if (!si)
 		return -EEXIST;
+	sync = data_race(si->flags & SWP_SYNCHRONOUS_IO);
 
 	mpol = get_task_policy(current);
 	folio = swap_cache_alloc_folio(swpentry, GFP_KERNEL, BIT(0), NULL, mpol,
-				       NO_INTERLEAVE_INDEX);
+				       NO_INTERLEAVE_INDEX, sync);
 	put_swap_device(si);
 
 	/*
@@ -1046,11 +1072,17 @@ static int zswap_writeback_entry(struct zswap_entry *entry,
 	folio_mark_uptodate(folio);
 
 	/* move it to the tail of the inactive list after end_writeback */
-	folio_set_reclaim(folio);
+	if (!sync)
+		folio_set_reclaim(folio);
 
 	/* start writeback */
 	__swap_writepage(folio, NULL);
 
+	if (sync) {
+		zswap_writeback_free_folio(folio);
+		return 0;
+	}
+
 out:
 	if (ret) {
 		swap_cache_del_folio(folio);
-- 
2.53.0-Meta