[RFC PATCH 08/10] mm/swapfile: manage ghost cluster_info via lazy vmalloc

[Baoquan He]

Replace the kvzalloc cluster_info allocation for ghost swap with a sparse
vmalloc area (VM_SPARSE). At swapon time, reserve up to 64MB of virtual
address space (supporting ~1TB ghost swap) but only allocate physical
pages for the initial size. The cluster_info pointer never changes —
__swap_offset_to_cluster() and cluster_index() are unchanged.

Add cluster_vm and ghost_name fields to struct swap_info_struct.
In setup_swap_clusters_info(), allocate zeroed physical pages and map
them into the reserved VA via vm_area_map_pages(). Handle teardown in
both swapoff and the swapon error path.

Signed-off-by: Baoquan He <baoquan.he@xxxxxxxxx>
---
 include/linux/swap.h |   4 +
 mm/swapfile.c        | 210 +++++++++++++++++++++++++++++++++++--------
 2 files changed, 179 insertions(+), 35 deletions(-)

diff --git a/include/linux/swap.h b/include/linux/swap.h
index a4d6851e0921..b87688b7d4ba 100644
--- a/include/linux/swap.h
+++ b/include/linux/swap.h
@@ -16,6 +16,8 @@
 #include <uapi/linux/mempolicy.h>
 #include <asm/page.h>
 
+struct vm_struct;
+
 #define SWAP_FLAG_PREFER	0x8000	/* set if swap priority specified */
 #define SWAP_FLAG_PRIO_MASK	0x7fff
 #define SWAP_FLAG_DISCARD	0x10000 /* enable discard for swap */
@@ -279,6 +281,8 @@ struct swap_info_struct {
 	struct plist_node avail_list;   /* entry in swap_avail_head */
 #ifdef CONFIG_ZSWAP
 	struct xarray redirect_xa;	/* ghost offset → phys swp_entry_t */
+	struct vm_struct *cluster_vm;	/* ghost: sparse vm_area for cluster_info */
+	char *ghost_name;		/* ghost: dentry name for sysfs */
 #endif
 };
 
diff --git a/mm/swapfile.c b/mm/swapfile.c
index c1686c2b2521..da085a75d7b5 100644
--- a/mm/swapfile.c
+++ b/mm/swapfile.c
@@ -56,6 +56,9 @@ static bool folio_swapcache_freeable(struct folio *folio);
 static void move_cluster(struct swap_info_struct *si,
 			 struct swap_cluster_info *ci, struct list_head *list,
 			 enum swap_cluster_flags new_flags);
+#ifdef CONFIG_ZSWAP
+#define GHOST_MAX_CLUSTER_BYTES (64UL * 1024 * 1024)
+#endif
 
 /*
  * Protects the swap_info array, and the SWP_USED flag. swap_info contains
@@ -3244,9 +3247,14 @@ SYSCALL_DEFINE1(swapoff, const char __user *, specialfile)
 	swap_file = p->swap_file;
 	p->swap_file = NULL;
 	maxpages = p->max;
-	cluster_info = p->cluster_info;
-	p->max = 0;
-	p->cluster_info = NULL;
+	if (p->flags & SWP_GHOST) {
+		p->max = 0;
+		cluster_info = NULL;
+	} else {
+		cluster_info = p->cluster_info;
+		p->max = 0;
+		p->cluster_info = NULL;
+	}
 	spin_unlock(&p->lock);
 	spin_unlock(&swap_lock);
 	arch_swap_invalidate_area(p->type);
@@ -3254,7 +3262,31 @@ SYSCALL_DEFINE1(swapoff, const char __user *, specialfile)
 	mutex_unlock(&swapon_mutex);
 	kfree(p->global_cluster);
 	p->global_cluster = NULL;
-	free_swap_cluster_info(cluster_info, maxpages);
+	if (p->flags & SWP_GHOST) {
+		unsigned long j, np = p->cluster_vm ?
+			p->cluster_vm->nr_pages : 0;
+		struct page **pglist = p->cluster_vm ?
+			(struct page **)p->cluster_vm->pages : NULL;
+
+		xa_destroy(&p->redirect_xa);
+		kfree(p->ghost_name);
+		p->ghost_name = NULL;
+		for (j = 0; j < np; j++)
+			__free_page(pglist[j]);
+		kvfree(pglist);
+		if (p->cluster_vm) {
+			vm_area_unmap_pages(p->cluster_vm,
+					    (unsigned long)p->cluster_vm->addr,
+					    (unsigned long)p->cluster_vm->addr +
+					    np * PAGE_SIZE);
+			remove_vm_area(p->cluster_vm->addr);
+			kfree(p->cluster_vm);
+			p->cluster_vm = NULL;
+		}
+		p->cluster_info = NULL;
+	} else {
+		free_swap_cluster_info(cluster_info, maxpages);
+	}
 
 	inode = mapping->host;
 
@@ -3624,16 +3656,48 @@ static int setup_swap_clusters_info(struct swap_info_struct *si,
 				    unsigned long maxpages)
 {
 	unsigned long nr_clusters = DIV_ROUND_UP(maxpages, SWAPFILE_CLUSTER);
-	struct swap_cluster_info *cluster_info;
+	struct swap_cluster_info *cluster_info = NULL;
 	int err = -ENOMEM;
 	unsigned long i;
 
-	cluster_info = kvzalloc_objs(*cluster_info, nr_clusters);
-	if (!cluster_info)
-		goto err;
+	if (si->flags & SWP_GHOST) {
+		unsigned long bytes, nr_pages;
+		struct page **pg;
 
-	for (i = 0; i < nr_clusters; i++)
-		spin_lock_init(&cluster_info[i].lock);
+		bytes = ALIGN(nr_clusters * sizeof(struct swap_cluster_info),
+			      PAGE_SIZE);
+		nr_pages = bytes / PAGE_SIZE;
+		pg = kvmalloc_array(nr_pages, sizeof(*pg), GFP_KERNEL);
+		if (!pg)
+			goto err;
+		for (i = 0; i < nr_pages; i++) {
+			pg[i] = alloc_page(GFP_KERNEL | __GFP_ZERO);
+			if (!pg[i]) {
+				while (i-- > 0)
+					__free_page(pg[i]);
+				kvfree(pg);
+				goto err;
+			}
+		}
+		err = vm_area_map_pages(si->cluster_vm,
+					(unsigned long)si->cluster_vm->addr,
+					(unsigned long)si->cluster_vm->addr + bytes, pg);
+		if (err) {
+			for (i = 0; i < nr_pages; i++)
+				__free_page(pg[i]);
+			kvfree(pg);
+			goto err;
+		}
+		si->cluster_vm->pages = (void *)pg;
+		si->cluster_vm->nr_pages = nr_pages;
+		cluster_info = si->cluster_info;
+	} else {
+		cluster_info = kvzalloc_objs(*cluster_info, nr_clusters);
+		if (!cluster_info)
+			goto err;
+		for (i = 0; i < nr_clusters; i++)
+			spin_lock_init(&cluster_info[i].lock);
+	}
 
 	if (!(si->flags & SWP_SOLIDSTATE)) {
 		si->global_cluster = kmalloc_obj(*si->global_cluster);
@@ -3644,37 +3708,64 @@ static int setup_swap_clusters_info(struct swap_info_struct *si,
 		spin_lock_init(&si->global_cluster_lock);
 	}
 
-	/*
-	 * Mark unusable pages (header page, bad pages, and the EOF part of
-	 * the last cluster) as unavailable. The clusters aren't marked free
-	 * yet, so no list operations are involved yet.
-	 */
-	err = swap_cluster_setup_bad_slot(si, cluster_info, 0, false);
-	if (err)
-		goto err;
-	for (i = 0; i < swap_header->info.nr_badpages; i++) {
-		unsigned int page_nr = swap_header->info.badpages[i];
-
-		if (!page_nr || page_nr > swap_header->info.last_page) {
-			pr_warn("Bad slot offset is out of border: %d (last_page: %d)\n",
-				page_nr, swap_header->info.last_page);
-			err = -EINVAL;
+	if (si->flags & SWP_GHOST) {
+		struct swap_cluster_info *ci;
+
+		ci = &cluster_info[0];
+		if (!ci->count && swap_cluster_alloc_table(ci, GFP_KERNEL)) {
+			err = -ENOMEM;
 			goto err;
 		}
-		err = swap_cluster_setup_bad_slot(si, cluster_info, page_nr, false);
-		if (err)
-			goto err;
-	}
-	for (i = maxpages; i < round_up(maxpages, SWAPFILE_CLUSTER); i++) {
-		err = swap_cluster_setup_bad_slot(si, cluster_info, i, true);
+		spin_lock(&ci->lock);
+		__swap_table_set(ci, 0, SWP_TB_BAD);
+		ci->count++;
+		spin_unlock(&ci->lock);
+		if (nr_clusters && (maxpages % SWAPFILE_CLUSTER)) {
+			unsigned long base = (maxpages - 1) /
+				SWAPFILE_CLUSTER * SWAPFILE_CLUSTER;
+			unsigned long idx = base / SWAPFILE_CLUSTER;
+
+			ci = &cluster_info[idx];
+			if (!ci->count && swap_cluster_alloc_table(ci, GFP_KERNEL)) {
+				err = -ENOMEM;
+				goto err;
+			}
+			spin_lock(&ci->lock);
+			for (i = maxpages % SWAPFILE_CLUSTER;
+			     i < SWAPFILE_CLUSTER; i++)
+				__swap_table_set(ci, i, SWP_TB_BAD);
+			spin_unlock(&ci->lock);
+		}
+	} else {
+		err = swap_cluster_setup_bad_slot(si, cluster_info, 0, false);
 		if (err)
 			goto err;
+		for (i = 0; i < swap_header->info.nr_badpages; i++) {
+			unsigned int page_nr = swap_header->info.badpages[i];
+
+			if (!page_nr || page_nr > swap_header->info.last_page) {
+				pr_warn("Bad slot offset is out of border: "
+					"%d (last_page: %d)\n",
+					page_nr, swap_header->info.last_page);
+				err = -EINVAL;
+				goto err;
+			}
+			err = swap_cluster_setup_bad_slot(si, cluster_info,
+							  page_nr, false);
+			if (err)
+				goto err;
+		}
+		for (i = maxpages; i < round_up(maxpages, SWAPFILE_CLUSTER); i++) {
+			err = swap_cluster_setup_bad_slot(si, cluster_info,
+							  i, true);
+			if (err)
+				goto err;
+		}
 	}
 
 	INIT_LIST_HEAD(&si->free_clusters);
 	INIT_LIST_HEAD(&si->full_clusters);
 	INIT_LIST_HEAD(&si->discard_clusters);
-
 	for (i = 0; i < SWAP_NR_ORDERS; i++) {
 		INIT_LIST_HEAD(&si->nonfull_clusters[i]);
 		INIT_LIST_HEAD(&si->frag_clusters[i]);
@@ -3692,9 +3783,12 @@ static int setup_swap_clusters_info(struct swap_info_struct *si,
 		}
 	}
 
-	si->cluster_info = cluster_info;
+	if (!(si->flags & SWP_GHOST))
+		si->cluster_info = cluster_info;
 	return 0;
 err:
+	if (si->flags & SWP_GHOST)
+		return err;
 	free_swap_cluster_info(cluster_info, maxpages);
 	return err;
 }
@@ -3790,6 +3884,28 @@ SYSCALL_DEFINE2(swapon, const char __user *, specialfile, int, swap_flags)
 
 	si->max = maxpages;
 	si->pages = maxpages - 1;
+	if (si->flags & SWP_GHOST) {
+		unsigned long max_cl_bytes;
+
+		max_cl_bytes = swapfile_maximum_size / SWAPFILE_CLUSTER *
+			       sizeof(struct swap_cluster_info);
+		max_cl_bytes = min(ALIGN(max_cl_bytes, PAGE_SIZE),
+				   GHOST_MAX_CLUSTER_BYTES);
+		si->cluster_vm = get_vm_area_caller(max_cl_bytes, VM_SPARSE,
+					__builtin_return_address(0));
+		if (!si->cluster_vm) {
+			error = -ENOMEM;
+			goto bad_swap_unlock_inode;
+		}
+		si->cluster_info = si->cluster_vm->addr;
+		si->cluster_vm->nr_pages = 0;
+		si->cluster_vm->pages = NULL;
+		si->ghost_name = kstrdup(dentry->d_name.name, GFP_KERNEL);
+		if (!si->ghost_name) {
+			error = -ENOMEM;
+			goto bad_swap_unlock_inode;
+		}
+	}
 	nr_extents = setup_swap_extents(si, swap_file, &span);
 	if (nr_extents < 0) {
 		error = nr_extents;
@@ -3907,8 +4023,32 @@ SYSCALL_DEFINE2(swapon, const char __user *, specialfile, int, swap_flags)
 	si->global_cluster = NULL;
 	inode = NULL;
 	destroy_swap_extents(si, swap_file);
-	free_swap_cluster_info(si->cluster_info, si->max);
-	si->cluster_info = NULL;
+	if (si->flags & SWP_GHOST) {
+		unsigned long j, np = si->cluster_vm ?
+			si->cluster_vm->nr_pages : 0;
+		struct page **pglist = si->cluster_vm ?
+			(struct page **)si->cluster_vm->pages : NULL;
+
+		xa_destroy(&si->redirect_xa);
+		kfree(si->ghost_name);
+		si->ghost_name = NULL;
+		for (j = 0; j < np; j++)
+			__free_page(pglist[j]);
+		kvfree(pglist);
+		if (si->cluster_vm) {
+			vm_area_unmap_pages(si->cluster_vm,
+					    (unsigned long)si->cluster_vm->addr,
+					    (unsigned long)si->cluster_vm->addr +
+					    np * PAGE_SIZE);
+			remove_vm_area(si->cluster_vm->addr);
+			kfree(si->cluster_vm);
+			si->cluster_vm = NULL;
+		}
+		si->cluster_info = NULL;
+	} else {
+		free_swap_cluster_info(si->cluster_info, si->max);
+		si->cluster_info = NULL;
+	}
 	/*
 	 * Clear the SWP_USED flag after all resources are freed so
 	 * alloc_swap_info can reuse this si safely.
-- 
2.54.0