Re: [PATCH v4 7/8] mm: zswap: Refactor and optimize zswap_store_pages() batch operations.

From: Nhat Pham

Date: Fri Nov 20 2026 - 18:46:15 EST


On Thu, Nov 19, 2026 at 8:19 PM Kanchana P. Sridhar
<kanchanapsridhar2026@xxxxxxxxx> wrote:
>
> Subsequent patch series will be submitted, that implement ideas building
> upon this batching framework:
>
> 1) Replacing zsmalloc with SLUB allocator, comprising of kmem_caches
> for a finite # of compressed size bins. This enables bulk allocation
> of compressed buffers in a batch that belong to the same size
> bin. It gets rid of significant zsmalloc latency overhead (locks,
> compressed data spanning zspages, etc.), without compromising memory
> efficiency.

I don't think this will be a good idea, because it will make
compressed memory unmovable.

>
> 2) vmscan reclaim batching of hybrid any-order folios, that will submit
> a batch of folios to zswap_store()'s batching framework to optimize
> storing of batches of pages from different folios, in addition to
> within large folios.

This one might be more complicated than its worth, but I'm not sure.

>
> Signed-off-by: Kanchana P. Sridhar <kanchanapsridhar2026@xxxxxxxxx>

Code mostly LGTM.

> ---
> include/linux/memcontrol.h | 12 +++
> mm/memcontrol.c | 23 ++++-
> mm/zswap.c | 181 +++++++++++++++++++++++++------------
> 3 files changed, 153 insertions(+), 63 deletions(-)
>
> diff --git a/include/linux/memcontrol.h b/include/linux/memcontrol.h
> index 980358bc6f49..93f060818c14 100644
> --- a/include/linux/memcontrol.h
> +++ b/include/linux/memcontrol.h
> @@ -1244,6 +1244,11 @@ struct mem_cgroup *mem_cgroup_from_css(struct cgroup_subsys_state *css)
> return NULL;
> }
>
> +static inline void obj_cgroup_get_many(struct obj_cgroup *objcg,
> + unsigned long nr)
> +{
> +}
> +
> static inline void obj_cgroup_get(struct obj_cgroup *objcg)
> {
> }
> @@ -1921,6 +1926,8 @@ static inline void mem_cgroup_calculate_protection_path(struct mem_cgroup *root,
> #if defined(CONFIG_MEMCG) && defined(CONFIG_ZSWAP)
> bool obj_cgroup_may_zswap(struct obj_cgroup *objcg);
> void obj_cgroup_charge_zswap(struct obj_cgroup *objcg, size_t size);
> +void obj_cgroup_charge_zswap_many(struct obj_cgroup *objcg, size_t size,
> + u8 nr_pages, u8 nr_incomp_pages);
> void obj_cgroup_uncharge_zswap(struct obj_cgroup *objcg, size_t size);
> bool mem_cgroup_zswap_writeback_enabled(const struct mem_cgroup *memcg);
> #else
> @@ -1932,6 +1939,11 @@ static inline void obj_cgroup_charge_zswap(struct obj_cgroup *objcg,
> size_t size)
> {
> }

Why do we still need obj_cgroup_charge_zswap() at all?

> +static inline void obj_cgroup_charge_zswap_many(struct obj_cgroup *objcg,
> + size_t size, u8 nr_pages,
> + u8 nr_incomp_pages)
> +{
> +}
> static inline void obj_cgroup_uncharge_zswap(struct obj_cgroup *objcg,
> size_t size)
> {
> diff --git a/mm/memcontrol.c b/mm/memcontrol.c
> index 091401cecf39..2236f5ba6304 100644
> --- a/mm/memcontrol.c
> +++ b/mm/memcontrol.c
> @@ -6345,6 +6345,23 @@ bool obj_cgroup_may_zswap(struct obj_cgroup *objcg)
> * compression and storage in zswap for this cgroup to go ahead.
> */
> void obj_cgroup_charge_zswap(struct obj_cgroup *objcg, size_t size)
> +{
> + obj_cgroup_charge_zswap_many(objcg, size, 1, (u8)(size == PAGE_SIZE));
> +}
> +
> +/**
> + * obj_cgroup_charge_zswap_many - bulk charge compression backend memory
> + * from compressing a batch of pages.
> + * @objcg: the object cgroup
> + * @size: total compressed length of a batch of compressed objects
> + * @nr_pages: # of pages that were compressed to store in zswap
> + * @nr_incomp_pages: # of incompressible pages to store in zswap
> + *
> + * This forces the charge after obj_cgroup_may_zswap() allowed
> + * compression and storage in zswap for this cgroup to go ahead.
> + */
> +void obj_cgroup_charge_zswap_many(struct obj_cgroup *objcg, size_t size,
> + u8 nr_pages, u8 nr_incomp_pages)
> {
> struct mem_cgroup *memcg;
>
> @@ -6363,9 +6380,9 @@ void obj_cgroup_charge_zswap(struct obj_cgroup *objcg, size_t size)
> rcu_read_lock();
> memcg = obj_cgroup_memcg(objcg);
> mod_memcg_state(memcg, MEMCG_ZSWAP_B, size);
> - mod_memcg_state(memcg, MEMCG_ZSWAPPED, 1);
> - if (size == PAGE_SIZE)
> - mod_memcg_state(memcg, MEMCG_ZSWAP_INCOMP, 1);
> + mod_memcg_state(memcg, MEMCG_ZSWAPPED, nr_pages);
> + if (nr_incomp_pages)
> + mod_memcg_state(memcg, MEMCG_ZSWAP_INCOMP, nr_incomp_pages);
> rcu_read_unlock();
> }
>
> diff --git a/mm/zswap.c b/mm/zswap.c
> index df59ab0c1029..5e1fb5624ffb 100644
> --- a/mm/zswap.c
> +++ b/mm/zswap.c
> @@ -491,12 +491,12 @@ static int __must_check zswap_pool_tryget(struct zswap_pool *pool)
> }
>
> /* The caller must already have a reference. */
> -static void zswap_pool_get(struct zswap_pool *pool)
> +static inline void zswap_pool_get_many(struct zswap_pool *pool, u8 nr)
> {
> - percpu_ref_get(&pool->ref);
> + percpu_ref_get_many(&pool->ref, nr);
> }
>
> -static void zswap_pool_put(struct zswap_pool *pool)
> +static inline void zswap_pool_put(struct zswap_pool *pool)
> {
> percpu_ref_put(&pool->ref);
> }
> @@ -1603,55 +1603,60 @@ static void shrink_worker(struct work_struct *w)
> * main API
> **********************************/
>
> +/* Functions called by zswap_store(), that operate on batches of pages. */
> +
> /*
> - * Store multiple pages in @zc->folio, starting from the page at index @start up
> - * to the page at index @end-1.
> + * zswap_init_entries() co-locates most of the zswap entry fields' setup under
> + * the assumption that all phases of store processing will succeed. An important
> + * marker to simplify catch-all cleanup in zswap_store_pages() is done by
> + * initializing each entry's @handle to ERR_PTR(-EINVAL).
> */
> -static bool zswap_store_pages(struct zswap_store_context *zc,
> - long start,
> - long end,
> - struct zswap_pool *pool,
> - struct obj_cgroup *objcg)
> +static inline void zswap_init_entries(struct zswap_store_context *zc,
> + struct zswap_pool *pool,
> + struct obj_cgroup *objcg)
> {
> - u8 i, store_fail_idx = 0;
> - bool ret;
> -
> - zc->nr_pages = end - start;
> - zc->folio_start = start;
> -
> - VM_WARN_ON_ONCE(zc->nr_pages <= 0 || zc->nr_pages > ZSWAP_MAX_BATCH_SIZE);
> -
> - ret = zswap_entries_cache_alloc_batch(zc, GFP_KERNEL);
> - if (unlikely(!ret))
> - return false;
> + struct zswap_entry *entry;
> + u8 i;
>
> /*
> * We co-locate entry initialization as much as possible here to
> * minimize potential cache misses.
> */
> for (i = 0; i < zc->nr_pages; ++i) {
> + entry = zc->entries[i];
> /*
> - * Set pool_idx before the xa_store() below publishes the entry, or a
> - * concurrent reader could resolve a stale pool_idx left by slab reuse
> - * to an unrelated live pool.
> + * Set pool_idx before the xa_store() in zswap_tree_stores()
> + * publishes the entry, or a concurrent reader could resolve a
> + * stale pool_idx left by slab reuse to an unrelated live pool.
> */
> - zc->entries[i]->pool_idx = pool->idx;
> - zc->entries[i]->swpentry = folio_swap_entry(zc->folio,
> + entry->pool_idx = pool->idx;
> + entry->swpentry = folio_swap_entry(zc->folio,
> zc->folio_start + i);
> - zc->entries[i]->objcg = objcg;
> - zc->entries[i]->referenced = true;
> - zc->entries[i]->nid = zc->nid;
> - INIT_LIST_HEAD(&zc->entries[i]->lru);
> + entry->objcg = objcg;
> + entry->referenced = true;
> + entry->nid = zc->nid;
> + INIT_LIST_HEAD(&entry->lru);
> }
> +}
>
> - for (i = 0; i < zc->nr_pages; ++i) {
> - if (!zswap_compress(zc->folio, zc->folio_start + i,
> - zc->entries[i], pool, zc->wb_enabled))
> - goto store_pages_failed;
> - }
> +/*
> + * zswap_tree_stores() stores the batch entries in xarray and adds them to the
> + * zswap LRU list. If an error occurs during xa_store(), localized state
> + * unwinding is done to erase prior batch entries from the xarray and delete
> + * them from the zswap LRU list. Remaining batch cleanup occurs in
> + * zswap_store_pages(). As batch entries are processed, we maintain aggregate
> + * statistics to do the accounting in one bulk operation.
> + */
> +static bool zswap_tree_stores(struct zswap_store_context *zc,
> + struct zswap_pool *pool,
> + struct obj_cgroup *objcg)
> +{
> + u8 i, store_fail_idx = 0, nr_incomp = 0;
> + size_t total_batch_comp_length = 0;
> + struct zswap_entry *old, *entry;
>
> for (i = 0; i < zc->nr_pages; ++i) {
> - struct zswap_entry *old, *entry = zc->entries[i];
> + entry = zc->entries[i];
>
> old = xa_store(swap_zswap_tree(entry->swpentry),
> swp_offset(entry->swpentry),
> @@ -1663,13 +1668,11 @@ static bool zswap_store_pages(struct zswap_store_context *zc,
> zswap_reject_alloc_fail++;
> /*
> * Entries up to this point have been stored in the
> - * xarray. zswap_store() will erase them from the xarray
> - * and call zswap_entry_free(). Local cleanup in
> - * 'store_pages_failed' only needs to happen for
> - * entries from [@i to @nr_pages).
> + * xarray. Local cleanup in 'tree_store_failed' needs to
> + * erase entries up until now.
> */
> store_fail_idx = i;
> - goto store_pages_failed;
> + goto tree_store_failed;
> }
>
> /*
> @@ -1680,22 +1683,10 @@ static bool zswap_store_pages(struct zswap_store_context *zc,
> if (unlikely(old))
> zswap_entry_free(old);
>
> - /*
> - * The entry is successfully compressed and stored in the tree,
> - * and further failures will be cleaned up in zswap_store().
> - * Grab refs to the pool and objcg, charge zswap memory, and
> - * increment zswap_stored_pages. The opposite actions will be
> - * performed by zswap_entry_free() when the entry is removed
> - * from the tree.
> - */
> - zswap_pool_get(pool);
> - if (objcg) {
> - obj_cgroup_get(objcg);
> - obj_cgroup_charge_zswap(objcg, entry->length);
> - }
> - atomic_long_inc(&zswap_stored_pages);
> + total_batch_comp_length += entry->length;
> +
> if (unlikely(entry->length == PAGE_SIZE))
> - atomic_long_inc(&zswap_stored_incompressible_pages);
> + ++nr_incomp;
>
> /*
> * We finish by adding the entry to the LRU while it's already
> @@ -1711,17 +1702,87 @@ static bool zswap_store_pages(struct zswap_store_context *zc,
> zswap_lru_add(entry);
> }
>
> + /*
> + * Each batch entry is successfully compressed and stored in the tree,
> + * and further failures will be cleaned up in zswap_store().
> + * Next, we do bulk accounting for the batch: grab refs to the pool and
> + * objcg, charge zswap memory, increment zswap_stored_pages and
> + * zswap_stored_incompressible_pages. The opposite actions will be
> + * performed by zswap_entry_free() when the entry is removed from the
> + * tree.
> + */
> + zswap_pool_get_many(pool, zc->nr_pages);
> + if (objcg) {
> + obj_cgroup_get_many(objcg, zc->nr_pages);
> + obj_cgroup_charge_zswap_many(objcg, total_batch_comp_length,
> + zc->nr_pages, nr_incomp);
> + }
> + atomic_long_add(zc->nr_pages, &zswap_stored_pages);
> + if (unlikely(nr_incomp))
> + atomic_long_add(nr_incomp, &zswap_stored_incompressible_pages);
> +
> + return true;
> +

Can you add a comment that we only clean up the current batch here,
and the clean up of other batches in the folio are deferred to
zswap_store() (or whereever up the stack it is handled?).

> +tree_store_failed:
> + for (i = 0; i < store_fail_idx; ++i) {
> + xa_erase(swap_zswap_tree(zc->entries[i]->swpentry),
> + swp_offset(zc->entries[i]->swpentry));

xas API?

> + zswap_lru_del(zc->entries[i]);
> + }
> +
> + return false;
> +}
> +
> +/*