[PATCH v4 11/12] zram: allocate bounce buffer for spanning objects
From: Sergey Senozhatsky
Date: Fri Oct 16 2026 - 00:51:00 EST
zsmalloc objects that span two physical pages need to be linearized
before decompression, which zcomp does using stream's ->local_copy
buffer. That buffer cannot be used concurrently, so such reads take
the stream in exclusive mode, even when the backend has stateless
decompression. And spanning objects are not rare: Bo Zhang [1]
measured 23.3% of reads on lzo-rle going through the exclusive path,
causing contention on ->excl_lock of up to 72-96 ms:
baseline patched w/o kmalloc bounce
read contended 24,749 1,959
read wait_ns total 239.8 s ~12 s
read wait_ns max 757 ms 72-96 ms
read acquire 3,948,150 3,510,610
Allocate a temporary bounce buffer for spanning objects instead, so
that backends with stateless decompression can always use shared mode.
If the allocation fails we fall back to ->local_copy under ->excl_lock.
Per-request bounce buffer fixes the remaining tail latency, as was
measured by Bo Zhang [2]:
baseline patched patched w/ kmalloc bounce
read contended 24,749 ~1,959 0
shared 0% 76.7% 100%
A quick synthetic fio test for Reader vs Reader inversion on 24 CPUs
(lz4, 48 low priority (nice 19) readers, 24 normal priority CPU hogs
and 24 high priority readers), average of 3 runs:
Metric Base Patched
----------------------------------------------------------
Read IOPS 1,452k 1,466k (+1.0%)
Read clat avg 3.46 us 3.30 us (-4.9%)
Read clat stdev 165.37 us 15.69 us (-10.5x)
Read clat max 306,679 us 7,060 us (-43.4x)
An alternative approach is to vm_map_ram() scattered objects
(two pages), which fixes the tail latency in the same way,
but deferred TLB flushes add more cost, so it was rejected.
[1] https://lore.kernel.org/linux-kernel/20261014153603.1365617-1-zhangbo56@xxxxxxxxxx
[2] https://lore.kernel.org/linux-kernel/20261015112440.1491168-1-zhangbo56@xxxxxxxxxx
Signed-off-by: Sergey Senozhatsky <senozhatsky@xxxxxxxxxxxx>
---
drivers/block/zram/zcomp.c | 47 +++++++++++++++++++++++++----------
drivers/block/zram/zcomp.h | 2 +-
drivers/block/zram/zram_drv.c | 2 +-
3 files changed, 36 insertions(+), 15 deletions(-)
diff --git a/drivers/block/zram/zcomp.c b/drivers/block/zram/zcomp.c
index da122bf4dd15..4fda13aac896 100644
--- a/drivers/block/zram/zcomp.c
+++ b/drivers/block/zram/zcomp.c
@@ -176,14 +176,13 @@ void zcomp_cstrm_put(struct zcomp_cstrm *zstrm)
mutex_unlock(&zstrm->lock);
}
-bool zcomp_shared_decomp(struct zcomp *comp, struct scatterlist *sg)
+bool zcomp_shared_decomp(struct zcomp *comp)
{
/*
* Shared decompression is only safe when the backend has no
- * mutable per-stream decompression context and the object is
- * contained within a single page (no ->local_copy bounce needed).
+ * mutable per-stream decompression context.
*/
- return !comp->ops->create_dctx && sg_is_last(sg);
+ return !comp->ops->create_dctx;
}
struct zcomp_dstrm *zcomp_dstrm_get(struct zcomp *comp,
@@ -239,6 +238,15 @@ int zcomp_compress(struct zcomp *comp, struct zcomp_cstrm *zstrm,
return ret;
}
+static int zcomp_decompress_buf(struct zcomp *comp, struct zcomp_dstrm *zstrm,
+ struct scatterlist *sg, void *buf,
+ struct zcomp_req *req)
+{
+ sg_copy_to_buffer(sg, 2, buf, req->src_len);
+ req->src = buf;
+ return comp->ops->decompress(comp->params, &zstrm->ctx, req);
+}
+
int zcomp_decompress(struct zcomp *comp, struct zcomp_dstrm *zstrm,
struct scatterlist *sg, unsigned int src_len, void *dst)
{
@@ -247,27 +255,40 @@ int zcomp_decompress(struct zcomp *comp, struct zcomp_dstrm *zstrm,
.src_len = src_len,
.dst_len = PAGE_SIZE,
};
- void *src = NULL;
+ void *src;
int ret;
might_sleep();
lockdep_assert_held(&zstrm->lock);
- if (!zcomp_shared_decomp(comp, sg))
+ if (!zcomp_shared_decomp(comp))
lockdep_assert_held(&zstrm->excl_lock);
if (sg_is_last(sg)) {
/* the object is contained within one page, read it in-place */
src = kmap_local_page(sg_page(sg));
req.src = src + sg->offset;
- } else {
- /* the object spans two pages, linearize it into local copy */
- sg_copy_to_buffer(sg, 2, zstrm->local_copy, src_len);
- req.src = zstrm->local_copy;
+ ret = comp->ops->decompress(comp->params, &zstrm->ctx, &req);
+ kunmap_local(src);
+ return ret;
}
- ret = comp->ops->decompress(comp->params, &zstrm->ctx, &req);
- if (src)
- kunmap_local(src);
+ /* exclusive mode already holds ->excl_lock */
+ if (!zcomp_shared_decomp(comp))
+ return zcomp_decompress_buf(comp, zstrm, sg, zstrm->local_copy,
+ &req);
+
+ /* shared mode: try a per-request bounce buffer first */
+ src = kmalloc(src_len, GFP_NOWAIT | __GFP_NOWARN);
+ if (src) {
+ ret = zcomp_decompress_buf(comp, zstrm, sg, src, &req);
+ kfree(src);
+ return ret;
+ }
+
+ /* fall back to ->local_copy serialized by ->excl_lock */
+ mutex_lock(&zstrm->excl_lock);
+ ret = zcomp_decompress_buf(comp, zstrm, sg, zstrm->local_copy, &req);
+ mutex_unlock(&zstrm->excl_lock);
return ret;
}
diff --git a/drivers/block/zram/zcomp.h b/drivers/block/zram/zcomp.h
index bf41b27958c4..362a6da014aa 100644
--- a/drivers/block/zram/zcomp.h
+++ b/drivers/block/zram/zcomp.h
@@ -112,7 +112,7 @@ void zcomp_destroy(struct zcomp *comp);
struct zcomp_cstrm *zcomp_cstrm_get(struct zcomp *comp);
void zcomp_cstrm_put(struct zcomp_cstrm *zstrm);
-bool zcomp_shared_decomp(struct zcomp *comp, struct scatterlist *sg);
+bool zcomp_shared_decomp(struct zcomp *comp);
struct zcomp_dstrm *zcomp_dstrm_get(struct zcomp *comp,
enum zcomp_dstrm_mode mode);
void zcomp_dstrm_put(struct zcomp_dstrm *zstrm, enum zcomp_dstrm_mode mode);
diff --git a/drivers/block/zram/zram_drv.c b/drivers/block/zram/zram_drv.c
index a34859ff0461..b429c52cb37d 100644
--- a/drivers/block/zram/zram_drv.c
+++ b/drivers/block/zram/zram_drv.c
@@ -2128,7 +2128,7 @@ static int read_compressed_page(struct zram *zram, struct page *page,
prio = get_slot_comp_priority(zram, index);
zs_obj_read_sg_begin(zram->mem_pool, handle, sg, size);
- if (zcomp_shared_decomp(zram->comps[prio], sg))
+ if (zcomp_shared_decomp(zram->comps[prio]))
mode = ZCOMP_DSTRM_SHARED;
zstrm = zcomp_dstrm_get(zram->comps[prio], mode);
--
2.56.0.176.g3575e65682-goog