[RFC PATCH v2 4/4] blk-cgroup: move async bio punt state to blkcg
From: Yu Kuai
Date: Tue Aug 11 2026 - 03:12:45 EST
From: Yu Kuai <yukuai@xxxxxxx>
blkcg_punt_bio_submit() currently queues punted bios on blkg->async_bios,
so it has to call bio_blkg() to find or create a queue-local blkg. Bios
now carry and pin the blkcg css, so punted bio lifetime no longer needs to
be anchored by a blkg.
Keeping the punt state in blkg can instantiate a blkg even when no blkcg
policy is enabled, just to bounce submission from a shared kthread. Move
async_bio_lock, async_bios and async_bio_work to struct blkcg, and queue
punted bios on bio_blkcg() for non-root cgroups. Root or unassociated bios
are submitted directly.
This preserves the priority-inversion avoidance while preventing
blkcg_punt_bio_submit() from creating blkgs that are not needed by any
policy.
Signed-off-by: Yu Kuai <yukuai@xxxxxxx>
---
block/blk-cgroup.c | 44 +++++++++++++++++++++-----------------------
block/blk-cgroup.h | 14 ++++++--------
2 files changed, 27 insertions(+), 31 deletions(-)
diff --git a/block/blk-cgroup.c b/block/blk-cgroup.c
index 1de2f2433d11..e76e9a52f95d 100644
--- a/block/blk-cgroup.c
+++ b/block/blk-cgroup.c
@@ -177,14 +177,10 @@ static void blkg_free(struct blkcg_gq *blkg)
static void __blkg_release(struct rcu_head *rcu)
{
struct blkcg_gq *blkg = container_of(rcu, struct blkcg_gq, rcu_head);
-#ifdef CONFIG_BLK_CGROUP_PUNT_BIO
- WARN_ON(!bio_list_empty(&blkg->async_bios));
-#endif
-
blkg_free(blkg);
}
/*
* A group is RCU protected, but having an rcu lock does not mean that one
@@ -218,23 +214,22 @@ static void blkg_release(struct percpu_ref *ref)
}
#ifdef CONFIG_BLK_CGROUP_PUNT_BIO
static struct workqueue_struct *blkcg_punt_bio_wq;
-static void blkg_async_bio_workfn(struct work_struct *work)
+static void blkcg_async_bio_workfn(struct work_struct *work)
{
- struct blkcg_gq *blkg = container_of(work, struct blkcg_gq,
- async_bio_work);
+ struct blkcg *blkcg = container_of(work, struct blkcg, async_bio_work);
struct bio_list bios = BIO_EMPTY_LIST;
struct bio *bio;
struct blk_plug plug;
bool need_plug = false;
- /* as long as there are pending bios, @blkg can't go away */
- spin_lock(&blkg->async_bio_lock);
- bio_list_merge_init(&bios, &blkg->async_bios);
- spin_unlock(&blkg->async_bio_lock);
+ /* as long as there are pending bios, @blkcg can't go away */
+ spin_lock(&blkcg->async_bio_lock);
+ bio_list_merge_init(&bios, &blkcg->async_bios);
+ spin_unlock(&blkcg->async_bio_lock);
/* start plug only when bio_list contains at least 2 bios */
if (bios.head && bios.head->bi_next) {
need_plug = true;
blk_start_plug(&plug);
@@ -251,19 +246,19 @@ static void blkg_async_bio_workfn(struct work_struct *work)
* cgroup. Use this helper instead of submit_bio to punt the actual issuing to
* a dedicated per-blkcg work item to avoid such priority inversions.
*/
void blkcg_punt_bio_submit(struct bio *bio)
{
- struct blkcg_gq *blkg = bio_blkg(bio);
+ struct blkcg *blkcg = bio_blkcg(bio);
- if (blkg && blkg->parent) {
- spin_lock(&blkg->async_bio_lock);
- bio_list_add(&blkg->async_bios, bio);
- spin_unlock(&blkg->async_bio_lock);
- queue_work(blkcg_punt_bio_wq, &blkg->async_bio_work);
+ if (blkcg && cgroup_parent(blkcg->css.cgroup)) {
+ spin_lock(&blkcg->async_bio_lock);
+ bio_list_add(&blkcg->async_bios, bio);
+ spin_unlock(&blkcg->async_bio_lock);
+ queue_work(blkcg_punt_bio_wq, &blkcg->async_bio_work);
} else {
- /* Never bounce if there is no non-root blkg to queue on. */
+ /* Never bounce if there is no non-root blkcg to queue on. */
submit_bio(bio);
}
}
EXPORT_SYMBOL_GPL(blkcg_punt_bio_submit);
@@ -342,15 +337,10 @@ static struct blkcg_gq *blkg_alloc(struct blkcg *blkcg, struct gendisk *disk,
blkg->q = disk->queue;
INIT_LIST_HEAD(&blkg->q_node);
blkg->blkcg = blkcg;
blkg->blkcg_id = blkcg->css.id;
blkg->iostat.blkg = blkg;
-#ifdef CONFIG_BLK_CGROUP_PUNT_BIO
- spin_lock_init(&blkg->async_bio_lock);
- bio_list_init(&blkg->async_bios);
- INIT_WORK(&blkg->async_bio_work, blkg_async_bio_workfn);
-#endif
u64_stats_init(&blkg->iostat.sync);
for_each_possible_cpu(cpu) {
u64_stats_init(&per_cpu_ptr(blkg->iostat_cpu, cpu)->sync);
per_cpu_ptr(blkg->iostat_cpu, cpu)->blkg = blkg;
@@ -1370,10 +1360,13 @@ static void blkcg_css_free(struct cgroup_subsys_state *css)
if (blkcg->cpd[i])
blkcg_policy[i]->cpd_free_fn(blkcg->cpd[i]);
mutex_unlock(&blkcg_pol_mutex);
+#ifdef CONFIG_BLK_CGROUP_PUNT_BIO
+ WARN_ON(!bio_list_empty(&blkcg->async_bios));
+#endif
free_percpu(blkcg->lhead);
kfree(blkcg);
}
static struct cgroup_subsys_state *
@@ -1418,10 +1411,15 @@ blkcg_css_alloc(struct cgroup_subsys_state *parent_css)
}
spin_lock_init(&blkcg->lock);
refcount_set(&blkcg->online_pin, 1);
INIT_HLIST_HEAD(&blkcg->blkg_list);
+#ifdef CONFIG_BLK_CGROUP_PUNT_BIO
+ spin_lock_init(&blkcg->async_bio_lock);
+ bio_list_init(&blkcg->async_bios);
+ INIT_WORK(&blkcg->async_bio_work, blkcg_async_bio_workfn);
+#endif
#ifdef CONFIG_CGROUP_WRITEBACK
INIT_LIST_HEAD(&blkcg->cgwb_list);
#endif
list_add_tail(&blkcg->all_blkcgs_node, &all_blkcgs);
diff --git a/block/blk-cgroup.h b/block/blk-cgroup.h
index 3d113f6f6d25..efef0967d39e 100644
--- a/block/blk-cgroup.h
+++ b/block/blk-cgroup.h
@@ -74,18 +74,11 @@ struct blkcg_gq {
struct blkg_iostat_set __percpu *iostat_cpu;
struct blkg_iostat_set iostat;
struct blkg_policy_data *pd[BLKCG_MAX_POLS];
-#ifdef CONFIG_BLK_CGROUP_PUNT_BIO
- spinlock_t async_bio_lock;
- struct bio_list async_bios;
-#endif
- union {
- struct work_struct async_bio_work;
- struct work_struct free_work;
- };
+ struct work_struct free_work;
atomic_t use_delay;
atomic64_t delay_nsec;
atomic64_t delay_start;
u64 last_delay;
@@ -110,10 +103,15 @@ struct blkcg {
/*
* List of updated percpu blkg_iostat_set's since the last flush.
*/
struct llist_head __percpu *lhead;
+#ifdef CONFIG_BLK_CGROUP_PUNT_BIO
+ spinlock_t async_bio_lock; /* protects async_bios */
+ struct bio_list async_bios;
+ struct work_struct async_bio_work;
+#endif
#ifdef CONFIG_BLK_CGROUP_FC_APPID
char fc_app_id[FC_APPID_LEN];
#endif
#ifdef CONFIG_CGROUP_WRITEBACK
struct list_head cgwb_list;
--
2.51.0