[PATCH 2/2] block/blk-iolatency: Fix inflight decrement for chained bio sub-bios

[Li kunyu]

Implement the blkcg_iolatency_done_split_bio callback in blk-iolatency to
correctly decrement the inflight count when the remaining bio headers
complete, excluding the one for which atomic_dec_and_test(&bio->
__bi_remaining) is true.

Signed-off-by: Li kunyu <likunyu@xxxxxxxxxxxxxx>
---
 block/blk-iolatency.c | 42 ++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 42 insertions(+)

diff --git a/block/blk-iolatency.c b/block/blk-iolatency.c
index 2caa79a008ad..e1ef3bb10572 100644
--- a/block/blk-iolatency.c
+++ b/block/blk-iolatency.c
@@ -632,6 +632,47 @@ static void blkcg_iolatency_done_bio(struct rq_qos *rqos, struct bio *bio)
 	}
 }
 
+static void blkcg_iolatency_done_split_bio(struct rq_qos *rqos, struct bio *bio)
+{
+	struct blkcg_gq *blkg;
+	struct rq_wait *rqw;
+	struct iolatency_grp *iolat;
+	int inflight = 0;
+
+	blkg = bio->bi_blkg;
+	if (!blkg || !bio_flagged(bio, BIO_QOS_THROTTLED) ||
+	    !bio_flagged(bio, BIO_QOS_CHAIN_CHILD))
+		return;
+
+	iolat = blkg_to_lat(bio->bi_blkg);
+	if (!iolat)
+		return;
+
+	if (!iolat->blkiolat->enabled)
+		return;
+
+	while (blkg && blkg->parent) {
+		iolat = blkg_to_lat(blkg);
+		if (!iolat) {
+			blkg = blkg->parent;
+			continue;
+		}
+		rqw = &iolat->rq_wait;
+
+		inflight = atomic_dec_return(&rqw->inflight);
+		WARN_ON_ONCE(inflight < 0);
+
+		/*
+		 * iolat->min_lat_nsec and other window-time equivalents
+		 * are not calculated here; they are calculated only when
+		 * blkcg_iolatency_done_bio() is called at the end of the
+		 * bio header.
+		 */
+		wake_up(&rqw->wait);
+		blkg = blkg->parent;
+	}
+}
+
 static void blkcg_iolatency_exit(struct rq_qos *rqos)
 {
 	struct blk_iolatency *blkiolat = BLKIOLATENCY(rqos);
@@ -645,6 +686,7 @@ static void blkcg_iolatency_exit(struct rq_qos *rqos)
 static const struct rq_qos_ops blkcg_iolatency_ops = {
 	.throttle = blkcg_iolatency_throttle,
 	.done_bio = blkcg_iolatency_done_bio,
+	.done_split_bio = blkcg_iolatency_done_split_bio,
 	.exit = blkcg_iolatency_exit,
 };
 
-- 
2.47.3