[PATCH] blk-mq: mark request tag accesses racing with busy iteration

[Zhu Jun]

Busy tag iteration can encounter a request retained in tags->rqs after
it releases its driver tag. The tag comparison in blk_mq_find_and_get_req()
therefore races with driver tag allocation and release, as KCSAN reports.

Mark the iterator's tag snapshot with READ_ONCE() and the corresponding
tag stores with WRITE_ONCE(), including the flush request stores. The
existing request reference and tag-set lifetime rules continue to govern
whether the request can be passed to the iterator callback.

Fixes: 67f3b2f822b7 ("blk-mq: avoid to iterate over stale request")
Signed-off-by: Zhu Jun <zhujun2@xxxxxxxxxxxxxxxxxxxx>
---
 block/blk-flush.c  | 4 ++--
 block/blk-mq-tag.c | 2 +-
 block/blk-mq.c     | 8 ++++----
 block/blk-mq.h     | 2 +-
 4 files changed, 8 insertions(+), 8 deletions(-)

diff --git a/block/blk-flush.c b/block/blk-flush.c
index 38bd4ae5d9e7..f8cea46775db 100644
--- a/block/blk-flush.c
+++ b/block/blk-flush.c
@@ -236,7 +236,7 @@ static enum rq_end_io_ret flush_end_io(struct request *flush_rq,
 	}
 
 	if (!q->elevator) {
-		flush_rq->tag = BLK_MQ_NO_TAG;
+		WRITE_ONCE(flush_rq->tag, BLK_MQ_NO_TAG);
 	} else {
 		blk_mq_put_driver_tag(flush_rq);
 		flush_rq->internal_tag = BLK_MQ_NO_TAG;
@@ -317,7 +317,7 @@ static void blk_kick_flush(struct request_queue *q, struct blk_flush_queue *fq,
 	flush_rq->mq_hctx = first_rq->mq_hctx;
 
 	if (!q->elevator)
-		flush_rq->tag = first_rq->tag;
+		WRITE_ONCE(flush_rq->tag, first_rq->tag);
 	else
 		flush_rq->internal_tag = first_rq->internal_tag;
 
diff --git a/block/blk-mq-tag.c b/block/blk-mq-tag.c
index 3515031b3226..84eea85f9217 100644
--- a/block/blk-mq-tag.c
+++ b/block/blk-mq-tag.c
@@ -271,7 +271,7 @@ static struct request *blk_mq_find_and_get_req(struct blk_mq_tags *tags,
 	struct request *rq;
 
 	rq = tags->rqs[bitnr];
-	if (!rq || rq->tag != bitnr || !req_ref_inc_not_zero(rq))
+	if (!rq || READ_ONCE(rq->tag) != bitnr || !req_ref_inc_not_zero(rq))
 		rq = NULL;
 	return rq;
 }
diff --git a/block/blk-mq.c b/block/blk-mq.c
index 5ae22d556540..95de1e4d015f 100644
--- a/block/blk-mq.c
+++ b/block/blk-mq.c
@@ -387,7 +387,7 @@ void blk_rq_init(struct request_queue *q, struct request *rq)
 	rq->phys_gap_bit = 0;
 	INIT_HLIST_NODE(&rq->hash);
 	RB_CLEAR_NODE(&rq->rb_node);
-	rq->tag = BLK_MQ_NO_TAG;
+	WRITE_ONCE(rq->tag, BLK_MQ_NO_TAG);
 	rq->internal_tag = BLK_MQ_NO_TAG;
 	rq->start_time_ns = blk_time_get_ns();
 	blk_crypto_rq_set_defaults(rq);
@@ -432,10 +432,10 @@ static struct request *blk_mq_rq_ctx_init(struct blk_mq_alloc_data *data,
 	rq->rq_flags = data->rq_flags;
 
 	if (data->rq_flags & RQF_SCHED_TAGS) {
-		rq->tag = BLK_MQ_NO_TAG;
+		WRITE_ONCE(rq->tag, BLK_MQ_NO_TAG);
 		rq->internal_tag = tag;
 	} else {
-		rq->tag = tag;
+		WRITE_ONCE(rq->tag, tag);
 		rq->internal_tag = BLK_MQ_NO_TAG;
 	}
 	rq->timeout = 0;
@@ -1877,7 +1877,7 @@ bool __blk_mq_alloc_driver_tag(struct request *rq)
 	if (tag == BLK_MQ_NO_TAG)
 		return false;
 
-	rq->tag = tag + tag_offset;
+	WRITE_ONCE(rq->tag, tag + tag_offset);
 	blk_mq_inc_active_requests(rq->mq_hctx);
 	return true;
 }
diff --git a/block/blk-mq.h b/block/blk-mq.h
index 1e64236d34d4..b21fba720225 100644
--- a/block/blk-mq.h
+++ b/block/blk-mq.h
@@ -358,7 +358,7 @@ static inline void __blk_mq_put_driver_tag(struct blk_mq_hw_ctx *hctx,
 {
 	blk_mq_dec_active_requests(hctx);
 	blk_mq_put_tag(hctx->tags, rq->mq_ctx, rq->tag);
-	rq->tag = BLK_MQ_NO_TAG;
+	WRITE_ONCE(rq->tag, BLK_MQ_NO_TAG);
 }
 
 static inline void blk_mq_put_driver_tag(struct request *rq)
-- 
2.51.0