[PATCH v4 1/2] block: suspend bio based disks around queue limits updates

[Jack Wang]

queue_attr_store() freezes the queue under q->limits_lock. For md this can
deadlock: a bio parked while the array is suspended still holds a
q_usage_counter reference, so the freeze waits for it, while the suspender
waits for reconfig_mutex held by md_start_sync(), which waits for
q->limits_lock.

Add optional ->suspend() and ->resume() block_device_operations. When
present, queue_attr_store() and the integrity flag_store() suspend the disk
before taking q->limits_lock and commit the limits without freezing the
queue. Other disks behave as before.

Fixes: c99f66e4084a ("block: fix queue freeze vs limits lock order in sysfs store methods")
Suggested-by: Yu Kuai <yukuai@xxxxxxx>
Signed-off-by: Jack Wang <jinpu.wang@xxxxxxxxxxxxxxx>
---
 block/blk-integrity.c  | 10 ++++++++--
 block/blk-sysfs.c      | 19 +++++++++++++------
 block/blk.h            | 20 ++++++++++++++++++++
 include/linux/blkdev.h |  7 +++++++
 4 files changed, 48 insertions(+), 8 deletions(-)

diff --git a/block/blk-integrity.c b/block/blk-integrity.c
index 964eebbee14d..b449e7c04eea 100644
--- a/block/blk-integrity.c
+++ b/block/blk-integrity.c
@@ -226,7 +226,8 @@ EXPORT_SYMBOL_GPL(blk_integrity_profile_name);
 static ssize_t flag_store(struct device *dev, const char *page, size_t count,
 		unsigned char flag)
 {
-	struct request_queue *q = dev_to_disk(dev)->queue;
+	struct gendisk *disk = dev_to_disk(dev);
+	struct request_queue *q = disk->queue;
 	struct queue_limits lim;
 	unsigned long val;
 	int err;
@@ -235,6 +236,10 @@ static ssize_t flag_store(struct device *dev, const char *page, size_t count,
 	if (err)
 		return err;
 
+	err = blk_disk_suspend(disk);
+	if (err)
+		return err;
+
 	/* note that the flags are inverted vs the values in the sysfs files */
 	lim = queue_limits_start_update(q);
 	if (val)
@@ -242,7 +247,8 @@ static ssize_t flag_store(struct device *dev, const char *page, size_t count,
 	else
 		lim.integrity.flags |= flag;
 
-	err = queue_limits_commit_update_frozen(q, &lim);
+	err = blk_disk_commit_limits(disk, &lim);
+	blk_disk_resume(disk);
 	if (err)
 		return err;
 	return count;
diff --git a/block/blk-sysfs.c b/block/blk-sysfs.c
index 520972676ab4..3cbe36d04699 100644
--- a/block/blk-sysfs.c
+++ b/block/blk-sysfs.c
@@ -887,20 +887,27 @@ queue_attr_store(struct kobject *kobj, struct attribute *attr,
 		return -EIO;
 
 	if (entry->store_limit) {
+		struct queue_limits lim;
 		ssize_t res;
 
-		struct queue_limits lim = queue_limits_start_update(q);
+		res = blk_disk_suspend(disk);
+		if (res)
+			return res;
+
+		lim = queue_limits_start_update(q);
 
 		res = entry->store_limit(disk, page, length, &lim);
 		if (res < 0) {
 			queue_limits_cancel_update(q);
-			return res;
+			goto out_resume;
 		}
 
-		res = queue_limits_commit_update_frozen(q, &lim);
-		if (res)
-			return res;
-		return length;
+		res = blk_disk_commit_limits(disk, &lim);
+		if (!res)
+			res = length;
+out_resume:
+		blk_disk_resume(disk);
+		return res;
 	}
 
 	return entry->store(disk, page, length);
diff --git a/block/blk.h b/block/blk.h
index ed2172472622..1c214f0bc178 100644
--- a/block/blk.h
+++ b/block/blk.h
@@ -661,6 +661,26 @@ void drop_partition(struct block_device *part);
 
 void bdev_set_nr_sectors(struct block_device *bdev, sector_t sectors);
 
+/* Suspend before queue_limits_start_update(), never under q->limits_lock. */
+static inline int blk_disk_suspend(struct gendisk *disk)
+{
+	return disk->fops->suspend ? disk->fops->suspend(disk) : 0;
+}
+
+static inline void blk_disk_resume(struct gendisk *disk)
+{
+	if (disk->fops->suspend)
+		disk->fops->resume(disk);
+}
+
+static inline int blk_disk_commit_limits(struct gendisk *disk,
+					 struct queue_limits *lim)
+{
+	if (disk->fops->suspend)
+		return queue_limits_commit_update(disk->queue, lim);
+	return queue_limits_commit_update_frozen(disk->queue, lim);
+}
+
 struct gendisk *__alloc_disk_node(struct request_queue *q, int node_id,
 		struct lock_class_key *lkclass);
 struct request_queue *blk_alloc_queue(struct queue_limits *lim, int node_id);
diff --git a/include/linux/blkdev.h b/include/linux/blkdev.h
index 0c2010230632..0011b7c0740f 100644
--- a/include/linux/blkdev.h
+++ b/include/linux/blkdev.h
@@ -1601,6 +1601,13 @@ struct block_device_operations {
 			enum blk_unique_id id_type);
 	int (*init_dma_buf_io_ctx)(struct block_device *,
 				   struct dma_buf_io_ctx *);
+	/*
+	 * Optional, for bio based drivers that cannot be frozen under
+	 * q->limits_lock: stop I/O around a sysfs limits update.  Called
+	 * before q->limits_lock is taken, with no queue frozen.
+	 */
+	int (*suspend)(struct gendisk *disk);
+	void (*resume)(struct gendisk *disk);
 	struct module *owner;
 	const struct pr_ops *pr_ops;
 
-- 
2.43.0