[PATCH v3 1/1] md: freeze the queue in mddev_suspend()

[Jack Wang]

From: Jack Wang <jinpu.wang@xxxxxxxxxxxxxxx>

queue_attr_store() holds q->limits_lock and waits in
blk_mq_freeze_queue() for q_usage_counter to drain. A bio parked in the
is_suspended() loop of md_handle_request() still holds a reference, so
the freeze never completes. If the suspender waits for reconfig_mutex
held by md_start_sync(), which waits for q->limits_lock, the tasks
deadlock.

Freeze the queue in mddev_suspend() and unfreeze it on resume, so bios
submitted while suspended block in bio_queue_enter() without holding a
reference.

Skip the drain wait when parked bios can only progress after the suspend
completes: writers waiting on the suspend_lo/suspend_hi window and
raid456 IO waiting on an interrupted reshape. dm-raid has no gendisk and
is not frozen.

Fixes: c99f66e4084a ("block: fix queue freeze vs limits lock order in sysfs store methods")
Suggested-by: Yu Kuai <yukuai@xxxxxxx>
Signed-off-by: Jack Wang <jinpu.wang@xxxxxxxxxxxxxxx>
---
 drivers/md/md.c | 45 +++++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 45 insertions(+)

diff --git a/drivers/md/md.c b/drivers/md/md.c
index 03e35386a802..29737a624844 100644
--- a/drivers/md/md.c
+++ b/drivers/md/md.c
@@ -472,6 +472,42 @@ static void md_submit_bio(struct bio *bio)
 	md_handle_request(mddev, bio);
 }
 
+/*
+ * Freeze the queue so a concurrent blk_mq_freeze_queue() (e.g. a limits
+ * update) can't wait on a bio parked in md_handle_request() while suspended.
+ * Skip the drain wait when parked bios hold a reference and only progress
+ * after suspend: the suspend_lo/hi window, or an interrupted raid456 reshape.
+ * Resume may run in another task, hence the non_owner variants.
+ */
+static int mddev_freeze_queue(struct mddev *mddev, bool interruptible)
+{
+	struct request_queue *q;
+
+	if (mddev_is_dm(mddev))
+		return 0;
+
+	q = mddev->gendisk->queue;
+	blk_freeze_queue_start_non_owner(q);
+
+	if (READ_ONCE(mddev->suspend_lo) < READ_ONCE(mddev->suspend_hi) ||
+	    (mddev->pers && mddev->pers->prepare_suspend &&
+	     reshape_interrupted(mddev)))
+		return 0;
+
+	if (!interruptible) {
+		blk_mq_freeze_queue_wait(q);
+		return 0;
+	}
+	return wait_event_interruptible(q->mq_freeze_wq,
+			percpu_ref_is_zero(&q->q_usage_counter));
+}
+
+static void mddev_unfreeze_queue(struct mddev *mddev)
+{
+	if (!mddev_is_dm(mddev))
+		blk_mq_unfreeze_queue_non_owner(mddev->gendisk->queue);
+}
+
 /*
  * Make sure no new requests are submitted to the device, and any requests that
  * have been submitted are completely handled.
@@ -500,6 +536,13 @@ int mddev_suspend(struct mddev *mddev, bool interruptible)
 		return 0;
 	}
 
+	err = mddev_freeze_queue(mddev, interruptible);
+	if (err) {
+		mddev_unfreeze_queue(mddev);
+		mutex_unlock(&mddev->suspend_mutex);
+		return err;
+	}
+
 	percpu_ref_kill(&mddev->active_io);
 
 	/*
@@ -520,6 +563,7 @@ int mddev_suspend(struct mddev *mddev, bool interruptible)
 				percpu_ref_is_zero(&mddev->active_io));
 	if (err) {
 		percpu_ref_resurrect(&mddev->active_io);
+		mddev_unfreeze_queue(mddev);
 		mutex_unlock(&mddev->suspend_mutex);
 		return err;
 	}
@@ -555,6 +599,7 @@ static void __mddev_resume(struct mddev *mddev, bool recovery_needed)
 	md_wakeup_thread(mddev->thread);
 	md_wakeup_thread(mddev->sync_thread); /* possibly kick off a reshape */
 
+	mddev_unfreeze_queue(mddev);
 	mutex_unlock(&mddev->suspend_mutex);
 }
 
-- 
2.43.0