[PATCH v7 11/17] nvme-rdma: Use CCR to recover controller that hits an error
From: Mohamed Khalfella
Date: Fri Nov 13 2026 - 22:07:10 EST
A LIVE controller that hits an error now moves to FENCING instead of
RESETTING. ctrl->fencing_work attempts CCR to terminate inflight IOs.
Regardless of the success or failure of the CCR operation, the
controller then moves to RESETTING to continue error recovery.
A request that times out while the controller is FENCING is not
completed. Its timer is rearmed, since inflight requests must not be
retried before the fence is over. nvme_rdma_stop_ctrl() flushes
fencing_work, so deleting a controller that is fencing waits for the
fence to finish.
Signed-off-by: Mohamed Khalfella <mkhalfella@xxxxxxxxxxxxxxx>
---
drivers/nvme/host/rdma.c | 39 ++++++++++++++++++++++++++++++++++-----
1 file changed, 34 insertions(+), 5 deletions(-)
diff --git a/drivers/nvme/host/rdma.c b/drivers/nvme/host/rdma.c
index 32189fdfc0f5..da1ec2dd6205 100644
--- a/drivers/nvme/host/rdma.c
+++ b/drivers/nvme/host/rdma.c
@@ -119,6 +119,7 @@ struct nvme_rdma_ctrl {
/* other member variables */
struct blk_mq_tag_set tag_set;
+ struct work_struct fencing_work;
struct work_struct err_work;
struct nvme_rdma_qe async_event_sqe;
@@ -992,6 +993,7 @@ static void nvme_rdma_stop_ctrl(struct nvme_ctrl *nctrl)
{
struct nvme_rdma_ctrl *ctrl = to_rdma_ctrl(nctrl);
+ flush_work(&ctrl->fencing_work);
flush_work(&ctrl->err_work);
cancel_delayed_work_sync(&ctrl->reconnect_work);
}
@@ -1153,6 +1155,22 @@ static void nvme_rdma_reconnect_ctrl_work(struct work_struct *work)
nvme_rdma_reconnect_or_remove(ctrl, ret);
}
+static void nvme_rdma_fencing_work(struct work_struct *work)
+{
+ struct nvme_rdma_ctrl *rdma_ctrl = container_of(work,
+ struct nvme_rdma_ctrl, fencing_work);
+ struct nvme_ctrl *ctrl = &rdma_ctrl->ctrl;
+ unsigned long rem;
+
+ rem = nvme_fence_ctrl(ctrl);
+ if (rem)
+ dev_info(ctrl->device, "CCR failed, starting error recovery\n");
+
+ nvme_change_ctrl_state(ctrl, NVME_CTRL_FENCED);
+ if (nvme_change_ctrl_state(ctrl, NVME_CTRL_RESETTING))
+ queue_work(nvme_reset_wq, &rdma_ctrl->err_work);
+}
+
static void nvme_rdma_error_recovery_work(struct work_struct *work)
{
struct nvme_rdma_ctrl *ctrl = container_of(work,
@@ -1180,6 +1198,12 @@ static void nvme_rdma_error_recovery_work(struct work_struct *work)
static void nvme_rdma_error_recovery(struct nvme_rdma_ctrl *ctrl)
{
+ if (nvme_change_ctrl_state(&ctrl->ctrl, NVME_CTRL_FENCING)) {
+ dev_warn(ctrl->ctrl.device, "starting controller fencing\n");
+ queue_work(nvme_wq, &ctrl->fencing_work);
+ return;
+ }
+
if (!nvme_change_ctrl_state(&ctrl->ctrl, NVME_CTRL_RESETTING))
return;
@@ -1991,17 +2015,19 @@ static enum blk_eh_timer_return nvme_rdma_timeout(struct request *rq)
struct nvme_rdma_ctrl *ctrl = queue->ctrl;
struct nvme_command *cmd = req->req.cmd;
int qid = nvme_rdma_queue_idx(queue);
+ enum nvme_ctrl_state state;
dev_warn(ctrl->ctrl.device,
"I/O tag %d (%04x) opcode %#x (%s) QID %d timeout\n",
rq->tag, nvme_cid(rq), cmd->common.opcode,
nvme_fabrics_opcode_str(qid, cmd), qid);
- if (nvme_ctrl_state(&ctrl->ctrl) != NVME_CTRL_LIVE) {
+ state = nvme_ctrl_state(&ctrl->ctrl);
+ if (state != NVME_CTRL_LIVE && state != NVME_CTRL_FENCING) {
/*
- * If we are resetting, connecting or deleting we should
- * complete immediately because we may block controller
- * teardown or setup sequence
+ * If fencing has finished or we are resetting, connecting,
+ * or deleting we should complete immediately because we may
+ * block controller teardown or setup sequence
* - ctrl disable/shutdown fabrics requests
* - connect requests
* - initialization admin requests
@@ -2017,7 +2043,9 @@ static enum blk_eh_timer_return nvme_rdma_timeout(struct request *rq)
/*
* LIVE state should trigger the normal error recovery which will
- * handle completing this request.
+ * handle completing this request. In FENCING state the request must
+ * not be completed before fencing has finished, so just rearm the
+ * timer and let the error recovery pick it up.
*/
nvme_rdma_error_recovery(ctrl);
return BLK_EH_RESET_TIMER;
@@ -2328,6 +2356,7 @@ static struct nvme_rdma_ctrl *nvme_rdma_alloc_ctrl(struct device *dev,
INIT_DELAYED_WORK(&ctrl->reconnect_work,
nvme_rdma_reconnect_ctrl_work);
+ INIT_WORK(&ctrl->fencing_work, nvme_rdma_fencing_work);
INIT_WORK(&ctrl->err_work, nvme_rdma_error_recovery_work);
INIT_WORK(&ctrl->ctrl.reset_work, nvme_rdma_reset_ctrl_work);
--
2.56.0