[PATCH v3 22/24] dmaengine: dw-edma: Add engine recovery infrastructure

[Koichiro Den]

For eDMA, resetting one direction affects every channel in it. Add a
worker to coordinate that recovery.

Gate publication and doorbells, and prevent channel setup from
re-enabling the direction during recovery. Wait for active LL channels
to stop. If a transfer is still active, compare two transfer-size
samples. Retry when it is moving; treat an unchanged value as frozen.

For a frozen direction, drain IRQ handling so pending ABORT cleanup can
finish. If recovery is still needed, reset the direction, rebuild every
exposed LL context while it remains disabled, publish remote LL writes,
and enable it again. Honor EDMA_REQ_STOP and EDMA_REQ_PAUSE during the
rebuild; leave channels gated if reset keeps failing.

Providers opt in with engine_reset(), engine_enable(), and
ch_transfer_size(). No caller requests recovery until the next patch.

Signed-off-by: Koichiro Den <den@xxxxxxxxxxxxx>
---
Changes in v3:
  - Drain pending IRQ and ABORT handling before and after reset.
  - Complete remote LL publication before re-enabling the direction.
  - Keep the direction disabled while rebuilding every exposed LL
    context, including those of unconfigured channels.
  - Queue every recovery request and stop recovery before hardware
    teardown. (Sashiko)
  - Avoid mixing scope-based lock guards with goto cleanup. (Sashiko)
  - Trim recovery and teardown details from the commit message.

 drivers/dma/dw-edma/dw-edma-core.c | 304 ++++++++++++++++++++++++++++-
 drivers/dma/dw-edma/dw-edma-core.h |  22 ++-
 2 files changed, 321 insertions(+), 5 deletions(-)

diff --git a/drivers/dma/dw-edma/dw-edma-core.c b/drivers/dma/dw-edma/dw-edma-core.c
index 98b8732a6cb0..56032137a4bb 100644
--- a/drivers/dma/dw-edma/dw-edma-core.c
+++ b/drivers/dma/dw-edma/dw-edma-core.c
@@ -28,6 +28,9 @@
 /* Empirically chosen progress interval. */
 #define DW_EDMA_LL_PROGRESS_INTERVAL	4
 #define DW_EDMA_LL_RECHECK_DELAY_MS		30
+#define DW_EDMA_ENGINE_QUIESCE_TIMEOUT_MS	250
+#define DW_EDMA_ENGINE_RESET_MAX_FAILS		5
+#define DW_EDMA_MAX_DIR_CH			MAX(HDMA_MAX_WR_CH, HDMA_MAX_RD_CH)
 
 static inline
 struct dw_edma_desc *vd2dw_edma_desc(struct virt_dma_desc *vd)
@@ -186,6 +189,15 @@ dw_edma_set_request(struct dw_edma_chan *chan, enum dw_edma_request request)
 	chan->request = request;
 }
 
+static void
+dw_edma_engine_recovery_queue(struct dw_edma_engine_recovery *rec)
+{
+	if (unlikely(READ_ONCE(rec->dw->teardown)))
+		return;
+
+	queue_work(rec->dw->wq, &rec->work);
+}
+
 static void dw_hdma_set_callback_result(struct virt_dma_desc *vd,
 					enum dmaengine_tx_result result)
 {
@@ -238,6 +250,7 @@ static void dw_edma_core_reset_ll(struct dw_edma_chan *chan)
 
 	chan->ll_head = 0;
 	chan->ll_done = 0;
+	chan->ll_recovery_pending = false;
 	/* Drop stale CB bits before reusing the circular LL ring. */
 	for (i = 0; i < chan->ll_max; i++)
 		dw_edma_core_ll_clear(chan, i);
@@ -398,7 +411,7 @@ static int dw_edma_start_transfer(struct dw_edma_chan *chan)
 		return 1;
 	}
 
-	if (dw_edma_abort_is_pending(chan))
+	if (chan->ll_recovering || dw_edma_abort_is_pending(chan))
 		return 0;
 
 	if (!chan->ll_valid)
@@ -548,8 +561,10 @@ static bool dw_edma_ll_clean_pending(struct dw_edma_chan *chan, int idx)
 	WARN_ON_ONCE(done);
 
 out:
-	if (advanced)
+	if (advanced) {
 		dw_edma_ll_recheck_cancel(chan);
+		chan->ll_recovery_pending = false;
+	}
 
 	return advanced;
 }
@@ -706,6 +721,7 @@ static void dw_edma_core_ch_kick(struct dw_edma_chan *chan)
 static bool dw_edma_core_ch_maybe_doorbell(struct dw_edma_chan *chan)
 {
 	if (chan->non_ll ||
+	    chan->ll_recovering ||
 	    chan->status != EDMA_ST_BUSY ||
 	    !dw_edma_ll_pending(chan) ||
 	    dw_edma_abort_is_pending(chan))
@@ -782,6 +798,279 @@ static void dw_edma_ll_recheck_work(struct work_struct *work)
 	dw_edma_core_ch_maybe_doorbell(chan);
 }
 
+static bool
+dw_edma_engine_recovery_needed(struct dw_edma_engine_recovery *rec)
+{
+	struct dw_edma *dw = rec->dw;
+	u16 off = rec->dir == EDMA_DIR_WRITE ? 0 : dw->wr_ch_cnt;
+	u16 cnt = rec->dir == EDMA_DIR_WRITE ? dw->wr_ch_cnt : dw->rd_ch_cnt;
+	struct dw_edma_chan *chan;
+	bool found = false;
+	bool needed;
+	u16 i;
+
+	for (i = 0; i < cnt; i++) {
+		chan = &dw->chan[off + i];
+		scoped_guard(spinlock_irqsave, &chan->vc.lock) {
+			needed = chan->ll_recovery_pending && chan->configured &&
+				 !chan->non_ll &&
+				 chan->status != EDMA_ST_PAUSE;
+			if (needed && chan->request == EDMA_REQ_NONE &&
+			    dw_edma_ll_reconcile_and_refill(chan))
+				dw_edma_core_ch_maybe_doorbell(chan);
+			needed = needed && chan->ll_recovery_pending &&
+				 dw_edma_ll_pending(chan);
+			if (!needed)
+				chan->ll_recovery_pending = false;
+		}
+		found |= needed;
+	}
+
+	return found;
+}
+
+static void
+dw_edma_engine_recovery_release(struct dw_edma_engine_recovery *rec)
+{
+	struct dw_edma *dw = rec->dw;
+	u16 off = rec->dir == EDMA_DIR_WRITE ? 0 : dw->wr_ch_cnt;
+	u16 cnt = rec->dir == EDMA_DIR_WRITE ? dw->wr_ch_cnt : dw->rd_ch_cnt;
+	struct dw_edma_chan *chan;
+	u16 i;
+
+	rec->fails = 0;
+
+	for (i = 0; i < cnt; i++) {
+		chan = &dw->chan[off + i];
+		guard(spinlock_irqsave)(&chan->vc.lock);
+		chan->ll_recovering = false;
+
+		if (!READ_ONCE(dw->teardown) &&
+		    chan->configured && !chan->non_ll &&
+		    chan->request == EDMA_REQ_NONE &&
+		    chan->status != EDMA_ST_PAUSE) {
+			dw_edma_start_transfer(chan);
+			chan->status = dw_edma_ll_pending(chan) ?
+				       EDMA_ST_BUSY : EDMA_ST_IDLE;
+			dw_edma_core_ch_maybe_doorbell(chan);
+		}
+	}
+
+	WRITE_ONCE(rec->active, false);
+}
+
+static void
+dw_edma_engine_recovery_drain_irqs(struct dw_edma_engine_recovery *rec)
+{
+	struct dw_edma *dw = rec->dw;
+	struct device *dev = dw->chip->dev;
+	u16 off = rec->dir == EDMA_DIR_WRITE ? 0 : dw->wr_ch_cnt;
+	u16 cnt = rec->dir == EDMA_DIR_WRITE ? dw->wr_ch_cnt : dw->rd_ch_cnt;
+	struct dw_edma_chan *chan;
+	u16 i;
+
+	/* Drain events observed before the engine context changes. */
+	for (i = 0; i < dw->nr_irqs; i++) {
+		if (bitmap_empty(rec->dir == EDMA_DIR_WRITE ?
+				 dw->irq[i].wr_mask : dw->irq[i].rd_mask, cnt))
+			continue;
+
+		synchronize_irq(dw->chip->ops->irq_vector(dev, i));
+	}
+
+	for (i = 0; i < cnt; i++) {
+		chan = &dw->chan[off + i];
+		flush_work(&chan->irq_work);
+	}
+}
+
+static void dw_edma_engine_recovery_work(struct work_struct *work)
+{
+	struct dw_edma_engine_recovery *rec =
+		container_of(work, struct dw_edma_engine_recovery, work);
+	struct dw_edma *dw = rec->dw;
+	const char *dir_name = str_write_read(rec->dir == EDMA_DIR_WRITE);
+	u16 off = rec->dir == EDMA_DIR_WRITE ? 0 : dw->wr_ch_cnt;
+	u16 cnt = rec->dir == EDMA_DIR_WRITE ? dw->wr_ch_cnt : dw->rd_ch_cnt;
+	unsigned long timeout =
+		jiffies + msecs_to_jiffies(DW_EDMA_ENGINE_QUIESCE_TIMEOUT_MS);
+	struct dw_edma_chan *chan;
+	bool configured_ll;
+	bool busy;
+	u16 i;
+
+	/*
+	 * Recovery resets a whole direction. The legacy eDMA register map does
+	 * not support partial ownership, while unrolled eDMA partial ownership
+	 * and delegation are direction-granular. Native HDMA permits per-channel
+	 * ownership, but does not provide direction recovery operations.
+	 */
+	if (unlikely(READ_ONCE(dw->teardown)))
+		return;
+
+	if (!dw_edma_engine_recovery_needed(rec)) {
+		if (READ_ONCE(rec->active))
+			dw_edma_engine_recovery_release(rec);
+		return;
+	}
+
+	WRITE_ONCE(rec->active, true);
+
+	/* Gate each channel before inspecting or resetting the direction. */
+	for (i = 0; i < cnt; i++) {
+		chan = &dw->chan[off + i];
+		guard(spinlock_irqsave)(&chan->vc.lock);
+		chan->ll_recovering = true;
+	}
+
+	/*
+	 * Gated channels stop at their first unpublished element. Wait for that
+	 * point before resetting; otherwise a restart could skip the remainder
+	 * of an in-flight element.
+	 */
+	do {
+		busy = false;
+		for (i = 0; i < cnt; i++) {
+			chan = &dw->chan[off + i];
+			scoped_guard(spinlock_irqsave, &chan->vc.lock)
+				configured_ll = chan->configured && !chan->non_ll;
+			if (configured_ll &&
+			    dw_edma_core_ch_status(chan) == DMA_IN_PROGRESS)
+				busy = true;
+		}
+		if (!busy)
+			break;
+		fsleep(1000);
+	} while (time_before(jiffies, timeout));
+
+	/* Progress after queueing makes this recovery request obsolete. */
+	if (!dw_edma_engine_recovery_needed(rec)) {
+		dw_edma_engine_recovery_release(rec);
+		return;
+	}
+
+	if (busy) {
+		DECLARE_BITMAP(busy_mask, DW_EDMA_MAX_DIR_CH) = { 0 };
+		u32 tsz[DW_EDMA_MAX_DIR_CH];
+		bool moving = false;
+
+		/*
+		 * A transfer_size change or a transition out of DMA_IN_PROGRESS
+		 * shows movement after the first sample. Retry while producers
+		 * remain gated. If neither occurs, treat the channel as frozen
+		 * within one element.
+		 */
+		for (i = 0; i < cnt; i++) {
+			chan = &dw->chan[off + i];
+			scoped_guard(spinlock_irqsave, &chan->vc.lock)
+				configured_ll = chan->configured && !chan->non_ll;
+			if (configured_ll &&
+			    dw_edma_core_ch_status(chan) == DMA_IN_PROGRESS) {
+				__set_bit(i, busy_mask);
+				tsz[i] = dw_edma_core_ch_transfer_size(chan);
+			}
+		}
+		fsleep(1000);
+		for_each_set_bit(i, busy_mask, cnt) {
+			chan = &dw->chan[off + i];
+			if (dw_edma_core_ch_status(chan) != DMA_IN_PROGRESS ||
+			    tsz[i] != dw_edma_core_ch_transfer_size(chan))
+				moving = true;
+		}
+		if (!dw_edma_engine_recovery_needed(rec)) {
+			dw_edma_engine_recovery_release(rec);
+			return;
+		}
+		if (moving) {
+			dev_warn_ratelimited(dw->chip->dev,
+					     "%s engine quiesce timed out with transfers still progressing, retrying\n",
+					     dir_name);
+			dw_edma_engine_recovery_queue(rec);
+			return;
+		}
+
+		dev_warn(dw->chip->dev,
+			 "%s engine reset with a channel frozen mid-element\n",
+			 dir_name);
+	}
+
+	if (!dw_edma_engine_recovery_needed(rec)) {
+		dw_edma_engine_recovery_release(rec);
+		return;
+	}
+
+	/*
+	 * Let a terminal ABORT finish its normal channel cleanup before a
+	 * direction-wide reset. It may also make recovery unnecessary.
+	 */
+	dw_edma_engine_recovery_drain_irqs(rec);
+	if (!dw_edma_engine_recovery_needed(rec)) {
+		dw_edma_engine_recovery_release(rec);
+		return;
+	}
+
+	if (!dw->core->engine_reset(dw, rec->dir)) {
+		/*
+		 * Keep channels gated and do not re-enable the engine unless
+		 * ENGINE_EN clears.
+		 */
+		if (++rec->fails >= DW_EDMA_ENGINE_RESET_MAX_FAILS) {
+			dev_err(dw->chip->dev,
+				"%s engine did not drain after %u attempts; leaving channels gated\n",
+				dir_name, rec->fails);
+			return;
+		}
+		dw_edma_engine_recovery_queue(rec);
+		return;
+	}
+
+	if (rec->fails)
+		dev_warn(dw->chip->dev, "%s engine drained after %u retries\n",
+			 dir_name, rec->fails);
+
+	/* Drain an event that raced with the engine reset. */
+	dw_edma_engine_recovery_drain_irqs(rec);
+
+	/*
+	 * Keep the engine disabled until every LL channel context is rebuilt.
+	 * On the tested integration, re-enabling a preserved context resumed an
+	 * old element without a doorbell, causing an IOMMU fault and stalling the
+	 * engine again. Reset each ring so it starts from a fresh context, and
+	 * complete remote LL publication before re-enabling the direction.
+	 */
+	for (i = 0; i < cnt; i++) {
+		bool stop;
+
+		chan = &dw->chan[off + i];
+		guard(spinlock_irqsave)(&chan->vc.lock);
+
+		if (chan->non_ll)
+			continue;
+
+		/* Honor EDMA_REQ_STOP and EDMA_REQ_PAUSE instead of republishing. */
+		stop = chan->request == EDMA_REQ_STOP;
+		if (stop)
+			dw_edma_terminate_all_descs(chan);
+
+		dw_edma_core_reset_ll(chan);
+		dw_edma_core_ll_sync(chan);
+
+		if (stop) {
+			dw_edma_set_request(chan, EDMA_REQ_NONE);
+			chan->status = EDMA_ST_IDLE;
+		} else if (chan->request == EDMA_REQ_PAUSE) {
+			dw_edma_set_request(chan, EDMA_REQ_NONE);
+			chan->status = EDMA_ST_PAUSE;
+		}
+	}
+
+	dw->core->engine_enable(dw, rec->dir);
+	dw_edma_engine_recovery_release(rec);
+
+	dev_warn(dw->chip->dev, "%s engine was reset to recover a stalled channel\n",
+		 dir_name);
+}
+
 static void dw_edma_device_caps(struct dma_chan *dchan,
 				struct dma_slave_caps *caps)
 {
@@ -1974,6 +2263,13 @@ int dw_edma_probe(struct dw_edma_chip *chip)
 	for (i = 0; i < ARRAY_SIZE(dw->event_lock); i++)
 		raw_spin_lock_init(&dw->event_lock[i]);
 
+	for (i = 0; i < ARRAY_SIZE(dw->eng_recovery); i++) {
+		dw->eng_recovery[i].dw = dw;
+		dw->eng_recovery[i].dir = i;
+		INIT_WORK(&dw->eng_recovery[i].work,
+			  dw_edma_engine_recovery_work);
+	}
+
 	/*
 	 * chip->ll_*_cnt describes the channels exposed by this instance. Keep
 	 * the usable hardware counts separate for partial ownership checks.
@@ -2079,6 +2375,10 @@ int dw_edma_remove(struct dw_edma_chip *chip)
 	WRITE_ONCE(dw->teardown, true);
 	dma_async_device_unregister(&dw->dma);
 
+	/* A recovery worker can re-enable the engine. */
+	disable_work_sync(&dw->eng_recovery[0].work);
+	disable_work_sync(&dw->eng_recovery[1].work);
+
 	/*
 	 * Drain channel work that may have passed the teardown gate before
 	 * stopping the hardware. IRQ handlers remain installed while it is
diff --git a/drivers/dma/dw-edma/dw-edma-core.h b/drivers/dma/dw-edma/dw-edma-core.h
index 46962a9bb4a6..a2699a663260 100644
--- a/drivers/dma/dw-edma/dw-edma-core.h
+++ b/drivers/dma/dw-edma/dw-edma-core.h
@@ -129,6 +129,9 @@ struct dw_edma_chan {
 
 	struct delayed_work		ll_recheck_work;
 	unsigned long			ll_recheck_at;
+	/* Per-channel recovery state. */
+	bool				ll_recovery_pending;
+	bool				ll_recovering;
 
 	u32				ll_max;		/* Data entries */
 	struct dw_edma_region		ll_region;	/* Linked list */
@@ -159,6 +162,18 @@ struct dw_edma_irq {
 	DECLARE_BITMAP(rd_mask, HDMA_MAX_RD_CH);
 };
 
+/*
+ * Direction-wide recovery state. active records that recovery has gated the
+ * channels; workqueue state coalesces duplicate requests.
+ */
+struct dw_edma_engine_recovery {
+	bool				active;
+	unsigned int			fails;
+	struct work_struct		work;
+	struct dw_edma			*dw;
+	enum dw_edma_dir		dir;
+};
+
 struct dw_edma {
 	char				name[32];
 
@@ -173,12 +188,13 @@ struct dw_edma {
 	struct dw_edma_chan		*chan;
 
 	/*
-	 * WQ_HIGHPRI keeps completion processing responsive under heavy load;
-	 * WQ_UNBOUND lets different channels run on different CPUs.
+	 * WQ_HIGHPRI keeps completion and recovery work responsive under heavy
+	 * load; WQ_UNBOUND lets independent work run on different CPUs.
 	 */
 	struct workqueue_struct		*wq;
 
 	bool				teardown;	/* Gate asynchronous hardware access */
+	struct dw_edma_engine_recovery	eng_recovery[2];
 
 	raw_spinlock_t			lock;		/* Protect v0 shared registers */
 	raw_spinlock_t			event_lock[2];	/* Per-direction event scopes */
@@ -378,7 +394,7 @@ static inline void dw_edma_core_ch_doorbell(struct dw_edma_chan *chan)
 static inline void dw_edma_core_ch_enable(struct dw_edma_chan *chan)
 {
 	chan->dw->core->ch_enable(chan);
-	if (chan->dw->core->engine_enable)
+	if (!chan->ll_recovering && chan->dw->core->engine_enable)
 		chan->dw->core->engine_enable(chan->dw, chan->dir);
 }
 
-- 
2.51.0