[PATCH v4 23/24] dmaengine: dw-edma: Detect and recover a stalled eDMA engine

[Koichiro Den]

Under sustained multi-channel traffic on one eDMA integration, a write
channel occasionally stopped with pending work, no progress, and no
error. Repeated doorbells did not restart it; clearing ENGINE_EN did.

LL progress accounting already schedules a bounded recheck when a DONE
or STOP/PAUSE drain leaves published work running. Reuse it for passive
stall detection. When a check finds the channel stopped, run the normal
restart or drain path, kick it, and arm one more check. If it is still
stopped with no progress 30 ms later, queue direction recovery.

Queue every recovery request and let the workqueue coalesce duplicates,
so a request raised while the worker releases channels is not lost.

The condition uses only generic eDMA state: published work, STOPPED
status, and lack of progress. Providers opt in by supplying the recovery
operations; HDMA compatibility mode is excluded because it has no
ENGINE_EN. Normal progress cancels the check before recovery runs. The
30 ms threshold is empirical and should be validated on more hardware.

Signed-off-by: Koichiro Den <den@xxxxxxxxxxxxx>
---
Changes in v4:
  - Keep stopped-channel rechecks independent of optional direction
    recovery support. (Sashiko)

 drivers/dma/dw-edma/dw-edma-core.c | 52 +++++++++++++++++++++++++++++-
 drivers/dma/dw-edma/dw-edma-core.h |  4 ++-
 2 files changed, 54 insertions(+), 2 deletions(-)

diff --git a/drivers/dma/dw-edma/dw-edma-core.c b/drivers/dma/dw-edma/dw-edma-core.c
index 616b0658b1ef..05b9ceb0b920 100644
--- a/drivers/dma/dw-edma/dw-edma-core.c
+++ b/drivers/dma/dw-edma/dw-edma-core.c
@@ -259,6 +259,7 @@ static void dw_edma_core_reset_ll(struct dw_edma_chan *chan)
 
 	chan->ll_head = 0;
 	chan->ll_done = 0;
+	chan->ll_stall_valid = false;
 	chan->ll_recovery_pending = false;
 	/* Drop stale CB bits before reusing the circular LL ring. */
 	for (i = 0; i < chan->ll_max; i++)
@@ -300,6 +301,52 @@ static bool dw_edma_ll_pending(struct dw_edma_chan *chan)
 	return chan->ll_head != chan->ll_done;
 }
 
+static bool dw_edma_engine_recovery_supported(struct dw_edma *dw)
+{
+	/*
+	 * DWC PCIe Controller Databook 6.10a-lca06, "Legacy DMA and HDMA
+	 * Software Compatibility":
+	 *
+	 * "HDMA does not implement engine enable, that is
+	 * DMA_[WRITE|READ]_ENGINE_EN_OFF.DMA_[WRITE|READ]_ENGINE field."
+	 */
+	if (dw->chip->mf == EDMA_MF_HDMA_COMPAT)
+		return false;
+
+	return dw->core->engine_reset && dw->core->engine_enable &&
+	       dw->core->ch_transfer_size;
+}
+
+/*
+ * Called with vc.lock held for a stopped channel with LL work pending. Queue
+ * direction recovery if repeated doorbells show no progress for one recheck
+ * interval.
+ */
+static void dw_edma_ll_stall_check(struct dw_edma_chan *chan)
+{
+	struct dw_edma_engine_recovery *rec;
+
+	if (unlikely(READ_ONCE(chan->dw->teardown)))
+		return;
+
+	if (!dw_edma_engine_recovery_supported(chan->dw))
+		return;
+
+	if (!chan->ll_stall_valid) {
+		chan->ll_stall_valid = true;
+		chan->ll_stall_since = jiffies;
+		return;
+	}
+
+	if (time_before(jiffies, chan->ll_stall_since +
+			msecs_to_jiffies(DW_EDMA_LL_RECHECK_DELAY_MS)))
+		return;
+
+	rec = &chan->dw->eng_recovery[chan->dir];
+	chan->ll_recovery_pending = true;
+	dw_edma_engine_recovery_queue(rec);
+}
+
 static u32 dw_edma_core_ch_transfer_size(struct dw_edma_chan *chan)
 {
 	if (!chan->dw->core->ch_transfer_size)
@@ -572,6 +619,7 @@ static bool dw_edma_ll_clean_pending(struct dw_edma_chan *chan, int idx)
 out:
 	if (advanced) {
 		dw_edma_ll_recheck_cancel(chan);
+		chan->ll_stall_valid = false;
 		chan->ll_recovery_pending = false;
 	}
 
@@ -763,8 +811,10 @@ static bool dw_edma_core_ch_maybe_doorbell(struct dw_edma_chan *chan)
 		}
 	}
 
+	dw_edma_ll_stall_check(chan);
 	dw_edma_core_ch_doorbell(chan);
-	if (!dw_edma_ll_has_hdma_stop_event(chan))
+	if (!dw_edma_ll_has_hdma_stop_event(chan) &&
+	    !chan->ll_recovery_pending)
 		dw_edma_ll_recheck_schedule(chan);
 
 	return false;
diff --git a/drivers/dma/dw-edma/dw-edma-core.h b/drivers/dma/dw-edma/dw-edma-core.h
index a902a24d20ba..cb3ff23143a2 100644
--- a/drivers/dma/dw-edma/dw-edma-core.h
+++ b/drivers/dma/dw-edma/dw-edma-core.h
@@ -124,9 +124,11 @@ struct dw_edma_chan {
 	spinlock_t			event_lock_per_chan;
 	spinlock_t			*event_lock;	/* Selected event lock */
 
+	/* Per-channel stall and recovery state. */
 	struct delayed_work		ll_recheck_work;
 	unsigned long			ll_recheck_at;
-	/* Per-channel recovery state. */
+	unsigned long			ll_stall_since;
+	bool				ll_stall_valid;
 	bool				ll_recovery_pending;
 	bool				ll_recovering;
 
-- 
2.51.0