[PATCH 15/17] dmaengine: dw-edma: Dynamically append requests while running

[Koichiro Den]

From: Frank Li <Frank.Li@xxxxxxx>

Use the LL producer-consumer state to append issued descriptors while
the channel is still running, instead of waiting for the current work to
drain.

Walk the issued list and append entries for any descriptor that still
has unstarted bursts. A descriptor that has already been fully appended
may still be pending in hardware, so keep walking; later descriptors can
use LL entries that have already been reclaimed.

Do not use dw_edma_start_transfer() as a channel-liveness test. It only
reports whether this pass appended new LL entries. Keep the software
state tied to pending LL entries, and keep doorbell decisions separate
from that state.

Doorbell only when runnable LL work was appended or when a stopped
channel still has pending LL entries. Recheck and kick under vc.lock so
the doorbell is serialized with request/status updates and LL reset.

Signed-off-by: Frank Li <Frank.Li@xxxxxxx>
Co-developed-by: Koichiro Den <den@xxxxxxxxxxxxx>
Signed-off-by: Koichiro Den <den@xxxxxxxxxxxxx>
---
Changes from Frank's original submission:
20260109-edma_dymatic-v1-4-9a98c9c98536@xxxxxxx
- Move LL progress cleanup, including dw_edma_ll_clean_pending(), into
  the earlier LL progress accounting patch.
- Continue past fully appended descriptors when appending later issued
  descriptors.
- Treat dw_edma_start_transfer() as "entries appended", not channel
  liveness.
- Derive BUSY/IDLE from pending LL entries.
- Tighten doorbelling in issue_pending(), resume(), and DONE handling so
  STOP/PAUSE paths are not re-kicked, and keep the kick under vc.lock.
- Keep Frank's eDMA ll_cur_idx() re-doorbell workaround for now; a later
  patch moves stopped-pending recovery into common tx_status() handling.
- Rephrase s/Need hold vc.lock/Must be called with vc.lock held./ for
  consistency.

 drivers/dma/dw-edma/dw-edma-core.c    | 121 ++++++++++++++++++++------
 drivers/dma/dw-edma/dw-edma-core.h    |   6 ++
 drivers/dma/dw-edma/dw-edma-v0-core.c |  22 ++++-
 3 files changed, 121 insertions(+), 28 deletions(-)

diff --git a/drivers/dma/dw-edma/dw-edma-core.c b/drivers/dma/dw-edma/dw-edma-core.c
index 4036adafedfa..477fc63e2778 100644
--- a/drivers/dma/dw-edma/dw-edma-core.c
+++ b/drivers/dma/dw-edma/dw-edma-core.c
@@ -84,6 +84,11 @@ static u32 dw_edma_core_get_ll_data_cnt(struct dw_edma_chan *chan)
 	return chan->ll_max - 1;
 }
 
+static bool dw_edma_core_has_flags(struct dw_edma_chan *chan, u32 flags)
+{
+	return chan->dw->core->flags & flags;
+}
+
 static u32 dw_edma_core_get_ll_dist(struct dw_edma_chan *chan, u32 from, u32 to)
 {
 	u32 cnt = dw_edma_core_get_ll_data_cnt(chan);
@@ -139,6 +144,31 @@ static bool dw_edma_ll_pending(struct dw_edma_chan *chan)
 	return dw_edma_core_get_used_num(chan);
 }
 
+static bool dw_edma_ll_stopped_pending(struct dw_edma_chan *chan)
+{
+	return dw_edma_ll_pending(chan) &&
+	       dw_edma_core_ch_status(chan) == DMA_COMPLETE;
+}
+
+static bool dw_edma_ll_recoverable_pending(struct dw_edma_chan *chan)
+{
+	return chan->request == EDMA_REQ_NONE &&
+	       chan->status != EDMA_ST_PAUSE &&
+	       dw_edma_ll_stopped_pending(chan);
+}
+
+/* Must be called with vc.lock held. */
+static void
+dw_edma_core_ch_doorbell_recheck(struct dw_edma_chan *chan, bool doorbell)
+{
+	if (!doorbell && !dw_edma_ll_recoverable_pending(chan))
+		return;
+
+	/* Serialize the kick with channel state changes and LL reset. */
+	dw_edma_core_ch_doorbell(chan);
+}
+
+/* Must be called with vc.lock held. */
 static void dw_edma_core_start(struct dw_edma_desc *desc)
 {
 	struct dw_edma_chan *chan = desc->chan;
@@ -173,30 +203,36 @@ static void dw_edma_core_start(struct dw_edma_desc *desc)
 
 	desc->start_burst = i;
 	desc->ll_end = chan->ll_head;
-
-	dw_edma_core_ch_doorbell(chan);
 }
 
+/* Must be called with vc.lock held. */
 static int dw_edma_start_transfer(struct dw_edma_chan *chan)
 {
 	struct dw_edma_desc *desc;
 	struct virt_dma_desc *vd;
 	int index = dw_edma_core_ll_cur_idx(chan);
+	int ret = 0;
 
 	if (index < 0)
 		dw_edma_core_reset_ll(chan);
 
-	vd = vchan_next_desc(&chan->vc);
-	if (!vd)
-		return 0;
+	list_for_each_entry(vd, &chan->vc.desc_issued, node) {
+		if (!dw_edma_core_get_free_num(chan))
+			return ret;
 
-	desc = vd2dw_edma_desc(vd);
-	if (!desc)
-		return 0;
+		desc = vd2dw_edma_desc(vd);
 
-	dw_edma_core_start(desc);
+		/*
+		 * Fully appended descriptors may still be pending. Keep walking
+		 * so later descriptors can use newly freed LL entries.
+		 */
+		if (desc->start_burst == desc->nburst)
+			continue;
+		dw_edma_core_start(desc);
+		ret = 1;
+	}
 
-	return 1;
+	return ret;
 }
 
 static void dw_hdma_set_callback_result(struct virt_dma_desc *vd,
@@ -375,6 +411,7 @@ static int dw_edma_device_resume(struct dma_chan *dchan)
 {
 	struct dw_edma_chan *chan = dchan2dw_edma_chan(dchan);
 	unsigned long flags;
+	bool doorbell = false;
 	int err = 0;
 
 	spin_lock_irqsave(&chan->vc.lock, flags);
@@ -385,8 +422,10 @@ static int dw_edma_device_resume(struct dma_chan *dchan)
 	} else if (chan->request != EDMA_REQ_NONE) {
 		err = -EPERM;
 	} else {
-		chan->status = EDMA_ST_BUSY;
 		dw_edma_start_transfer(chan);
+		doorbell = dw_edma_ll_pending(chan);
+		chan->status = doorbell ? EDMA_ST_BUSY : EDMA_ST_IDLE;
+		dw_edma_core_ch_doorbell_recheck(chan, doorbell);
 	}
 	spin_unlock_irqrestore(&chan->vc.lock, flags);
 
@@ -430,13 +469,19 @@ static void dw_edma_device_issue_pending(struct dma_chan *dchan)
 {
 	struct dw_edma_chan *chan = dchan2dw_edma_chan(dchan);
 	unsigned long flags;
+	bool doorbell = false;
+	bool pending;
 
 	spin_lock_irqsave(&chan->vc.lock, flags);
-	if (chan->configured && vchan_issue_pending(&chan->vc) &&
-	    chan->request == EDMA_REQ_NONE &&
-	    chan->status == EDMA_ST_IDLE) {
+	if (!chan->configured)
+		pending = false;
+	else
+		pending = vchan_issue_pending(&chan->vc);
+	if (pending && chan->request == EDMA_REQ_NONE &&
+	    chan->status != EDMA_ST_PAUSE) {
 		chan->status = EDMA_ST_BUSY;
-		dw_edma_start_transfer(chan);
+		doorbell = dw_edma_start_transfer(chan);
+		dw_edma_core_ch_doorbell_recheck(chan, doorbell);
 	}
 	spin_unlock_irqrestore(&chan->vc.lock, flags);
 }
@@ -451,7 +496,18 @@ dw_edma_device_tx_status(struct dma_chan *dchan, dma_cookie_t cookie,
 	unsigned long flags;
 	enum dma_status ret;
 	u32 residue = 0;
+	int idx;
 
+	ret = dma_cookie_status(dchan, cookie, txstate);
+	if (ret == DMA_COMPLETE)
+		return ret;
+
+	spin_lock_irqsave(&chan->vc.lock, flags);
+	idx = dw_edma_core_ll_cur_idx(chan);
+	dw_edma_ll_recycle_and_refill(chan, idx);
+	spin_unlock_irqrestore(&chan->vc.lock, flags);
+
+	/* check again because dw_edma_ll_clean_pending() may update cookie */
 	ret = dma_cookie_status(dchan, cookie, txstate);
 	if (ret == DMA_COMPLETE)
 		return ret;
@@ -705,9 +761,9 @@ dw_edma_device_prep_interleaved_dma(struct dma_chan *dchan,
 
 static void dw_edma_done_interrupt(struct dw_edma_chan *chan)
 {
-	struct dw_edma_desc *desc;
 	struct virt_dma_desc *vd;
 	unsigned long flags;
+	bool doorbell = false;
 	int idx;
 
 	spin_lock_irqsave(&chan->vc.lock, flags);
@@ -718,22 +774,23 @@ static void dw_edma_done_interrupt(struct dw_edma_chan *chan)
 		dw_edma_ll_recycle(chan, idx);
 		vd = vchan_next_desc(&chan->vc);
 		if (vd) {
-			desc = vd2dw_edma_desc(vd);
-			if (desc->start_burst >= desc->nburst) {
-				dw_hdma_set_callback_result(vd,
-							    DMA_TRANS_NOERROR);
-				list_del(&vd->node);
-				vchan_cookie_complete(vd);
-				chan->ll_end = desc->ll_end;
-			}
-
-			/* Continue transferring if there are remaining chunks or issued requests.
+			/*
+			 * dw_edma_start_transfer() reports whether new entries
+			 * were appended. Channel liveness follows the LL
+			 * producer/consumer state.
 			 */
-			chan->status = dw_edma_start_transfer(chan) ? EDMA_ST_BUSY : EDMA_ST_IDLE;
+			if (dw_edma_core_has_flags(chan, DW_EDMA_CORE_FLAG_DONE_IRQ_DOORBELL))
+				doorbell = true;
+			doorbell |= dw_edma_start_transfer(chan);
+			chan->status = dw_edma_ll_pending(chan) ?
+					EDMA_ST_BUSY : EDMA_ST_IDLE;
 		} else {
 			chan->status = dw_edma_ll_pending(chan) ?
 				       EDMA_ST_BUSY : EDMA_ST_IDLE;
 		}
+
+		if (!doorbell && dw_edma_ll_recoverable_pending(chan))
+			doorbell = true;
 		break;
 
 	case EDMA_REQ_STOP:
@@ -753,6 +810,7 @@ static void dw_edma_done_interrupt(struct dw_edma_chan *chan)
 	default:
 		break;
 	}
+	dw_edma_core_ch_doorbell_recheck(chan, doorbell);
 
 	spin_unlock_irqrestore(&chan->vc.lock, flags);
 }
@@ -760,6 +818,7 @@ static void dw_edma_done_interrupt(struct dw_edma_chan *chan)
 static void dw_edma_progress_interrupt(struct dw_edma_chan *chan)
 {
 	unsigned long flags;
+	bool doorbell = false;
 	int idx;
 
 	spin_lock_irqsave(&chan->vc.lock, flags);
@@ -768,7 +827,15 @@ static void dw_edma_progress_interrupt(struct dw_edma_chan *chan)
 		dw_edma_ll_recycle_and_refill(chan, idx);
 		chan->status = dw_edma_ll_pending(chan) ?
 			       EDMA_ST_BUSY : EDMA_ST_IDLE;
+
+		/*
+		 * The channel may have stopped after the progress point was
+		 * sampled. Re-kick it if LL work remains pending.
+		 */
+		if (dw_edma_ll_recoverable_pending(chan))
+			doorbell = true;
 	}
+	dw_edma_core_ch_doorbell_recheck(chan, doorbell);
 
 	spin_unlock_irqrestore(&chan->vc.lock, flags);
 }
diff --git a/drivers/dma/dw-edma/dw-edma-core.h b/drivers/dma/dw-edma/dw-edma-core.h
index 1252d264c1ca..27a0521c989c 100644
--- a/drivers/dma/dw-edma/dw-edma-core.h
+++ b/drivers/dma/dw-edma/dw-edma-core.h
@@ -9,11 +9,15 @@
 #ifndef _DW_EDMA_CORE_H
 #define _DW_EDMA_CORE_H
 
+#include <linux/bits.h>
 #include <linux/msi.h>
 #include <linux/dma/edma.h>
 
 #include "../virt-dma.h"
 
+/* Force a doorbell after DONE IRQ handling to recover lost starts. */
+#define DW_EDMA_CORE_FLAG_DONE_IRQ_DOORBELL		BIT(0)
+
 #define EDMA_LL_SZ					24
 
 enum dw_edma_dir {
@@ -140,6 +144,8 @@ struct dw_edma {
 typedef void (*dw_edma_handler_t)(struct dw_edma_chan *);
 
 struct dw_edma_core_ops {
+	u32 flags;
+
 	void (*off)(struct dw_edma *dw);
 	u16 (*ch_count)(struct dw_edma *dw, enum dw_edma_dir dir);
 	enum dma_status (*ch_status)(struct dw_edma_chan *chan);
diff --git a/drivers/dma/dw-edma/dw-edma-v0-core.c b/drivers/dma/dw-edma/dw-edma-v0-core.c
index 265eefbf2ead..a5ffb0e77602 100644
--- a/drivers/dma/dw-edma/dw-edma-v0-core.c
+++ b/drivers/dma/dw-edma/dw-edma-v0-core.c
@@ -516,7 +516,6 @@ static void dw_edma_v0_core_ch_doorbell(struct dw_edma_chan *chan)
 
 	dw_edma_v0_sync_ll_data(chan);
 
-	/* Doorbell */
 	SET_RW_32(dw, chan->dir, doorbell,
 		  FIELD_PREP(EDMA_V0_DOORBELL_CH_MASK, chan->id));
 }
@@ -534,6 +533,26 @@ static int dw_edma_v0_core_ll_cur_idx(struct dw_edma_chan *chan)
 	if (!val)
 		return -EINVAL;
 
+	/*
+	 * Doorbell will be missed if DMA engine running, so last update
+	 * descriptor have not fetched by DMA engine, so DMA engine stop.
+	 *
+	 *	Most like issue happen at
+	 *
+	 *	  DMA Engine		|	SW
+	 *        ======================================
+	 *  1     send Read req for LL
+	 *  2					update LL
+	 *  3					doorbell
+	 *  4	  *Missed doorbell*
+	 *  5     Get old LL data
+	 *  6     DMA stop
+	 *
+	 * Workaround: Push doorbell again when found DMA stop.
+	 */
+	if (dw_edma_v0_core_ch_status(chan) != DMA_IN_PROGRESS)
+		dw_edma_v0_core_ch_doorbell(chan);
+
 	return (val - (paddr & 0xFFFFFFFF)) / EDMA_LL_SZ;
 }
 
@@ -553,6 +572,7 @@ static void dw_edma_v0_core_debugfs_on(struct dw_edma *dw)
 }
 
 static const struct dw_edma_core_ops dw_edma_v0_core = {
+	.flags = DW_EDMA_CORE_FLAG_DONE_IRQ_DOORBELL,
 	.off = dw_edma_v0_core_off,
 	.ch_count = dw_edma_v0_core_ch_count,
 	.ch_status = dw_edma_v0_core_ch_status,
-- 
2.51.0