[PATCH v8 6/7] dmaengine: dw-edma: Dynamically append requests while running

[Koichiro Den]

From: Frank Li <Frank.Li@xxxxxxx>

Allow issue_pending() to fill free LL entries while the channel is busy,
then check progress and restart it if it has stopped. Publish requests
in descriptor order.

Multiple descriptors can now have entries in the LL ring. When resetting
the ring after an abort, mark the other descriptors with pending entries
as aborted too, so they cannot be replayed.

Since dw_edma_start_transfer() can now return zero with LL entries still
pending, keep such channels BUSY on resume.

Signed-off-by: Frank Li <Frank.Li@xxxxxxx>
Co-developed-by: Koichiro Den <den@xxxxxxxxxxxxx>
Signed-off-by: Koichiro Den <den@xxxxxxxxxxxxx>
---
Changes from v7:
  - No changes. The premature completion Sashiko reported on v7 is fixed in
    patches 2 and 4.
    https://lore.kernel.org/r/rtfjvblptmdez2znjgx7t4v5fumj6wabp2ft6zmnsm353nsves@vow63wcrnpol/

 drivers/dma/dw-edma/dw-edma-core.c | 65 ++++++++++++++++++++++--------
 1 file changed, 49 insertions(+), 16 deletions(-)

diff --git a/drivers/dma/dw-edma/dw-edma-core.c b/drivers/dma/dw-edma/dw-edma-core.c
index 1d13a25663ef..bcf4fca34591 100644
--- a/drivers/dma/dw-edma/dw-edma-core.c
+++ b/drivers/dma/dw-edma/dw-edma-core.c
@@ -111,8 +111,22 @@ static void dw_edma_complete_vdesc(struct virt_dma_desc *vd,
 
 static void dw_edma_core_reset_ll(struct dw_edma_chan *chan)
 {
+	struct virt_dma_desc *vd, *tmp;
 	u32 i;
 
+	/*
+	 * We cannot tell which pending entries completed before the reset.
+	 * Abort their descriptors to avoid repeating a transfer.
+	 */
+	list_for_each_entry_safe(vd, tmp, &chan->vc.desc_issued, node) {
+		struct dw_edma_desc *desc = vd2dw_edma_desc(vd);
+
+		if (desc->start_burst == desc->done_burst)
+			continue;
+
+		dw_edma_complete_vdesc(vd, DMA_TRANS_ABORTED);
+	}
+
 	chan->ll_head = 0;
 	chan->ll_done = 0;
 	/* Drop stale CB bits before reusing the circular LL ring. */
@@ -301,21 +315,34 @@ static int dw_edma_start_transfer(struct dw_edma_chan *chan)
 {
 	struct dw_edma_desc *desc;
 	struct virt_dma_desc *vd;
+	int ret = 0;
 
-	vd = vchan_next_desc(&chan->vc);
-	if (!vd)
-		return 0;
+	if (chan->non_ll) {
+		vd = vchan_next_desc(&chan->vc);
+		if (!vd)
+			return 0;
 
-	desc = vd2dw_edma_desc(vd);
-	if (!desc)
-		return 0;
+		dw_edma_core_start(vd2dw_edma_desc(vd));
+		return 1;
+	}
 
-	if (!chan->non_ll && !chan->ll_valid)
-		dw_edma_core_reset_ll(chan);
+	list_for_each_entry(vd, &chan->vc.desc_issued, node) {
+		if (!dw_edma_core_get_free_num(chan))
+			break;
+
+		desc = vd2dw_edma_desc(vd);
+		/* Fully published descriptors may still be in flight. */
+		if (desc->start_burst == desc->nburst)
+			continue;
 
-	dw_edma_core_start(desc);
+		if (!chan->ll_valid)
+			dw_edma_core_reset_ll(chan);
 
-	return 1;
+		dw_edma_core_start(desc);
+		ret = 1;
+	}
+
+	return ret;
 }
 
 static void dw_edma_terminate_vdesc(struct virt_dma_desc *vd)
@@ -593,7 +620,8 @@ static int dw_edma_device_resume(struct dma_chan *dchan)
 		err = -EPERM;
 	} else {
 		chan->status = EDMA_ST_BUSY;
-		if (!dw_edma_start_transfer(chan))
+		if (!dw_edma_start_transfer(chan) &&
+		    (chan->non_ll || !dw_edma_ll_pending(chan)))
 			chan->status = EDMA_ST_IDLE;
 		dw_edma_core_ch_maybe_doorbell(chan);
 	}
@@ -642,11 +670,16 @@ static void dw_edma_device_issue_pending(struct dma_chan *dchan)
 
 	spin_lock_irqsave(&chan->vc.lock, flags);
 	if (chan->configured && vchan_issue_pending(&chan->vc) &&
-	    chan->request == EDMA_REQ_NONE &&
-	    chan->status == EDMA_ST_IDLE) {
-		chan->status = EDMA_ST_BUSY;
-		dw_edma_start_transfer(chan);
-		dw_edma_core_ch_maybe_doorbell(chan);
+	    chan->request == EDMA_REQ_NONE) {
+		if (chan->status == EDMA_ST_IDLE) {
+			chan->status = EDMA_ST_BUSY;
+			dw_edma_start_transfer(chan);
+			dw_edma_core_ch_maybe_doorbell(chan);
+		} else if (!chan->non_ll &&
+			   chan->status == EDMA_ST_BUSY) {
+			dw_edma_start_transfer(chan);
+			dw_edma_ll_advance(chan, false);
+		}
 	}
 	spin_unlock_irqrestore(&chan->vc.lock, flags);
 }
-- 
2.54.0