[PATCH v8 4/7] dmaengine: dw-edma: Reclaim LL entries from hardware progress

[Koichiro Den]

With dynamic appends, a DONE interrupt no longer means that all
published LL entries have completed. Prepare for this by using LLP to
advance ll_done and complete descriptors in issue order.

Reclaim completed entries and refill the ring as space becomes
available. Restart a stopped channel when published work remains. This
also allows descriptors larger than the ring to run in batches.

Hardware can keep running while the driver handles an interrupt, and
LLP does not always track payload completion. Use the following
exclusive completion boundaries, where idx is the raw LLP entry index
and prev(idx) is the previous data entry in the ring:

  State/event          Meaning of raw LLP               Boundary
  -------------------  -------------------------------  ---------
  eDMA/HDMA stopped    Next entry to process             idx

  eDMA running         Observed to advance before        prev(idx)
                       payload completion

  HDMA watermark       Inclusive LLE recycling limit     prev(idx)

A stopped sample requires CS=STOPPED and a zero transfer size.

While running, HDMA updates LLP only at watermark events. Its documented
recycling limit translates to an exclusive boundary of next(idx).
However, tests using that boundary for dmaengine completion reported
IOMMU faults on HDMA 6.30a SpacemiT K3. Use prev(idx) for both engines
while running to keep the handling simple and conservative. Entries held
back by this rule are eventually reclaimed on later progress or a
confirmed stop, where idx gives the exact boundary.

Handle termination and pause as follows:
- On termination, stop publishing new entries and drain those already
  published, restarting the channel if needed. Terminate descriptors
  without callbacks, as in the non-LL path.
- On pause, stop publishing new entries and enter the paused state once
  the channel stops. Keep unfinished work for resume. Reclaim delayed
  completions without restarting the channel.

Signed-off-by: Koichiro Den <den@xxxxxxxxxxxxx>
---
Changes from v7:
  - Read LLP before checking for a stop, and use the same sample for
    completion and termination (Sashiko).
  - Reclaim delayed completions while paused without restarting the
    channel (spotted during code inspection, not observed in hardware
    testing).

@Frank: Dropped your R-b due to the changes; would appreciate another
look.

 drivers/dma/dw-edma/dw-edma-core.c | 160 +++++++++++++++++++++++++----
 1 file changed, 140 insertions(+), 20 deletions(-)

diff --git a/drivers/dma/dw-edma/dw-edma-core.c b/drivers/dma/dw-edma/dw-edma-core.c
index 346ec8ee631d..1d13a25663ef 100644
--- a/drivers/dma/dw-edma/dw-edma-core.c
+++ b/drivers/dma/dw-edma/dw-edma-core.c
@@ -53,13 +53,6 @@ dw_edma_alloc_desc(struct dw_edma_chan *chan, size_t nburst)
 {
 	struct dw_edma_desc *desc;
 
-	/*
-	 * For now, a descriptor that does not fit would stall the channel
-	 * forever: reject it up front.
-	 */
-	if (!chan->non_ll && nburst > chan->ll_max - 1)
-		return NULL;
-
 	desc = kzalloc_flex(*desc, burst, nburst, GFP_NOWAIT);
 	if (unlikely(!desc))
 		return NULL;
@@ -181,18 +174,83 @@ static int dw_edma_ll_read_idx(struct dw_edma_chan *chan, bool *stopped)
 	return idx;
 }
 
+/* Return the first entry we cannot reclaim yet. */
+static int dw_edma_ll_boundary(struct dw_edma_chan *chan, int idx,
+			       bool stopped)
+{
+	if (idx < 0 || (u32)idx > chan->ll_max)
+		return -EINVAL;
+
+	/* After a stop, LLP points past the completed entries. */
+	if (stopped)
+		return idx == chan->ll_max ? 0 : idx;
+
+	/*
+	 * DMA_LLP can advance before payload completion. Keep the boundary
+	 * one entry behind it, wrapping at the start of the ring.
+	 */
+	return (idx + chan->ll_max - 1) % chan->ll_max;
+}
+
 static bool dw_edma_ll_drained(struct dw_edma_chan *chan)
 {
 	bool stopped;
 	int idx;
 
 	idx = dw_edma_ll_read_idx(chan, &stopped);
-	if (idx == chan->ll_max)
-		idx = 0;
+	idx = dw_edma_ll_boundary(chan, idx, true);
 
 	return stopped && idx >= 0 && (u32)idx == chan->ll_head;
 }
 
+static void dw_edma_ll_consume(struct dw_edma_chan *chan, int idx,
+			       bool stopped)
+{
+	struct virt_dma_desc *vd, *_vd;
+	u32 done, used;
+
+	lockdep_assert_held(&chan->vc.lock);
+
+	idx = dw_edma_ll_boundary(chan, idx, stopped);
+	if (idx < 0)
+		return;
+
+	done = dw_edma_core_get_ll_dist(chan, chan->ll_done, idx);
+	used = dw_edma_core_get_used_num(chan);
+
+	if (!done || done > used)
+		return;
+
+	/*
+	 * LLP may still point to the first pending entry. Subtracting one
+	 * can wrap the index and falsely mark all pending entries complete.
+	 * Reclaim them all only after the channel has stopped.
+	 */
+	if (!stopped && done == used)
+		return;
+
+	chan->ll_done = idx;
+
+	/* Match reclaimed entries to descriptors in issue order. */
+	list_for_each_entry_safe(vd, _vd, &chan->vc.desc_issued, node) {
+		struct dw_edma_desc *desc = vd2dw_edma_desc(vd);
+		u32 consumed = min_t(u32, done,
+				     desc->start_burst - desc->done_burst);
+
+		if (!consumed)
+			break;
+
+		desc->done_burst += consumed;
+		done -= consumed;
+		if (desc->done_burst != desc->nburst)
+			break;
+
+		dw_edma_complete_vdesc(vd, DMA_TRANS_NOERROR);
+	}
+
+	WARN_ON_ONCE(done);
+}
+
 static void dw_edma_core_ll_start(struct dw_edma_desc *desc)
 {
 	struct dw_edma_chan *chan = desc->chan;
@@ -221,7 +279,6 @@ static void dw_edma_core_ll_start(struct dw_edma_desc *desc)
 		}
 	}
 
-	desc->done_burst = desc->start_burst;
 	desc->start_burst = i;
 }
 
@@ -330,6 +387,61 @@ static void dw_edma_core_ch_maybe_doorbell(struct dw_edma_chan *chan)
 	dw_edma_core_ch_doorbell(chan);
 }
 
+/* Must be called with vc.lock held. */
+static void dw_edma_ll_advance(struct dw_edma_chan *chan, bool event)
+{
+	bool stopped;
+	int idx;
+
+	lockdep_assert_held(&chan->vc.lock);
+
+	idx = dw_edma_ll_read_idx(chan, &stopped);
+
+	/* Terminate without scheduling completion callbacks. */
+	if (chan->request == EDMA_REQ_STOP) {
+		if (!stopped)
+			return;
+
+		idx = dw_edma_ll_boundary(chan, idx, true);
+		if (idx >= 0 && (u32)idx == chan->ll_head)
+			dw_edma_finish_termination(chan);
+		else
+			/* A newly appended tail may have missed the last stop. */
+			dw_edma_core_ch_doorbell(chan);
+		return;
+	}
+
+	/*
+	 * HDMA updates LLP at watermark events while running. Use running
+	 * samples only when processing an LL interrupt.
+	 */
+	if (stopped || event)
+		dw_edma_ll_consume(chan, idx, stopped);
+
+	/* An interrupt may arrive after we have entered PAUSE. */
+	if (chan->status == EDMA_ST_PAUSE)
+		return;
+
+	if (chan->request == EDMA_REQ_PAUSE) {
+		if (!stopped)
+			return;
+
+		dw_edma_set_request(chan, EDMA_REQ_NONE);
+		chan->status = EDMA_ST_PAUSE;
+		return;
+	}
+
+	dw_edma_start_transfer(chan);
+	if (!dw_edma_ll_pending(chan)) {
+		chan->status = EDMA_ST_IDLE;
+		return;
+	}
+
+	chan->status = EDMA_ST_BUSY;
+	if (stopped)
+		dw_edma_core_ch_doorbell(chan);
+}
+
 static void dw_edma_device_caps(struct dma_chan *dchan,
 				struct dma_slave_caps *caps)
 {
@@ -514,6 +626,8 @@ static int dw_edma_device_terminate_all(struct dma_chan *dchan)
 		err = -EPERM;
 	} else {
 		dw_edma_set_request(chan, EDMA_REQ_STOP);
+		if (!chan->non_ll)
+			dw_edma_ll_advance(chan, false);
 	}
 	if (chan->status == EDMA_ST_IDLE)
 		dw_edma_set_request(chan, EDMA_REQ_NONE);
@@ -798,7 +912,7 @@ dw_edma_device_prep_interleaved_dma(struct dma_chan *dchan,
 	return dw_edma_device_transfer(&xfer, dw_edma_device_get_config(dchan, NULL));
 }
 
-/* Must be called with vc.lock held. */
+/* Non-LL completion; vc.lock must be held. */
 static void dw_edma_done_interrupt_locked(struct dw_edma_chan *chan)
 {
 	struct dw_edma_desc *desc;
@@ -817,11 +931,8 @@ static void dw_edma_done_interrupt_locked(struct dw_edma_chan *chan)
 			break;
 
 		desc = vd2dw_edma_desc(vd);
-		if (desc->start_burst >= desc->nburst) {
+		if (desc->start_burst >= desc->nburst)
 			dw_edma_complete_vdesc(vd, DMA_TRANS_NOERROR);
-			if (!chan->non_ll)
-				chan->ll_done = chan->ll_head;
-		}
 
 		if (chan->request == EDMA_REQ_PAUSE) {
 			dw_edma_set_request(chan, EDMA_REQ_NONE);
@@ -837,14 +948,12 @@ static void dw_edma_done_interrupt_locked(struct dw_edma_chan *chan)
 		if (!vd)
 			break;
 
-		if (chan->non_ll || dw_edma_ll_drained(chan))
-			dw_edma_finish_termination(chan);
+		dw_edma_finish_termination(chan);
 		break;
 
 	default:
 		break;
 	}
-	dw_edma_core_ch_maybe_doorbell(chan);
 }
 
 static void dw_edma_done_interrupt(struct dw_edma_chan *chan)
@@ -856,6 +965,13 @@ static void dw_edma_done_interrupt(struct dw_edma_chan *chan)
 	spin_unlock_irqrestore(&chan->vc.lock, flags);
 }
 
+static void dw_edma_ll_interrupt(struct dw_edma_chan *chan)
+{
+	guard(spinlock_irqsave)(&chan->vc.lock);
+
+	dw_edma_ll_advance(chan, true);
+}
+
 static void dw_edma_abort_interrupt(struct dw_edma_chan *chan)
 {
 	struct virt_dma_desc *vd;
@@ -884,8 +1000,12 @@ static void dw_edma_irq_work(struct work_struct *work)
 	do {
 		events = atomic_xchg(&chan->irq_pending, 0);
 
-		if (events & DW_EDMA_DEFERRED_DONE)
-			dw_edma_done_interrupt(chan);
+		if (events & DW_EDMA_DEFERRED_DONE) {
+			if (chan->non_ll)
+				dw_edma_done_interrupt(chan);
+			else
+				dw_edma_ll_interrupt(chan);
+		}
 		if (events & DW_EDMA_DEFERRED_ABORT)
 			dw_edma_abort_interrupt(chan);
 	} while (atomic_read(&chan->irq_pending));
-- 
2.54.0