[PATCH v3 19/24] dmaengine: dw-edma: Drain LL entries for STOP and PAUSE

[Koichiro Den]

An EDMA_REQ_STOP or EDMA_REQ_PAUSE can arrive while hardware still owns
published LL entries. Keep the request pending until hardware consumes
them instead of discarding the ring state immediately.

With the eDMA-compatible interrupt interface, a DONE interrupt can
report progress rather than a stop, and STOPPED status may still be
visible while a doorbell-triggered start takes effect. Before kicking
the remaining entries, serialize the stopped-state and
zero-transfer-size checks, the DMA_LLP sample and the interrupt-status
clear against IRQ capture. Consume that boundary without refilling so
the producer boundary stays fixed and the kick path cannot clear its
DONE status before accounting for the stop.

If the channel has already stopped when the request arrives, complete
the request once the stopped boundary consumes all published entries.
Otherwise, kick only the entries published before the request. If status
still reports the channel running, arm one delayed recheck to catch a
later stop at a CB mismatch. Native HDMA reports STOP separately; non-LL
completion and EDMA_ST_PAUSE also identify a stopped channel.

If an ABORT is pending, leave cleanup to its handler and do not ring
another doorbell. Once EDMA_REQ_STOP completes, terminate submitted and
issued descriptors without callbacks. EDMA_REQ_PAUSE leaves the channel
stopped for resume.

Signed-off-by: Koichiro Den <den@xxxxxxxxxxxxx>
---
Changes in v3:
  - Rework and rename v2's "Reset LL state after terminate and abort":
    move abnormal-path ring reset to the circular-ring patch and drain
    published entries before completing EDMA_REQ_STOP or EDMA_REQ_PAUSE.
    (Frank, Sashiko)
  - Preserve an IRQ-captured LL event across request changes and leave
    terminal cleanup to a pending ABORT. (Sashiko)
  - Reconcile a stopped boundary not yet captured by the IRQ path before
    kicking a request drain.
  - Use the common kick helper for normal and request-drain paths.
  - Continue waiting after the five-second warning in
    device_synchronize(). (Sashiko)
  - Arm one delayed stop recheck while draining EDMA_REQ_STOP or
    EDMA_REQ_PAUSE. (Sashiko)

 drivers/dma/dw-edma/dw-edma-core.c | 111 +++++++++++++++++++++--------
 1 file changed, 81 insertions(+), 30 deletions(-)

diff --git a/drivers/dma/dw-edma/dw-edma-core.c b/drivers/dma/dw-edma/dw-edma-core.c
index 0c30c0f51fea..bbd417e98a9d 100644
--- a/drivers/dma/dw-edma/dw-edma-core.c
+++ b/drivers/dma/dw-edma/dw-edma-core.c
@@ -460,6 +460,13 @@ static void dw_edma_finish_termination(struct dw_edma_chan *chan)
 	chan->status = EDMA_ST_IDLE;
 }
 
+/* Must be called with vc.lock held after the channel has stopped. */
+static void dw_edma_finish_pause(struct dw_edma_chan *chan)
+{
+	dw_edma_set_request(chan, EDMA_REQ_NONE);
+	chan->status = EDMA_ST_PAUSE;
+}
+
 /* Must be called with vc.lock held. */
 static bool dw_edma_ll_clean_pending(struct dw_edma_chan *chan, int idx)
 {
@@ -691,8 +698,9 @@ static void dw_edma_core_ch_kick(struct dw_edma_chan *chan)
  */
 static bool dw_edma_core_ch_maybe_doorbell(struct dw_edma_chan *chan)
 {
-	if (chan->non_ll || chan->request != EDMA_REQ_NONE ||
-	    chan->status != EDMA_ST_BUSY || !dw_edma_ll_pending(chan) ||
+	if (chan->non_ll ||
+	    chan->status != EDMA_ST_BUSY ||
+	    !dw_edma_ll_pending(chan) ||
 	    dw_edma_abort_is_pending(chan))
 		return false;
 
@@ -703,9 +711,25 @@ static bool dw_edma_core_ch_maybe_doorbell(struct dw_edma_chan *chan)
 	if (dw_edma_core_ch_status(chan) == DMA_IN_PROGRESS)
 		return true;
 
-	dw_edma_ll_reconcile_and_refill(chan);
-	if (!dw_edma_ll_pending(chan))
-		return false;
+	/*
+	 * Normal work may refill a stopped tail. STOP and PAUSE freeze the
+	 * producer boundary, so only kick entries published before the request.
+	 */
+	if (chan->request == EDMA_REQ_NONE) {
+		dw_edma_ll_reconcile_and_refill(chan);
+		if (!dw_edma_ll_pending(chan))
+			return false;
+	} else {
+		dw_edma_ll_reconcile_stopped(chan);
+		if (!dw_edma_ll_pending(chan)) {
+			if (chan->request == EDMA_REQ_STOP)
+				dw_edma_finish_termination(chan);
+			else
+				dw_edma_finish_pause(chan);
+
+			return false;
+		}
+	}
 
 	dw_edma_core_ch_kick(chan);
 
@@ -748,8 +772,7 @@ static void dw_edma_ll_recheck_work(struct work_struct *work)
 	}
 
 	chan->ll_recheck_at = 0;
-	if (chan->request == EDMA_REQ_NONE)
-		dw_edma_core_ch_maybe_doorbell(chan);
+	dw_edma_core_ch_maybe_doorbell(chan);
 }
 
 static void dw_edma_device_caps(struct dma_chan *dchan,
@@ -841,6 +864,16 @@ dw_edma_device_get_config(struct dma_chan *dchan,
 	return &chan->config;
 }
 
+/* Must be called with vc.lock held. */
+static void dw_edma_request_pause(struct dw_edma_chan *chan)
+{
+	dw_edma_set_request(chan, EDMA_REQ_PAUSE);
+	if (!chan->non_ll && !dw_edma_ll_pending(chan))
+		dw_edma_finish_pause(chan);
+	else
+		dw_edma_core_ch_maybe_doorbell_or_recheck(chan);
+}
+
 static int dw_edma_device_pause(struct dma_chan *dchan)
 {
 	struct dw_edma_chan *chan = dchan2dw_edma_chan(dchan);
@@ -855,7 +888,7 @@ static int dw_edma_device_pause(struct dma_chan *dchan)
 	else if (chan->request != EDMA_REQ_NONE)
 		err = -EPERM;
 	else
-		dw_edma_set_request(chan, EDMA_REQ_PAUSE);
+		dw_edma_request_pause(chan);
 
 	return err;
 }
@@ -883,35 +916,52 @@ static int dw_edma_device_resume(struct dma_chan *dchan)
 	return err;
 }
 
+/* Must be called with vc.lock held. */
+static bool dw_edma_may_be_active(struct dw_edma_chan *chan)
+{
+	if (chan->non_ll)
+		return dw_edma_core_ch_status(chan) == DMA_IN_PROGRESS;
+
+	if (!dw_edma_ll_pending(chan))
+		return false;
+
+	/*
+	 * eDMA can report STOPPED while a doorbell-triggered start is
+	 * still pending. Native HDMA has a dedicated STOP event.
+	 */
+	return !dw_edma_ll_has_hdma_stop_event(chan) ||
+	       dw_edma_core_ch_status(chan) == DMA_IN_PROGRESS;
+}
+
 static int dw_edma_device_terminate_all(struct dma_chan *dchan)
 {
 	struct dw_edma_chan *chan = dchan2dw_edma_chan(dchan);
-	int err = 0;
 
 	guard(spinlock_irqsave)(&chan->vc.lock);
 
 	if (!chan->configured) {
 		dw_edma_terminate_all_descs(chan);
+		chan->request = EDMA_REQ_NONE;
 	} else if (chan->status == EDMA_ST_PAUSE) {
+		/* A paused channel has already stopped. */
 		dw_edma_finish_termination(chan);
-	} else if (chan->status == EDMA_ST_IDLE) {
-		dw_edma_finish_termination(chan);
-	} else if (dw_edma_core_ch_status(chan) == DMA_COMPLETE) {
+	} else if (dw_edma_may_be_active(chan)) {
 		/*
-		 * The channel is in a false BUSY state, probably didn't
-		 * receive or lost an interrupt
+		 * Keep published entries intact until an IRQ sample confirms
+		 * that the channel stopped. An outstanding EDMA_REQ_PAUSE is
+		 * replaced by EDMA_REQ_STOP.
 		 */
-		dw_edma_finish_termination(chan);
-	} else if (chan->request > EDMA_REQ_PAUSE) {
-		err = -EPERM;
-	} else {
 		dw_edma_set_request(chan, EDMA_REQ_STOP);
+		dw_edma_core_ch_maybe_doorbell_or_recheck(chan);
+	} else {
+		/*
+		 * No LL entry is hardware-owned, or non-LL status confirms that
+		 * the one programmed burst is not running.
+		 */
+		dw_edma_finish_termination(chan);
 	}
-	if (chan->status == EDMA_ST_IDLE &&
-	    !dw_edma_abort_is_pending(chan))
-		dw_edma_set_request(chan, EDMA_REQ_NONE);
 
-	return err;
+	return 0;
 }
 
 static void dw_edma_device_issue_pending(struct dma_chan *dchan)
@@ -1252,8 +1302,7 @@ static void dw_edma_done_interrupt(struct dw_edma_chan *chan)
 		}
 
 		if (chan->request == EDMA_REQ_PAUSE) {
-			dw_edma_set_request(chan, EDMA_REQ_NONE);
-			chan->status = EDMA_ST_PAUSE;
+			dw_edma_finish_pause(chan);
 			break;
 		}
 
@@ -1296,8 +1345,7 @@ static void dw_edma_ll_interrupt(struct dw_edma_chan *chan)
 		break;
 
 	case EDMA_REQ_PAUSE:
-		dw_edma_set_request(chan, EDMA_REQ_NONE);
-		chan->status = EDMA_ST_PAUSE;
+		dw_edma_finish_pause(chan);
 		break;
 
 	case EDMA_REQ_STOP:
@@ -1591,18 +1639,21 @@ static void dw_edma_wait_termination(struct dma_chan *dchan)
 	 * A STOP may be deferred to a later interrupt while the channel is still
 	 * running. Wait until that handler completes the termination.
 	 */
-	while (time_before(jiffies, timeout)) {
+	for (;;) {
 		scoped_guard(spinlock_irqsave, &chan->vc.lock)
 			stopping = chan->request == EDMA_REQ_STOP;
 
 		if (!stopping)
 			return;
 
+		if (time_after_eq(jiffies, timeout)) {
+			dev_warn(chan->dw->chip->dev,
+				 "timeout waiting for channel termination; still waiting\n");
+			timeout = jiffies + msecs_to_jiffies(5000);
+		}
+
 		fsleep(1000);
 	}
-
-	dev_warn(chan->dw->chip->dev,
-		 "timeout waiting for channel termination\n");
 }
 
 static void dw_edma_device_synchronize(struct dma_chan *dchan)
-- 
2.51.0