[PATCH v3 12/24] dmaengine: dw-edma: Keep channels stopped while ABORT is pending

[Koichiro Den]

The hard IRQ path clears the ABORT interrupt status before deferred
handling. Keep the event pending until the worker has handled it;
otherwise a new start can race ahead after the status has been cleared.

Set abort_pending before clearing the status. Before a non-LL start or
LL doorbell, check both the pending flag and hardware status while
holding the event lock. This covers the case where the start path gets
the lock before the IRQ handler has latched the event.

Handle ABORT before coalesced DONE or LL work. Also block LL progress
consumers while ABORT is pending; tx_status() can otherwise complete a
descriptor before the ABORT worker runs. ABORT cleanup makes that
progress stale by completing or terminating descriptors and, for an LL
channel, resetting the ring. Clear abort_pending only after cleanup.

If termination sees a pending ABORT, leave EDMA_REQ_STOP set until the
ABORT worker finishes. This prevents dw_edma_device_synchronize() from
discarding the event before cleanup.

Signed-off-by: Koichiro Den <den@xxxxxxxxxxxxx>
---
Changes in v3:
  - New patch to latch ABORT before clearing its interrupt status, block
    channel restart and LL progress consumers while it is pending, and
    keep termination pending until deferred cleanup completes. (Sashiko)

 drivers/dma/dw-edma/dw-edma-core.c    | 82 ++++++++++++++++++++++++---
 drivers/dma/dw-edma/dw-edma-core.h    | 17 ++++++
 drivers/dma/dw-edma/dw-edma-v0-core.c |  9 +++
 drivers/dma/dw-edma/dw-hdma-v0-core.c |  8 +++
 4 files changed, 107 insertions(+), 9 deletions(-)

diff --git a/drivers/dma/dw-edma/dw-edma-core.c b/drivers/dma/dw-edma/dw-edma-core.c
index 3970b066dca2..5bf08ff0fe46 100644
--- a/drivers/dma/dw-edma/dw-edma-core.c
+++ b/drivers/dma/dw-edma/dw-edma-core.c
@@ -82,11 +82,30 @@ static void dw_edma_ll_snapshot_discard_locked(struct dw_edma_chan *chan)
 	chan->ll_irq.event = DW_EDMA_LL_EVENT_NONE;
 }
 
-static void dw_edma_ll_snapshot_discard(struct dw_edma_chan *chan)
+static void dw_edma_irq_events_discard(struct dw_edma_chan *chan)
 {
 	guard(raw_spinlock_irqsave)(dw_edma_event_lock(chan));
 
 	dw_edma_ll_snapshot_discard_locked(chan);
+	chan->abort_pending = false;
+}
+
+static bool dw_edma_abort_latch_locked(struct dw_edma_chan *chan)
+{
+	lockdep_assert_held(dw_edma_event_lock(chan));
+
+	if (!chan->abort_pending &&
+	    dw_edma_core_ch_abort_int_pending(chan))
+		dw_edma_abort_event_mark(chan);
+
+	return chan->abort_pending;
+}
+
+static bool dw_edma_abort_is_pending(struct dw_edma_chan *chan)
+{
+	guard(raw_spinlock_irqsave)(dw_edma_event_lock(chan));
+
+	return chan->abort_pending;
 }
 
 /* Must be called with vc.lock held. */
@@ -99,12 +118,23 @@ dw_edma_ll_snapshot_take(struct dw_edma_chan *chan,
 	if (chan->ll_irq.event == DW_EDMA_LL_EVENT_NONE)
 		return false;
 
+	if (dw_edma_abort_latch_locked(chan))
+		return false;
+
 	*snapshot = chan->ll_irq;
 	dw_edma_ll_snapshot_discard_locked(chan);
 
 	return true;
 }
 
+/* Must be called with vc.lock held. */
+static void dw_edma_abort_event_clear(struct dw_edma_chan *chan)
+{
+	guard(raw_spinlock_irqsave)(dw_edma_event_lock(chan));
+
+	chan->abort_pending = false;
+}
+
 static void dw_edma_ll_event_discard_locked(struct dw_edma_chan *chan)
 {
 	lockdep_assert_held(dw_edma_event_lock(chan));
@@ -253,7 +283,20 @@ static int dw_edma_start_transfer(struct dw_edma_chan *chan)
 	if (!desc)
 		return 0;
 
-	if (!chan->non_ll && !chan->ll_valid)
+	if (chan->non_ll) {
+		guard(raw_spinlock_irqsave)(dw_edma_event_lock(chan));
+
+		if (dw_edma_abort_latch_locked(chan))
+			return 0;
+
+		dw_edma_core_start(desc);
+		return 1;
+	}
+
+	if (dw_edma_abort_is_pending(chan))
+		return 0;
+
+	if (!chan->ll_valid)
 		dw_edma_core_reset_ll(chan);
 
 	dw_edma_core_start(desc);
@@ -293,6 +336,11 @@ static void dw_edma_terminate_all_descs(struct dw_edma_chan *chan)
 /* Must be called with vc.lock held after the channel has stopped. */
 static void dw_edma_finish_termination(struct dw_edma_chan *chan)
 {
+	if (dw_edma_abort_is_pending(chan)) {
+		dw_edma_set_request(chan, EDMA_REQ_STOP);
+		return;
+	}
+
 	dw_edma_terminate_all_descs(chan);
 
 	/* Preserve a clean ring; resync only if entries remain published. */
@@ -320,6 +368,9 @@ static void dw_edma_core_ch_kick(struct dw_edma_chan *chan)
 
 	guard(raw_spinlock_irqsave)(dw_edma_event_lock(chan));
 
+	if (dw_edma_abort_latch_locked(chan))
+		return;
+
 	/*
 	 * A recorded event belongs to the current ring and must be consumed
 	 * before starting another hardware run.
@@ -496,7 +547,8 @@ static int dw_edma_device_terminate_all(struct dma_chan *dchan)
 	} else {
 		dw_edma_set_request(chan, EDMA_REQ_STOP);
 	}
-	if (chan->status == EDMA_ST_IDLE)
+	if (chan->status == EDMA_ST_IDLE &&
+	    !dw_edma_abort_is_pending(chan))
 		dw_edma_set_request(chan, EDMA_REQ_NONE);
 
 	return err;
@@ -850,14 +902,19 @@ static void dw_edma_ll_interrupt(struct dw_edma_chan *chan)
 	dw_edma_done_interrupt_locked(chan);
 }
 
-static void dw_edma_abort_interrupt(struct dw_edma_chan *chan)
+static bool dw_edma_abort_interrupt(struct dw_edma_chan *chan)
 {
 	struct virt_dma_desc *vd;
 	unsigned long flags;
 
 	spin_lock_irqsave(&chan->vc.lock, flags);
+	if (!dw_edma_abort_is_pending(chan)) {
+		spin_unlock_irqrestore(&chan->vc.lock, flags);
+		return false;
+	}
+
 	vd = vchan_next_desc(&chan->vc);
-	if (vd && chan->request == EDMA_REQ_STOP) {
+	if (chan->request == EDMA_REQ_STOP) {
 		dw_edma_terminate_all_descs(chan);
 	} else if (vd) {
 		dw_hdma_set_callback_result(vd, DMA_TRANS_ABORTED);
@@ -868,7 +925,11 @@ static void dw_edma_abort_interrupt(struct dw_edma_chan *chan)
 		dw_edma_core_reset_ll(chan);
 	dw_edma_set_request(chan, EDMA_REQ_NONE);
 	chan->status = EDMA_ST_IDLE;
+	dw_edma_abort_event_clear(chan);
+
 	spin_unlock_irqrestore(&chan->vc.lock, flags);
+
+	return true;
 }
 
 static void dw_edma_irq_work(struct work_struct *work)
@@ -880,12 +941,14 @@ static void dw_edma_irq_work(struct work_struct *work)
 	do {
 		events = atomic_xchg(&chan->irq_pending, 0);
 
+		/* ABORT cleanup makes coalesced DONE and LL work stale. */
+		if ((events & DW_EDMA_DEFERRED_ABORT) &&
+		    dw_edma_abort_interrupt(chan))
+			continue;
 		if (events & DW_EDMA_DEFERRED_DONE)
 			dw_edma_done_interrupt(chan);
 		if (events & DW_EDMA_DEFERRED_LL)
 			dw_edma_ll_interrupt(chan);
-		if (events & DW_EDMA_DEFERRED_ABORT)
-			dw_edma_abort_interrupt(chan);
 	} while (atomic_read(&chan->irq_pending));
 }
 
@@ -906,7 +969,7 @@ static void dw_edma_record_irq(struct dw_edma_chan *chan, unsigned int events)
 
 	lockdep_assert_held(dw_edma_event_lock(chan));
 
-	if (events & DW_EDMA_IRQ_ABORT)
+	if ((events & DW_EDMA_IRQ_ABORT) && chan->abort_pending)
 		pending |= DW_EDMA_DEFERRED_ABORT;
 
 	if (chan->non_ll) {
@@ -1108,7 +1171,7 @@ static void dw_edma_device_synchronize(struct dma_chan *dchan)
 	dw_edma_wait_termination(dchan);
 	cancel_work_sync(&chan->irq_work);
 	atomic_set(&chan->irq_pending, 0);
-	dw_edma_ll_snapshot_discard(chan);
+	dw_edma_irq_events_discard(chan);
 	vchan_synchronize(&chan->vc);
 }
 
@@ -1161,6 +1224,7 @@ static int dw_edma_channel_setup(struct dw_edma *dw, u32 wr_alloc, u32 rd_alloc)
 		INIT_WORK(&chan->irq_work, dw_edma_irq_work);
 		atomic_set(&chan->irq_pending, 0);
 		chan->ll_irq.event = DW_EDMA_LL_EVENT_NONE;
+		chan->abort_pending = false;
 		raw_spin_lock_init(&chan->event_lock);
 
 		if (chan->dir == EDMA_DIR_WRITE)
diff --git a/drivers/dma/dw-edma/dw-edma-core.h b/drivers/dma/dw-edma/dw-edma-core.h
index 9f528f160cea..2bfbd1b25703 100644
--- a/drivers/dma/dw-edma/dw-edma-core.h
+++ b/drivers/dma/dw-edma/dw-edma-core.h
@@ -120,6 +120,8 @@ struct dw_edma_chan {
 	 * its consumption with LL state.
 	 */
 	struct dw_edma_ll_snapshot	ll_irq;
+	/* ABORT is terminal and remains pending across LL state changes. */
+	bool				abort_pending;
 	raw_spinlock_t			event_lock;
 
 	u32				ll_max;		/* Data entries */
@@ -187,6 +189,8 @@ struct dw_edma_core_ops {
 	int (*ch_quiesce)(struct dw_edma_chan *chan);
 	u16 (*ch_count)(struct dw_edma *dw, enum dw_edma_dir dir);
 	enum dma_status (*ch_status)(struct dw_edma_chan *chan);
+	/* Called with the event scope locked. */
+	bool (*ch_abort_int_pending)(struct dw_edma_chan *chan);
 	enum dw_edma_event_scope event_scope;
 	irqreturn_t (*handle_int)(struct dw_edma_irq *dw_irq, enum dw_edma_dir dir,
 				  dw_edma_handler_t handler);
@@ -258,6 +262,13 @@ static inline raw_spinlock_t *dw_edma_event_lock(struct dw_edma_chan *chan)
 	return &chan->event_lock;
 }
 
+static inline void dw_edma_abort_event_mark(struct dw_edma_chan *chan)
+{
+	lockdep_assert_held(dw_edma_event_lock(chan));
+
+	chan->abort_pending = true;
+}
+
 /*
  * Return the current LL entry index. A negative value means that the channel
  * context is not initialized or was lost after a link reset.
@@ -305,6 +316,12 @@ enum dma_status dw_edma_core_ch_status(struct dw_edma_chan *chan)
 	return chan->dw->core->ch_status(chan);
 }
 
+static inline bool
+dw_edma_core_ch_abort_int_pending(struct dw_edma_chan *chan)
+{
+	return chan->dw->core->ch_abort_int_pending(chan);
+}
+
 static inline irqreturn_t
 dw_edma_core_handle_int(struct dw_edma_irq *dw_irq, enum dw_edma_dir dir,
 			dw_edma_handler_t handler)
diff --git a/drivers/dma/dw-edma/dw-edma-v0-core.c b/drivers/dma/dw-edma/dw-edma-v0-core.c
index 43e33a400a3e..91b5d331cfa3 100644
--- a/drivers/dma/dw-edma/dw-edma-v0-core.c
+++ b/drivers/dma/dw-edma/dw-edma-v0-core.c
@@ -318,6 +318,13 @@ static enum dma_status dw_edma_v0_core_ch_status(struct dw_edma_chan *chan)
 		return DMA_ERROR;
 }
 
+static bool dw_edma_v0_core_ch_abort_int_pending(struct dw_edma_chan *chan)
+{
+	u32 sts = GET_RW_32(chan->dw, chan->dir, int_status);
+
+	return FIELD_GET(EDMA_V0_ABORT_INT_MASK, sts) & BIT(chan->id);
+}
+
 static void dw_edma_v0_core_clear_done_int(struct dw_edma_chan *chan)
 {
 	struct dw_edma *dw = chan->dw;
@@ -387,6 +394,7 @@ dw_edma_v0_core_handle_int(struct dw_edma_irq *dw_irq, enum dw_edma_dir dir,
 
 			events[pos] |= DW_EDMA_IRQ_ABORT;
 			active |= BIT(pos);
+			dw_edma_abort_event_mark(chan);
 			dw_edma_v0_core_clear_abort_int(chan);
 		}
 
@@ -664,6 +672,7 @@ static const struct dw_edma_core_ops dw_edma_v0_core = {
 	.ch_quiesce = dw_edma_v0_core_ch_quiesce,
 	.ch_count = dw_edma_v0_core_ch_count,
 	.ch_status = dw_edma_v0_core_ch_status,
+	.ch_abort_int_pending = dw_edma_v0_core_ch_abort_int_pending,
 	.event_scope = DW_EDMA_EVENT_PER_DIR,
 	.handle_int = dw_edma_v0_core_handle_int,
 	.ll_data = dw_edma_v0_core_ll_data,
diff --git a/drivers/dma/dw-edma/dw-hdma-v0-core.c b/drivers/dma/dw-edma/dw-hdma-v0-core.c
index bbf9ee0392f8..5d0d7575bae9 100644
--- a/drivers/dma/dw-edma/dw-hdma-v0-core.c
+++ b/drivers/dma/dw-edma/dw-hdma-v0-core.c
@@ -172,6 +172,12 @@ static u32 dw_hdma_v0_core_status_int(struct dw_edma_chan *chan)
 	return GET_CH_32(dw, chan->dir, chan->id, int_stat);
 }
 
+static bool dw_hdma_v0_core_ch_abort_int_pending(struct dw_edma_chan *chan)
+{
+	return FIELD_GET(HDMA_V0_ABORT_INT_MASK,
+			 dw_hdma_v0_core_status_int(chan));
+}
+
 static irqreturn_t
 dw_hdma_v0_core_handle_int(struct dw_edma_irq *dw_irq, enum dw_edma_dir dir,
 			   dw_edma_handler_t handler)
@@ -210,6 +216,7 @@ dw_hdma_v0_core_handle_int(struct dw_edma_irq *dw_irq, enum dw_edma_dir dir,
 
 			if (FIELD_GET(HDMA_V0_ABORT_INT_MASK, val)) {
 				events |= DW_EDMA_IRQ_ABORT;
+				dw_edma_abort_event_mark(chan);
 				dw_hdma_v0_core_clear_abort_int(chan);
 			}
 
@@ -429,6 +436,7 @@ static const struct dw_edma_core_ops dw_hdma_v0_core = {
 	.ch_quiesce = dw_hdma_v0_core_ch_quiesce,
 	.ch_count = dw_hdma_v0_core_ch_count,
 	.ch_status = dw_hdma_v0_core_ch_status,
+	.ch_abort_int_pending = dw_hdma_v0_core_ch_abort_int_pending,
 	.event_scope = DW_EDMA_EVENT_PER_CHAN,
 	.handle_int = dw_hdma_v0_core_handle_int,
 	.non_ll_start = dw_hdma_v0_core_non_ll_start,
-- 
2.51.0