[PATCH v4 12/24] dmaengine: dw-edma: Keep channels stopped while ABORT is pending

[Koichiro Den]

The hard IRQ path clears the ABORT interrupt status before deferred
handling. Keep the event pending until the worker has handled it;
otherwise a new start can race ahead after the status has been cleared.

Set abort_pending before clearing the status. Before a non-LL start or
LL doorbell, check both the pending flag and hardware status while
holding the event lock. This covers the case where the start path gets
the lock before the IRQ handler has latched the event.

Handle ABORT before coalesced DONE or LL work. Also block LL progress
consumers while ABORT is pending; tx_status() can otherwise complete a
descriptor before the ABORT worker runs. ABORT cleanup makes that
progress stale by completing or terminating descriptors and, for an LL
channel, resetting the ring. Clear abort_pending only after cleanup.

If termination sees a pending ABORT, leave EDMA_REQ_STOP set until the
ABORT worker finishes. This prevents dw_edma_device_synchronize() from
discarding the event before cleanup.

Note also that only one issued descriptor can have LL entries published
at this point. Before multi-descriptor publication is enabled, the ring
reset is extended to abort every descriptor with outstanding published
entries. Therefore, discarding coalesced DONE or LL work cannot strand
another published descriptor.

Signed-off-by: Koichiro Den <den@xxxxxxxxxxxxx>
---
Changes in v4:
  - Clarify in commit message why discarding DONE or LL work coalesced
    with ABORT cannot strand a published descriptor. (Sashiko)

 drivers/dma/dw-edma/dw-edma-core.c    | 82 ++++++++++++++++++++++++---
 drivers/dma/dw-edma/dw-edma-core.h    | 17 ++++++
 drivers/dma/dw-edma/dw-edma-v0-core.c |  9 +++
 drivers/dma/dw-edma/dw-hdma-v0-core.c |  8 +++
 4 files changed, 107 insertions(+), 9 deletions(-)

diff --git a/drivers/dma/dw-edma/dw-edma-core.c b/drivers/dma/dw-edma/dw-edma-core.c
index 9ef7342a1093..4c4f19bf3c16 100644
--- a/drivers/dma/dw-edma/dw-edma-core.c
+++ b/drivers/dma/dw-edma/dw-edma-core.c
@@ -82,11 +82,30 @@ static void dw_edma_ll_snapshot_discard_locked(struct dw_edma_chan *chan)
 	chan->ll_irq.event = DW_EDMA_LL_EVENT_NONE;
 }
 
-static void dw_edma_ll_snapshot_discard(struct dw_edma_chan *chan)
+static void dw_edma_irq_events_discard(struct dw_edma_chan *chan)
 {
 	guard(spinlock_irqsave)(dw_edma_event_lock(chan));
 
 	dw_edma_ll_snapshot_discard_locked(chan);
+	chan->abort_pending = false;
+}
+
+static bool dw_edma_abort_latch_locked(struct dw_edma_chan *chan)
+{
+	lockdep_assert_held(dw_edma_event_lock(chan));
+
+	if (!chan->abort_pending &&
+	    dw_edma_core_ch_abort_int_pending(chan))
+		dw_edma_abort_event_mark(chan);
+
+	return chan->abort_pending;
+}
+
+static bool dw_edma_abort_is_pending(struct dw_edma_chan *chan)
+{
+	guard(spinlock_irqsave)(dw_edma_event_lock(chan));
+
+	return chan->abort_pending;
 }
 
 /* Must be called with vc.lock held. */
@@ -99,12 +118,23 @@ dw_edma_ll_snapshot_take(struct dw_edma_chan *chan,
 	if (chan->ll_irq.event == DW_EDMA_LL_EVENT_NONE)
 		return false;
 
+	if (dw_edma_abort_latch_locked(chan))
+		return false;
+
 	*snapshot = chan->ll_irq;
 	dw_edma_ll_snapshot_discard_locked(chan);
 
 	return true;
 }
 
+/* Must be called with vc.lock held. */
+static void dw_edma_abort_event_clear(struct dw_edma_chan *chan)
+{
+	guard(spinlock_irqsave)(dw_edma_event_lock(chan));
+
+	chan->abort_pending = false;
+}
+
 static void dw_edma_ll_event_discard_locked(struct dw_edma_chan *chan)
 {
 	lockdep_assert_held(dw_edma_event_lock(chan));
@@ -253,7 +283,20 @@ static int dw_edma_start_transfer(struct dw_edma_chan *chan)
 	if (!desc)
 		return 0;
 
-	if (!chan->non_ll && !chan->ll_valid)
+	if (chan->non_ll) {
+		guard(spinlock_irqsave)(dw_edma_event_lock(chan));
+
+		if (dw_edma_abort_latch_locked(chan))
+			return 0;
+
+		dw_edma_core_start(desc);
+		return 1;
+	}
+
+	if (dw_edma_abort_is_pending(chan))
+		return 0;
+
+	if (!chan->ll_valid)
 		dw_edma_core_reset_ll(chan);
 
 	dw_edma_core_start(desc);
@@ -293,6 +336,11 @@ static void dw_edma_terminate_all_descs(struct dw_edma_chan *chan)
 /* Must be called with vc.lock held after the channel has stopped. */
 static void dw_edma_finish_termination(struct dw_edma_chan *chan)
 {
+	if (dw_edma_abort_is_pending(chan)) {
+		dw_edma_set_request(chan, EDMA_REQ_STOP);
+		return;
+	}
+
 	dw_edma_terminate_all_descs(chan);
 
 	/* Preserve a clean ring; resync only if entries remain published. */
@@ -320,6 +368,9 @@ static void dw_edma_core_ch_doorbell(struct dw_edma_chan *chan)
 
 	guard(spinlock_irqsave)(dw_edma_event_lock(chan));
 
+	if (dw_edma_abort_latch_locked(chan))
+		return;
+
 	/*
 	 * A recorded event belongs to the current ring and must be consumed
 	 * before starting another hardware run.
@@ -496,7 +547,8 @@ static int dw_edma_device_terminate_all(struct dma_chan *dchan)
 	} else {
 		dw_edma_set_request(chan, EDMA_REQ_STOP);
 	}
-	if (chan->status == EDMA_ST_IDLE)
+	if (chan->status == EDMA_ST_IDLE &&
+	    !dw_edma_abort_is_pending(chan))
 		dw_edma_set_request(chan, EDMA_REQ_NONE);
 
 	return err;
@@ -850,14 +902,19 @@ static void dw_edma_ll_interrupt(struct dw_edma_chan *chan)
 	dw_edma_done_interrupt_locked(chan);
 }
 
-static void dw_edma_abort_interrupt(struct dw_edma_chan *chan)
+static bool dw_edma_abort_interrupt(struct dw_edma_chan *chan)
 {
 	struct virt_dma_desc *vd;
 	unsigned long flags;
 
 	spin_lock_irqsave(&chan->vc.lock, flags);
+	if (!dw_edma_abort_is_pending(chan)) {
+		spin_unlock_irqrestore(&chan->vc.lock, flags);
+		return false;
+	}
+
 	vd = vchan_next_desc(&chan->vc);
-	if (vd && chan->request == EDMA_REQ_STOP) {
+	if (chan->request == EDMA_REQ_STOP) {
 		dw_edma_terminate_all_descs(chan);
 	} else if (vd) {
 		dw_hdma_set_callback_result(vd, DMA_TRANS_ABORTED);
@@ -868,7 +925,11 @@ static void dw_edma_abort_interrupt(struct dw_edma_chan *chan)
 		dw_edma_core_reset_ll(chan);
 	dw_edma_set_request(chan, EDMA_REQ_NONE);
 	chan->status = EDMA_ST_IDLE;
+	dw_edma_abort_event_clear(chan);
+
 	spin_unlock_irqrestore(&chan->vc.lock, flags);
+
+	return true;
 }
 
 static void dw_edma_irq_work(struct work_struct *work)
@@ -880,12 +941,14 @@ static void dw_edma_irq_work(struct work_struct *work)
 	do {
 		events = atomic_xchg(&chan->irq_pending, 0);
 
+		/* ABORT cleanup makes coalesced DONE and LL work stale. */
+		if ((events & DW_EDMA_DEFERRED_ABORT) &&
+		    dw_edma_abort_interrupt(chan))
+			continue;
 		if (events & DW_EDMA_DEFERRED_DONE)
 			dw_edma_done_interrupt(chan);
 		if (events & DW_EDMA_DEFERRED_LL)
 			dw_edma_ll_interrupt(chan);
-		if (events & DW_EDMA_DEFERRED_ABORT)
-			dw_edma_abort_interrupt(chan);
 	} while (atomic_read(&chan->irq_pending));
 }
 
@@ -906,7 +969,7 @@ static void dw_edma_record_irq(struct dw_edma_chan *chan, unsigned int events)
 
 	lockdep_assert_held(dw_edma_event_lock(chan));
 
-	if (events & DW_EDMA_IRQ_ABORT)
+	if ((events & DW_EDMA_IRQ_ABORT) && chan->abort_pending)
 		pending |= DW_EDMA_DEFERRED_ABORT;
 
 	if (chan->non_ll) {
@@ -1108,7 +1171,7 @@ static void dw_edma_device_synchronize(struct dma_chan *dchan)
 	dw_edma_wait_termination(dchan);
 	cancel_work_sync(&chan->irq_work);
 	atomic_set(&chan->irq_pending, 0);
-	dw_edma_ll_snapshot_discard(chan);
+	dw_edma_irq_events_discard(chan);
 	vchan_synchronize(&chan->vc);
 }
 
@@ -1161,6 +1224,7 @@ static int dw_edma_channel_setup(struct dw_edma *dw, u32 wr_alloc, u32 rd_alloc)
 		INIT_WORK(&chan->irq_work, dw_edma_irq_work);
 		atomic_set(&chan->irq_pending, 0);
 		chan->ll_irq.event = DW_EDMA_LL_EVENT_NONE;
+		chan->abort_pending = false;
 
 		if (chan->dir == EDMA_DIR_WRITE)
 			chan->ll_region = chip->ll_region_wr[chan->id];
diff --git a/drivers/dma/dw-edma/dw-edma-core.h b/drivers/dma/dw-edma/dw-edma-core.h
index b6de4ac9a8aa..82c0d66747be 100644
--- a/drivers/dma/dw-edma/dw-edma-core.h
+++ b/drivers/dma/dw-edma/dw-edma-core.h
@@ -115,6 +115,8 @@ struct dw_edma_chan {
 	 * its consumption with LL state.
 	 */
 	struct dw_edma_ll_snapshot	ll_irq;
+	/* ABORT is terminal and remains pending across LL state changes. */
+	bool				abort_pending;
 	/* Native HDMA lock storage. */
 	spinlock_t			event_lock_per_chan;
 	spinlock_t			*event_lock;	/* Selected event lock */
@@ -185,6 +187,8 @@ struct dw_edma_core_ops {
 	int (*ch_quiesce)(struct dw_edma_chan *chan);
 	u16 (*ch_count)(struct dw_edma *dw, enum dw_edma_dir dir);
 	enum dma_status (*ch_status)(struct dw_edma_chan *chan);
+	/* Called with the event scope locked. */
+	bool (*ch_abort_int_pending)(struct dw_edma_chan *chan);
 	irqreturn_t (*handle_int)(struct dw_edma_irq *dw_irq, enum dw_edma_dir dir,
 				  dw_edma_handler_t handler);
 	void (*non_ll_start)(struct dw_edma_chan *chan, struct dw_edma_burst *child);
@@ -252,6 +256,13 @@ static inline spinlock_t *dw_edma_event_lock(struct dw_edma_chan *chan)
 	return chan->event_lock;
 }
 
+static inline void dw_edma_abort_event_mark(struct dw_edma_chan *chan)
+{
+	lockdep_assert_held(dw_edma_event_lock(chan));
+
+	chan->abort_pending = true;
+}
+
 /*
  * Return the current LL entry index. A negative value means that the channel
  * context is not initialized or was lost after a link reset.
@@ -299,6 +310,12 @@ enum dma_status dw_edma_core_ch_status(struct dw_edma_chan *chan)
 	return chan->dw->core->ch_status(chan);
 }
 
+static inline bool
+dw_edma_core_ch_abort_int_pending(struct dw_edma_chan *chan)
+{
+	return chan->dw->core->ch_abort_int_pending(chan);
+}
+
 static inline irqreturn_t
 dw_edma_core_handle_int(struct dw_edma_irq *dw_irq, enum dw_edma_dir dir,
 			dw_edma_handler_t handler)
diff --git a/drivers/dma/dw-edma/dw-edma-v0-core.c b/drivers/dma/dw-edma/dw-edma-v0-core.c
index 3f3e94b9073f..451e421b11b8 100644
--- a/drivers/dma/dw-edma/dw-edma-v0-core.c
+++ b/drivers/dma/dw-edma/dw-edma-v0-core.c
@@ -318,6 +318,13 @@ static enum dma_status dw_edma_v0_core_ch_status(struct dw_edma_chan *chan)
 		return DMA_ERROR;
 }
 
+static bool dw_edma_v0_core_ch_abort_int_pending(struct dw_edma_chan *chan)
+{
+	u32 sts = GET_RW_32(chan->dw, chan->dir, int_status);
+
+	return FIELD_GET(EDMA_V0_ABORT_INT_MASK, sts) & BIT(chan->id);
+}
+
 static void dw_edma_v0_core_clear_done_int(struct dw_edma_chan *chan)
 {
 	struct dw_edma *dw = chan->dw;
@@ -390,6 +397,7 @@ dw_edma_v0_core_handle_int(struct dw_edma_irq *dw_irq, enum dw_edma_dir dir,
 
 			events[pos] |= DW_EDMA_IRQ_ABORT;
 			active |= BIT(pos);
+			dw_edma_abort_event_mark(chan);
 			dw_edma_v0_core_clear_abort_int(chan);
 		}
 
@@ -667,6 +675,7 @@ static const struct dw_edma_core_ops dw_edma_v0_core = {
 	.ch_quiesce = dw_edma_v0_core_ch_quiesce,
 	.ch_count = dw_edma_v0_core_ch_count,
 	.ch_status = dw_edma_v0_core_ch_status,
+	.ch_abort_int_pending = dw_edma_v0_core_ch_abort_int_pending,
 	.handle_int = dw_edma_v0_core_handle_int,
 	.ll_data = dw_edma_v0_core_ll_data,
 	.ll_link = dw_edma_v0_core_ll_link,
diff --git a/drivers/dma/dw-edma/dw-hdma-v0-core.c b/drivers/dma/dw-edma/dw-hdma-v0-core.c
index 617e39f7a8b7..0ed4e1b8240a 100644
--- a/drivers/dma/dw-edma/dw-hdma-v0-core.c
+++ b/drivers/dma/dw-edma/dw-hdma-v0-core.c
@@ -172,6 +172,12 @@ static u32 dw_hdma_v0_core_status_int(struct dw_edma_chan *chan)
 	return GET_CH_32(dw, chan->dir, chan->id, int_stat);
 }
 
+static bool dw_hdma_v0_core_ch_abort_int_pending(struct dw_edma_chan *chan)
+{
+	return FIELD_GET(HDMA_V0_ABORT_INT_MASK,
+			 dw_hdma_v0_core_status_int(chan));
+}
+
 static irqreturn_t
 dw_hdma_v0_core_handle_int(struct dw_edma_irq *dw_irq, enum dw_edma_dir dir,
 			   dw_edma_handler_t handler)
@@ -210,6 +216,7 @@ dw_hdma_v0_core_handle_int(struct dw_edma_irq *dw_irq, enum dw_edma_dir dir,
 
 			if (FIELD_GET(HDMA_V0_ABORT_INT_MASK, val)) {
 				events |= DW_EDMA_IRQ_ABORT;
+				dw_edma_abort_event_mark(chan);
 				dw_hdma_v0_core_clear_abort_int(chan);
 			}
 
@@ -429,6 +436,7 @@ static const struct dw_edma_core_ops dw_hdma_v0_core = {
 	.ch_quiesce = dw_hdma_v0_core_ch_quiesce,
 	.ch_count = dw_hdma_v0_core_ch_count,
 	.ch_status = dw_hdma_v0_core_ch_status,
+	.ch_abort_int_pending = dw_hdma_v0_core_ch_abort_int_pending,
 	.handle_int = dw_hdma_v0_core_handle_int,
 	.non_ll_start = dw_hdma_v0_core_non_ll_start,
 	.ll_data = dw_hdma_v0_core_ll_data,
-- 
2.51.0