[PATCH 10/17] dmaengine: dw-edma: Reclaim issued descriptors from LL progress

[Koichiro Den]

Dynamic append can leave LL entries for more than one issued descriptor
in the ring at the same time. A DONE or progress interrupt therefore
cannot complete work by looking only at the first issued descriptor.

Track the hardware consumption point separately from descriptor
completion. ll_done follows the LL pointer reported by hardware; ll_end
remains the boundary of the last completed descriptor. Measure free
space from ll_done, so consumed LL entries can be reused even before the
descriptor that owns them has completed. This keeps descriptors larger
than the LL ring moving.

Keep start_burst as the appended boundary and done_burst as the consumed
boundary. Complete issued descriptors whose fully appended LL range is
covered by the latest progress segment. The ring keeps one data entry
free, so a reported physical LL index is unique within the current
ll_done..ll_head producer window.

For eDMA, LL element LIE/RIE interrupts are reported through the DONE
interrupt status, and the documented producer-consumer flow reads
DMA_LLP_* on such interrupts to recycle elements up to the reported
location. HDMA STOP handling also uses this common accounting. The
following patch wires HDMA watermark interrupts as running progress
events.

[den: dw_edma_ll_clean_pending() naming and core idea are borrowed from
 20260109-edma_dymatic-v1-3-9a98c9c98536@xxxxxxx]
Signed-off-by: Koichiro Den <den@xxxxxxxxxxxxx>
---
 drivers/dma/dw-edma/dw-edma-core.c | 111 +++++++++++++++++++++++++++--
 drivers/dma/dw-edma/dw-edma-core.h |  18 ++---
 2 files changed, 116 insertions(+), 13 deletions(-)

diff --git a/drivers/dma/dw-edma/dw-edma-core.c b/drivers/dma/dw-edma/dw-edma-core.c
index 2165f2fa5398..839bafc762a1 100644
--- a/drivers/dma/dw-edma/dw-edma-core.c
+++ b/drivers/dma/dw-edma/dw-edma-core.c
@@ -65,6 +65,7 @@ static void dw_edma_core_reset_ll(struct dw_edma_chan *chan)
 {
 	chan->ll_head = 0;
 	chan->ll_end = 0;
+	chan->ll_done = 0;
 	chan->cb = true;
 
 	dw_edma_core_ll_link(chan, chan->ll_max - 1, chan->cb,
@@ -73,13 +74,34 @@ static void dw_edma_core_reset_ll(struct dw_edma_chan *chan)
 	dw_edma_core_ch_enable(chan);
 }
 
+static u32 dw_edma_core_get_ll_data_cnt(struct dw_edma_chan *chan)
+{
+	return chan->ll_max - 1;
+}
+
+static u32 dw_edma_core_get_ll_dist(struct dw_edma_chan *chan, u32 from, u32 to)
+{
+	u32 cnt = dw_edma_core_get_ll_data_cnt(chan);
+
+	return (to + cnt - from) % cnt;
+}
+
+static u32 dw_edma_core_get_used_num(struct dw_edma_chan *chan)
+{
+	return dw_edma_core_get_ll_dist(chan, chan->ll_done, chan->ll_head);
+}
+
 static u32 dw_edma_core_get_free_num(struct dw_edma_chan *chan)
 {
 	/*
+	 * Measure occupancy from ll_done, not ll_end. Entries consumed by the
+	 * hardware can be reused even if the descriptor that owns them has not
+	 * completed yet; this lets descriptors larger than the ring move forward.
+	 *
 	 * Max entries is ll_max - 1 because last one used for link back to
 	 * start of ll_region.
 	 */
-	return (chan->ll_end + chan->ll_max - 2 - chan->ll_head) %
+	return (chan->ll_done + chan->ll_max - 2 - chan->ll_head) %
 		(chan->ll_max - 1);
 }
 
@@ -89,6 +111,29 @@ static bool dw_edma_core_enable_ll_irq(struct dw_edma_desc *desc, u32 i,
 	return desc->chan->dw->core->ll_irq(desc, i, free);
 }
 
+static bool dw_edma_ll_advance(struct dw_edma_chan *chan, int idx, u32 *old_done)
+{
+	u32 cnt = dw_edma_core_get_ll_data_cnt(chan);
+	u32 done;
+
+	if (idx < 0 || (u32)idx >= cnt)
+		return false;
+
+	done = dw_edma_core_get_ll_dist(chan, chan->ll_done, idx);
+	if (!done || done > dw_edma_core_get_used_num(chan))
+		return false;
+
+	*old_done = chan->ll_done;
+	chan->ll_done = idx;
+
+	return true;
+}
+
+static bool dw_edma_ll_pending(struct dw_edma_chan *chan)
+{
+	return dw_edma_core_get_used_num(chan);
+}
+
 static void dw_edma_core_start(struct dw_edma_desc *desc)
 {
 	struct dw_edma_chan *chan = desc->chan;
@@ -121,7 +166,6 @@ static void dw_edma_core_start(struct dw_edma_desc *desc)
 		}
 	}
 
-	desc->done_burst = desc->start_burst;
 	desc->start_burst = i;
 	desc->ll_end = chan->ll_head;
 
@@ -164,9 +208,7 @@ static void dw_hdma_set_callback_result(struct virt_dma_desc *vd,
 	if (desc) {
 		residue = desc->alloc_sz;
 
-		if (result == DMA_TRANS_NOERROR)
-			residue -= desc->burst[desc->start_burst - 1].xfer_sz;
-		else if (desc->done_burst)
+		if (desc->done_burst)
 			residue -= desc->burst[desc->done_burst - 1].xfer_sz;
 	}
 
@@ -204,6 +246,59 @@ static void dw_edma_terminate_all_descs(struct dw_edma_chan *chan)
 	dw_edma_terminate_vdesc_list(&chan->vc.desc_submitted);
 }
 
+/* Must be called with vc.lock held. */
+static void dw_edma_ll_clean_pending(struct dw_edma_chan *chan, u32 old_done)
+{
+	struct virt_dma_desc *vd, *_vd;
+	u32 done = dw_edma_core_get_ll_dist(chan, old_done, chan->ll_done);
+
+	list_for_each_entry_safe(vd, _vd, &chan->vc.desc_issued, node) {
+		struct dw_edma_desc *desc = vd2dw_edma_desc(vd);
+		u32 consumed, started;
+
+		if (!done)
+			break;
+
+		/*
+		 * start_burst is the append boundary. done_burst is the
+		 * hardware-consumed boundary reported through LL progress.
+		 */
+		started = desc->start_burst - desc->done_burst;
+		if (!started)
+			break;
+
+		consumed = min(done, started);
+		desc->done_burst += consumed;
+		done -= consumed;
+
+		/*
+		 * Descriptors are appended in list order, so later descriptors
+		 * cannot be complete if this one has not been fully consumed.
+		 */
+		if (desc->done_burst != desc->nburst)
+			break;
+
+		/* Hardware has consumed this descriptor's LL entries. */
+		dw_hdma_set_callback_result(vd, DMA_TRANS_NOERROR);
+		list_del(&vd->node);
+		vchan_cookie_complete(vd);
+		chan->ll_end = desc->ll_end;
+	}
+}
+
+/* Must be called with vc.lock held. */
+static bool dw_edma_ll_recycle(struct dw_edma_chan *chan, int idx)
+{
+	u32 old_done;
+
+	if (!dw_edma_ll_advance(chan, idx, &old_done))
+		return false;
+
+	dw_edma_ll_clean_pending(chan, old_done);
+
+	return true;
+}
+
 static void dw_edma_device_caps(struct dma_chan *dchan,
 				struct dma_slave_caps *caps)
 {
@@ -600,8 +695,11 @@ static void dw_edma_done_interrupt(struct dw_edma_chan *chan)
 	struct dw_edma_desc *desc;
 	struct virt_dma_desc *vd;
 	unsigned long flags;
+	int idx;
 
 	spin_lock_irqsave(&chan->vc.lock, flags);
+	idx = dw_edma_core_ll_cur_idx(chan);
+	dw_edma_ll_recycle(chan, idx);
 	vd = vchan_next_desc(&chan->vc);
 	if (vd) {
 		switch (chan->request) {
@@ -634,6 +732,9 @@ static void dw_edma_done_interrupt(struct dw_edma_chan *chan)
 		default:
 			break;
 		}
+	} else if (chan->request == EDMA_REQ_NONE) {
+		chan->status = dw_edma_ll_pending(chan) ?
+			       EDMA_ST_BUSY : EDMA_ST_IDLE;
 	}
 	spin_unlock_irqrestore(&chan->vc.lock, flags);
 }
diff --git a/drivers/dma/dw-edma/dw-edma-core.h b/drivers/dma/dw-edma/dw-edma-core.h
index ea9f4292c40e..dbc4af0eab59 100644
--- a/drivers/dma/dw-edma/dw-edma-core.h
+++ b/drivers/dma/dw-edma/dw-edma-core.h
@@ -75,24 +75,26 @@ struct dw_edma_chan {
 	enum dw_edma_dir		dir;
 
 	/*
-	 * New LL entries are appended at ll_head. Entries between ll_end
-	 * and ll_head, modulo the LL ring, are owned by DMA; the rest are
-	 * owned by software.
+	 * New LL entries are appended at ll_head. Entries between ll_done and
+	 * ll_head, modulo the LL ring, are owned by DMA; the rest have already
+	 * been consumed and may be overwritten by software. ll_end trails behind
+	 * at the boundary of the last completed descriptor.
 	 *
 	 *   software-owned      DMA-owned       software-owned
 	 * +---------------+-------------------+---------------+
 	 * ^               ^                   ^
-	 * 0             ll_end              ll_head
+	 * 0            ll_done             ll_head
 	 *
-	 * The link entry points back to the region start. ll_head == ll_end
-	 * means all entries are software-owned and previous DMA work is
-	 * done.
+	 * The link entry points back to the region start. No DMA-owned entries
+	 * remain once ll_done catches up with ll_head.
 	 *
 	 * Software always keeps at least one free entry, so the ring is
-	 * never completely DMA-owned.
+	 * never completely DMA-owned. That keeps a hardware-reported physical
+	 * LL index unique within the current ll_done..ll_head producer window.
 	 */
 	u32				ll_head;
 	u32				ll_end;
+	u32				ll_done;
 
 	u32				ll_max;
 	struct dw_edma_region		ll_region;	/* Linked list */
-- 
2.51.0