Re: [PATCH v8 4/7] dmaengine: dw-edma: Reclaim LL entries from hardware progress
From: Frank Li
Date: Thu Nov 12 2026 - 14:13:44 EST
On Sat, Nov 07, 2026 at 10:30:24PM +0900, Koichiro Den wrote:
> With dynamic appends, a DONE interrupt no longer means that all
> published LL entries have completed. Prepare for this by using LLP to
> advance ll_done and complete descriptors in issue order.
>
> Reclaim completed entries and refill the ring as space becomes
> available. Restart a stopped channel when published work remains. This
> also allows descriptors larger than the ring to run in batches.
>
> Hardware can keep running while the driver handles an interrupt, and
> LLP does not always track payload completion. Use the following
> exclusive completion boundaries, where idx is the raw LLP entry index
> and prev(idx) is the previous data entry in the ring:
>
> State/event Meaning of raw LLP Boundary
> ------------------- ------------------------------- ---------
> eDMA/HDMA stopped Next entry to process idx
>
> eDMA running Observed to advance before prev(idx)
> payload completion
>
> HDMA watermark Inclusive LLE recycling limit prev(idx)
>
> A stopped sample requires CS=STOPPED and a zero transfer size.
>
> While running, HDMA updates LLP only at watermark events. Its documented
> recycling limit translates to an exclusive boundary of next(idx).
> However, tests using that boundary for dmaengine completion reported
> IOMMU faults on HDMA 6.30a SpacemiT K3. Use prev(idx) for both engines
> while running to keep the handling simple and conservative. Entries held
> back by this rule are eventually reclaimed on later progress or a
> confirmed stop, where idx gives the exact boundary.
>
> Handle termination and pause as follows:
> - On termination, stop publishing new entries and drain those already
> published, restarting the channel if needed. Terminate descriptors
> without callbacks, as in the non-LL path.
> - On pause, stop publishing new entries and enter the paused state once
> the channel stops. Keep unfinished work for resume. Reclaim delayed
> completions without restarting the channel.
>
> Signed-off-by: Koichiro Den <den@xxxxxxxxxxxxx>
> ---
Reviewed-by: Frank Li <Frank.Li@xxxxxxx>
> Changes from v7:
> - Read LLP before checking for a stop, and use the same sample for
> completion and termination (Sashiko).
> - Reclaim delayed completions while paused without restarting the
> channel (spotted during code inspection, not observed in hardware
> testing).
>
> @Frank: Dropped your R-b due to the changes; would appreciate another
> look.
>
> drivers/dma/dw-edma/dw-edma-core.c | 160 +++++++++++++++++++++++++----
> 1 file changed, 140 insertions(+), 20 deletions(-)
>
> diff --git a/drivers/dma/dw-edma/dw-edma-core.c b/drivers/dma/dw-edma/dw-edma-core.c
> index 346ec8ee631d..1d13a25663ef 100644
> --- a/drivers/dma/dw-edma/dw-edma-core.c
> +++ b/drivers/dma/dw-edma/dw-edma-core.c
> @@ -53,13 +53,6 @@ dw_edma_alloc_desc(struct dw_edma_chan *chan, size_t nburst)
> {
> struct dw_edma_desc *desc;
>
> - /*
> - * For now, a descriptor that does not fit would stall the channel
> - * forever: reject it up front.
> - */
> - if (!chan->non_ll && nburst > chan->ll_max - 1)
> - return NULL;
> -
> desc = kzalloc_flex(*desc, burst, nburst, GFP_NOWAIT);
> if (unlikely(!desc))
> return NULL;
> @@ -181,18 +174,83 @@ static int dw_edma_ll_read_idx(struct dw_edma_chan *chan, bool *stopped)
> return idx;
> }
>
> +/* Return the first entry we cannot reclaim yet. */
> +static int dw_edma_ll_boundary(struct dw_edma_chan *chan, int idx,
> + bool stopped)
> +{
> + if (idx < 0 || (u32)idx > chan->ll_max)
> + return -EINVAL;
> +
> + /* After a stop, LLP points past the completed entries. */
> + if (stopped)
> + return idx == chan->ll_max ? 0 : idx;
> +
> + /*
> + * DMA_LLP can advance before payload completion. Keep the boundary
> + * one entry behind it, wrapping at the start of the ring.
> + */
> + return (idx + chan->ll_max - 1) % chan->ll_max;
> +}
> +
> static bool dw_edma_ll_drained(struct dw_edma_chan *chan)
> {
> bool stopped;
> int idx;
>
> idx = dw_edma_ll_read_idx(chan, &stopped);
> - if (idx == chan->ll_max)
> - idx = 0;
> + idx = dw_edma_ll_boundary(chan, idx, true);
>
> return stopped && idx >= 0 && (u32)idx == chan->ll_head;
> }
>
> +static void dw_edma_ll_consume(struct dw_edma_chan *chan, int idx,
> + bool stopped)
> +{
> + struct virt_dma_desc *vd, *_vd;
> + u32 done, used;
> +
> + lockdep_assert_held(&chan->vc.lock);
> +
> + idx = dw_edma_ll_boundary(chan, idx, stopped);
> + if (idx < 0)
> + return;
> +
> + done = dw_edma_core_get_ll_dist(chan, chan->ll_done, idx);
> + used = dw_edma_core_get_used_num(chan);
> +
> + if (!done || done > used)
> + return;
> +
> + /*
> + * LLP may still point to the first pending entry. Subtracting one
> + * can wrap the index and falsely mark all pending entries complete.
> + * Reclaim them all only after the channel has stopped.
> + */
> + if (!stopped && done == used)
> + return;
> +
> + chan->ll_done = idx;
> +
> + /* Match reclaimed entries to descriptors in issue order. */
> + list_for_each_entry_safe(vd, _vd, &chan->vc.desc_issued, node) {
> + struct dw_edma_desc *desc = vd2dw_edma_desc(vd);
> + u32 consumed = min_t(u32, done,
> + desc->start_burst - desc->done_burst);
> +
> + if (!consumed)
> + break;
> +
> + desc->done_burst += consumed;
> + done -= consumed;
> + if (desc->done_burst != desc->nburst)
> + break;
> +
> + dw_edma_complete_vdesc(vd, DMA_TRANS_NOERROR);
> + }
> +
> + WARN_ON_ONCE(done);
> +}
> +
> static void dw_edma_core_ll_start(struct dw_edma_desc *desc)
> {
> struct dw_edma_chan *chan = desc->chan;
> @@ -221,7 +279,6 @@ static void dw_edma_core_ll_start(struct dw_edma_desc *desc)
> }
> }
>
> - desc->done_burst = desc->start_burst;
> desc->start_burst = i;
> }
>
> @@ -330,6 +387,61 @@ static void dw_edma_core_ch_maybe_doorbell(struct dw_edma_chan *chan)
> dw_edma_core_ch_doorbell(chan);
> }
>
> +/* Must be called with vc.lock held. */
> +static void dw_edma_ll_advance(struct dw_edma_chan *chan, bool event)
> +{
> + bool stopped;
> + int idx;
> +
> + lockdep_assert_held(&chan->vc.lock);
> +
> + idx = dw_edma_ll_read_idx(chan, &stopped);
> +
> + /* Terminate without scheduling completion callbacks. */
> + if (chan->request == EDMA_REQ_STOP) {
> + if (!stopped)
> + return;
> +
> + idx = dw_edma_ll_boundary(chan, idx, true);
> + if (idx >= 0 && (u32)idx == chan->ll_head)
> + dw_edma_finish_termination(chan);
> + else
> + /* A newly appended tail may have missed the last stop. */
> + dw_edma_core_ch_doorbell(chan);
> + return;
> + }
> +
> + /*
> + * HDMA updates LLP at watermark events while running. Use running
> + * samples only when processing an LL interrupt.
> + */
> + if (stopped || event)
> + dw_edma_ll_consume(chan, idx, stopped);
> +
> + /* An interrupt may arrive after we have entered PAUSE. */
> + if (chan->status == EDMA_ST_PAUSE)
> + return;
> +
> + if (chan->request == EDMA_REQ_PAUSE) {
> + if (!stopped)
> + return;
> +
> + dw_edma_set_request(chan, EDMA_REQ_NONE);
> + chan->status = EDMA_ST_PAUSE;
> + return;
> + }
> +
> + dw_edma_start_transfer(chan);
> + if (!dw_edma_ll_pending(chan)) {
> + chan->status = EDMA_ST_IDLE;
> + return;
> + }
> +
> + chan->status = EDMA_ST_BUSY;
> + if (stopped)
> + dw_edma_core_ch_doorbell(chan);
> +}
> +
> static void dw_edma_device_caps(struct dma_chan *dchan,
> struct dma_slave_caps *caps)
> {
> @@ -514,6 +626,8 @@ static int dw_edma_device_terminate_all(struct dma_chan *dchan)
> err = -EPERM;
> } else {
> dw_edma_set_request(chan, EDMA_REQ_STOP);
> + if (!chan->non_ll)
> + dw_edma_ll_advance(chan, false);
> }
> if (chan->status == EDMA_ST_IDLE)
> dw_edma_set_request(chan, EDMA_REQ_NONE);
> @@ -798,7 +912,7 @@ dw_edma_device_prep_interleaved_dma(struct dma_chan *dchan,
> return dw_edma_device_transfer(&xfer, dw_edma_device_get_config(dchan, NULL));
> }
>
> -/* Must be called with vc.lock held. */
> +/* Non-LL completion; vc.lock must be held. */
> static void dw_edma_done_interrupt_locked(struct dw_edma_chan *chan)
> {
> struct dw_edma_desc *desc;
> @@ -817,11 +931,8 @@ static void dw_edma_done_interrupt_locked(struct dw_edma_chan *chan)
> break;
>
> desc = vd2dw_edma_desc(vd);
> - if (desc->start_burst >= desc->nburst) {
> + if (desc->start_burst >= desc->nburst)
> dw_edma_complete_vdesc(vd, DMA_TRANS_NOERROR);
> - if (!chan->non_ll)
> - chan->ll_done = chan->ll_head;
> - }
>
> if (chan->request == EDMA_REQ_PAUSE) {
> dw_edma_set_request(chan, EDMA_REQ_NONE);
> @@ -837,14 +948,12 @@ static void dw_edma_done_interrupt_locked(struct dw_edma_chan *chan)
> if (!vd)
> break;
>
> - if (chan->non_ll || dw_edma_ll_drained(chan))
> - dw_edma_finish_termination(chan);
> + dw_edma_finish_termination(chan);
> break;
>
> default:
> break;
> }
> - dw_edma_core_ch_maybe_doorbell(chan);
> }
>
> static void dw_edma_done_interrupt(struct dw_edma_chan *chan)
> @@ -856,6 +965,13 @@ static void dw_edma_done_interrupt(struct dw_edma_chan *chan)
> spin_unlock_irqrestore(&chan->vc.lock, flags);
> }
>
> +static void dw_edma_ll_interrupt(struct dw_edma_chan *chan)
> +{
> + guard(spinlock_irqsave)(&chan->vc.lock);
> +
> + dw_edma_ll_advance(chan, true);
> +}
> +
> static void dw_edma_abort_interrupt(struct dw_edma_chan *chan)
> {
> struct virt_dma_desc *vd;
> @@ -884,8 +1000,12 @@ static void dw_edma_irq_work(struct work_struct *work)
> do {
> events = atomic_xchg(&chan->irq_pending, 0);
>
> - if (events & DW_EDMA_DEFERRED_DONE)
> - dw_edma_done_interrupt(chan);
> + if (events & DW_EDMA_DEFERRED_DONE) {
> + if (chan->non_ll)
> + dw_edma_done_interrupt(chan);
> + else
> + dw_edma_ll_interrupt(chan);
> + }
> if (events & DW_EDMA_DEFERRED_ABORT)
> dw_edma_abort_interrupt(chan);
> } while (atomic_read(&chan->irq_pending));
> --
> 2.54.0
>