[PATCH 11/17] dmaengine: dw-edma: Use HDMA watermarks as progress events
From: Koichiro Den
Date: Mon Jun 15 2026 - 11:44:57 EST
HDMA updates its running LL pointer at watermark points, so route HDMA
watermark interrupts to a progress handler instead of treating progress
as a STOP-only event.
Wire HDMA watermark interrupt setup, clearing, MSI address programming,
and LL LWIE/RWIE bits into the core interrupt path. STOP remains the
HDMA done event, while WATERMARK only reports running progress.
The previous patch decides where LL interrupt bits are placed. This
patch makes HDMA honor those bits as hardware watermark events and
dispatch them to the common progress handler.
The progress handler reuses the common LL recycling helper to reclaim
completed descriptors and refill the ring while the channel keeps
running. Do this only while no channel request is pending; STOP and
PAUSE requests are handled by their request paths, not as normal running
progress.
The legacy eDMA dispatcher keeps routing its interrupt status only to
DONE and ABORT handlers; the new progress handler is used only by HDMA
watermark interrupts.
Signed-off-by: Koichiro Den <den@xxxxxxxxxxxxx>
---
drivers/dma/dw-edma/dw-edma-core.c | 27 ++++++++++++
drivers/dma/dw-edma/dw-edma-core.h | 9 ++--
drivers/dma/dw-edma/dw-edma-v0-core.c | 4 +-
drivers/dma/dw-edma/dw-hdma-v0-core.c | 59 +++++++++++++++++++++++----
drivers/dma/dw-edma/dw-hdma-v0-regs.h | 1 +
5 files changed, 87 insertions(+), 13 deletions(-)
diff --git a/drivers/dma/dw-edma/dw-edma-core.c b/drivers/dma/dw-edma/dw-edma-core.c
index 839bafc762a1..a289d8f8cc17 100644
--- a/drivers/dma/dw-edma/dw-edma-core.c
+++ b/drivers/dma/dw-edma/dw-edma-core.c
@@ -299,6 +299,15 @@ static bool dw_edma_ll_recycle(struct dw_edma_chan *chan, int idx)
return true;
}
+/* Must be called with vc.lock held. */
+static bool dw_edma_ll_recycle_and_refill(struct dw_edma_chan *chan, int idx)
+{
+ if (!dw_edma_ll_recycle(chan, idx))
+ return false;
+
+ return dw_edma_start_transfer(chan);
+}
+
static void dw_edma_device_caps(struct dma_chan *dchan,
struct dma_slave_caps *caps)
{
@@ -739,6 +748,22 @@ static void dw_edma_done_interrupt(struct dw_edma_chan *chan)
spin_unlock_irqrestore(&chan->vc.lock, flags);
}
+static void dw_edma_progress_interrupt(struct dw_edma_chan *chan)
+{
+ unsigned long flags;
+ int idx;
+
+ spin_lock_irqsave(&chan->vc.lock, flags);
+ idx = dw_edma_core_ll_cur_idx(chan);
+ if (chan->request == EDMA_REQ_NONE && chan->status != EDMA_ST_PAUSE) {
+ dw_edma_ll_recycle_and_refill(chan, idx);
+ chan->status = dw_edma_ll_pending(chan) ?
+ EDMA_ST_BUSY : EDMA_ST_IDLE;
+ }
+
+ spin_unlock_irqrestore(&chan->vc.lock, flags);
+}
+
static void dw_edma_abort_interrupt(struct dw_edma_chan *chan)
{
struct virt_dma_desc *vd;
@@ -762,6 +787,7 @@ static inline irqreturn_t dw_edma_interrupt_write(int irq, void *data)
return dw_edma_core_handle_int(dw_irq, EDMA_DIR_WRITE,
dw_edma_done_interrupt,
+ dw_edma_progress_interrupt,
dw_edma_abort_interrupt);
}
@@ -771,6 +797,7 @@ static inline irqreturn_t dw_edma_interrupt_read(int irq, void *data)
return dw_edma_core_handle_int(dw_irq, EDMA_DIR_READ,
dw_edma_done_interrupt,
+ dw_edma_progress_interrupt,
dw_edma_abort_interrupt);
}
diff --git a/drivers/dma/dw-edma/dw-edma-core.h b/drivers/dma/dw-edma/dw-edma-core.h
index dbc4af0eab59..9bd0a5f2f08b 100644
--- a/drivers/dma/dw-edma/dw-edma-core.h
+++ b/drivers/dma/dw-edma/dw-edma-core.h
@@ -144,7 +144,9 @@ struct dw_edma_core_ops {
u16 (*ch_count)(struct dw_edma *dw, enum dw_edma_dir dir);
enum dma_status (*ch_status)(struct dw_edma_chan *chan);
irqreturn_t (*handle_int)(struct dw_edma_irq *dw_irq, enum dw_edma_dir dir,
- dw_edma_handler_t done, dw_edma_handler_t abort);
+ dw_edma_handler_t done,
+ dw_edma_handler_t progress,
+ dw_edma_handler_t abort);
void (*ll_data)(struct dw_edma_chan *chan, struct dw_edma_burst *burst,
u32 idx, bool cb, bool irq);
void (*ll_link)(struct dw_edma_chan *chan, u32 idx, bool cb, u64 addr);
@@ -228,9 +230,10 @@ enum dma_status dw_edma_core_ch_status(struct dw_edma_chan *chan)
static inline irqreturn_t
dw_edma_core_handle_int(struct dw_edma_irq *dw_irq, enum dw_edma_dir dir,
- dw_edma_handler_t done, dw_edma_handler_t abort)
+ dw_edma_handler_t done, dw_edma_handler_t progress,
+ dw_edma_handler_t abort)
{
- return dw_irq->dw->core->handle_int(dw_irq, dir, done, abort);
+ return dw_irq->dw->core->handle_int(dw_irq, dir, done, progress, abort);
}
static inline
diff --git a/drivers/dma/dw-edma/dw-edma-v0-core.c b/drivers/dma/dw-edma/dw-edma-v0-core.c
index 47faedd14dc2..dfe0483896d3 100644
--- a/drivers/dma/dw-edma/dw-edma-v0-core.c
+++ b/drivers/dma/dw-edma/dw-edma-v0-core.c
@@ -232,7 +232,9 @@ static u32 dw_edma_v0_core_status_abort_int(struct dw_edma *dw, enum dw_edma_dir
static irqreturn_t
dw_edma_v0_core_handle_int(struct dw_edma_irq *dw_irq, enum dw_edma_dir dir,
- dw_edma_handler_t done, dw_edma_handler_t abort)
+ dw_edma_handler_t done,
+ dw_edma_handler_t progress,
+ dw_edma_handler_t abort)
{
struct dw_edma *dw = dw_irq->dw;
unsigned long total, pos, val;
diff --git a/drivers/dma/dw-edma/dw-hdma-v0-core.c b/drivers/dma/dw-edma/dw-hdma-v0-core.c
index b9e193774714..9f5b11350f23 100644
--- a/drivers/dma/dw-edma/dw-hdma-v0-core.c
+++ b/drivers/dma/dw-edma/dw-hdma-v0-core.c
@@ -59,9 +59,13 @@ static void dw_hdma_v0_core_off(struct dw_edma *dw)
for (id = 0; id < HDMA_V0_MAX_NR_CH; id++) {
SET_BOTH_CH_32(dw, id, int_setup,
- HDMA_V0_STOP_INT_MASK | HDMA_V0_ABORT_INT_MASK);
+ HDMA_V0_STOP_INT_MASK |
+ HDMA_V0_WATERMARK_INT_MASK |
+ HDMA_V0_ABORT_INT_MASK);
SET_BOTH_CH_32(dw, id, int_clear,
- HDMA_V0_STOP_INT_MASK | HDMA_V0_ABORT_INT_MASK);
+ HDMA_V0_STOP_INT_MASK |
+ HDMA_V0_WATERMARK_INT_MASK |
+ HDMA_V0_ABORT_INT_MASK);
SET_BOTH_CH_32(dw, id, ch_en, 0);
}
}
@@ -99,6 +103,14 @@ static void dw_hdma_v0_core_clear_done_int(struct dw_edma_chan *chan)
SET_CH_32(dw, chan->dir, chan->id, int_clear, HDMA_V0_STOP_INT_MASK);
}
+static void dw_hdma_v0_core_clear_watermark_int(struct dw_edma_chan *chan)
+{
+ struct dw_edma *dw = chan->dw;
+
+ SET_CH_32(dw, chan->dir, chan->id, int_clear,
+ HDMA_V0_WATERMARK_INT_MASK);
+}
+
static void dw_hdma_v0_core_clear_abort_int(struct dw_edma_chan *chan)
{
struct dw_edma *dw = chan->dw;
@@ -115,7 +127,9 @@ static u32 dw_hdma_v0_core_status_int(struct dw_edma_chan *chan)
static irqreturn_t
dw_hdma_v0_core_handle_int(struct dw_edma_irq *dw_irq, enum dw_edma_dir dir,
- dw_edma_handler_t done, dw_edma_handler_t abort)
+ dw_edma_handler_t done,
+ dw_edma_handler_t progress,
+ dw_edma_handler_t abort)
{
struct dw_edma *dw = dw_irq->dw;
unsigned long total, pos, val;
@@ -134,16 +148,29 @@ dw_hdma_v0_core_handle_int(struct dw_edma_irq *dw_irq, enum dw_edma_dir dir,
}
for_each_set_bit(pos, &mask, total) {
+ bool has_stop, has_watermark;
+
chan = &dw->chan[pos + off];
val = dw_hdma_v0_core_status_int(chan);
- if (FIELD_GET(HDMA_V0_STOP_INT_MASK, val)) {
- dw_hdma_v0_core_clear_done_int(chan);
- done(chan);
+ has_stop = FIELD_GET(HDMA_V0_STOP_INT_MASK, val);
+ has_watermark = FIELD_GET(HDMA_V0_WATERMARK_INT_MASK, val);
+ if (has_watermark) {
+ dw_hdma_v0_core_clear_watermark_int(chan);
ret = IRQ_HANDLED;
}
+ if (has_stop) {
+ dw_hdma_v0_core_clear_done_int(chan);
+ ret = IRQ_HANDLED;
+ }
+
+ if (has_stop)
+ done(chan);
+ else if (has_watermark)
+ progress(chan);
+
if (FIELD_GET(HDMA_V0_ABORT_INT_MASK, val)) {
dw_hdma_v0_core_clear_abort_int(chan);
abort(chan);
@@ -204,10 +231,12 @@ static void dw_hdma_v0_core_ch_enable(struct dw_edma_chan *chan)
/* Enable engine */
SET_CH_32(dw, chan->dir, chan->id, ch_en, BIT(0));
- /* Interrupt unmask - stop, abort */
+ /* Interrupt unmask - stop, watermark, abort */
tmp = GET_CH_32(dw, chan->dir, chan->id, int_setup);
- tmp &= ~(HDMA_V0_STOP_INT_MASK | HDMA_V0_ABORT_INT_MASK);
- /* Interrupt enable - stop, abort */
+ tmp &= ~(HDMA_V0_STOP_INT_MASK | HDMA_V0_WATERMARK_INT_MASK |
+ HDMA_V0_ABORT_INT_MASK);
+ /* Interrupt enable - stop, abort. */
+ /* Watermark is enabled per LL element. */
tmp |= HDMA_V0_LOCAL_STOP_INT_EN | HDMA_V0_LOCAL_ABORT_INT_EN;
if (!(dw->chip->flags & DW_EDMA_CHIP_LOCAL))
tmp |= HDMA_V0_REMOTE_STOP_INT_EN | HDMA_V0_REMOTE_ABORT_INT_EN;
@@ -247,6 +276,11 @@ static void dw_hdma_v0_core_ch_config(struct dw_edma_chan *chan)
/* MSI done addr - low, high */
SET_CH_32(dw, chan->dir, chan->id, msi_stop.lsb, chan->msi.address_lo);
SET_CH_32(dw, chan->dir, chan->id, msi_stop.msb, chan->msi.address_hi);
+ /* MSI watermark addr - low, high */
+ SET_CH_32(dw, chan->dir, chan->id, msi_watermark.lsb,
+ chan->msi.address_lo);
+ SET_CH_32(dw, chan->dir, chan->id, msi_watermark.msb,
+ chan->msi.address_hi);
/* MSI abort addr - low, high */
SET_CH_32(dw, chan->dir, chan->id, msi_abort.lsb, chan->msi.address_lo);
SET_CH_32(dw, chan->dir, chan->id, msi_abort.msb, chan->msi.address_hi);
@@ -263,6 +297,13 @@ dw_hdma_v0_core_ll_data(struct dw_edma_chan *chan, struct dw_edma_burst *burst,
if (cb)
control |= DW_HDMA_V0_CB;
+ if (irq) {
+ control |= DW_HDMA_V0_LWIE;
+
+ if (!(chan->dw->chip->flags & DW_EDMA_CHIP_LOCAL))
+ control |= DW_HDMA_V0_RWIE;
+ }
+
dw_hdma_v0_write_ll_data(chan, idx, control, burst->sz, burst->sar,
burst->dar);
}
diff --git a/drivers/dma/dw-edma/dw-hdma-v0-regs.h b/drivers/dma/dw-edma/dw-hdma-v0-regs.h
index eab5fd7177e5..bf13e451b1a9 100644
--- a/drivers/dma/dw-edma/dw-hdma-v0-regs.h
+++ b/drivers/dma/dw-edma/dw-hdma-v0-regs.h
@@ -17,6 +17,7 @@
#define HDMA_V0_LOCAL_STOP_INT_EN BIT(4)
#define HDMA_V0_REMOTE_STOP_INT_EN BIT(3)
#define HDMA_V0_ABORT_INT_MASK BIT(2)
+#define HDMA_V0_WATERMARK_INT_MASK BIT(1)
#define HDMA_V0_STOP_INT_MASK BIT(0)
#define HDMA_V0_LINKLIST_EN BIT(0)
#define HDMA_V0_CONSUMER_CYCLE_STAT BIT(1)
--
2.51.0