[PATCH v2 10/19] dmaengine: dw-edma: Use HDMA watermarks as progress events
From: Koichiro Den
Date: Thu Jul 23 2026 - 04:48:37 EST
HDMA reports running LL progress through WATERMARK and channel stop
through STOP. Wire watermark setup, clearing, MSI programming, and
LWIE/RWIE into the core interrupt path. Route WATERMARK to the common
progress handler; STOP remains the done event.
DWC PCIe Controller Databook 6.10a-lca06, Section 7.2.1, permits recycling
LLEs through the watermark LLP. Testing on a 6.30a integration found that
boundary unsafe for DMAengine completion: clients released mappings while
hardware still accessed them. Keep completion one entry behind the raw
LLP. Once the channel has stopped, LLP points to the next entry and needs
no such margin.
Leave EDMA_REQ_STOP and EDMA_REQ_PAUSE to the STOP path instead of
handling their watermark as normal running progress. The eDMA path is
unchanged.
Signed-off-by: Koichiro Den <den@xxxxxxxxxxxxx>
---
Changes in v2:
- Keep completion one entry behind the raw LLP; v1 used the reported
boundary.
- Fold watermark masks into dw_hdma_v0_core_int_setup() while keeping
v1's local and remote routing.
- Use the deferred per-channel IRQ work from the groundwork series.
- Mask local watermark interrupts only for remote LL channels, and
document the STOP/ABORT and WATERMARK distinction.
drivers/dma/dw-edma/dw-edma-core.c | 32 ++++++++++-
drivers/dma/dw-edma/dw-edma-core.h | 9 ++--
drivers/dma/dw-edma/dw-edma-v0-core.c | 4 +-
drivers/dma/dw-edma/dw-hdma-v0-core.c | 76 ++++++++++++++++++++++-----
drivers/dma/dw-edma/dw-hdma-v0-regs.h | 1 +
5 files changed, 104 insertions(+), 18 deletions(-)
diff --git a/drivers/dma/dw-edma/dw-edma-core.c b/drivers/dma/dw-edma/dw-edma-core.c
index 44a0ff906f2f..c6c2655c6c43 100644
--- a/drivers/dma/dw-edma/dw-edma-core.c
+++ b/drivers/dma/dw-edma/dw-edma-core.c
@@ -32,7 +32,8 @@ struct dw_edma_desc *vd2dw_edma_desc(struct virt_dma_desc *vd)
enum dw_edma_irq_event {
DW_EDMA_IRQ_DONE = BIT(0),
- DW_EDMA_IRQ_ABORT = BIT(1),
+ DW_EDMA_IRQ_PROGRESS = BIT(1),
+ DW_EDMA_IRQ_ABORT = BIT(2),
};
static inline
@@ -912,6 +913,24 @@ static void dw_edma_done_interrupt(struct dw_edma_chan *chan)
spin_unlock_irqrestore(&chan->vc.lock, flags);
}
+static void dw_edma_progress_interrupt(struct dw_edma_chan *chan)
+{
+ if (chan->non_ll)
+ return;
+
+ guard(spinlock_irqsave)(&chan->vc.lock);
+
+ if (chan->request == EDMA_REQ_NONE && chan->status != EDMA_ST_PAUSE) {
+ if (dw_edma_ll_consume_progress(chan))
+ dw_edma_start_transfer(chan);
+ chan->status = dw_edma_ll_pending(chan) ?
+ EDMA_ST_BUSY : EDMA_ST_IDLE;
+ }
+
+ /* The channel may have stopped after the progress point was sampled. */
+ dw_edma_core_ch_maybe_doorbell(chan);
+}
+
static void dw_edma_abort_interrupt(struct dw_edma_chan *chan)
{
struct virt_dma_desc *vd;
@@ -942,6 +961,8 @@ static void dw_edma_irq_work(struct work_struct *work)
if (events & DW_EDMA_IRQ_DONE)
dw_edma_done_interrupt(chan);
+ if (events & DW_EDMA_IRQ_PROGRESS)
+ dw_edma_progress_interrupt(chan);
if (events & DW_EDMA_IRQ_ABORT)
dw_edma_abort_interrupt(chan);
} while (atomic_read(&chan->irq_pending));
@@ -980,6 +1001,13 @@ static void dw_edma_done_interrupt_deferred(struct dw_edma_chan *chan,
dw_edma_queue_irq_work(chan, DW_EDMA_IRQ_DONE);
}
+static void dw_edma_progress_interrupt_deferred(struct dw_edma_chan *chan,
+ bool stopped)
+{
+ dw_edma_record_irq_idx(chan, stopped);
+ dw_edma_queue_irq_work(chan, DW_EDMA_IRQ_PROGRESS);
+}
+
static void dw_edma_abort_interrupt_deferred(struct dw_edma_chan *chan,
bool stopped)
{
@@ -1084,6 +1112,7 @@ static inline irqreturn_t dw_edma_interrupt_write_inner(int irq, void *data)
return dw_edma_core_handle_int(dw_irq, EDMA_DIR_WRITE,
dw_edma_done_interrupt_deferred,
+ dw_edma_progress_interrupt_deferred,
dw_edma_abort_interrupt_deferred);
}
@@ -1093,6 +1122,7 @@ static inline irqreturn_t dw_edma_interrupt_read_inner(int irq, void *data)
return dw_edma_core_handle_int(dw_irq, EDMA_DIR_READ,
dw_edma_done_interrupt_deferred,
+ dw_edma_progress_interrupt_deferred,
dw_edma_abort_interrupt_deferred);
}
diff --git a/drivers/dma/dw-edma/dw-edma-core.h b/drivers/dma/dw-edma/dw-edma-core.h
index 90dcbe415f57..d074506f63ed 100644
--- a/drivers/dma/dw-edma/dw-edma-core.h
+++ b/drivers/dma/dw-edma/dw-edma-core.h
@@ -167,7 +167,9 @@ struct dw_edma_core_ops {
enum dma_status (*ch_status)(struct dw_edma_chan *chan);
u32 (*ch_transfer_size)(struct dw_edma_chan *chan);
irqreturn_t (*handle_int)(struct dw_edma_irq *dw_irq, enum dw_edma_dir dir,
- dw_edma_handler_t done, dw_edma_handler_t abort);
+ dw_edma_handler_t done,
+ dw_edma_handler_t progress,
+ dw_edma_handler_t abort);
void (*non_ll_start)(struct dw_edma_chan *chan, struct dw_edma_burst *child);
void (*ll_data)(struct dw_edma_chan *chan, struct dw_edma_burst *burst,
u32 idx, bool cb, bool irq);
@@ -268,9 +270,10 @@ enum dma_status dw_edma_core_ch_status(struct dw_edma_chan *chan)
static inline irqreturn_t
dw_edma_core_handle_int(struct dw_edma_irq *dw_irq, enum dw_edma_dir dir,
- dw_edma_handler_t done, dw_edma_handler_t abort)
+ dw_edma_handler_t done, dw_edma_handler_t progress,
+ dw_edma_handler_t abort)
{
- return dw_irq->dw->core->handle_int(dw_irq, dir, done, abort);
+ return dw_irq->dw->core->handle_int(dw_irq, dir, done, progress, abort);
}
static inline
diff --git a/drivers/dma/dw-edma/dw-edma-v0-core.c b/drivers/dma/dw-edma/dw-edma-v0-core.c
index 3b69fa26bf1c..42036239ba79 100644
--- a/drivers/dma/dw-edma/dw-edma-v0-core.c
+++ b/drivers/dma/dw-edma/dw-edma-v0-core.c
@@ -341,7 +341,9 @@ static void dw_edma_v0_core_clear_abort_int(struct dw_edma_chan *chan)
static irqreturn_t
dw_edma_v0_core_handle_int(struct dw_edma_irq *dw_irq, enum dw_edma_dir dir,
- dw_edma_handler_t done, dw_edma_handler_t abort)
+ dw_edma_handler_t done,
+ dw_edma_handler_t progress,
+ dw_edma_handler_t abort)
{
struct dw_edma *dw = dw_irq->dw;
unsigned long total, pos, val;
diff --git a/drivers/dma/dw-edma/dw-hdma-v0-core.c b/drivers/dma/dw-edma/dw-hdma-v0-core.c
index 4440e3ebaeac..535832e1ed0d 100644
--- a/drivers/dma/dw-edma/dw-hdma-v0-core.c
+++ b/drivers/dma/dw-edma/dw-hdma-v0-core.c
@@ -56,7 +56,8 @@ static u32 dw_hdma_v0_core_int_setup(struct dw_edma_chan *chan, u32 val)
{
val &= ~(HDMA_V0_LOCAL_ABORT_INT_EN | HDMA_V0_REMOTE_ABORT_INT_EN |
HDMA_V0_LOCAL_STOP_INT_EN | HDMA_V0_REMOTE_STOP_INT_EN |
- HDMA_V0_ABORT_INT_MASK | HDMA_V0_STOP_INT_MASK);
+ HDMA_V0_ABORT_INT_MASK | HDMA_V0_WATERMARK_INT_MASK |
+ HDMA_V0_STOP_INT_MASK);
/*
* HDMA_INT_STATUS.STOP and .ABORT are latched only when LSIE and
@@ -66,10 +67,14 @@ static u32 dw_hdma_v0_core_int_setup(struct dw_edma_chan *chan, u32 val)
*/
val |= HDMA_V0_LOCAL_ABORT_INT_EN | HDMA_V0_LOCAL_STOP_INT_EN;
- if (chan->irq_mode == DW_EDMA_CH_IRQ_REMOTE)
+ if (chan->irq_mode == DW_EDMA_CH_IRQ_REMOTE) {
+ if (!chan->non_ll)
+ val |= HDMA_V0_WATERMARK_INT_MASK;
+
val |= HDMA_V0_REMOTE_ABORT_INT_EN |
HDMA_V0_REMOTE_STOP_INT_EN |
HDMA_V0_ABORT_INT_MASK | HDMA_V0_STOP_INT_MASK;
+ }
return val;
}
@@ -79,10 +84,12 @@ static void dw_hdma_v0_core_ch_off(struct dw_edma *dw, enum dw_edma_dir dir,
u16 id)
{
SET_CH_32(dw, dir, id, int_setup,
- HDMA_V0_STOP_INT_MASK | HDMA_V0_ABORT_INT_MASK);
+ HDMA_V0_STOP_INT_MASK | HDMA_V0_WATERMARK_INT_MASK |
+ HDMA_V0_ABORT_INT_MASK);
SET_CH_32(dw, dir, id, ch_en, 0);
SET_CH_32(dw, dir, id, int_clear,
- HDMA_V0_STOP_INT_MASK | HDMA_V0_ABORT_INT_MASK);
+ HDMA_V0_STOP_INT_MASK | HDMA_V0_WATERMARK_INT_MASK |
+ HDMA_V0_ABORT_INT_MASK);
}
static void dw_hdma_v0_core_off(struct dw_edma *dw)
@@ -93,18 +100,26 @@ static void dw_hdma_v0_core_off(struct dw_edma *dw)
dir = EDMA_DIR_WRITE;
for (id = 0; id < dw->wr_ch_cnt; id++) {
SET_CH_32(dw, dir, id, int_setup,
- HDMA_V0_STOP_INT_MASK | HDMA_V0_ABORT_INT_MASK);
+ HDMA_V0_STOP_INT_MASK |
+ HDMA_V0_WATERMARK_INT_MASK |
+ HDMA_V0_ABORT_INT_MASK);
SET_CH_32(dw, dir, id, int_clear,
- HDMA_V0_STOP_INT_MASK | HDMA_V0_ABORT_INT_MASK);
+ HDMA_V0_STOP_INT_MASK |
+ HDMA_V0_WATERMARK_INT_MASK |
+ HDMA_V0_ABORT_INT_MASK);
SET_CH_32(dw, dir, id, ch_en, 0);
}
dir = EDMA_DIR_READ;
for (id = 0; id < dw->rd_ch_cnt; id++) {
SET_CH_32(dw, dir, id, int_setup,
- HDMA_V0_STOP_INT_MASK | HDMA_V0_ABORT_INT_MASK);
+ HDMA_V0_STOP_INT_MASK |
+ HDMA_V0_WATERMARK_INT_MASK |
+ HDMA_V0_ABORT_INT_MASK);
SET_CH_32(dw, dir, id, int_clear,
- HDMA_V0_STOP_INT_MASK | HDMA_V0_ABORT_INT_MASK);
+ HDMA_V0_STOP_INT_MASK |
+ HDMA_V0_WATERMARK_INT_MASK |
+ HDMA_V0_ABORT_INT_MASK);
SET_CH_32(dw, dir, id, ch_en, 0);
}
}
@@ -161,6 +176,14 @@ static void dw_hdma_v0_core_clear_done_int(struct dw_edma_chan *chan)
SET_CH_32(dw, chan->dir, chan->id, int_clear, HDMA_V0_STOP_INT_MASK);
}
+static void dw_hdma_v0_core_clear_watermark_int(struct dw_edma_chan *chan)
+{
+ struct dw_edma *dw = chan->dw;
+
+ SET_CH_32(dw, chan->dir, chan->id, int_clear,
+ HDMA_V0_WATERMARK_INT_MASK);
+}
+
static void dw_hdma_v0_core_clear_abort_int(struct dw_edma_chan *chan)
{
struct dw_edma *dw = chan->dw;
@@ -177,7 +200,9 @@ static u32 dw_hdma_v0_core_status_int(struct dw_edma_chan *chan)
static irqreturn_t
dw_hdma_v0_core_handle_int(struct dw_edma_irq *dw_irq, enum dw_edma_dir dir,
- dw_edma_handler_t done, dw_edma_handler_t abort)
+ dw_edma_handler_t done,
+ dw_edma_handler_t progress,
+ dw_edma_handler_t abort)
{
struct dw_edma *dw = dw_irq->dw;
unsigned long total, pos, val;
@@ -196,18 +221,31 @@ dw_hdma_v0_core_handle_int(struct dw_edma_irq *dw_irq, enum dw_edma_dir dir,
}
for_each_set_bit(pos, mask, total) {
+ bool has_stop, has_watermark;
+
chan = &dw->chan[pos + off];
if (unlikely(dw_edma_core_ch_ignore_irq(chan)))
continue;
val = dw_hdma_v0_core_status_int(chan);
- if (FIELD_GET(HDMA_V0_STOP_INT_MASK, val)) {
- dw_hdma_v0_core_clear_done_int(chan);
- done(chan, true);
+ has_stop = FIELD_GET(HDMA_V0_STOP_INT_MASK, val);
+ has_watermark = FIELD_GET(HDMA_V0_WATERMARK_INT_MASK, val);
+ if (has_watermark) {
+ dw_hdma_v0_core_clear_watermark_int(chan);
ret = IRQ_HANDLED;
}
+ if (has_stop) {
+ dw_hdma_v0_core_clear_done_int(chan);
+ ret = IRQ_HANDLED;
+ }
+
+ if (has_stop)
+ done(chan, true);
+ else if (has_watermark)
+ progress(chan, false);
+
if (FIELD_GET(HDMA_V0_ABORT_INT_MASK, val)) {
dw_hdma_v0_core_clear_abort_int(chan);
abort(chan, false);
@@ -268,7 +306,7 @@ static void dw_hdma_v0_core_ch_enable(struct dw_edma_chan *chan)
/* Enable engine */
SET_CH_32(dw, chan->dir, chan->id, ch_en, BIT(0));
- /* Interrupt unmask - stop, abort */
+ /* Interrupt unmask - stop, watermark, abort */
tmp = GET_CH_32(dw, chan->dir, chan->id, int_setup);
tmp = dw_hdma_v0_core_int_setup(chan, tmp);
SET_CH_32(dw, chan->dir, chan->id, int_setup, tmp);
@@ -343,6 +381,11 @@ static void dw_hdma_v0_core_ch_config(struct dw_edma_chan *chan)
/* MSI done addr - low, high */
SET_CH_32(dw, chan->dir, chan->id, msi_stop.lsb, chan->msi.address_lo);
SET_CH_32(dw, chan->dir, chan->id, msi_stop.msb, chan->msi.address_hi);
+ /* MSI watermark addr - low, high */
+ SET_CH_32(dw, chan->dir, chan->id, msi_watermark.lsb,
+ chan->msi.address_lo);
+ SET_CH_32(dw, chan->dir, chan->id, msi_watermark.msb,
+ chan->msi.address_hi);
/* MSI abort addr - low, high */
SET_CH_32(dw, chan->dir, chan->id, msi_abort.lsb, chan->msi.address_lo);
SET_CH_32(dw, chan->dir, chan->id, msi_abort.msb, chan->msi.address_hi);
@@ -362,6 +405,13 @@ dw_hdma_v0_core_ll_data(struct dw_edma_chan *chan, struct dw_edma_burst *burst,
if (cb)
control |= DW_HDMA_V0_CB;
+ if (irq) {
+ control |= DW_HDMA_V0_LWIE;
+
+ if (chan->irq_mode == DW_EDMA_CH_IRQ_REMOTE)
+ control |= DW_HDMA_V0_RWIE;
+ }
+
dw_hdma_v0_write_ll_data(chan, idx, control, burst->sz, burst->sar,
burst->dar);
}
diff --git a/drivers/dma/dw-edma/dw-hdma-v0-regs.h b/drivers/dma/dw-edma/dw-hdma-v0-regs.h
index 2124c162a62f..60dc054adafa 100644
--- a/drivers/dma/dw-edma/dw-hdma-v0-regs.h
+++ b/drivers/dma/dw-edma/dw-hdma-v0-regs.h
@@ -18,6 +18,7 @@
#define HDMA_V0_LOCAL_STOP_INT_EN BIT(4)
#define HDMA_V0_REMOTE_STOP_INT_EN BIT(3)
#define HDMA_V0_ABORT_INT_MASK BIT(2)
+#define HDMA_V0_WATERMARK_INT_MASK BIT(1)
#define HDMA_V0_STOP_INT_MASK BIT(0)
#define HDMA_V0_LINKLIST_EN BIT(0)
#define HDMA_V0_CONSUMER_CYCLE_STAT BIT(1)
--
2.51.0