[PATCH v2 04/19] dmaengine: dw-edma: Make DMA link list work as a circular buffer
From: Koichiro Den
Date: Thu Jul 23 2026 - 04:51:26 EST
From: Frank Li <Frank.Li@xxxxxxx>
The driver currently rebuilds the whole linked list for every transfer.
Use it as a circular ring instead. Append entries at ll_head with the
current cycle bit, and reserve the final entry for the link back to the
start.
Clear control words before first use so stale cycle bits cannot become
valid entries. Reject rings without usable data slots and, until reclaim
support lands, descriptors that exceed the usable ring capacity.
This prepares the driver for appending requests while the engine runs.
Signed-off-by: Frank Li <Frank.Li@xxxxxxx>
Co-developed-by: Koichiro Den <den@xxxxxxxxxxxxx>
Signed-off-by: Koichiro Den <den@xxxxxxxxxxxxx>
---
Changes in v2:
- Track first use in software instead of inferring it from DMA_LLP.
- Clear stale entries and validate the LL layout; leave non-LL
configurations unchanged.
- Reject oversized descriptors until reclaim support lands.
drivers/dma/dw-edma/dw-edma-core.c | 86 ++++++++++++++++++++++++------
drivers/dma/dw-edma/dw-edma-core.h | 26 ++++++++-
2 files changed, 94 insertions(+), 18 deletions(-)
diff --git a/drivers/dma/dw-edma/dw-edma-core.c b/drivers/dma/dw-edma/dw-edma-core.c
index 6a25a050b89c..a9f25a65294e 100644
--- a/drivers/dma/dw-edma/dw-edma-core.c
+++ b/drivers/dma/dw-edma/dw-edma-core.c
@@ -51,13 +51,19 @@ dw_edma_alloc_desc(struct dw_edma_chan *chan, size_t nburst)
{
struct dw_edma_desc *desc;
+ /*
+ * For now, a descriptor that does not fit would stall the channel
+ * forever: reject it up front.
+ */
+ if (!chan->non_ll && nburst > chan->ll_max - 1)
+ return NULL;
+
desc = kzalloc_flex(*desc, burst, nburst, GFP_NOWAIT);
if (unlikely(!desc))
return NULL;
desc->chan = chan;
desc->nburst = nburst;
- desc->cb = true;
return desc;
}
@@ -67,30 +73,65 @@ static void vchan_free_desc(struct virt_dma_desc *vdesc)
kfree(vd2dw_edma_desc(vdesc));
}
+static void dw_edma_core_reset_ll(struct dw_edma_chan *chan)
+{
+ u32 i;
+
+ chan->ll_head = 0;
+ chan->ll_end = 0;
+ /* Drop stale CB bits before reusing the circular LL ring. */
+ for (i = 0; i < chan->ll_max; i++)
+ dw_edma_core_ll_clear(chan, i);
+ chan->cb = true;
+
+ dw_edma_core_ll_link(chan, chan->ll_max, chan->cb,
+ chan->ll_region.paddr);
+
+ dw_edma_core_ch_enable(chan);
+ chan->ll_valid = true;
+}
+
+static u32 dw_edma_core_get_free_num(struct dw_edma_chan *chan)
+{
+ /* Keep one data entry free so equal indices mean an empty ring. */
+ return (chan->ll_end + chan->ll_max - 1 - chan->ll_head) %
+ chan->ll_max;
+}
+
static void dw_edma_core_ll_start(struct dw_edma_desc *desc)
{
struct dw_edma_chan *chan = desc->chan;
size_t i;
- bool first = !desc->start_burst;
+ u32 free;
- for (i = 0; i + desc->start_burst < desc->nburst; i++) {
- u32 idx = i + desc->start_burst;
+ for (i = desc->start_burst; i < desc->nburst; i++) {
+ free = dw_edma_core_get_free_num(chan);
- if (i == chan->ll_max)
+ if (!free)
break;
- dw_edma_core_ll_data(chan, &desc->burst[idx],
- i, desc->cb,
- idx == desc->nburst - 1 || i == chan->ll_max - 1);
- }
+ /*
+ * Refresh the link element before filling the last data slot so
+ * the next lap has the updated CB value.
+ */
+ if (chan->ll_head == chan->ll_max - 1)
+ dw_edma_core_ll_link(chan, chan->ll_max, chan->cb,
+ chan->ll_region.paddr);
- desc->done_burst = desc->start_burst;
- desc->start_burst += i;
+ dw_edma_core_ll_data(chan, &desc->burst[i],
+ chan->ll_head, chan->cb,
+ i == desc->nburst - 1 || free == 1);
- dw_edma_core_ll_link(chan, i, desc->cb, chan->ll_region.paddr);
+ chan->ll_head++;
- if (first)
- dw_edma_core_ch_enable(chan);
+ if (chan->ll_head == chan->ll_max) {
+ chan->cb = !chan->cb;
+ chan->ll_head = 0;
+ }
+ }
+
+ desc->done_burst = desc->start_burst;
+ desc->start_burst = i;
dw_edma_core_ch_doorbell(chan);
}
@@ -123,9 +164,10 @@ static int dw_edma_start_transfer(struct dw_edma_chan *chan)
if (!desc)
return 0;
- dw_edma_core_start(desc);
+ if (!chan->non_ll && !chan->ll_valid)
+ dw_edma_core_reset_ll(chan);
- desc->cb = !desc->cb;
+ dw_edma_core_start(desc);
return 1;
}
@@ -645,6 +687,8 @@ static void dw_edma_done_interrupt(struct dw_edma_chan *chan)
DMA_TRANS_NOERROR);
list_del(&vd->node);
vchan_cookie_complete(vd);
+ if (!chan->non_ll)
+ chan->ll_end = chan->ll_head;
}
if (chan->request == EDMA_REQ_PAUSE) {
@@ -871,6 +915,9 @@ static int dw_edma_alloc_chan_resources(struct dma_chan *dchan)
if (chan->status != EDMA_ST_IDLE)
return -EBUSY;
+ /* The hardware context may have been invalidated while unowned. */
+ chan->ll_valid = false;
+
return 0;
}
@@ -962,6 +1009,13 @@ static int dw_edma_channel_setup(struct dw_edma *dw, u32 wr_alloc, u32 rd_alloc)
else
chan->ll_region = chip->ll_region_rd[chan->id];
+ if (!chip->cfg_non_ll && chan->ll_region.sz < 3 * EDMA_LL_SZ) {
+ dev_err(dev,
+ "channel %s[%u]: LL region has fewer than 2 data entries\n",
+ str_write_read(chan->dir == EDMA_DIR_WRITE),
+ chan->id);
+ return -EINVAL;
+ }
chan->ll_max = chan->ll_region.sz / EDMA_LL_SZ - 1;
dev_vdbg(dev, "L. List:\tChannel %s[%u] max_cnt=%u\n",
diff --git a/drivers/dma/dw-edma/dw-edma-core.h b/drivers/dma/dw-edma/dw-edma-core.h
index 089f913fd247..f2c2d5af1fff 100644
--- a/drivers/dma/dw-edma/dw-edma-core.h
+++ b/drivers/dma/dw-edma/dw-edma-core.h
@@ -60,7 +60,6 @@ struct dw_edma_desc {
size_t done_burst;
size_t start_burst;
- u8 cb;
size_t nburst;
struct dw_edma_burst burst[] __counted_by(nburst);
};
@@ -72,8 +71,31 @@ struct dw_edma_chan {
enum dw_edma_dir dir;
u8 func_no;
- u32 ll_max;
+ /*
+ * New LL entries are appended at ll_head. Entries between ll_end
+ * and ll_head, modulo the LL ring, are owned by DMA; the rest are
+ * owned by software.
+ *
+ * software-owned DMA-owned software-owned
+ * +---------------+-------------------+---------------+
+ * ^ ^ ^
+ * 0 ll_end ll_head
+ *
+ * The link entry points back to the region start. ll_head == ll_end
+ * means all entries are software-owned and previous DMA work is
+ * done.
+ *
+ * Software always keeps at least one free entry, so the ring is
+ * never completely DMA-owned.
+ */
+ u32 ll_head;
+ u32 ll_end;
+
+ u32 ll_max; /* Data entries */
struct dw_edma_region ll_region; /* Linked list */
+ bool ll_valid; /* LL context programmed */
+
+ bool cb;
struct msi_msg msi;
--
2.51.0