[PATCH v4 2/2] dmaengine: fsl-edma: Support dynamic scatter/gather chaining
From: Benoît Monin
Date: Mon May 18 2026 - 08:40:31 EST
Implement dynamic linking of scatter/gather transfers to enable
chaining multiple DMA descriptors without stopping the channel.
This avoids waiting for the channel to go idle if there is another
transaction already issued.
Add fsl_edma_link_sg() to dynamically link the last TCD of a previously
submitted descriptor to the first TCD of a new descriptor by setting
the scatter/gather address and the E_SG flag, and keeping the channel
active by clearing the DREQ bit.
Linking is done when the transaction is submitted by fsl_edma_tx_submit().
To do so, the .tx_submit() callback is overridden for non-cyclic
transactions prepared by fsl_edma_prep_peripheral_dma_vec() and
fsl_edma_prep_slave_sg(). This ensures that transactions are linked
in the order they are submitted.
Update fsl_edma_xfer_desc() to avoid re-initializing the hardware when a
transfer is already in progress, allowing seamless chaining of descriptors.
Modify the transfer completion handler to check the DONE flag in the
channel CSR before marking the transfer complete. Since this flag is
only available on SoC with the split registers layout, we only link
transactions for DMA controllers flagged with FSL_EDMA_DRV_SPLIT_REG.
Add trace event for scatter/gather linking operations.
Signed-off-by: Benoît Monin <benoit.monin@xxxxxxxxxxx>
---
drivers/dma/fsl-edma-common.c | 90 +++++++++++++++++++++++++++++++++++++++----
drivers/dma/fsl-edma-trace.h | 5 +++
2 files changed, 88 insertions(+), 7 deletions(-)
diff --git a/drivers/dma/fsl-edma-common.c b/drivers/dma/fsl-edma-common.c
index c10190164926..6e5820051f29 100644
--- a/drivers/dma/fsl-edma-common.c
+++ b/drivers/dma/fsl-edma-common.c
@@ -58,7 +58,10 @@ void fsl_edma_tx_chan_handler(struct fsl_edma_chan *fsl_chan)
list_del(&fsl_chan->edesc->vdesc.node);
vchan_cookie_complete(&fsl_chan->edesc->vdesc);
fsl_chan->edesc = NULL;
- fsl_chan->status = DMA_COMPLETE;
+ if (!(fsl_edma_drvflags(fsl_chan) & FSL_EDMA_DRV_SPLIT_REG) ||
+ (edma_readl_chreg(fsl_chan, ch_csr) & EDMA_V3_CH_CSR_DONE)) {
+ fsl_chan->status = DMA_COMPLETE;
+ }
} else {
vchan_cyclic_callback(&fsl_chan->edesc->vdesc);
}
@@ -673,6 +676,68 @@ struct dma_async_tx_descriptor *fsl_edma_prep_dma_cyclic(
return vchan_tx_prep(&fsl_chan->vchan, &fsl_desc->vdesc, flags);
}
+static void fsl_edma_link_sg(struct fsl_edma_chan *fsl_chan, struct fsl_edma_desc *fsl_desc)
+{
+ u32 flags = fsl_edma_drvflags(fsl_chan);
+ struct fsl_edma_hw_tcd *last_tcd;
+ struct fsl_edma_desc *prev_desc;
+ struct virt_dma_desc *vdesc;
+ u16 csr;
+
+ lockdep_assert_held(&fsl_chan->vchan.lock);
+
+ if (!(flags & FSL_EDMA_DRV_SPLIT_REG))
+ return;
+
+ vdesc = list_last_entry_or_null(&fsl_chan->vchan.desc_submitted,
+ struct virt_dma_desc, node);
+ if (!vdesc)
+ vdesc = list_last_entry_or_null(&fsl_chan->vchan.desc_issued,
+ struct virt_dma_desc, node);
+ if (!vdesc)
+ return;
+
+ prev_desc = to_fsl_edma_desc(vdesc);
+ last_tcd = prev_desc->tcd[prev_desc->n_tcds - 1].vtcd;
+
+ csr = fsl_edma_get_tcd_to_cpu(fsl_chan, last_tcd, csr);
+ if (!(csr & EDMA_TCD_CSR_D_REQ))
+ return;
+
+ fsl_edma_set_tcd_to_le(fsl_chan, last_tcd, fsl_desc->tcd[0].ptcd, dlast_sga);
+
+ csr &= ~EDMA_TCD_CSR_D_REQ;
+ csr |= EDMA_TCD_CSR_E_SG;
+ fsl_edma_set_tcd_to_le(fsl_chan, last_tcd, csr, csr);
+
+ if (prev_desc == fsl_chan->edesc && prev_desc->n_tcds == 1) {
+ if (flags & FSL_EDMA_DRV_CLEAR_DONE_E_SG)
+ edma_writel_chreg(fsl_chan, edma_readl_chreg(fsl_chan, ch_csr), ch_csr);
+
+ edma_cp_tcd_to_reg(fsl_chan, last_tcd, dlast_sga);
+ edma_cp_tcd_to_reg(fsl_chan, last_tcd, csr);
+ }
+
+ trace_edma_link_sg(fsl_chan, last_tcd);
+}
+
+static dma_cookie_t fsl_edma_tx_submit(struct dma_async_tx_descriptor *tx)
+{
+ struct virt_dma_desc *vd = container_of(tx, struct virt_dma_desc, tx);
+ struct fsl_edma_chan *fsl_chan = to_fsl_edma_chan(tx->chan);
+ struct fsl_edma_desc *fsl_desc = to_fsl_edma_desc(vd);
+ struct virt_dma_chan *vc = to_virt_chan(tx->chan);
+ dma_cookie_t cookie;
+
+ guard(spinlock_irqsave)(&fsl_chan->vchan.lock);
+
+ fsl_edma_link_sg(fsl_chan, fsl_desc);
+ cookie = dma_cookie_assign(tx);
+ list_move_tail(&vd->node, &vc->desc_submitted);
+
+ return cookie;
+}
+
struct dma_async_tx_descriptor *
fsl_edma_prep_peripheral_dma_vec(struct dma_chan *chan, const struct dma_vec *vecs,
size_t nb, enum dma_transfer_direction direction,
@@ -680,6 +745,7 @@ fsl_edma_prep_peripheral_dma_vec(struct dma_chan *chan, const struct dma_vec *ve
{
struct fsl_edma_chan *fsl_chan = to_fsl_edma_chan(chan);
dma_addr_t src_addr, dst_addr, last_sg;
+ struct dma_async_tx_descriptor *tx;
struct fsl_edma_desc *fsl_desc;
u16 soff, doff, iter;
u32 nbytes;
@@ -779,7 +845,10 @@ fsl_edma_prep_peripheral_dma_vec(struct dma_chan *chan, const struct dma_vec *ve
}
}
- return vchan_tx_prep(&fsl_chan->vchan, &fsl_desc->vdesc, flags);
+ tx = vchan_tx_prep(&fsl_chan->vchan, &fsl_desc->vdesc, flags);
+ if (!fsl_desc->iscyclic)
+ tx->tx_submit = fsl_edma_tx_submit;
+ return tx;
}
struct dma_async_tx_descriptor *fsl_edma_prep_slave_sg(
@@ -788,9 +857,10 @@ struct dma_async_tx_descriptor *fsl_edma_prep_slave_sg(
unsigned long flags, void *context)
{
struct fsl_edma_chan *fsl_chan = to_fsl_edma_chan(chan);
+ dma_addr_t src_addr, dst_addr, last_sg;
+ struct dma_async_tx_descriptor *tx;
struct fsl_edma_desc *fsl_desc;
struct scatterlist *sg;
- dma_addr_t src_addr, dst_addr, last_sg;
u16 soff, doff, iter;
u32 nbytes;
int i;
@@ -882,7 +952,10 @@ struct dma_async_tx_descriptor *fsl_edma_prep_slave_sg(
}
}
- return vchan_tx_prep(&fsl_chan->vchan, &fsl_desc->vdesc, flags);
+ tx = vchan_tx_prep(&fsl_chan->vchan, &fsl_desc->vdesc, flags);
+ tx->tx_submit = fsl_edma_tx_submit;
+
+ return tx;
}
struct dma_async_tx_descriptor *fsl_edma_prep_memcpy(struct dma_chan *chan,
@@ -924,9 +997,12 @@ void fsl_edma_xfer_desc(struct fsl_edma_chan *fsl_chan)
if (!vdesc)
return;
fsl_chan->edesc = to_fsl_edma_desc(vdesc);
- fsl_edma_set_tcd_regs(fsl_chan, fsl_chan->edesc->tcd[0].vtcd);
- fsl_edma_enable_request(fsl_chan);
- fsl_chan->status = DMA_IN_PROGRESS;
+
+ if (fsl_chan->status != DMA_IN_PROGRESS) {
+ fsl_edma_set_tcd_regs(fsl_chan, fsl_chan->edesc->tcd[0].vtcd);
+ fsl_edma_enable_request(fsl_chan);
+ fsl_chan->status = DMA_IN_PROGRESS;
+ }
}
void fsl_edma_issue_pending(struct dma_chan *chan)
diff --git a/drivers/dma/fsl-edma-trace.h b/drivers/dma/fsl-edma-trace.h
index d3541301a247..ac319d2dbb90 100644
--- a/drivers/dma/fsl-edma-trace.h
+++ b/drivers/dma/fsl-edma-trace.h
@@ -119,6 +119,11 @@ DEFINE_EVENT(edma_log_tcd, edma_fill_tcd,
TP_ARGS(chan, tcd)
);
+DEFINE_EVENT(edma_log_tcd, edma_link_sg,
+ TP_PROTO(struct fsl_edma_chan *chan, void *tcd),
+ TP_ARGS(chan, tcd)
+);
+
#endif
/* this part must be outside header guard */
--
2.54.0