[PATCH v2 05/19] dmaengine: dw-edma: Add LL interrupt placement policy

[Koichiro Den]

Move linked-list interrupt placement behind a core callback so eDMA and
HDMA can use different policies.

Keep the eDMA behavior. For HDMA, place watermarks at descriptor ends,
before link entries, and every four entries when a descriptor exceeds the
usable ring capacity or another issued descriptor follows. A later patch
handles these watermarks as progress events.

Four entries is an empirically chosen coalescing interval.

Signed-off-by: Koichiro Den <den@xxxxxxxxxxxxx>
---
Changes in v2:
  - Fix the zero-based interval so the first watermark is on the fourth
    entry, not the fifth.
  - Revise the commit message and source comments.

 drivers/dma/dw-edma/dw-edma-core.c    |  8 +++++++-
 drivers/dma/dw-edma/dw-edma-core.h    |  2 ++
 drivers/dma/dw-edma/dw-edma-v0-core.c | 10 ++++++++++
 drivers/dma/dw-edma/dw-hdma-v0-core.c | 28 +++++++++++++++++++++++++++
 4 files changed, 47 insertions(+), 1 deletion(-)

diff --git a/drivers/dma/dw-edma/dw-edma-core.c b/drivers/dma/dw-edma/dw-edma-core.c
index a9f25a65294e..78d1bb6302fb 100644
--- a/drivers/dma/dw-edma/dw-edma-core.c
+++ b/drivers/dma/dw-edma/dw-edma-core.c
@@ -98,6 +98,12 @@ static u32 dw_edma_core_get_free_num(struct dw_edma_chan *chan)
 		chan->ll_max;
 }
 
+static bool dw_edma_core_enable_ll_irq(struct dw_edma_desc *desc, u32 i,
+				       u32 free)
+{
+	return desc->chan->dw->core->ll_irq(desc, i, free);
+}
+
 static void dw_edma_core_ll_start(struct dw_edma_desc *desc)
 {
 	struct dw_edma_chan *chan = desc->chan;
@@ -120,7 +126,7 @@ static void dw_edma_core_ll_start(struct dw_edma_desc *desc)
 
 		dw_edma_core_ll_data(chan, &desc->burst[i],
 				     chan->ll_head, chan->cb,
-				     i == desc->nburst - 1 || free == 1);
+				     dw_edma_core_enable_ll_irq(desc, i, free));
 
 		chan->ll_head++;
 
diff --git a/drivers/dma/dw-edma/dw-edma-core.h b/drivers/dma/dw-edma/dw-edma-core.h
index f2c2d5af1fff..8ed37e8a12cb 100644
--- a/drivers/dma/dw-edma/dw-edma-core.h
+++ b/drivers/dma/dw-edma/dw-edma-core.h
@@ -162,6 +162,8 @@ struct dw_edma_core_ops {
 	void (*ll_link)(struct dw_edma_chan *chan, u32 idx, bool cb, u64 addr);
 	void (*ll_clear)(struct dw_edma_chan *chan, u32 idx);
 	int (*ll_cur_idx)(struct dw_edma_chan *chan);
+	/* Called with chan->vc.lock held. */
+	bool (*ll_irq)(struct dw_edma_desc *desc, u32 i, u32 free);
 	void (*ch_doorbell)(struct dw_edma_chan *chan);
 	void (*ch_enable)(struct dw_edma_chan *chan);
 	void (*ch_config)(struct dw_edma_chan *chan);
diff --git a/drivers/dma/dw-edma/dw-edma-v0-core.c b/drivers/dma/dw-edma/dw-edma-v0-core.c
index c31fff095b4f..62141aa32e50 100644
--- a/drivers/dma/dw-edma/dw-edma-v0-core.c
+++ b/drivers/dma/dw-edma/dw-edma-v0-core.c
@@ -649,6 +649,15 @@ static int dw_edma_v0_core_ll_cur_idx(struct dw_edma_chan *chan)
 	return (val - base) / EDMA_LL_SZ;
 }
 
+static bool dw_edma_v0_core_ll_irq(struct dw_edma_desc *desc, u32 i, u32 free)
+{
+	/*
+	 * eDMA reports LL interrupts through DONE. Keep them at
+	 * descriptor ends, plus the last free slot to refill the ring.
+	 */
+	return i == desc->nburst - 1 || free == 1;
+}
+
 /* eDMA debugfs callbacks */
 static void dw_edma_v0_core_debugfs_on(struct dw_edma *dw)
 {
@@ -685,6 +694,7 @@ static const struct dw_edma_core_ops dw_edma_v0_core = {
 	.ll_link = dw_edma_v0_core_ll_link,
 	.ll_clear = dw_edma_v0_core_ll_clear,
 	.ll_cur_idx = dw_edma_v0_core_ll_cur_idx,
+	.ll_irq = dw_edma_v0_core_ll_irq,
 	.ch_doorbell = dw_edma_v0_core_ch_doorbell,
 	.ch_enable = dw_edma_v0_core_ch_enable,
 	.ch_config = dw_edma_v0_core_ch_config,
diff --git a/drivers/dma/dw-edma/dw-hdma-v0-core.c b/drivers/dma/dw-edma/dw-hdma-v0-core.c
index b2d35f0b7b6d..fb651a83f0f3 100644
--- a/drivers/dma/dw-edma/dw-hdma-v0-core.c
+++ b/drivers/dma/dw-edma/dw-hdma-v0-core.c
@@ -13,6 +13,9 @@
 #include "dw-hdma-v0-regs.h"
 #include "dw-hdma-v0-debugfs.h"
 
+/* Empirically chosen watermark interval. */
+#define HDMA_V0_WATERMARK_INTERVAL			4
+
 enum dw_hdma_control {
 	DW_HDMA_V0_CB					= BIT(0),
 	DW_HDMA_V0_TCB					= BIT(1),
@@ -417,6 +420,30 @@ static int dw_hdma_v0_core_ll_cur_idx(struct dw_edma_chan *chan)
 	return (val - base) / EDMA_LL_SZ;
 }
 
+static bool dw_hdma_v0_core_ll_irq(struct dw_edma_desc *desc, u32 i, u32 free)
+{
+	struct dw_edma_chan *chan = desc->chan;
+	bool needs_progress;
+
+	/*
+	 * Always place a watermark at a descriptor end and before the link
+	 * element. The first reports descriptor progress; the second provides
+	 * one progress point per ring lap.
+	 */
+	if (i == desc->nburst - 1 || chan->ll_head == chan->ll_max - 1)
+		return true;
+
+	/*
+	 * Use periodic watermarks when a descriptor exceeds the usable ring or
+	 * when more issued work follows it.
+	 */
+	needs_progress = desc->nburst > chan->ll_max - 1 ||
+			 !list_is_last(&desc->vd.node, &chan->vc.desc_issued);
+
+	return needs_progress &&
+	       (chan->ll_head + 1) % HDMA_V0_WATERMARK_INTERVAL == 0;
+}
+
 /* HDMA debugfs callbacks */
 static void dw_hdma_v0_core_debugfs_on(struct dw_edma *dw)
 {
@@ -441,6 +468,7 @@ static const struct dw_edma_core_ops dw_hdma_v0_core = {
 	.ll_link = dw_hdma_v0_core_ll_link,
 	.ll_clear = dw_hdma_v0_core_ll_clear,
 	.ll_cur_idx = dw_hdma_v0_core_ll_cur_idx,
+	.ll_irq = dw_hdma_v0_core_ll_irq,
 	.ch_doorbell = dw_hdma_v0_core_ch_doorbell,
 	.ch_enable = dw_hdma_v0_core_ch_enable,
 	.ch_config = dw_hdma_v0_core_ch_config,
-- 
2.51.0