[PATCH 2/3] dmaengine: ae4dma: register AE4DMA controller as a DMA resource

[Sanjay R Mehta]

From: Sanjay R Mehta <sanju.mehta@xxxxxxx>

Register ae4dma queue's to Linux dmaengine framework as general-purpose
DMA channels.

Signed-off-by: Sanjay R Mehta <sanju.mehta@xxxxxxx>
---
 drivers/dma/ae4dma/Kconfig            |   2 +
 drivers/dma/ae4dma/Makefile           |   2 +-
 drivers/dma/ae4dma/ae4dma-dev.c       |  17 ++
 drivers/dma/ae4dma/ae4dma-dmaengine.c | 425 ++++++++++++++++++++++++++++++++++
 drivers/dma/ae4dma/ae4dma.h           |  28 +++
 5 files changed, 473 insertions(+), 1 deletion(-)
 create mode 100644 drivers/dma/ae4dma/ae4dma-dmaengine.c

diff --git a/drivers/dma/ae4dma/Kconfig b/drivers/dma/ae4dma/Kconfig
index 1cda9de..3d1dbb7 100644
--- a/drivers/dma/ae4dma/Kconfig
+++ b/drivers/dma/ae4dma/Kconfig
@@ -2,6 +2,8 @@
 config AMD_AE4DMA
 	tristate  "AMD AE4DMA Engine"
 	depends on X86_64 && PCI
+	select DMA_ENGINE
+	select DMA_VIRTUAL_CHANNELS
 	help
 	  Enabling this option provides support for the AMD AE4DMA controller,
 	  which offers DMA capabilities for high-bandwidth memory-to-memory and
diff --git a/drivers/dma/ae4dma/Makefile b/drivers/dma/ae4dma/Makefile
index db9cab1..b1e4318 100644
--- a/drivers/dma/ae4dma/Makefile
+++ b/drivers/dma/ae4dma/Makefile
@@ -5,6 +5,6 @@
 
 obj-$(CONFIG_AMD_AE4DMA) += ae4dma.o
 
-ae4dma-objs := ae4dma-dev.o
+ae4dma-objs := ae4dma-dev.o ae4dma-dmaengine.o
 
 ae4dma-$(CONFIG_PCI) += ae4dma-pci.o
diff --git a/drivers/dma/ae4dma/ae4dma-dev.c b/drivers/dma/ae4dma/ae4dma-dev.c
index a3c50a2..af7e510 100644
--- a/drivers/dma/ae4dma/ae4dma-dev.c
+++ b/drivers/dma/ae4dma/ae4dma-dev.c
@@ -25,6 +25,11 @@ static unsigned int max_hw_q = 1;
 module_param(max_hw_q, uint, 0444);
 MODULE_PARM_DESC(max_hw_q, "Max queues per engine (any non-zero value less than or equal to 16, default: 1)");
 
+static inline struct ae4_dma_chan *to_ae4_chan(struct dma_chan *dma_chan)
+{
+	return container_of(dma_chan, struct ae4_dma_chan, vc.chan);
+}
+
 /* Human-readable error strings */
 static char *ae4_error_codes[] = {
 	"",
@@ -273,8 +278,17 @@ int ae4_core_init(struct ae4_device *ae4)
 		tasklet_setup(&cmd_q->irq_tasklet, ae4_do_cmd_complete);
 	}
 
+	/* register to the DMA engine */
+	ret = ae4_dmaengine_register(ae4);
+	if (ret)
+		goto e_free_irq;
+
 	return 0;
 
+e_free_irq:
+	for (i = 0; i < ae4->cmd_q_count; i++)
+		free_irq(ae4->ae4_irq[i], ae4);
+
 e_free_dma:
 	dma_free_coherent(dev, cmd_q->qsize, cmd_q->qbase, cmd_q->qbase_dma);
 
@@ -292,6 +306,9 @@ void ae4_core_destroy(struct ae4_device *ae4)
 	struct ae4_cmd *cmd;
 	unsigned int i;
 
+	/* Unregister the DMA engine */
+	ae4_dmaengine_unregister(ae4);
+
 	for (i = 0; i < ae4->cmd_q_count; i++) {
 		cmd_q = &ae4->cmd_q[i];
 
diff --git a/drivers/dma/ae4dma/ae4dma-dmaengine.c b/drivers/dma/ae4dma/ae4dma-dmaengine.c
new file mode 100644
index 0000000..a50e4f5
--- /dev/null
+++ b/drivers/dma/ae4dma/ae4dma-dmaengine.c
@@ -0,0 +1,425 @@
+// SPDX-License-Identifier: GPL-2.0-only
+/*
+ * AMD AE4DMA device driver
+ * -- Based on the PTDMA driver
+ *
+ * Copyright (C) 2023 Advanced Micro Devices, Inc.
+ *
+ * Author: Sanjay R Mehta <sanju.mehta@xxxxxxx>
+ */
+#include <linux/delay.h>
+#include "ae4dma.h"
+#include "../dmaengine.h"
+#include "../virt-dma.h"
+
+static inline struct ae4_dma_chan *to_ae4_chan(struct dma_chan *dma_chan)
+{
+	return container_of(dma_chan, struct ae4_dma_chan, vc.chan);
+}
+
+static inline struct ae4_dma_desc *to_ae4_desc(struct virt_dma_desc *vd)
+{
+	return container_of(vd, struct ae4_dma_desc, vd);
+}
+
+static void ae4_free_chan_resources(struct dma_chan *dma_chan)
+{
+	struct ae4_dma_chan *chan = to_ae4_chan(dma_chan);
+
+	vchan_free_chan_resources(&chan->vc);
+}
+
+static void ae4_synchronize(struct dma_chan *dma_chan)
+{
+	struct ae4_dma_chan *chan = to_ae4_chan(dma_chan);
+
+	vchan_synchronize(&chan->vc);
+}
+
+static void ae4_do_cleanup(struct virt_dma_desc *vd)
+{
+	struct ae4_dma_desc *desc = to_ae4_desc(vd);
+	struct ae4_device *ae4 = desc->ae4;
+
+	kmem_cache_free(ae4->dma_desc_cache, desc);
+}
+
+static int ae4_dma_start_desc(struct ae4_dma_desc *desc, struct ae4_dma_chan *chan)
+{
+	struct ae4_passthru_engine *ae4_engine;
+	struct ae4_device *ae4;
+	struct ae4_cmd *ae4_cmd;
+	struct ae4_cmd_queue *cmd_q;
+
+	desc->issued_to_hw = 1;
+	list_del(&desc->vd.node);
+
+	ae4_cmd = &desc->ae4_cmd;
+	ae4 = ae4_cmd->ae4;
+	cmd_q = chan->cmd_q;
+	ae4_engine = &ae4_cmd->passthru;
+
+	ae4_cmd->qid = cmd_q->qidx;
+	cmd_q->tdata.cmd = ae4_cmd;
+
+	/* Execute the command */
+	ae4_cmd->ret = ae4_core_perform_passthru(cmd_q, ae4_engine);
+
+	return 0;
+}
+
+static struct ae4_dma_desc *ae4_next_dma_desc(struct ae4_dma_chan *chan)
+{
+	/* Get the next DMA descriptor on the active list */
+	struct virt_dma_desc *vd = vchan_next_desc(&chan->vc);
+
+	return vd ? to_ae4_desc(vd) : NULL;
+}
+
+static void ae4_cmd_callback_tasklet(void *data, int err)
+{
+	struct ae4_dma_desc *desc = data;
+	struct dma_chan *dma_chan;
+	struct ae4_dma_chan *chan;
+	struct dma_async_tx_descriptor *tx_desc;
+	struct virt_dma_desc *vd;
+	unsigned long flags;
+
+	dma_chan = desc->vd.tx.chan;
+	chan = to_ae4_chan(dma_chan);
+
+	if (err == -EINPROGRESS)
+		return;
+
+	tx_desc = &desc->vd.tx;
+	vd = &desc->vd;
+
+	if (err)
+		desc->status = DMA_ERROR;
+
+	spin_lock_irqsave(&chan->vc.lock, flags);
+	if (desc) {
+		if (desc->status != DMA_COMPLETE) {
+			if (desc->status != DMA_ERROR)
+				desc->status = DMA_COMPLETE;
+
+			dma_cookie_complete(tx_desc);
+			dma_descriptor_unmap(tx_desc);
+		} else {
+			/* Don't handle it twice */
+			tx_desc = NULL;
+		}
+	}
+	spin_unlock_irqrestore(&chan->vc.lock, flags);
+
+	if (tx_desc) {
+		dmaengine_desc_get_callback_invoke(tx_desc, NULL);
+		dma_run_dependencies(tx_desc);
+	}
+}
+
+static struct ae4_dma_desc *ae4_handle_active_desc(struct ae4_dma_chan *chan,
+						   struct ae4_dma_desc *desc)
+{
+	struct dma_async_tx_descriptor *tx_desc;
+	struct virt_dma_desc *vd;
+	unsigned long flags;
+
+	/* Loop over descriptors until one is found with commands */
+	do {
+		if (desc) {
+			if (!desc->issued_to_hw) {
+				/* No errors, keep going */
+				if (desc->status != DMA_ERROR)
+					return desc;
+			}
+			tx_desc = &desc->vd.tx;
+			vd = &desc->vd;
+		} else {
+			tx_desc = NULL;
+		}
+		spin_lock_irqsave(&chan->vc.lock, flags);
+		desc = ae4_next_dma_desc(chan);
+		spin_unlock_irqrestore(&chan->vc.lock, flags);
+	} while (desc);
+
+	return NULL;
+}
+
+static void ae4_cmd_callback(void *data, int err)
+{
+	struct ae4_dma_desc *desc = data;
+	struct dma_chan *dma_chan;
+	struct ae4_dma_chan *chan;
+	struct ae4_device *ae4;
+	int ret;
+
+	if (err == -EINPROGRESS)
+		return;
+
+	dma_chan = desc->vd.tx.chan;
+	chan = to_ae4_chan(dma_chan);
+	ae4 = chan->ae4;
+
+	if (err)
+		desc->status = DMA_ERROR;
+
+	while (true) {
+		/* Check for DMA descriptor completion */
+		desc = ae4_handle_active_desc(chan, desc);
+
+		/* Don't submit cmd if no descriptor or DMA is paused */
+		if (!desc)
+			break;
+
+		/* if queue is full dont submit to queue */
+		if (ae4_core_queue_full(ae4, chan->cmd_q)) {
+			usleep_range(100, 200);
+			continue;
+		}
+
+		ret = ae4_dma_start_desc(desc, chan);
+		if (!ret)
+			break;
+
+		desc->status = DMA_ERROR;
+	}
+}
+
+static struct ae4_dma_desc *ae4_alloc_dma_desc(struct ae4_dma_chan *chan, unsigned long flags)
+{
+	struct ae4_dma_desc *desc;
+	struct ae4_cmd_queue *cmd_q = chan->cmd_q;
+
+	desc = kmem_cache_zalloc(chan->ae4->dma_desc_cache, GFP_NOWAIT);
+	if (!desc)
+		return NULL;
+
+	vchan_tx_prep(&chan->vc, &desc->vd, flags);
+
+	desc->ae4 = chan->ae4;
+	cmd_q->int_en = !!(flags & DMA_PREP_INTERRUPT);
+	desc->issued_to_hw = 0;
+	desc->status = DMA_IN_PROGRESS;
+
+	return desc;
+}
+
+static struct ae4_dma_desc *ae4_create_desc(struct dma_chan *dma_chan,
+					    dma_addr_t dst,
+					    dma_addr_t src,
+					    unsigned int len,
+					    unsigned long flags)
+{
+	struct ae4_dma_chan *chan = to_ae4_chan(dma_chan);
+	struct ae4_cmd_queue *cmd_q = chan->cmd_q;
+	struct ae4_passthru_engine *ae4_engine;
+	struct ae4_dma_desc *desc;
+	struct ae4_cmd *ae4_cmd;
+
+	desc = ae4_alloc_dma_desc(chan, flags);
+	if (!desc)
+		return NULL;
+
+	ae4_cmd = &desc->ae4_cmd;
+	ae4_cmd->ae4 = chan->ae4;
+	ae4_engine = &ae4_cmd->passthru;
+	ae4_cmd->engine = AE4_ENGINE_PASSTHRU;
+	ae4_engine->src_dma = src;
+	ae4_engine->dst_dma = dst;
+	ae4_engine->src_len = len;
+	ae4_cmd->ae4_cmd_callback = ae4_cmd_callback_tasklet;
+	ae4_cmd->data = desc;
+
+	desc->len = len;
+
+	spin_lock_irqsave(&cmd_q->cmd_lock, flags);
+	list_add_tail(&ae4_cmd->entry, &cmd_q->cmd);
+	spin_unlock_irqrestore(&cmd_q->cmd_lock, flags);
+
+	return desc;
+}
+
+static struct dma_async_tx_descriptor *
+ae4_prep_dma_memcpy(struct dma_chan *dma_chan, dma_addr_t dst,
+		    dma_addr_t src, size_t len, unsigned long flags)
+{
+	struct ae4_dma_desc *desc;
+
+	desc = ae4_create_desc(dma_chan, dst, src, len, flags);
+	if (!desc)
+		return NULL;
+
+	return &desc->vd.tx;
+}
+
+static struct dma_async_tx_descriptor *
+ae4_prep_dma_interrupt(struct dma_chan *dma_chan, unsigned long flags)
+{
+	struct ae4_dma_chan *chan = to_ae4_chan(dma_chan);
+	struct ae4_dma_desc *desc;
+
+	desc = ae4_alloc_dma_desc(chan, flags);
+	if (!desc)
+		return NULL;
+
+	return &desc->vd.tx;
+}
+
+static void ae4_issue_pending(struct dma_chan *dma_chan)
+{
+	struct ae4_dma_chan *chan = to_ae4_chan(dma_chan);
+	struct ae4_dma_desc *desc;
+	unsigned long flags;
+
+	spin_lock_irqsave(&chan->vc.lock, flags);
+	vchan_issue_pending(&chan->vc);
+	desc = ae4_next_dma_desc(chan);
+	spin_unlock_irqrestore(&chan->vc.lock, flags);
+
+	ae4_cmd_callback(desc, 0);
+}
+
+static int ae4_pause(struct dma_chan *dma_chan)
+{
+	struct ae4_dma_chan *chan = to_ae4_chan(dma_chan);
+	unsigned long flags;
+
+	spin_lock_irqsave(&chan->vc.lock, flags);
+	//TODO : implemnt pause
+	spin_unlock_irqrestore(&chan->vc.lock, flags);
+
+	return 0;
+}
+
+static int ae4_resume(struct dma_chan *dma_chan)
+{
+	struct ae4_dma_chan *chan = to_ae4_chan(dma_chan);
+	struct ae4_dma_desc *desc = NULL;
+	unsigned long flags;
+
+	spin_lock_irqsave(&chan->vc.lock, flags);
+	ae4_start_queue(chan->cmd_q);
+	desc = ae4_next_dma_desc(chan);
+	spin_unlock_irqrestore(&chan->vc.lock, flags);
+
+	/* If there was something active, re-start */
+	if (desc)
+		ae4_cmd_callback(desc, 0);
+
+	return 0;
+}
+
+static int ae4_terminate_all(struct dma_chan *dma_chan)
+{
+	struct ae4_dma_chan *chan = to_ae4_chan(dma_chan);
+	unsigned long flags;
+	LIST_HEAD(head);
+
+	spin_lock_irqsave(&chan->vc.lock, flags);
+	vchan_get_all_descriptors(&chan->vc, &head);
+	spin_unlock_irqrestore(&chan->vc.lock, flags);
+
+	vchan_dma_desc_free_list(&chan->vc, &head);
+	vchan_free_chan_resources(&chan->vc);
+
+	return 0;
+}
+
+int ae4_dmaengine_register(struct ae4_device *ae4)
+{
+	struct ae4_dma_chan *chan;
+	struct ae4_cmd_queue *cmd_q;
+	struct dma_device *dma_dev = &ae4->dma_dev;
+	char *cmd_cache_name;
+	char *desc_cache_name;
+	unsigned int i;
+	int ret;
+
+	ae4->ae4_dma_chan = devm_kcalloc(ae4->dev, ae4->cmd_q_count,
+					 sizeof(*ae4->ae4_dma_chan), GFP_KERNEL);
+	if (!ae4->ae4_dma_chan)
+		return -ENOMEM;
+
+	cmd_cache_name = devm_kasprintf(ae4->dev, GFP_KERNEL,
+					"%s-dmaengine-cmd-cache",
+					dev_name(ae4->dev));
+	if (!cmd_cache_name)
+		return -ENOMEM;
+
+	desc_cache_name = devm_kasprintf(ae4->dev, GFP_KERNEL,
+					 "%s-dmaengine-desc-cache",
+					 dev_name(ae4->dev));
+	if (!desc_cache_name) {
+		ret = -ENOMEM;
+		goto err_cache;
+	}
+
+	ae4->dma_desc_cache = kmem_cache_create(desc_cache_name,
+						sizeof(struct ae4_dma_desc), 0,
+						SLAB_HWCACHE_ALIGN, NULL);
+	if (!ae4->dma_desc_cache) {
+		ret = -ENOMEM;
+		goto err_cache;
+	}
+
+	dma_dev->dev = ae4->dev;
+	dma_dev->src_addr_widths = DMA_SLAVE_BUSWIDTH_64_BYTES;
+	dma_dev->dst_addr_widths = DMA_SLAVE_BUSWIDTH_64_BYTES;
+	dma_dev->directions = DMA_MEM_TO_MEM;
+	dma_dev->residue_granularity = DMA_RESIDUE_GRANULARITY_DESCRIPTOR;
+	dma_cap_set(DMA_MEMCPY, dma_dev->cap_mask);
+	dma_cap_set(DMA_INTERRUPT, dma_dev->cap_mask);
+
+	/*
+	 * AE4DMA is intended to be used with the AMD NTB devices, hence
+	 * marking it as DMA_PRIVATE.
+	 */
+	dma_cap_set(DMA_PRIVATE, dma_dev->cap_mask);
+
+	/* Set base and prep routines */
+	dma_dev->device_free_chan_resources = ae4_free_chan_resources;
+	dma_dev->device_prep_dma_memcpy = ae4_prep_dma_memcpy;
+	dma_dev->device_prep_dma_interrupt = ae4_prep_dma_interrupt;
+	dma_dev->device_issue_pending = ae4_issue_pending;
+	dma_dev->device_tx_status = dma_cookie_status;
+	dma_dev->device_pause = ae4_pause;
+	dma_dev->device_resume = ae4_resume;
+	dma_dev->device_terminate_all = ae4_terminate_all;
+	dma_dev->device_synchronize = ae4_synchronize;
+
+	INIT_LIST_HEAD(&dma_dev->channels);
+	for (i = 0; i < ae4->cmd_q_count; i++) {
+		chan = ae4->ae4_dma_chan + i;
+		cmd_q = &ae4->cmd_q[i];
+		chan->cmd_q = cmd_q;
+		chan->id = cmd_q->id;
+		chan->ae4 = ae4;
+		chan->vc.desc_free = ae4_do_cleanup;
+		vchan_init(&chan->vc, dma_dev);
+	}
+
+	dma_set_mask_and_coherent(ae4->dev, DMA_BIT_MASK(64));
+
+	ret = dma_async_device_register(dma_dev);
+	if (ret)
+		goto err_reg;
+
+	return 0;
+
+err_reg:
+	kmem_cache_destroy(ae4->dma_desc_cache);
+
+err_cache:
+	kmem_cache_destroy(ae4->dma_cmd_cache);
+
+	return ret;
+}
+
+void ae4_dmaengine_unregister(struct ae4_device *ae4)
+{
+	struct dma_device *dma_dev = &ae4->dma_dev;
+
+	dma_async_device_unregister(dma_dev);
+	kmem_cache_destroy(ae4->dma_cmd_cache);
+}
diff --git a/drivers/dma/ae4dma/ae4dma.h b/drivers/dma/ae4dma/ae4dma.h
index 0ae46ee..d341c35 100644
--- a/drivers/dma/ae4dma/ae4dma.h
+++ b/drivers/dma/ae4dma/ae4dma.h
@@ -11,6 +11,7 @@
 #define __AE4_DEV_H__
 
 #include <linux/device.h>
+#include <linux/dmaengine.h>
 #include <linux/pci.h>
 #include <linux/spinlock.h>
 #include <linux/mutex.h>
@@ -18,6 +19,8 @@
 #include <linux/wait.h>
 #include <linux/dmapool.h>
 
+#include "../virt-dma.h"
+
 #define MAX_AE4_NAME_LEN			16
 #define MAX_DMAPOOL_NAME_LEN		32
 
@@ -168,6 +171,22 @@ struct ae4_cmd {
 	u8 qid;
 };
 
+struct ae4_dma_desc {
+	struct virt_dma_desc vd;
+	struct ae4_device *ae4;
+	enum dma_status status;
+	size_t len;
+	bool issued_to_hw;
+	struct ae4_cmd ae4_cmd;
+};
+
+struct ae4_dma_chan {
+	struct virt_dma_chan vc;
+	struct ae4_device *ae4;
+	struct ae4_cmd_queue *cmd_q;
+	u32 id;
+};
+
 struct ae4_cmd_queue {
 	struct ae4_device *ae4;
 
@@ -248,6 +267,12 @@ struct ae4_device {
 	struct ae4_cmd_queue cmd_q[MAX_HW_QUEUES];
 	unsigned int cmd_q_count;
 
+	/* Support for the DMA Engine capabilities */
+	struct dma_device dma_dev;
+	struct ae4_dma_chan *ae4_dma_chan;
+	struct kmem_cache *dma_cmd_cache;
+	struct kmem_cache *dma_desc_cache;
+
 	wait_queue_head_t lsb_queue;
 };
 
@@ -298,6 +323,9 @@ struct ae4_dev_vdata {
 	const unsigned int bar;
 };
 
+int ae4_dmaengine_register(struct ae4_device *ae4);
+void ae4_dmaengine_unregister(struct ae4_device *ae4);
+
 int ae4_core_init(struct ae4_device *ae4);
 void ae4_core_destroy(struct ae4_device *ae4);
 
-- 
2.7.4