[PATCH 2/2] dma: Add Spreadtrum DMA controller driver
From: Baolin Wang
Date: Tue Jul 18 2017 - 03:12:15 EST
This patch adds the DMA controller driver for Spreadtrum SC9860 platform.
Signed-off-by: Baolin Wang <baolin.wang@xxxxxxxxxxxxxx>
---
drivers/dma/Kconfig | 7 +
drivers/dma/Makefile | 1 +
drivers/dma/sprd-dma.c | 1451 ++++++++++++++++++++++++++++++++++++++++++
include/linux/dma/sprd-dma.h | 270 ++++++++
4 files changed, 1729 insertions(+)
create mode 100644 drivers/dma/sprd-dma.c
create mode 100644 include/linux/dma/sprd-dma.h
diff --git a/drivers/dma/Kconfig b/drivers/dma/Kconfig
index fa8f9c0..961f6ea 100644
--- a/drivers/dma/Kconfig
+++ b/drivers/dma/Kconfig
@@ -477,6 +477,13 @@ config STM32_DMA
If you have a board based on such a MCU and wish to use DMA say Y
here.
+config SPRD_DMA
+ bool "Spreadtrum DMA support"
+ depends on ARCH_SPRD
+ select DMA_ENGINE
+ help
+ Enable support for the on-chip DMA controller on Spreadtrum platform.
+
config S3C24XX_DMAC
bool "Samsung S3C24XX DMA support"
depends on ARCH_S3C24XX || COMPILE_TEST
diff --git a/drivers/dma/Makefile b/drivers/dma/Makefile
index d12ab29..0fee561 100644
--- a/drivers/dma/Makefile
+++ b/drivers/dma/Makefile
@@ -58,6 +58,7 @@ obj-$(CONFIG_RENESAS_DMA) += sh/
obj-$(CONFIG_SIRF_DMA) += sirf-dma.o
obj-$(CONFIG_STE_DMA40) += ste_dma40.o ste_dma40_ll.o
obj-$(CONFIG_STM32_DMA) += stm32-dma.o
+obj-$(CONFIG_SPRD_DMA) += sprd-dma.o
obj-$(CONFIG_S3C24XX_DMAC) += s3c24xx-dma.o
obj-$(CONFIG_TXX9_DMAC) += txx9dmac.o
obj-$(CONFIG_TEGRA20_APB_DMA) += tegra20-apb-dma.o
diff --git a/drivers/dma/sprd-dma.c b/drivers/dma/sprd-dma.c
new file mode 100644
index 0000000..64eaef7
--- /dev/null
+++ b/drivers/dma/sprd-dma.c
@@ -0,0 +1,1451 @@
+/*
+ * Copyright (C) 2017 Spreadtrum Communications Inc.
+ *
+ * This software is licensed under the terms of the GNU General Public
+ * License version 2, as published by the Free Software Foundation, and
+ * may be copied, distributed, and modified under those terms.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ */
+
+#include <linux/clk.h>
+#include <linux/dma-mapping.h>
+#include <linux/dma/sprd-dma.h>
+#include <linux/errno.h>
+#include <linux/init.h>
+#include <linux/interrupt.h>
+#include <linux/io.h>
+#include <linux/kernel.h>
+#include <linux/module.h>
+#include <linux/of.h>
+#include <linux/of_dma.h>
+#include <linux/of_device.h>
+#include <linux/pm_runtime.h>
+#include <linux/slab.h>
+
+#include "dmaengine.h"
+
+#define SPRD_DMA_DESCRIPTORS 16
+#define SPRD_DMA_CFG_COUNT 32
+#define SPRD_DMA_CHN_REG_OFFSET 0x1000
+#define SPRD_DMA_CHN_REG_LENGTH 0x40
+#define SPRD_DMA_MEMCPY_MIN_SIZE 64
+
+/* DMA global registers definition */
+#define DMA_GLB_PAUSE 0x0
+#define DMA_GLB_FRAG_WAIT 0x4
+#define DMA_GLB_REQ_PEND0_EN 0x8
+#define DMA_GLB_REQ_PEND1_EN 0xc
+#define DMA_GLB_INT_RAW_STS 0x10
+#define DMA_GLB_INT_MSK_STS 0x14
+#define DMA_GLB_REQ_STS 0x18
+#define DMA_GLB_CHN_EN_STS 0x1c
+#define DMA_GLB_DEBUG_STS 0x20
+#define DMA_GLB_ARB_SEL_STS 0x24
+#define DMA_GLB_CHN_START_CHN_CFG1 0x28
+#define DMA_GLB_CHN_START_CHN_CFG2 0x2c
+#define DMA_CHN_LLIST_OFFSET 0x10
+#define DMA_GLB_REQ_CID(base, uid) \
+ ((unsigned long)(base) + 0x2000 + 0x4 * ((uid) - 1))
+
+/* DMA_GLB_CHN_START_CHN_CFG register definition */
+#define SRC_CHN_OFFSET 0
+#define DST_CHN_OFFSET 8
+#define START_MODE_OFFSET 16
+#define CHN_START_CHN BIT(24)
+
+/* DMA channel registers definition */
+#define DMA_CHN_PAUSE 0x0
+#define DMA_CHN_REQ 0x4
+#define DMA_CHN_CFG 0x8
+#define DMA_CHN_INTC 0xc
+#define DMA_CHN_SRC_ADDR 0x10
+#define DMA_CHN_DES_ADDR 0x14
+#define DMA_CHN_FRG_LEN 0x18
+#define DMA_CHN_BLK_LEN 0x1c
+#define DMA_CHN_TRSC_LEN 0x20
+#define DMA_CHN_TRSF_STEP 0x24
+#define DMA_CHN_WARP_PTR 0x28
+#define DMA_CHN_WARP_TO 0x2c
+#define DMA_CHN_LLIST_PTR 0x30
+#define DMA_CHN_FRAG_STEP 0x34
+#define DMA_CHN_SRC_BLK_STEP 0x38
+#define DMA_CHN_DES_BLK_STEP 0x3c
+
+/* DMA_CHN_INTC register definition */
+#define DMA_CHN_INT_MASK GENMASK(4, 0)
+#define DMA_CHN_INT_CLR_OFFSET 24
+#define FRAG_INT_EN BIT(0)
+#define BLK_INT_EN BIT(1)
+#define TRANS_INT_EN BIT(2)
+#define LIST_INT_EN BIT(3)
+#define CFG_ERROR_INT_EN BIT(4)
+
+/* DMA_CHN_CFG register definition */
+#define DMA_CHN_EN BIT(0)
+#define DMA_CHN_PRIORITY_OFFSET 12
+#define LLIST_EN_OFFSET 4
+#define CHN_WAIT_BDONE 24
+#define DMA_DONOT_WAIT_BDONE 1
+
+/* DMA_CHN_REQ register definition */
+#define DMA_CHN_REQ_EN BIT(0)
+
+/* DMA_CHN_PAUSE register definition */
+#define DMA_CHN_PAUSE_EN BIT(0)
+#define DMA_CHN_PAUSE_STS BIT(2)
+#define DMA_CHN_PAUSE_CNT 0x2000
+
+/* DMA_CHN_WARP_* register definition */
+#define DMA_CHN_ADDR_MASK GENMASK(31, 28)
+#define DMA_CHN_HIGH_ADDR_OFFSET 4
+#define WRAP_DATA_MASK GENMASK(27, 0)
+
+/* DMA_CHN_INTC register definition */
+#define FRAG_INT_STS BIT(8)
+#define BLK_INT_STS BIT(9)
+#define TRSC_INT_STS BIT(10)
+#define LLIST_INT_STS BIT(11)
+#define CFGERR_INT_STS BIT(12)
+
+/* DMA_CHN_FRG_LEN register definition */
+#define SRC_DATAWIDTH_OFFSET 30
+#define DES_DATAWIDTH_OFFSET 28
+#define SWT_MODE_OFFSET 26
+#define REQ_MODE_OFFSET 24
+#define REQ_MODE_MASK 0x3
+#define ADDR_WRAP_SEL_OFFSET 23
+#define ADDR_WRAP_EN_OFFSET 22
+#define ADDR_FIX_SEL_OFFSET 21
+#define ADDR_FIX_SEL_EN 20
+#define LLIST_END_OFFSET 19
+#define BLK_LEN_REC_H_OFFSET 17
+#define FRG_LEN_OFFSET 0
+#define FRG_LEN_MASK GENMASK(16, 0)
+
+/* DMA_CHN_BLK_LEN register definition */
+#define BLK_LEN_MASK GENMASK(16, 0)
+
+/* DMA_CHN_TRSC_LEN register definition */
+#define TRSC_LEN_MASK GENMASK(27, 0)
+
+/* DMA_CHN_TRSF_STEP register definition */
+#define DEST_TRSF_STEP_OFFSET 16
+#define SRC_TRSF_STEP_OFFSET 0
+#define TRSF_STEP_MASK GENMASK(15, 0)
+
+/* DMA_CHN_FRAG_STEP register definition */
+#define DEST_FRAG_STEP_OFFSET 16
+#define SRC_FRAG_STEP_OFFSET 0
+#define FRAG_STEP_MASK GENMASK(15, 0)
+
+/* DMA_CHN_SRC_BLK_STEP register definition */
+#define PTR_HIGH_ADDR_MASK GENMASK(31, 28)
+#define PTR_HIGH_ADDR_OFFSET 4
+
+enum dma_config_type {
+ SINGLE_CONFIG,
+ LINKLIST_CONFIG,
+};
+
+/* dma channel configuration */
+struct sprd_dma_chn_config {
+ u32 pause;
+ u32 req;
+ u32 cfg;
+ u32 intc;
+ u32 src_addr;
+ u32 des_addr;
+ u32 frg_len;
+ u32 blk_len;
+ u32 trsc_len;
+ u32 trsf_step;
+ u32 wrap_ptr;
+ u32 wrap_to;
+ u32 llist_ptr;
+ u32 frg_step;
+ u32 src_blk_step;
+ u32 des_blk_step;
+};
+
+/* dma request description */
+struct sprd_dma_desc {
+ struct dma_async_tx_descriptor desc;
+ struct sprd_dma_chn_config *chn_config;
+ struct list_head node;
+ enum dma_flags dma_flags;
+ int cycle;
+};
+
+/* dma channel description */
+struct sprd_dma_chn {
+ struct dma_chan chan;
+ struct list_head free;
+ struct list_head prepared;
+ struct list_head queued;
+ struct list_head active;
+ struct list_head completed;
+ void __iomem *chn_base;
+ spinlock_t chn_lock;
+ int chn_num;
+ u32 dev_id;
+ void *dma_desc_configs;
+ struct sprd_dma_cfg dma_cfg[SPRD_DMA_CFG_COUNT];
+ int cfg_count;
+};
+
+/* SPRD dma device */
+struct sprd_dma_dev {
+ struct dma_device dma_dev;
+ void __iomem *glb_base;
+ struct clk *clk;
+ struct clk *ashb_clk;
+ int irq;
+ struct tasklet_struct tasklet;
+ struct kmem_cache *dma_desc_node_cachep;
+ u32 total_chns;
+ struct sprd_dma_chn channels[0];
+};
+
+static bool sprd_dma_filter_fn(struct dma_chan *chan, void *param);
+static struct of_dma_filter_info sprd_dma_info = {
+ .filter_fn = sprd_dma_filter_fn,
+};
+
+static inline struct sprd_dma_chn *to_sprd_dma_chan(struct dma_chan *c)
+{
+ return container_of(c, struct sprd_dma_chn, chan);
+}
+
+static inline struct sprd_dma_dev *to_sprd_dma_dev(struct dma_chan *c)
+{
+ struct sprd_dma_chn *mchan = to_sprd_dma_chan(c);
+
+ return container_of(mchan, struct sprd_dma_dev, channels[c->chan_id]);
+}
+
+static inline struct sprd_dma_desc *
+to_sprd_dma_desc(struct dma_async_tx_descriptor *tx)
+{
+ return container_of(tx, struct sprd_dma_desc, desc);
+}
+
+static int sprd_dma_enable(struct sprd_dma_dev *sdev)
+{
+ int ret;
+
+ ret = clk_prepare_enable(sdev->clk);
+ if (ret)
+ return ret;
+
+ if (!IS_ERR(sdev->ashb_clk))
+ ret = clk_prepare_enable(sdev->ashb_clk);
+
+ return ret;
+}
+
+static void sprd_dma_disable(struct sprd_dma_dev *sdev)
+{
+ clk_disable_unprepare(sdev->clk);
+
+ if (!IS_ERR(sdev->ashb_clk))
+ clk_disable_unprepare(sdev->ashb_clk);
+}
+
+static void sprd_dma_set_uid(struct sprd_dma_chn *mchan)
+{
+ struct sprd_dma_dev *sdev = to_sprd_dma_dev(&mchan->chan);
+ u32 dev_id = mchan->dev_id;
+
+ if (dev_id != DMA_SOFTWARE_UID)
+ writel(mchan->chn_num + 1, (void __iomem *)DMA_GLB_REQ_CID(
+ sdev->glb_base, dev_id));
+}
+
+static void sprd_dma_unset_uid(struct sprd_dma_chn *mchan)
+{
+ struct sprd_dma_dev *sdev = to_sprd_dma_dev(&mchan->chan);
+ u32 dev_id = mchan->dev_id;
+
+ if (dev_id != DMA_SOFTWARE_UID)
+ writel(0, (void __iomem *)DMA_GLB_REQ_CID(sdev->glb_base,
+ dev_id));
+}
+
+static void sprd_dma_clear_int(struct sprd_dma_chn *mchan)
+{
+ u32 intc = readl(mchan->chn_base + DMA_CHN_INTC);
+
+ intc |= DMA_CHN_INT_MASK << DMA_CHN_INT_CLR_OFFSET;
+ writel(intc, mchan->chn_base + DMA_CHN_INTC);
+}
+
+static void sprd_dma_enable_chn(struct sprd_dma_chn *mchan)
+{
+ u32 cfg = readl(mchan->chn_base + DMA_CHN_CFG);
+
+ cfg |= DMA_CHN_EN;
+ writel(cfg, mchan->chn_base + DMA_CHN_CFG);
+}
+
+static void sprd_dma_disable_chn(struct sprd_dma_chn *mchan)
+{
+ u32 cfg = readl(mchan->chn_base + DMA_CHN_CFG);
+
+ cfg &= ~DMA_CHN_EN;
+ writel(cfg, mchan->chn_base + DMA_CHN_CFG);
+}
+
+static void sprd_dma_soft_request(struct sprd_dma_chn *mchan)
+{
+ u32 req = readl(mchan->chn_base + DMA_CHN_REQ);
+
+ req |= DMA_CHN_REQ_EN;
+ writel(req, mchan->chn_base + DMA_CHN_REQ);
+}
+
+static void sprd_dma_stop_and_disable(struct sprd_dma_chn *mchan)
+{
+ u32 cfg = readl(mchan->chn_base + DMA_CHN_CFG);
+ u32 pause, timeout = DMA_CHN_PAUSE_CNT;
+
+ if (!(cfg & DMA_CHN_EN))
+ return;
+
+ pause = readl(mchan->chn_base + DMA_CHN_PAUSE);
+ pause |= DMA_CHN_PAUSE_EN;
+ writel(pause, mchan->chn_base + DMA_CHN_PAUSE);
+
+ do {
+ pause = readl(mchan->chn_base + DMA_CHN_PAUSE);
+ if (pause & DMA_CHN_PAUSE_STS)
+ break;
+
+ cpu_relax();
+ } while (--timeout > 0);
+
+ sprd_dma_disable_chn(mchan);
+ writel(0, mchan->chn_base + DMA_CHN_PAUSE);
+}
+
+static unsigned long sprd_dma_get_dst_addr(struct sprd_dma_chn *mchan)
+{
+ unsigned long addr;
+
+ addr = readl(mchan->chn_base + DMA_CHN_DES_ADDR);
+ addr |= (readl(mchan->chn_base + DMA_CHN_WARP_TO) &
+ DMA_CHN_ADDR_MASK) << DMA_CHN_HIGH_ADDR_OFFSET;
+ return addr;
+}
+
+static enum dma_int_type sprd_dma_get_int_type(struct sprd_dma_chn *mchan)
+{
+ u32 intc_reg = readl(mchan->chn_base + DMA_CHN_INTC);
+
+ if (intc_reg & CFGERR_INT_STS)
+ return CONFIG_ERR;
+ else if (intc_reg & LLIST_INT_STS)
+ return LIST_DONE;
+ else if (intc_reg & TRSC_INT_STS)
+ return TRANS_DONE;
+ else if (intc_reg & BLK_INT_STS)
+ return BLK_DONE;
+ else if (intc_reg & FRAG_INT_STS)
+ return FRAG_DONE;
+ else
+ return NO_INT;
+}
+
+static enum dma_request_mode sprd_dma_get_req_type(struct sprd_dma_chn *mchan)
+{
+ u32 frag_reg = readl(mchan->chn_base + DMA_CHN_FRG_LEN);
+
+ switch ((frag_reg >> REQ_MODE_OFFSET) & REQ_MODE_MASK) {
+ case 0:
+ return FRAG_REQ_MODE;
+ case 1:
+ return BLOCK_REQ_MODE;
+ case 2:
+ return TRANS_REQ_MODE;
+ case 3:
+ return LIST_REQ_MODE;
+ default:
+ return FRAG_REQ_MODE;
+ }
+}
+
+static int sprd_dma_chn_start_chn(struct sprd_dma_chn *mchan,
+ struct sprd_dma_desc *mdesc)
+{
+ struct sprd_dma_dev *sdev = to_sprd_dma_dev(&mchan->chan);
+ enum dma_flags flag = mdesc->dma_flags;
+ int chn = mchan->chn_num + 1;
+ unsigned int cfg_group1, cfg_group2, start_mode = 0;
+
+ if (!(flag & (DMA_GROUP1_SRC | DMA_GROUP2_SRC | DMA_GROUP1_DST |
+ DMA_GROUP2_DST)))
+ return 0;
+
+ if (flag & (DMA_GROUP1_SRC | DMA_GROUP2_SRC)) {
+ switch (flag & (DMA_MUTL_FRAG_DONE | DMA_MUTL_BLK_DONE |
+ DMA_MUTL_TRANS_DONE | DMA_MUTL_LIST_DONE)) {
+ case DMA_MUTL_FRAG_DONE:
+ start_mode = 0;
+ break;
+ case DMA_MUTL_BLK_DONE:
+ start_mode = 1;
+ break;
+ case DMA_MUTL_TRANS_DONE:
+ start_mode = 2;
+ break;
+ case DMA_MUTL_LIST_DONE:
+ start_mode = 3;
+ break;
+ default:
+ dev_err(sdev->dma_dev.dev,
+ "chn stat chn mode incorrect\n");
+ return -EINVAL;
+ }
+ }
+
+ cfg_group1 = readl(sdev->glb_base + DMA_GLB_CHN_START_CHN_CFG1);
+ cfg_group2 = readl(sdev->glb_base + DMA_GLB_CHN_START_CHN_CFG2);
+
+ switch (flag & (DMA_GROUP1_SRC | DMA_GROUP2_SRC |
+ DMA_GROUP1_DST | DMA_GROUP2_DST)) {
+ case DMA_GROUP1_SRC:
+ cfg_group1 |= chn << SRC_CHN_OFFSET;
+ cfg_group1 |= (1 << start_mode) << START_MODE_OFFSET;
+ cfg_group1 |= CHN_START_CHN;
+ writel(cfg_group1, sdev->glb_base + DMA_GLB_CHN_START_CHN_CFG1);
+ break;
+ case DMA_GROUP2_SRC:
+ cfg_group2 |= chn << SRC_CHN_OFFSET;
+ cfg_group2 |= (1 << start_mode) << START_MODE_OFFSET;
+ cfg_group2 |= CHN_START_CHN;
+ writel(cfg_group2, sdev->glb_base + DMA_GLB_CHN_START_CHN_CFG2);
+ break;
+ case DMA_GROUP1_DST:
+ cfg_group1 |= chn << DST_CHN_OFFSET;
+ cfg_group1 |= CHN_START_CHN;
+ writel(cfg_group1, sdev->glb_base + DMA_GLB_CHN_START_CHN_CFG1);
+ break;
+ case DMA_GROUP2_DST:
+ cfg_group2 |= chn << DST_CHN_OFFSET;
+ cfg_group2 |= CHN_START_CHN;
+ writel(cfg_group2, sdev->glb_base + DMA_GLB_CHN_START_CHN_CFG2);
+ break;
+ default:
+ break;
+ }
+
+ return 0;
+}
+
+static int sprd_dma_set_chn_config(struct sprd_dma_chn *mchan,
+ struct sprd_dma_desc *mdesc)
+{
+ struct sprd_dma_chn_config *cfg = mdesc->chn_config;
+ int ret;
+
+ ret = sprd_dma_chn_start_chn(mchan, mdesc);
+ if (ret)
+ return ret;
+
+ writel(cfg->pause, mchan->chn_base + DMA_CHN_PAUSE);
+ writel(cfg->cfg, mchan->chn_base + DMA_CHN_CFG);
+ writel(cfg->intc, mchan->chn_base + DMA_CHN_INTC);
+ writel(cfg->src_addr, mchan->chn_base + DMA_CHN_SRC_ADDR);
+ writel(cfg->des_addr, mchan->chn_base + DMA_CHN_DES_ADDR);
+ writel(cfg->frg_len, mchan->chn_base + DMA_CHN_FRG_LEN);
+ writel(cfg->blk_len, mchan->chn_base + DMA_CHN_BLK_LEN);
+ writel(cfg->trsc_len, mchan->chn_base + DMA_CHN_TRSC_LEN);
+ writel(cfg->trsf_step, mchan->chn_base + DMA_CHN_TRSF_STEP);
+ writel(cfg->wrap_ptr, mchan->chn_base + DMA_CHN_WARP_PTR);
+ writel(cfg->wrap_to, mchan->chn_base + DMA_CHN_WARP_TO);
+ writel(cfg->llist_ptr, mchan->chn_base + DMA_CHN_LLIST_PTR);
+ writel(cfg->frg_step, mchan->chn_base + DMA_CHN_FRAG_STEP);
+ writel(cfg->src_blk_step, mchan->chn_base + DMA_CHN_SRC_BLK_STEP);
+ writel(cfg->des_blk_step, mchan->chn_base + DMA_CHN_DES_BLK_STEP);
+ writel(cfg->req, mchan->chn_base + DMA_CHN_REQ);
+
+ return 0;
+}
+
+static int sprd_dma_start(struct sprd_dma_chn *mchan)
+{
+ struct sprd_dma_desc *first;
+ int ret;
+
+ if (list_empty(&mchan->active))
+ return 0;
+
+ /*
+ * Get the first DMA descriptor from active list, and copy the DMA
+ * configuration from DMA descriptor to this DMA channel.
+ */
+ first = list_first_entry(&mchan->active, struct sprd_dma_desc, node);
+ ret = sprd_dma_set_chn_config(mchan, first);
+ if (ret)
+ return ret;
+
+ sprd_dma_set_uid(mchan);
+ sprd_dma_enable_chn(mchan);
+
+ if (mchan->dev_id == DMA_SOFTWARE_UID &&
+ !(first->dma_flags & (DMA_GROUP1_DST | DMA_GROUP2_DST)))
+ sprd_dma_soft_request(mchan);
+
+ return 0;
+}
+
+static void sprd_dma_stop(struct sprd_dma_chn *mchan)
+{
+ sprd_dma_unset_uid(mchan);
+ sprd_dma_stop_and_disable(mchan);
+ sprd_dma_clear_int(mchan);
+}
+
+static int sprd_dma_process_completed(struct sprd_dma_dev *sdev)
+{
+ struct sprd_dma_chn *mchan;
+ struct sprd_dma_desc *mdesc;
+ struct sprd_dma_desc *first;
+ struct dma_async_tx_descriptor *desc;
+ u32 dma_total_chns = sdev->total_chns;
+ dma_cookie_t last_cookie = 0;
+ unsigned long flags;
+ LIST_HEAD(list);
+ int i;
+
+ for (i = 0; i < dma_total_chns; i++) {
+ mchan = &sdev->channels[i];
+
+ spin_lock_irqsave(&mchan->chn_lock, flags);
+ if (!list_empty(&mchan->completed))
+ list_splice_tail_init(&mchan->completed, &list);
+ spin_unlock_irqrestore(&mchan->chn_lock, flags);
+
+ if (list_empty(&list))
+ continue;
+
+ list_for_each_entry(mdesc, &list, node) {
+ desc = &mdesc->desc;
+
+ if (desc->callback)
+ desc->callback(desc->callback_param);
+
+ dma_run_dependencies(desc);
+ last_cookie = desc->cookie;
+ }
+
+ spin_lock_irqsave(&mchan->chn_lock, flags);
+ list_splice_tail_init(&list, &mchan->free);
+
+ /*
+ * Check if there are pending DMA descriptor in queued list
+ * need to be queued.
+ */
+ if (!list_empty(&mchan->queued)) {
+ list_for_each_entry(mdesc, &mchan->queued, node) {
+ /*
+ * If the DMA descriptor flag is set
+ * DMA_GROUP1_DST or DMA_GROUP1_DST flag, which
+ * means this DMA descriptor do not need to be
+ * activated and it will be invoked by another
+ * DMA descriptor which set DMA_GROUP1_SRC or
+ * DMA_GROUP2_SRC flag.
+ */
+ if (!(mdesc->dma_flags &
+ (DMA_GROUP1_DST | DMA_GROUP2_DST)) &&
+ list_empty(&mchan->active)) {
+ first = list_first_entry(&mchan->queued,
+ struct sprd_dma_desc, node);
+ list_move_tail(&first->node,
+ &mchan->active);
+ sprd_dma_start(mchan);
+ break;
+ }
+ }
+ } else {
+ mchan->chan.completed_cookie = last_cookie;
+ }
+
+ spin_unlock_irqrestore(&mchan->chn_lock, flags);
+ }
+
+ return 0;
+}
+
+static void sprd_dma_tasklet(unsigned long data)
+{
+ struct sprd_dma_dev *sdev = (void *)data;
+
+ sprd_dma_process_completed(sdev);
+}
+
+static bool sprd_dma_check_trans_done(struct sprd_dma_desc *mdesc,
+ enum dma_int_type int_type,
+ enum dma_request_mode req_mode)
+{
+ if (mdesc->cycle == 1)
+ return false;
+
+ if ((unsigned int)int_type >= ((unsigned int)req_mode + 1))
+ return true;
+ else
+ return false;
+}
+
+static irqreturn_t dma_irq_handle(int irq, void *dev_id)
+{
+ struct sprd_dma_dev *sdev = (struct sprd_dma_dev *)dev_id;
+ u32 irq_status = readl(sdev->glb_base + DMA_GLB_INT_MSK_STS);
+ struct sprd_dma_chn *mchan;
+ struct sprd_dma_desc *mdesc;
+ struct dma_async_tx_descriptor *desc;
+ enum dma_request_mode req_type;
+ enum dma_int_type int_type;
+ bool trans_done = false;
+ u32 i;
+
+ while (irq_status) {
+ i = __ffs(irq_status);
+ irq_status &= (irq_status - 1);
+ mchan = &sdev->channels[i];
+
+ spin_lock(&mchan->chn_lock);
+ int_type = sprd_dma_get_int_type(mchan);
+ req_type = sprd_dma_get_req_type(mchan);
+ sprd_dma_clear_int(mchan);
+
+ if (!list_empty(&mchan->active)) {
+ mdesc = list_first_entry(&mchan->active,
+ struct sprd_dma_desc, node);
+ /*
+ * Check if the dma request descriptor is done, if it
+ * is done we should move this descriptor into
+ * complete list to handle.
+ */
+ trans_done = sprd_dma_check_trans_done(mdesc, int_type,
+ req_type);
+ if (trans_done == true)
+ list_splice_tail_init(&mchan->active,
+ &mchan->completed);
+ /*
+ * If the cycle is 1, which means this DMA descriptor
+ * will always in active state until user free this DMA
+ * channel's resources. But user may need to be notified
+ * when transfer interrupt is generated.
+ */
+ if (mdesc->cycle == 1) {
+ desc = &mdesc->desc;
+ spin_unlock(&mchan->chn_lock);
+ if (desc->callback)
+ desc->callback(desc->callback_param);
+ continue;
+ }
+ }
+ spin_unlock(&mchan->chn_lock);
+ }
+
+ tasklet_schedule(&sdev->tasklet);
+ return IRQ_HANDLED;
+}
+
+static dma_cookie_t sprd_desc_submit(struct dma_async_tx_descriptor *tx)
+{
+ struct sprd_dma_chn *mchan = to_sprd_dma_chan(tx->chan);
+ struct sprd_dma_desc *mdesc = to_sprd_dma_desc(tx);
+ struct sprd_dma_desc *first;
+ unsigned long flags;
+ dma_cookie_t cookie;
+ int ret;
+
+ spin_lock_irqsave(&mchan->chn_lock, flags);
+ cookie = dma_cookie_assign(tx);
+ list_move_tail(&mdesc->node, &mchan->queued);
+
+ /*
+ * If we set DMA_GROUP1_DST or DMA_GROUP2_DST flags, which means we do
+ * not need start this DMA transfer and it will be invoked by other DMA
+ * transfer setting DMA_GROUP1_SRC or DMA_GROUP2_SRC flag.
+ */
+ if (mdesc->dma_flags & (DMA_GROUP1_DST | DMA_GROUP2_DST)) {
+ ret = sprd_dma_set_chn_config(mchan, mdesc);
+ if (ret) {
+ spin_unlock_irqrestore(&mchan->chn_lock, flags);
+ return ret;
+ }
+
+ sprd_dma_enable_chn(mchan);
+ } else if (list_empty(&mchan->active)) {
+ first = list_first_entry(&mchan->queued, struct sprd_dma_desc,
+ node);
+ list_move_tail(&first->node, &mchan->active);
+ ret = sprd_dma_start(mchan);
+ if (ret)
+ cookie = ret;
+ }
+ spin_unlock_irqrestore(&mchan->chn_lock, flags);
+
+ return cookie;
+}
+
+static int sprd_dma_alloc_chan_resources(struct dma_chan *chan)
+{
+ struct sprd_dma_chn *mchan = to_sprd_dma_chan(chan);
+ struct sprd_dma_dev *sdev = to_sprd_dma_dev(chan);
+ struct sprd_dma_desc *mdesc, *tmp;
+ struct sprd_dma_chn_config *chn_configs;
+ LIST_HEAD(descs);
+ unsigned long flags;
+ int ret, i;
+
+ ret = pm_runtime_get_sync(chan->device->dev);
+ if (ret < 0) {
+ dev_err(sdev->dma_dev.dev, "pm runtime get failed: %d\n", ret);
+ return ret;
+ }
+
+ chn_configs = devm_kzalloc(sdev->dma_dev.dev,
+ SPRD_DMA_DESCRIPTORS *
+ sizeof(struct sprd_dma_chn_config),
+ GFP_KERNEL);
+ if (!chn_configs) {
+ ret = -ENOMEM;
+ goto err_alloc;
+ }
+
+ for (i = 0; i < SPRD_DMA_DESCRIPTORS; i++) {
+ mdesc = (struct sprd_dma_desc *)kmem_cache_zalloc(
+ sdev->dma_desc_node_cachep,
+ GFP_ATOMIC);
+ if (!mdesc) {
+ ret = -ENOMEM;
+ goto err_cache_alloc;
+ }
+
+ dma_async_tx_descriptor_init(&mdesc->desc, chan);
+ mdesc->desc.flags = DMA_CTRL_ACK;
+ mdesc->desc.tx_submit = sprd_desc_submit;
+ mdesc->chn_config = &chn_configs[i];
+ mdesc->cycle = 0;
+ mdesc->dma_flags = 0;
+ INIT_LIST_HEAD(&mdesc->node);
+ list_add_tail(&mdesc->node, &descs);
+ }
+
+ spin_lock_irqsave(&mchan->chn_lock, flags);
+ list_splice_tail_init(&descs, &mchan->free);
+ spin_unlock_irqrestore(&mchan->chn_lock, flags);
+
+ mchan->dma_desc_configs = chn_configs;
+ mchan->dev_id = 0;
+ mchan->cfg_count = 0;
+ memset(mchan->dma_cfg, 0, sizeof(struct sprd_dma_cfg) *
+ SPRD_DMA_CFG_COUNT);
+
+ return 0;
+
+err_cache_alloc:
+ list_for_each_entry_safe(mdesc, tmp, &descs, node)
+ kmem_cache_free(sdev->dma_desc_node_cachep, mdesc);
+ devm_kfree(sdev->dma_dev.dev, chn_configs);
+err_alloc:
+ pm_runtime_put_sync(chan->device->dev);
+ return ret;
+}
+
+static void sprd_dma_free_chan_resources(struct dma_chan *chan)
+{
+ struct sprd_dma_chn *mchan = to_sprd_dma_chan(chan);
+ struct sprd_dma_dev *sdev = to_sprd_dma_dev(chan);
+ struct sprd_dma_desc *mdesc, *tmp;
+ unsigned long flags;
+ LIST_HEAD(descs);
+
+ spin_lock_irqsave(&mchan->chn_lock, flags);
+ list_splice_tail_init(&mchan->prepared, &mchan->free);
+ list_splice_tail_init(&mchan->queued, &mchan->free);
+ list_splice_tail_init(&mchan->active, &mchan->free);
+ list_splice_tail_init(&mchan->completed, &mchan->free);
+ list_splice_tail_init(&mchan->free, &descs);
+ spin_unlock_irqrestore(&mchan->chn_lock, flags);
+
+ list_for_each_entry_safe(mdesc, tmp, &descs, node)
+ kmem_cache_free(sdev->dma_desc_node_cachep, mdesc);
+ devm_kfree(sdev->dma_dev.dev, mchan->dma_desc_configs);
+
+ spin_lock_irqsave(&mchan->chn_lock, flags);
+ sprd_dma_stop(mchan);
+ spin_unlock_irqrestore(&mchan->chn_lock, flags);
+
+ pm_runtime_put_sync(chan->device->dev);
+}
+
+static enum dma_status sprd_dma_tx_status(struct dma_chan *chan,
+ dma_cookie_t cookie,
+ struct dma_tx_state *txstate)
+{
+ struct sprd_dma_chn *mchan = to_sprd_dma_chan(chan);
+ unsigned long flags;
+ enum dma_status ret;
+
+ ret = dma_cookie_status(chan, cookie, txstate);
+
+ spin_lock_irqsave(&mchan->chn_lock, flags);
+ txstate->residue = sprd_dma_get_dst_addr(mchan);
+ spin_unlock_irqrestore(&mchan->chn_lock, flags);
+
+ return ret;
+}
+
+static void sprd_dma_issue_pending(struct dma_chan *chan)
+{
+ /*
+ * We are posting descriptors to the hardware as soon as
+ * they are ready, so this function does nothing.
+ */
+}
+
+static int sprd_dma_config(struct dma_chan *chan, struct sprd_dma_desc *mdesc,
+ struct sprd_dma_cfg *cfg_list,
+ struct sprd_dma_chn_config *chn_config,
+ enum dma_config_type type)
+{
+ struct sprd_dma_chn *mchan = to_sprd_dma_chan(chan);
+ struct sprd_dma_dev *sdev = to_sprd_dma_dev(&mchan->chan);
+ struct sprd_dma_cfg *cfg = cfg_list;
+ struct sprd_dma_chn_config *dma_chn_config;
+ static unsigned long first_link_p;
+ unsigned long linklist_ptr = 0;
+ u32 fix_mode = 0, fix_en = 0;
+ u32 wrap_en = 0, wrap_mode = 0;
+ u32 llist_en = 0, list_end = 0;
+
+ if (!IS_ALIGNED(cfg->src_step, 1 << cfg->datawidth)) {
+ dev_err(sdev->dma_dev.dev, "source step is not aligned\n");
+ return -EINVAL;
+ }
+
+ if (!IS_ALIGNED(cfg->des_step, 1 << cfg->datawidth)) {
+ dev_err(sdev->dma_dev.dev, "destination step is not aligned\n");
+ return -EINVAL;
+ }
+
+ if (type == SINGLE_CONFIG)
+ dma_chn_config = mdesc->chn_config;
+ else if (type == LINKLIST_CONFIG)
+ dma_chn_config = chn_config;
+ else {
+ dev_err(sdev->dma_dev.dev,
+ "check configuration type failed %d\n", type);
+ return -EINVAL;
+ }
+
+ if (!mchan->dev_id)
+ mchan->dev_id = cfg->dev_id;
+
+ dma_chn_config->pause = 0;
+ dma_chn_config->req = 0;
+
+ if (cfg->link_cfg_p)
+ llist_en = 1;
+
+ dma_chn_config->cfg = cfg->chn_pri << DMA_CHN_PRIORITY_OFFSET |
+ llist_en << LLIST_EN_OFFSET |
+ DMA_DONOT_WAIT_BDONE << CHN_WAIT_BDONE;
+
+ if (cfg->wrap_ptr && cfg->wrap_to) {
+ wrap_en = 1;
+ dma_chn_config->wrap_ptr = cfg->wrap_ptr & WRAP_DATA_MASK;
+ dma_chn_config->wrap_to = cfg->wrap_to & WRAP_DATA_MASK;
+ }
+
+ dma_chn_config->wrap_ptr |=
+ (u32)((cfg->src_addr >> DMA_CHN_HIGH_ADDR_OFFSET) &
+ DMA_CHN_ADDR_MASK);
+ dma_chn_config->wrap_to |=
+ (u32)((cfg->des_addr >> DMA_CHN_HIGH_ADDR_OFFSET) &
+ DMA_CHN_ADDR_MASK);
+
+ dma_chn_config->src_addr = (u32)cfg->src_addr;
+ dma_chn_config->des_addr = (u32)cfg->des_addr;
+
+ if ((cfg->src_step != 0 && cfg->des_step != 0) ||
+ ((cfg->src_step | cfg->des_step) == 0)) {
+ fix_en = 0;
+ } else {
+ fix_en = 1;
+ if (cfg->src_step)
+ fix_mode = 1;
+ else
+ fix_mode = 0;
+ }
+
+ if (wrap_en == 1) {
+ if (cfg->wrap_to == cfg->src_addr) {
+ wrap_mode = 0;
+ } else if (cfg->wrap_to == cfg->des_addr) {
+ wrap_mode = 1;
+ } else {
+ dev_err(sdev->dma_dev.dev,
+ "check wrap config failed\n");
+ return -EINVAL;
+ }
+ }
+
+ if (llist_en == 1 && cfg->is_end == DMA_END)
+ list_end = 1;
+
+ dma_chn_config->frg_len = (cfg->datawidth << SRC_DATAWIDTH_OFFSET) |
+ (cfg->datawidth << DES_DATAWIDTH_OFFSET) |
+ (cfg->swt_mode << SWT_MODE_OFFSET) |
+ (cfg->req_mode << REQ_MODE_OFFSET) |
+ (wrap_mode << ADDR_WRAP_SEL_OFFSET) |
+ (wrap_en << ADDR_WRAP_EN_OFFSET) |
+ (fix_mode << ADDR_FIX_SEL_OFFSET) |
+ (fix_en << ADDR_FIX_SEL_EN) |
+ (list_end << LLIST_END_OFFSET) |
+ (cfg->fragmens_len & FRG_LEN_MASK);
+
+ dma_chn_config->blk_len = cfg->block_len & BLK_LEN_MASK;
+
+ if (type == SINGLE_CONFIG) {
+ dma_chn_config->intc &= ~DMA_CHN_INT_MASK;
+ dma_chn_config->intc |= CFG_ERROR_INT_EN;
+
+ switch (cfg->irq_mode) {
+ case NO_INT:
+ break;
+ case FRAG_DONE:
+ dma_chn_config->intc |= FRAG_INT_EN;
+ break;
+ case BLK_DONE:
+ dma_chn_config->intc |= BLK_INT_EN;
+ break;
+ case BLOCK_FRAG_DONE:
+ dma_chn_config->intc |= BLK_INT_EN | FRAG_INT_EN;
+ break;
+ case TRANS_DONE:
+ dma_chn_config->intc |= TRANS_INT_EN;
+ break;
+ case TRANS_FRAG_DONE:
+ dma_chn_config->intc |= TRANS_INT_EN | FRAG_INT_EN;
+ break;
+ case TRANS_BLOCK_DONE:
+ dma_chn_config->intc |= TRANS_INT_EN | BLK_INT_EN;
+ break;
+ case LIST_DONE:
+ dma_chn_config->intc |= LIST_INT_EN;
+ break;
+ case CONFIG_ERR:
+ dma_chn_config->intc |= CFG_ERROR_INT_EN;
+ break;
+ default:
+ dev_err(sdev->dma_dev.dev, "irq mode failed\n");
+ return -EINVAL;
+ }
+ } else {
+ dma_chn_config->intc = 0;
+ }
+
+ if (cfg->transcation_len == 0)
+ dma_chn_config->trsc_len = cfg->block_len & TRSC_LEN_MASK;
+ else
+ dma_chn_config->trsc_len = cfg->transcation_len &
+ TRSC_LEN_MASK;
+
+ dma_chn_config->trsf_step = (cfg->des_step & TRSF_STEP_MASK) <<
+ DEST_TRSF_STEP_OFFSET |
+ (cfg->src_step & TRSF_STEP_MASK) << SRC_TRSF_STEP_OFFSET;
+
+ dma_chn_config->frg_step = (cfg->dst_frag_step & FRAG_STEP_MASK)
+ << DEST_FRAG_STEP_OFFSET |
+ (cfg->src_frag_step & FRAG_STEP_MASK)
+ << SRC_FRAG_STEP_OFFSET;
+
+ dma_chn_config->src_blk_step = cfg->src_blk_step;
+ dma_chn_config->des_blk_step = cfg->dst_blk_step;
+
+ if (first_link_p == 0)
+ first_link_p = cfg->link_cfg_p;
+
+ if (type == SINGLE_CONFIG) {
+ linklist_ptr = first_link_p + DMA_CHN_LLIST_OFFSET;
+ first_link_p = 0;
+ } else if (type == LINKLIST_CONFIG) {
+ if (cfg->is_end == DMA_LINK)
+ linklist_ptr = first_link_p + DMA_CHN_LLIST_OFFSET;
+ else
+ linklist_ptr = cfg->link_cfg_p +
+ sizeof(struct sprd_dma_chn_config) +
+ DMA_CHN_LLIST_OFFSET;
+ }
+
+ dma_chn_config->src_blk_step |=
+ (u32)((linklist_ptr >> PTR_HIGH_ADDR_OFFSET) &
+ PTR_HIGH_ADDR_MASK);
+ dma_chn_config->llist_ptr = (u32)linklist_ptr;
+ return 0;
+}
+
+static int sprd_dma_config_linklist(struct dma_chan *chan,
+ struct sprd_dma_desc *mdesc,
+ struct sprd_dma_cfg *cfg_list,
+ u32 node_size)
+{
+ struct sprd_dma_dev *sdev = to_sprd_dma_dev(chan);
+ struct sprd_dma_chn_config *chn_config_list_v;
+ struct sprd_dma_cfg start_list;
+ int ret, i;
+
+ chn_config_list_v = (struct sprd_dma_chn_config *)cfg_list[0].link_cfg_v;
+
+ for (i = 0; i < node_size; i++) {
+ if (cfg_list[i].link_cfg_v == 0 || cfg_list[i].link_cfg_p == 0) {
+ dev_err(sdev->dma_dev.dev,
+ "no allocated memory for list node\n");
+ return -EINVAL;
+ }
+
+ ret = sprd_dma_config(chan, NULL, cfg_list + i,
+ chn_config_list_v + i,
+ LINKLIST_CONFIG);
+ if (ret < 0) {
+ dev_err(sdev->dma_dev.dev,
+ "linklist configuration failed\n");
+ return ret;
+ }
+ }
+
+ memset((void *)&start_list, 0, sizeof(struct sprd_dma_cfg));
+ start_list.link_cfg_p = cfg_list[0].link_cfg_p;
+ start_list.irq_mode = cfg_list[0].irq_mode;
+ start_list.src_addr = cfg_list[0].src_addr;
+ start_list.des_addr = cfg_list[0].des_addr;
+
+ if (cfg_list[node_size - 1].is_end == DMA_LINK)
+ mdesc->cycle = 1;
+
+ return sprd_dma_config(chan, mdesc, &start_list, NULL, SINGLE_CONFIG);
+}
+
+struct dma_async_tx_descriptor *sprd_dma_prep_dma_memcpy(struct dma_chan *chan,
+ dma_addr_t dest,
+ dma_addr_t src,
+ size_t len,
+ unsigned long flags)
+{
+ struct sprd_dma_dev *sdev = to_sprd_dma_dev(chan);
+ struct sprd_dma_chn *mchan = to_sprd_dma_chan(chan);
+ struct sprd_dma_desc *mdesc = NULL;
+ struct sprd_dma_cfg dma_cfg;
+ u32 datawidth, src_step, des_step;
+ int cfg_count = mchan->cfg_count;
+ unsigned long irq_flags;
+ int ret;
+
+ /*
+ * If user did not have configuration for DMA transfer, then we should
+ * configure the DMA controller as default configuration.
+ */
+ if (cfg_count == 0) {
+ if (IS_ALIGNED(len, 4)) {
+ datawidth = 2;
+ src_step = 4;
+ des_step = 4;
+ } else if (IS_ALIGNED(len, 2)) {
+ datawidth = 1;
+ src_step = 2;
+ des_step = 2;
+ } else {
+ datawidth = 0;
+ src_step = 1;
+ des_step = 1;
+ }
+
+ memset(&dma_cfg, 0, sizeof(struct sprd_dma_cfg));
+ dma_cfg.src_addr = src;
+ dma_cfg.des_addr = dest;
+ dma_cfg.datawidth = datawidth;
+ dma_cfg.src_step = src_step;
+ dma_cfg.des_step = src_step;
+ dma_cfg.fragmens_len = SPRD_DMA_MEMCPY_MIN_SIZE;
+ if (len <= BLK_LEN_MASK) {
+ dma_cfg.block_len = len;
+ dma_cfg.req_mode = BLOCK_REQ_MODE;
+ dma_cfg.irq_mode = BLK_DONE;
+ } else {
+ dma_cfg.block_len = SPRD_DMA_MEMCPY_MIN_SIZE;
+ dma_cfg.transcation_len = len;
+ dma_cfg.req_mode = TRANS_REQ_MODE;
+ dma_cfg.irq_mode = TRANS_DONE;
+ }
+ cfg_count = 1;
+ } else if (cfg_count == 1) {
+ memcpy(&dma_cfg, mchan->dma_cfg, sizeof(struct sprd_dma_cfg));
+ }
+
+ /* get one free DMA descriptor for this DMA transfer. */
+ spin_lock_irqsave(&mchan->chn_lock, irq_flags);
+ if (!list_empty(&mchan->free)) {
+ mdesc = list_first_entry(&mchan->free,
+ struct sprd_dma_desc, node);
+ list_del(&mdesc->node);
+ }
+ spin_unlock_irqrestore(&mchan->chn_lock, irq_flags);
+
+ if (!mdesc) {
+ dev_err(sdev->dma_dev.dev, "get free descriptor failed\n");
+ sprd_dma_process_completed(sdev);
+ goto out;
+ }
+
+ mdesc->dma_flags = flags;
+
+ /*
+ * Set the DMA configuration into channel register of this DMA
+ * descriptor. If we have multiple configuration, which means
+ * need enable the link-list function, then we should set these
+ * configuration into link-list address.
+ */
+ if (cfg_count == 1)
+ ret = sprd_dma_config(chan, mdesc, &dma_cfg, NULL,
+ SINGLE_CONFIG);
+ else
+ ret = sprd_dma_config_linklist(chan, mdesc, mchan->dma_cfg,
+ cfg_count);
+
+ if (ret < 0) {
+ spin_lock_irqsave(&mchan->chn_lock, irq_flags);
+ list_add_tail(&mdesc->node, &mchan->free);
+ spin_unlock_irqrestore(&mchan->chn_lock, irq_flags);
+ dev_err(sdev->dma_dev.dev, "DMA configuration is failed\n");
+ goto out;
+ }
+
+ if (!(flags & DMA_HARDWARE_REQ))
+ mchan->dev_id = DMA_SOFTWARE_UID;
+
+ /* add the free descriptor to the prepared list. */
+ spin_lock_irqsave(&mchan->chn_lock, irq_flags);
+ list_add_tail(&mdesc->node, &mchan->prepared);
+ spin_unlock_irqrestore(&mchan->chn_lock, irq_flags);
+
+ mchan->cfg_count = 0;
+ memset(mchan->dma_cfg, 0, sizeof(struct sprd_dma_cfg) *
+ SPRD_DMA_CFG_COUNT);
+ return &mdesc->desc;
+
+out:
+ mchan->cfg_count = 0;
+ memset(mchan->dma_cfg, 0, sizeof(struct sprd_dma_cfg) *
+ SPRD_DMA_CFG_COUNT);
+ return NULL;
+}
+
+static int sprd_dma_pause(struct dma_chan *chan)
+{
+ struct sprd_dma_chn *mchan =
+ container_of(chan, struct sprd_dma_chn, chan);
+ unsigned long flags;
+
+ spin_lock_irqsave(&mchan->chn_lock, flags);
+ sprd_dma_stop(mchan);
+ spin_unlock_irqrestore(&mchan->chn_lock, flags);
+
+ return 0;
+}
+
+static int sprd_dma_resume(struct dma_chan *chan)
+{
+ struct sprd_dma_chn *mchan =
+ container_of(chan, struct sprd_dma_chn, chan);
+ unsigned long flags;
+ int ret;
+
+ spin_lock_irqsave(&mchan->chn_lock, flags);
+ ret = sprd_dma_start(mchan);
+ spin_unlock_irqrestore(&mchan->chn_lock, flags);
+
+ return ret;
+}
+
+static int sprd_terminate_all(struct dma_chan *chan)
+{
+ struct sprd_dma_chn *mchan = to_sprd_dma_chan(chan);
+ unsigned long flags;
+
+ spin_lock_irqsave(&mchan->chn_lock, flags);
+ list_splice_tail_init(&mchan->prepared, &mchan->free);
+ list_splice_tail_init(&mchan->queued, &mchan->free);
+ list_splice_tail_init(&mchan->active, &mchan->free);
+ list_splice_tail_init(&mchan->completed, &mchan->free);
+
+ sprd_dma_stop(mchan);
+ spin_unlock_irqrestore(&mchan->chn_lock, flags);
+
+ return 0;
+}
+
+static int sprd_dma_slave_config(struct dma_chan *chan,
+ struct dma_slave_config *config)
+{
+ struct sprd_dma_chn *mchan = to_sprd_dma_chan(chan);
+ struct sprd_dma_cfg *sprd_cfg = container_of(config,
+ struct sprd_dma_cfg,
+ config);
+ int i = 0;
+
+ do {
+ memcpy(&mchan->dma_cfg[i], sprd_cfg, sizeof(*sprd_cfg));
+ sprd_cfg++;
+ } while (mchan->dma_cfg[i++].is_end == DMA_NOT_END
+ && i < (SPRD_DMA_CFG_COUNT - 1));
+
+ mchan->cfg_count = i;
+ return 0;
+}
+
+static bool sprd_dma_filter_fn(struct dma_chan *chan, void *param)
+{
+ struct sprd_dma_chn *mchan = to_sprd_dma_chan(chan);
+ struct of_phandle_args *dma_spec = (struct of_phandle_args *)param;
+ unsigned int req = (unsigned int)dma_spec->args[0];
+
+ if (chan->device->dev->of_node == dma_spec->np)
+ return req == (mchan->chn_num + 1);
+ else
+ return false;
+}
+
+static struct dma_chan *sprd_dma_simple_xlate(struct of_phandle_args *dma_spec,
+ struct of_dma *of_dma)
+{
+ struct of_dma_filter_info *info = of_dma->of_dma_data;
+ int count = dma_spec->args_count;
+
+ if (!info || !info->filter_fn)
+ return NULL;
+
+ if (count != 1)
+ return NULL;
+
+ return dma_request_channel(info->dma_cap, info->filter_fn, dma_spec);
+}
+
+static int sprd_dma_probe(struct platform_device *pdev)
+{
+ struct device_node *np = pdev->dev.of_node;
+ struct sprd_dma_dev *sdev;
+ struct sprd_dma_chn *dma_chn;
+ struct resource *res;
+ u32 chn_count;
+ int ret, i;
+
+ if (of_property_read_u32(np, "#dma-channels", &chn_count)) {
+ dev_err(&pdev->dev, "get dma channels count failed\n");
+ return -ENODEV;
+ }
+
+ sdev = devm_kzalloc(&pdev->dev, (sizeof(*sdev) +
+ (sizeof(struct sprd_dma_chn) * chn_count)),
+ GFP_KERNEL);
+ if (!sdev)
+ return -ENOMEM;
+
+ sdev->clk = devm_clk_get(&pdev->dev, "enable");
+ if (IS_ERR(sdev->clk)) {
+ dev_err(&pdev->dev, "get enable clock failed\n");
+ return PTR_ERR(sdev->clk);
+ }
+
+ /* ashb clock is optional for AGCP DMA */
+ sdev->ashb_clk = devm_clk_get(&pdev->dev, "ashb_eb");
+ if (IS_ERR(sdev->ashb_clk))
+ dev_warn(&pdev->dev, "get ashb eb clock failed\n");
+
+ sdev->irq = platform_get_irq(pdev, 0);
+ if (sdev->irq > 0) {
+ ret = devm_request_irq(&pdev->dev, sdev->irq, dma_irq_handle,
+ 0, "sprd_dma", (void *)sdev);
+ if (ret < 0) {
+ dev_err(&pdev->dev, "request dma irq failed\n");
+ return ret;
+ }
+
+ tasklet_init(&sdev->tasklet, sprd_dma_tasklet,
+ (unsigned long)sdev);
+ } else {
+ dev_warn(&pdev->dev, "no interrupts for the DMA controller\n");
+ }
+
+ res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
+ sdev->glb_base = devm_ioremap_nocache(&pdev->dev, res->start,
+ resource_size(res));
+ if (!sdev->glb_base)
+ return -ENOMEM;
+
+ dma_cap_set(DMA_MEMCPY, sdev->dma_dev.cap_mask);
+ sdev->total_chns = chn_count;
+ sdev->dma_dev.chancnt = chn_count;
+ INIT_LIST_HEAD(&sdev->dma_dev.channels);
+ INIT_LIST_HEAD(&sdev->dma_dev.global_node);
+ sdev->dma_dev.dev = &pdev->dev;
+ sdev->dma_dev.device_alloc_chan_resources = sprd_dma_alloc_chan_resources;
+ sdev->dma_dev.device_free_chan_resources = sprd_dma_free_chan_resources;
+ sdev->dma_dev.device_tx_status = sprd_dma_tx_status;
+ sdev->dma_dev.device_issue_pending = sprd_dma_issue_pending;
+ sdev->dma_dev.device_prep_dma_memcpy = sprd_dma_prep_dma_memcpy;
+ sdev->dma_dev.device_config = sprd_dma_slave_config;
+ sdev->dma_dev.device_pause = sprd_dma_pause;
+ sdev->dma_dev.device_resume = sprd_dma_resume;
+ sdev->dma_dev.device_terminate_all = sprd_terminate_all;
+
+ for (i = 0; i < chn_count; i++) {
+ dma_chn = &sdev->channels[i];
+ dma_chn->chan.device = &sdev->dma_dev;
+ dma_cookie_init(&dma_chn->chan);
+ list_add_tail(&dma_chn->chan.device_node,
+ &sdev->dma_dev.channels);
+
+ dma_chn->chn_num = i;
+ /* get each channel's registers base address. */
+ dma_chn->chn_base = (void __iomem *)
+ ((unsigned long)sdev->glb_base +
+ SPRD_DMA_CHN_REG_OFFSET +
+ SPRD_DMA_CHN_REG_LENGTH * i);
+
+ spin_lock_init(&dma_chn->chn_lock);
+ INIT_LIST_HEAD(&dma_chn->free);
+ INIT_LIST_HEAD(&dma_chn->prepared);
+ INIT_LIST_HEAD(&dma_chn->queued);
+ INIT_LIST_HEAD(&dma_chn->active);
+ INIT_LIST_HEAD(&dma_chn->completed);
+ }
+
+ sdev->dma_desc_node_cachep = kmem_cache_create("dma_desc_node",
+ sizeof(struct sprd_dma_desc), 0,
+ SLAB_HWCACHE_ALIGN, NULL);
+ if (!sdev->dma_desc_node_cachep) {
+ dev_err(&pdev->dev, "allocate memory cache failed\n");
+ return -ENOMEM;
+ }
+
+ platform_set_drvdata(pdev, sdev);
+ ret = sprd_dma_enable(sdev);
+ if (ret)
+ goto err_enable;
+
+ pm_runtime_set_active(&pdev->dev);
+ pm_runtime_enable(&pdev->dev);
+
+ ret = pm_runtime_get_sync(&pdev->dev);
+ if (ret < 0)
+ goto err_rpm;
+
+ ret = dma_async_device_register(&sdev->dma_dev);
+ if (ret < 0) {
+ dev_err(&pdev->dev, "register dma device failed:%d\n", ret);
+ goto err_register;
+ }
+
+ sprd_dma_info.dma_cap = sdev->dma_dev.cap_mask;
+ ret = of_dma_controller_register(np, sprd_dma_simple_xlate,
+ &sprd_dma_info);
+ if (ret) {
+ dev_err(&pdev->dev, "failed to register of DMA controller\n");
+ goto err_of_register;
+ }
+
+ pm_runtime_put_sync(&pdev->dev);
+ return 0;
+
+err_of_register:
+ dma_async_device_unregister(&sdev->dma_dev);
+err_register:
+ pm_runtime_put_noidle(&pdev->dev);
+ pm_runtime_disable(&pdev->dev);
+err_rpm:
+ sprd_dma_disable(sdev);
+err_enable:
+ kmem_cache_destroy(sdev->dma_desc_node_cachep);
+ return ret;
+}
+
+static int sprd_dma_remove(struct platform_device *pdev)
+{
+ struct sprd_dma_dev *sdev = platform_get_drvdata(pdev);
+ int ret;
+
+ ret = pm_runtime_get_sync(&pdev->dev);
+ if (ret < 0)
+ return ret;
+
+ kmem_cache_destroy(sdev->dma_desc_node_cachep);
+ dma_async_device_unregister(&sdev->dma_dev);
+ sprd_dma_disable(sdev);
+
+ pm_runtime_put_noidle(&pdev->dev);
+ pm_runtime_disable(&pdev->dev);
+ return 0;
+}
+
+static const struct of_device_id sprd_dma_match[] = {
+ { .compatible = "sprd,sc9860-dma", },
+ {},
+};
+
+static int __maybe_unused sprd_dma_runtime_suspend(struct device *dev)
+{
+ struct sprd_dma_dev *sdev = dev_get_drvdata(dev);
+
+ sprd_dma_disable(sdev);
+ return 0;
+}
+
+static int __maybe_unused sprd_dma_runtime_resume(struct device *dev)
+{
+ struct sprd_dma_dev *sdev = dev_get_drvdata(dev);
+
+ return sprd_dma_enable(sdev);
+}
+
+static const struct dev_pm_ops sprd_dma_pm_ops = {
+ SET_RUNTIME_PM_OPS(sprd_dma_runtime_suspend,
+ sprd_dma_runtime_resume,
+ NULL)
+};
+
+static struct platform_driver sprd_dma_driver = {
+ .probe = sprd_dma_probe,
+ .remove = sprd_dma_remove,
+ .driver = {
+ .name = "sprd-dma",
+ .owner = THIS_MODULE,
+ .of_match_table = sprd_dma_match,
+ .pm = &sprd_dma_pm_ops,
+ },
+};
+
+static int __init sprd_dma_init(void)
+{
+ return platform_driver_register(&sprd_dma_driver);
+}
+arch_initcall_sync(sprd_dma_init);
+
+static void __exit sprd_dma_exit(void)
+{
+ platform_driver_unregister(&sprd_dma_driver);
+}
+module_exit(sprd_dma_exit);
+
+MODULE_LICENSE("GPL");
+MODULE_DESCRIPTION("DMA driver for Spreadtrum");
+MODULE_AUTHOR("Baolin Wang <baolin.wang@xxxxxxxxxxxxxx>");
+MODULE_AUTHOR("Eric Long <eric.long@xxxxxxxxxxxxxx>");
diff --git a/include/linux/dma/sprd-dma.h b/include/linux/dma/sprd-dma.h
new file mode 100644
index 0000000..40a8b1c
--- /dev/null
+++ b/include/linux/dma/sprd-dma.h
@@ -0,0 +1,270 @@
+/*
+ * Copyright (C) 2017 Spreadtrum Communications Inc.
+ *
+ * This software is licensed under the terms of the GNU General Public
+ * License version 2, as published by the Free Software Foundation, and
+ * may be copied, distributed, and modified under those terms.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ */
+
+#ifndef __SPRD_DMA_H
+#define __SPRD_DMA_H
+#include <linux/dmaengine.h>
+
+/* DMA request ID definition */
+#define DMA_SOFTWARE_UID 0
+#define DMA_SIM_RX 1
+#define DMA_SIM_TX 2
+#define DMA_IIS0_RX 3
+#define DMA_IIS0_TX 4
+#define DMA_IIS1_RX 5
+#define DMA_IIS1_TX 6
+#define DMA_IIS2_RX 7
+#define DMA_IIS2_TX 8
+#define DMA_IIS3_RX 9
+#define DMA_IIS3_TX 10
+#define DMA_SPI0_RX 11
+#define DMA_SPI0_TX 12
+#define DMA_SPI1_RX 13
+#define DMA_SPI1_TX 14
+#define DMA_SPI2_RX 15
+#define DMA_SPI2_TX 16
+#define DMA_UART0_RX 17
+#define DMA_UART0_TX 18
+#define DMA_UART1_RX 19
+#define DMA_UART1_TX 20
+#define DMA_UART2_RX 21
+#define DMA_UART2_TX 22
+#define DMA_UART3_RX 23
+#define DMA_UART3_TX 24
+#define DMA_UART4_RX 25
+#define DMA_UART4_TX 26
+#define DMA_DRM_CPT 27
+#define DMA_DRM_RAW 28
+#define DMA_VB_DA0 29
+#define DMA_VB_DA1 30
+#define DMA_VB_AD0 31
+#define DMA_VB_AD1 32
+#define DMA_VB_AD2 33
+#define DMA_VB_AD3 34
+#define DMA_GPS 35
+#define DMA_SDIO0_RD 36
+#define DMA_SDIO0_WR 37
+#define DMA_SDIO1_RD 38
+#define DMA_SDIO1_WR 39
+#define DMA_SDIO2_RD 40
+#define DMA_SDIO2_WR 41
+#define DMA_EMMC_RD 42
+#define DMA_EMMC_WR 43
+
+/*
+ * enum dma_datawidth: define the DMA transfer data width
+ * @BYTE_WIDTH: 1 byte width
+ * @SHORT_WIDTH: 2 bytes width
+ * @WORD_WIDTH: 4 bytes width
+ * @DWORD_WIDTH: 8 bytes width
+ */
+enum dma_datawidth {
+ BYTE_WIDTH,
+ SHORT_WIDTH,
+ WORD_WIDTH,
+ DWORD_WIDTH,
+};
+
+/*
+ * enum dma_request_mode: define the DMA request mode
+ * @FRAG_REQ_MODE: fragment request mode
+ * @BLOCK_REQ_MODE: block request mode
+ * @TRANS_REQ_MODE: transaction request mode
+ * @LIST_REQ_MODE: link-list request mode
+ *
+ * We have 4 types request mode: fragment mode, block mode, transaction mode
+ * and linklist mode. One transaction can contain several blocks, one block can
+ * contain several fragments. Link-list mode means we can save several DMA
+ * configuration into one reserved memory, then DMA can fetch each DMA
+ * configuration automatically to start transfer.
+ */
+enum dma_request_mode {
+ FRAG_REQ_MODE,
+ BLOCK_REQ_MODE,
+ TRANS_REQ_MODE,
+ LIST_REQ_MODE,
+};
+
+/*
+ * enum dma_int_type: define the DMA interrupt type
+ * @NO_INT: do not need generate DMA interrupt.
+ * @FRAG_DONE: fragment done interrupt when one fragment request is done.
+ * @BLK_DONE: block done interrupt when one block request is done.
+ * @TRANS_DONE: tansaction done interrupt when one transaction request is done.
+ * @LIST_DONE: link-list done interrupt when one link-list request is done.
+ * @CONFIG_ERR: configure error interrupt when configuration is incorrect
+ * @BLOCK_FRAG_DONE: block and fragment interrupt when one fragment or block
+ * request is done.
+ * @TRANS_FRAG_DONE: transaction and fragment interrupt when one transaction
+ * request or fragment request is done.
+ * @TRANS_BLOCK_DONE: transaction and block interrupt when one transaction
+ * request or block request is done.
+ */
+enum dma_int_type {
+ NO_INT,
+ FRAG_DONE,
+ BLK_DONE,
+ TRANS_DONE,
+ LIST_DONE,
+ CONFIG_ERR,
+ BLOCK_FRAG_DONE,
+ TRANS_FRAG_DONE,
+ TRANS_BLOCK_DONE,
+};
+
+/*
+ * enum dma_pri_level: define the DMA channel priority level
+ * @DMA_PRI_0: level 0
+ * @DMA_PRI_1: level 1
+ * @DMA_PRI_2: level 2
+ * @DMA_PRI_3: level 3
+ *
+ * When there are several DMA channels need to start, the DMA controller's
+ * arbitration will choose the high priority channel to start firstly.
+ */
+enum dma_pri_level {
+ DMA_PRI_0,
+ DMA_PRI_1,
+ DMA_PRI_2,
+ DMA_PRI_3,
+};
+
+/*
+ * enum dma_switch_mode: define the DMA transfer format
+ * @DATA_ABCD: ABCD to ABCD
+ * @DATA_DCBA: ABCD to DCBA
+ * @DATA_BADC: ABCD to BADC
+ * @DATA_CDAB: ABCD to CDAB
+ */
+enum dma_switch_mode {
+ DATA_ABCD,
+ DATA_DCBA,
+ DATA_BADC,
+ DATA_CDAB,
+};
+
+/*
+ * enum dma_end_type: define the DMA configuration end type
+ * @DMA_NOT_END: DMA configuration is not end
+ * @DMA_END: DMA configuration is end but not one link-list cycle configuration
+ * @DMA_LINK: DMA configuration is end but it is one link-list cycle
+ * configuration
+ *
+ * Since DMA contrller can support link-list transfer mode, that means user can
+ * supply several DMA configuration and each configuration can be pointed by
+ * previous link pointer register, then DMA controller will start to transfer
+ * for each DMA configuration automatically. DMA_END and DMA_LINK flag can
+ * indicate these several configuration is end, but DMA_LINK can also indicate
+ * these listed configuration is one cycle. For example if we have 4 group DMA
+ * configuration and we set DMA_LINK, which means it will start to transfer from
+ * cfg0--->cfg1--->cfg2--->cfg3, then back to cfg0 as one cycle.
+ */
+enum dma_end_type {
+ DMA_NOT_END,
+ DMA_END,
+ DMA_LINK,
+};
+
+/*
+ * enum dma_flags: define the DMA flags
+ * @DMA_HARDWARE_REQ: hardware request channel to start transfer by hardware id.
+ * @DMA_SOFTWARE_REQ: software request channel to start transfer.
+ * @DMA_GROUP1_SRC: indicate this is source channel of group 1 which can be
+ * start another channel.
+ * @DMA_GROUP1_DST: indicate this is destination channel of group 1 which will
+ * be started by source channel.
+ * @DMA_GROUP2_SRC: indicate this is source channel of group 2 which can be
+ * start another channel.
+ * @DMA_GROUP2_DST: indicate this is destination channel of group 2 which will
+ * be started by source channel.
+ * @DMA_MUTL_FRAG_DONE: when fragment is done of source channel which will
+ * start another destination channel.
+ * @DMA_MUTL_BLK_DONE: when block is done of source channel which will start
+ * another destination channel.
+ * @DMA_MUTL_TRANS_DONE: when transaction is done of source channel which will
+ * start another destination channel.
+ * @DMA_MUTL_LIST_DONE: when link-list is done of source channel which will
+ * start another destination channel.
+ *
+ * Since DMA controller can support 2 stage transfer which means when one
+ * channel transfer is done, then it can start another channel's transfer
+ * automatically by interrupt type.
+ */
+enum dma_flags {
+ DMA_HARDWARE_REQ = BIT(0),
+ DMA_SOFTWARE_REQ = BIT(1),
+ DMA_GROUP1_SRC = BIT(2),
+ DMA_GROUP1_DST = BIT(3),
+ DMA_GROUP2_SRC = BIT(4),
+ DMA_GROUP2_DST = BIT(5),
+ DMA_MUTL_FRAG_DONE = BIT(6),
+ DMA_MUTL_BLK_DONE = BIT(7),
+ DMA_MUTL_TRANS_DONE = BIT(8),
+ DMA_MUTL_LIST_DONE = BIT(9),
+};
+
+/*
+ * struct sprd_dma_cfg: DMA configuration for users
+ * @config: salve config structure
+ * @chn_pri: the channel priority
+ * @datawidth: the data width
+ * @req_mode: the request mode
+ * @irq_mode: the interrupt mode
+ * @swt_mode: the switch mode
+ * @link_cfg_v: point to the virtual memory address to save link-list DMA
+ * configuration
+ * @link_cfg_p: point to the physical memory address to save link-list DMA
+ * configuration
+ * @src_addr: the source address
+ * @des_addr: the destination address
+ * @fragmens_len: one fragment request length
+ * @block_len; one block request length
+ * @transcation_len: one transaction request length
+ * @src_step: source side transfer step
+ * @des_step: destination side transfer step
+ * @src_frag_step: source fragment transfer step
+ * @dst_frag_step: destination fragment transfer step
+ * @src_blk_step: source block transfer step
+ * @dst_blk_step: destination block transfer step
+ * @wrap_ptr: wrap jump pointer address
+ * @wrap_to: wrap jump to address
+ * @dev_id: hardware device id to start DMA transfer
+ * @is_end: DMA configuration end type
+ */
+struct sprd_dma_cfg {
+ struct dma_slave_config config;
+ enum dma_pri_level chn_pri;
+ enum dma_datawidth datawidth;
+ enum dma_request_mode req_mode;
+ enum dma_int_type irq_mode;
+ enum dma_switch_mode swt_mode;
+ unsigned long link_cfg_v;
+ unsigned long link_cfg_p;
+ unsigned long src_addr;
+ unsigned long des_addr;
+ u32 fragmens_len;
+ u32 block_len;
+ u32 transcation_len;
+ u32 src_step;
+ u32 des_step;
+ u32 src_frag_step;
+ u32 dst_frag_step;
+ u32 src_blk_step;
+ u32 dst_blk_step;
+ u32 wrap_ptr;
+ u32 wrap_to;
+ u32 dev_id;
+ enum dma_end_type is_end;
+};
+
+#endif
--
1.7.9.5