[PATCH v2 2/3] Added Xilinx XDMA IP core driver
From: tumic
Date: Mon Sep 19 2022 - 14:16:08 EST
From: Martin Tůma <martin.tuma@xxxxxxxxxxxxxxxxxxxxx>
Added support for the Xilinx XDMA PCIe DMA IP core. The XDMA IP core is used in
many FPGA PCIe card designs for DMA transfers between the PCIe card and the host
system. This driver can be incorporated into any PCIe card (that contains
the XDMA IP core) driver to initialize the XDMA HW and process DMA transfers.
The driver is originally based on the code provided by Xilinx at
https://github.com/Xilinx/dma_ip_drivers
Signed-off-by: Martin Tůma <martin.tuma@xxxxxxxxxxxxxxxxxxxxx>
---
drivers/dma/Kconfig | 7 +
drivers/dma/xilinx/Makefile | 1 +
drivers/dma/xilinx/xilinx_xdma.c | 2042 ++++++++++++++++++++++++++++++
include/linux/dma/xilinx_xdma.h | 44 +
4 files changed, 2094 insertions(+)
create mode 100644 drivers/dma/xilinx/xilinx_xdma.c
create mode 100644 include/linux/dma/xilinx_xdma.h
diff --git a/drivers/dma/Kconfig b/drivers/dma/Kconfig
index a06d2a7627aa..932086cd5962 100644
--- a/drivers/dma/Kconfig
+++ b/drivers/dma/Kconfig
@@ -804,4 +804,11 @@ config DMATEST
config DMA_ENGINE_RAID
bool
+config XILINX_XDMA
+ tristate "Xilinx XDMA Engine"
+ depends on PCI
+ select DMA_ENGINE
+ help
+ Enable support for Xilinx XDMA IP controller.
+
endif
diff --git a/drivers/dma/xilinx/Makefile b/drivers/dma/xilinx/Makefile
index 767bb45f641f..55e97686f8ea 100644
--- a/drivers/dma/xilinx/Makefile
+++ b/drivers/dma/xilinx/Makefile
@@ -2,3 +2,4 @@
obj-$(CONFIG_XILINX_DMA) += xilinx_dma.o
obj-$(CONFIG_XILINX_ZYNQMP_DMA) += zynqmp_dma.o
obj-$(CONFIG_XILINX_ZYNQMP_DPDMA) += xilinx_dpdma.o
+obj-$(CONFIG_XILINX_XDMA) += xilinx_xdma.o
diff --git a/drivers/dma/xilinx/xilinx_xdma.c b/drivers/dma/xilinx/xilinx_xdma.c
new file mode 100644
index 000000000000..9db637c25045
--- /dev/null
+++ b/drivers/dma/xilinx/xilinx_xdma.c
@@ -0,0 +1,2042 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * This file is part of the Xilinx DMA IP Core driver for Linux
+ *
+ * Copyright (c) 2016-2021, Xilinx, Inc.
+ * Copyright (c) 2022, Digiteq Automotive s.r.o.
+ */
+
+#include <linux/module.h>
+#include <linux/kernel.h>
+#include <linux/string.h>
+#include <linux/mm.h>
+#include <linux/errno.h>
+#include <linux/sched.h>
+#include <linux/vmalloc.h>
+#include <linux/pci.h>
+#include <linux/dma/xilinx_xdma.h>
+
+
+static unsigned int enable_credit_mp = 1;
+module_param(enable_credit_mp, uint, 0644);
+MODULE_PARM_DESC(enable_credit_mp,
+ "Set 0 to disable credit feature, default is 1 (enabled)");
+
+#define XDMA_BAR_SIZE 0x8000UL
+
+#define XDMA_CHANNEL_NUM_MAX 4
+#define XDMA_ENG_IRQ_NUM 1
+#define XDMA_MAX_ADJ_BLOCK_SIZE 0x40
+#define XDMA_PAGE_SIZE 0x1000
+#define RX_STATUS_EOP 1
+
+#define XDMA_OFS_INT_CTRL 0x2000UL
+#define XDMA_OFS_CONFIG 0x3000UL
+
+#define XDMA_TRANSFER_MAX_DESC 2048
+
+#define XDMA_DESC_BLEN_BITS 28
+#define XDMA_DESC_BLEN_MAX ((1 << (XDMA_DESC_BLEN_BITS)) - 1)
+
+/* bits of the SG DMA control register */
+#define XDMA_CTRL_RUN_STOP (1UL << 0)
+#define XDMA_CTRL_IE_DESC_STOPPED (1UL << 1)
+#define XDMA_CTRL_IE_DESC_COMPLETED (1UL << 2)
+#define XDMA_CTRL_IE_DESC_ALIGN_MISMATCH (1UL << 3)
+#define XDMA_CTRL_IE_MAGIC_STOPPED (1UL << 4)
+#define XDMA_CTRL_IE_IDLE_STOPPED (1UL << 6)
+#define XDMA_CTRL_IE_READ_ERROR (0x1FUL << 9)
+#define XDMA_CTRL_IE_DESC_ERROR (0x1FUL << 19)
+#define XDMA_CTRL_NON_INCR_ADDR (1UL << 25)
+#define XDMA_CTRL_POLL_MODE_WB (1UL << 26)
+#define XDMA_CTRL_STM_MODE_WB (1UL << 27)
+
+/* bits of the SG DMA status register */
+#define XDMA_STAT_BUSY (1UL << 0)
+#define XDMA_STAT_DESC_STOPPED (1UL << 1)
+#define XDMA_STAT_DESC_COMPLETED (1UL << 2)
+#define XDMA_STAT_ALIGN_MISMATCH (1UL << 3)
+#define XDMA_STAT_MAGIC_STOPPED (1UL << 4)
+#define XDMA_STAT_INVALID_LEN (1UL << 5)
+#define XDMA_STAT_IDLE_STOPPED (1UL << 6)
+
+#define XDMA_STAT_COMMON_ERR_MASK \
+ (XDMA_STAT_ALIGN_MISMATCH | XDMA_STAT_MAGIC_STOPPED | \
+ XDMA_STAT_INVALID_LEN)
+
+/* desc_error, C2H & H2C */
+#define XDMA_STAT_DESC_UNSUPP_REQ (1UL << 19)
+#define XDMA_STAT_DESC_COMPL_ABORT (1UL << 20)
+#define XDMA_STAT_DESC_PARITY_ERR (1UL << 21)
+#define XDMA_STAT_DESC_HEADER_EP (1UL << 22)
+#define XDMA_STAT_DESC_UNEXP_COMPL (1UL << 23)
+
+#define XDMA_STAT_DESC_ERR_MASK \
+ (XDMA_STAT_DESC_UNSUPP_REQ | XDMA_STAT_DESC_COMPL_ABORT | \
+ XDMA_STAT_DESC_PARITY_ERR | XDMA_STAT_DESC_HEADER_EP | \
+ XDMA_STAT_DESC_UNEXP_COMPL)
+
+/* read error: H2C */
+#define XDMA_STAT_H2C_R_UNSUPP_REQ (1UL << 9)
+#define XDMA_STAT_H2C_R_COMPL_ABORT (1UL << 10)
+#define XDMA_STAT_H2C_R_PARITY_ERR (1UL << 11)
+#define XDMA_STAT_H2C_R_HEADER_EP (1UL << 12)
+#define XDMA_STAT_H2C_R_UNEXP_COMPL (1UL << 13)
+
+#define XDMA_STAT_H2C_R_ERR_MASK \
+ (XDMA_STAT_H2C_R_UNSUPP_REQ | XDMA_STAT_H2C_R_COMPL_ABORT | \
+ XDMA_STAT_H2C_R_PARITY_ERR | XDMA_STAT_H2C_R_HEADER_EP | \
+ XDMA_STAT_H2C_R_UNEXP_COMPL)
+
+/* write error, H2C only */
+#define XDMA_STAT_H2C_W_DECODE_ERR (1UL << 14)
+#define XDMA_STAT_H2C_W_SLAVE_ERR (1UL << 15)
+
+#define XDMA_STAT_H2C_W_ERR_MASK \
+ (XDMA_STAT_H2C_W_DECODE_ERR | XDMA_STAT_H2C_W_SLAVE_ERR)
+
+/* read error: C2H */
+#define XDMA_STAT_C2H_R_DECODE_ERR (1UL << 9)
+#define XDMA_STAT_C2H_R_SLAVE_ERR (1UL << 10)
+
+#define XDMA_STAT_C2H_R_ERR_MASK \
+ (XDMA_STAT_C2H_R_DECODE_ERR | XDMA_STAT_C2H_R_SLAVE_ERR)
+
+/* all combined */
+#define XDMA_STAT_H2C_ERR_MASK \
+ (XDMA_STAT_COMMON_ERR_MASK | XDMA_STAT_DESC_ERR_MASK | \
+ XDMA_STAT_H2C_R_ERR_MASK | XDMA_STAT_H2C_W_ERR_MASK)
+
+#define XDMA_STAT_C2H_ERR_MASK \
+ (XDMA_STAT_COMMON_ERR_MASK | XDMA_STAT_DESC_ERR_MASK | \
+ XDMA_STAT_C2H_R_ERR_MASK)
+
+/* bits of the SGDMA descriptor control field */
+#define XDMA_DESC_STOPPED (1UL << 0)
+#define XDMA_DESC_COMPLETED (1UL << 1)
+#define XDMA_DESC_EOP (1UL << 4)
+
+/* upper 16-bits of engine identifier register */
+#define XDMA_ID_H2C 0x1fc0U
+#define XDMA_ID_C2H 0x1fc1U
+
+#define LS_BYTE_MASK 0x000000FFUL
+
+#define BLOCK_ID_MASK 0xFFF00000
+#define BLOCK_ID_HEAD 0x1FC00000
+
+#define IRQ_BLOCK_ID 0x1fc20000UL
+#define CONFIG_BLOCK_ID 0x1fc30000UL
+
+#define WB_COUNT_MASK 0x00ffffffUL
+#define WB_ERR_MASK (1UL << 31)
+
+#define MAX_USER_IRQ 16
+
+#define DESC_MAGIC 0xAD4B0000UL
+
+#define C2H_WB 0x52B4UL
+
+#define H2C_CHANNEL_OFFSET 0x1000
+#define SGDMA_OFFSET_FROM_CHANNEL 0x4000
+#define CHANNEL_SPACING 0x100
+#define TARGET_SPACING 0x1000
+
+/* obtain the 32 most significant (high) bits of a 32-bit or 64-bit address */
+#define PCI_DMA_H(addr) ((addr >> 16) >> 16)
+/* obtain the 32 least significant (low) bits of a 32-bit or 64-bit address */
+#define PCI_DMA_L(addr) (addr & 0xffffffffUL)
+
+
+enum transfer_state {
+ TRANSFER_STATE_NEW = 0,
+ TRANSFER_STATE_SUBMITTED,
+ TRANSFER_STATE_COMPLETED,
+ TRANSFER_STATE_FAILED,
+ TRANSFER_STATE_ABORTED
+};
+
+enum shutdown_state {
+ ENGINE_SHUTDOWN_NONE = 0, /* No shutdown in progress */
+ ENGINE_SHUTDOWN_REQUEST = 1, /* engine requested to shutdown */
+ ENGINE_SHUTDOWN_IDLE = 2 /* engine has shutdown and is idle */
+};
+
+struct config_regs {
+ u32 identifier;
+ u32 reserved_1[4];
+ u32 msi_enable;
+};
+
+struct engine_regs {
+ u32 identifier;
+ u32 control;
+ u32 control_w1s;
+ u32 control_w1c;
+ u32 reserved_1[12]; /* padding */
+
+ u32 status;
+ u32 status_rc;
+ u32 completed_desc_count;
+ u32 alignments;
+ u32 reserved_2[14]; /* padding */
+
+ u32 poll_mode_wb_lo;
+ u32 poll_mode_wb_hi;
+ u32 interrupt_enable_mask;
+ u32 interrupt_enable_mask_w1s;
+ u32 interrupt_enable_mask_w1c;
+ u32 reserved_3[9]; /* padding */
+
+ u32 perf_ctrl;
+ u32 perf_cyc_lo;
+ u32 perf_cyc_hi;
+ u32 perf_dat_lo;
+ u32 perf_dat_hi;
+ u32 perf_pnd_lo;
+ u32 perf_pnd_hi;
+} __packed;
+
+struct engine_sgdma_regs {
+ u32 identifier;
+ u32 reserved_1[31]; /* padding */
+
+ /* bus address to first descriptor in Root Complex Memory */
+ u32 first_desc_lo;
+ u32 first_desc_hi;
+ /* number of adjacent descriptors at first_desc */
+ u32 first_desc_adjacent;
+ u32 credits;
+} __packed;
+
+struct interrupt_regs {
+ u32 identifier;
+ u32 user_int_enable;
+ u32 user_int_enable_w1s;
+ u32 user_int_enable_w1c;
+ u32 channel_int_enable;
+ u32 channel_int_enable_w1s;
+ u32 channel_int_enable_w1c;
+ u32 reserved_1[9]; /* padding */
+
+ u32 user_int_request;
+ u32 channel_int_request;
+ u32 user_int_pending;
+ u32 channel_int_pending;
+ u32 reserved_2[12]; /* padding */
+
+ u32 user_msi_vector[8];
+ u32 channel_msi_vector[8];
+} __packed;
+
+struct sgdma_common_regs {
+ u32 padding[8];
+ u32 credit_mode_enable;
+ u32 credit_mode_enable_w1s;
+ u32 credit_mode_enable_w1c;
+} __packed;
+
+
+/*
+ * Descriptor for a single contiguous memory block transfer.
+ *
+ * Multiple descriptors are linked by means of the next pointer. An additional
+ * extra adjacent number gives the amount of extra contiguous descriptors.
+ *
+ * The descriptors are in root complex memory, and the bytes in the 32-bit
+ * words must be in little-endian byte ordering.
+ */
+struct xdma_desc {
+ u32 control;
+ u32 bytes; /* transfer length in bytes */
+ u32 src_addr_lo; /* source address (low 32-bit) */
+ u32 src_addr_hi; /* source address (high 32-bit) */
+ u32 dst_addr_lo; /* destination address (low 32-bit) */
+ u32 dst_addr_hi; /* destination address (high 32-bit) */
+ /*
+ * next descriptor in the single-linked list of descriptors;
+ * this is the PCIe (bus) address of the next descriptor in the
+ * root complex memory
+ */
+ u32 next_lo; /* next desc address (low 32-bit) */
+ u32 next_hi; /* next desc address (high 32-bit) */
+} __packed;
+
+/* 32 bytes (four 32-bit words) or 64 bytes (eight 32-bit words) */
+struct xdma_result {
+ u32 status;
+ u32 length;
+ u32 reserved_1[6]; /* padding */
+} __packed;
+
+struct sw_desc {
+ dma_addr_t addr;
+ unsigned int len;
+};
+
+/* Describes a (SG DMA) single transfer for the engine */
+#define XFER_FLAG_NEED_UNMAP 0x1
+#define XFER_FLAG_ST_C2H_EOP_RCVED 0x2 /* ST c2h only */
+struct xdma_transfer {
+ struct list_head entry; /* queue of non-completed transfers */
+ struct xdma_desc *desc_virt; /* virt addr of the 1st descriptor */
+ struct xdma_result *res_virt; /* virt addr of result, c2h streaming */
+ dma_addr_t res_bus; /* bus addr for result descriptors */
+ dma_addr_t desc_bus; /* bus addr of the first descriptor */
+ int desc_adjacent; /* adjacent descriptors at desc_bus */
+ int desc_num; /* number of descriptors in transfer */
+ int desc_index; /* index for 1st desc. in transfer */
+ int desc_cmpl; /* completed descriptors */
+ int desc_cmpl_th; /* completed descriptor threshold */
+ enum dma_data_direction dir;
+ struct swait_queue_head wq; /* wait queue for transfer completion */
+
+ enum transfer_state state; /* state of the transfer */
+ unsigned int flags;
+ int cyclic; /* flag if transfer is cyclic */
+ int last_in_request; /* flag if last within request */
+ unsigned int len;
+ struct sg_table *sgt;
+};
+
+struct xdma_request_cb {
+ struct sg_table *sgt;
+ unsigned int total_len;
+ u64 ep_addr;
+
+ struct xdma_transfer tfer;
+
+ unsigned int sw_desc_idx;
+ unsigned int sw_desc_cnt;
+ struct sw_desc sdesc[0];
+};
+
+struct xdma_engine {
+ struct xdma_dev *xdev; /* parent device */
+ char name[16]; /* name of this engine */
+
+ /* HW register address offsets */
+ struct engine_regs *regs; /* Control reg BAR offset */
+ struct engine_sgdma_regs *sgdma_regs; /* SGDAM reg BAR offset */
+
+ /* Engine state, configuration and flags */
+ enum shutdown_state shutdown; /* engine shutdown mode */
+ enum dma_data_direction dir;
+ u8 addr_align; /* source/dest alignment in bytes */
+ u8 len_granularity; /* transfer length multiple */
+ u8 addr_bits; /* HW datapath address width */
+ u8 channel:2; /* engine indices */
+ u8 streaming:1;
+ u8 device_open:1; /* flag if engine node open, ST mode only */
+ u8 running:1; /* flag if the driver started engine */
+ u8 non_incr_addr:1; /* flag if non-incremental addressing used */
+ u8 eop_flush:1; /* st c2h only, flush up the data with eop */
+ u8 filler:1;
+
+ int max_extra_adj; /* descriptor prefetch capability */
+ int desc_dequeued; /* num descriptors of completed transfers */
+ u32 status; /* last known status of device */
+ u32 interrupt_enable_mask_value; /* per-engine interrupt mask value */
+
+ /* Transfer list management */
+ struct list_head transfer_list; /* queue of transfers */
+
+ /* Members applicable to AXI-ST C2H (cyclic) transfers */
+ struct xdma_result *cyclic_result;
+ dma_addr_t cyclic_result_bus; /* bus addr for transfer */
+
+ /* Members associated with interrupt mode support */
+ struct swait_queue_head shutdown_wq;
+ spinlock_t lock; /* protects concurrent access */
+ int prev_cpu; /* remember CPU# of (last) locker */
+ int irq_line; /* IRQ vector for this engine */
+ u32 irq_bitmask; /* IRQ bit mask for this engine */
+ struct work_struct work; /* Work queue for interrupt handling */
+
+ struct mutex desc_lock; /* protects concurrent access */
+ dma_addr_t desc_bus;
+ struct xdma_desc *desc;
+ int desc_idx; /* current descriptor index */
+ int desc_used; /* total descriptors used */
+};
+
+struct xdma_dev {
+ struct pci_dev *pdev;
+ void __iomem *config_bar;
+ unsigned int mask_irq_user;
+ int engines_num;
+ struct xdma_engine engine_h2c[XDMA_CHANNEL_NUM_MAX];
+ struct xdma_engine engine_c2h[XDMA_CHANNEL_NUM_MAX];
+};
+
+
+static void channel_interrupts_enable(struct xdma_dev *xdev, u32 mask)
+{
+ struct interrupt_regs *reg =
+ (struct interrupt_regs *)(xdev->config_bar + XDMA_OFS_INT_CTRL);
+
+ iowrite32(mask, ®->channel_int_enable_w1s);
+}
+
+static void channel_interrupts_disable(struct xdma_dev *xdev, u32 mask)
+{
+ struct interrupt_regs *reg =
+ (struct interrupt_regs *)(xdev->config_bar + XDMA_OFS_INT_CTRL);
+
+ iowrite32(mask, ®->channel_int_enable_w1c);
+}
+
+static void user_interrupts_enable(struct xdma_dev *xdev, u32 mask)
+{
+ struct interrupt_regs *reg =
+ (struct interrupt_regs *)(xdev->config_bar + XDMA_OFS_INT_CTRL);
+
+ iowrite32(mask, ®->user_int_enable_w1s);
+}
+
+static void user_interrupts_disable(struct xdma_dev *xdev, u32 mask)
+{
+ struct interrupt_regs *reg =
+ (struct interrupt_regs *)(xdev->config_bar + XDMA_OFS_INT_CTRL);
+
+ iowrite32(mask, ®->user_int_enable_w1c);
+}
+
+static void read_interrupts(struct xdma_dev *xdev)
+{
+ struct interrupt_regs *reg =
+ (struct interrupt_regs *)(xdev->config_bar + XDMA_OFS_INT_CTRL);
+ u32 lo, hi;
+
+ hi = ioread32(®->user_int_request);
+ lo = ioread32(®->channel_int_request);
+}
+
+static void engine_reg_dump(struct xdma_engine *engine)
+{
+ u32 w;
+
+ w = ioread32(&engine->regs->identifier);
+ if ((w & BLOCK_ID_MASK) != BLOCK_ID_HEAD) {
+ pr_warn("XDMA: %s: 0x%08x: invalid engine id\n",
+ engine->name, w);
+ return;
+ }
+
+ pr_info("XDMA: %s: ENGINE REGISTER DUMP\n", engine->name);
+ pr_info("%s: ioread32(0x%p) = 0x%08x (id).\n",
+ engine->name, &engine->regs->identifier, w);
+ w = ioread32(&engine->regs->status);
+ pr_info("%s: ioread32(0x%p) = 0x%08x (status).\n",
+ engine->name, &engine->regs->status, w);
+ w = ioread32(&engine->regs->control);
+ pr_info("%s: ioread32(0x%p) = 0x%08x (control)\n",
+ engine->name, &engine->regs->control, w);
+ w = ioread32(&engine->sgdma_regs->first_desc_lo);
+ pr_info("%s: ioread32(0x%p) = 0x%08x (first_desc_lo)\n",
+ engine->name, &engine->sgdma_regs->first_desc_lo, w);
+ w = ioread32(&engine->sgdma_regs->first_desc_hi);
+ pr_info("%s: ioread32(0x%p) = 0x%08x (first_desc_hi)\n",
+ engine->name, &engine->sgdma_regs->first_desc_hi, w);
+ w = ioread32(&engine->sgdma_regs->first_desc_adjacent);
+ pr_info("%s: ioread32(0x%p) = 0x%08x (first_desc_adjacent).\n",
+ engine->name, &engine->sgdma_regs->first_desc_adjacent, w);
+ w = ioread32(&engine->regs->completed_desc_count);
+ pr_info("%s: ioread32(0x%p) = 0x%08x (completed_desc_count).\n",
+ engine->name, &engine->regs->completed_desc_count, w);
+ w = ioread32(&engine->regs->interrupt_enable_mask);
+ pr_info("%s: ioread32(0x%p) = 0x%08x (interrupt_enable_mask)\n",
+ engine->name, &engine->regs->interrupt_enable_mask, w);
+}
+
+static void engine_status_dump(struct xdma_engine *engine)
+{
+ u32 v = engine->status;
+ char buffer[256];
+ char *buf = buffer;
+ int len = 0;
+
+ len = sprintf(buf, "XDMA: %s: status: 0x%08x: ", engine->name, v);
+
+ if ((v & XDMA_STAT_BUSY))
+ len += sprintf(buf + len, "BUSY,");
+ if ((v & XDMA_STAT_DESC_STOPPED))
+ len += sprintf(buf + len, "DESC_STOPPED,");
+ if ((v & XDMA_STAT_DESC_COMPLETED))
+ len += sprintf(buf + len, "DESC_COMPL,");
+
+ /* common H2C & C2H */
+ if ((v & XDMA_STAT_COMMON_ERR_MASK)) {
+ if ((v & XDMA_STAT_ALIGN_MISMATCH))
+ len += sprintf(buf + len, "ALIGN_MISMATCH ");
+ if ((v & XDMA_STAT_MAGIC_STOPPED))
+ len += sprintf(buf + len, "MAGIC_STOPPED ");
+ if ((v & XDMA_STAT_INVALID_LEN))
+ len += sprintf(buf + len, "INVLIAD_LEN ");
+ if ((v & XDMA_STAT_IDLE_STOPPED))
+ len += sprintf(buf + len, "IDLE_STOPPED ");
+ buf[len - 1] = ',';
+ }
+
+ if (engine->dir == DMA_TO_DEVICE) {
+ /* H2C only */
+ if ((v & XDMA_STAT_H2C_R_ERR_MASK)) {
+ len += sprintf(buf + len, "R:");
+ if ((v & XDMA_STAT_H2C_R_UNSUPP_REQ))
+ len += sprintf(buf + len, "UNSUPP_REQ ");
+ if ((v & XDMA_STAT_H2C_R_COMPL_ABORT))
+ len += sprintf(buf + len, "COMPL_ABORT ");
+ if ((v & XDMA_STAT_H2C_R_PARITY_ERR))
+ len += sprintf(buf + len, "PARITY ");
+ if ((v & XDMA_STAT_H2C_R_HEADER_EP))
+ len += sprintf(buf + len, "HEADER_EP ");
+ if ((v & XDMA_STAT_H2C_R_UNEXP_COMPL))
+ len += sprintf(buf + len, "UNEXP_COMPL ");
+ buf[len - 1] = ',';
+ }
+
+ if ((v & XDMA_STAT_H2C_W_ERR_MASK)) {
+ len += sprintf(buf + len, "W:");
+ if ((v & XDMA_STAT_H2C_W_DECODE_ERR))
+ len += sprintf(buf + len, "DECODE_ERR ");
+ if ((v & XDMA_STAT_H2C_W_SLAVE_ERR))
+ len += sprintf(buf + len, "SLAVE_ERR ");
+ buf[len - 1] = ',';
+ }
+
+ } else {
+ /* C2H only */
+ if ((v & XDMA_STAT_C2H_R_ERR_MASK)) {
+ len += sprintf(buf + len, "R:");
+ if ((v & XDMA_STAT_C2H_R_DECODE_ERR))
+ len += sprintf(buf + len, "DECODE_ERR ");
+ if ((v & XDMA_STAT_C2H_R_SLAVE_ERR))
+ len += sprintf(buf + len, "SLAVE_ERR ");
+ buf[len - 1] = ',';
+ }
+ }
+
+ /* common H2C & C2H */
+ if ((v & XDMA_STAT_DESC_ERR_MASK)) {
+ len += sprintf(buf + len, "DESC_ERR:");
+ if ((v & XDMA_STAT_DESC_UNSUPP_REQ))
+ len += sprintf(buf + len, "UNSUPP_REQ ");
+ if ((v & XDMA_STAT_DESC_COMPL_ABORT))
+ len += sprintf(buf + len, "COMPL_ABORT ");
+ if ((v & XDMA_STAT_DESC_PARITY_ERR))
+ len += sprintf(buf + len, "PARITY ");
+ if ((v & XDMA_STAT_DESC_HEADER_EP))
+ len += sprintf(buf + len, "HEADER_EP ");
+ if ((v & XDMA_STAT_DESC_UNEXP_COMPL))
+ len += sprintf(buf + len, "UNEXP_COMPL ");
+ buf[len - 1] = ',';
+ }
+
+ buf[len - 1] = '\0';
+ pr_info("%s\n", buffer);
+}
+
+static void engine_status_read(struct xdma_engine *engine, bool clear, bool dump)
+{
+ if (dump)
+ engine_reg_dump(engine);
+
+ if (clear)
+ engine->status = ioread32(&engine->regs->status_rc);
+ else
+ engine->status = ioread32(&engine->regs->status);
+
+ if (dump)
+ engine_status_dump(engine);
+}
+
+static void engine_stop(struct xdma_engine *engine)
+{
+ u32 w;
+
+ if (enable_credit_mp && engine->streaming &&
+ engine->dir == DMA_FROM_DEVICE)
+ iowrite32(0, &engine->sgdma_regs->credits);
+
+ w = 0;
+ w |= (u32)XDMA_CTRL_IE_DESC_ALIGN_MISMATCH;
+ w |= (u32)XDMA_CTRL_IE_MAGIC_STOPPED;
+ w |= (u32)XDMA_CTRL_IE_READ_ERROR;
+ w |= (u32)XDMA_CTRL_IE_DESC_ERROR;
+
+ w |= (u32)XDMA_CTRL_IE_DESC_STOPPED;
+ w |= (u32)XDMA_CTRL_IE_DESC_COMPLETED;
+
+ iowrite32(w, &engine->regs->control);
+
+ engine->running = 0;
+}
+
+static int engine_start_mode_config(struct xdma_engine *engine)
+{
+ u32 w;
+
+ /* write control register of SG DMA engine */
+ w = (u32)XDMA_CTRL_RUN_STOP;
+ w |= (u32)XDMA_CTRL_IE_READ_ERROR;
+ w |= (u32)XDMA_CTRL_IE_DESC_ERROR;
+ w |= (u32)XDMA_CTRL_IE_DESC_ALIGN_MISMATCH;
+ w |= (u32)XDMA_CTRL_IE_MAGIC_STOPPED;
+
+ w |= (u32)XDMA_CTRL_IE_DESC_STOPPED;
+ w |= (u32)XDMA_CTRL_IE_DESC_COMPLETED;
+
+ /* set non-incremental addressing mode */
+ if (engine->non_incr_addr)
+ w |= (u32)XDMA_CTRL_NON_INCR_ADDR;
+
+ /* start the engine */
+ iowrite32(w, &engine->regs->control);
+ /* dummy read of status register to flush all previous writes */
+ w = ioread32(&engine->regs->status);
+
+ return 0;
+}
+
+/*
+ * Get the number for adjacent descriptors to set in a descriptor, based on the
+ * remaining number of descriptors and the lower bits of the address of the
+ * next descriptor.
+ * Since the number of descriptors in a page (XDMA_PAGE_SIZE) is 128 and the
+ * maximum size of a block of adjacent descriptors is 64 (63 max adjacent
+ * descriptors for any descriptor), align the blocks of adjacent descriptors
+ * to the block size.
+ */
+static u32 xdma_get_next_adj(unsigned int remaining, u32 next_lo)
+{
+ unsigned int next_index;
+
+ if (remaining <= 1)
+ return 0;
+
+ /* shift right 5 times corresponds to a division by
+ * sizeof(xdma_desc) = 32
+ */
+ next_index = ((next_lo & (XDMA_PAGE_SIZE - 1)) >> 5) %
+ XDMA_MAX_ADJ_BLOCK_SIZE;
+ return min(XDMA_MAX_ADJ_BLOCK_SIZE - next_index - 1, remaining - 1);
+}
+
+/*
+ * start an idle engine with its first transfer on queue
+ *
+ * The engine will run and process all transfers that are queued using
+ * transfer_queue() and thus have their descriptor lists chained.
+ *
+ * During the run, new transfers will be processed if transfer_queue() has
+ * chained the descriptors before the hardware fetches the last descriptor.
+ * A transfer that was chained too late will invoke a new run of the engine
+ * initiated from the engine_service() routine.
+ *
+ * The engine must be idle and at least one transfer must be queued.
+ */
+static int engine_start(struct xdma_engine *engine)
+{
+ struct xdma_transfer *transfer;
+ u32 w, next_adj;
+ int rv;
+
+ /* engine transfer queue must not be empty */
+ if (list_empty(&engine->transfer_list)) {
+ pr_warn("XDMA: %s: transfer queue must not be empty\n",
+ engine->name);
+ return -EIO;
+ }
+ /* inspect first transfer queued on the engine */
+ transfer = list_entry(engine->transfer_list.next, struct xdma_transfer,
+ entry);
+ if (!transfer) {
+ pr_warn("XDMA: %s: queued transfer must not be empty\n",
+ engine->name);
+ return -EIO;
+ }
+
+ /* engine is no longer shutdown */
+ engine->shutdown = ENGINE_SHUTDOWN_NONE;
+
+ /* Add credits for Streaming mode C2H */
+ if (enable_credit_mp && engine->streaming &&
+ engine->dir == DMA_FROM_DEVICE)
+ iowrite32(engine->desc_used, &engine->sgdma_regs->credits);
+
+ /* initialize number of descriptors of dequeued transfers */
+ engine->desc_dequeued = 0;
+
+ /* write lower 32-bit of bus address of transfer first descriptor */
+ w = cpu_to_le32(PCI_DMA_L(transfer->desc_bus));
+ iowrite32(w, &engine->sgdma_regs->first_desc_lo);
+ /* write upper 32-bit of bus address of transfer first descriptor */
+ w = cpu_to_le32(PCI_DMA_H(transfer->desc_bus));
+ iowrite32(w, &engine->sgdma_regs->first_desc_hi);
+
+ next_adj = xdma_get_next_adj(transfer->desc_adjacent,
+ cpu_to_le32(PCI_DMA_L(transfer->desc_bus)));
+ iowrite32(next_adj, &engine->sgdma_regs->first_desc_adjacent);
+
+ rv = engine_start_mode_config(engine);
+ if (rv < 0)
+ return rv;
+ engine_status_read(engine, 0, 0);
+
+ engine->running = 1;
+
+ return 0;
+}
+
+static void engine_service_shutdown(struct xdma_engine *engine)
+{
+ engine_stop(engine);
+ /* awake task on engine's shutdown wait queue */
+ swake_up_one(&engine->shutdown_wq);
+}
+
+static struct xdma_transfer *engine_transfer_completion(
+ struct xdma_engine *engine,
+ struct xdma_transfer *transfer)
+{
+ if (unlikely(!transfer)) {
+ pr_warn("XDMA: %s empty xfer\n", engine->name);
+ return NULL;
+ }
+
+ /* synchronous I/O? */
+ /* awake task on transfer's wait queue */
+ swake_up_one(&transfer->wq);
+
+ return transfer;
+}
+
+static struct xdma_transfer *engine_service_transfer_list(
+ struct xdma_engine *engine,
+ struct xdma_transfer *transfer,
+ u32 *pdesc_completed)
+{
+ if (unlikely(!transfer)) {
+ pr_warn("XDMA: %s empty xfer\n", engine->name);
+ return NULL;
+ }
+
+ /*
+ * iterate over all the transfers completed by the engine,
+ * except for the last
+ */
+ while (transfer && (!transfer->cyclic) &&
+ (*pdesc_completed > transfer->desc_num)) {
+ /* remove this transfer from pdesc_completed */
+ *pdesc_completed -= transfer->desc_num;
+
+ /* remove completed transfer from list */
+ list_del(engine->transfer_list.next);
+ /* add to dequeued number of descriptors during this run */
+ engine->desc_dequeued += transfer->desc_num;
+ /* mark transfer as successfully completed */
+ transfer->state = TRANSFER_STATE_COMPLETED;
+
+ /*
+ * Complete transfer - sets transfer to NULL if an async
+ * transfer has completed
+ */
+ transfer = engine_transfer_completion(engine, transfer);
+
+ /* if exists, get the next transfer on the list */
+ if (!list_empty(&engine->transfer_list)) {
+ transfer = list_entry(engine->transfer_list.next,
+ struct xdma_transfer, entry);
+ } else {
+ /* no further transfers? */
+ transfer = NULL;
+ }
+ }
+
+ return transfer;
+}
+
+static void engine_err_handle(struct xdma_engine *engine,
+ struct xdma_transfer *transfer)
+{
+ u32 value;
+
+ /*
+ * The BUSY bit is expected to be clear now but older HW has a race
+ * condition which could cause it to be still set. If it's set, re-read
+ * and check again. If it's still set, log the issue.
+ */
+ if (engine->status & XDMA_STAT_BUSY) {
+ value = ioread32(&engine->regs->status);
+ if ((value & XDMA_STAT_BUSY))
+ pr_warn("XDMA: %s has errors but is still BUSY\n",
+ engine->name);
+ }
+
+ /* mark transfer as failed */
+ transfer->state = TRANSFER_STATE_FAILED;
+ engine_stop(engine);
+}
+
+static struct xdma_transfer *
+engine_service_final_transfer(struct xdma_engine *engine,
+ struct xdma_transfer *transfer,
+ u32 *pdesc_completed)
+{
+ /* inspect the current transfer */
+ if (unlikely(!transfer)) {
+ pr_warn("XDMA: %s: empty xfer\n", engine->name);
+ return NULL;
+ }
+
+ if (((engine->dir == DMA_FROM_DEVICE) &&
+ (engine->status & XDMA_STAT_C2H_ERR_MASK)) ||
+ ((engine->dir == DMA_TO_DEVICE) &&
+ (engine->status & XDMA_STAT_H2C_ERR_MASK))) {
+ pr_warn("XDMA: %s: status error 0x%x.\n", engine->name,
+ engine->status);
+ engine_status_dump(engine);
+ engine_err_handle(engine, transfer);
+ goto transfer_del;
+ }
+
+ if (engine->status & XDMA_STAT_BUSY)
+ pr_info("XDMA: %s: engine unexpectedly busy, ignoring\n",
+ engine->name);
+
+ /* the engine stopped on current transfer? */
+ if (*pdesc_completed < transfer->desc_num) {
+ if (engine->eop_flush) {
+ /* check if eop received */
+ struct xdma_result *result = transfer->res_virt;
+ int i;
+ int max = *pdesc_completed;
+
+ for (i = 0; i < max; i++) {
+ if ((result[i].status & RX_STATUS_EOP) != 0) {
+ transfer->flags |=
+ XFER_FLAG_ST_C2H_EOP_RCVED;
+ break;
+ }
+ }
+
+ transfer->desc_cmpl += *pdesc_completed;
+ if (!(transfer->flags & XFER_FLAG_ST_C2H_EOP_RCVED))
+ return NULL;
+
+ /* mark transfer as successfully completed */
+ engine_service_shutdown(engine);
+ transfer->state = TRANSFER_STATE_COMPLETED;
+ engine->desc_dequeued += transfer->desc_cmpl;
+ } else {
+ transfer->state = TRANSFER_STATE_FAILED;
+ pr_warn("XDMA: %s: xfer stopped half-way\n",
+ engine->name);
+
+ /* add dequeued number of descriptors during this run */
+ engine->desc_dequeued += transfer->desc_num;
+ transfer->desc_cmpl = *pdesc_completed;
+ }
+ } else {
+ if (!transfer->cyclic) {
+ /*
+ * if the engine stopped on this transfer,
+ * it should be the last
+ */
+ WARN_ON(*pdesc_completed > transfer->desc_num);
+ }
+ /* mark transfer as successfully completed */
+ transfer->state = TRANSFER_STATE_COMPLETED;
+ transfer->desc_cmpl = transfer->desc_num;
+ /* add dequeued number of descriptors during this run */
+ engine->desc_dequeued += transfer->desc_num;
+ }
+
+transfer_del:
+ /* remove completed transfer from list */
+ list_del(engine->transfer_list.next);
+
+ /*
+ * Complete transfer - sets transfer to NULL if an asynchronous
+ * transfer has completed
+ */
+ transfer = engine_transfer_completion(engine, transfer);
+
+ return transfer;
+}
+
+static int engine_service_resume(struct xdma_engine *engine)
+{
+ int rv;
+
+ if (!engine->running) {
+ /* in the case of shutdown, let it finish what's in the Q */
+ if (!list_empty(&engine->transfer_list)) {
+ /* (re)start engine */
+ rv = engine_start(engine);
+ if (rv)
+ return rv;
+ /* engine was requested to be shutdown? */
+ } else if (engine->shutdown & ENGINE_SHUTDOWN_REQUEST) {
+ engine->shutdown |= ENGINE_SHUTDOWN_IDLE;
+ /* awake task on engine's shutdown wait queue */
+ swake_up_one(&engine->shutdown_wq);
+ }
+ } else if (list_empty(&engine->transfer_list)) {
+ engine_service_shutdown(engine);
+ }
+
+ return 0;
+}
+
+static int engine_service(struct xdma_engine *engine, int desc_writeback)
+{
+ struct xdma_transfer *transfer = NULL;
+ u32 desc_count = desc_writeback & WB_COUNT_MASK;
+ u32 err_flag = desc_writeback & WB_ERR_MASK;
+ int rv;
+
+ if (!engine->running) {
+ engine_status_read(engine, 1, 0);
+ return 0;
+ }
+
+ /*
+ * If called by the ISR detected an error, read and clear
+ * engine status.
+ */
+ if ((desc_count == 0) || (err_flag != 0))
+ engine_status_read(engine, 1, 0);
+
+ /*
+ * engine was running but is no longer busy, or writeback occurred,
+ * shut down
+ */
+ if ((engine->running && !(engine->status & XDMA_STAT_BUSY)) ||
+ (!engine->eop_flush && desc_count != 0))
+ engine_service_shutdown(engine);
+
+ /*
+ * If called from the ISR, or if an error occurred, the descriptor
+ * count will be zero. In this scenario, read the descriptor count
+ * from HW.
+ */
+ if (!desc_count)
+ desc_count = ioread32(&engine->regs->completed_desc_count);
+ if (!desc_count)
+ goto done;
+
+ /* transfers on queue? */
+ if (!list_empty(&engine->transfer_list)) {
+ /* pick first transfer on queue (was submitted to the engine) */
+ transfer = list_entry(engine->transfer_list.next,
+ struct xdma_transfer, entry);
+ }
+
+ /* account for already dequeued transfers during this engine run */
+ desc_count -= engine->desc_dequeued;
+
+ /* Process all but the last transfer */
+ transfer = engine_service_transfer_list(engine, transfer, &desc_count);
+
+ /*
+ * Process final transfer - includes checks of number of descriptors to
+ * detect faulty completion
+ */
+ transfer = engine_service_final_transfer(engine, transfer, &desc_count);
+
+ /* Restart the engine following the servicing */
+ if (!engine->eop_flush) {
+ rv = engine_service_resume(engine);
+ if (rv)
+ return rv;
+ }
+
+done:
+ return err_flag ? -1 : 0;
+}
+
+static void engine_service_work(struct work_struct *work)
+{
+ struct xdma_engine *engine;
+ unsigned long flags;
+ int rv;
+
+ engine = container_of(work, struct xdma_engine, work);
+
+ spin_lock_irqsave(&engine->lock, flags);
+
+ rv = engine_service(engine, 0);
+ if (rv < 0)
+ goto unlock;
+
+ /* re-enable interrupts for this engine */
+ iowrite32(engine->interrupt_enable_mask_value,
+ &engine->regs->interrupt_enable_mask_w1s);
+
+unlock:
+ spin_unlock_irqrestore(&engine->lock, flags);
+}
+
+static irqreturn_t xdma_isr(int irq, void *dev_id)
+{
+ struct xdma_dev *xdev;
+ struct xdma_engine *engine;
+ struct interrupt_regs *irq_regs;
+
+ engine = (struct xdma_engine *)dev_id;
+ xdev = engine->xdev;
+
+ irq_regs = (struct interrupt_regs *)(xdev->config_bar + XDMA_OFS_INT_CTRL);
+
+ /* Disable the interrupt for this engine */
+ iowrite32(engine->interrupt_enable_mask_value,
+ &engine->regs->interrupt_enable_mask_w1c);
+ /* Dummy read to flush the above write */
+ ioread32(&irq_regs->channel_int_pending);
+ schedule_work(&engine->work);
+
+ return IRQ_HANDLED;
+}
+
+static int is_config_bar(void *bar)
+{
+ u32 irq_id = 0;
+ u32 cfg_id = 0;
+ u32 mask = 0xffff0000; /* Compare only XDMA ID's not Version number */
+ struct interrupt_regs *irq_regs =
+ (struct interrupt_regs *)(bar + XDMA_OFS_INT_CTRL);
+ struct config_regs *cfg_regs =
+ (struct config_regs *)(bar + XDMA_OFS_CONFIG);
+
+ irq_id = ioread32(&irq_regs->identifier);
+ cfg_id = ioread32(&cfg_regs->identifier);
+
+ if (((irq_id & mask) == IRQ_BLOCK_ID)
+ && ((cfg_id & mask) == CONFIG_BLOCK_ID))
+ return 1;
+
+ return 0;
+}
+
+static void unmap_config_bar(struct xdma_dev *xdev, int config_bar_id)
+{
+ pci_iounmap(xdev->pdev, xdev->config_bar);
+ pci_release_selected_regions(xdev->pdev, 1U<<config_bar_id);
+}
+
+static int map_config_bar(struct xdma_dev *xdev, int config_bar_id)
+{
+ int rv, bar_len;
+
+ bar_len = pci_resource_len(xdev->pdev, config_bar_id);
+ if (bar_len < XDMA_BAR_SIZE) {
+ pr_err("XDMA: %d: Not a config BAR\n", config_bar_id);
+ return -EINVAL;
+ }
+ rv = pci_request_selected_regions(xdev->pdev, 1U<<config_bar_id,
+ "xdma");
+ if (rv) {
+ pr_err("XDMA: Failed to request config BAR memory\n");
+ return rv;
+ }
+ xdev->config_bar = pci_iomap(xdev->pdev, config_bar_id, bar_len);
+ if (!xdev->config_bar) {
+ pr_err("XDMA: Failed to map config BAR memory\n");
+ rv = -ENOMEM;
+ goto err_map;
+ }
+ if (!is_config_bar(xdev->config_bar)) {
+ pr_err("XDMA: %d: Not a config BAR\n", config_bar_id);
+ rv = -EINVAL;
+ goto err_bar;
+ }
+
+ pr_debug("XDMA: Config BAR %d mapped at %p\n",
+ config_bar_id, xdev->config_bar);
+
+ return 0;
+
+err_bar:
+ pci_iounmap(xdev->pdev, xdev->config_bar);
+err_map:
+ pci_release_selected_regions(xdev->pdev, 1U<<config_bar_id);
+
+ return rv;
+}
+
+static void prog_irq_user(struct xdma_dev *xdev, int num_channel, int num_irq,
+ bool clear)
+{
+ struct interrupt_regs *int_regs =
+ (struct interrupt_regs *)(xdev->config_bar + XDMA_OFS_INT_CTRL);
+ int i = num_channel;
+ int max = i + num_irq;
+ int j;
+
+ for (j = 0; i < max; j++) {
+ u32 val = 0;
+ int k, shift = 0;
+
+ if (clear)
+ i += 4;
+ else
+ for (k = 0; k < 4 && i < max; i++, k++, shift += 8)
+ val |= (i & 0x1f) << shift;
+
+ iowrite32(val, &int_regs->user_msi_vector[j]);
+ }
+}
+
+static void prog_irq_channel(struct xdma_dev *xdev, int num_channel, bool clear)
+{
+ struct interrupt_regs *int_regs =
+ (struct interrupt_regs *)(xdev->config_bar + XDMA_OFS_INT_CTRL);
+ int i, j;
+
+ for (i = 0, j = 0; i < num_channel; j++) {
+ u32 val = 0;
+ int k, shift = 0;
+
+ if (clear)
+ i += 4;
+ else
+ for (k = 0; k < 4 && i < num_channel; i++, k++, shift += 8)
+ val |= (i & 0x1f) << shift;
+
+ iowrite32(val, &int_regs->channel_msi_vector[j]);
+ }
+}
+
+static void irq_channel_teardown(struct xdma_dev *xdev, int h2c_channel_max,
+ int c2h_channel_max)
+{
+ struct xdma_engine *engine;
+ int i = 0, j = 0;
+
+ engine = xdev->engine_h2c;
+ for (i = 0; i < h2c_channel_max; i++, j++, engine++) {
+ if (!engine->irq_line)
+ break;
+ free_irq(engine->irq_line, engine);
+ }
+
+ engine = xdev->engine_c2h;
+ for (i = 0; i < c2h_channel_max; i++, j++, engine++) {
+ if (!engine->irq_line)
+ break;
+ free_irq(engine->irq_line, engine);
+ }
+}
+
+static int irq_channel_setup(struct xdma_dev *xdev, int h2c_channel_max,
+ int c2h_channel_max)
+{
+ int i, j, rv;
+ u32 vector;
+ struct xdma_engine *engine;
+
+ j = h2c_channel_max;
+ engine = xdev->engine_h2c;
+ for (i = 0; i < h2c_channel_max; i++, engine++) {
+ vector = pci_irq_vector(xdev->pdev, i);
+ rv = request_irq(vector, xdma_isr, 0, engine->name, engine);
+ if (rv) {
+ pr_err("XDMA: %s: error requesting irq#%d\n",
+ engine->name, vector);
+ return rv;
+ }
+ pr_info("XDMA: %s: irq#%d\n", engine->name, vector);
+ engine->irq_line = vector;
+ }
+
+ engine = xdev->engine_c2h;
+ for (i = 0; i < c2h_channel_max; i++, j++, engine++) {
+ vector = pci_irq_vector(xdev->pdev, j);
+ rv = request_irq(vector, xdma_isr, 0, engine->name, engine);
+ if (rv) {
+ pr_err("XDMA: %s: error requesting irq#%d\n",
+ engine->name, vector);
+ return rv;
+ }
+ pr_info("XDMA: %s: irq#%d\n", engine->name, vector);
+ engine->irq_line = vector;
+ }
+
+ return 0;
+}
+
+static void irq_teardown(struct xdma_dev *xdev, int h2c_channel_max,
+ int c2h_channel_max, int user_irq_max)
+{
+ int num_channel = h2c_channel_max + c2h_channel_max;
+
+ prog_irq_user(xdev, num_channel, user_irq_max, 1);
+ prog_irq_channel(xdev, num_channel, 1);
+
+ irq_channel_teardown(xdev, h2c_channel_max, c2h_channel_max);
+}
+
+static int irq_setup(struct xdma_dev *xdev, int h2c_channel_max,
+ int c2h_channel_max, int user_irq_max)
+{
+ int rv;
+ int num_channel = h2c_channel_max + c2h_channel_max;
+
+ rv = irq_channel_setup(xdev, h2c_channel_max, c2h_channel_max);
+ if (rv)
+ return rv;
+
+ prog_irq_channel(xdev, num_channel, 0);
+ prog_irq_user(xdev, num_channel, user_irq_max, 0);
+
+ return 0;
+}
+
+/* Chains the descriptors as a singly-linked list
+ *
+ * Each descriptor's next pointer specifies the bus address of the next
+ * descriptor.
+ * Terminates the last descriptor to form a singly-linked list.
+ */
+static void transfer_desc_init(struct xdma_transfer *transfer, int count)
+{
+ struct xdma_desc *desc_virt = transfer->desc_virt;
+ dma_addr_t desc_bus = transfer->desc_bus;
+ int i;
+
+ BUG_ON(count > XDMA_TRANSFER_MAX_DESC);
+
+ /* create singly-linked list for SG DMA controller */
+ for (i = 0; i < count - 1; i++) {
+ /* increment bus address to next in array */
+ desc_bus += sizeof(struct xdma_desc);
+
+ /* singly-linked list uses bus addresses */
+ desc_virt[i].next_lo = cpu_to_le32(PCI_DMA_L(desc_bus));
+ desc_virt[i].next_hi = cpu_to_le32(PCI_DMA_H(desc_bus));
+ desc_virt[i].bytes = cpu_to_le32(0);
+
+ desc_virt[i].control = cpu_to_le32(DESC_MAGIC);
+ }
+
+ /* zero the last descriptor next pointer */
+ desc_virt[i].next_lo = cpu_to_le32(0);
+ desc_virt[i].next_hi = cpu_to_le32(0);
+ desc_virt[i].bytes = cpu_to_le32(0);
+ desc_virt[i].control = cpu_to_le32(DESC_MAGIC);
+}
+
+/* Set how many descriptors are adjacent to this one */
+static void xdma_desc_adjacent(struct xdma_desc *desc, u32 next_adjacent)
+{
+ /* remember reserved and control bits */
+ u32 control = le32_to_cpu(desc->control) & 0x0000f0ffUL;
+ /* merge adjacent and control field */
+ control |= 0xAD4B0000UL | (next_adjacent << 8);
+ /* write control and next_adjacent */
+ desc->control = cpu_to_le32(control);
+}
+
+/* Set complete control field of a descriptor */
+static void xdma_desc_control_set(struct xdma_desc *first, u32 control_field)
+{
+ /* remember magic and adjacent number */
+ u32 control = le32_to_cpu(first->control) & ~(LS_BYTE_MASK);
+
+ /* merge adjacent and control field */
+ control |= control_field;
+ /* write control and next_adjacent */
+ first->control = cpu_to_le32(control);
+}
+
+static inline void xdma_desc_done(struct xdma_desc *desc_virt, int count)
+{
+ memset(desc_virt, 0, count * sizeof(struct xdma_desc));
+}
+
+/* Fill a descriptor with the transfer details */
+static void xdma_desc_set(struct xdma_desc *desc, dma_addr_t rc_bus_addr,
+ u64 ep_addr, int len, int dir)
+{
+ /* transfer length */
+ desc->bytes = cpu_to_le32(len);
+ if (dir == DMA_TO_DEVICE) {
+ /* read from root complex memory (source address) */
+ desc->src_addr_lo = cpu_to_le32(PCI_DMA_L(rc_bus_addr));
+ desc->src_addr_hi = cpu_to_le32(PCI_DMA_H(rc_bus_addr));
+ /* write to end point address (destination address) */
+ desc->dst_addr_lo = cpu_to_le32(PCI_DMA_L(ep_addr));
+ desc->dst_addr_hi = cpu_to_le32(PCI_DMA_H(ep_addr));
+ } else {
+ /* read from end point address (source address) */
+ desc->src_addr_lo = cpu_to_le32(PCI_DMA_L(ep_addr));
+ desc->src_addr_hi = cpu_to_le32(PCI_DMA_H(ep_addr));
+ /* write to root complex memory (destination address) */
+ desc->dst_addr_lo = cpu_to_le32(PCI_DMA_L(rc_bus_addr));
+ desc->dst_addr_hi = cpu_to_le32(PCI_DMA_H(rc_bus_addr));
+ }
+}
+
+static void transfer_abort(struct xdma_engine *engine,
+ struct xdma_transfer *transfer)
+{
+ struct xdma_transfer *head;
+
+ head = list_entry(engine->transfer_list.next, struct xdma_transfer,
+ entry);
+ if (head == transfer)
+ list_del(engine->transfer_list.next);
+ else
+ pr_warn("XDMA: %s: transfer for abort NOT found\n",
+ engine->name);
+
+ if (transfer->state == TRANSFER_STATE_SUBMITTED)
+ transfer->state = TRANSFER_STATE_ABORTED;
+}
+
+static int transfer_queue(struct xdma_engine *engine,
+ struct xdma_transfer *transfer)
+{
+ int rv = 0;
+ unsigned long flags;
+
+ /* lock the engine state */
+ spin_lock_irqsave(&engine->lock, flags);
+
+ engine->prev_cpu = get_cpu();
+ put_cpu();
+
+ /* engine is being shutdown; do not accept new transfers */
+ if (engine->shutdown & ENGINE_SHUTDOWN_REQUEST) {
+ pr_info("XDMA: %s: engine offline, transfer not queued\n",
+ engine->name);
+ rv = -EBUSY;
+ goto shutdown;
+ }
+
+ /* mark the transfer as submitted */
+ transfer->state = TRANSFER_STATE_SUBMITTED;
+ /* add transfer to the tail of the engine transfer queue */
+ list_add_tail(&transfer->entry, &engine->transfer_list);
+
+ if (!engine->running)
+ rv = engine_start(engine);
+
+shutdown:
+ spin_unlock_irqrestore(&engine->lock, flags);
+
+ return rv;
+}
+
+static void engine_alignments(struct xdma_engine *engine)
+{
+ u32 w = ioread32(&engine->regs->alignments);
+
+ if (w) {
+ engine->addr_align = (w & 0x00ff0000U) >> 16;
+ engine->len_granularity = (w & 0x0000ff00U) >> 8;
+ engine->addr_bits = (w & 0x000000ffU);
+ } else {
+ /* Some default values if alignments are unspecified */
+ engine->addr_align = 1;
+ engine->len_granularity = 1;
+ engine->addr_bits = 64;
+ }
+}
+
+static void engine_free_resource(struct xdma_engine *engine)
+{
+ struct xdma_dev *xdev = engine->xdev;
+
+ if (engine->desc) {
+ dma_free_coherent(&xdev->pdev->dev,
+ XDMA_TRANSFER_MAX_DESC *
+ sizeof(struct xdma_desc),
+ engine->desc, engine->desc_bus);
+ engine->desc = NULL;
+ }
+
+ if (engine->cyclic_result) {
+ dma_free_coherent(
+ &xdev->pdev->dev,
+ XDMA_TRANSFER_MAX_DESC * sizeof(struct xdma_result),
+ engine->cyclic_result, engine->cyclic_result_bus);
+ engine->cyclic_result = NULL;
+ }
+}
+
+static void engine_destroy(struct xdma_dev *xdev, struct xdma_engine *engine)
+{
+ /* Disable interrupts to stop processing new events during shutdown */
+ iowrite32(0x0, &engine->regs->interrupt_enable_mask);
+
+ if (enable_credit_mp && engine->streaming &&
+ engine->dir == DMA_FROM_DEVICE) {
+ u32 reg_value = (0x1 << engine->channel) << 16;
+ struct sgdma_common_regs *reg =
+ (struct sgdma_common_regs *)
+ (xdev->config_bar + (0x6 * TARGET_SPACING));
+ iowrite32(reg_value, ®->credit_mode_enable_w1c);
+ }
+
+ /* Release memory use for descriptor writebacks */
+ engine_free_resource(engine);
+
+ memset(engine, 0, sizeof(struct xdma_engine));
+ /* Decrement the number of engines available */
+ xdev->engines_num--;
+}
+
+static void engine_init_regs(struct xdma_engine *engine)
+{
+ u32 reg_value;
+
+ iowrite32(XDMA_CTRL_NON_INCR_ADDR, &engine->regs->control_w1c);
+
+ engine_alignments(engine);
+
+ /* Configure error interrupts by default */
+ reg_value = XDMA_CTRL_IE_DESC_ALIGN_MISMATCH;
+ reg_value |= XDMA_CTRL_IE_MAGIC_STOPPED;
+ reg_value |= XDMA_CTRL_IE_MAGIC_STOPPED;
+ reg_value |= XDMA_CTRL_IE_READ_ERROR;
+ reg_value |= XDMA_CTRL_IE_DESC_ERROR;
+
+ /* enable the relevant completion interrupts */
+ reg_value |= XDMA_CTRL_IE_DESC_STOPPED;
+ reg_value |= XDMA_CTRL_IE_DESC_COMPLETED;
+
+ /* Apply engine configurations */
+ iowrite32(reg_value, &engine->regs->interrupt_enable_mask);
+
+ engine->interrupt_enable_mask_value = reg_value;
+
+ /* only enable credit mode for AXI-ST C2H */
+ if (enable_credit_mp && engine->streaming &&
+ engine->dir == DMA_FROM_DEVICE) {
+ struct xdma_dev *xdev = engine->xdev;
+ u32 reg_value = (0x1 << engine->channel) << 16;
+ struct sgdma_common_regs *reg =
+ (struct sgdma_common_regs *)
+ (xdev->config_bar + (0x6 * TARGET_SPACING));
+
+ iowrite32(reg_value, ®->credit_mode_enable_w1s);
+ }
+}
+
+static int engine_alloc_resource(struct xdma_engine *engine)
+{
+ struct xdma_dev *xdev = engine->xdev;
+
+ engine->desc = dma_alloc_coherent(&xdev->pdev->dev,
+ XDMA_TRANSFER_MAX_DESC *
+ sizeof(struct xdma_desc),
+ &engine->desc_bus, GFP_KERNEL);
+ if (!engine->desc)
+ goto err_out;
+
+ if (engine->streaming && engine->dir == DMA_FROM_DEVICE) {
+ engine->cyclic_result = dma_alloc_coherent(
+ &xdev->pdev->dev,
+ XDMA_TRANSFER_MAX_DESC * sizeof(struct xdma_result),
+ &engine->cyclic_result_bus, GFP_KERNEL);
+
+ if (!engine->cyclic_result)
+ goto err_out;
+ }
+
+ return 0;
+
+err_out:
+ engine_free_resource(engine);
+ return -ENOMEM;
+}
+
+static int engine_init(struct xdma_engine *engine, struct xdma_dev *xdev,
+ int offset, enum dma_data_direction dir, int channel)
+{
+ int rv;
+ u32 val;
+
+ engine->channel = channel;
+ engine->xdev = xdev;
+
+ /* engine interrupt request bit */
+ engine->irq_bitmask = (1 << XDMA_ENG_IRQ_NUM) - 1;
+ engine->irq_bitmask <<= (xdev->engines_num * XDMA_ENG_IRQ_NUM);
+
+ /* register address */
+ engine->regs = xdev->config_bar + offset;
+ engine->sgdma_regs = xdev->config_bar + offset +
+ SGDMA_OFFSET_FROM_CHANNEL;
+ val = ioread32(&engine->regs->identifier);
+ if (val & 0x8000U)
+ engine->streaming = 1;
+
+ /* remember SG DMA direction */
+ engine->dir = dir;
+ sprintf(engine->name, "xdma-%s%d%s", (dir == DMA_TO_DEVICE) ? "H2C" : "C2H",
+ channel, engine->streaming ? "ST" : "MM");
+
+ /* initialize the deferred work for transfer completion */
+ INIT_WORK(&engine->work, engine_service_work);
+
+ xdev->engines_num++;
+
+ rv = engine_alloc_resource(engine);
+ if (rv)
+ return rv;
+ engine_init_regs(engine);
+
+ return 0;
+}
+
+static void transfer_destroy(struct xdma_dev *xdev, struct xdma_transfer *xfer)
+{
+ xdma_desc_done(xfer->desc_virt, xfer->desc_num);
+
+ if (xfer->last_in_request && (xfer->flags & XFER_FLAG_NEED_UNMAP)) {
+ struct sg_table *sgt = xfer->sgt;
+
+ if (sgt->nents) {
+ dma_unmap_sg(&xdev->pdev->dev, sgt->sgl, sgt->nents,
+ xfer->dir);
+ sgt->nents = 0;
+ }
+ }
+}
+
+static void transfer_build(struct xdma_engine *engine,
+ struct xdma_request_cb *req, struct xdma_transfer *xfer,
+ unsigned int desc_max)
+{
+ struct sw_desc *sdesc = &(req->sdesc[req->sw_desc_idx]);
+ int i, j;
+ dma_addr_t bus = xfer->res_bus;
+
+ for (i = 0, j = 0; i < desc_max; i++, j++, sdesc++) {
+ /* fill in descriptor entry j with transfer details */
+ xdma_desc_set(xfer->desc_virt + j, sdesc->addr, req->ep_addr,
+ sdesc->len, xfer->dir);
+ xfer->len += sdesc->len;
+
+ /* for non-inc-add mode don't increment ep_addr */
+ if (!engine->non_incr_addr)
+ req->ep_addr += sdesc->len;
+
+ if (engine->streaming && engine->dir == DMA_FROM_DEVICE) {
+ memset(xfer->res_virt + j, 0,
+ sizeof(struct xdma_result));
+ xfer->desc_virt[j].src_addr_lo =
+ cpu_to_le32(PCI_DMA_L(bus));
+ xfer->desc_virt[j].src_addr_hi =
+ cpu_to_le32(PCI_DMA_H(bus));
+ bus += sizeof(struct xdma_result);
+ }
+
+ }
+
+ req->sw_desc_idx += desc_max;
+}
+
+static void transfer_init(struct xdma_engine *engine,
+ struct xdma_request_cb *req, struct xdma_transfer *xfer)
+{
+ unsigned int desc_max = min_t(unsigned int,
+ req->sw_desc_cnt - req->sw_desc_idx,
+ XDMA_TRANSFER_MAX_DESC);
+ int i, last;
+ u32 control;
+ unsigned long flags;
+
+ memset(xfer, 0, sizeof(*xfer));
+
+ spin_lock_irqsave(&engine->lock, flags);
+ init_swait_queue_head(&xfer->wq);
+
+ /* remember direction of transfer */
+ xfer->dir = engine->dir;
+ xfer->desc_virt = engine->desc + engine->desc_idx;
+ xfer->res_virt = engine->cyclic_result + engine->desc_idx;
+ xfer->desc_bus = engine->desc_bus +
+ (sizeof(struct xdma_desc) * engine->desc_idx);
+ xfer->res_bus = engine->cyclic_result_bus +
+ (sizeof(struct xdma_result) * engine->desc_idx);
+ xfer->desc_index = engine->desc_idx;
+
+ if ((engine->desc_idx + desc_max) >= XDMA_TRANSFER_MAX_DESC)
+ desc_max = XDMA_TRANSFER_MAX_DESC - engine->desc_idx;
+
+ transfer_desc_init(xfer, desc_max);
+ transfer_build(engine, req, xfer, desc_max);
+
+ xfer->desc_adjacent = desc_max;
+
+ /* terminate last descriptor */
+ last = desc_max - 1;
+ /* stop engine, EOP for AXI ST, req IRQ on last descriptor */
+ control = XDMA_DESC_STOPPED;
+ control |= XDMA_DESC_EOP;
+ control |= XDMA_DESC_COMPLETED;
+ xdma_desc_control_set(xfer->desc_virt + last, control);
+
+ if (engine->eop_flush) {
+ for (i = 0; i < last; i++)
+ xdma_desc_control_set(xfer->desc_virt + i,
+ XDMA_DESC_COMPLETED);
+ xfer->desc_cmpl_th = 1;
+ } else
+ xfer->desc_cmpl_th = desc_max;
+
+ xfer->desc_num = desc_max;
+ engine->desc_idx = (engine->desc_idx + desc_max) % XDMA_TRANSFER_MAX_DESC;
+ engine->desc_used += desc_max;
+
+ /* fill in adjacent numbers */
+ for (i = 0; i < xfer->desc_num; i++) {
+ u32 next_adj = xdma_get_next_adj(xfer->desc_num - i - 1,
+ (xfer->desc_virt + i)->next_lo);
+ xdma_desc_adjacent(xfer->desc_virt + i, next_adj);
+ }
+
+ spin_unlock_irqrestore(&engine->lock, flags);
+}
+
+static void xdma_request_free(struct xdma_request_cb *req)
+{
+ kvfree(req);
+}
+
+static struct xdma_request_cb *xdma_request_alloc(struct xdma_dev *xdev,
+ unsigned int sdesc_nr)
+{
+ unsigned int size = sizeof(struct xdma_request_cb) +
+ sdesc_nr * sizeof(struct sw_desc);
+
+ return kvzalloc(size, GFP_KERNEL);
+}
+
+static struct xdma_request_cb *xdma_init_request(struct xdma_dev *xdev,
+ struct sg_table *sgt,
+ u64 ep_addr)
+{
+ struct xdma_request_cb *req;
+ struct scatterlist *sg = sgt->sgl;
+ int max = sgt->nents;
+ int extra = 0;
+ int i, j = 0;
+
+ for (i = 0; i < max; i++, sg = sg_next(sg)) {
+ unsigned int len = sg_dma_len(sg);
+
+ if (unlikely(len > XDMA_DESC_BLEN_MAX))
+ extra += (len + XDMA_DESC_BLEN_MAX - 1) / XDMA_DESC_BLEN_MAX;
+ }
+
+ max += extra;
+ req = xdma_request_alloc(xdev, max);
+ if (!req)
+ return NULL;
+
+ req->sgt = sgt;
+ req->ep_addr = ep_addr;
+
+ for (i = 0, sg = sgt->sgl; i < sgt->nents; i++, sg = sg_next(sg)) {
+ unsigned int tlen = sg_dma_len(sg);
+ dma_addr_t addr = sg_dma_address(sg);
+
+ req->total_len += tlen;
+ while (tlen) {
+ req->sdesc[j].addr = addr;
+ if (tlen > XDMA_DESC_BLEN_MAX) {
+ req->sdesc[j].len = XDMA_DESC_BLEN_MAX;
+ addr += XDMA_DESC_BLEN_MAX;
+ tlen -= XDMA_DESC_BLEN_MAX;
+ } else {
+ req->sdesc[j].len = tlen;
+ tlen = 0;
+ }
+ j++;
+ }
+ }
+
+ if (j > max) {
+ pr_err("XDMA: Max. transfer length (%d) exceeded",
+ XDMA_DESC_BLEN_MAX);
+ xdma_request_free(req);
+ return NULL;
+ }
+ req->sw_desc_cnt = j;
+
+ return req;
+}
+
+static struct xdma_engine *channel_engine(struct xdma_core *xdma, int channel,
+ bool write)
+{
+ if (write) {
+ if (channel >= xdma->h2c_channel_max) {
+ pr_err("XDMA: %d: invalid H2C channel\n", channel);
+ return NULL;
+ } else
+ return &xdma->xdev->engine_h2c[channel];
+ } else {
+ if (channel >= xdma->c2h_channel_max) {
+ pr_err("XDMA: %d: invalid C2H channel\n", channel);
+ return NULL;
+ } else
+ return &xdma->xdev->engine_c2h[channel];
+ }
+}
+
+static struct xdma_dev *alloc_dev(struct pci_dev *pdev)
+{
+ int i;
+ struct xdma_dev *xdev;
+ struct xdma_engine *engine;
+
+ xdev = kzalloc(sizeof(struct xdma_dev), GFP_KERNEL);
+ if (!xdev)
+ return NULL;
+
+ xdev->pdev = pdev;
+
+ engine = xdev->engine_h2c;
+ for (i = 0; i < XDMA_CHANNEL_NUM_MAX; i++, engine++) {
+ spin_lock_init(&engine->lock);
+ mutex_init(&engine->desc_lock);
+ INIT_LIST_HEAD(&engine->transfer_list);
+ init_swait_queue_head(&engine->shutdown_wq);
+ }
+
+ engine = xdev->engine_c2h;
+ for (i = 0; i < XDMA_CHANNEL_NUM_MAX; i++, engine++) {
+ spin_lock_init(&engine->lock);
+ mutex_init(&engine->desc_lock);
+ INIT_LIST_HEAD(&engine->transfer_list);
+ init_swait_queue_head(&engine->shutdown_wq);
+ }
+
+ return xdev;
+}
+
+static int set_dma_mask(struct xdma_dev *xdev)
+{
+ if (!dma_set_mask(&xdev->pdev->dev, DMA_BIT_MASK(64))) {
+ pr_devel("XDMA: Using a 64-bit DMA mask\n");
+ /* use 32-bit DMA for descriptors */
+ dma_set_coherent_mask(&xdev->pdev->dev, DMA_BIT_MASK(32));
+ } else if (!dma_set_mask(&xdev->pdev->dev, DMA_BIT_MASK(32))) {
+ pr_devel("XDMA: Using a 32-bit DMA mask\n");
+ dma_set_coherent_mask(&xdev->pdev->dev, DMA_BIT_MASK(32));
+ } else {
+ pr_err("XDMA: No suitable DMA possible.\n");
+ return -EINVAL;
+ }
+
+ return 0;
+}
+
+static int get_engine_channel_id(struct engine_regs *regs)
+{
+ int value = ioread32(®s->identifier);
+
+ return (value & 0x00000f00U) >> 8;
+}
+
+static int get_engine_id(struct engine_regs *regs)
+{
+ int value = ioread32(®s->identifier);
+
+ return (value & 0xffff0000U) >> 16;
+}
+
+static void remove_engines(struct xdma_dev *xdev, int h2c_channel_max,
+ int c2h_channel_max)
+{
+ int i;
+
+ for (i = 0; i < h2c_channel_max; i++)
+ engine_destroy(xdev, &xdev->engine_h2c[i]);
+
+ for (i = 0; i < c2h_channel_max; i++)
+ engine_destroy(xdev, &xdev->engine_c2h[i]);
+}
+
+static int probe_for_engine(struct xdma_dev *xdev, enum dma_data_direction dir,
+ int channel)
+{
+ struct engine_regs *regs;
+ int offset = channel * CHANNEL_SPACING;
+ u32 engine_id;
+ u32 engine_id_expected;
+ u32 channel_id;
+ struct xdma_engine *engine;
+
+ if (dir == DMA_TO_DEVICE) {
+ engine_id_expected = XDMA_ID_H2C;
+ engine = &xdev->engine_h2c[channel];
+ } else {
+ offset += H2C_CHANNEL_OFFSET;
+ engine_id_expected = XDMA_ID_C2H;
+ engine = &xdev->engine_c2h[channel];
+ }
+
+ regs = xdev->config_bar + offset;
+ engine_id = get_engine_id(regs);
+ channel_id = get_engine_channel_id(regs);
+
+ if ((engine_id != engine_id_expected) || (channel_id != channel)) {
+ pr_err("XDMA: %s engine #%d not found\n",
+ dir == DMA_TO_DEVICE ? "H2C" : "C2H", channel);
+ return -EINVAL;
+ }
+
+ engine_init(engine, xdev, offset, dir, channel);
+
+ return 0;
+}
+
+static int probe_engines(struct xdma_dev *xdev, int h2c_channel_max,
+ int c2h_channel_max)
+{
+ int i, rv;
+
+ for (i = 0; i < h2c_channel_max; i++) {
+ rv = probe_for_engine(xdev, DMA_TO_DEVICE, i);
+ if (rv)
+ return rv;
+ }
+
+ for (i = 0; i < c2h_channel_max; i++) {
+ rv = probe_for_engine(xdev, DMA_FROM_DEVICE, i);
+ if (rv)
+ return rv;
+ }
+
+ return 0;
+}
+
+
+int xdma_probe(struct xdma_core *xdma)
+{
+ int rv;
+
+ if (xdma->user_irq_max > MAX_USER_IRQ) {
+ pr_err("XDMA: %d: Invalid number of user IRQs\n",
+ xdma->user_irq_max);
+ return -EINVAL;
+ }
+ if (xdma->h2c_channel_max > XDMA_CHANNEL_NUM_MAX) {
+ pr_err("XDMA: %d: Invalid number of H2C channels\n",
+ xdma->h2c_channel_max);
+ return -EINVAL;
+ }
+ if (xdma->c2h_channel_max > XDMA_CHANNEL_NUM_MAX) {
+ pr_err("XDMA: %d: Invalid number of C2H channels\n",
+ xdma->c2h_channel_max);
+ return -EINVAL;
+ }
+
+ xdma->xdev = alloc_dev(xdma->pdev);
+ if (!xdma->xdev)
+ return -ENOMEM;
+
+ rv = map_config_bar(xdma->xdev, xdma->config_bar_id);
+ if (rv)
+ goto err_map;
+
+ rv = set_dma_mask(xdma->xdev);
+ if (rv)
+ goto err_mask;
+
+ channel_interrupts_disable(xdma->xdev, ~0);
+ user_interrupts_disable(xdma->xdev, ~0);
+ /* Flush writes */
+ read_interrupts(xdma->xdev);
+
+ rv = probe_engines(xdma->xdev, xdma->h2c_channel_max, xdma->c2h_channel_max);
+ if (rv)
+ goto err_engines;
+
+ rv = irq_setup(xdma->xdev, xdma->h2c_channel_max, xdma->c2h_channel_max,
+ xdma->user_irq_max);
+ if (rv < 0)
+ goto err_interrupts;
+ channel_interrupts_enable(xdma->xdev, ~0);
+ /* Flush writes */
+ read_interrupts(xdma->xdev);
+
+ return 0;
+
+err_interrupts:
+ irq_teardown(xdma->xdev, xdma->h2c_channel_max, xdma->c2h_channel_max,
+ xdma->user_irq_max);
+err_engines:
+ remove_engines(xdma->xdev, xdma->h2c_channel_max, xdma->c2h_channel_max);
+err_mask:
+ unmap_config_bar(xdma->xdev, xdma->config_bar_id);
+err_map:
+ kfree(xdma->xdev);
+
+ return rv;
+}
+EXPORT_SYMBOL_GPL(xdma_probe);
+
+void xdma_remove(struct xdma_core *xdma)
+{
+ channel_interrupts_disable(xdma->xdev, ~0);
+ user_interrupts_disable(xdma->xdev, ~0);
+ /* Flush writes */
+ read_interrupts(xdma->xdev);
+
+ irq_teardown(xdma->xdev, xdma->h2c_channel_max, xdma->c2h_channel_max,
+ xdma->user_irq_max);
+
+ remove_engines(xdma->xdev, xdma->h2c_channel_max, xdma->c2h_channel_max);
+ unmap_config_bar(xdma->xdev, xdma->config_bar_id);
+
+ kfree(xdma->xdev);
+}
+EXPORT_SYMBOL_GPL(xdma_remove);
+
+/**
+ * xdma_irq_enable - enable XDMA user interrupt(s)
+ * @xdma: XDMA device handle
+ * @mask: bitmask of user interrupts (0 ~ 15) to be registered
+ */
+void xdma_irq_enable(struct xdma_core *xdma, unsigned int mask)
+{
+ xdma->xdev->mask_irq_user |= mask;
+ user_interrupts_enable(xdma->xdev, mask);
+ /* Flush writes */
+ read_interrupts(xdma->xdev);
+}
+EXPORT_SYMBOL_GPL(xdma_irq_enable);
+
+/**
+ * xdma_irq_disable - disable XDMA user interrupt(s)
+ * @xdma: XDMA device handle
+ * @mask: bitmask of user interrupts (0 ~ 15) to be unregistered
+ */
+void xdma_irq_disable(struct xdma_core *xdma, unsigned int mask)
+{
+ xdma->xdev->mask_irq_user &= ~mask;
+ user_interrupts_disable(xdma->xdev, mask);
+ /* Flush writes */
+ read_interrupts(xdma->xdev);
+}
+EXPORT_SYMBOL_GPL(xdma_irq_disable);
+
+/**
+ * xdma_transfer - do a DMA transfer
+ * @xdma: XDMA device handle
+ * @channel: channel number
+ * @write: slecets read/write operation
+ * @ep_addr: offset into the DDR/BRAM (card) memory to read from or write to
+ * @sg_table: the scatter-gather list of data buffers
+ * @timeout_ms: timeout in mili-seconds
+ *
+ * Returns # of bytes transferred on success, negative on failure
+ */
+ssize_t xdma_transfer(struct xdma_core *xdma, int channel, bool write,
+ u64 ep_addr, struct sg_table *sgt, int timeout_ms)
+{
+ struct xdma_engine *engine;
+ int rv = 0, i, nents;
+ ssize_t done = 0;
+ struct xdma_request_cb *req = NULL;
+
+
+ engine = channel_engine(xdma, channel, write);
+ if (!engine)
+ return -EINVAL;
+
+ req = xdma_init_request(xdma->xdev, sgt, ep_addr);
+ if (!req)
+ return -ENOMEM;
+
+ nents = req->sw_desc_cnt;
+ mutex_lock(&engine->desc_lock);
+
+ while (nents) {
+ unsigned long flags;
+ struct xdma_transfer *xfer;
+
+ /* build transfer */
+ transfer_init(engine, req, &req->tfer);
+ xfer = &req->tfer;
+
+ /* last transfer for the given request? */
+ nents -= xfer->desc_num;
+ if (!nents) {
+ xfer->last_in_request = 1;
+ xfer->sgt = sgt;
+ }
+
+ rv = transfer_queue(engine, xfer);
+ if (rv < 0)
+ break;
+
+ if (timeout_ms > 0)
+ swait_event_interruptible_timeout_exclusive(xfer->wq,
+ (xfer->state != TRANSFER_STATE_SUBMITTED),
+ msecs_to_jiffies(timeout_ms));
+ else
+ swait_event_interruptible_exclusive(xfer->wq,
+ (xfer->state != TRANSFER_STATE_SUBMITTED));
+
+ spin_lock_irqsave(&engine->lock, flags);
+
+ switch (xfer->state) {
+ case TRANSFER_STATE_COMPLETED:
+ spin_unlock_irqrestore(&engine->lock, flags);
+ /* For C2H streaming use writeback results */
+ if (engine->streaming &&
+ engine->dir == DMA_FROM_DEVICE) {
+ struct xdma_result *result = xfer->res_virt;
+
+ for (i = 0; i < xfer->desc_cmpl; i++)
+ done += result[i].length;
+
+ /* finish the whole request */
+ if (engine->eop_flush)
+ nents = 0;
+ } else
+ done += xfer->len;
+ rv = 0;
+ break;
+ case TRANSFER_STATE_FAILED:
+ pr_warn("XDMA: transfer failed\n");
+ spin_unlock_irqrestore(&engine->lock, flags);
+ rv = -EIO;
+ break;
+ default:
+ /* transfer can still be in-flight */
+ pr_warn("XDMA: transfer timed out\n");
+ engine_status_read(engine, 0, 1);
+ transfer_abort(engine, xfer);
+ engine_stop(engine);
+ spin_unlock_irqrestore(&engine->lock, flags);
+ rv = -ERESTARTSYS;
+ break;
+ }
+
+ engine->desc_used -= xfer->desc_num;
+ transfer_destroy(xdma->xdev, xfer);
+
+ if (rv < 0)
+ break;
+ }
+
+ mutex_unlock(&engine->desc_lock);
+ xdma_request_free(req);
+
+ return rv ? rv : done;
+}
+EXPORT_SYMBOL_GPL(xdma_transfer);
+
+MODULE_AUTHOR("Digiteq Automotive s.r.o.");
+MODULE_DESCRIPTION("Xilinx XDMA Driver");
+MODULE_LICENSE("GPL");
diff --git a/include/linux/dma/xilinx_xdma.h b/include/linux/dma/xilinx_xdma.h
new file mode 100644
index 000000000000..c63dc7768e66
--- /dev/null
+++ b/include/linux/dma/xilinx_xdma.h
@@ -0,0 +1,44 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+/*
+ * This file is part of the Xilinx DMA IP Core driver for Linux
+ *
+ * Copyright (c) 2016-2021, Xilinx, Inc.
+ * Copyright (c) 2020-2022, Digiteq Automotive s.r.o.
+ */
+
+#ifndef XILINX_XDMA_H
+#define XILINX_XDMA_H
+
+#include <linux/pci.h>
+#include <linux/scatterlist.h>
+
+struct xdma_dev;
+
+/**
+ * struct xdma_core - representation of XDMA hardware
+ * @pdev: The parent PCIe device which contains the XDMA core
+ * @config_bar_id: PCI BAR id where XDMA config regs are located
+ * @user_irq_max: number of user IRQs
+ * @c2h_channel_max: number of C2H DMA channels
+ * @h2c_channel_max: number of H2C DMA channels
+ * @xdev: struct xdma_dev that is filed by ->probe()
+ */
+struct xdma_core {
+ struct pci_dev *pdev;
+ int config_bar_id;
+ unsigned int user_irq_max;
+ unsigned int c2h_channel_max;
+ unsigned int h2c_channel_max;
+ struct xdma_dev *xdev;
+};
+
+int xdma_probe(struct xdma_core *xdma);
+void xdma_remove(struct xdma_core *xdma);
+
+void xdma_irq_enable(struct xdma_core *xdma, unsigned int mask);
+void xdma_irq_disable(struct xdma_core *xdma, unsigned int mask);
+
+ssize_t xdma_transfer(struct xdma_core *xdma, int channel, bool write,
+ u64 ep_addr, struct sg_table *sgt, int timeout_ms);
+
+#endif /* XILINX_XDMA_H */
--
2.37.2