[PATCH 09/15] PCI: endpoint: Add pci-ep-dma helper for exported DMA ABI v1

[Koichiro Den]

Add a generic helper that packages controller-owned DMA resources into a
peer-visible BAR slice described by exported DMA ABI v1.

pci_epf_alloc_dma() queries EPC auxiliary resources, delegates the
requested DMA read channels, builds an ABI header in coherent memory,
and assembles the BAR region list covering the header, controller
register window, and per-channel descriptor windows. If the controller
control window is not already BAR-backed, map it into the exported slice
so the peer still sees a self-contained layout.

The first ABI is designed based on the DesignWare unrolled eDMA model,
but it is intended to be vendor-neutral. It exports delegated READ
channels only, which are the channels the host uses to send data from
host memory into the endpoint.

Signed-off-by: Koichiro Den <den@xxxxxxxxxxxxx>
---
 drivers/pci/endpoint/Makefile     |   2 +-
 drivers/pci/endpoint/pci-ep-dma.c | 342 ++++++++++++++++++++++++++++++
 include/linux/pci-ep-dma.h        | 130 ++++++++++++
 3 files changed, 473 insertions(+), 1 deletion(-)
 create mode 100644 drivers/pci/endpoint/pci-ep-dma.c
 create mode 100644 include/linux/pci-ep-dma.h

diff --git a/drivers/pci/endpoint/Makefile b/drivers/pci/endpoint/Makefile
index b4869d52053a..94824f3ed5a1 100644
--- a/drivers/pci/endpoint/Makefile
+++ b/drivers/pci/endpoint/Makefile
@@ -5,5 +5,5 @@
 
 obj-$(CONFIG_PCI_ENDPOINT_CONFIGFS)	+= pci-ep-cfs.o
 obj-$(CONFIG_PCI_ENDPOINT)		+= pci-epc-core.o pci-epf-core.o\
-					   pci-epc-mem.o functions/
+					   pci-epc-mem.o pci-ep-dma.o functions/
 obj-$(CONFIG_PCI_ENDPOINT_MSI_DOORBELL)	+= pci-ep-msi.o
diff --git a/drivers/pci/endpoint/pci-ep-dma.c b/drivers/pci/endpoint/pci-ep-dma.c
new file mode 100644
index 000000000000..2a996f9b1424
--- /dev/null
+++ b/drivers/pci/endpoint/pci-ep-dma.c
@@ -0,0 +1,342 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Generic exported DMA helper for PCI endpoint functions
+ */
+
+#include <linux/align.h>
+#include <linux/dma-mapping.h>
+#include <linux/err.h>
+#include <linux/kernel.h>
+#include <linux/pci-ep-dma.h>
+#include <linux/pci-epc.h>
+#include <linux/slab.h>
+
+static const struct pci_epc_aux_resource *
+pci_ep_dma_find_ctrl(const struct pci_epc_aux_resource *resources, int count)
+{
+	int i;
+
+	for (i = 0; i < count; i++)
+		if (resources[i].type == PCI_EPC_AUX_DMA_CTRL_MMIO)
+			return &resources[i];
+
+	return NULL;
+}
+
+static const struct pci_epc_aux_resource *
+pci_ep_dma_find_desc(const struct pci_epc_aux_resource *resources, int count,
+		     int chan_id)
+{
+	int i;
+
+	/*
+	 * ABI v1 exports delegated READ channels only. A remote client uses
+	 * those channels to pull host memory into the endpoint, so ignore WRITE
+	 * channel descriptors here.
+	 */
+	for (i = 0; i < count; i++) {
+		if (resources[i].type != PCI_EPC_AUX_DMA_CHAN_DESC)
+			continue;
+		if (resources[i].u.dma_chan.dir != PCI_EPC_AUX_DMA_DIR_READ)
+			continue;
+		if (resources[i].u.dma_chan.chan_id != chan_id)
+			continue;
+		return &resources[i];
+	}
+
+	return NULL;
+}
+
+static int pci_ep_dma_undelegate_chans(struct pci_epf *epf, const int *chan_ids,
+				       u32 num_chans)
+{
+	if (!num_chans)
+		return 0;
+
+	return pci_epc_undelegate_dma_channels(epf->epc, epf->func_no,
+					       epf->vfunc_no,
+					       PCI_EPC_AUX_DMA_DIR_READ,
+					       chan_ids, num_chans);
+}
+
+static int pci_ep_dma_map_resource(struct device *dev, phys_addr_t phys,
+				   size_t size, size_t align,
+				   dma_addr_t *dma_addr, size_t *map_size,
+				   u32 *map_delta)
+{
+	phys_addr_t base;
+	size_t map_align;
+
+	map_align = max_t(size_t, PAGE_SIZE, align);
+	base = ALIGN_DOWN(phys, map_align);
+	*map_delta = phys - base;
+	*map_size = ALIGN(size + *map_delta, map_align);
+	*dma_addr = dma_map_resource(dev, base, *map_size,
+				     DMA_BIDIRECTIONAL, 0);
+	if (dma_mapping_error(dev, *dma_addr))
+		return -ENOMEM;
+
+	return 0;
+}
+
+struct pci_ep_dma *pci_epf_alloc_dma(struct pci_epf *epf, enum pci_barno bar,
+				     u32 offset, u32 req_chans)
+{
+	const struct pci_epc_aux_resource *descs[PCI_EP_DMA_MAX_CHANS] = { };
+	const struct pci_epc_features *epc_features;
+	int chan_ids[PCI_EP_DMA_MAX_CHANS] = { 0 };
+	int kept_chan_ids[PCI_EP_DMA_MAX_CHANS] = { 0 };
+	int rejected_chan_ids[PCI_EP_DMA_MAX_CHANS] = { 0 };
+	const struct pci_epc_aux_resource *ctrl;
+	struct pci_epc_aux_resource *res = NULL;
+	struct pci_ep_dma_hdr_v1 *hdr;
+	size_t align, hdr_sz, cur;
+	unsigned int delegated_chans = 0;
+	unsigned int rejected_chans = 0;
+	unsigned int num_chans = 0;
+	struct pci_ep_dma *dma;
+	struct device *dma_dev;
+	struct pci_epc *epc;
+	int count, ret;
+	unsigned int i;
+
+	if (!epf || !epf->epc)
+		return ERR_PTR(-EINVAL);
+
+	epc = epf->epc;
+	epc_features = pci_epc_get_features(epc, epf->func_no, epf->vfunc_no);
+	if (!epc_features)
+		return ERR_PTR(-ENODEV);
+
+	count = pci_epc_get_aux_resources(epc, epf->func_no, epf->vfunc_no,
+					  NULL, 0);
+	if (count < 0)
+		return ERR_PTR(count);
+
+	res = kcalloc(count, sizeof(*res), GFP_KERNEL);
+	if (!res)
+		return ERR_PTR(-ENOMEM);
+
+	ret = pci_epc_get_aux_resources(epc, epf->func_no, epf->vfunc_no,
+					res, count);
+	if (ret < 0)
+		goto err_free_res;
+	count = ret;
+
+	ret = pci_epc_delegate_dma_channels(epc, epf->func_no, epf->vfunc_no,
+					    PCI_EPC_AUX_DMA_DIR_READ,
+					    max_t(u32, 1, req_chans),
+					    chan_ids,
+					    ARRAY_SIZE(chan_ids));
+	if (ret < 0)
+		goto err_free_res;
+	delegated_chans = ret;
+
+	ctrl = pci_ep_dma_find_ctrl(res, count);
+	if (!ctrl) {
+		ret = -ENODEV;
+		goto err_undelegate;
+	}
+
+	for (i = 0; i < delegated_chans; i++) {
+		const struct pci_epc_aux_resource *desc;
+
+		desc = pci_ep_dma_find_desc(res, count, chan_ids[i]);
+		if (!desc) {
+			rejected_chan_ids[rejected_chans++] = chan_ids[i];
+			continue;
+		}
+
+		descs[num_chans] = desc;
+		kept_chan_ids[num_chans++] = chan_ids[i];
+	}
+
+	ret = pci_ep_dma_undelegate_chans(epf, rejected_chan_ids, rejected_chans);
+	if (ret)
+		goto err_undelegate;
+	memcpy(chan_ids, kept_chan_ids, num_chans * sizeof(chan_ids[0]));
+	delegated_chans = num_chans;
+
+	if (!num_chans) {
+		ret = -ENODEV;
+		goto err_undelegate;
+	}
+
+	if (num_chans < req_chans)
+		dev_warn(&epf->dev,
+			 "requested %u DMA channels, delegating %u\n",
+			 req_chans, num_chans);
+
+	dma = kzalloc_obj(*dma);
+	if (!dma) {
+		ret = -ENOMEM;
+		goto err_undelegate;
+	}
+
+	dma_dev = epc->dev.parent;
+	align = epc_features->align ? epc_features->align : SZ_4K;
+	hdr_sz = ALIGN(sizeof(*hdr), align);
+
+	dma->hdr_virt = dma_alloc_coherent(dma_dev, hdr_sz, &dma->hdr_phys,
+					   GFP_KERNEL);
+	if (!dma->hdr_virt) {
+		ret = -ENOMEM;
+		goto err_free_dma;
+	}
+
+	dma->epf = epf;
+	dma->bar = bar;
+	dma->hdr_alloc_size = hdr_sz;
+	dma->num_chans = num_chans;
+	dma->delegated_num_chans = delegated_chans;
+	memcpy(dma->delegated_chan_ids, chan_ids,
+	       delegated_chans * sizeof(dma->delegated_chan_ids[0]));
+
+	cur = offset;
+	dma->regions[dma->num_regions++] = (struct pci_ep_dma_region) {
+		.offset = cur,
+		.phys_addr = dma->hdr_phys,
+		.size = hdr_sz,
+	};
+	cur += hdr_sz;
+
+	hdr = dma->hdr_virt;
+	memset(hdr, 0, sizeof(*hdr));
+	hdr->magic = cpu_to_le32(PCI_EP_DMA_MAGIC);
+	hdr->version = cpu_to_le16(1);
+	hdr->hdr_size = cpu_to_le16(sizeof(*hdr));
+
+	/*
+	 * If there is a fixed mapped DMA register block, reuse it. If the
+	 * controller only reports a raw physical MMIO resource, map it into
+	 * the exported slice so the peer still sees a self-contained BAR layout.
+	 */
+	if (ctrl->bar != NO_BAR) {
+		hdr->ctrl_bar = cpu_to_le32(ctrl->bar);
+		hdr->ctrl_offset = cpu_to_le32(ctrl->bar_offset);
+		hdr->ctrl_size = cpu_to_le32(ctrl->size);
+	} else {
+		size_t map_sz;
+		u32 map_delta;
+		dma_addr_t map_addr;
+
+		ret = pci_ep_dma_map_resource(dma_dev, ctrl->phys_addr, ctrl->size,
+					      align, &map_addr, &map_sz,
+					      &map_delta);
+		if (ret)
+			goto err_free_hdr;
+
+		dma->ctrl_map_addr = map_addr;
+		dma->ctrl_map_size = map_sz;
+		dma->regions[dma->num_regions++] = (struct pci_ep_dma_region) {
+			.offset = cur,
+			.phys_addr = map_addr,
+			.size = map_sz,
+		};
+		hdr->ctrl_bar = cpu_to_le32(bar);
+		hdr->ctrl_offset = cpu_to_le32(cur + map_delta);
+		hdr->ctrl_size = cpu_to_le32(ctrl->size);
+		cur += map_sz;
+	}
+
+	hdr->irq_count = cpu_to_le32(num_chans);
+	hdr->num_chans = cpu_to_le32(num_chans);
+	/*
+	 * Preserve the delegated-channel order in @hdr->chans[]. ABI v1 currently
+	 * assumes the exported READ channels form a dense prefix of the remote
+	 * hardware READ-channel space, so dw-edma-aux consumes @chans[i] as remote
+	 * READ channel i.
+	 */
+	for (i = 0; i < num_chans; i++) {
+		size_t desc_sz = ALIGN(descs[i]->size, align);
+
+		dma->regions[dma->num_regions++] = (struct pci_ep_dma_region) {
+			.offset = cur,
+			.phys_addr = descs[i]->phys_addr,
+			.size = desc_sz,
+		};
+		hdr->chans[i].desc_bar = cpu_to_le32(bar);
+		hdr->chans[i].desc_offset = cpu_to_le32(cur);
+		hdr->chans[i].desc_size = cpu_to_le32(descs[i]->size);
+		hdr->chans[i].desc_phys_addr =
+			cpu_to_le64(descs[i]->phys_addr);
+		cur += desc_sz;
+	}
+
+	hdr->total_size = cpu_to_le32(cur - offset);
+	dma->loc.abi = PCI_EP_DMA_ABI_V1;
+	dma->loc.bar = bar;
+	dma->loc.offset = offset;
+	dma->loc.size = cur - offset;
+
+	kfree(res);
+	return dma;
+
+err_free_hdr:
+	if (dma->ctrl_map_addr)
+		dma_unmap_resource(dma_dev, dma->ctrl_map_addr,
+				   dma->ctrl_map_size,
+				   DMA_BIDIRECTIONAL, 0);
+	if (dma->hdr_virt)
+		dma_free_coherent(dma_dev, dma->hdr_alloc_size,
+				  dma->hdr_virt, dma->hdr_phys);
+err_free_dma:
+	kfree(dma);
+err_undelegate:
+	{
+		int unret;
+
+		unret = pci_ep_dma_undelegate_chans(epf, chan_ids, delegated_chans);
+		if (unret)
+			dev_warn(&epf->dev,
+				 "failed to undelegate DMA channels: %d\n",
+				 unret);
+	}
+err_free_res:
+	kfree(res);
+	return ERR_PTR(ret);
+}
+EXPORT_SYMBOL_GPL(pci_epf_alloc_dma);
+
+void pci_epf_free_dma(struct pci_ep_dma *dma)
+{
+	struct device *dma_dev;
+	int ret;
+
+	if (!dma)
+		return;
+
+	dma_dev = dma->epf->epc->dev.parent;
+	ret = pci_ep_dma_undelegate_chans(dma->epf, dma->delegated_chan_ids,
+					  dma->delegated_num_chans);
+	if (ret)
+		dev_warn(&dma->epf->dev,
+			 "failed to undelegate DMA channels: %d\n", ret);
+	if (dma->ctrl_map_addr)
+		dma_unmap_resource(dma_dev, dma->ctrl_map_addr,
+				   dma->ctrl_map_size,
+				   DMA_BIDIRECTIONAL, 0);
+	if (dma->hdr_virt)
+		dma_free_coherent(dma_dev, dma->hdr_alloc_size,
+				  dma->hdr_virt, dma->hdr_phys);
+	kfree(dma);
+}
+EXPORT_SYMBOL_GPL(pci_epf_free_dma);
+
+const struct pci_ep_dma_locator *pci_epf_get_dma_locator(const struct pci_ep_dma *dma)
+{
+	return dma ? &dma->loc : NULL;
+}
+EXPORT_SYMBOL_GPL(pci_epf_get_dma_locator);
+
+unsigned int pci_epf_get_dma_region_count(const struct pci_ep_dma *dma)
+{
+	return dma ? dma->num_regions : 0;
+}
+EXPORT_SYMBOL_GPL(pci_epf_get_dma_region_count);
+
+const struct pci_ep_dma_region *pci_epf_get_dma_regions(const struct pci_ep_dma *dma)
+{
+	return dma ? dma->regions : NULL;
+}
+EXPORT_SYMBOL_GPL(pci_epf_get_dma_regions);
diff --git a/include/linux/pci-ep-dma.h b/include/linux/pci-ep-dma.h
new file mode 100644
index 000000000000..0ef6f9eb8593
--- /dev/null
+++ b/include/linux/pci-ep-dma.h
@@ -0,0 +1,130 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+/*
+ * Generic exported DMA helper for PCI endpoint functions
+ */
+
+#ifndef __LINUX_PCI_EP_DMA_H
+#define __LINUX_PCI_EP_DMA_H
+
+#include <linux/dma-mapping.h>
+#include <linux/pci-epf.h>
+
+#define PCI_EP_DMA_MAGIC	0x4d445045 /* "EPDM" */
+#define PCI_EP_DMA_MAX_CHANS	8
+#define PCI_EP_DMA_MAX_REGIONS	(2 + PCI_EP_DMA_MAX_CHANS)
+
+enum pci_ep_dma_abi {
+	PCI_EP_DMA_ABI_NONE = 0,
+	PCI_EP_DMA_ABI_V1 = 1,
+};
+
+/**
+ * struct pci_ep_dma_locator - peer-visible location of an exported DMA slice
+ * @abi: exported-DMA ABI identifier from &enum pci_ep_dma_abi
+ * @bar: BAR number that carries the exported slice
+ * @flags: ABI-specific locator flags, reserved for future use in v1
+ * @offset: BAR-relative start offset of the exported slice
+ * @size: total size of the exported slice in bytes
+ */
+struct pci_ep_dma_locator {
+	u8 abi;
+	u8 bar;
+	u16 flags;
+	u32 offset;
+	u32 size;
+};
+
+/**
+ * struct pci_ep_dma_region - one physical region mapped into the exported slice
+ * @offset: BAR-relative start offset of the region within the exported slice
+ * @phys_addr: DMA address to program into the EPC BAR mapping
+ * @size: mapped size in bytes
+ */
+struct pci_ep_dma_region {
+	u32 offset;
+	dma_addr_t phys_addr;
+	size_t size;
+};
+
+/**
+ * struct pci_ep_dma_chan_info - per-channel descriptor metadata in ABI v1
+ * @desc_bar: BAR number that exposes the descriptor window
+ * @desc_offset: BAR-relative start offset of the descriptor window
+ * @desc_size: descriptor window size in bytes
+ * @desc_phys_addr: physical/DMA address used for the EPC-side BAR mapping
+ */
+struct pci_ep_dma_chan_info {
+	__le32 desc_bar;
+	__le32 desc_offset;
+	__le32 desc_size;
+	__le64 desc_phys_addr;
+};
+
+/**
+ * struct pci_ep_dma_hdr_v1 - exported DMA header format, version 1
+ * @magic: fixed signature, must be %PCI_EP_DMA_MAGIC
+ * @version: header version, must be 1 for this structure
+ * @hdr_size: size of the populated header structure in bytes
+ * @total_size: total exported-slice size starting at &struct pci_ep_dma_locator.offset
+ * @ctrl_bar: BAR that exposes the live DMA control registers
+ * @ctrl_offset: BAR-relative start offset of the control-register window
+ * @ctrl_size: size of the control-register window in bytes
+ * @irq_count: number of IRQ vectors reserved for the exported DMA provider
+ * @num_chans: number of valid entries in @chans
+ * @chans: per-channel descriptor metadata
+ *
+ * Exported DMA ABI v1 lays out the peer-visible slice as:
+ *
+ *   [header][controller window?][descriptor window 0]...[descriptor window N]
+ *
+ * The controller window is optional in that slice. When the live register
+ * block is already exposed through another BAR, @ctrl_bar/@ctrl_offset point at
+ * that BAR directly and no controller subrange is embedded in the exported
+ * slice.
+ *
+ * @chans[] describes a dense prefix of the remote hardware READ-channel
+ * space, ordered by remote hardware READ-channel index starting at 0. A
+ * consumer may map @chans[i] directly to remote READ channel i.
+ */
+struct pci_ep_dma_hdr_v1 {
+	__le32 magic;
+	__le16 version;
+	__le16 hdr_size;
+	__le32 total_size;
+	__le32 ctrl_bar;
+	__le32 ctrl_offset;
+	__le32 ctrl_size;
+	__le32 irq_count;
+	__le32 num_chans;
+	struct pci_ep_dma_chan_info chans[PCI_EP_DMA_MAX_CHANS];
+};
+
+struct pci_ep_dma {
+	struct pci_epf *epf;
+	enum pci_barno bar;
+	void *hdr_virt;
+	dma_addr_t hdr_phys;
+	size_t hdr_alloc_size;
+	struct pci_ep_dma_locator loc;
+	unsigned int num_regions;
+	u32 num_chans;
+	struct pci_ep_dma_region regions[PCI_EP_DMA_MAX_REGIONS];
+	dma_addr_t ctrl_map_addr;
+	size_t ctrl_map_size;
+	int delegated_chan_ids[PCI_EP_DMA_MAX_CHANS];
+	u8 delegated_num_chans;
+};
+
+struct pci_ep_dma *pci_epf_alloc_dma(struct pci_epf *epf, enum pci_barno bar,
+				     u32 offset, u32 req_chans);
+void pci_epf_free_dma(struct pci_ep_dma *dma);
+
+const struct pci_ep_dma_locator *
+pci_epf_get_dma_locator(const struct pci_ep_dma *dma);
+
+unsigned int pci_epf_get_dma_region_count(const struct pci_ep_dma *dma);
+
+const struct pci_ep_dma_region *
+pci_epf_get_dma_regions(const struct pci_ep_dma *dma);
+
+#endif /* __LINUX_PCI_EP_DMA_H */
-- 
2.51.0