[PATCH v3 04/10] ARM: dma-mapping: Implement and expose DMA setup functions

[Mikko Perttunen]

ARM DMA domains are currently allocated by the ARM DMA code, with struct
dma_iommu_mapping owning the domain and the IOVA bookkeeping. To allow
the IOMMU core to own DMA domains as is the case with other
architectures, we have to allow that code to allocate the domain and get
the cookie from the ARM DMA code.

Implement ARM versions of iommu_get/put_dma_cookie that create a struct
dma_iommu_mapping that doesn't own its domain, as well as
iommu_setup/teardown_dma_ops that set the device's archdata.mapping,
allowing ARM DMA ops to seamlessly work on these devices.

For core-owned DMA domains, the IOVA bitmap is allocated at
iommu_setup_dma_ops, after the IOMMU driver has been able to establish
domain geometry, matching arm64.

The code is protected by CONFIG_ARM_DMA_USE_IOMMU and mutually exclusive
with the same-named functions in the dma-iommu code.

arm_iommu_* APIs stay functional with DMA default domains, restoring the
default domain when detaching an explicitly created mapping.

Signed-off-by: Mikko Perttunen <mperttunen@xxxxxxxxxx>
---
 arch/arm/include/asm/dma-iommu.h |   9 +++
 arch/arm/mm/dma-mapping.c        | 145 ++++++++++++++++++++++++++++++++++++---
 drivers/iommu/dma-iommu.h        |  50 +++++++++-----
 3 files changed, 177 insertions(+), 27 deletions(-)

diff --git a/arch/arm/include/asm/dma-iommu.h b/arch/arm/include/asm/dma-iommu.h
index 2ce4c5683e6d..d8d37980794f 100644
--- a/arch/arm/include/asm/dma-iommu.h
+++ b/arch/arm/include/asm/dma-iommu.h
@@ -32,5 +32,14 @@ int arm_iommu_attach_device(struct device *dev,
 					struct dma_iommu_mapping *mapping);
 void arm_iommu_detach_device(struct device *dev);
 
+#ifdef CONFIG_ARM_DMA_USE_IOMMU
+struct iommu_domain;
+
+void iommu_setup_dma_ops(struct device *dev, struct iommu_domain *domain);
+void iommu_teardown_dma_ops(struct device *dev);
+int iommu_get_dma_cookie(struct iommu_domain *domain);
+void iommu_put_dma_cookie(struct iommu_domain *domain);
+#endif
+
 #endif /* __KERNEL__ */
 #endif
diff --git a/arch/arm/mm/dma-mapping.c b/arch/arm/mm/dma-mapping.c
index 7ef7c268c583..48b9a9f26095 100644
--- a/arch/arm/mm/dma-mapping.c
+++ b/arch/arm/mm/dma-mapping.c
@@ -1521,6 +1521,96 @@ static void __arm_iommu_free_mapping(struct dma_iommu_mapping *mapping)
 	kfree(mapping);
 }
 
+/**
+ * iommu_get_dma_cookie - Allocate an ARM DMA domain cookie
+ * @domain: the domain
+ */
+int iommu_get_dma_cookie(struct iommu_domain *domain)
+{
+	struct dma_iommu_mapping *mapping;
+
+	if (domain->cookie_type != IOMMU_COOKIE_NONE)
+		return -EEXIST;
+
+	mapping = kzalloc_obj(*mapping);
+	if (!mapping)
+		return -ENOMEM;
+
+	mapping->domain = domain;
+	spin_lock_init(&mapping->lock);
+	domain->cookie_type = IOMMU_COOKIE_ARM_DMA;
+	domain->arm_cookie = mapping;
+
+	return 0;
+}
+
+/**
+ * iommu_put_dma_cookie - Free domain cookie for ARM IOMMU-based DMA API
+ * @domain: the domain being freed
+ */
+void iommu_put_dma_cookie(struct iommu_domain *domain)
+{
+	__arm_iommu_free_mapping(domain->arm_cookie);
+}
+
+/**
+ * iommu_setup_dma_ops - Initialize device's DMA domain cookie and ops
+ * @dev: the device
+ * @domain: the group's new default domain
+ *
+ * Allocates IOVA bookkeeping based on the domain's geometry, which must be
+ * initialized by attaching the domain first. The aperture is limited to within
+ * the 32-bit DMA address space.
+ */
+void iommu_setup_dma_ops(struct device *dev, struct iommu_domain *domain)
+{
+	const struct iommu_domain_geometry *geo = &domain->geometry;
+	struct dma_iommu_mapping *mapping;
+	u64 end, size;
+
+	dev->archdata.dma_ops_setup = false;
+
+	/* Bail out for non-DMA default domains */
+	if (!iommu_is_dma_domain(domain)) {
+		to_dma_iommu_mapping(dev) = NULL;
+		set_dma_ops(dev, NULL);
+		return;
+	}
+
+	mapping = domain->arm_cookie;
+	to_dma_iommu_mapping(dev) = mapping;
+
+	end = min_t(u64, geo->aperture_end, DMA_BIT_MASK(32));
+
+	/* Are we the first in the group to allocate the bitmaps? */
+	if (!mapping->bitmaps) {
+		if (geo->aperture_start > end)
+			goto out_err;
+
+		size = end - geo->aperture_start + 1;
+		if (__arm_iommu_init_mapping(mapping, geo->aperture_start, size))
+			goto out_err;
+	}
+
+	set_dma_ops(dev, &iommu_ops);
+
+	return;
+
+out_err:
+	set_dma_ops(dev, NULL);
+	dev_warn(dev, "Failed to initialize IOMMU DMA mapping; retaining platform DMA ops\n");
+}
+
+/**
+ * iommu_teardown_dma_ops - Clear a device's DMA domain cookie and ops
+ * @dev: the device
+ */
+void iommu_teardown_dma_ops(struct device *dev)
+{
+	to_dma_iommu_mapping(dev) = NULL;
+	set_dma_ops(dev, NULL);
+}
+
 /**
  * arm_iommu_create_mapping
  * @dev: pointer to the client device (for IOMMU calls)
@@ -1531,6 +1621,9 @@ static void __arm_iommu_free_mapping(struct dma_iommu_mapping *mapping)
  * IO address ranges, which is required to perform memory allocation and
  * mapping with IOMMU aware functions.
  *
+ * Unlike a default domain set up by the IOMMU core, the returned mapping owns
+ * the domain it describes and releases it along with the last reference.
+ *
  * The client device need to be attached to the mapping with
  * arm_iommu_attach_device function.
  */
@@ -1594,8 +1687,10 @@ static int extend_iommu_mapping(struct dma_iommu_mapping *mapping)
 
 void arm_iommu_release_mapping(struct dma_iommu_mapping *mapping)
 {
-	if (mapping)
-		kref_put(&mapping->kref, release_iommu_mapping);
+	if (!mapping || iommu_is_dma_domain(mapping->domain))
+		return;
+
+	kref_put(&mapping->kref, release_iommu_mapping);
 }
 EXPORT_SYMBOL_GPL(arm_iommu_release_mapping);
 
@@ -1604,11 +1699,17 @@ static int __arm_iommu_attach_device(struct device *dev,
 {
 	int err;
 
+	/* Avoid attaching if IOVA bookkeeping initialization failed earlier. */
+	if (!mapping->bitmaps)
+		return -EINVAL;
+
 	err = iommu_attach_device(mapping->domain, dev);
 	if (err)
 		return err;
 
-	kref_get(&mapping->kref);
+	if (!iommu_is_dma_domain(mapping->domain))
+		kref_get(&mapping->kref);
+
 	to_dma_iommu_mapping(dev) = mapping;
 
 	pr_debug("Attached IOMMU controller to %s device.\n", dev_name(dev));
@@ -1646,12 +1747,14 @@ EXPORT_SYMBOL_GPL(arm_iommu_attach_device);
  * arm_iommu_detach_device
  * @dev: valid struct device pointer
  *
- * Detaches the provided device from a previously attached map.
- * This overwrites the dma_ops pointer with appropriate non-IOMMU ops.
+ * Detaches an explicitly attached ARM DMA domain and restores the default
+ * domain's DMA cookie and ops. Does nothing if already using a DMA default
+ * domain.
  */
 void arm_iommu_detach_device(struct device *dev)
 {
 	struct dma_iommu_mapping *mapping;
+	struct iommu_domain *def_domain;
 
 	mapping = to_dma_iommu_mapping(dev);
 	if (!mapping) {
@@ -1659,10 +1762,25 @@ void arm_iommu_detach_device(struct device *dev)
 		return;
 	}
 
+	/*
+	 * Allow callers who think they're detaching the legacy UNMANAGED
+	 * domain to work transparently.
+	 */
+	if (iommu_is_dma_domain(mapping->domain))
+		return;
+
 	iommu_detach_device(mapping->domain, dev);
 	kref_put(&mapping->kref, release_iommu_mapping);
-	to_dma_iommu_mapping(dev) = NULL;
-	set_dma_ops(dev, NULL);
+
+	def_domain = iommu_get_domain_for_dev(dev);
+	if (iommu_is_dma_domain(def_domain)) {
+		to_dma_iommu_mapping(dev) = def_domain->arm_cookie;
+		/* Restore NULL ops if default domain initialization failed. */
+		set_dma_ops(dev, def_domain->arm_cookie->bitmaps ? &iommu_ops : NULL);
+	} else {
+		to_dma_iommu_mapping(dev) = NULL;
+		set_dma_ops(dev, NULL);
+	}
 
 	pr_debug("Detached IOMMU controller from %s device.\n", dev_name(dev));
 }
@@ -1717,6 +1835,8 @@ static void arm_teardown_iommu_dma_ops(struct device *dev) { }
 
 void arch_setup_dma_ops(struct device *dev, bool coherent)
 {
+	struct dma_iommu_mapping *mapping;
+
 	/*
 	 * Due to legacy code that sets the dma_coherent flag from a bus
 	 * notifier we can't just assign coherent to the dma_coherent flag
@@ -1727,10 +1847,14 @@ void arch_setup_dma_ops(struct device *dev, bool coherent)
 		dev_set_dma_coherent(dev);
 
 	/*
-	 * Don't override the dma_ops if they have already been set. Ideally
-	 * this should be the only location where dma_ops are set, remove this
-	 * check when all other callers of set_dma_ops will have disappeared.
+	 * For DMA default domains, iommu_setup_dma_ops manages the DMA ops.
+	 * Don't set them here.
 	 */
+	mapping = to_dma_iommu_mapping(dev);
+	if (mapping && iommu_is_dma_domain(mapping->domain))
+		return;
+
+	/* Preserve DMA ops installed by another owner. */
 	if (dev->dma_ops)
 		return;
 
@@ -1749,6 +1873,7 @@ void arch_teardown_dma_ops(struct device *dev)
 	arm_teardown_iommu_dma_ops(dev);
 	/* Let arch_setup_dma_ops() start again from scratch upon re-probe */
 	set_dma_ops(dev, NULL);
+	dev->archdata.dma_ops_setup = false;
 }
 
 void *arch_dma_alloc(struct device *dev, size_t size, dma_addr_t *dma_handle,
diff --git a/drivers/iommu/dma-iommu.h b/drivers/iommu/dma-iommu.h
index 349d6a57e5a0..71b23cb698c0 100644
--- a/drivers/iommu/dma-iommu.h
+++ b/drivers/iommu/dma-iommu.h
@@ -7,25 +7,24 @@
 
 #include <linux/iommu.h>
 
-#ifdef CONFIG_IOMMU_DMA
+/*
+ * Both dma-iommu.c and arch/arm/mm/dma-mapping.c implement functions to set up
+ * IOMMU_DOMAIN_DMA default domains. Only one provider can be compiled in at a
+ * time.
+ */
+#if defined(CONFIG_ARM_DMA_USE_IOMMU)
+
+#include <asm/dma-iommu.h>
+
+#elif defined(CONFIG_IOMMU_DMA)
 
 void iommu_setup_dma_ops(struct device *dev, struct iommu_domain *domain);
 void iommu_teardown_dma_ops(struct device *dev);
 
 int iommu_get_dma_cookie(struct iommu_domain *domain);
 void iommu_put_dma_cookie(struct iommu_domain *domain);
-void iommu_put_msi_cookie(struct iommu_domain *domain);
 
-int iommu_dma_init_fq(struct iommu_domain *domain);
-
-void iommu_dma_get_resv_regions(struct device *dev, struct list_head *list);
-
-int iommu_dma_sw_msi(struct iommu_domain *domain, struct msi_desc *desc,
-		     phys_addr_t msi_addr);
-
-extern bool iommu_dma_forcedac;
-
-#else /* CONFIG_IOMMU_DMA */
+#else
 
 static inline void iommu_setup_dma_ops(struct device *dev,
 				       struct iommu_domain *domain)
@@ -36,11 +35,6 @@ static inline void iommu_teardown_dma_ops(struct device *dev)
 {
 }
 
-static inline int iommu_dma_init_fq(struct iommu_domain *domain)
-{
-	return -EINVAL;
-}
-
 static inline int iommu_get_dma_cookie(struct iommu_domain *domain)
 {
 	return -ENODEV;
@@ -50,6 +44,28 @@ static inline void iommu_put_dma_cookie(struct iommu_domain *domain)
 {
 }
 
+#endif
+
+#ifdef CONFIG_IOMMU_DMA
+
+void iommu_put_msi_cookie(struct iommu_domain *domain);
+
+int iommu_dma_init_fq(struct iommu_domain *domain);
+
+void iommu_dma_get_resv_regions(struct device *dev, struct list_head *list);
+
+int iommu_dma_sw_msi(struct iommu_domain *domain, struct msi_desc *desc,
+		     phys_addr_t msi_addr);
+
+extern bool iommu_dma_forcedac;
+
+#else /* CONFIG_IOMMU_DMA */
+
+static inline int iommu_dma_init_fq(struct iommu_domain *domain)
+{
+	return -EINVAL;
+}
+
 static inline void iommu_put_msi_cookie(struct iommu_domain *domain)
 {
 }

-- 
2.55.0