[PATCH 4/5] iommu: Separate IOVA rcache memories from iova_domain structure

From: John Garry
Date: Fri Sep 24 2021 - 06:07:27 EST

Only dma-iommu.c and vdpa actually use the "fast" mode of IOVA alloc and
free. As such, it's wasteful that all other IOVA domains hold the rcache
memories.

In addition, the current IOVA domain init implementation is poor
(init_iova_domain()), in that errors are ignored and not passed to the
caller. The only errors can come from the IOVA rcache init, and fixing up
all the IOVA domain init callsites to handle the errors would take some
work.

Separate the IOVA rache out of the IOVA domain, and create a new IOVA
domain structure, iova_caching_domain.

Signed-off-by: John Garry <john.garry@xxxxxxxxxx>
---
drivers/iommu/dma-iommu.c | 56 +++++++-----
drivers/iommu/iova.c | 125 ++++++++++++++++++---------
drivers/vdpa/vdpa_user/iova_domain.c | 53 +++++++-----
drivers/vdpa/vdpa_user/iova_domain.h | 4 +-
include/linux/iova.h | 18 ++--
5 files changed, 166 insertions(+), 90 deletions(-)

diff --git a/drivers/iommu/dma-iommu.c b/drivers/iommu/dma-iommu.c
index fd669bad96e1..70651f1a688d 100644
--- a/drivers/iommu/dma-iommu.c
+++ b/drivers/iommu/dma-iommu.c
@@ -92,8 +92,8 @@ struct iommu_dma_cookie {
union {
/* Full allocator for IOMMU_DMA_IOVA_COOKIE */
struct {
- struct iova_domain iovad;
- struct fq_domain fq;
+ struct iova_caching_domain rcached;
+ struct fq_domain fq;
};
/* Trivial linear page allocator for IOMMU_DMA_MSI_COOKIE */
dma_addr_t msi_iova;
@@ -197,7 +197,6 @@ static void fq_ring_free(struct fq_domain *fq_domain, struct fq *fq)
struct iommu_dma_cookie *cookie = container_of(fq_domain,
struct iommu_dma_cookie,
fq);
- struct iova_domain *iovad = &cookie->iovad;
u64 counter = atomic64_read(&fq_domain->fq_flush_finish_cnt);
unsigned idx;

@@ -211,7 +210,7 @@ static void fq_ring_free(struct fq_domain *fq_domain, struct fq *fq)
if (fq_domain->entry_dtor)
fq_domain->entry_dtor(fq->entries[idx].data);

- free_iova_fast(iovad,
+ free_iova_fast(&cookie->rcached,
fq->entries[idx].iova_pfn,
fq->entries[idx].pages);

@@ -330,7 +329,7 @@ static int init_flush_queue(struct fq_domain *fq_domain,
static inline size_t cookie_msi_granule(struct iommu_dma_cookie *cookie)
{
if (cookie->type == IOMMU_DMA_IOVA_COOKIE)
- return cookie->iovad.granule;
+ return cookie->rcached.iovad.granule;
return PAGE_SIZE;
}

@@ -413,9 +412,10 @@ void iommu_put_dma_cookie(struct iommu_domain *domain)
if (!cookie)
return;

- if (cookie->type == IOMMU_DMA_IOVA_COOKIE && cookie->iovad.granule) {
+ if (cookie->type == IOMMU_DMA_IOVA_COOKIE &&
+ cookie->rcached.iovad.granule) {
free_flush_queue(&cookie->fq);
- put_iova_domain(&cookie->iovad);
+ put_iova_caching_domain(&cookie->rcached);
}

list_for_each_entry_safe(msi, tmp, &cookie->msi_page_list, list) {
@@ -449,7 +449,7 @@ EXPORT_SYMBOL(iommu_dma_get_resv_regions);
static int cookie_init_hw_msi_region(struct iommu_dma_cookie *cookie,
phys_addr_t start, phys_addr_t end)
{
- struct iova_domain *iovad = &cookie->iovad;
+ struct iova_domain *iovad = &cookie->rcached.iovad;
struct iommu_dma_msi_page *msi_page;
int i, num_pages;

@@ -520,7 +520,8 @@ static int iova_reserve_iommu_regions(struct device *dev,
struct iommu_domain *domain)
{
struct iommu_dma_cookie *cookie = domain->iova_cookie;
- struct iova_domain *iovad = &cookie->iovad;
+ struct iova_caching_domain *rcached = &cookie->rcached;
+ struct iova_domain *iovad = &rcached->iovad;
struct iommu_resv_region *region;
LIST_HEAD(resv_regions);
int ret = 0;
@@ -612,14 +613,17 @@ static int iommu_dma_init_domain(struct iommu_domain *domain, dma_addr_t base,
dma_addr_t limit, struct device *dev)
{
struct iommu_dma_cookie *cookie = domain->iova_cookie;
+ struct iova_caching_domain *rcached;
unsigned long order, base_pfn;
struct iova_domain *iovad;
struct fq_domain *fq;
+ int ret;

if (!cookie || cookie->type != IOMMU_DMA_IOVA_COOKIE)
return -EINVAL;

- iovad = &cookie->iovad;
+ rcached = &cookie->rcached;
+ iovad = &rcached->iovad;
fq = &cookie->fq;

/* Use the smallest supported page size for IOVA granularity */
@@ -652,7 +656,11 @@ static int iommu_dma_init_domain(struct iommu_domain *domain, dma_addr_t base,
fq->flush_cb = NULL;
fq->fq = NULL;

- init_iova_domain(iovad, 1UL << order, base_pfn);
+ ret = init_iova_caching_domain(rcached, 1UL << order, base_pfn);
+ if (ret) {
+ dev_err(dev, "init_iova_caching_domain failed (%d)\n", ret);
+ return ret;
+ }

/* If the FQ fails we can simply fall back to strict mode */
if (domain->type == IOMMU_DOMAIN_DMA_FQ && iommu_dma_init_fq(domain))
@@ -694,13 +702,16 @@ static dma_addr_t iommu_dma_alloc_iova(struct iommu_domain *domain,
size_t size, u64 dma_limit, struct device *dev)
{
struct iommu_dma_cookie *cookie = domain->iova_cookie;
- struct iova_domain *iovad = &cookie->iovad;
+ struct iova_caching_domain *rcached;
+ struct iova_domain *iovad;
unsigned long shift, iova_len, iova = 0;

if (cookie->type == IOMMU_DMA_MSI_COOKIE) {
cookie->msi_iova += size;
return cookie->msi_iova - size;
}
+ rcached = &cookie->rcached;
+ iovad = &rcached->iovad;

shift = iova_shift(iovad);
iova_len = size >> shift;
@@ -712,11 +723,11 @@ static dma_addr_t iommu_dma_alloc_iova(struct iommu_domain *domain,

/* Try to get PCI devices a SAC address */
if (dma_limit > DMA_BIT_MASK(32) && !iommu_dma_forcedac && dev_is_pci(dev))
- iova = alloc_iova_fast(iovad, iova_len,
+ iova = alloc_iova_fast(rcached, iova_len,
DMA_BIT_MASK(32) >> shift, false);

if (!iova)
- iova = alloc_iova_fast(iovad, iova_len, dma_limit >> shift,
+ iova = alloc_iova_fast(rcached, iova_len, dma_limit >> shift,
true);

return (dma_addr_t)iova << shift;
@@ -725,7 +736,8 @@ static dma_addr_t iommu_dma_alloc_iova(struct iommu_domain *domain,
static void iommu_dma_free_iova(struct iommu_dma_cookie *cookie,
dma_addr_t iova, size_t size, struct iommu_iotlb_gather *gather)
{
- struct iova_domain *iovad = &cookie->iovad;
+ struct iova_caching_domain *rcached = &cookie->rcached;
+ struct iova_domain *iovad = &rcached->iovad;

/* The MSI case is only ever cleaning up its most recent allocation */
if (cookie->type == IOMMU_DMA_MSI_COOKIE) {
@@ -735,7 +747,7 @@ static void iommu_dma_free_iova(struct iommu_dma_cookie *cookie,
size >> iova_shift(iovad),
(unsigned long)gather->freelist);
} else {
- free_iova_fast(iovad, iova_pfn(iovad, iova),
+ free_iova_fast(rcached, iova_pfn(iovad, iova),
size >> iova_shift(iovad));
}
}
@@ -745,7 +757,7 @@ static void __iommu_dma_unmap(struct device *dev, dma_addr_t dma_addr,
{
struct iommu_domain *domain = iommu_get_dma_domain(dev);
struct iommu_dma_cookie *cookie = domain->iova_cookie;
- struct iova_domain *iovad = &cookie->iovad;
+ struct iova_domain *iovad = &cookie->rcached.iovad;
size_t iova_off = iova_offset(iovad, dma_addr);
struct iommu_iotlb_gather iotlb_gather;
size_t unmapped;
@@ -785,7 +797,7 @@ static dma_addr_t __iommu_dma_map(struct device *dev, phys_addr_t phys,
{
struct iommu_domain *domain = iommu_get_dma_domain(dev);
struct iommu_dma_cookie *cookie = domain->iova_cookie;
- struct iova_domain *iovad = &cookie->iovad;
+ struct iova_domain *iovad = &cookie->rcached.iovad;
size_t iova_off = iova_offset(iovad, phys);
dma_addr_t iova;

@@ -813,7 +825,7 @@ static dma_addr_t __iommu_dma_map_swiotlb(struct device *dev, phys_addr_t phys,
int prot = dma_info_to_prot(dir, coherent, attrs);
struct iommu_domain *domain = iommu_get_dma_domain(dev);
struct iommu_dma_cookie *cookie = domain->iova_cookie;
- struct iova_domain *iovad = &cookie->iovad;
+ struct iova_domain *iovad = &cookie->rcached.iovad;
size_t aligned_size = org_size;
void *padding_start;
size_t padding_size;
@@ -924,7 +936,8 @@ static struct page **__iommu_dma_alloc_noncontiguous(struct device *dev,
{
struct iommu_domain *domain = iommu_get_dma_domain(dev);
struct iommu_dma_cookie *cookie = domain->iova_cookie;
- struct iova_domain *iovad = &cookie->iovad;
+ struct iova_caching_domain *rcached = &cookie->rcached;
+ struct iova_domain *iovad = &rcached->iovad;
bool coherent = dev_is_dma_coherent(dev);
int ioprot = dma_info_to_prot(DMA_BIDIRECTIONAL, coherent, attrs);
unsigned int count, min_size, alloc_sizes = domain->pgsize_bitmap;
@@ -1258,7 +1271,8 @@ static int iommu_dma_map_sg(struct device *dev, struct scatterlist *sg,
{
struct iommu_domain *domain = iommu_get_dma_domain(dev);
struct iommu_dma_cookie *cookie = domain->iova_cookie;
- struct iova_domain *iovad = &cookie->iovad;
+ struct iova_caching_domain *rcached = &cookie->rcached;
+ struct iova_domain *iovad = &rcached->iovad;
struct scatterlist *s, *prev = NULL;
int prot = dma_info_to_prot(dir, dev_is_dma_coherent(dev), attrs);
dma_addr_t iova;
diff --git a/drivers/iommu/iova.c b/drivers/iommu/iova.c
index 104fdc9d6c6a..30eb128b1581 100644
--- a/drivers/iommu/iova.c
+++ b/drivers/iommu/iova.c
@@ -15,27 +15,29 @@
/* The anchor node sits above the top of the usable address space */
#define IOVA_ANCHOR ~0UL

-static bool iova_rcache_insert(struct iova_domain *iovad,
+static bool iova_rcache_insert(struct iova_caching_domain *rcached,
unsigned long pfn,
unsigned long size);
-static unsigned long iova_rcache_get(struct iova_domain *iovad,
+static unsigned long iova_rcache_get(struct iova_caching_domain *rcached,
unsigned long size,
unsigned long limit_pfn);
-static void init_iova_rcaches(struct iova_domain *iovad);
-static void free_cpu_cached_iovas(unsigned int cpu, struct iova_domain *iovad);
-static void free_iova_rcaches(struct iova_domain *iovad);
+static int init_iova_rcaches(struct iova_caching_domain *rcached);
+static void free_cpu_cached_iovas(unsigned int cpu,
+ struct iova_caching_domain *rcached);
+static void free_iova_rcaches(struct iova_caching_domain *rcached);

static int iova_cpuhp_dead(unsigned int cpu, struct hlist_node *node)
{
- struct iova_domain *iovad;
+ struct iova_caching_domain *rcached;

- iovad = hlist_entry_safe(node, struct iova_domain, cpuhp_dead);
+ rcached = hlist_entry_safe(node, struct iova_caching_domain,
+ cpuhp_dead);

- free_cpu_cached_iovas(cpu, iovad);
+ free_cpu_cached_iovas(cpu, rcached);
return 0;
}

-static void free_global_cached_iovas(struct iova_domain *iovad);
+static void free_global_cached_iovas(struct iova_caching_domain *rcached);

static struct iova *to_iova(struct rb_node *node)
{
@@ -64,11 +66,32 @@ init_iova_domain(struct iova_domain *iovad, unsigned long granule,
iovad->anchor.pfn_lo = iovad->anchor.pfn_hi = IOVA_ANCHOR;
rb_link_node(&iovad->anchor.node, NULL, &iovad->rbroot.rb_node);
rb_insert_color(&iovad->anchor.node, &iovad->rbroot);
- cpuhp_state_add_instance_nocalls(CPUHP_IOMMU_IOVA_DEAD, &iovad->cpuhp_dead);
- init_iova_rcaches(iovad);
}
EXPORT_SYMBOL_GPL(init_iova_domain);

+int init_iova_caching_domain(struct iova_caching_domain *rcached,
+ unsigned long granule, unsigned long start_pfn)
+{
+ int ret;
+
+ ret = cpuhp_state_add_instance_nocalls(CPUHP_IOMMU_IOVA_DEAD,
+ &rcached->cpuhp_dead);
+ if (ret)
+ return ret;
+
+ ret = init_iova_rcaches(rcached);
+ if (ret) {
+ cpuhp_state_remove_instance_nocalls(CPUHP_IOMMU_IOVA_DEAD,
+ &rcached->cpuhp_dead);
+ return ret;
+ }
+
+ init_iova_domain(&rcached->iovad, granule, start_pfn);
+
+ return 0;
+}
+EXPORT_SYMBOL_GPL(init_iova_caching_domain);
+
static struct rb_node *
__get_cached_rbnode(struct iova_domain *iovad, unsigned long limit_pfn)
{
@@ -422,7 +445,7 @@ EXPORT_SYMBOL_GPL(free_iova);

/**
* alloc_iova_fast - allocates an iova from rcache
- * @iovad: - iova domain in question
+ * @rcached: - iova caching domain in question
* @size: - size of page frames to allocate
* @limit_pfn: - max limit address
* @flush_rcache: - set to flush rcache on regular allocation failure
@@ -431,7 +454,7 @@ EXPORT_SYMBOL_GPL(free_iova);
* fails too and the flush_rcache flag is set then the rcache will be flushed.
*/
unsigned long
-alloc_iova_fast(struct iova_domain *iovad, unsigned long size,
+alloc_iova_fast(struct iova_caching_domain *rcached, unsigned long size,
unsigned long limit_pfn, bool flush_rcache)
{
unsigned long iova_pfn;
@@ -446,12 +469,12 @@ alloc_iova_fast(struct iova_domain *iovad, unsigned long size,
if (size < (1 << (IOVA_RANGE_CACHE_MAX_SIZE - 1)))
size = roundup_pow_of_two(size);

- iova_pfn = iova_rcache_get(iovad, size, limit_pfn + 1);
+ iova_pfn = iova_rcache_get(rcached, size, limit_pfn + 1);
if (iova_pfn)
return iova_pfn;

retry:
- new_iova = alloc_iova(iovad, size, limit_pfn, true);
+ new_iova = alloc_iova(&rcached->iovad, size, limit_pfn, true);
if (!new_iova) {
unsigned int cpu;

@@ -461,8 +484,8 @@ alloc_iova_fast(struct iova_domain *iovad, unsigned long size,
/* Try replenishing IOVAs by flushing rcache. */
flush_rcache = false;
for_each_online_cpu(cpu)
- free_cpu_cached_iovas(cpu, iovad);
- free_global_cached_iovas(iovad);
+ free_cpu_cached_iovas(cpu, rcached);
+ free_global_cached_iovas(rcached);
goto retry;
}

@@ -472,21 +495,22 @@ EXPORT_SYMBOL_GPL(alloc_iova_fast);

/**
* free_iova_fast - free iova pfn range into rcache
- * @iovad: - iova domain in question.
+ * @rcached: - iova caching domain in question.
* @pfn: - pfn that is allocated previously
* @size: - # of pages in range
* This functions frees an iova range by trying to put it into the rcache,
* falling back to regular iova deallocation via free_iova() if this fails.
*/
-void
-free_iova_fast(struct iova_domain *iovad, unsigned long pfn, unsigned long size)
+void free_iova_fast(struct iova_caching_domain *rcached, unsigned long pfn,
+ unsigned long size)
{
- if (iova_rcache_insert(iovad, pfn, size))
+ if (iova_rcache_insert(rcached, pfn, size))
return;

- free_iova(iovad, pfn);
+ free_iova(&rcached->iovad, pfn);
}
EXPORT_SYMBOL_GPL(free_iova_fast);
+
/**
* put_iova_domain - destroys the iova domain
* @iovad: - iova domain in question.
@@ -496,15 +520,23 @@ void put_iova_domain(struct iova_domain *iovad)
{
struct iova *iova, *tmp;

- cpuhp_state_remove_instance_nocalls(CPUHP_IOMMU_IOVA_DEAD,
- &iovad->cpuhp_dead);
-
- free_iova_rcaches(iovad);
rbtree_postorder_for_each_entry_safe(iova, tmp, &iovad->rbroot, node)
free_iova_mem(iova);
}
EXPORT_SYMBOL_GPL(put_iova_domain);

+void put_iova_caching_domain(struct iova_caching_domain *rcached)
+{
+ struct iova_domain *iovad = &rcached->iovad;
+
+ cpuhp_state_remove_instance_nocalls(CPUHP_IOMMU_IOVA_DEAD,
+ &rcached->cpuhp_dead);
+ free_iova_rcaches(rcached);
+
+ put_iova_domain(iovad);
+}
+EXPORT_SYMBOL_GPL(put_iova_caching_domain);
+
static int
__is_range_overlap(struct rb_node *node,
unsigned long pfn_lo, unsigned long pfn_hi)
@@ -693,7 +725,7 @@ static void iova_magazine_push(struct iova_magazine *mag, unsigned long pfn)
mag->pfns[mag->size++] = pfn;
}

-static void init_iova_rcaches(struct iova_domain *iovad)
+static int init_iova_rcaches(struct iova_caching_domain *rcached)
{
struct iova_cpu_rcache *cpu_rcache;
struct iova_rcache *rcache;
@@ -701,12 +733,12 @@ static void init_iova_rcaches(struct iova_domain *iovad)
int i;

for (i = 0; i < IOVA_RANGE_CACHE_MAX_SIZE; ++i) {
- rcache = &iovad->rcaches[i];
+ rcache = &rcached->rcaches[i];
spin_lock_init(&rcache->lock);
rcache->depot_size = 0;
rcache->cpu_rcaches = __alloc_percpu(sizeof(*cpu_rcache), cache_line_size());
- if (WARN_ON(!rcache->cpu_rcaches))
- continue;
+ if (!rcache->cpu_rcaches)
+ goto err;
for_each_possible_cpu(cpu) {
cpu_rcache = per_cpu_ptr(rcache->cpu_rcaches, cpu);
spin_lock_init(&cpu_rcache->lock);
@@ -714,6 +746,12 @@ static void init_iova_rcaches(struct iova_domain *iovad)
cpu_rcache->prev = iova_magazine_alloc(GFP_KERNEL);
}
}
+
+ return 0;
+
+err:
+ free_iova_rcaches(rcached);
+ return -ENOMEM;
}

/*
@@ -722,7 +760,7 @@ static void init_iova_rcaches(struct iova_domain *iovad)
* space, and free_iova() (our only caller) will then return the IOVA
* range to the rbtree instead.
*/
-static bool __iova_rcache_insert(struct iova_domain *iovad,
+static bool __iova_rcache_insert(struct iova_caching_domain *rcached,
struct iova_rcache *rcache,
unsigned long iova_pfn)
{
@@ -763,14 +801,14 @@ static bool __iova_rcache_insert(struct iova_domain *iovad,
spin_unlock_irqrestore(&cpu_rcache->lock, flags);

if (mag_to_free) {
- iova_magazine_free_pfns(mag_to_free, iovad);
+ iova_magazine_free_pfns(mag_to_free, &rcached->iovad);
iova_magazine_free(mag_to_free);
}

return can_insert;
}

-static bool iova_rcache_insert(struct iova_domain *iovad, unsigned long pfn,
+static bool iova_rcache_insert(struct iova_caching_domain *rcached, unsigned long pfn,
unsigned long size)
{
unsigned int log_size = order_base_2(size);
@@ -778,7 +816,7 @@ static bool iova_rcache_insert(struct iova_domain *iovad, unsigned long pfn,
if (log_size >= IOVA_RANGE_CACHE_MAX_SIZE)
return false;

- return __iova_rcache_insert(iovad, &iovad->rcaches[log_size], pfn);
+ return __iova_rcache_insert(rcached, &rcached->rcaches[log_size], pfn);
}

/*
@@ -825,7 +863,7 @@ static unsigned long __iova_rcache_get(struct iova_rcache *rcache,
* size is too big or the DMA limit we are given isn't satisfied by the
* top element in the magazine.
*/
-static unsigned long iova_rcache_get(struct iova_domain *iovad,
+static unsigned long iova_rcache_get(struct iova_caching_domain *rcached,
unsigned long size,
unsigned long limit_pfn)
{
@@ -834,13 +872,13 @@ static unsigned long iova_rcache_get(struct iova_domain *iovad,
if (log_size >= IOVA_RANGE_CACHE_MAX_SIZE)
return 0;

- return __iova_rcache_get(&iovad->rcaches[log_size], limit_pfn - size);
+ return __iova_rcache_get(&rcached->rcaches[log_size], limit_pfn - size);
}

/*
* free rcache data structures.
*/
-static void free_iova_rcaches(struct iova_domain *iovad)
+static void free_iova_rcaches(struct iova_caching_domain *rcached)
{
struct iova_rcache *rcache;
struct iova_cpu_rcache *cpu_rcache;
@@ -848,7 +886,7 @@ static void free_iova_rcaches(struct iova_domain *iovad)
int i, j;

for (i = 0; i < IOVA_RANGE_CACHE_MAX_SIZE; ++i) {
- rcache = &iovad->rcaches[i];
+ rcache = &rcached->rcaches[i];
for_each_possible_cpu(cpu) {
cpu_rcache = per_cpu_ptr(rcache->cpu_rcaches, cpu);
iova_magazine_free(cpu_rcache->loaded);
@@ -863,15 +901,17 @@ static void free_iova_rcaches(struct iova_domain *iovad)
/*
* free all the IOVA ranges cached by a cpu (used when cpu is unplugged)
*/
-static void free_cpu_cached_iovas(unsigned int cpu, struct iova_domain *iovad)
+static void free_cpu_cached_iovas(unsigned int cpu,
+ struct iova_caching_domain *rcached)
{
+ struct iova_domain *iovad = &rcached->iovad;
struct iova_cpu_rcache *cpu_rcache;
struct iova_rcache *rcache;
unsigned long flags;
int i;

for (i = 0; i < IOVA_RANGE_CACHE_MAX_SIZE; ++i) {
- rcache = &iovad->rcaches[i];
+ rcache = &rcached->rcaches[i];
cpu_rcache = per_cpu_ptr(rcache->cpu_rcaches, cpu);
spin_lock_irqsave(&cpu_rcache->lock, flags);
iova_magazine_free_pfns(cpu_rcache->loaded, iovad);
@@ -883,14 +923,15 @@ static void free_cpu_cached_iovas(unsigned int cpu, struct iova_domain *iovad)
/*
* free all the IOVA ranges of global cache
*/
-static void free_global_cached_iovas(struct iova_domain *iovad)
+static void free_global_cached_iovas(struct iova_caching_domain *rcached)
{
+ struct iova_domain *iovad = &rcached->iovad;
struct iova_rcache *rcache;
unsigned long flags;
int i, j;

for (i = 0; i < IOVA_RANGE_CACHE_MAX_SIZE; ++i) {
- rcache = &iovad->rcaches[i];
+ rcache = &rcached->rcaches[i];
spin_lock_irqsave(&rcache->lock, flags);
for (j = 0; j < rcache->depot_size; ++j) {
iova_magazine_free_pfns(rcache->depot[j], iovad);
diff --git a/drivers/vdpa/vdpa_user/iova_domain.c b/drivers/vdpa/vdpa_user/iova_domain.c
index 2b1143f11d8f..d2ffdbf5f29c 100644
--- a/drivers/vdpa/vdpa_user/iova_domain.c
+++ b/drivers/vdpa/vdpa_user/iova_domain.c
@@ -285,25 +285,28 @@ static int vduse_domain_init_bounce_map(struct vduse_iova_domain *domain)
}

static dma_addr_t
-vduse_domain_alloc_iova(struct iova_domain *iovad,
+vduse_domain_alloc_iova(struct iova_caching_domain *rcached,
unsigned long size, unsigned long limit)
{
+ struct iova_domain *iovad = &rcached->iovad;
unsigned long shift = iova_shift(iovad);
unsigned long iova_len = iova_align(iovad, size) >> shift;
unsigned long iova_pfn;

- iova_pfn = alloc_iova_fast(iovad, iova_len, limit >> shift, true);
+ iova_pfn = alloc_iova_fast(rcached, iova_len, limit >> shift, true);

return iova_pfn << shift;
}

-static void vduse_domain_free_iova(struct iova_domain *iovad,
+static void vduse_domain_free_iova(struct iova_caching_domain *rcached,
dma_addr_t iova, size_t size)
{
+ struct iova_domain *iovad = &rcached->iovad;
+
unsigned long shift = iova_shift(iovad);
unsigned long iova_len = iova_align(iovad, size) >> shift;

- free_iova_fast(iovad, iova >> shift, iova_len);
+ free_iova_fast(rcached, iova >> shift, iova_len);
}

dma_addr_t vduse_domain_map_page(struct vduse_iova_domain *domain,
@@ -311,10 +314,10 @@ dma_addr_t vduse_domain_map_page(struct vduse_iova_domain *domain,
size_t size, enum dma_data_direction dir,
unsigned long attrs)
{
- struct iova_domain *iovad = &domain->stream_iovad;
+ struct iova_caching_domain *rcached = &domain->stream_iovad;
unsigned long limit = domain->bounce_size - 1;
phys_addr_t pa = page_to_phys(page) + offset;
- dma_addr_t iova = vduse_domain_alloc_iova(iovad, size, limit);
+ dma_addr_t iova = vduse_domain_alloc_iova(rcached, size, limit);

if (!iova)
return DMA_MAPPING_ERROR;
@@ -330,7 +333,7 @@ dma_addr_t vduse_domain_map_page(struct vduse_iova_domain *domain,

return iova;
err:
- vduse_domain_free_iova(iovad, iova, size);
+ vduse_domain_free_iova(rcached, iova, size);
return DMA_MAPPING_ERROR;
}

@@ -338,22 +341,22 @@ void vduse_domain_unmap_page(struct vduse_iova_domain *domain,
dma_addr_t dma_addr, size_t size,
enum dma_data_direction dir, unsigned long attrs)
{
- struct iova_domain *iovad = &domain->stream_iovad;
+ struct iova_caching_domain *rcached = &domain->stream_iovad;

if (dir == DMA_FROM_DEVICE || dir == DMA_BIDIRECTIONAL)
vduse_domain_bounce(domain, dma_addr, size, DMA_FROM_DEVICE);

vduse_domain_unmap_bounce_page(domain, (u64)dma_addr, (u64)size);
- vduse_domain_free_iova(iovad, dma_addr, size);
+ vduse_domain_free_iova(rcached, dma_addr, size);
}

void *vduse_domain_alloc_coherent(struct vduse_iova_domain *domain,
size_t size, dma_addr_t *dma_addr,
gfp_t flag, unsigned long attrs)
{
- struct iova_domain *iovad = &domain->consistent_iovad;
+ struct iova_caching_domain *rcached = &domain->consistent_iovad;
unsigned long limit = domain->iova_limit;
- dma_addr_t iova = vduse_domain_alloc_iova(iovad, size, limit);
+ dma_addr_t iova = vduse_domain_alloc_iova(rcached, size, limit);
void *orig = alloc_pages_exact(size, flag);

if (!iova || !orig)
@@ -376,7 +379,7 @@ void *vduse_domain_alloc_coherent(struct vduse_iova_domain *domain,
if (orig)
free_pages_exact(orig, size);
if (iova)
- vduse_domain_free_iova(iovad, iova, size);
+ vduse_domain_free_iova(rcached, iova, size);

return NULL;
}
@@ -385,7 +388,7 @@ void vduse_domain_free_coherent(struct vduse_iova_domain *domain, size_t size,
void *vaddr, dma_addr_t dma_addr,
unsigned long attrs)
{
- struct iova_domain *iovad = &domain->consistent_iovad;
+ struct iova_caching_domain *rcached = &domain->consistent_iovad;
struct vhost_iotlb_map *map;
struct vdpa_map_file *map_file;
phys_addr_t pa;
@@ -404,7 +407,7 @@ void vduse_domain_free_coherent(struct vduse_iova_domain *domain, size_t size,
vhost_iotlb_map_free(domain->iotlb, map);
spin_unlock(&domain->iotlb_lock);

- vduse_domain_free_iova(iovad, dma_addr, size);
+ vduse_domain_free_iova(rcached, dma_addr, size);
free_pages_exact(phys_to_virt(pa), size);
}

@@ -453,8 +456,8 @@ static int vduse_domain_release(struct inode *inode, struct file *file)
vduse_iotlb_del_range(domain, 0, ULLONG_MAX);
vduse_domain_free_bounce_pages(domain);
spin_unlock(&domain->iotlb_lock);
- put_iova_domain(&domain->stream_iovad);
- put_iova_domain(&domain->consistent_iovad);
+ put_iova_caching_domain(&domain->stream_iovad);
+ put_iova_caching_domain(&domain->consistent_iovad);
vhost_iotlb_free(domain->iotlb);
vfree(domain->bounce_maps);
kfree(domain);
@@ -480,6 +483,7 @@ vduse_domain_create(unsigned long iova_limit, size_t bounce_size)
struct file *file;
struct vduse_bounce_map *map;
unsigned long pfn, bounce_pfns;
+ int ret;

bounce_pfns = PAGE_ALIGN(bounce_size) >> PAGE_SHIFT;
if (iova_limit <= bounce_size)
@@ -511,12 +515,21 @@ vduse_domain_create(unsigned long iova_limit, size_t bounce_size)

domain->file = file;
spin_lock_init(&domain->iotlb_lock);
- init_iova_domain(&domain->stream_iovad,
- PAGE_SIZE, IOVA_START_PFN);
- init_iova_domain(&domain->consistent_iovad,
- PAGE_SIZE, bounce_pfns);
+ ret = init_iova_caching_domain(&domain->stream_iovad,
+ PAGE_SIZE, IOVA_START_PFN);
+ if (ret)
+ goto err_stream_domain;
+ ret = init_iova_caching_domain(&domain->consistent_iovad,
+ PAGE_SIZE, bounce_pfns);
+ if (ret)
+ goto err_consistent_domain;

return domain;
+
+err_consistent_domain:
+ put_iova_caching_domain(&domain->stream_iovad);
+err_stream_domain:
+ fput(domain->file);
err_file:
vfree(domain->bounce_maps);
err_map:
diff --git a/drivers/vdpa/vdpa_user/iova_domain.h b/drivers/vdpa/vdpa_user/iova_domain.h
index 2722d9b8e21a..38576e1d3b2c 100644
--- a/drivers/vdpa/vdpa_user/iova_domain.h
+++ b/drivers/vdpa/vdpa_user/iova_domain.h
@@ -25,8 +25,8 @@ struct vduse_bounce_map {
};

struct vduse_iova_domain {
- struct iova_domain stream_iovad;
- struct iova_domain consistent_iovad;
+ struct iova_caching_domain stream_iovad;
+ struct iova_caching_domain consistent_iovad;
struct vduse_bounce_map *bounce_maps;
size_t bounce_size;
unsigned long iova_limit;
diff --git a/include/linux/iova.h b/include/linux/iova.h
index ef3b0f8f8a31..858ca7a5f1fa 100644
--- a/include/linux/iova.h
+++ b/include/linux/iova.h
@@ -48,6 +48,10 @@ struct iova_domain {
unsigned long dma_32bit_pfn;
unsigned long max32_alloc_size; /* Size of last failed allocation */
struct iova anchor; /* rbtree lookup anchor */
+};
+
+struct iova_caching_domain {
+ struct iova_domain iovad;
struct iova_rcache rcaches[IOVA_RANGE_CACHE_MAX_SIZE]; /* IOVA range caches */
struct hlist_node cpuhp_dead;
};
@@ -96,16 +100,20 @@ void __free_iova(struct iova_domain *iovad, struct iova *iova);
struct iova *alloc_iova(struct iova_domain *iovad, unsigned long size,
unsigned long limit_pfn,
bool size_aligned);
-void free_iova_fast(struct iova_domain *iovad, unsigned long pfn,
+void free_iova_fast(struct iova_caching_domain *rcached, unsigned long pfn,
unsigned long size);
-unsigned long alloc_iova_fast(struct iova_domain *iovad, unsigned long size,
- unsigned long limit_pfn, bool flush_rcache);
+unsigned long alloc_iova_fast(struct iova_caching_domain *rcached,
+ unsigned long size, unsigned long limit_pfn,
+ bool flush_rcache);
struct iova *reserve_iova(struct iova_domain *iovad, unsigned long pfn_lo,
unsigned long pfn_hi);
void init_iova_domain(struct iova_domain *iovad, unsigned long granule,
unsigned long start_pfn);
struct iova *find_iova(struct iova_domain *iovad, unsigned long pfn);
void put_iova_domain(struct iova_domain *iovad);
+void put_iova_caching_domain(struct iova_caching_domain *rcached);
+int init_iova_caching_domain(struct iova_caching_domain *rcached,
+ unsigned long granule, unsigned long start_pfn);
#else
static inline int iova_cache_get(void)
{
@@ -132,13 +140,13 @@ static inline struct iova *alloc_iova(struct iova_domain *iovad,
return NULL;
}

-static inline void free_iova_fast(struct iova_domain *iovad,
+static inline void free_iova_fast(struct iova_caching_domain *rcached,
unsigned long pfn,
unsigned long size)
{
}

-static inline unsigned long alloc_iova_fast(struct iova_domain *iovad,
+static inline unsigned long alloc_iova_fast(struct iova_caching_domain *rcached,
unsigned long size,
unsigned long limit_pfn,
bool flush_rcache)
--
2.26.2