---
drivers/iommu/dma-iommu.c | 56 +++++++-----
drivers/iommu/iova.c | 125 ++++++++++++++++++---------
drivers/vdpa/vdpa_user/iova_domain.c | 53 +++++++-----
drivers/vdpa/vdpa_user/iova_domain.h | 4 +-
include/linux/iova.h | 18 ++--
5 files changed, 166 insertions(+), 90 deletions(-)
diff --git a/drivers/iommu/dma-iommu.c b/drivers/iommu/dma-iommu.c
index fd669bad96e1..70651f1a688d 100644
--- a/drivers/iommu/dma-iommu.c
+++ b/drivers/iommu/dma-iommu.c
@@ -92,8 +92,8 @@ struct iommu_dma_cookie {
union {
/* Full allocator for IOMMU_DMA_IOVA_COOKIE */
struct {
- struct iova_domain iovad;
- struct fq_domain fq;
+ struct iova_caching_domain rcached;
+ struct fq_domain fq;
};
/* Trivial linear page allocator for IOMMU_DMA_MSI_COOKIE */
dma_addr_t msi_iova;
@@ -197,7 +197,6 @@ static void fq_ring_free(struct fq_domain *fq_domain,
struct fq *fq)
struct iommu_dma_cookie *cookie = container_of(fq_domain,
struct iommu_dma_cookie,
fq);
- struct iova_domain *iovad = &cookie->iovad;
u64 counter = atomic64_read(&fq_domain->fq_flush_finish_cnt);
unsigned idx;
@@ -211,7 +210,7 @@ static void fq_ring_free(struct fq_domain *fq_domain, struct fq *fq)
if (fq_domain->entry_dtor)
fq_domain->entry_dtor(fq->entries[idx].data);
- free_iova_fast(iovad,
+ free_iova_fast(&cookie->rcached,
fq->entries[idx].iova_pfn,
fq->entries[idx].pages);
@@ -330,7 +329,7 @@ static int init_flush_queue(struct fq_domain *fq_domain,
static inline size_t cookie_msi_granule(struct iommu_dma_cookie *cookie)
{
if (cookie->type == IOMMU_DMA_IOVA_COOKIE)
- return cookie->iovad.granule;
+ return cookie->rcached.iovad.granule;
return PAGE_SIZE;
}
@@ -413,9 +412,10 @@ void iommu_put_dma_cookie(struct iommu_domain *domain)
if (!cookie)
return;
- if (cookie->type == IOMMU_DMA_IOVA_COOKIE && cookie->iovad.granule) {
+ if (cookie->type == IOMMU_DMA_IOVA_COOKIE &&
+ cookie->rcached.iovad.granule) {
free_flush_queue(&cookie->fq);
- put_iova_domain(&cookie->iovad);
+ put_iova_caching_domain(&cookie->rcached);
}
list_for_each_entry_safe(msi, tmp, &cookie->msi_page_list, list) {
@@ -449,7 +449,7 @@ EXPORT_SYMBOL(iommu_dma_get_resv_regions);
static int cookie_init_hw_msi_region(struct iommu_dma_cookie *cookie,
phys_addr_t start, phys_addr_t end)
{
- struct iova_domain *iovad = &cookie->iovad;
+ struct iova_domain *iovad = &cookie->rcached.iovad;
struct iommu_dma_msi_page *msi_page;
int i, num_pages;
@@ -520,7 +520,8 @@ static int iova_reserve_iommu_regions(struct device *dev,
struct iommu_domain *domain)
{
struct iommu_dma_cookie *cookie = domain->iova_cookie;
- struct iova_domain *iovad = &cookie->iovad;
+ struct iova_caching_domain *rcached = &cookie->rcached;
+ struct iova_domain *iovad = &rcached->iovad;
struct iommu_resv_region *region;
LIST_HEAD(resv_regions);
int ret = 0;
@@ -612,14 +613,17 @@ static int iommu_dma_init_domain(struct iommu_domain
*domain, dma_addr_t base,
dma_addr_t limit, struct device *dev)
{
struct iommu_dma_cookie *cookie = domain->iova_cookie;
+ struct iova_caching_domain *rcached;
unsigned long order, base_pfn;
struct iova_domain *iovad;
struct fq_domain *fq;
+ int ret;
if (!cookie || cookie->type != IOMMU_DMA_IOVA_COOKIE)
return -EINVAL;
- iovad = &cookie->iovad;
+ rcached = &cookie->rcached;
+ iovad = &rcached->iovad;
fq = &cookie->fq;
/* Use the smallest supported page size for IOVA granularity */
@@ -652,7 +656,11 @@ static int iommu_dma_init_domain(struct iommu_domain
*domain, dma_addr_t base,
fq->flush_cb = NULL;
fq->fq = NULL;
- init_iova_domain(iovad, 1UL << order, base_pfn);
+ ret = init_iova_caching_domain(rcached, 1UL << order, base_pfn);
+ if (ret) {
+ dev_err(dev, "init_iova_caching_domain failed (%d)\n", ret);
+ return ret;
+ }
/* If the FQ fails we can simply fall back to strict mode */
if (domain->type == IOMMU_DOMAIN_DMA_FQ && iommu_dma_init_fq(domain))
@@ -694,13 +702,16 @@ static dma_addr_t iommu_dma_alloc_iova(struct
iommu_domain *domain,
size_t size, u64 dma_limit, struct device *dev)
{
struct iommu_dma_cookie *cookie = domain->iova_cookie;
- struct iova_domain *iovad = &cookie->iovad;
+ struct iova_caching_domain *rcached;
+ struct iova_domain *iovad;
unsigned long shift, iova_len, iova = 0;
if (cookie->type == IOMMU_DMA_MSI_COOKIE) {
cookie->msi_iova += size;
return cookie->msi_iova - size;
}
+ rcached = &cookie->rcached;
+ iovad = &rcached->iovad;
shift = iova_shift(iovad);
iova_len = size >> shift;
@@ -712,11 +723,11 @@ static dma_addr_t iommu_dma_alloc_iova(struct
iommu_domain *domain,
/* Try to get PCI devices a SAC address */
if (dma_limit > DMA_BIT_MASK(32) && !iommu_dma_forcedac &&
dev_is_pci(dev))
- iova = alloc_iova_fast(iovad, iova_len,
+ iova = alloc_iova_fast(rcached, iova_len,
DMA_BIT_MASK(32) >> shift, false);
if (!iova)
- iova = alloc_iova_fast(iovad, iova_len, dma_limit >> shift,
+ iova = alloc_iova_fast(rcached, iova_len, dma_limit >> shift,
true);
return (dma_addr_t)iova << shift;
@@ -725,7 +736,8 @@ static dma_addr_t iommu_dma_alloc_iova(struct iommu_domain
*domain,
static void iommu_dma_free_iova(struct iommu_dma_cookie *cookie,
dma_addr_t iova, size_t size, struct iommu_iotlb_gather *gather)
{
- struct iova_domain *iovad = &cookie->iovad;
+ struct iova_caching_domain *rcached = &cookie->rcached;
+ struct iova_domain *iovad = &rcached->iovad;
/* The MSI case is only ever cleaning up its most recent allocation */
if (cookie->type == IOMMU_DMA_MSI_COOKIE) {
@@ -735,7 +747,7 @@ static void iommu_dma_free_iova(struct iommu_dma_cookie
*cookie,
size >> iova_shift(iovad),
(unsigned long)gather->freelist);
} else {
- free_iova_fast(iovad, iova_pfn(iovad, iova),
+ free_iova_fast(rcached, iova_pfn(iovad, iova),
size >> iova_shift(iovad));
}
}
@@ -745,7 +757,7 @@ static void __iommu_dma_unmap(struct device *dev,
dma_addr_t dma_addr,
{
struct iommu_domain *domain = iommu_get_dma_domain(dev);
struct iommu_dma_cookie *cookie = domain->iova_cookie;
- struct iova_domain *iovad = &cookie->iovad;
+ struct iova_domain *iovad = &cookie->rcached.iovad;
size_t iova_off = iova_offset(iovad, dma_addr);
struct iommu_iotlb_gather iotlb_gather;
size_t unmapped;
@@ -785,7 +797,7 @@ static dma_addr_t __iommu_dma_map(struct device *dev,
phys_addr_t phys,
{
struct iommu_domain *domain = iommu_get_dma_domain(dev);
struct iommu_dma_cookie *cookie = domain->iova_cookie;
- struct iova_domain *iovad = &cookie->iovad;
+ struct iova_domain *iovad = &cookie->rcached.iovad;
size_t iova_off = iova_offset(iovad, phys);
dma_addr_t iova;
@@ -813,7 +825,7 @@ static dma_addr_t __iommu_dma_map_swiotlb(struct device *dev, phys_addr_t phys,
int prot = dma_info_to_prot(dir, coherent, attrs);
struct iommu_domain *domain = iommu_get_dma_domain(dev);
struct iommu_dma_cookie *cookie = domain->iova_cookie;
- struct iova_domain *iovad = &cookie->iovad;
+ struct iova_domain *iovad = &cookie->rcached.iovad;
size_t aligned_size = org_size;
void *padding_start;
size_t padding_size;
@@ -924,7 +936,8 @@ static struct page **__iommu_dma_alloc_noncontiguous(struct
device *dev,
{
struct iommu_domain *domain = iommu_get_dma_domain(dev);
struct iommu_dma_cookie *cookie = domain->iova_cookie;
- struct iova_domain *iovad = &cookie->iovad;
+ struct iova_caching_domain *rcached = &cookie->rcached;
+ struct iova_domain *iovad = &rcached->iovad;
bool coherent = dev_is_dma_coherent(dev);
int ioprot = dma_info_to_prot(DMA_BIDIRECTIONAL, coherent, attrs);
unsigned int count, min_size, alloc_sizes = domain->pgsize_bitmap;
@@ -1258,7 +1271,8 @@ static int iommu_dma_map_sg(struct device *dev, struct
scatterlist *sg,
{
struct iommu_domain *domain = iommu_get_dma_domain(dev);
struct iommu_dma_cookie *cookie = domain->iova_cookie;
- struct iova_domain *iovad = &cookie->iovad;
+ struct iova_caching_domain *rcached = &cookie->rcached;
+ struct iova_domain *iovad = &rcached->iovad;
struct scatterlist *s, *prev = NULL;
int prot = dma_info_to_prot(dir, dev_is_dma_coherent(dev), attrs);
dma_addr_t iova;
diff --git a/drivers/iommu/iova.c b/drivers/iommu/iova.c
index 104fdc9d6c6a..30eb128b1581 100644
--- a/drivers/iommu/iova.c
+++ b/drivers/iommu/iova.c
@@ -15,27 +15,29 @@
/* The anchor node sits above the top of the usable address space */
#define IOVA_ANCHOR ~0UL
-static bool iova_rcache_insert(struct iova_domain *iovad,
+static bool iova_rcache_insert(struct iova_caching_domain *rcached,
unsigned long pfn,
unsigned long size);
-static unsigned long iova_rcache_get(struct iova_domain *iovad,
+static unsigned long iova_rcache_get(struct iova_caching_domain *rcached,
unsigned long size,
unsigned long limit_pfn);
-static void init_iova_rcaches(struct iova_domain *iovad);
-static void free_cpu_cached_iovas(unsigned int cpu, struct iova_domain *iovad);
-static void free_iova_rcaches(struct iova_domain *iovad);
+static int init_iova_rcaches(struct iova_caching_domain *rcached);
+static void free_cpu_cached_iovas(unsigned int cpu,
+ struct iova_caching_domain *rcached);
+static void free_iova_rcaches(struct iova_caching_domain *rcached);
static int iova_cpuhp_dead(unsigned int cpu, struct hlist_node *node)
{
- struct iova_domain *iovad;
+ struct iova_caching_domain *rcached;
- iovad = hlist_entry_safe(node, struct iova_domain, cpuhp_dead);
+ rcached = hlist_entry_safe(node, struct iova_caching_domain,
+ cpuhp_dead);
- free_cpu_cached_iovas(cpu, iovad);
+ free_cpu_cached_iovas(cpu, rcached);
return 0;
}
-static void free_global_cached_iovas(struct iova_domain *iovad);
+static void free_global_cached_iovas(struct iova_caching_domain *rcached);
static struct iova *to_iova(struct rb_node *node)
{
@@ -64,11 +66,32 @@ init_iova_domain(struct iova_domain *iovad, unsigned long
granule,
iovad->anchor.pfn_lo = iovad->anchor.pfn_hi = IOVA_ANCHOR;
rb_link_node(&iovad->anchor.node, NULL, &iovad->rbroot.rb_node);
rb_insert_color(&iovad->anchor.node, &iovad->rbroot);
- cpuhp_state_add_instance_nocalls(CPUHP_IOMMU_IOVA_DEAD,
&iovad->cpuhp_dead);
- init_iova_rcaches(iovad);
}
EXPORT_SYMBOL_GPL(init_iova_domain);
+int init_iova_caching_domain(struct iova_caching_domain *rcached,
+ unsigned long granule, unsigned long start_pfn)
+{
+ int ret;
+
+ ret = cpuhp_state_add_instance_nocalls(CPUHP_IOMMU_IOVA_DEAD,
+ &rcached->cpuhp_dead);
+ if (ret)
+ return ret;
+
+ ret = init_iova_rcaches(rcached);
+ if (ret) {
+ cpuhp_state_remove_instance_nocalls(CPUHP_IOMMU_IOVA_DEAD,
+ &rcached->cpuhp_dead);
+ return ret;
+ }
+
+ init_iova_domain(&rcached->iovad, granule, start_pfn);
+
+ return 0;
+}
+EXPORT_SYMBOL_GPL(init_iova_caching_domain);
+
static struct rb_node *
__get_cached_rbnode(struct iova_domain *iovad, unsigned long limit_pfn)
{
@@ -422,7 +445,7 @@ EXPORT_SYMBOL_GPL(free_iova);
/**
* alloc_iova_fast - allocates an iova from rcache
- * @iovad: - iova domain in question
+ * @rcached: - iova caching domain in question
* @size: - size of page frames to allocate
* @limit_pfn: - max limit address
* @flush_rcache: - set to flush rcache on regular allocation failure
@@ -431,7 +454,7 @@ EXPORT_SYMBOL_GPL(free_iova);
* fails too and the flush_rcache flag is set then the rcache will be flushed.
*/
unsigned long
-alloc_iova_fast(struct iova_domain *iovad, unsigned long size,
+alloc_iova_fast(struct iova_caching_domain *rcached, unsigned long size,
unsigned long limit_pfn, bool flush_rcache)
{
unsigned long iova_pfn;
@@ -446,12 +469,12 @@ alloc_iova_fast(struct iova_domain *iovad, unsigned long
size,
if (size < (1 << (IOVA_RANGE_CACHE_MAX_SIZE - 1)))
size = roundup_pow_of_two(size);
- iova_pfn = iova_rcache_get(iovad, size, limit_pfn + 1);
+ iova_pfn = iova_rcache_get(rcached, size, limit_pfn + 1);
if (iova_pfn)
return iova_pfn;
retry:
- new_iova = alloc_iova(iovad, size, limit_pfn, true);
+ new_iova = alloc_iova(&rcached->iovad, size, limit_pfn, true);
if (!new_iova) {
unsigned int cpu;
@@ -461,8 +484,8 @@ alloc_iova_fast(struct iova_domain *iovad, unsigned long size,
/* Try replenishing IOVAs by flushing rcache. */
flush_rcache = false;
for_each_online_cpu(cpu)
- free_cpu_cached_iovas(cpu, iovad);
- free_global_cached_iovas(iovad);
+ free_cpu_cached_iovas(cpu, rcached);
+ free_global_cached_iovas(rcached);
goto retry;
}
@@ -472,21 +495,22 @@ EXPORT_SYMBOL_GPL(alloc_iova_fast);
/**
* free_iova_fast - free iova pfn range into rcache
- * @iovad: - iova domain in question.
+ * @rcached: - iova caching domain in question.
* @pfn: - pfn that is allocated previously
* @size: - # of pages in range
* This functions frees an iova range by trying to put it into the rcache,
* falling back to regular iova deallocation via free_iova() if this fails.
*/
-void
-free_iova_fast(struct iova_domain *iovad, unsigned long pfn, unsigned long
size)
+void free_iova_fast(struct iova_caching_domain *rcached, unsigned long pfn,
+ unsigned long size)
{
- if (iova_rcache_insert(iovad, pfn, size))
+ if (iova_rcache_insert(rcached, pfn, size))
return;
- free_iova(iovad, pfn);
+ free_iova(&rcached->iovad, pfn);
}
EXPORT_SYMBOL_GPL(free_iova_fast);
+
/**
* put_iova_domain - destroys the iova domain
* @iovad: - iova domain in question.
@@ -496,15 +520,23 @@ void put_iova_domain(struct iova_domain *iovad)
{
struct iova *iova, *tmp;
- cpuhp_state_remove_instance_nocalls(CPUHP_IOMMU_IOVA_DEAD,
- &iovad->cpuhp_dead);
-
- free_iova_rcaches(iovad);
rbtree_postorder_for_each_entry_safe(iova, tmp, &iovad->rbroot, node)
free_iova_mem(iova);
}
EXPORT_SYMBOL_GPL(put_iova_domain);
+void put_iova_caching_domain(struct iova_caching_domain *rcached)
+{
+ struct iova_domain *iovad = &rcached->iovad;
+
+ cpuhp_state_remove_instance_nocalls(CPUHP_IOMMU_IOVA_DEAD,
+ &rcached->cpuhp_dead);
+ free_iova_rcaches(rcached);
+
+ put_iova_domain(iovad);
+}
+EXPORT_SYMBOL_GPL(put_iova_caching_domain);
+
static int
__is_range_overlap(struct rb_node *node,
unsigned long pfn_lo, unsigned long pfn_hi)
@@ -693,7 +725,7 @@ static void iova_magazine_push(struct iova_magazine *mag,
unsigned long pfn)
mag->pfns[mag->size++] = pfn;
}
-static void init_iova_rcaches(struct iova_domain *iovad)
+static int init_iova_rcaches(struct iova_caching_domain *rcached)
{
struct iova_cpu_rcache *cpu_rcache;
struct iova_rcache *rcache;
@@ -701,12 +733,12 @@ static void init_iova_rcaches(struct iova_domain *iovad)
int i;
for (i = 0; i < IOVA_RANGE_CACHE_MAX_SIZE; ++i) {
- rcache = &iovad->rcaches[i];
+ rcache = &rcached->rcaches[i];
spin_lock_init(&rcache->lock);
rcache->depot_size = 0;
rcache->cpu_rcaches = __alloc_percpu(sizeof(*cpu_rcache),
cache_line_size());
- if (WARN_ON(!rcache->cpu_rcaches))
- continue;
+ if (!rcache->cpu_rcaches)
+ goto err;
for_each_possible_cpu(cpu) {
cpu_rcache = per_cpu_ptr(rcache->cpu_rcaches, cpu);
spin_lock_init(&cpu_rcache->lock);
@@ -714,6 +746,12 @@ static void init_iova_rcaches(struct iova_domain *iovad)
cpu_rcache->prev = iova_magazine_alloc(GFP_KERNEL);
}
}
+
+ return 0;
+
+err:
+ free_iova_rcaches(rcached);
+ return -ENOMEM;
}
/*
@@ -722,7 +760,7 @@ static void init_iova_rcaches(struct iova_domain *iovad)
* space, and free_iova() (our only caller) will then return the IOVA
* range to the rbtree instead.
*/
-static bool __iova_rcache_insert(struct iova_domain *iovad,
+static bool __iova_rcache_insert(struct iova_caching_domain *rcached,
struct iova_rcache *rcache,
unsigned long iova_pfn)
{
@@ -763,14 +801,14 @@ static bool __iova_rcache_insert(struct iova_domain
*iovad,
spin_unlock_irqrestore(&cpu_rcache->lock, flags);
if (mag_to_free) {
- iova_magazine_free_pfns(mag_to_free, iovad);
+ iova_magazine_free_pfns(mag_to_free, &rcached->iovad);
iova_magazine_free(mag_to_free);
}
return can_insert;
}
-static bool iova_rcache_insert(struct iova_domain *iovad, unsigned long pfn,
+static bool iova_rcache_insert(struct iova_caching_domain *rcached, unsigned
long pfn,
unsigned long size)
{
unsigned int log_size = order_base_2(size);
@@ -778,7 +816,7 @@ static bool iova_rcache_insert(struct iova_domain *iovad,
unsigned long pfn,
if (log_size >= IOVA_RANGE_CACHE_MAX_SIZE)
return false;
- return __iova_rcache_insert(iovad, &iovad->rcaches[log_size], pfn);
+ return __iova_rcache_insert(rcached, &rcached->rcaches[log_size], pfn);
}
/*
@@ -825,7 +863,7 @@ static unsigned long __iova_rcache_get(struct iova_rcache
*rcache,
* size is too big or the DMA limit we are given isn't satisfied by the
* top element in the magazine.
*/
-static unsigned long iova_rcache_get(struct iova_domain *iovad,
+static unsigned long iova_rcache_get(struct iova_caching_domain *rcached,
unsigned long size,
unsigned long limit_pfn)
{
@@ -834,13 +872,13 @@ static unsigned long iova_rcache_get(struct iova_domain
*iovad,
if (log_size >= IOVA_RANGE_CACHE_MAX_SIZE)
return 0;
- return __iova_rcache_get(&iovad->rcaches[log_size], limit_pfn - size);
+ return __iova_rcache_get(&rcached->rcaches[log_size], limit_pfn - size);
}
/*
* free rcache data structures.
*/
-static void free_iova_rcaches(struct iova_domain *iovad)
+static void free_iova_rcaches(struct iova_caching_domain *rcached)
{
struct iova_rcache *rcache;
struct iova_cpu_rcache *cpu_rcache;
@@ -848,7 +886,7 @@ static void free_iova_rcaches(struct iova_domain *iovad)
int i, j;
for (i = 0; i < IOVA_RANGE_CACHE_MAX_SIZE; ++i) {
- rcache = &iovad->rcaches[i];
+ rcache = &rcached->rcaches[i];
for_each_possible_cpu(cpu) {
cpu_rcache = per_cpu_ptr(rcache->cpu_rcaches, cpu);
iova_magazine_free(cpu_rcache->loaded);
@@ -863,15 +901,17 @@ static void free_iova_rcaches(struct iova_domain *iovad)
/*
* free all the IOVA ranges cached by a cpu (used when cpu is unplugged)
*/
-static void free_cpu_cached_iovas(unsigned int cpu, struct iova_domain *iovad)
+static void free_cpu_cached_iovas(unsigned int cpu,
+ struct iova_caching_domain *rcached)
{
+ struct iova_domain *iovad = &rcached->iovad;
struct iova_cpu_rcache *cpu_rcache;
struct iova_rcache *rcache;
unsigned long flags;
int i;
for (i = 0; i < IOVA_RANGE_CACHE_MAX_SIZE; ++i) {
- rcache = &iovad->rcaches[i];
+ rcache = &rcached->rcaches[i];
cpu_rcache = per_cpu_ptr(rcache->cpu_rcaches, cpu);
spin_lock_irqsave(&cpu_rcache->lock, flags);
iova_magazine_free_pfns(cpu_rcache->loaded, iovad);
@@ -883,14 +923,15 @@ static void free_cpu_cached_iovas(unsigned int cpu,
struct iova_domain *iovad)
/*
* free all the IOVA ranges of global cache
*/
-static void free_global_cached_iovas(struct iova_domain *iovad)
+static void free_global_cached_iovas(struct iova_caching_domain *rcached)
{
+ struct iova_domain *iovad = &rcached->iovad;
struct iova_rcache *rcache;
unsigned long flags;
int i, j;
for (i = 0; i < IOVA_RANGE_CACHE_MAX_SIZE; ++i) {
- rcache = &iovad->rcaches[i];
+ rcache = &rcached->rcaches[i];
spin_lock_irqsave(&rcache->lock, flags);
for (j = 0; j < rcache->depot_size; ++j) {
iova_magazine_free_pfns(rcache->depot[j], iovad);
diff --git a/drivers/vdpa/vdpa_user/iova_domain.c
b/drivers/vdpa/vdpa_user/iova_domain.c
index 2b1143f11d8f..d2ffdbf5f29c 100644
--- a/drivers/vdpa/vdpa_user/iova_domain.c
+++ b/drivers/vdpa/vdpa_user/iova_domain.c
@@ -285,25 +285,28 @@ static int vduse_domain_init_bounce_map(struct
vduse_iova_domain *domain)
}
static dma_addr_t
-vduse_domain_alloc_iova(struct iova_domain *iovad,
+vduse_domain_alloc_iova(struct iova_caching_domain *rcached,
unsigned long size, unsigned long limit)
{
+ struct iova_domain *iovad = &rcached->iovad;
unsigned long shift = iova_shift(iovad);
unsigned long iova_len = iova_align(iovad, size) >> shift;
unsigned long iova_pfn;
- iova_pfn = alloc_iova_fast(iovad, iova_len, limit >> shift, true);
+ iova_pfn = alloc_iova_fast(rcached, iova_len, limit >> shift, true);
return iova_pfn << shift;
}
-static void vduse_domain_free_iova(struct iova_domain *iovad,
+static void vduse_domain_free_iova(struct iova_caching_domain *rcached,
dma_addr_t iova, size_t size)
{
+ struct iova_domain *iovad = &rcached->iovad;
+
unsigned long shift = iova_shift(iovad);
unsigned long iova_len = iova_align(iovad, size) >> shift;
- free_iova_fast(iovad, iova >> shift, iova_len);
+ free_iova_fast(rcached, iova >> shift, iova_len);
}
dma_addr_t vduse_domain_map_page(struct vduse_iova_domain *domain,
@@ -311,10 +314,10 @@ dma_addr_t vduse_domain_map_page(struct vduse_iova_domain
*domain,
size_t size, enum dma_data_direction dir,
unsigned long attrs)
{
- struct iova_domain *iovad = &domain->stream_iovad;
+ struct iova_caching_domain *rcached = &domain->stream_iovad;
unsigned long limit = domain->bounce_size - 1;
phys_addr_t pa = page_to_phys(page) + offset;
- dma_addr_t iova = vduse_domain_alloc_iova(iovad, size, limit);
+ dma_addr_t iova = vduse_domain_alloc_iova(rcached, size, limit);
if (!iova)
return DMA_MAPPING_ERROR;
@@ -330,7 +333,7 @@ dma_addr_t vduse_domain_map_page(struct vduse_iova_domain
*domain,
return iova;
err:
- vduse_domain_free_iova(iovad, iova, size);
+ vduse_domain_free_iova(rcached, iova, size);
return DMA_MAPPING_ERROR;
}
@@ -338,22 +341,22 @@ void vduse_domain_unmap_page(struct vduse_iova_domain *domain,
dma_addr_t dma_addr, size_t size,
enum dma_data_direction dir, unsigned long attrs)
{
- struct iova_domain *iovad = &domain->stream_iovad;
+ struct iova_caching_domain *rcached = &domain->stream_iovad;
if (dir == DMA_FROM_DEVICE || dir == DMA_BIDIRECTIONAL)
vduse_domain_bounce(domain, dma_addr, size, DMA_FROM_DEVICE);
vduse_domain_unmap_bounce_page(domain, (u64)dma_addr, (u64)size);
- vduse_domain_free_iova(iovad, dma_addr, size);
+ vduse_domain_free_iova(rcached, dma_addr, size);
}
void *vduse_domain_alloc_coherent(struct vduse_iova_domain *domain,
size_t size, dma_addr_t *dma_addr,
gfp_t flag, unsigned long attrs)
{
- struct iova_domain *iovad = &domain->consistent_iovad;
+ struct iova_caching_domain *rcached = &domain->consistent_iovad;
unsigned long limit = domain->iova_limit;
- dma_addr_t iova = vduse_domain_alloc_iova(iovad, size, limit);
+ dma_addr_t iova = vduse_domain_alloc_iova(rcached, size, limit);
void *orig = alloc_pages_exact(size, flag);
if (!iova || !orig)
@@ -376,7 +379,7 @@ void *vduse_domain_alloc_coherent(struct vduse_iova_domain
*domain,
if (orig)
free_pages_exact(orig, size);
if (iova)
- vduse_domain_free_iova(iovad, iova, size);
+ vduse_domain_free_iova(rcached, iova, size);
return NULL;
}
@@ -385,7 +388,7 @@ void vduse_domain_free_coherent(struct vduse_iova_domain
*domain, size_t size,
void *vaddr, dma_addr_t dma_addr,
unsigned long attrs)
{
- struct iova_domain *iovad = &domain->consistent_iovad;
+ struct iova_caching_domain *rcached = &domain->consistent_iovad;
struct vhost_iotlb_map *map;
struct vdpa_map_file *map_file;
phys_addr_t pa;
@@ -404,7 +407,7 @@ void vduse_domain_free_coherent(struct vduse_iova_domain
*domain, size_t size,
vhost_iotlb_map_free(domain->iotlb, map);
spin_unlock(&domain->iotlb_lock);
- vduse_domain_free_iova(iovad, dma_addr, size);
+ vduse_domain_free_iova(rcached, dma_addr, size);
free_pages_exact(phys_to_virt(pa), size);
}
@@ -453,8 +456,8 @@ static int vduse_domain_release(struct inode *inode, struct file *file)
vduse_iotlb_del_range(domain, 0, ULLONG_MAX);
vduse_domain_free_bounce_pages(domain);
spin_unlock(&domain->iotlb_lock);
- put_iova_domain(&domain->stream_iovad);
- put_iova_domain(&domain->consistent_iovad);
+ put_iova_caching_domain(&domain->stream_iovad);
+ put_iova_caching_domain(&domain->consistent_iovad);
vhost_iotlb_free(domain->iotlb);
vfree(domain->bounce_maps);
kfree(domain);
@@ -480,6 +483,7 @@ vduse_domain_create(unsigned long iova_limit, size_t
bounce_size)
struct file *file;
struct vduse_bounce_map *map;
unsigned long pfn, bounce_pfns;
+ int ret;
bounce_pfns = PAGE_ALIGN(bounce_size) >> PAGE_SHIFT;
if (iova_limit <= bounce_size)
@@ -511,12 +515,21 @@ vduse_domain_create(unsigned long iova_limit, size_t
bounce_size)
domain->file = file;
spin_lock_init(&domain->iotlb_lock);
- init_iova_domain(&domain->stream_iovad,
- PAGE_SIZE, IOVA_START_PFN);
- init_iova_domain(&domain->consistent_iovad,
- PAGE_SIZE, bounce_pfns);
+ ret = init_iova_caching_domain(&domain->stream_iovad,
+ PAGE_SIZE, IOVA_START_PFN);
+ if (ret)
+ goto err_stream_domain;
+ ret = init_iova_caching_domain(&domain->consistent_iovad,
+ PAGE_SIZE, bounce_pfns);
+ if (ret)
+ goto err_consistent_domain;
return domain;
+
+err_consistent_domain:
+ put_iova_caching_domain(&domain->stream_iovad);
+err_stream_domain:
+ fput(domain->file);
err_file:
vfree(domain->bounce_maps);
err_map:
diff --git a/drivers/vdpa/vdpa_user/iova_domain.h
b/drivers/vdpa/vdpa_user/iova_domain.h
index 2722d9b8e21a..38576e1d3b2c 100644
--- a/drivers/vdpa/vdpa_user/iova_domain.h
+++ b/drivers/vdpa/vdpa_user/iova_domain.h
@@ -25,8 +25,8 @@ struct vduse_bounce_map {
};
struct vduse_iova_domain {
- struct iova_domain stream_iovad;
- struct iova_domain consistent_iovad;
+ struct iova_caching_domain stream_iovad;
+ struct iova_caching_domain consistent_iovad;
struct vduse_bounce_map *bounce_maps;
size_t bounce_size;
unsigned long iova_limit;
diff --git a/include/linux/iova.h b/include/linux/iova.h
index ef3b0f8f8a31..858ca7a5f1fa 100644
--- a/include/linux/iova.h
+++ b/include/linux/iova.h
@@ -48,6 +48,10 @@ struct iova_domain {
unsigned long dma_32bit_pfn;
unsigned long max32_alloc_size; /* Size of last failed allocation */
struct iova anchor; /* rbtree lookup anchor */
+};
+
+struct iova_caching_domain {
+ struct iova_domain iovad;
struct iova_rcache rcaches[IOVA_RANGE_CACHE_MAX_SIZE]; /* IOVA range
caches */
struct hlist_node cpuhp_dead;
};
@@ -96,16 +100,20 @@ void __free_iova(struct iova_domain *iovad, struct iova
*iova);
struct iova *alloc_iova(struct iova_domain *iovad, unsigned long size,
unsigned long limit_pfn,
bool size_aligned);
-void free_iova_fast(struct iova_domain *iovad, unsigned long pfn,
+void free_iova_fast(struct iova_caching_domain *rcached, unsigned long pfn,
unsigned long size);
-unsigned long alloc_iova_fast(struct iova_domain *iovad, unsigned long size,
- unsigned long limit_pfn, bool flush_rcache);
+unsigned long alloc_iova_fast(struct iova_caching_domain *rcached,
+ unsigned long size, unsigned long limit_pfn,
+ bool flush_rcache);
struct iova *reserve_iova(struct iova_domain *iovad, unsigned long pfn_lo,
unsigned long pfn_hi);
void init_iova_domain(struct iova_domain *iovad, unsigned long granule,
unsigned long start_pfn);
struct iova *find_iova(struct iova_domain *iovad, unsigned long pfn);
void put_iova_domain(struct iova_domain *iovad);
+void put_iova_caching_domain(struct iova_caching_domain *rcached);
+int init_iova_caching_domain(struct iova_caching_domain *rcached,
+ unsigned long granule, unsigned long start_pfn);
#else
static inline int iova_cache_get(void)
{
@@ -132,13 +140,13 @@ static inline struct iova *alloc_iova(struct iova_domain
*iovad,
return NULL;
}
-static inline void free_iova_fast(struct iova_domain *iovad,
+static inline void free_iova_fast(struct iova_caching_domain *rcached,
unsigned long pfn,
unsigned long size)
{
}
-static inline unsigned long alloc_iova_fast(struct iova_domain *iovad,
+static inline unsigned long alloc_iova_fast(struct iova_caching_domain
*rcached,
unsigned long size,
unsigned long limit_pfn,
bool flush_rcache)