[PATCH RFC v2 1/1] wiotlb: split buffer into 32-bit default and 64-bit extra zones
From: Dongli Zhang
Date: Sat Aug 20 2022 - 03:44:34 EST
Hello,
I used to send out RFC v1 to introduce an extra io_tlb_mem (created with
SWIOTLB_ANY) in addition to the default io_tlb_mem (32-bit). The
dev->dma_io_tlb_mem is set to either default or the extra io_tlb_mem,
depending on dma mask. However, that is not good for setting
dev->dma_io_tlb_mem at swiotlb layer transparently as suggested by
Christoph Hellwig.
https://lore.kernel.org/all/20220609005553.30954-1-dongli.zhang@xxxxxxxxxx/
Therefore, this is another RFC v2 implementation following a different
direction. The core ideas are:
1. The swiotlb is splited into two zones, io_tlb_mem->zone[0] (32-bit) and
io_tlb_mem->zone[1] (64-bit).
struct io_tlb_mem {
struct io_tlb_zone zone[SWIOTLB_NR];
struct dentry *debugfs;
bool late_alloc;
bool force_bounce;
bool for_alloc;
bool has_extra;
};
struct io_tlb_zone {
phys_addr_t start;
phys_addr_t end;
void *vaddr;
unsigned long nslabs;
unsigned long used;
unsigned int nareas;
unsigned int area_nslabs;
struct io_tlb_area *areas;
struct io_tlb_slot *slots;
};
2. By default, only io_tlb_mem->zone[0] is available. The
io_tlb_mem->zone[1] is allocated conditionally if:
- the "swiotlb=" is configured to allocate extra buffer, and
- the SWIOTLB_EXTRA is set in the flag (this is to make sure arch(s) other
than x86/sev/xen will not enable it until it is fully tested by each
arch, e.g., mips/powerpc). Currently it is enabled for x86 and xen.
3. During swiotlb map, whether zone[0] (32-bit) or zone[1] (64-bit
SWIOTLB_ANY)
is used depends on min_not_zero(*dev->dma_mask, dev->bus_dma_limit).
To test the RFC v2, here is the QEMU command line.
qemu-system-x86_64 -smp 8 -m 32G -enable-kvm -vnc :5 -hda disk.img \
-kernel path-to-linux/arch/x86_64/boot/bzImage \
-append "root=/dev/sda1 init=/sbin/init text console=ttyS0 loglevel=7 swiotlb=32768,4194304,force" \
-net nic -net user,hostfwd=tcp::5025-:22 \
-device nvme,drive=nvme01,serial=helloworld -drive file=test.qcow2,if=none,id=nvme01 \
-serial stdio
There is below in syslog. The extra 8GB buffer is allocated.
[ 0.152251] software IO TLB: area num 8.
... ...
[ 3.706088] PCI-DMA: Using software bounce buffering for IO (SWIOTLB)
[ 3.707334] software IO TLB: mapped default [mem 0x00000000bbfd7000-0x00000000bffd7000] (64MB)
[ 3.708585] software IO TLB: mapped extra [mem 0x000000061cc00000-0x000000081cc00000] (8192MB)
After the FIO is triggered over NVMe, the 64-bit buffer is used.
$ cat /sys/kernel/debug/swiotlb/io_tlb_nslabs_extra
4194304
$ cat /sys/kernel/debug/swiotlb/io_tlb_used_extra
327552
Would you mind helping if this is the right direction to go?
Thank you very much!
Cc: Konrad Wilk <konrad.wilk@xxxxxxxxxx>
Cc: Joe Jin <joe.jin@xxxxxxxxxx>
Signed-off-by: Dongli Zhang <dongli.zhang@xxxxxxxxxx>
---
arch/arm/xen/mm.c | 2 +-
arch/mips/pci/pci-octeon.c | 5 +-
arch/x86/include/asm/xen/swiotlb-xen.h | 2 +-
arch/x86/kernel/pci-dma.c | 6 +-
drivers/xen/swiotlb-xen.c | 18 +-
include/linux/swiotlb.h | 73 +++--
kernel/dma/swiotlb.c | 499 +++++++++++++++++++++------------
7 files changed, 388 insertions(+), 217 deletions(-)
diff --git a/arch/arm/xen/mm.c b/arch/arm/xen/mm.c
index 3d826c0..4edfa42 100644
--- a/arch/arm/xen/mm.c
+++ b/arch/arm/xen/mm.c
@@ -125,7 +125,7 @@ static int __init xen_mm_init(void)
return 0;
/* we can work with the default swiotlb */
- if (!io_tlb_default_mem.nslabs) {
+ if (!io_tlb_default_mem.zone[SWIOTLB_DF].nslabs) {
rc = swiotlb_init_late(swiotlb_size_or_default(),
xen_swiotlb_gfp(), NULL);
if (rc < 0)
diff --git a/arch/mips/pci/pci-octeon.c b/arch/mips/pci/pci-octeon.c
index e457a18..0bf0859 100644
--- a/arch/mips/pci/pci-octeon.c
+++ b/arch/mips/pci/pci-octeon.c
@@ -654,6 +654,9 @@ static int __init octeon_pci_setup(void)
octeon_pci_mem_resource.end =
octeon_pci_mem_resource.start + (1ul << 30);
} else {
+ struct io_tlb_mem *mem = &io_tlb_default_mem;
+ struct io_tlb_zone *zone = &mem->zone[SWIOTLB_DF];
+
/* Remap the Octeon BAR 0 to map 128MB-(128MB+4KB) */
octeon_npi_write32(CVMX_NPI_PCI_CFG04, 128ul << 20);
octeon_npi_write32(CVMX_NPI_PCI_CFG05, 0);
@@ -664,7 +667,7 @@ static int __init octeon_pci_setup(void)
/* BAR1 movable regions contiguous to cover the swiotlb */
octeon_bar1_pci_phys =
- io_tlb_default_mem.start & ~((1ull << 22) - 1);
+ zone->start & ~((1ull << 22) - 1);
for (index = 0; index < 32; index++) {
union cvmx_pci_bar1_indexx bar1_index;
diff --git a/arch/x86/include/asm/xen/swiotlb-xen.h b/arch/x86/include/asm/xen/swiotlb-xen.h
index 77a2d19..9476a9f 100644
--- a/arch/x86/include/asm/xen/swiotlb-xen.h
+++ b/arch/x86/include/asm/xen/swiotlb-xen.h
@@ -8,7 +8,7 @@
static inline int pci_xen_swiotlb_init_late(void) { return -ENXIO; }
#endif
-int xen_swiotlb_fixup(void *buf, unsigned long nslabs);
+int xen_swiotlb_fixup(void *buf, unsigned long nslabs, unsigned int flags);
int xen_create_contiguous_region(phys_addr_t pstart, unsigned int order,
unsigned int address_bits,
dma_addr_t *dma_handle);
diff --git a/arch/x86/kernel/pci-dma.c b/arch/x86/kernel/pci-dma.c
index 30bbe4a..a8736e7 100644
--- a/arch/x86/kernel/pci-dma.c
+++ b/arch/x86/kernel/pci-dma.c
@@ -77,7 +77,7 @@ static void __init pci_xen_swiotlb_init(void)
if (!xen_initial_domain() && !x86_swiotlb_enable)
return;
x86_swiotlb_enable = true;
- x86_swiotlb_flags |= SWIOTLB_ANY;
+ x86_swiotlb_flags |= SWIOTLB_ANY | SWIOTLB_EXTRA;
swiotlb_init_remap(true, x86_swiotlb_flags, xen_swiotlb_fixup);
dma_ops = &xen_swiotlb_dma_ops;
if (IS_ENABLED(CONFIG_PCI))
@@ -90,7 +90,7 @@ int pci_xen_swiotlb_init_late(void)
return 0;
/* we can work with the default swiotlb */
- if (!io_tlb_default_mem.nslabs) {
+ if (!io_tlb_default_mem.zone[SWIOTLB_DF].nslabs) {
int rc = swiotlb_init_late(swiotlb_size_or_default(),
GFP_KERNEL, xen_swiotlb_fixup);
if (rc < 0)
@@ -120,6 +120,8 @@ void __init pci_iommu_alloc(void)
gart_iommu_hole_init();
amd_iommu_detect();
detect_intel_iommu();
+
+ x86_swiotlb_flags |= SWIOTLB_EXTRA;
swiotlb_init(x86_swiotlb_enable, x86_swiotlb_flags);
}
diff --git a/drivers/xen/swiotlb-xen.c b/drivers/xen/swiotlb-xen.c
index 67aa74d..3ff082b 100644
--- a/drivers/xen/swiotlb-xen.c
+++ b/drivers/xen/swiotlb-xen.c
@@ -38,7 +38,6 @@
#include <asm/dma-mapping.h>
#include <trace/events/swiotlb.h>
-#define MAX_DMA_BITS 32
/*
* Quick lookup value of the bus address of the IOTLB.
@@ -104,24 +103,32 @@ static int is_xen_swiotlb_buffer(struct device *dev, dma_addr_t dma_addr)
}
#ifdef CONFIG_X86
-int xen_swiotlb_fixup(void *buf, unsigned long nslabs)
+int xen_swiotlb_fixup(void *buf, unsigned long nslabs, unsigned int flags)
{
int rc;
unsigned int order = get_order(IO_TLB_SEGSIZE << IO_TLB_SHIFT);
unsigned int i, dma_bits = order + PAGE_SHIFT;
dma_addr_t dma_handle;
phys_addr_t p = virt_to_phys(buf);
+ unsigned int max_dma_bits = 32;
BUILD_BUG_ON(IO_TLB_SEGSIZE & (IO_TLB_SEGSIZE - 1));
BUG_ON(nslabs % IO_TLB_SEGSIZE);
+#ifdef CONFIG_X86_64
+ if (flags & SWIOTLB_EXTRA) {
+ max_dma_bits = 64;
+ dma_bits = 64;
+ }
+#endif
+
i = 0;
do {
do {
rc = xen_create_contiguous_region(
p + (i << IO_TLB_SHIFT), order,
dma_bits, &dma_handle);
- } while (rc && dma_bits++ < MAX_DMA_BITS);
+ } while (rc && dma_bits++ < max_dma_bits);
if (rc)
return rc;
@@ -381,7 +388,10 @@ static void xen_swiotlb_unmap_page(struct device *hwdev, dma_addr_t dev_addr,
static int
xen_swiotlb_dma_supported(struct device *hwdev, u64 mask)
{
- return xen_phys_to_dma(hwdev, io_tlb_default_mem.end - 1) <= mask;
+ struct io_tlb_mem *mem = &io_tlb_default_mem;
+ struct io_tlb_zone *zone = &mem->zone[SWIOTLB_DF];
+
+ return xen_phys_to_dma(hwdev, zone->end - 1) <= mask;
}
const struct dma_map_ops xen_swiotlb_dma_ops = {
diff --git a/include/linux/swiotlb.h b/include/linux/swiotlb.h
index 35bc4e2..365e5d7 100644
--- a/include/linux/swiotlb.h
+++ b/include/linux/swiotlb.h
@@ -16,6 +16,11 @@
#define SWIOTLB_VERBOSE (1 << 0) /* verbose initialization */
#define SWIOTLB_FORCE (1 << 1) /* force bounce buffering */
#define SWIOTLB_ANY (1 << 2) /* allow any memory for the buffer */
+#define SWIOTLB_EXTRA (1 << 3) /* allow extra buffer if supported */
+
+#define SWIOTLB_DF 0
+#define SWIOTLB_EX 1
+#define SWIOTLB_NR 2
/*
* Maximum allowable number of contiguous slabs to map,
@@ -36,9 +41,9 @@
unsigned long swiotlb_size_or_default(void);
void __init swiotlb_init_remap(bool addressing_limit, unsigned int flags,
- int (*remap)(void *tlb, unsigned long nslabs));
+ int (*remap)(void *tlb, unsigned long nslabs, unsigned int flags));
int swiotlb_init_late(size_t size, gfp_t gfp_mask,
- int (*remap)(void *tlb, unsigned long nslabs));
+ int (*remap)(void *tlb, unsigned long nslabs, unsigned int flags));
extern void __init swiotlb_update_mem_attributes(void);
phys_addr_t swiotlb_tbl_map_single(struct device *hwdev, phys_addr_t phys,
@@ -62,54 +67,78 @@ dma_addr_t swiotlb_map(struct device *dev, phys_addr_t phys,
#ifdef CONFIG_SWIOTLB
/**
- * struct io_tlb_mem - IO TLB Memory Pool Descriptor
+ * struct io_tlb_zone - IO TLB Memory Zone Descriptor
*
- * @start: The start address of the swiotlb memory pool. Used to do a quick
+ * @start: The start address of the swiotlb memory zone. Used to do a quick
* range check to see if the memory was in fact allocated by this
* API.
- * @end: The end address of the swiotlb memory pool. Used to do a quick
+ * @end: The end address of the swiotlb memory zone. Used to do a quick
* range check to see if the memory was in fact allocated by this
* API.
- * @vaddr: The vaddr of the swiotlb memory pool. The swiotlb memory pool
+ * @vaddr: The vaddr of the swiotlb memory zone. The swiotlb memory zone
* may be remapped in the memory encrypted case and store virtual
* address for bounce buffer operation.
* @nslabs: The number of IO TLB blocks (in groups of 64) between @start and
* @end. For default swiotlb, this is command line adjustable via
* setup_io_tlb_npages.
* @used: The number of used IO TLB block.
- * @list: The free list describing the number of free entries available
- * from each index.
- * @orig_addr: The original address corresponding to a mapped entry.
- * @alloc_size: Size of the allocated buffer.
- * @debugfs: The dentry to debugfs.
- * @late_alloc: %true if allocated using the page allocator
- * @force_bounce: %true if swiotlb bouncing is forced
- * @for_alloc: %true if the pool is used for memory allocation
- * @nareas: The area number in the pool.
- * @area_nslabs: The slot number in the area.
+ * @nareas: The area number in the zone.
+ * @area_nslabs:The slot number in the area.
+ * @area: The IO TLB memory area descriptor.
+ * @slots: The data to track memory allocation.
*/
-struct io_tlb_mem {
+struct io_tlb_zone {
phys_addr_t start;
phys_addr_t end;
void *vaddr;
unsigned long nslabs;
unsigned long used;
- struct dentry *debugfs;
- bool late_alloc;
- bool force_bounce;
- bool for_alloc;
unsigned int nareas;
unsigned int area_nslabs;
struct io_tlb_area *areas;
struct io_tlb_slot *slots;
};
+
+/**
+ * struct io_tlb_mem - IO TLB Memory Pool Descriptor
+ *
+ * @zone: IO TLB memory zone descriptor.
+ * @debugfs: The dentry to debugfs.
+ * @late_alloc: %true if allocated using the page allocator.
+ * @force_bounce: %true if swiotlb bouncing is forced.
+ * @for_alloc: %true if the pool is used for memory allocation.
+ * @has_extra: %true if the extra zone (e.g., 64-bit) is available.
+ */
+struct io_tlb_mem {
+ struct io_tlb_zone zone[SWIOTLB_NR];
+ struct dentry *debugfs;
+ bool late_alloc;
+ bool force_bounce;
+ bool for_alloc;
+ bool has_extra;
+};
extern struct io_tlb_mem io_tlb_default_mem;
static inline bool is_swiotlb_buffer(struct device *dev, phys_addr_t paddr)
{
struct io_tlb_mem *mem = dev->dma_io_tlb_mem;
+ struct io_tlb_zone *zone;
+ int end_zone;
+ int i;
+
+ if (!mem)
+ return false;
+
+ end_zone = mem->has_extra ? SWIOTLB_EX : SWIOTLB_DF;
- return mem && paddr >= mem->start && paddr < mem->end;
+ for (i = end_zone ; i >= 0 ; i--) {
+ zone = &mem->zone[i];
+
+ if (paddr >= zone->start && paddr < zone->end)
+ return true;
+ }
+
+ return false;
}
static inline bool is_swiotlb_force_bounce(struct device *dev)
diff --git a/kernel/dma/swiotlb.c b/kernel/dma/swiotlb.c
index c5a9190..392f21b 100644
--- a/kernel/dma/swiotlb.c
+++ b/kernel/dma/swiotlb.c
@@ -76,6 +76,7 @@ struct io_tlb_slot {
phys_addr_t swiotlb_unencrypted_base;
static unsigned long default_nslabs = IO_TLB_DEFAULT_SIZE >> IO_TLB_SHIFT;
+static unsigned long extra_nslabs;
static unsigned long default_nareas;
/**
@@ -96,24 +97,24 @@ struct io_tlb_area {
/*
* Round up number of slabs to the next power of 2. The last area is going
- * be smaller than the rest if default_nslabs is not power of two.
+ * be smaller than the rest if *nslabs_ptr is not power of two.
* The number of slot in an area should be a multiple of IO_TLB_SEGSIZE,
* otherwise a segment may span two or more areas. It conflicts with free
* contiguous slots tracking: free slots are treated contiguous no matter
* whether they cross an area boundary.
*
- * Return true if default_nslabs is rounded up.
+ * Return true if *nslabs_ptr is rounded up.
*/
-static bool round_up_default_nslabs(void)
+static bool round_up_nslabs_config(unsigned long *nslabs_ptr)
{
if (!default_nareas)
return false;
- if (default_nslabs < IO_TLB_SEGSIZE * default_nareas)
- default_nslabs = IO_TLB_SEGSIZE * default_nareas;
- else if (is_power_of_2(default_nslabs))
+ if (*nslabs_ptr < IO_TLB_SEGSIZE * default_nareas)
+ *nslabs_ptr = IO_TLB_SEGSIZE * default_nareas;
+ else if (is_power_of_2(*nslabs_ptr))
return false;
- default_nslabs = roundup_pow_of_two(default_nslabs);
+ *nslabs_ptr = roundup_pow_of_two(*nslabs_ptr);
return true;
}
@@ -128,9 +129,14 @@ static void swiotlb_adjust_nareas(unsigned int nareas)
default_nareas = nareas;
pr_info("area num %d.\n", nareas);
- if (round_up_default_nslabs())
+
+ if (round_up_nslabs_config(&default_nslabs))
pr_info("SWIOTLB bounce buffer size roundup to %luMB",
(default_nslabs << IO_TLB_SHIFT) >> 20);
+
+ if (extra_nslabs && round_up_nslabs_config(&extra_nslabs))
+ pr_info("SWIOTLB extra bounce buffer size roundup to %luMB",
+ (extra_nslabs << IO_TLB_SHIFT) >> 20);
}
static int __init
@@ -144,6 +150,11 @@ static void swiotlb_adjust_nareas(unsigned int nareas)
if (*str == ',')
++str;
if (isdigit(*str))
+ extra_nslabs =
+ ALIGN(simple_strtoul(str, &str, 0), IO_TLB_SEGSIZE);
+ if (*str == ',')
+ ++str;
+ if (isdigit(*str))
swiotlb_adjust_nareas(simple_strtoul(str, &str, 0));
if (*str == ',')
++str;
@@ -158,9 +169,11 @@ static void swiotlb_adjust_nareas(unsigned int nareas)
unsigned int swiotlb_max_segment(void)
{
- if (!io_tlb_default_mem.nslabs)
+ struct io_tlb_zone *zone = &io_tlb_default_mem.zone[SWIOTLB_DF];
+
+ if (!zone->nslabs)
return 0;
- return rounddown(io_tlb_default_mem.nslabs << IO_TLB_SHIFT, PAGE_SIZE);
+ return rounddown(zone->nslabs << IO_TLB_SHIFT, PAGE_SIZE);
}
EXPORT_SYMBOL_GPL(swiotlb_max_segment);
@@ -181,22 +194,49 @@ void __init swiotlb_adjust_size(unsigned long size)
size = ALIGN(size, IO_TLB_SIZE);
default_nslabs = ALIGN(size >> IO_TLB_SHIFT, IO_TLB_SEGSIZE);
- if (round_up_default_nslabs())
+ if (round_up_nslabs_config(&default_nslabs))
size = default_nslabs << IO_TLB_SHIFT;
- pr_info("SWIOTLB bounce buffer size adjusted to %luMB", size >> 20);
+ pr_info("SWIOTLB default bounce buffer size adjusted to %luMB",
+ size >> 20);
}
void swiotlb_print_info(void)
{
struct io_tlb_mem *mem = &io_tlb_default_mem;
+ int nr_zone = mem->has_extra ? SWIOTLB_NR : SWIOTLB_NR - 1;
+ struct io_tlb_zone *zone;
+ int i;
- if (!mem->nslabs) {
+ if (!mem->zone[SWIOTLB_DF].nslabs) {
pr_warn("No low mem\n");
return;
}
- pr_info("mapped [mem %pa-%pa] (%luMB)\n", &mem->start, &mem->end,
- (mem->nslabs << IO_TLB_SHIFT) >> 20);
+ for (i = 0; i < nr_zone; i++) {
+ zone = &mem->zone[i];
+ pr_info("mapped %s [mem %pa-%pa] (%luMB)\n",
+ i == 0 ? "default" : "extra",
+ &zone->start, &zone->end,
+ (zone->nslabs << IO_TLB_SHIFT) >> 20);
+ }
+}
+
+static int swiotlb_zone_index(struct io_tlb_mem *mem, phys_addr_t paddr)
+{
+ struct io_tlb_zone *zone;
+ int end_zone;
+ int i;
+
+ end_zone = mem->has_extra ? SWIOTLB_EX : SWIOTLB_DF;
+
+ for (i = end_zone ; i >= 0 ; i--) {
+ zone = &mem->zone[i];
+
+ if (paddr >= zone->start && paddr < zone->end)
+ return i;
+ }
+
+ return -EINVAL;
}
static inline unsigned long io_tlb_offset(unsigned long val)
@@ -215,12 +255,13 @@ static inline unsigned long nr_slots(u64 val)
* Isolation VMs).
*/
#ifdef CONFIG_HAS_IOMEM
-static void *swiotlb_mem_remap(struct io_tlb_mem *mem, unsigned long bytes)
+static void *swiotlb_mem_remap(struct io_tlb_zone *zone,
+ unsigned long bytes)
{
void *vaddr = NULL;
if (swiotlb_unencrypted_base) {
- phys_addr_t paddr = mem->start + swiotlb_unencrypted_base;
+ phys_addr_t paddr = zone->start + swiotlb_unencrypted_base;
vaddr = memremap(paddr, bytes, MEMREMAP_WB);
if (!vaddr)
@@ -231,7 +272,8 @@ static void *swiotlb_mem_remap(struct io_tlb_mem *mem, unsigned long bytes)
return vaddr;
}
#else
-static void *swiotlb_mem_remap(struct io_tlb_mem *mem, unsigned long bytes)
+static void *swiotlb_mem_remap(struct io_tlb_zone *zone,
+ unsigned long bytes)
{
return NULL;
}
@@ -246,46 +288,53 @@ static void *swiotlb_mem_remap(struct io_tlb_mem *mem, unsigned long bytes)
void __init swiotlb_update_mem_attributes(void)
{
struct io_tlb_mem *mem = &io_tlb_default_mem;
+ int nr_zone = mem->has_extra ? SWIOTLB_NR : SWIOTLB_NR - 1;
void *vaddr;
unsigned long bytes;
+ int i;
- if (!mem->nslabs || mem->late_alloc)
+ if (mem->late_alloc)
return;
- vaddr = phys_to_virt(mem->start);
- bytes = PAGE_ALIGN(mem->nslabs << IO_TLB_SHIFT);
- set_memory_decrypted((unsigned long)vaddr, bytes >> PAGE_SHIFT);
- mem->vaddr = swiotlb_mem_remap(mem, bytes);
- if (!mem->vaddr)
- mem->vaddr = vaddr;
+ for (i = 0; i < nr_zone; i++) {
+ struct io_tlb_zone *zone = &mem->zone[i];
+
+ if (!zone->nslabs)
+ continue;
+ vaddr = phys_to_virt(zone->start);
+ bytes = PAGE_ALIGN(zone->nslabs << IO_TLB_SHIFT);
+ set_memory_decrypted((unsigned long)vaddr, bytes >> PAGE_SHIFT);
+
+ zone->vaddr = swiotlb_mem_remap(zone, bytes);
+ if (!zone->vaddr)
+ zone->vaddr = vaddr;
+ }
}
-static void swiotlb_init_io_tlb_mem(struct io_tlb_mem *mem, phys_addr_t start,
- unsigned long nslabs, unsigned int flags,
- bool late_alloc, unsigned int nareas)
+static void swiotlb_init_io_tlb_zone(struct io_tlb_zone *zone,
+ phys_addr_t start,
+ unsigned long nslabs,
+ unsigned int nareas)
{
void *vaddr = phys_to_virt(start);
unsigned long bytes = nslabs << IO_TLB_SHIFT, i;
- mem->nslabs = nslabs;
- mem->start = start;
- mem->end = mem->start + bytes;
- mem->late_alloc = late_alloc;
- mem->nareas = nareas;
- mem->area_nslabs = nslabs / mem->nareas;
-
- mem->force_bounce = swiotlb_force_bounce || (flags & SWIOTLB_FORCE);
+ zone->nslabs = nslabs;
+ zone->start = start;
+ zone->end = zone->start + bytes;
+ zone->nareas = nareas;
+ zone->area_nslabs = nslabs / zone->nareas;
- for (i = 0; i < mem->nareas; i++) {
- spin_lock_init(&mem->areas[i].lock);
- mem->areas[i].index = 0;
- mem->areas[i].used = 0;
+ for (i = 0; i < zone->nareas; i++) {
+ spin_lock_init(&zone->areas[i].lock);
+ zone->areas[i].index = 0;
+ zone->areas[i].used = 0;
}
- for (i = 0; i < mem->nslabs; i++) {
- mem->slots[i].list = IO_TLB_SEGSIZE - io_tlb_offset(i);
- mem->slots[i].orig_addr = INVALID_PHYS_ADDR;
- mem->slots[i].alloc_size = 0;
+ for (i = 0; i < zone->nslabs; i++) {
+ zone->slots[i].list = IO_TLB_SEGSIZE - io_tlb_offset(i);
+ zone->slots[i].orig_addr = INVALID_PHYS_ADDR;
+ zone->slots[i].alloc_size = 0;
}
/*
@@ -296,44 +345,19 @@ static void swiotlb_init_io_tlb_mem(struct io_tlb_mem *mem, phys_addr_t start,
return;
memset(vaddr, 0, bytes);
- mem->vaddr = vaddr;
+ zone->vaddr = vaddr;
return;
}
-/*
- * Statically reserve bounce buffer space and initialize bounce buffer data
- * structures for the software IO TLB used to implement the DMA API.
- */
-void __init swiotlb_init_remap(bool addressing_limit, unsigned int flags,
- int (*remap)(void *tlb, unsigned long nslabs))
+void __init swiotlb_init_remap_zone(struct io_tlb_zone *zone,
+ unsigned long nslabs, unsigned int flags,
+ int (*remap)(void *tlb, unsigned long nslabs,
+ unsigned int flags))
{
- struct io_tlb_mem *mem = &io_tlb_default_mem;
- unsigned long nslabs;
size_t alloc_size;
size_t bytes;
void *tlb;
- if (!addressing_limit && !swiotlb_force_bounce)
- return;
- if (swiotlb_force_disable)
- return;
-
- /*
- * default_nslabs maybe changed when adjust area number.
- * So allocate bounce buffer after adjusting area number.
- */
- if (!default_nareas)
- swiotlb_adjust_nareas(num_possible_cpus());
-
- nslabs = default_nslabs;
- if (nslabs < IO_TLB_MIN_SLABS)
- panic("%s: nslabs = %lu too small\n", __func__, nslabs);
-
- /*
- * By default allocate the bounce buffer memory from low memory, but
- * allow to pick a location everywhere for hypervisors with guest
- * memory encryption.
- */
retry:
bytes = PAGE_ALIGN(nslabs << IO_TLB_SHIFT);
if (flags & SWIOTLB_ANY)
@@ -346,7 +370,7 @@ void __init swiotlb_init_remap(bool addressing_limit, unsigned int flags,
return;
}
- if (remap && remap(tlb, nslabs) < 0) {
+ if (remap && remap(tlb, nslabs, flags) < 0) {
memblock_free(tlb, PAGE_ALIGN(bytes));
nslabs = ALIGN(nslabs >> 1, IO_TLB_SEGSIZE);
@@ -356,19 +380,58 @@ void __init swiotlb_init_remap(bool addressing_limit, unsigned int flags,
goto retry;
}
- alloc_size = PAGE_ALIGN(array_size(sizeof(*mem->slots), nslabs));
- mem->slots = memblock_alloc(alloc_size, PAGE_SIZE);
- if (!mem->slots)
+ alloc_size = PAGE_ALIGN(array_size(sizeof(*zone->slots), nslabs));
+ zone->slots = memblock_alloc(alloc_size, PAGE_SIZE);
+ if (!zone->slots)
panic("%s: Failed to allocate %zu bytes align=0x%lx\n",
__func__, alloc_size, PAGE_SIZE);
- mem->areas = memblock_alloc(array_size(sizeof(struct io_tlb_area),
+ zone->areas = memblock_alloc(array_size(sizeof(struct io_tlb_area),
default_nareas), SMP_CACHE_BYTES);
- if (!mem->areas)
- panic("%s: Failed to allocate mem->areas.\n", __func__);
+ if (!zone->areas)
+ panic("%s: Failed to allocate zone->areas.\n", __func__);
+
+ swiotlb_init_io_tlb_zone(zone, __pa(tlb), nslabs, default_nareas);
+}
+
+/*
+ * Statically reserve bounce buffer space and initialize bounce buffer data
+ * structures for the software IO TLB used to implement the DMA API.
+ */
+void __init swiotlb_init_remap(bool addressing_limit, unsigned int flags,
+ int (*remap)(void *tlb, unsigned long nslabs,
+ unsigned int flags))
+{
+ struct io_tlb_mem *mem = &io_tlb_default_mem;
- swiotlb_init_io_tlb_mem(mem, __pa(tlb), nslabs, flags, false,
- default_nareas);
+ if (!addressing_limit && !swiotlb_force_bounce)
+ return;
+ if (swiotlb_force_disable)
+ return;
+
+ /*
+ * Both default_nslabs and extra_nslabs maybe changed when adjust
+ * area number.
+ * So allocate bounce buffer after adjusting area number.
+ */
+ if (!default_nareas)
+ swiotlb_adjust_nareas(num_possible_cpus());
+
+ if (default_nslabs < IO_TLB_MIN_SLABS)
+ panic("%s: default_nslabs = %lu too small\n", __func__,
+ default_nslabs);
+
+ swiotlb_init_remap_zone(&mem->zone[SWIOTLB_DF], default_nslabs,
+ flags & ~SWIOTLB_EXTRA, remap);
+ if (extra_nslabs && (flags & SWIOTLB_EXTRA)) {
+ swiotlb_init_remap_zone(&mem->zone[SWIOTLB_EX],
+ extra_nslabs,
+ flags | SWIOTLB_ANY, remap);
+ mem->has_extra = true;
+ }
+
+ mem->late_alloc = false;
+ mem->force_bounce = swiotlb_force_bounce || (flags & SWIOTLB_FORCE);
if (flags & SWIOTLB_VERBOSE)
swiotlb_print_info();
@@ -385,9 +448,11 @@ void __init swiotlb_init(bool addressing_limit, unsigned int flags)
* This should be just like above, but with some error catching.
*/
int swiotlb_init_late(size_t size, gfp_t gfp_mask,
- int (*remap)(void *tlb, unsigned long nslabs))
+ int (*remap)(void *tlb, unsigned long nslabs,
+ unsigned int flags))
{
struct io_tlb_mem *mem = &io_tlb_default_mem;
+ struct io_tlb_zone *zone = &mem->zone[SWIOTLB_DF];
unsigned long nslabs = ALIGN(size >> IO_TLB_SHIFT, IO_TLB_SEGSIZE);
unsigned char *vstart = NULL;
unsigned int order, area_order;
@@ -415,7 +480,7 @@ int swiotlb_init_late(size_t size, gfp_t gfp_mask,
return -ENOMEM;
if (remap)
- rc = remap(vstart, nslabs);
+ rc = remap(vstart, nslabs, 0);
if (rc) {
free_pages((unsigned long)vstart, order);
@@ -434,28 +499,31 @@ int swiotlb_init_late(size_t size, gfp_t gfp_mask,
if (!default_nareas)
swiotlb_adjust_nareas(num_possible_cpus());
- area_order = get_order(array_size(sizeof(*mem->areas),
+ area_order = get_order(array_size(sizeof(*zone->areas),
default_nareas));
- mem->areas = (struct io_tlb_area *)
+ zone->areas = (struct io_tlb_area *)
__get_free_pages(GFP_KERNEL | __GFP_ZERO, area_order);
- if (!mem->areas)
+ if (!zone->areas)
goto error_area;
- mem->slots = (void *)__get_free_pages(GFP_KERNEL | __GFP_ZERO,
- get_order(array_size(sizeof(*mem->slots), nslabs)));
- if (!mem->slots)
+ zone->slots = (void *)__get_free_pages(GFP_KERNEL | __GFP_ZERO,
+ get_order(array_size(sizeof(*zone->slots), nslabs)));
+ if (!zone->slots)
goto error_slots;
set_memory_decrypted((unsigned long)vstart,
(nslabs << IO_TLB_SHIFT) >> PAGE_SHIFT);
- swiotlb_init_io_tlb_mem(mem, virt_to_phys(vstart), nslabs, 0, true,
- default_nareas);
+ swiotlb_init_io_tlb_zone(zone, virt_to_phys(vstart), nslabs,
+ default_nareas);
+
+ mem->late_alloc = true;
+ mem->force_bounce = swiotlb_force_bounce;
swiotlb_print_info();
return 0;
error_slots:
- free_pages((unsigned long)mem->areas, area_order);
+ free_pages((unsigned long)zone->areas, area_order);
error_area:
free_pages((unsigned long)vstart, order);
return -ENOMEM;
@@ -464,33 +532,40 @@ int swiotlb_init_late(size_t size, gfp_t gfp_mask,
void __init swiotlb_exit(void)
{
struct io_tlb_mem *mem = &io_tlb_default_mem;
+ int nr_zone = mem->has_extra ? SWIOTLB_NR : SWIOTLB_NR - 1;
unsigned long tbl_vaddr;
size_t tbl_size, slots_size;
unsigned int area_order;
+ int i;
if (swiotlb_force_bounce)
return;
- if (!mem->nslabs)
- return;
+ for (i = 0 ; i < nr_zone ; i++) {
+ struct io_tlb_zone *zone = &mem->zone[i];
- pr_info("tearing down default memory pool\n");
- tbl_vaddr = (unsigned long)phys_to_virt(mem->start);
- tbl_size = PAGE_ALIGN(mem->end - mem->start);
- slots_size = PAGE_ALIGN(array_size(sizeof(*mem->slots), mem->nslabs));
-
- set_memory_encrypted(tbl_vaddr, tbl_size >> PAGE_SHIFT);
- if (mem->late_alloc) {
- area_order = get_order(array_size(sizeof(*mem->areas),
- mem->nareas));
- free_pages((unsigned long)mem->areas, area_order);
- free_pages(tbl_vaddr, get_order(tbl_size));
- free_pages((unsigned long)mem->slots, get_order(slots_size));
- } else {
- memblock_free_late(__pa(mem->areas),
- array_size(sizeof(*mem->areas), mem->nareas));
- memblock_free_late(mem->start, tbl_size);
- memblock_free_late(__pa(mem->slots), slots_size);
+ if (!zone->nslabs)
+ continue;
+
+ pr_info("tearing down %s memory pool\n",
+ i == 0 ? "default" : "extra");
+ tbl_vaddr = (unsigned long)phys_to_virt(zone->start);
+ tbl_size = PAGE_ALIGN(zone->end - zone->start);
+ slots_size = PAGE_ALIGN(array_size(sizeof(*zone->slots), zone->nslabs));
+
+ set_memory_encrypted(tbl_vaddr, tbl_size >> PAGE_SHIFT);
+ if (mem->late_alloc) {
+ area_order = get_order(array_size(sizeof(*zone->areas),
+ zone->nareas));
+ free_pages((unsigned long)zone->areas, area_order);
+ free_pages(tbl_vaddr, get_order(tbl_size));
+ free_pages((unsigned long)zone->slots, get_order(slots_size));
+ } else {
+ memblock_free_late(__pa(zone->areas),
+ array_size(sizeof(*zone->areas), zone->nareas));
+ memblock_free_late(zone->start, tbl_size);
+ memblock_free_late(__pa(zone->slots), slots_size);
+ }
}
memset(mem, 0, sizeof(*mem));
@@ -507,15 +582,15 @@ static unsigned int swiotlb_align_offset(struct device *dev, u64 addr)
/*
* Bounce: copy the swiotlb buffer from or back to the original dma location
*/
-static void swiotlb_bounce(struct device *dev, phys_addr_t tlb_addr, size_t size,
+static void swiotlb_bounce(struct device *dev, struct io_tlb_zone *zone,
+ phys_addr_t tlb_addr, size_t size,
enum dma_data_direction dir)
{
- struct io_tlb_mem *mem = dev->dma_io_tlb_mem;
- int index = (tlb_addr - mem->start) >> IO_TLB_SHIFT;
- phys_addr_t orig_addr = mem->slots[index].orig_addr;
- size_t alloc_size = mem->slots[index].alloc_size;
+ int index = (tlb_addr - zone->start) >> IO_TLB_SHIFT;
+ phys_addr_t orig_addr = zone->slots[index].orig_addr;
+ size_t alloc_size = zone->slots[index].alloc_size;
unsigned long pfn = PFN_DOWN(orig_addr);
- unsigned char *vaddr = mem->vaddr + tlb_addr - mem->start;
+ unsigned char *vaddr = zone->vaddr + tlb_addr - zone->start;
unsigned int tlb_offset, orig_addr_offset;
if (orig_addr == INVALID_PHYS_ADDR)
@@ -579,7 +654,10 @@ static void swiotlb_bounce(struct device *dev, phys_addr_t tlb_addr, size_t size
}
}
-#define slot_addr(start, idx) ((start) + ((idx) << IO_TLB_SHIFT))
+static inline phys_addr_t slot_addr(phys_addr_t start, phys_addr_t idx)
+{
+ return start + (idx << IO_TLB_SHIFT);
+}
/*
* Carefully handle integer overflow which can occur when boundary_mask == ~0UL.
@@ -591,9 +669,10 @@ static inline unsigned long get_max_slots(unsigned long boundary_mask)
return nr_slots(boundary_mask + 1);
}
-static unsigned int wrap_area_index(struct io_tlb_mem *mem, unsigned int index)
+static unsigned int wrap_area_index(struct io_tlb_zone *zone,
+ unsigned int index)
{
- if (index >= mem->area_nslabs)
+ if (index >= zone->area_nslabs)
return 0;
return index;
}
@@ -602,15 +681,15 @@ static unsigned int wrap_area_index(struct io_tlb_mem *mem, unsigned int index)
* Find a suitable number of IO TLB entries size that will fit this request and
* allocate a buffer from that IO TLB pool.
*/
-static int swiotlb_do_find_slots(struct device *dev, int area_index,
+static int swiotlb_do_find_slots(struct device *dev,
+ struct io_tlb_zone *zone, int area_index,
phys_addr_t orig_addr, size_t alloc_size,
unsigned int alloc_align_mask)
{
- struct io_tlb_mem *mem = dev->dma_io_tlb_mem;
- struct io_tlb_area *area = mem->areas + area_index;
+ struct io_tlb_area *area = zone->areas + area_index;
unsigned long boundary_mask = dma_get_seg_boundary(dev);
dma_addr_t tbl_dma_addr =
- phys_to_dma_unencrypted(dev, mem->start) & boundary_mask;
+ phys_to_dma_unencrypted(dev, zone->start) & boundary_mask;
unsigned long max_slots = get_max_slots(boundary_mask);
unsigned int iotlb_align_mask =
dma_get_min_align_mask(dev) & ~(IO_TLB_SIZE - 1);
@@ -622,7 +701,7 @@ static int swiotlb_do_find_slots(struct device *dev, int area_index,
unsigned int slot_index;
BUG_ON(!nslots);
- BUG_ON(area_index >= mem->nareas);
+ BUG_ON(area_index >= zone->nareas);
/*
* For mappings with an alignment requirement don't bother looping to
@@ -635,11 +714,11 @@ static int swiotlb_do_find_slots(struct device *dev, int area_index,
stride = max(stride, (alloc_align_mask >> IO_TLB_SHIFT) + 1);
spin_lock_irqsave(&area->lock, flags);
- if (unlikely(nslots > mem->area_nslabs - area->used))
+ if (unlikely(nslots > zone->area_nslabs - area->used))
goto not_found;
- slot_base = area_index * mem->area_nslabs;
- index = wrap = wrap_area_index(mem, ALIGN(area->index, stride));
+ slot_base = area_index * zone->area_nslabs;
+ index = wrap = wrap_area_index(zone, ALIGN(area->index, stride));
do {
slot_index = slot_base + index;
@@ -647,7 +726,7 @@ static int swiotlb_do_find_slots(struct device *dev, int area_index,
if (orig_addr &&
(slot_addr(tbl_dma_addr, slot_index) &
iotlb_align_mask) != (orig_addr & iotlb_align_mask)) {
- index = wrap_area_index(mem, index + 1);
+ index = wrap_area_index(zone, index + 1);
continue;
}
@@ -659,10 +738,10 @@ static int swiotlb_do_find_slots(struct device *dev, int area_index,
if (!iommu_is_span_boundary(slot_index, nslots,
nr_slots(tbl_dma_addr),
max_slots)) {
- if (mem->slots[slot_index].list >= nslots)
+ if (zone->slots[slot_index].list >= nslots)
goto found;
}
- index = wrap_area_index(mem, index + stride);
+ index = wrap_area_index(zone, index + stride);
} while (index != wrap);
not_found:
@@ -671,19 +750,19 @@ static int swiotlb_do_find_slots(struct device *dev, int area_index,
found:
for (i = slot_index; i < slot_index + nslots; i++) {
- mem->slots[i].list = 0;
- mem->slots[i].alloc_size = alloc_size - (offset +
+ zone->slots[i].list = 0;
+ zone->slots[i].alloc_size = alloc_size - (offset +
((i - slot_index) << IO_TLB_SHIFT));
}
for (i = slot_index - 1;
io_tlb_offset(i) != IO_TLB_SEGSIZE - 1 &&
- mem->slots[i].list; i--)
- mem->slots[i].list = ++count;
+ zone->slots[i].list; i--)
+ zone->slots[i].list = ++count;
/*
* Update the indices to avoid searching in the next round.
*/
- if (index + nslots < mem->area_nslabs)
+ if (index + nslots < zone->area_nslabs)
area->index = index + nslots;
else
area->index = 0;
@@ -692,32 +771,33 @@ static int swiotlb_do_find_slots(struct device *dev, int area_index,
return slot_index;
}
-static int swiotlb_find_slots(struct device *dev, phys_addr_t orig_addr,
+static int swiotlb_find_slots(struct device *dev,
+ struct io_tlb_zone *zone, phys_addr_t orig_addr,
size_t alloc_size, unsigned int alloc_align_mask)
{
- struct io_tlb_mem *mem = dev->dma_io_tlb_mem;
- int start = raw_smp_processor_id() & (mem->nareas - 1);
+ int start = raw_smp_processor_id() & (zone->nareas - 1);
int i = start, index;
do {
- index = swiotlb_do_find_slots(dev, i, orig_addr, alloc_size,
+ index = swiotlb_do_find_slots(dev, zone, i,
+ orig_addr, alloc_size,
alloc_align_mask);
if (index >= 0)
return index;
- if (++i >= mem->nareas)
+ if (++i >= zone->nareas)
i = 0;
} while (i != start);
return -1;
}
-static unsigned long mem_used(struct io_tlb_mem *mem)
+static unsigned long mem_used(struct io_tlb_zone *zone)
{
int i;
unsigned long used = 0;
- for (i = 0; i < mem->nareas; i++)
- used += mem->areas[i].used;
+ for (i = 0; i < zone->nareas; i++)
+ used += zone->areas[i].used;
return used;
}
@@ -728,11 +808,13 @@ phys_addr_t swiotlb_tbl_map_single(struct device *dev, phys_addr_t orig_addr,
{
struct io_tlb_mem *mem = dev->dma_io_tlb_mem;
unsigned int offset = swiotlb_align_offset(dev, orig_addr);
+ struct io_tlb_zone *zone;
unsigned int i;
int index;
phys_addr_t tlb_addr;
+ unsigned long dma_mask;
- if (!mem || !mem->nslabs)
+ if (!mem || !mem->zone[SWIOTLB_DF].nslabs)
panic("Can not allocate SWIOTLB buffer earlier and can't now provide you with the DMA bounce buffer");
if (cc_platform_has(CC_ATTR_MEM_ENCRYPT))
@@ -744,13 +826,20 @@ phys_addr_t swiotlb_tbl_map_single(struct device *dev, phys_addr_t orig_addr,
return (phys_addr_t)DMA_MAPPING_ERROR;
}
- index = swiotlb_find_slots(dev, orig_addr,
+ dma_mask = min_not_zero(*dev->dma_mask, dev->bus_dma_limit);
+ if (mem->has_extra && mem->zone[SWIOTLB_EX].nslabs &&
+ dma_mask == DMA_BIT_MASK(64))
+ zone = &mem->zone[SWIOTLB_EX];
+ else
+ zone = &mem->zone[SWIOTLB_DF];
+
+ index = swiotlb_find_slots(dev, zone, orig_addr,
alloc_size + offset, alloc_align_mask);
if (index == -1) {
if (!(attrs & DMA_ATTR_NO_WARN))
dev_warn_ratelimited(dev,
"swiotlb buffer is full (sz: %zd bytes), total %lu (slots), used %lu (slots)\n",
- alloc_size, mem->nslabs, mem_used(mem));
+ alloc_size, zone->nslabs, mem_used(zone));
return (phys_addr_t)DMA_MAPPING_ERROR;
}
@@ -760,8 +849,8 @@ phys_addr_t swiotlb_tbl_map_single(struct device *dev, phys_addr_t orig_addr,
* needed.
*/
for (i = 0; i < nr_slots(alloc_size + offset); i++)
- mem->slots[index + i].orig_addr = slot_addr(orig_addr, i);
- tlb_addr = slot_addr(mem->start, index) + offset;
+ zone->slots[index + i].orig_addr = slot_addr(orig_addr, i);
+ tlb_addr = slot_addr(zone->start, index) + offset;
/*
* When dir == DMA_FROM_DEVICE we could omit the copy from the orig
* to the tlb buffer, if we knew for sure the device will
@@ -769,19 +858,19 @@ phys_addr_t swiotlb_tbl_map_single(struct device *dev, phys_addr_t orig_addr,
* unconditional bounce may prevent leaking swiotlb content (i.e.
* kernel memory) to user-space.
*/
- swiotlb_bounce(dev, tlb_addr, mapping_size, DMA_TO_DEVICE);
+ swiotlb_bounce(dev, zone, tlb_addr, mapping_size, DMA_TO_DEVICE);
return tlb_addr;
}
-static void swiotlb_release_slots(struct device *dev, phys_addr_t tlb_addr)
+static void swiotlb_release_slots(struct device *dev,
+ struct io_tlb_zone *zone, phys_addr_t tlb_addr)
{
- struct io_tlb_mem *mem = dev->dma_io_tlb_mem;
unsigned long flags;
unsigned int offset = swiotlb_align_offset(dev, tlb_addr);
- int index = (tlb_addr - offset - mem->start) >> IO_TLB_SHIFT;
- int nslots = nr_slots(mem->slots[index].alloc_size + offset);
- int aindex = index / mem->area_nslabs;
- struct io_tlb_area *area = &mem->areas[aindex];
+ int index = (tlb_addr - offset - zone->start) >> IO_TLB_SHIFT;
+ int nslots = nr_slots(zone->slots[index].alloc_size + offset);
+ int aindex = index / zone->area_nslabs;
+ struct io_tlb_area *area = &zone->areas[aindex];
int count, i;
/*
@@ -790,11 +879,11 @@ static void swiotlb_release_slots(struct device *dev, phys_addr_t tlb_addr)
* While returning the entries to the free list, we merge the entries
* with slots below and above the pool being returned.
*/
- BUG_ON(aindex >= mem->nareas);
+ BUG_ON(aindex >= zone->nareas);
spin_lock_irqsave(&area->lock, flags);
if (index + nslots < ALIGN(index + 1, IO_TLB_SEGSIZE))
- count = mem->slots[index + nslots].list;
+ count = zone->slots[index + nslots].list;
else
count = 0;
@@ -803,9 +892,9 @@ static void swiotlb_release_slots(struct device *dev, phys_addr_t tlb_addr)
* superceeding slots
*/
for (i = index + nslots - 1; i >= index; i--) {
- mem->slots[i].list = ++count;
- mem->slots[i].orig_addr = INVALID_PHYS_ADDR;
- mem->slots[i].alloc_size = 0;
+ zone->slots[i].list = ++count;
+ zone->slots[i].orig_addr = INVALID_PHYS_ADDR;
+ zone->slots[i].alloc_size = 0;
}
/*
@@ -813,9 +902,9 @@ static void swiotlb_release_slots(struct device *dev, phys_addr_t tlb_addr)
* available (non zero)
*/
for (i = index - 1;
- io_tlb_offset(i) != IO_TLB_SEGSIZE - 1 && mem->slots[i].list;
+ io_tlb_offset(i) != IO_TLB_SEGSIZE - 1 && zone->slots[i].list;
i--)
- mem->slots[i].list = ++count;
+ zone->slots[i].list = ++count;
area->used -= nslots;
spin_unlock_irqrestore(&area->lock, flags);
}
@@ -827,21 +916,30 @@ void swiotlb_tbl_unmap_single(struct device *dev, phys_addr_t tlb_addr,
size_t mapping_size, enum dma_data_direction dir,
unsigned long attrs)
{
+ struct io_tlb_mem *mem = dev->dma_io_tlb_mem;
+ int zone_index = swiotlb_zone_index(mem, tlb_addr);
+ struct io_tlb_zone *zone = &mem->zone[zone_index];
+
/*
* First, sync the memory before unmapping the entry
*/
if (!(attrs & DMA_ATTR_SKIP_CPU_SYNC) &&
(dir == DMA_FROM_DEVICE || dir == DMA_BIDIRECTIONAL))
- swiotlb_bounce(dev, tlb_addr, mapping_size, DMA_FROM_DEVICE);
+ swiotlb_bounce(dev, zone, tlb_addr, mapping_size,
+ DMA_FROM_DEVICE);
- swiotlb_release_slots(dev, tlb_addr);
+ swiotlb_release_slots(dev, zone, tlb_addr);
}
void swiotlb_sync_single_for_device(struct device *dev, phys_addr_t tlb_addr,
size_t size, enum dma_data_direction dir)
{
+ struct io_tlb_mem *mem = dev->dma_io_tlb_mem;
+ int zone_index = swiotlb_zone_index(mem, tlb_addr);
+ struct io_tlb_zone *zone = &mem->zone[zone_index];
+
if (dir == DMA_TO_DEVICE || dir == DMA_BIDIRECTIONAL)
- swiotlb_bounce(dev, tlb_addr, size, DMA_TO_DEVICE);
+ swiotlb_bounce(dev, zone, tlb_addr, size, DMA_TO_DEVICE);
else
BUG_ON(dir != DMA_FROM_DEVICE);
}
@@ -849,8 +947,12 @@ void swiotlb_sync_single_for_device(struct device *dev, phys_addr_t tlb_addr,
void swiotlb_sync_single_for_cpu(struct device *dev, phys_addr_t tlb_addr,
size_t size, enum dma_data_direction dir)
{
+ struct io_tlb_mem *mem = dev->dma_io_tlb_mem;
+ int zone_index = swiotlb_zone_index(mem, tlb_addr);
+ struct io_tlb_zone *zone = &mem->zone[zone_index];
+
if (dir == DMA_FROM_DEVICE || dir == DMA_BIDIRECTIONAL)
- swiotlb_bounce(dev, tlb_addr, size, DMA_FROM_DEVICE);
+ swiotlb_bounce(dev, zone, tlb_addr, size, DMA_FROM_DEVICE);
else
BUG_ON(dir != DMA_TO_DEVICE);
}
@@ -908,13 +1010,14 @@ bool is_swiotlb_active(struct device *dev)
{
struct io_tlb_mem *mem = dev->dma_io_tlb_mem;
- return mem && mem->nslabs;
+ return mem && mem->zone[SWIOTLB_DF].nslabs;
}
EXPORT_SYMBOL_GPL(is_swiotlb_active);
static int io_tlb_used_get(void *data, u64 *val)
{
- *val = mem_used(&io_tlb_default_mem);
+ struct io_tlb_zone *zone = (struct io_tlb_zone *) data;
+ *val = mem_used(zone);
return 0;
}
DEFINE_DEBUGFS_ATTRIBUTE(fops_io_tlb_used, io_tlb_used_get, NULL, "%llu\n");
@@ -922,13 +1025,26 @@ static int io_tlb_used_get(void *data, u64 *val)
static void swiotlb_create_debugfs_files(struct io_tlb_mem *mem,
const char *dirname)
{
+ struct io_tlb_zone *zone;
+
mem->debugfs = debugfs_create_dir(dirname, io_tlb_default_mem.debugfs);
- if (!mem->nslabs)
- return;
- debugfs_create_ulong("io_tlb_nslabs", 0400, mem->debugfs, &mem->nslabs);
- debugfs_create_file("io_tlb_used", 0400, mem->debugfs, NULL,
- &fops_io_tlb_used);
+ zone = &mem->zone[SWIOTLB_DF];
+ if (zone->nslabs) {
+ debugfs_create_ulong("io_tlb_nslabs", 0400,
+ mem->debugfs, &zone->nslabs);
+ debugfs_create_file("io_tlb_used", 0400, mem->debugfs,
+ zone, &fops_io_tlb_used);
+ }
+
+ if (mem->has_extra) {
+ zone = &mem->zone[SWIOTLB_EX];
+
+ debugfs_create_ulong("io_tlb_nslabs_extra", 0400,
+ mem->debugfs, &zone->nslabs);
+ debugfs_create_file("io_tlb_used_extra", 0400,
+ mem->debugfs, zone, &fops_io_tlb_used);
+ }
}
static int __init __maybe_unused swiotlb_create_default_debugfs(void)
@@ -946,17 +1062,20 @@ static int __init __maybe_unused swiotlb_create_default_debugfs(void)
struct page *swiotlb_alloc(struct device *dev, size_t size)
{
struct io_tlb_mem *mem = dev->dma_io_tlb_mem;
+ struct io_tlb_zone *zone;
phys_addr_t tlb_addr;
int index;
if (!mem)
return NULL;
- index = swiotlb_find_slots(dev, 0, size, 0);
+ zone = &mem->zone[SWIOTLB_DF];
+
+ index = swiotlb_find_slots(dev, zone, 0, size, 0);
if (index == -1)
return NULL;
- tlb_addr = slot_addr(mem->start, index);
+ tlb_addr = slot_addr(zone->start, index);
return pfn_to_page(PFN_DOWN(tlb_addr));
}
@@ -964,11 +1083,14 @@ struct page *swiotlb_alloc(struct device *dev, size_t size)
bool swiotlb_free(struct device *dev, struct page *page, size_t size)
{
phys_addr_t tlb_addr = page_to_phys(page);
+ struct io_tlb_mem *mem = &dev->dma_io_tlb_mem;
+ int zone_index = swiotlb_zone_index(mem, tlb_addr);
+ struct io_tlb_zone *zone = mem->zone[zone_index];
if (!is_swiotlb_buffer(dev, tlb_addr))
return false;
- swiotlb_release_slots(dev, tlb_addr);
+ swiotlb_release_slots(dev, zone, tlb_addr);
return true;
}
@@ -977,6 +1099,7 @@ static int rmem_swiotlb_device_init(struct reserved_mem *rmem,
struct device *dev)
{
struct io_tlb_mem *mem = rmem->priv;
+ struct io_tlb_zone *zone;
unsigned long nslabs = rmem->size >> IO_TLB_SHIFT;
/* Set Per-device io tlb area to one */
@@ -992,25 +1115,29 @@ static int rmem_swiotlb_device_init(struct reserved_mem *rmem,
if (!mem)
return -ENOMEM;
- mem->slots = kcalloc(nslabs, sizeof(*mem->slots), GFP_KERNEL);
- if (!mem->slots) {
+ zone = &mem->zone[SWIOTLB_DF];
+
+ zone->slots = kcalloc(nslabs, sizeof(*zone->slots), GFP_KERNEL);
+ if (!zone->slots) {
kfree(mem);
return -ENOMEM;
}
- mem->areas = kcalloc(nareas, sizeof(*mem->areas),
+ zone->areas = kcalloc(nareas, sizeof(*zone->areas),
GFP_KERNEL);
- if (!mem->areas) {
- kfree(mem->slots);
+ if (!zone->areas) {
+ kfree(zone->slots);
kfree(mem);
return -ENOMEM;
}
set_memory_decrypted((unsigned long)phys_to_virt(rmem->base),
rmem->size >> PAGE_SHIFT);
- swiotlb_init_io_tlb_mem(mem, rmem->base, nslabs, SWIOTLB_FORCE,
- false, nareas);
+ swiotlb_init_io_tlb_zone(zone, rmem->base, nslabs, nareas);
mem->for_alloc = true;
+ mem->has_extra = false;
+ mem->late_alloc = false;
+ mem->force_bounce = flags & SWIOTLB_FORCE;
rmem->priv = mem;
--
1.8.3.1