[PATCH 10/22] swiotlb: refactor coherent buffer allocation
Factor out a new swiotlb_alloc_buffer helper that allocates DMA coherent
memory from the swiotlb bounce buffer.
This allows to simplify the swiotlb_alloc implemenation that uses
dma_direct_alloc to try to allocate a reachable buffer first.
Signed-off-by: Christoph Hellwig
---
lib/swiotlb.c | 122 +++---
1 file changed, 65 insertions(+), 57 deletions(-)
diff --git a/lib/swiotlb.c b/lib/swiotlb.c
index 1a147f1354a1..bf2d19ee91c1 100644
--- a/lib/swiotlb.c
+++ b/lib/swiotlb.c
@@ -709,75 +709,79 @@ void swiotlb_tbl_sync_single(struct device *hwdev,
phys_addr_t tlb_addr,
}
EXPORT_SYMBOL_GPL(swiotlb_tbl_sync_single);
-void *
-swiotlb_alloc_coherent(struct device *hwdev, size_t size,
- dma_addr_t *dma_handle, gfp_t flags)
+static inline bool dma_coherent_ok(struct device *dev, dma_addr_t addr,
+ size_t size)
{
- bool warn = !(flags & __GFP_NOWARN);
- dma_addr_t dev_addr;
- void *ret;
- int order = get_order(size);
- u64 dma_mask = DMA_BIT_MASK(32);
+ u64 mask = DMA_BIT_MASK(32);
- if (hwdev && hwdev->coherent_dma_mask)
- dma_mask = hwdev->coherent_dma_mask;
+ if (dev && dev->coherent_dma_mask)
+ mask = dev->coherent_dma_mask;
+ return addr + size - 1 <= mask;
+}
- ret = (void *)__get_free_pages(flags, order);
- if (ret) {
- dev_addr = swiotlb_virt_to_bus(hwdev, ret);
- if (dev_addr + size - 1 > dma_mask) {
- /*
-* The allocated memory isn't reachable by the device.
-*/
- free_pages((unsigned long) ret, order);
- ret = NULL;
- }
- }
- if (!ret) {
- /*
-* We are either out of memory or the device can't DMA to
-* GFP_DMA memory; fall back on map_single(), which
-* will grab memory from the lowest available address range.
-*/
- phys_addr_t paddr = map_single(hwdev, 0, size, DMA_FROM_DEVICE,
- warn ? 0 : DMA_ATTR_NO_WARN);
- if (paddr == SWIOTLB_MAP_ERROR)
- goto err_warn;
+static void *
+swiotlb_alloc_buffer(struct device *dev, size_t size, dma_addr_t *dma_handle,
+ unsigned long attrs)
+{
+ phys_addr_t phys_addr;
+
+ if (swiotlb_force == SWIOTLB_NO_FORCE)
+ goto out_warn;
- ret = phys_to_virt(paddr);
- dev_addr = swiotlb_phys_to_dma(hwdev, paddr);
+ phys_addr = swiotlb_tbl_map_single(dev,
+ swiotlb_phys_to_dma(dev, io_tlb_start),
+ 0, size, DMA_FROM_DEVICE, 0);
+ if (phys_addr == SWIOTLB_MAP_ERROR)
+ goto out_warn;
- /* Confirm address can be DMA'd by device */
- if (dev_addr + size - 1 > dma_mask) {
- printk("hwdev DMA mask = 0x%016Lx, dev_addr =
0x%016Lx\n",
- (unsigned long long)dma_mask,
- (unsigned long long)dev_addr);
+ *dma_handle = swiotlb_phys_to_dma(dev, phys_addr);
- /*
-* DMA_TO_DEVICE to avoid memcpy in unmap_single.
-* The DMA_ATTR_SKIP_CPU_SYNC is optional.
-*/
- swiotlb_tbl_unmap_single(hwdev, paddr,
-size, DMA_TO_DEVICE,
-DMA_ATTR_SKIP_CPU_SYNC);
- goto err_warn;
- }
- }
+ if (dma_coherent_ok(dev, *dma_handle, size))
+ goto out_unmap;
- *dma_handle = dev_addr;
- memset(ret, 0, size);
+ memset(phys_to_virt(phys_addr), 0, size);
+ return phys_to_virt(phys_addr);
- return ret;
+out_unmap:
+ dev_warn(dev, "hwdev DMA mask = 0x%016Lx, dev_addr = 0x%016Lx\n",
+ (unsigned long long)(dev ? dev->coherent_dma_mask : 0),
+ (unsigned long long)*dma_handle);
-err_warn:
- if (warn && printk_ratelimit()) {
- pr_warn("swiotlb: coherent allocation failed for device %s
size=%zu\n",
- dev_name(hwdev), size);
+ /*
+* DMA_TO_DEVICE to avoid memcpy in unmap_single.
+* DMA_ATTR_SKIP_CPU_SYNC is optional.
+*/
+ swiotlb_tbl_unmap_single(dev, phys_addr, size, DMA_TO_DEVICE,
+ DMA_ATTR_SKIP_CPU_SYNC);
+out_warn:
+ if ((attrs & DMA_ATTR_NO_WARN) && printk_ratelimit()) {
+ dev_warn(dev,
+ "swiotlb: coherent allocation failed, size=%zu\n",
+ size);
dump_stack();
}
-
return NULL;
}
+
+void *
+swiotlb_alloc_coherent(str
Re: [PATCH 10/22] swiotlb: refactor coherent buffer allocation
On 10/01/18 08:09, Christoph Hellwig wrote:
Factor out a new swiotlb_alloc_buffer helper that allocates DMA coherent
memory from the swiotlb bounce buffer.
This allows to simplify the swiotlb_alloc implemenation that uses
dma_direct_alloc to try to allocate a reachable buffer first.
Signed-off-by: Christoph Hellwig
---
lib/swiotlb.c | 122 +++---
1 file changed, 65 insertions(+), 57 deletions(-)
diff --git a/lib/swiotlb.c b/lib/swiotlb.c
index 1a147f1354a1..bf2d19ee91c1 100644
--- a/lib/swiotlb.c
+++ b/lib/swiotlb.c
@@ -709,75 +709,79 @@ void swiotlb_tbl_sync_single(struct device *hwdev,
phys_addr_t tlb_addr,
}
EXPORT_SYMBOL_GPL(swiotlb_tbl_sync_single);
-void *
-swiotlb_alloc_coherent(struct device *hwdev, size_t size,
- dma_addr_t *dma_handle, gfp_t flags)
+static inline bool dma_coherent_ok(struct device *dev, dma_addr_t addr,
+ size_t size)
{
- bool warn = !(flags & __GFP_NOWARN);
- dma_addr_t dev_addr;
- void *ret;
- int order = get_order(size);
- u64 dma_mask = DMA_BIT_MASK(32);
+ u64 mask = DMA_BIT_MASK(32);
- if (hwdev && hwdev->coherent_dma_mask)
- dma_mask = hwdev->coherent_dma_mask;
+ if (dev && dev->coherent_dma_mask)
+ mask = dev->coherent_dma_mask;
+ return addr + size - 1 <= mask;
+}
- ret = (void *)__get_free_pages(flags, order);
- if (ret) {
- dev_addr = swiotlb_virt_to_bus(hwdev, ret);
- if (dev_addr + size - 1 > dma_mask) {
- /*
-* The allocated memory isn't reachable by the device.
-*/
- free_pages((unsigned long) ret, order);
- ret = NULL;
- }
- }
- if (!ret) {
- /*
-* We are either out of memory or the device can't DMA to
-* GFP_DMA memory; fall back on map_single(), which
-* will grab memory from the lowest available address range.
-*/
- phys_addr_t paddr = map_single(hwdev, 0, size, DMA_FROM_DEVICE,
- warn ? 0 : DMA_ATTR_NO_WARN);
- if (paddr == SWIOTLB_MAP_ERROR)
- goto err_warn;
+static void *
+swiotlb_alloc_buffer(struct device *dev, size_t size, dma_addr_t *dma_handle,
+ unsigned long attrs)
+{
+ phys_addr_t phys_addr;
+
+ if (swiotlb_force == SWIOTLB_NO_FORCE)
+ goto out_warn;
- ret = phys_to_virt(paddr);
- dev_addr = swiotlb_phys_to_dma(hwdev, paddr);
+ phys_addr = swiotlb_tbl_map_single(dev,
+ swiotlb_phys_to_dma(dev, io_tlb_start),
+ 0, size, DMA_FROM_DEVICE, 0);
+ if (phys_addr == SWIOTLB_MAP_ERROR)
+ goto out_warn;
- /* Confirm address can be DMA'd by device */
- if (dev_addr + size - 1 > dma_mask) {
- printk("hwdev DMA mask = 0x%016Lx, dev_addr =
0x%016Lx\n",
- (unsigned long long)dma_mask,
- (unsigned long long)dev_addr);
+ *dma_handle = swiotlb_phys_to_dma(dev, phys_addr);
nit: this should probably go after the dma_coherent_ok() check (as with
the original logic).
- /*
-* DMA_TO_DEVICE to avoid memcpy in unmap_single.
-* The DMA_ATTR_SKIP_CPU_SYNC is optional.
-*/
- swiotlb_tbl_unmap_single(hwdev, paddr,
-size, DMA_TO_DEVICE,
-DMA_ATTR_SKIP_CPU_SYNC);
- goto err_warn;
- }
- }
+ if (dma_coherent_ok(dev, *dma_handle, size))
+ goto out_unmap;
- *dma_handle = dev_addr;
- memset(ret, 0, size);
+ memset(phys_to_virt(phys_addr), 0, size);
+ return phys_to_virt(phys_addr);
- return ret;
+out_unmap:
+ dev_warn(dev, "hwdev DMA mask = 0x%016Lx, dev_addr = 0x%016Lx\n",
+ (unsigned long long)(dev ? dev->coherent_dma_mask : 0),
+ (unsigned long long)*dma_handle);
-err_warn:
- if (warn && printk_ratelimit()) {
- pr_warn("swiotlb: coherent allocation failed for device %s
size=%zu\n",
- dev_name(hwdev), size);
+ /*
+* DMA_TO_DEVICE to avoid memcpy in unmap_single.
+* DMA_ATTR_SKIP_CPU_SYNC is optional.
+*/
+ swiotlb_tbl_unmap_single(dev, phys_addr, size, DMA_TO_DEVICE,
+ DMA_ATTR_SKIP_CPU_SYNC);
+out_warn:
+ if ((attrs & DMA_ATTR_NO_WARN) && printk_ratelimit()) {
+ dev_warn(dev,
+ "swiotlb: coherent allocation failed, size=%zu\n",
+ size);
Re: [PATCH 10/22] swiotlb: refactor coherent buffer allocation
On Wed, Jan 10, 2018 at 12:22:18PM +, Robin Murphy wrote:
>> +if (phys_addr == SWIOTLB_MAP_ERROR)
>> +goto out_warn;
>> - /* Confirm address can be DMA'd by device */
>> -if (dev_addr + size - 1 > dma_mask) {
>> -printk("hwdev DMA mask = 0x%016Lx, dev_addr =
>> 0x%016Lx\n",
>> - (unsigned long long)dma_mask,
>> - (unsigned long long)dev_addr);
>> +*dma_handle = swiotlb_phys_to_dma(dev, phys_addr);
>
> nit: this should probably go after the dma_coherent_ok() check (as with the
> original logic).
But the originall logic also needs the dma_addr_t for the
dma_coherent_ok check:
dev_addr = swiotlb_phys_to_dma(hwdev, paddr);
/* Confirm address can be DMA'd by device */
if (dev_addr + size - 1 > dma_mask) {
...
goto err_warn;
}
or do you mean assining to *dma_handle? The dma_handle is not
valid for a failure return, so I don't think this should matter.
>> +if (ret) {
>> +*dma_handle = swiotlb_virt_to_bus(hwdev, ret);
>> +if (dma_coherent_ok(hwdev, *dma_handle, size)) {
>> +memset(ret, 0, size);
>> +return ret;
>> +}
>
> Aren't we leaking the pages here?
Yes, that free_pages got lost somewhere in the rebases, I've added
it back.
Re: [PATCH 10/22] swiotlb: refactor coherent buffer allocation
On 10/01/18 15:46, Christoph Hellwig wrote:
On Wed, Jan 10, 2018 at 12:22:18PM +, Robin Murphy wrote:
+ if (phys_addr == SWIOTLB_MAP_ERROR)
+ goto out_warn;
-/* Confirm address can be DMA'd by device */
- if (dev_addr + size - 1 > dma_mask) {
- printk("hwdev DMA mask = 0x%016Lx, dev_addr =
0x%016Lx\n",
- (unsigned long long)dma_mask,
- (unsigned long long)dev_addr);
+ *dma_handle = swiotlb_phys_to_dma(dev, phys_addr);
nit: this should probably go after the dma_coherent_ok() check (as with the
original logic).
But the originall logic also needs the dma_addr_t for the
dma_coherent_ok check:
dev_addr = swiotlb_phys_to_dma(hwdev, paddr);
/* Confirm address can be DMA'd by device */
if (dev_addr + size - 1 > dma_mask) {
...
goto err_warn;
}
or do you mean assining to *dma_handle? The dma_handle is not
valid for a failure return, so I don't think this should matter.
Yeah, only the assignment - as I said, it's just a stylistic nit; no big
deal either way.
+ if (ret) {
+ *dma_handle = swiotlb_virt_to_bus(hwdev, ret);
+ if (dma_coherent_ok(hwdev, *dma_handle, size)) {
+ memset(ret, 0, size);
+ return ret;
+ }
Aren't we leaking the pages here?
Yes, that free_pages got lost somewhere in the rebases, I've added
it back.
Cool.
Robin.
Re: [PATCH 10/22] swiotlb: refactor coherent buffer allocation
On Wed, Jan 10, 2018 at 05:02:30PM +, Robin Murphy wrote: >>> >>> Aren't we leaking the pages here? >> >> Yes, that free_pages got lost somewhere in the rebases, I've added >> it back. > > Cool. FYI, here is the fixed version, I don't want to re-send the whole series for this fix: http://git.infradead.org/users/hch/misc.git/commitdiff/0176adb004065d6815a8e67946752df4cd947c5b
