[PATCH 10/22] swiotlb: refactor coherent buffer allocation
Factor out a new swiotlb_alloc_buffer helper that allocates DMA coherent memory from the swiotlb bounce buffer. This allows to simplify the swiotlb_alloc implemenation that uses dma_direct_alloc to try to allocate a reachable buffer first. Signed-off-by: Christoph Hellwig --- lib/swiotlb.c | 122 +++--- 1 file changed, 65 insertions(+), 57 deletions(-) diff --git a/lib/swiotlb.c b/lib/swiotlb.c index 1a147f1354a1..bf2d19ee91c1 100644 --- a/lib/swiotlb.c +++ b/lib/swiotlb.c @@ -709,75 +709,79 @@ void swiotlb_tbl_sync_single(struct device *hwdev, phys_addr_t tlb_addr, } EXPORT_SYMBOL_GPL(swiotlb_tbl_sync_single); -void * -swiotlb_alloc_coherent(struct device *hwdev, size_t size, - dma_addr_t *dma_handle, gfp_t flags) +static inline bool dma_coherent_ok(struct device *dev, dma_addr_t addr, + size_t size) { - bool warn = !(flags & __GFP_NOWARN); - dma_addr_t dev_addr; - void *ret; - int order = get_order(size); - u64 dma_mask = DMA_BIT_MASK(32); + u64 mask = DMA_BIT_MASK(32); - if (hwdev && hwdev->coherent_dma_mask) - dma_mask = hwdev->coherent_dma_mask; + if (dev && dev->coherent_dma_mask) + mask = dev->coherent_dma_mask; + return addr + size - 1 <= mask; +} - ret = (void *)__get_free_pages(flags, order); - if (ret) { - dev_addr = swiotlb_virt_to_bus(hwdev, ret); - if (dev_addr + size - 1 > dma_mask) { - /* -* The allocated memory isn't reachable by the device. -*/ - free_pages((unsigned long) ret, order); - ret = NULL; - } - } - if (!ret) { - /* -* We are either out of memory or the device can't DMA to -* GFP_DMA memory; fall back on map_single(), which -* will grab memory from the lowest available address range. -*/ - phys_addr_t paddr = map_single(hwdev, 0, size, DMA_FROM_DEVICE, - warn ? 0 : DMA_ATTR_NO_WARN); - if (paddr == SWIOTLB_MAP_ERROR) - goto err_warn; +static void * +swiotlb_alloc_buffer(struct device *dev, size_t size, dma_addr_t *dma_handle, + unsigned long attrs) +{ + phys_addr_t phys_addr; + + if (swiotlb_force == SWIOTLB_NO_FORCE) + goto out_warn; - ret = phys_to_virt(paddr); - dev_addr = swiotlb_phys_to_dma(hwdev, paddr); + phys_addr = swiotlb_tbl_map_single(dev, + swiotlb_phys_to_dma(dev, io_tlb_start), + 0, size, DMA_FROM_DEVICE, 0); + if (phys_addr == SWIOTLB_MAP_ERROR) + goto out_warn; - /* Confirm address can be DMA'd by device */ - if (dev_addr + size - 1 > dma_mask) { - printk("hwdev DMA mask = 0x%016Lx, dev_addr = 0x%016Lx\n", - (unsigned long long)dma_mask, - (unsigned long long)dev_addr); + *dma_handle = swiotlb_phys_to_dma(dev, phys_addr); - /* -* DMA_TO_DEVICE to avoid memcpy in unmap_single. -* The DMA_ATTR_SKIP_CPU_SYNC is optional. -*/ - swiotlb_tbl_unmap_single(hwdev, paddr, -size, DMA_TO_DEVICE, -DMA_ATTR_SKIP_CPU_SYNC); - goto err_warn; - } - } + if (dma_coherent_ok(dev, *dma_handle, size)) + goto out_unmap; - *dma_handle = dev_addr; - memset(ret, 0, size); + memset(phys_to_virt(phys_addr), 0, size); + return phys_to_virt(phys_addr); - return ret; +out_unmap: + dev_warn(dev, "hwdev DMA mask = 0x%016Lx, dev_addr = 0x%016Lx\n", + (unsigned long long)(dev ? dev->coherent_dma_mask : 0), + (unsigned long long)*dma_handle); -err_warn: - if (warn && printk_ratelimit()) { - pr_warn("swiotlb: coherent allocation failed for device %s size=%zu\n", - dev_name(hwdev), size); + /* +* DMA_TO_DEVICE to avoid memcpy in unmap_single. +* DMA_ATTR_SKIP_CPU_SYNC is optional. +*/ + swiotlb_tbl_unmap_single(dev, phys_addr, size, DMA_TO_DEVICE, + DMA_ATTR_SKIP_CPU_SYNC); +out_warn: + if ((attrs & DMA_ATTR_NO_WARN) && printk_ratelimit()) { + dev_warn(dev, + "swiotlb: coherent allocation failed, size=%zu\n", + size); dump_stack(); } - return NULL; } + +void * +swiotlb_alloc_coherent(str
Re: [PATCH 10/22] swiotlb: refactor coherent buffer allocation
On 10/01/18 08:09, Christoph Hellwig wrote: Factor out a new swiotlb_alloc_buffer helper that allocates DMA coherent memory from the swiotlb bounce buffer. This allows to simplify the swiotlb_alloc implemenation that uses dma_direct_alloc to try to allocate a reachable buffer first. Signed-off-by: Christoph Hellwig --- lib/swiotlb.c | 122 +++--- 1 file changed, 65 insertions(+), 57 deletions(-) diff --git a/lib/swiotlb.c b/lib/swiotlb.c index 1a147f1354a1..bf2d19ee91c1 100644 --- a/lib/swiotlb.c +++ b/lib/swiotlb.c @@ -709,75 +709,79 @@ void swiotlb_tbl_sync_single(struct device *hwdev, phys_addr_t tlb_addr, } EXPORT_SYMBOL_GPL(swiotlb_tbl_sync_single); -void * -swiotlb_alloc_coherent(struct device *hwdev, size_t size, - dma_addr_t *dma_handle, gfp_t flags) +static inline bool dma_coherent_ok(struct device *dev, dma_addr_t addr, + size_t size) { - bool warn = !(flags & __GFP_NOWARN); - dma_addr_t dev_addr; - void *ret; - int order = get_order(size); - u64 dma_mask = DMA_BIT_MASK(32); + u64 mask = DMA_BIT_MASK(32); - if (hwdev && hwdev->coherent_dma_mask) - dma_mask = hwdev->coherent_dma_mask; + if (dev && dev->coherent_dma_mask) + mask = dev->coherent_dma_mask; + return addr + size - 1 <= mask; +} - ret = (void *)__get_free_pages(flags, order); - if (ret) { - dev_addr = swiotlb_virt_to_bus(hwdev, ret); - if (dev_addr + size - 1 > dma_mask) { - /* -* The allocated memory isn't reachable by the device. -*/ - free_pages((unsigned long) ret, order); - ret = NULL; - } - } - if (!ret) { - /* -* We are either out of memory or the device can't DMA to -* GFP_DMA memory; fall back on map_single(), which -* will grab memory from the lowest available address range. -*/ - phys_addr_t paddr = map_single(hwdev, 0, size, DMA_FROM_DEVICE, - warn ? 0 : DMA_ATTR_NO_WARN); - if (paddr == SWIOTLB_MAP_ERROR) - goto err_warn; +static void * +swiotlb_alloc_buffer(struct device *dev, size_t size, dma_addr_t *dma_handle, + unsigned long attrs) +{ + phys_addr_t phys_addr; + + if (swiotlb_force == SWIOTLB_NO_FORCE) + goto out_warn; - ret = phys_to_virt(paddr); - dev_addr = swiotlb_phys_to_dma(hwdev, paddr); + phys_addr = swiotlb_tbl_map_single(dev, + swiotlb_phys_to_dma(dev, io_tlb_start), + 0, size, DMA_FROM_DEVICE, 0); + if (phys_addr == SWIOTLB_MAP_ERROR) + goto out_warn; - /* Confirm address can be DMA'd by device */ - if (dev_addr + size - 1 > dma_mask) { - printk("hwdev DMA mask = 0x%016Lx, dev_addr = 0x%016Lx\n", - (unsigned long long)dma_mask, - (unsigned long long)dev_addr); + *dma_handle = swiotlb_phys_to_dma(dev, phys_addr); nit: this should probably go after the dma_coherent_ok() check (as with the original logic). - /* -* DMA_TO_DEVICE to avoid memcpy in unmap_single. -* The DMA_ATTR_SKIP_CPU_SYNC is optional. -*/ - swiotlb_tbl_unmap_single(hwdev, paddr, -size, DMA_TO_DEVICE, -DMA_ATTR_SKIP_CPU_SYNC); - goto err_warn; - } - } + if (dma_coherent_ok(dev, *dma_handle, size)) + goto out_unmap; - *dma_handle = dev_addr; - memset(ret, 0, size); + memset(phys_to_virt(phys_addr), 0, size); + return phys_to_virt(phys_addr); - return ret; +out_unmap: + dev_warn(dev, "hwdev DMA mask = 0x%016Lx, dev_addr = 0x%016Lx\n", + (unsigned long long)(dev ? dev->coherent_dma_mask : 0), + (unsigned long long)*dma_handle); -err_warn: - if (warn && printk_ratelimit()) { - pr_warn("swiotlb: coherent allocation failed for device %s size=%zu\n", - dev_name(hwdev), size); + /* +* DMA_TO_DEVICE to avoid memcpy in unmap_single. +* DMA_ATTR_SKIP_CPU_SYNC is optional. +*/ + swiotlb_tbl_unmap_single(dev, phys_addr, size, DMA_TO_DEVICE, + DMA_ATTR_SKIP_CPU_SYNC); +out_warn: + if ((attrs & DMA_ATTR_NO_WARN) && printk_ratelimit()) { + dev_warn(dev, + "swiotlb: coherent allocation failed, size=%zu\n", + size);
Re: [PATCH 10/22] swiotlb: refactor coherent buffer allocation
On Wed, Jan 10, 2018 at 12:22:18PM +, Robin Murphy wrote: >> +if (phys_addr == SWIOTLB_MAP_ERROR) >> +goto out_warn; >> - /* Confirm address can be DMA'd by device */ >> -if (dev_addr + size - 1 > dma_mask) { >> -printk("hwdev DMA mask = 0x%016Lx, dev_addr = >> 0x%016Lx\n", >> - (unsigned long long)dma_mask, >> - (unsigned long long)dev_addr); >> +*dma_handle = swiotlb_phys_to_dma(dev, phys_addr); > > nit: this should probably go after the dma_coherent_ok() check (as with the > original logic). But the originall logic also needs the dma_addr_t for the dma_coherent_ok check: dev_addr = swiotlb_phys_to_dma(hwdev, paddr); /* Confirm address can be DMA'd by device */ if (dev_addr + size - 1 > dma_mask) { ... goto err_warn; } or do you mean assining to *dma_handle? The dma_handle is not valid for a failure return, so I don't think this should matter. >> +if (ret) { >> +*dma_handle = swiotlb_virt_to_bus(hwdev, ret); >> +if (dma_coherent_ok(hwdev, *dma_handle, size)) { >> +memset(ret, 0, size); >> +return ret; >> +} > > Aren't we leaking the pages here? Yes, that free_pages got lost somewhere in the rebases, I've added it back.
Re: [PATCH 10/22] swiotlb: refactor coherent buffer allocation
On 10/01/18 15:46, Christoph Hellwig wrote: On Wed, Jan 10, 2018 at 12:22:18PM +, Robin Murphy wrote: + if (phys_addr == SWIOTLB_MAP_ERROR) + goto out_warn; -/* Confirm address can be DMA'd by device */ - if (dev_addr + size - 1 > dma_mask) { - printk("hwdev DMA mask = 0x%016Lx, dev_addr = 0x%016Lx\n", - (unsigned long long)dma_mask, - (unsigned long long)dev_addr); + *dma_handle = swiotlb_phys_to_dma(dev, phys_addr); nit: this should probably go after the dma_coherent_ok() check (as with the original logic). But the originall logic also needs the dma_addr_t for the dma_coherent_ok check: dev_addr = swiotlb_phys_to_dma(hwdev, paddr); /* Confirm address can be DMA'd by device */ if (dev_addr + size - 1 > dma_mask) { ... goto err_warn; } or do you mean assining to *dma_handle? The dma_handle is not valid for a failure return, so I don't think this should matter. Yeah, only the assignment - as I said, it's just a stylistic nit; no big deal either way. + if (ret) { + *dma_handle = swiotlb_virt_to_bus(hwdev, ret); + if (dma_coherent_ok(hwdev, *dma_handle, size)) { + memset(ret, 0, size); + return ret; + } Aren't we leaking the pages here? Yes, that free_pages got lost somewhere in the rebases, I've added it back. Cool. Robin.
Re: [PATCH 10/22] swiotlb: refactor coherent buffer allocation
On Wed, Jan 10, 2018 at 05:02:30PM +, Robin Murphy wrote: >>> >>> Aren't we leaking the pages here? >> >> Yes, that free_pages got lost somewhere in the rebases, I've added >> it back. > > Cool. FYI, here is the fixed version, I don't want to re-send the whole series for this fix: http://git.infradead.org/users/hch/misc.git/commitdiff/0176adb004065d6815a8e67946752df4cd947c5b