Commit 0176adb0 authored by Christoph Hellwig's avatar Christoph Hellwig

swiotlb: refactor coherent buffer allocation

Factor out a new swiotlb_alloc_buffer helper that allocates DMA coherent
memory from the swiotlb bounce buffer.

This allows to simplify the swiotlb_alloc implemenation that uses
dma_direct_alloc to try to allocate a reachable buffer first.
Signed-off-by: default avatarChristoph Hellwig <hch@lst.de>
Acked-by: default avatarChristian König <christian.koenig@amd.com>
parent a25381aa
...@@ -709,74 +709,78 @@ void swiotlb_tbl_sync_single(struct device *hwdev, phys_addr_t tlb_addr, ...@@ -709,74 +709,78 @@ void swiotlb_tbl_sync_single(struct device *hwdev, phys_addr_t tlb_addr,
} }
EXPORT_SYMBOL_GPL(swiotlb_tbl_sync_single); EXPORT_SYMBOL_GPL(swiotlb_tbl_sync_single);
void * static inline bool dma_coherent_ok(struct device *dev, dma_addr_t addr,
swiotlb_alloc_coherent(struct device *hwdev, size_t size, size_t size)
dma_addr_t *dma_handle, gfp_t flags)
{ {
bool warn = !(flags & __GFP_NOWARN); u64 mask = DMA_BIT_MASK(32);
dma_addr_t dev_addr;
void *ret;
int order = get_order(size);
u64 dma_mask = DMA_BIT_MASK(32);
if (hwdev && hwdev->coherent_dma_mask) if (dev && dev->coherent_dma_mask)
dma_mask = hwdev->coherent_dma_mask; mask = dev->coherent_dma_mask;
return addr + size - 1 <= mask;
}
ret = (void *)__get_free_pages(flags, order); static void *
if (ret) { swiotlb_alloc_buffer(struct device *dev, size_t size, dma_addr_t *dma_handle,
dev_addr = swiotlb_virt_to_bus(hwdev, ret); unsigned long attrs)
if (dev_addr + size - 1 > dma_mask) { {
/* phys_addr_t phys_addr;
* The allocated memory isn't reachable by the device.
*/
free_pages((unsigned long) ret, order);
ret = NULL;
}
}
if (!ret) {
/*
* We are either out of memory or the device can't DMA to
* GFP_DMA memory; fall back on map_single(), which
* will grab memory from the lowest available address range.
*/
phys_addr_t paddr = map_single(hwdev, 0, size, DMA_FROM_DEVICE,
warn ? 0 : DMA_ATTR_NO_WARN);
if (paddr == SWIOTLB_MAP_ERROR)
goto err_warn;
ret = phys_to_virt(paddr); if (swiotlb_force == SWIOTLB_NO_FORCE)
dev_addr = swiotlb_phys_to_dma(hwdev, paddr); goto out_warn;
phys_addr = swiotlb_tbl_map_single(dev,
swiotlb_phys_to_dma(dev, io_tlb_start),
0, size, DMA_FROM_DEVICE, 0);
if (phys_addr == SWIOTLB_MAP_ERROR)
goto out_warn;
*dma_handle = swiotlb_phys_to_dma(dev, phys_addr);
if (dma_coherent_ok(dev, *dma_handle, size))
goto out_unmap;
/* Confirm address can be DMA'd by device */ memset(phys_to_virt(phys_addr), 0, size);
if (dev_addr + size - 1 > dma_mask) { return phys_to_virt(phys_addr);
printk("hwdev DMA mask = 0x%016Lx, dev_addr = 0x%016Lx\n",
(unsigned long long)dma_mask, out_unmap:
(unsigned long long)dev_addr); dev_warn(dev, "hwdev DMA mask = 0x%016Lx, dev_addr = 0x%016Lx\n",
(unsigned long long)(dev ? dev->coherent_dma_mask : 0),
(unsigned long long)*dma_handle);
/* /*
* DMA_TO_DEVICE to avoid memcpy in unmap_single. * DMA_TO_DEVICE to avoid memcpy in unmap_single.
* The DMA_ATTR_SKIP_CPU_SYNC is optional. * DMA_ATTR_SKIP_CPU_SYNC is optional.
*/ */
swiotlb_tbl_unmap_single(hwdev, paddr, swiotlb_tbl_unmap_single(dev, phys_addr, size, DMA_TO_DEVICE,
size, DMA_TO_DEVICE,
DMA_ATTR_SKIP_CPU_SYNC); DMA_ATTR_SKIP_CPU_SYNC);
goto err_warn; out_warn:
} if ((attrs & DMA_ATTR_NO_WARN) && printk_ratelimit()) {
dev_warn(dev,
"swiotlb: coherent allocation failed, size=%zu\n",
size);
dump_stack();
} }
return NULL;
}
*dma_handle = dev_addr; void *
memset(ret, 0, size); swiotlb_alloc_coherent(struct device *hwdev, size_t size,
dma_addr_t *dma_handle, gfp_t flags)
{
int order = get_order(size);
unsigned long attrs = (flags & __GFP_NOWARN) ? DMA_ATTR_NO_WARN : 0;
void *ret;
ret = (void *)__get_free_pages(flags, order);
if (ret) {
*dma_handle = swiotlb_virt_to_bus(hwdev, ret);
if (dma_coherent_ok(hwdev, *dma_handle, size)) {
memset(ret, 0, size);
return ret; return ret;
}
err_warn: free_pages((unsigned long)ret, order);
if (warn && printk_ratelimit()) {
pr_warn("swiotlb: coherent allocation failed for device %s size=%zu\n",
dev_name(hwdev), size);
dump_stack();
} }
return NULL; return swiotlb_alloc_buffer(hwdev, size, dma_handle, attrs);
} }
EXPORT_SYMBOL(swiotlb_alloc_coherent); EXPORT_SYMBOL(swiotlb_alloc_coherent);
...@@ -1103,6 +1107,10 @@ void *swiotlb_alloc(struct device *dev, size_t size, dma_addr_t *dma_handle, ...@@ -1103,6 +1107,10 @@ void *swiotlb_alloc(struct device *dev, size_t size, dma_addr_t *dma_handle,
{ {
void *vaddr; void *vaddr;
/* temporary workaround: */
if (gfp & __GFP_NOWARN)
attrs |= DMA_ATTR_NO_WARN;
/* /*
* Don't print a warning when the first allocation attempt fails. * Don't print a warning when the first allocation attempt fails.
* swiotlb_alloc_coherent() will print a warning when the DMA memory * swiotlb_alloc_coherent() will print a warning when the DMA memory
...@@ -1112,7 +1120,7 @@ void *swiotlb_alloc(struct device *dev, size_t size, dma_addr_t *dma_handle, ...@@ -1112,7 +1120,7 @@ void *swiotlb_alloc(struct device *dev, size_t size, dma_addr_t *dma_handle,
vaddr = dma_direct_alloc(dev, size, dma_handle, gfp, attrs); vaddr = dma_direct_alloc(dev, size, dma_handle, gfp, attrs);
if (!vaddr) if (!vaddr)
vaddr = swiotlb_alloc_coherent(dev, size, dma_handle, gfp); vaddr = swiotlb_alloc_buffer(dev, size, dma_handle, attrs);
return vaddr; return vaddr;
} }
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment