Skip to content

L
linux
Commit cff22949 authored by Linus Torvalds's avatar Linus Torvalds
Browse files
Options

Merge tag 'dma-mapping-4.20' of git://git.infradead.org/users/hch/dma-mapping 

Pull dma mapping updates from Christoph Hellwig:
 "First batch of dma-mapping changes for 4.20.

  There will be a second PR as some big changes were only applied just
  before the end of the merge window, and I want to give them a few more
  days in linux-next.

  Summary:

   - mostly more consolidation of the direct mapping code, including
     converting over hexagon, and merging the coherent and non-coherent
     code into a single dma_map_ops instance (me)

   - cleanups for the dma_configure/dma_unconfigure callchains (me)

   - better handling of dma_masks in odd setups (me, Alexander Duyck)

   - better debugging of passing vmalloc address to the DMA API (Stephen
     Boyd)

   - CMA command line parsing fix (He Zhe)"

* tag 'dma-mapping-4.20' of git://git.infradead.org/users/hch/dma-mapping: (27 commits)
  dma-direct: respect DMA_ATTR_NO_WARN
  dma-mapping: translate __GFP_NOFAIL to DMA_ATTR_NO_WARN
  dma-direct: document the zone selection logic
  dma-debug: Check for drivers mapping invalid addresses in dma_map_single()
  dma-direct: fix return value of dma_direct_supported
  dma-mapping: move dma_default_get_required_mask under ifdef
  dma-direct: always allow dma mask <= physiscal memory size
  dma-direct: implement complete bus_dma_mask handling
  dma-direct: refine dma_direct_alloc zone selection
  dma-direct: add an explicit dma_direct_get_required_mask
  dma-mapping: make the get_required_mask method available unconditionally
  unicore32: remove swiotlb support
  Revert "dma-mapping: clear dev->dma_ops in arch_teardown_dma_ops"
  dma-mapping: support non-coherent devices in dma_common_get_sgtable
  dma-mapping: consolidate the dma mmap implementations
  dma-mapping: merge direct and noncoherent ops
  dma-mapping: move the dma_coherent flag to struct device
  MIPS: don't select DMA_MAYBE_COHERENT from DMA_PERDEV_COHERENT
  dma-mapping: add the missing ARCH_HAS_SYNC_DMA_FOR_CPU_ALL declaration
  dma-mapping: fix panic caused by passing empty cma command line argument
  ...
parents 13775dac b9fd0426
Show whitespace changes
Inline Side-by-side
Showing with 420 additions and 698 deletions
arch/arc/Kconfig
View file @ cff22949
...@@ -9,6 +9,7 @@ ...@@ -9,6 +9,7 @@
config ARC config ARC
def_bool y def_bool y
select ARC_TIMERS select ARC_TIMERS
select ARCH_HAS_DMA_COHERENT_TO_PFN
select ARCH_HAS_PTE_SPECIAL select ARCH_HAS_PTE_SPECIAL
select ARCH_HAS_SYNC_DMA_FOR_CPU select ARCH_HAS_SYNC_DMA_FOR_CPU
select ARCH_HAS_SYNC_DMA_FOR_DEVICE select ARCH_HAS_SYNC_DMA_FOR_DEVICE
...@@ -17,8 +18,7 @@ config ARC ...@@ -17,8 +18,7 @@ config ARC
select BUILDTIME_EXTABLE_SORT select BUILDTIME_EXTABLE_SORT
select CLONE_BACKWARDS select CLONE_BACKWARDS
select COMMON_CLK select COMMON_CLK
select DMA_NONCOHERENT_OPS select DMA_DIRECT_OPS
select DMA_NONCOHERENT_MMAP
select GENERIC_ATOMIC64 if !ISA_ARCV2 || !(ARC_HAS_LL64 && ARC_HAS_LLSC) select GENERIC_ATOMIC64 if !ISA_ARCV2 || !(ARC_HAS_LL64 && ARC_HAS_LLSC)
select GENERIC_CLOCKEVENTS select GENERIC_CLOCKEVENTS
select GENERIC_FIND_FIRST_BIT select GENERIC_FIND_FIRST_BIT
......
arch/arc/mm/dma.c
View file @ cff22949
...@@ -84,29 +84,10 @@ void arch_dma_free(struct device *dev, size_t size, void *vaddr, ...@@ -84,29 +84,10 @@ void arch_dma_free(struct device *dev, size_t size, void *vaddr,
__free_pages(page, get_order(size)); __free_pages(page, get_order(size));
} }
int arch_dma_mmap(struct device *dev, struct vm_area_struct *vma, long arch_dma_coherent_to_pfn(struct device *dev, void *cpu_addr,
void *cpu_addr, dma_addr_t dma_addr, size_t size, dma_addr_t dma_addr)
unsigned long attrs)
{ {
unsigned long user_count = vma_pages(vma); return __phys_to_pfn(dma_addr);
unsigned long count = PAGE_ALIGN(size) >> PAGE_SHIFT;
unsigned long pfn = __phys_to_pfn(dma_addr);
unsigned long off = vma->vm_pgoff;
int ret = -ENXIO;
vma->vm_page_prot = pgprot_noncached(vma->vm_page_prot);
if (dma_mmap_from_dev_coherent(dev, vma, cpu_addr, size, &ret))
return ret;
if (off < count && user_count <= (count - off)) {
ret = remap_pfn_range(vma, vma->vm_start,
pfn + off,
user_count << PAGE_SHIFT,
vma->vm_page_prot);
}
return ret;
} }
/* /*
...@@ -167,7 +148,7 @@ void arch_sync_dma_for_cpu(struct device *dev, phys_addr_t paddr, ...@@ -167,7 +148,7 @@ void arch_sync_dma_for_cpu(struct device *dev, phys_addr_t paddr,
} }
/* /*
* Plug in coherent or noncoherent dma ops * Plug in direct dma map ops.
*/ */
void arch_setup_dma_ops(struct device *dev, u64 dma_base, u64 size, void arch_setup_dma_ops(struct device *dev, u64 dma_base, u64 size,
const struct iommu_ops *iommu, bool coherent) const struct iommu_ops *iommu, bool coherent)
...@@ -175,13 +156,11 @@ void arch_setup_dma_ops(struct device *dev, u64 dma_base, u64 size, ...@@ -175,13 +156,11 @@ void arch_setup_dma_ops(struct device *dev, u64 dma_base, u64 size,
/* /*
* IOC hardware snoops all DMA traffic keeping the caches consistent * IOC hardware snoops all DMA traffic keeping the caches consistent
* with memory - eliding need for any explicit cache maintenance of * with memory - eliding need for any explicit cache maintenance of
* DMA buffers - so we can use dma_direct cache ops. * DMA buffers.
*/ */
if (is_isa_arcv2() && ioc_enable && coherent) { if (is_isa_arcv2() && ioc_enable && coherent)
set_dma_ops(dev, &dma_direct_ops); dev->dma_coherent = true;
dev_info(dev, "use dma_direct_ops cache ops\n");
} else { dev_info(dev, "use %sncoherent DMA ops\n",
set_dma_ops(dev, &dma_noncoherent_ops); dev->dma_coherent ? "" : "non");
dev_info(dev, "use dma_noncoherent_ops cache ops\n");
}
} }
arch/arm/include/asm/dma-mapping.h
View file @ cff22949
...@@ -100,8 +100,10 @@ static inline unsigned long dma_max_pfn(struct device *dev) ...@@ -100,8 +100,10 @@ static inline unsigned long dma_max_pfn(struct device *dev)
extern void arch_setup_dma_ops(struct device *dev, u64 dma_base, u64 size, extern void arch_setup_dma_ops(struct device *dev, u64 dma_base, u64 size,
const struct iommu_ops *iommu, bool coherent); const struct iommu_ops *iommu, bool coherent);
#ifdef CONFIG_MMU
#define arch_teardown_dma_ops arch_teardown_dma_ops #define arch_teardown_dma_ops arch_teardown_dma_ops
extern void arch_teardown_dma_ops(struct device *dev); extern void arch_teardown_dma_ops(struct device *dev);
#endif
/* do not use this function in a driver */ /* do not use this function in a driver */
static inline bool is_device_dma_coherent(struct device *dev) static inline bool is_device_dma_coherent(struct device *dev)
......
arch/arm/mm/dma-mapping-nommu.c
View file @ cff22949
...@@ -47,7 +47,8 @@ static void *arm_nommu_dma_alloc(struct device *dev, size_t size, ...@@ -47,7 +47,8 @@ static void *arm_nommu_dma_alloc(struct device *dev, size_t size,
*/ */
if (attrs & DMA_ATTR_NON_CONSISTENT) if (attrs & DMA_ATTR_NON_CONSISTENT)
return dma_direct_alloc(dev, size, dma_handle, gfp, attrs); return dma_direct_alloc_pages(dev, size, dma_handle, gfp,
attrs);
ret = dma_alloc_from_global_coherent(size, dma_handle); ret = dma_alloc_from_global_coherent(size, dma_handle);
...@@ -70,7 +71,7 @@ static void arm_nommu_dma_free(struct device *dev, size_t size, ...@@ -70,7 +71,7 @@ static void arm_nommu_dma_free(struct device *dev, size_t size,
unsigned long attrs) unsigned long attrs)
{ {
if (attrs & DMA_ATTR_NON_CONSISTENT) { if (attrs & DMA_ATTR_NON_CONSISTENT) {
dma_direct_free(dev, size, cpu_addr, dma_addr, attrs); dma_direct_free_pages(dev, size, cpu_addr, dma_addr, attrs);
} else { } else {
int ret = dma_release_from_global_coherent(get_order(size), int ret = dma_release_from_global_coherent(get_order(size),
cpu_addr); cpu_addr);
...@@ -90,7 +91,7 @@ static int arm_nommu_dma_mmap(struct device *dev, struct vm_area_struct *vma, ...@@ -90,7 +91,7 @@ static int arm_nommu_dma_mmap(struct device *dev, struct vm_area_struct *vma,
if (dma_mmap_from_global_coherent(vma, cpu_addr, size, &ret)) if (dma_mmap_from_global_coherent(vma, cpu_addr, size, &ret))
return ret; return ret;
return dma_common_mmap(dev, vma, cpu_addr, dma_addr, size); return dma_common_mmap(dev, vma, cpu_addr, dma_addr, size, attrs);
} }
...@@ -237,7 +238,3 @@ void arch_setup_dma_ops(struct device *dev, u64 dma_base, u64 size, ...@@ -237,7 +238,3 @@ void arch_setup_dma_ops(struct device *dev, u64 dma_base, u64 size,
set_dma_ops(dev, dma_ops); set_dma_ops(dev, dma_ops);
} }
void arch_teardown_dma_ops(struct device *dev)
{
}
arch/c6x/Kconfig
View file @ cff22949
...@@ -9,7 +9,7 @@ config C6X ...@@ -9,7 +9,7 @@ config C6X
select ARCH_HAS_SYNC_DMA_FOR_CPU select ARCH_HAS_SYNC_DMA_FOR_CPU
select ARCH_HAS_SYNC_DMA_FOR_DEVICE select ARCH_HAS_SYNC_DMA_FOR_DEVICE
select CLKDEV_LOOKUP select CLKDEV_LOOKUP
select DMA_NONCOHERENT_OPS select DMA_DIRECT_OPS
select GENERIC_ATOMIC64 select GENERIC_ATOMIC64
select GENERIC_IRQ_SHOW select GENERIC_IRQ_SHOW
select HAVE_ARCH_TRACEHOOK select HAVE_ARCH_TRACEHOOK
......
arch/hexagon/Kconfig
View file @ cff22949
...@@ -4,6 +4,7 @@ comment "Linux Kernel Configuration for Hexagon" ...@@ -4,6 +4,7 @@ comment "Linux Kernel Configuration for Hexagon"
config HEXAGON config HEXAGON
def_bool y def_bool y
select ARCH_HAS_SYNC_DMA_FOR_DEVICE
select ARCH_NO_PREEMPT select ARCH_NO_PREEMPT
select HAVE_OPROFILE select HAVE_OPROFILE
# Other pending projects/to-do items. # Other pending projects/to-do items.
...@@ -29,6 +30,7 @@ config HEXAGON ...@@ -29,6 +30,7 @@ config HEXAGON
select GENERIC_CLOCKEVENTS_BROADCAST select GENERIC_CLOCKEVENTS_BROADCAST
select MODULES_USE_ELF_RELA select MODULES_USE_ELF_RELA
select GENERIC_CPU_DEVICES select GENERIC_CPU_DEVICES
select DMA_DIRECT_OPS
---help--- ---help---
Qualcomm Hexagon is a processor architecture designed for high Qualcomm Hexagon is a processor architecture designed for high
performance and low power across a wide variety of applications. performance and low power across a wide variety of applications.
......
arch/hexagon/include/asm/Kbuild
View file @ cff22949
...@@ -6,6 +6,7 @@ generic-y += compat.h ...@@ -6,6 +6,7 @@ generic-y += compat.h
generic-y += current.h generic-y += current.h
generic-y += device.h generic-y += device.h
generic-y += div64.h generic-y += div64.h
generic-y += dma-mapping.h
generic-y += emergency-restart.h generic-y += emergency-restart.h
generic-y += extable.h generic-y += extable.h
generic-y += fb.h generic-y += fb.h
......
arch/hexagon/include/asm/dma-mapping.h deleted 100644 → 0
View file @ 13775dac
/*
* DMA operations for the Hexagon architecture
*
* Copyright (c) 2010-2011, The Linux Foundation. All rights reserved.
*
* This program is free software; you can redistribute it and/or modify
* it under the terms of the GNU General Public License version 2 and
* only version 2 as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program; if not, write to the Free Software
* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
* 02110-1301, USA.
*/
#ifndef _ASM_DMA_MAPPING_H
#define _ASM_DMA_MAPPING_H
#include <linux/types.h>
#include <linux/cache.h>
#include <linux/mm.h>
#include <linux/scatterlist.h>
#include <linux/dma-debug.h>
#include <asm/io.h>
struct device;
extern const struct dma_map_ops *dma_ops;
static inline const struct dma_map_ops *get_arch_dma_ops(struct bus_type *bus)
{
return dma_ops;
}
#endif
arch/hexagon/kernel/dma.c
View file @ cff22949
...@@ -18,32 +18,19 @@ ...@@ -18,32 +18,19 @@
* 02110-1301, USA. * 02110-1301, USA.
*/ */
#include <linux/dma-mapping.h> #include <linux/dma-noncoherent.h>
#include <linux/dma-direct.h>
#include <linux/bootmem.h> #include <linux/bootmem.h>
#include <linux/genalloc.h> #include <linux/genalloc.h>
#include <asm/dma-mapping.h>
#include <linux/module.h> #include <linux/module.h>
#include <asm/page.h> #include <asm/page.h>
#define HEXAGON_MAPPING_ERROR 0
const struct dma_map_ops *dma_ops;
EXPORT_SYMBOL(dma_ops);
static inline void *dma_addr_to_virt(dma_addr_t dma_addr)
{
return phys_to_virt((unsigned long) dma_addr);
}
static struct gen_pool *coherent_pool; static struct gen_pool *coherent_pool;
/* Allocates from a pool of uncached memory that was reserved at boot time */ /* Allocates from a pool of uncached memory that was reserved at boot time */
static void *hexagon_dma_alloc_coherent(struct device *dev, size_t size, void *arch_dma_alloc(struct device *dev, size_t size, dma_addr_t *dma_addr,
dma_addr_t *dma_addr, gfp_t flag, gfp_t flag, unsigned long attrs)
unsigned long attrs)
{ {
void *ret; void *ret;
...@@ -75,58 +62,17 @@ static void *hexagon_dma_alloc_coherent(struct device *dev, size_t size, ...@@ -75,58 +62,17 @@ static void *hexagon_dma_alloc_coherent(struct device *dev, size_t size,
return ret; return ret;
} }
static void hexagon_free_coherent(struct device *dev, size_t size, void *vaddr, void arch_dma_free(struct device *dev, size_t size, void *vaddr,
dma_addr_t dma_addr, unsigned long attrs) dma_addr_t dma_addr, unsigned long attrs)
{ {
gen_pool_free(coherent_pool, (unsigned long) vaddr, size); gen_pool_free(coherent_pool, (unsigned long) vaddr, size);
} }
static int check_addr(const char *name, struct device *hwdev, void arch_sync_dma_for_device(struct device *dev, phys_addr_t paddr,
dma_addr_t bus, size_t size) size_t size, enum dma_data_direction dir)
{
if (hwdev && hwdev->dma_mask && !dma_capable(hwdev, bus, size)) {
if (*hwdev->dma_mask >= DMA_BIT_MASK(32))
printk(KERN_ERR
"%s: overflow %Lx+%zu of device mask %Lx\n",
name, (long long)bus, size,
(long long)*hwdev->dma_mask);
return 0;
}
return 1;
}
static int hexagon_map_sg(struct device *hwdev, struct scatterlist *sg,
int nents, enum dma_data_direction dir,
unsigned long attrs)
{ {
struct scatterlist *s; void *addr = phys_to_virt(paddr);
int i;
WARN_ON(nents == 0 || sg[0].length == 0);
for_each_sg(sg, s, nents, i) {
s->dma_address = sg_phys(s);
if (!check_addr("map_sg", hwdev, s->dma_address, s->length))
return 0;
s->dma_length = s->length;
if (attrs & DMA_ATTR_SKIP_CPU_SYNC)
continue;
flush_dcache_range(dma_addr_to_virt(s->dma_address),
dma_addr_to_virt(s->dma_address + s->length));
}
return nents;
}
/*
* address is virtual
*/
static inline void dma_sync(void *addr, size_t size,
enum dma_data_direction dir)
{
switch (dir) { switch (dir) {
case DMA_TO_DEVICE: case DMA_TO_DEVICE:
hexagon_clean_dcache_range((unsigned long) addr, hexagon_clean_dcache_range((unsigned long) addr,
...@@ -144,76 +90,3 @@ static inline void dma_sync(void *addr, size_t size, ...@@ -144,76 +90,3 @@ static inline void dma_sync(void *addr, size_t size,
BUG(); BUG();
} }
} }
/**
* hexagon_map_page() - maps an address for device DMA
* @dev: pointer to DMA device
* @page: pointer to page struct of DMA memory
* @offset: offset within page
* @size: size of memory to map
* @dir: transfer direction
* @attrs: pointer to DMA attrs (not used)
*
* Called to map a memory address to a DMA address prior
* to accesses to/from device.
*
* We don't particularly have many hoops to jump through
* so far. Straight translation between phys and virtual.
*
* DMA is not cache coherent so sync is necessary; this
* seems to be a convenient place to do it.
*
*/
static dma_addr_t hexagon_map_page(struct device *dev, struct page *page,
unsigned long offset, size_t size,
enum dma_data_direction dir,
unsigned long attrs)
{
dma_addr_t bus = page_to_phys(page) + offset;
WARN_ON(size == 0);
if (!check_addr("map_single", dev, bus, size))
return HEXAGON_MAPPING_ERROR;
if (!(attrs & DMA_ATTR_SKIP_CPU_SYNC))
dma_sync(dma_addr_to_virt(bus), size, dir);
return bus;
}
static void hexagon_sync_single_for_cpu(struct device *dev,
dma_addr_t dma_handle, size_t size,
enum dma_data_direction dir)
{
dma_sync(dma_addr_to_virt(dma_handle), size, dir);
}
static void hexagon_sync_single_for_device(struct device *dev,
dma_addr_t dma_handle, size_t size,
enum dma_data_direction dir)
{
dma_sync(dma_addr_to_virt(dma_handle), size, dir);
}
static int hexagon_mapping_error(struct device *dev, dma_addr_t dma_addr)
{
return dma_addr == HEXAGON_MAPPING_ERROR;
}
const struct dma_map_ops hexagon_dma_ops = {
.alloc = hexagon_dma_alloc_coherent,
.free = hexagon_free_coherent,
.map_sg = hexagon_map_sg,
.map_page = hexagon_map_page,
.sync_single_for_cpu = hexagon_sync_single_for_cpu,
.sync_single_for_device = hexagon_sync_single_for_device,
.mapping_error = hexagon_mapping_error,
};
void __init hexagon_dma_init(void)
{
if (dma_ops)
return;
dma_ops = &hexagon_dma_ops;
}
arch/ia64/include/asm/dma-mapping.h
View file @ cff22949
...@@ -10,8 +10,6 @@ ...@@ -10,8 +10,6 @@
#include <linux/scatterlist.h> #include <linux/scatterlist.h>
#include <linux/dma-debug.h> #include <linux/dma-debug.h>
#define ARCH_HAS_DMA_GET_REQUIRED_MASK
extern const struct dma_map_ops *dma_ops; extern const struct dma_map_ops *dma_ops;
extern struct ia64_machine_vector ia64_mv; extern struct ia64_machine_vector ia64_mv;
extern void set_iommu_machvec(void); extern void set_iommu_machvec(void);
......
arch/ia64/include/asm/machvec.h
View file @ cff22949
...@@ -44,7 +44,6 @@ typedef void ia64_mv_kernel_launch_event_t(void); ...@@ -44,7 +44,6 @@ typedef void ia64_mv_kernel_launch_event_t(void);
/* DMA-mapping interface: */ /* DMA-mapping interface: */
typedef void ia64_mv_dma_init (void); typedef void ia64_mv_dma_init (void);
typedef u64 ia64_mv_dma_get_required_mask (struct device *);
typedef const struct dma_map_ops *ia64_mv_dma_get_ops(struct device *); typedef const struct dma_map_ops *ia64_mv_dma_get_ops(struct device *);
/* /*
...@@ -127,7 +126,6 @@ extern void machvec_tlb_migrate_finish (struct mm_struct *); ...@@ -127,7 +126,6 @@ extern void machvec_tlb_migrate_finish (struct mm_struct *);
# define platform_global_tlb_purge ia64_mv.global_tlb_purge # define platform_global_tlb_purge ia64_mv.global_tlb_purge
# define platform_tlb_migrate_finish ia64_mv.tlb_migrate_finish # define platform_tlb_migrate_finish ia64_mv.tlb_migrate_finish
# define platform_dma_init ia64_mv.dma_init # define platform_dma_init ia64_mv.dma_init
# define platform_dma_get_required_mask ia64_mv.dma_get_required_mask
# define platform_dma_get_ops ia64_mv.dma_get_ops # define platform_dma_get_ops ia64_mv.dma_get_ops
# define platform_irq_to_vector ia64_mv.irq_to_vector # define platform_irq_to_vector ia64_mv.irq_to_vector
# define platform_local_vector_to_irq ia64_mv.local_vector_to_irq # define platform_local_vector_to_irq ia64_mv.local_vector_to_irq
...@@ -171,7 +169,6 @@ struct ia64_machine_vector { ...@@ -171,7 +169,6 @@ struct ia64_machine_vector {
ia64_mv_global_tlb_purge_t *global_tlb_purge; ia64_mv_global_tlb_purge_t *global_tlb_purge;
ia64_mv_tlb_migrate_finish_t *tlb_migrate_finish; ia64_mv_tlb_migrate_finish_t *tlb_migrate_finish;
ia64_mv_dma_init *dma_init; ia64_mv_dma_init *dma_init;
ia64_mv_dma_get_required_mask *dma_get_required_mask;
ia64_mv_dma_get_ops *dma_get_ops; ia64_mv_dma_get_ops *dma_get_ops;
ia64_mv_irq_to_vector *irq_to_vector; ia64_mv_irq_to_vector *irq_to_vector;
ia64_mv_local_vector_to_irq *local_vector_to_irq; ia64_mv_local_vector_to_irq *local_vector_to_irq;
...@@ -211,7 +208,6 @@ struct ia64_machine_vector { ...@@ -211,7 +208,6 @@ struct ia64_machine_vector {
platform_global_tlb_purge, \ platform_global_tlb_purge, \
platform_tlb_migrate_finish, \ platform_tlb_migrate_finish, \
platform_dma_init, \ platform_dma_init, \
platform_dma_get_required_mask, \
platform_dma_get_ops, \ platform_dma_get_ops, \
platform_irq_to_vector, \ platform_irq_to_vector, \
platform_local_vector_to_irq, \ platform_local_vector_to_irq, \
...@@ -286,9 +282,6 @@ extern const struct dma_map_ops *dma_get_ops(struct device *); ...@@ -286,9 +282,6 @@ extern const struct dma_map_ops *dma_get_ops(struct device *);
#ifndef platform_dma_get_ops #ifndef platform_dma_get_ops
# define platform_dma_get_ops dma_get_ops # define platform_dma_get_ops dma_get_ops
#endif #endif
#ifndef platform_dma_get_required_mask
# define platform_dma_get_required_mask ia64_dma_get_required_mask
#endif
#ifndef platform_irq_to_vector #ifndef platform_irq_to_vector
# define platform_irq_to_vector __ia64_irq_to_vector # define platform_irq_to_vector __ia64_irq_to_vector
#endif #endif
......
arch/ia64/include/asm/machvec_init.h
View file @ cff22949
...@@ -4,7 +4,6 @@ ...@@ -4,7 +4,6 @@
extern ia64_mv_send_ipi_t ia64_send_ipi; extern ia64_mv_send_ipi_t ia64_send_ipi;
extern ia64_mv_global_tlb_purge_t ia64_global_tlb_purge; extern ia64_mv_global_tlb_purge_t ia64_global_tlb_purge;
extern ia64_mv_dma_get_required_mask ia64_dma_get_required_mask;
extern ia64_mv_irq_to_vector __ia64_irq_to_vector; extern ia64_mv_irq_to_vector __ia64_irq_to_vector;
extern ia64_mv_local_vector_to_irq __ia64_local_vector_to_irq; extern ia64_mv_local_vector_to_irq __ia64_local_vector_to_irq;
extern ia64_mv_pci_get_legacy_mem_t ia64_pci_get_legacy_mem; extern ia64_mv_pci_get_legacy_mem_t ia64_pci_get_legacy_mem;
......
arch/ia64/include/asm/machvec_sn2.h
View file @ cff22949
...@@ -55,7 +55,6 @@ extern ia64_mv_readb_t __sn_readb_relaxed; ...@@ -55,7 +55,6 @@ extern ia64_mv_readb_t __sn_readb_relaxed;
extern ia64_mv_readw_t __sn_readw_relaxed; extern ia64_mv_readw_t __sn_readw_relaxed;
extern ia64_mv_readl_t __sn_readl_relaxed; extern ia64_mv_readl_t __sn_readl_relaxed;
extern ia64_mv_readq_t __sn_readq_relaxed; extern ia64_mv_readq_t __sn_readq_relaxed;
extern ia64_mv_dma_get_required_mask sn_dma_get_required_mask;
extern ia64_mv_dma_init sn_dma_init; extern ia64_mv_dma_init sn_dma_init;
extern ia64_mv_migrate_t sn_migrate; extern ia64_mv_migrate_t sn_migrate;
extern ia64_mv_kernel_launch_event_t sn_kernel_launch_event; extern ia64_mv_kernel_launch_event_t sn_kernel_launch_event;
...@@ -100,7 +99,6 @@ extern ia64_mv_pci_fixup_bus_t sn_pci_fixup_bus; ...@@ -100,7 +99,6 @@ extern ia64_mv_pci_fixup_bus_t sn_pci_fixup_bus;
#define platform_pci_get_legacy_mem sn_pci_get_legacy_mem #define platform_pci_get_legacy_mem sn_pci_get_legacy_mem
#define platform_pci_legacy_read sn_pci_legacy_read #define platform_pci_legacy_read sn_pci_legacy_read
#define platform_pci_legacy_write sn_pci_legacy_write #define platform_pci_legacy_write sn_pci_legacy_write
#define platform_dma_get_required_mask sn_dma_get_required_mask
#define platform_dma_init sn_dma_init #define platform_dma_init sn_dma_init
#define platform_migrate sn_migrate #define platform_migrate sn_migrate
#define platform_kernel_launch_event sn_kernel_launch_event #define platform_kernel_launch_event sn_kernel_launch_event
......
arch/ia64/pci/pci.c
View file @ cff22949
...@@ -568,32 +568,6 @@ static void __init set_pci_dfl_cacheline_size(void) ...@@ -568,32 +568,6 @@ static void __init set_pci_dfl_cacheline_size(void)
pci_dfl_cache_line_size = (1 << cci.pcci_line_size) / 4; pci_dfl_cache_line_size = (1 << cci.pcci_line_size) / 4;
} }
u64 ia64_dma_get_required_mask(struct device *dev)
{
u32 low_totalram = ((max_pfn - 1) << PAGE_SHIFT);
u32 high_totalram = ((max_pfn - 1) >> (32 - PAGE_SHIFT));
u64 mask;
if (!high_totalram) {
/* convert to mask just covering totalram */
low_totalram = (1 << (fls(low_totalram) - 1));
low_totalram += low_totalram - 1;
mask = low_totalram;
} else {
high_totalram = (1 << (fls(high_totalram) - 1));
high_totalram += high_totalram - 1;
mask = (((u64)high_totalram) << 32) + 0xffffffff;
}
return mask;
}
EXPORT_SYMBOL_GPL(ia64_dma_get_required_mask);
u64 dma_get_required_mask(struct device *dev)
{
return platform_dma_get_required_mask(dev);
}
EXPORT_SYMBOL_GPL(dma_get_required_mask);
static int __init pcibios_init(void) static int __init pcibios_init(void)
{ {
set_pci_dfl_cacheline_size(); set_pci_dfl_cacheline_size();
......
arch/ia64/sn/pci/pci_dma.c
View file @ cff22949
...@@ -344,11 +344,10 @@ static int sn_dma_mapping_error(struct device *dev, dma_addr_t dma_addr) ...@@ -344,11 +344,10 @@ static int sn_dma_mapping_error(struct device *dev, dma_addr_t dma_addr)
return 0; return 0;
} }
u64 sn_dma_get_required_mask(struct device *dev) static u64 sn_dma_get_required_mask(struct device *dev)
{ {
return DMA_BIT_MASK(64); return DMA_BIT_MASK(64);
} }
EXPORT_SYMBOL_GPL(sn_dma_get_required_mask);
char *sn_pci_get_legacy_mem(struct pci_bus *bus) char *sn_pci_get_legacy_mem(struct pci_bus *bus)
{ {
...@@ -473,6 +472,7 @@ static struct dma_map_ops sn_dma_ops = { ...@@ -473,6 +472,7 @@ static struct dma_map_ops sn_dma_ops = {
.sync_sg_for_device = sn_dma_sync_sg_for_device, .sync_sg_for_device = sn_dma_sync_sg_for_device,
.mapping_error = sn_dma_mapping_error, .mapping_error = sn_dma_mapping_error,
.dma_supported = sn_dma_supported, .dma_supported = sn_dma_supported,
.get_required_mask = sn_dma_get_required_mask,
}; };
void sn_dma_init(void) void sn_dma_init(void)
......
arch/m68k/Kconfig
View file @ cff22949
...@@ -26,7 +26,7 @@ config M68K ...@@ -26,7 +26,7 @@ config M68K
select MODULES_USE_ELF_RELA select MODULES_USE_ELF_RELA
select OLD_SIGSUSPEND3 select OLD_SIGSUSPEND3
select OLD_SIGACTION select OLD_SIGACTION
select DMA_NONCOHERENT_OPS if HAS_DMA select DMA_DIRECT_OPS if HAS_DMA
select HAVE_MEMBLOCK select HAVE_MEMBLOCK
select ARCH_DISCARD_MEMBLOCK select ARCH_DISCARD_MEMBLOCK
select NO_BOOTMEM select NO_BOOTMEM
......
arch/microblaze/Kconfig
View file @ cff22949
config MICROBLAZE config MICROBLAZE
def_bool y def_bool y
select ARCH_NO_SWAP select ARCH_NO_SWAP
select ARCH_HAS_DMA_COHERENT_TO_PFN if MMU
select ARCH_HAS_GCOV_PROFILE_ALL select ARCH_HAS_GCOV_PROFILE_ALL
select ARCH_HAS_SYNC_DMA_FOR_CPU select ARCH_HAS_SYNC_DMA_FOR_CPU
select ARCH_HAS_SYNC_DMA_FOR_DEVICE select ARCH_HAS_SYNC_DMA_FOR_DEVICE
...@@ -11,8 +12,7 @@ config MICROBLAZE ...@@ -11,8 +12,7 @@ config MICROBLAZE
select TIMER_OF select TIMER_OF
select CLONE_BACKWARDS3 select CLONE_BACKWARDS3
select COMMON_CLK select COMMON_CLK
select DMA_NONCOHERENT_OPS select DMA_DIRECT_OPS
select DMA_NONCOHERENT_MMAP
select GENERIC_ATOMIC64 select GENERIC_ATOMIC64
select GENERIC_CLOCKEVENTS select GENERIC_CLOCKEVENTS
select GENERIC_CPU_DEVICES select GENERIC_CPU_DEVICES
......
arch/microblaze/include/asm/pgtable.h
View file @ cff22949
...@@ -553,8 +553,6 @@ void __init *early_get_page(void); ...@@ -553,8 +553,6 @@ void __init *early_get_page(void);
extern unsigned long ioremap_bot, ioremap_base; extern unsigned long ioremap_bot, ioremap_base;
unsigned long consistent_virt_to_pfn(void *vaddr);
void setup_memory(void); void setup_memory(void);
#endif /* __ASSEMBLY__ */ #endif /* __ASSEMBLY__ */
......
arch/microblaze/kernel/dma.c
View file @ cff22949
...@@ -42,25 +42,3 @@ void arch_sync_dma_for_cpu(struct device *dev, phys_addr_t paddr, ...@@ -42,25 +42,3 @@ void arch_sync_dma_for_cpu(struct device *dev, phys_addr_t paddr,
{ {
__dma_sync(dev, paddr, size, dir); __dma_sync(dev, paddr, size, dir);
} }
int arch_dma_mmap(struct device *dev, struct vm_area_struct *vma,
void *cpu_addr, dma_addr_t handle, size_t size,
unsigned long attrs)
{
#ifdef CONFIG_MMU
unsigned long user_count = vma_pages(vma);
unsigned long count = PAGE_ALIGN(size) >> PAGE_SHIFT;
unsigned long off = vma->vm_pgoff;
unsigned long pfn;
if (off >= count || user_count > (count - off))
return -ENXIO;
vma->vm_page_prot = pgprot_noncached(vma->vm_page_prot);
pfn = consistent_virt_to_pfn(cpu_addr);
return remap_pfn_range(vma, vma->vm_start, pfn + off,
vma->vm_end - vma->vm_start, vma->vm_page_prot);
#else
return -ENXIO;
#endif
}
arch/microblaze/mm/consistent.c
View file @ cff22949
...@@ -165,7 +165,8 @@ static pte_t *consistent_virt_to_pte(void *vaddr) ...@@ -165,7 +165,8 @@ static pte_t *consistent_virt_to_pte(void *vaddr)
return pte_offset_kernel(pmd_offset(pgd_offset_k(addr), addr), addr); return pte_offset_kernel(pmd_offset(pgd_offset_k(addr), addr), addr);
} }
unsigned long consistent_virt_to_pfn(void *vaddr) long arch_dma_coherent_to_pfn(struct device *dev, void *vaddr,
dma_addr_t dma_addr)
{ {
pte_t *ptep = consistent_virt_to_pte(vaddr); pte_t *ptep = consistent_virt_to_pte(vaddr);
......
arch/mips/Kconfig
View file @ cff22949
...@@ -1106,21 +1106,22 @@ config ARCH_SUPPORTS_UPROBES ...@@ -1106,21 +1106,22 @@ config ARCH_SUPPORTS_UPROBES
bool bool
config DMA_MAYBE_COHERENT config DMA_MAYBE_COHERENT
select ARCH_HAS_DMA_COHERENCE_H
select DMA_NONCOHERENT select DMA_NONCOHERENT
bool bool
config DMA_PERDEV_COHERENT config DMA_PERDEV_COHERENT
bool bool
select DMA_MAYBE_COHERENT select DMA_NONCOHERENT
config DMA_NONCOHERENT config DMA_NONCOHERENT
bool bool
select ARCH_HAS_DMA_MMAP_PGPROT
select ARCH_HAS_SYNC_DMA_FOR_DEVICE select ARCH_HAS_SYNC_DMA_FOR_DEVICE
select ARCH_HAS_SYNC_DMA_FOR_CPU select ARCH_HAS_SYNC_DMA_FOR_CPU
select NEED_DMA_MAP_STATE select NEED_DMA_MAP_STATE
select DMA_NONCOHERENT_MMAP select ARCH_HAS_DMA_COHERENT_TO_PFN
select DMA_NONCOHERENT_CACHE_SYNC select DMA_NONCOHERENT_CACHE_SYNC
select DMA_NONCOHERENT_OPS
config SYS_HAS_EARLY_PRINTK config SYS_HAS_EARLY_PRINTK
bool bool
......
arch/mips/include/asm/Kbuild
View file @ cff22949
# MIPS headers # MIPS headers
generic-(CONFIG_GENERIC_CSUM) += checksum.h generic-(CONFIG_GENERIC_CSUM) += checksum.h
generic-y += current.h generic-y += current.h
generic-y += device.h
generic-y += dma-contiguous.h generic-y += dma-contiguous.h
generic-y += emergency-restart.h generic-y += emergency-restart.h
generic-y += export.h generic-y += export.h
......
arch/mips/include/asm/device.h deleted 100644 → 0
View file @ 13775dac
/*
* Arch specific extensions to struct device
*
* This file is released under the GPLv2
*/
#ifndef _ASM_MIPS_DEVICE_H
#define _ASM_MIPS_DEVICE_H
struct dev_archdata {
#ifdef CONFIG_DMA_PERDEV_COHERENT
/* Non-zero if DMA is coherent with CPU caches */
bool dma_coherent;
#endif
};
struct pdev_archdata {
};
#endif /* _ASM_MIPS_DEVICE_H*/
arch/mips/include/asm/dma-coherence.h
View file @ cff22949
...@@ -20,6 +20,12 @@ enum coherent_io_user_state { ...@@ -20,6 +20,12 @@ enum coherent_io_user_state {
#elif defined(CONFIG_DMA_MAYBE_COHERENT) #elif defined(CONFIG_DMA_MAYBE_COHERENT)
extern enum coherent_io_user_state coherentio; extern enum coherent_io_user_state coherentio;
extern int hw_coherentio; extern int hw_coherentio;
static inline bool dev_is_dma_coherent(struct device *dev)
{
return coherentio == IO_COHERENCE_ENABLED ||
(coherentio == IO_COHERENCE_DEFAULT && hw_coherentio);
}
#else #else
#ifdef CONFIG_DMA_NONCOHERENT #ifdef CONFIG_DMA_NONCOHERENT
#define coherentio IO_COHERENCE_DISABLED #define coherentio IO_COHERENCE_DISABLED
......
arch/mips/include/asm/dma-mapping.h
View file @ cff22949
...@@ -12,8 +12,6 @@ static inline const struct dma_map_ops *get_arch_dma_ops(struct bus_type *bus) ...@@ -12,8 +12,6 @@ static inline const struct dma_map_ops *get_arch_dma_ops(struct bus_type *bus)
return &jazz_dma_ops; return &jazz_dma_ops;
#elif defined(CONFIG_SWIOTLB) #elif defined(CONFIG_SWIOTLB)
return &swiotlb_dma_ops; return &swiotlb_dma_ops;
#elif defined(CONFIG_DMA_NONCOHERENT_OPS)
return &dma_noncoherent_ops;
#else #else
return &dma_direct_ops; return &dma_direct_ops;
#endif #endif
...@@ -25,7 +23,7 @@ static inline void arch_setup_dma_ops(struct device *dev, u64 dma_base, ...@@ -25,7 +23,7 @@ static inline void arch_setup_dma_ops(struct device *dev, u64 dma_base,
bool coherent) bool coherent)
{ {
#ifdef CONFIG_DMA_PERDEV_COHERENT #ifdef CONFIG_DMA_PERDEV_COHERENT
dev->archdata.dma_coherent = coherent; dev->dma_coherent = coherent;
#endif #endif
} }
......
arch/mips/jazz/jazzdma.c
View file @ cff22949
...@@ -564,13 +564,13 @@ static void *jazz_dma_alloc(struct device *dev, size_t size, ...@@ -564,13 +564,13 @@ static void *jazz_dma_alloc(struct device *dev, size_t size,
{ {
void *ret; void *ret;
ret = dma_direct_alloc(dev, size, dma_handle, gfp, attrs); ret = dma_direct_alloc_pages(dev, size, dma_handle, gfp, attrs);
if (!ret) if (!ret)
return NULL; return NULL;
*dma_handle = vdma_alloc(virt_to_phys(ret), size); *dma_handle = vdma_alloc(virt_to_phys(ret), size);
if (*dma_handle == VDMA_ERROR) { if (*dma_handle == VDMA_ERROR) {
dma_direct_free(dev, size, ret, *dma_handle, attrs); dma_direct_free_pages(dev, size, ret, *dma_handle, attrs);
return NULL; return NULL;
} }
...@@ -587,7 +587,7 @@ static void jazz_dma_free(struct device *dev, size_t size, void *vaddr, ...@@ -587,7 +587,7 @@ static void jazz_dma_free(struct device *dev, size_t size, void *vaddr,
vdma_free(dma_handle); vdma_free(dma_handle);
if (!(attrs & DMA_ATTR_NON_CONSISTENT)) if (!(attrs & DMA_ATTR_NON_CONSISTENT))
vaddr = (void *)CAC_ADDR((unsigned long)vaddr); vaddr = (void *)CAC_ADDR((unsigned long)vaddr);
return dma_direct_free(dev, size, vaddr, dma_handle, attrs); dma_direct_free_pages(dev, size, vaddr, dma_handle, attrs);
} }
static dma_addr_t jazz_dma_map_page(struct device *dev, struct page *page, static dma_addr_t jazz_dma_map_page(struct device *dev, struct page *page,
...@@ -682,7 +682,6 @@ static int jazz_dma_mapping_error(struct device *dev, dma_addr_t dma_addr) ...@@ -682,7 +682,6 @@ static int jazz_dma_mapping_error(struct device *dev, dma_addr_t dma_addr)
const struct dma_map_ops jazz_dma_ops = { const struct dma_map_ops jazz_dma_ops = {
.alloc = jazz_dma_alloc, .alloc = jazz_dma_alloc,
.free = jazz_dma_free, .free = jazz_dma_free,
.mmap = arch_dma_mmap,
.map_page = jazz_dma_map_page, .map_page = jazz_dma_map_page,
.unmap_page = jazz_dma_unmap_page, .unmap_page = jazz_dma_unmap_page,
.map_sg = jazz_dma_map_sg, .map_sg = jazz_dma_map_sg,
......
arch/mips/kernel/setup.c
View file @ cff22949
...@@ -1075,7 +1075,7 @@ static int __init debugfs_mips(void) ...@@ -1075,7 +1075,7 @@ static int __init debugfs_mips(void)
arch_initcall(debugfs_mips); arch_initcall(debugfs_mips);
#endif #endif
#if defined(CONFIG_DMA_MAYBE_COHERENT) && !defined(CONFIG_DMA_PERDEV_COHERENT) #ifdef CONFIG_DMA_MAYBE_COHERENT
/* User defined DMA coherency from command line. */ /* User defined DMA coherency from command line. */
enum coherent_io_user_state coherentio = IO_COHERENCE_DEFAULT; enum coherent_io_user_state coherentio = IO_COHERENCE_DEFAULT;
EXPORT_SYMBOL_GPL(coherentio); EXPORT_SYMBOL_GPL(coherentio);
......
arch/mips/mm/c-r4k.c
View file @ cff22949
...@@ -1955,22 +1955,21 @@ void r4k_cache_init(void) ...@@ -1955,22 +1955,21 @@ void r4k_cache_init(void)
__flush_icache_user_range = r4k_flush_icache_user_range; __flush_icache_user_range = r4k_flush_icache_user_range;
__local_flush_icache_user_range = local_r4k_flush_icache_user_range; __local_flush_icache_user_range = local_r4k_flush_icache_user_range;
#if defined(CONFIG_DMA_NONCOHERENT) || defined(CONFIG_DMA_MAYBE_COHERENT) #ifdef CONFIG_DMA_NONCOHERENT
# if defined(CONFIG_DMA_PERDEV_COHERENT) #ifdef CONFIG_DMA_MAYBE_COHERENT
if (0) { if (coherentio == IO_COHERENCE_ENABLED ||
# else (coherentio == IO_COHERENCE_DEFAULT && hw_coherentio)) {
if ((coherentio == IO_COHERENCE_ENABLED) ||
((coherentio == IO_COHERENCE_DEFAULT) && hw_coherentio)) {
# endif
_dma_cache_wback_inv = (void *)cache_noop; _dma_cache_wback_inv = (void *)cache_noop;
_dma_cache_wback = (void *)cache_noop; _dma_cache_wback = (void *)cache_noop;
_dma_cache_inv = (void *)cache_noop; _dma_cache_inv = (void *)cache_noop;
} else { } else
#endif /* CONFIG_DMA_MAYBE_COHERENT */
{
_dma_cache_wback_inv = r4k_dma_cache_wback_inv; _dma_cache_wback_inv = r4k_dma_cache_wback_inv;
_dma_cache_wback = r4k_dma_cache_wback_inv; _dma_cache_wback = r4k_dma_cache_wback_inv;
_dma_cache_inv = r4k_dma_cache_inv; _dma_cache_inv = r4k_dma_cache_inv;
} }
#endif #endif /* CONFIG_DMA_NONCOHERENT */
build_clear_page(); build_clear_page();
build_copy_page(); build_copy_page();
......
arch/mips/mm/dma-noncoherent.c
View file @ cff22949
...@@ -14,26 +14,6 @@ ...@@ -14,26 +14,6 @@
#include <asm/dma-coherence.h> #include <asm/dma-coherence.h>
#include <asm/io.h> #include <asm/io.h>
#ifdef CONFIG_DMA_PERDEV_COHERENT
static inline int dev_is_coherent(struct device *dev)
{
return dev->archdata.dma_coherent;
}
#else
static inline int dev_is_coherent(struct device *dev)
{
switch (coherentio) {
default:
case IO_COHERENCE_DEFAULT:
return hw_coherentio;
case IO_COHERENCE_ENABLED:
return 1;
case IO_COHERENCE_DISABLED:
return 0;
}
}
#endif /* CONFIG_DMA_PERDEV_COHERENT */
/* /*
* The affected CPUs below in 'cpu_needs_post_dma_flush()' can speculatively * The affected CPUs below in 'cpu_needs_post_dma_flush()' can speculatively
* fill random cachelines with stale data at any time, requiring an extra * fill random cachelines with stale data at any time, requiring an extra
...@@ -49,9 +29,6 @@ static inline int dev_is_coherent(struct device *dev) ...@@ -49,9 +29,6 @@ static inline int dev_is_coherent(struct device *dev)
*/ */
static inline bool cpu_needs_post_dma_flush(struct device *dev) static inline bool cpu_needs_post_dma_flush(struct device *dev)
{ {
if (dev_is_coherent(dev))
return false;
switch (boot_cpu_type()) { switch (boot_cpu_type()) {
case CPU_R10000: case CPU_R10000:
case CPU_R12000: case CPU_R12000:
...@@ -72,11 +49,8 @@ void *arch_dma_alloc(struct device *dev, size_t size, ...@@ -72,11 +49,8 @@ void *arch_dma_alloc(struct device *dev, size_t size,
{ {
void *ret; void *ret;
ret = dma_direct_alloc(dev, size, dma_handle, gfp, attrs); ret = dma_direct_alloc_pages(dev, size, dma_handle, gfp, attrs);
if (!ret) if (!ret && !(attrs & DMA_ATTR_NON_CONSISTENT)) {
return NULL;
if (!dev_is_coherent(dev) && !(attrs & DMA_ATTR_NON_CONSISTENT)) {
dma_cache_wback_inv((unsigned long) ret, size); dma_cache_wback_inv((unsigned long) ret, size);
ret = (void *)UNCAC_ADDR(ret); ret = (void *)UNCAC_ADDR(ret);
} }
...@@ -87,43 +61,24 @@ void *arch_dma_alloc(struct device *dev, size_t size, ...@@ -87,43 +61,24 @@ void *arch_dma_alloc(struct device *dev, size_t size,
void arch_dma_free(struct device *dev, size_t size, void *cpu_addr, void arch_dma_free(struct device *dev, size_t size, void *cpu_addr,
dma_addr_t dma_addr, unsigned long attrs) dma_addr_t dma_addr, unsigned long attrs)
{ {
if (!(attrs & DMA_ATTR_NON_CONSISTENT) && !dev_is_coherent(dev)) if (!(attrs & DMA_ATTR_NON_CONSISTENT))
cpu_addr = (void *)CAC_ADDR((unsigned long)cpu_addr); cpu_addr = (void *)CAC_ADDR((unsigned long)cpu_addr);
dma_direct_free(dev, size, cpu_addr, dma_addr, attrs); dma_direct_free_pages(dev, size, cpu_addr, dma_addr, attrs);
} }
int arch_dma_mmap(struct device *dev, struct vm_area_struct *vma, long arch_dma_coherent_to_pfn(struct device *dev, void *cpu_addr,
void *cpu_addr, dma_addr_t dma_addr, size_t size, dma_addr_t dma_addr)
unsigned long attrs)
{ {
unsigned long user_count = vma_pages(vma); unsigned long addr = CAC_ADDR((unsigned long)cpu_addr);
unsigned long count = PAGE_ALIGN(size) >> PAGE_SHIFT; return page_to_pfn(virt_to_page((void *)addr));
unsigned long addr = (unsigned long)cpu_addr; }
unsigned long off = vma->vm_pgoff;
unsigned long pfn;
int ret = -ENXIO;
if (!dev_is_coherent(dev))
addr = CAC_ADDR(addr);
pfn = page_to_pfn(virt_to_page((void *)addr));
pgprot_t arch_dma_mmap_pgprot(struct device *dev, pgprot_t prot,
unsigned long attrs)
{
if (attrs & DMA_ATTR_WRITE_COMBINE) if (attrs & DMA_ATTR_WRITE_COMBINE)
vma->vm_page_prot = pgprot_writecombine(vma->vm_page_prot); return pgprot_writecombine(prot);
else return pgprot_noncached(prot);
vma->vm_page_prot = pgprot_noncached(vma->vm_page_prot);
if (dma_mmap_from_dev_coherent(dev, vma, cpu_addr, size, &ret))
return ret;
if (off < count && user_count <= (count - off)) {
ret = remap_pfn_range(vma, vma->vm_start,
pfn + off,
user_count << PAGE_SHIFT,
vma->vm_page_prot);
}
return ret;
} }
static inline void dma_sync_virt(void *addr, size_t size, static inline void dma_sync_virt(void *addr, size_t size,
...@@ -187,7 +142,6 @@ static inline void dma_sync_phys(phys_addr_t paddr, size_t size, ...@@ -187,7 +142,6 @@ static inline void dma_sync_phys(phys_addr_t paddr, size_t size,
void arch_sync_dma_for_device(struct device *dev, phys_addr_t paddr, void arch_sync_dma_for_device(struct device *dev, phys_addr_t paddr,
size_t size, enum dma_data_direction dir) size_t size, enum dma_data_direction dir)
{ {
if (!dev_is_coherent(dev))
dma_sync_phys(paddr, size, dir); dma_sync_phys(paddr, size, dir);
} }
...@@ -203,6 +157,5 @@ void arch_dma_cache_sync(struct device *dev, void *vaddr, size_t size, ...@@ -203,6 +157,5 @@ void arch_dma_cache_sync(struct device *dev, void *vaddr, size_t size,
{ {
BUG_ON(direction == DMA_NONE); BUG_ON(direction == DMA_NONE);
if (!dev_is_coherent(dev))
dma_sync_virt(vaddr, size, direction); dma_sync_virt(vaddr, size, direction);
} }
arch/nds32/Kconfig
View file @ cff22949
...@@ -11,7 +11,7 @@ config NDS32 ...@@ -11,7 +11,7 @@ config NDS32
select CLKSRC_MMIO select CLKSRC_MMIO
select CLONE_BACKWARDS select CLONE_BACKWARDS
select COMMON_CLK select COMMON_CLK
select DMA_NONCOHERENT_OPS select DMA_DIRECT_OPS
select GENERIC_ATOMIC64 select GENERIC_ATOMIC64
select GENERIC_CPU_DEVICES select GENERIC_CPU_DEVICES
select GENERIC_CLOCKEVENTS select GENERIC_CLOCKEVENTS
......
arch/nios2/Kconfig
View file @ cff22949
...@@ -4,7 +4,7 @@ config NIOS2 ...@@ -4,7 +4,7 @@ config NIOS2
select ARCH_HAS_SYNC_DMA_FOR_CPU select ARCH_HAS_SYNC_DMA_FOR_CPU
select ARCH_HAS_SYNC_DMA_FOR_DEVICE select ARCH_HAS_SYNC_DMA_FOR_DEVICE
select ARCH_NO_SWAP select ARCH_NO_SWAP
select DMA_NONCOHERENT_OPS select DMA_DIRECT_OPS
select TIMER_OF select TIMER_OF
select GENERIC_ATOMIC64 select GENERIC_ATOMIC64
select GENERIC_CLOCKEVENTS select GENERIC_CLOCKEVENTS
......
arch/openrisc/Kconfig
View file @ cff22949
...@@ -7,7 +7,7 @@ ...@@ -7,7 +7,7 @@
config OPENRISC config OPENRISC
def_bool y def_bool y
select ARCH_HAS_SYNC_DMA_FOR_DEVICE select ARCH_HAS_SYNC_DMA_FOR_DEVICE
select DMA_NONCOHERENT_OPS select DMA_DIRECT_OPS
select OF select OF
select OF_EARLY_FLATTREE select OF_EARLY_FLATTREE
select IRQ_DOMAIN select IRQ_DOMAIN
......
arch/parisc/Kconfig
View file @ cff22949
...@@ -186,7 +186,7 @@ config PA11 ...@@ -186,7 +186,7 @@ config PA11
depends on PA7000 || PA7100LC || PA7200 || PA7300LC depends on PA7000 || PA7100LC || PA7200 || PA7300LC
select ARCH_HAS_SYNC_DMA_FOR_CPU select ARCH_HAS_SYNC_DMA_FOR_CPU
select ARCH_HAS_SYNC_DMA_FOR_DEVICE select ARCH_HAS_SYNC_DMA_FOR_DEVICE
select DMA_NONCOHERENT_OPS select DMA_DIRECT_OPS
select DMA_NONCOHERENT_CACHE_SYNC select DMA_NONCOHERENT_CACHE_SYNC
config PREFETCH config PREFETCH
......
arch/parisc/kernel/setup.c
View file @ cff22949
...@@ -102,7 +102,7 @@ void __init dma_ops_init(void) ...@@ -102,7 +102,7 @@ void __init dma_ops_init(void)
case pcxl: /* falls through */ case pcxl: /* falls through */
case pcxs: case pcxs:
case pcxt: case pcxt:
hppa_dma_ops = &dma_noncoherent_ops; hppa_dma_ops = &dma_direct_ops;
break; break;
default: default:
break; break;
......
arch/sh/Kconfig
View file @ cff22949
...@@ -7,6 +7,7 @@ config SUPERH ...@@ -7,6 +7,7 @@ config SUPERH
select ARCH_NO_COHERENT_DMA_MMAP if !MMU select ARCH_NO_COHERENT_DMA_MMAP if !MMU
select HAVE_PATA_PLATFORM select HAVE_PATA_PLATFORM
select CLKDEV_LOOKUP select CLKDEV_LOOKUP
select DMA_DIRECT_OPS
select HAVE_IDE if HAS_IOPORT_MAP select HAVE_IDE if HAS_IOPORT_MAP
select HAVE_MEMBLOCK select HAVE_MEMBLOCK
select HAVE_MEMBLOCK_NODE_MAP select HAVE_MEMBLOCK_NODE_MAP
...@@ -158,13 +159,11 @@ config SWAP_IO_SPACE ...@@ -158,13 +159,11 @@ config SWAP_IO_SPACE
bool bool
config DMA_COHERENT config DMA_COHERENT
select DMA_DIRECT_OPS
bool bool
config DMA_NONCOHERENT config DMA_NONCOHERENT
def_bool !DMA_COHERENT def_bool !DMA_COHERENT
select ARCH_HAS_SYNC_DMA_FOR_DEVICE select ARCH_HAS_SYNC_DMA_FOR_DEVICE
select DMA_NONCOHERENT_OPS
config PGTABLE_LEVELS config PGTABLE_LEVELS
default 3 if X2TLB default 3 if X2TLB
......
arch/sparc/Kconfig
View file @ cff22949
...@@ -51,7 +51,7 @@ config SPARC ...@@ -51,7 +51,7 @@ config SPARC
config SPARC32 config SPARC32
def_bool !64BIT def_bool !64BIT
select ARCH_HAS_SYNC_DMA_FOR_CPU select ARCH_HAS_SYNC_DMA_FOR_CPU
select DMA_NONCOHERENT_OPS select DMA_DIRECT_OPS
select GENERIC_ATOMIC64 select GENERIC_ATOMIC64
select CLZ_TAB select CLZ_TAB
select HAVE_UID16 select HAVE_UID16
......
arch/sparc/include/asm/dma-mapping.h
View file @ cff22949
...@@ -14,11 +14,11 @@ static inline const struct dma_map_ops *get_arch_dma_ops(struct bus_type *bus) ...@@ -14,11 +14,11 @@ static inline const struct dma_map_ops *get_arch_dma_ops(struct bus_type *bus)
{ {
#ifdef CONFIG_SPARC_LEON #ifdef CONFIG_SPARC_LEON
if (sparc_cpu_model == sparc_leon) if (sparc_cpu_model == sparc_leon)
return &dma_noncoherent_ops; return &dma_direct_ops;
#endif #endif
#if defined(CONFIG_SPARC32) && defined(CONFIG_PCI) #if defined(CONFIG_SPARC32) && defined(CONFIG_PCI)
if (bus == &pci_bus_type) if (bus == &pci_bus_type)
return &dma_noncoherent_ops; return &dma_direct_ops;
#endif #endif
return dma_ops; return dma_ops;
} }
......
arch/unicore32/Kconfig
View file @ cff22949
...@@ -4,6 +4,7 @@ config UNICORE32 ...@@ -4,6 +4,7 @@ config UNICORE32
select ARCH_HAS_DEVMEM_IS_ALLOWED select ARCH_HAS_DEVMEM_IS_ALLOWED
select ARCH_MIGHT_HAVE_PC_PARPORT select ARCH_MIGHT_HAVE_PC_PARPORT
select ARCH_MIGHT_HAVE_PC_SERIO select ARCH_MIGHT_HAVE_PC_SERIO
select DMA_DIRECT_OPS
select HAVE_MEMBLOCK select HAVE_MEMBLOCK
select HAVE_GENERIC_DMA_COHERENT select HAVE_GENERIC_DMA_COHERENT
select HAVE_KERNEL_GZIP select HAVE_KERNEL_GZIP
...@@ -20,7 +21,6 @@ config UNICORE32 ...@@ -20,7 +21,6 @@ config UNICORE32
select GENERIC_IOMAP select GENERIC_IOMAP
select MODULES_USE_ELF_REL select MODULES_USE_ELF_REL
select NEED_DMA_MAP_STATE select NEED_DMA_MAP_STATE
select SWIOTLB
help help
UniCore-32 is 32-bit Instruction Set Architecture, UniCore-32 is 32-bit Instruction Set Architecture,
including a series of low-power-consumption RISC chip including a series of low-power-consumption RISC chip
......
arch/unicore32/include/asm/Kbuild
View file @ cff22949
...@@ -4,6 +4,7 @@ generic-y += compat.h ...@@ -4,6 +4,7 @@ generic-y += compat.h
generic-y += current.h generic-y += current.h
generic-y += device.h generic-y += device.h
generic-y += div64.h generic-y += div64.h
generic-y += dma-mapping.h
generic-y += emergency-restart.h generic-y += emergency-restart.h
generic-y += exec.h generic-y += exec.h
generic-y += extable.h generic-y += extable.h
......
arch/unicore32/include/asm/dma-mapping.h deleted 100644 → 0
View file @ 13775dac
/*
* linux/arch/unicore32/include/asm/dma-mapping.h
*
* Code specific to PKUnity SoC and UniCore ISA
*
* Copyright (C) 2001-2010 GUAN Xue-tao
*
* This program is free software; you can redistribute it and/or modify
* it under the terms of the GNU General Public License version 2 as
* published by the Free Software Foundation.
*/
#ifndef __UNICORE_DMA_MAPPING_H__
#define __UNICORE_DMA_MAPPING_H__
#include <linux/swiotlb.h>
static inline const struct dma_map_ops *get_arch_dma_ops(struct bus_type *bus)
{
return &swiotlb_dma_ops;
}
#endif
arch/unicore32/mm/init.c
View file @ cff22949
...@@ -234,9 +234,6 @@ void __init bootmem_init(void) ...@@ -234,9 +234,6 @@ void __init bootmem_init(void)
uc32_bootmem_init(min, max_low); uc32_bootmem_init(min, max_low);
#ifdef CONFIG_SWIOTLB
swiotlb_init(1);
#endif
/* /*
* Sparsemem tries to allocate bootmem in memory_present(), * Sparsemem tries to allocate bootmem in memory_present(),
* so must be done after the fixed reservations * so must be done after the fixed reservations
......
arch/x86/kernel/amd_gart_64.c
View file @ cff22949
...@@ -482,7 +482,7 @@ gart_alloc_coherent(struct device *dev, size_t size, dma_addr_t *dma_addr, ...@@ -482,7 +482,7 @@ gart_alloc_coherent(struct device *dev, size_t size, dma_addr_t *dma_addr,
{ {
void *vaddr; void *vaddr;
vaddr = dma_direct_alloc(dev, size, dma_addr, flag, attrs); vaddr = dma_direct_alloc_pages(dev, size, dma_addr, flag, attrs);
if (!vaddr || if (!vaddr ||
!force_iommu || dev->coherent_dma_mask <= DMA_BIT_MASK(24)) !force_iommu || dev->coherent_dma_mask <= DMA_BIT_MASK(24))
return vaddr; return vaddr;
...@@ -494,7 +494,7 @@ gart_alloc_coherent(struct device *dev, size_t size, dma_addr_t *dma_addr, ...@@ -494,7 +494,7 @@ gart_alloc_coherent(struct device *dev, size_t size, dma_addr_t *dma_addr,
goto out_free; goto out_free;
return vaddr; return vaddr;
out_free: out_free:
dma_direct_free(dev, size, vaddr, *dma_addr, attrs); dma_direct_free_pages(dev, size, vaddr, *dma_addr, attrs);
return NULL; return NULL;
} }
...@@ -504,7 +504,7 @@ gart_free_coherent(struct device *dev, size_t size, void *vaddr, ...@@ -504,7 +504,7 @@ gart_free_coherent(struct device *dev, size_t size, void *vaddr,
dma_addr_t dma_addr, unsigned long attrs) dma_addr_t dma_addr, unsigned long attrs)
{ {
gart_unmap_page(dev, dma_addr, size, DMA_BIDIRECTIONAL, 0); gart_unmap_page(dev, dma_addr, size, DMA_BIDIRECTIONAL, 0);
dma_direct_free(dev, size, vaddr, dma_addr, attrs); dma_direct_free_pages(dev, size, vaddr, dma_addr, attrs);
} }
static int gart_mapping_error(struct device *dev, dma_addr_t dma_addr) static int gart_mapping_error(struct device *dev, dma_addr_t dma_addr)
......
arch/xtensa/Kconfig
View file @ cff22949
...@@ -13,7 +13,7 @@ config XTENSA ...@@ -13,7 +13,7 @@ config XTENSA
select BUILDTIME_EXTABLE_SORT select BUILDTIME_EXTABLE_SORT
select CLONE_BACKWARDS select CLONE_BACKWARDS
select COMMON_CLK select COMMON_CLK
select DMA_NONCOHERENT_OPS select DMA_DIRECT_OPS
select GENERIC_ATOMIC64 select GENERIC_ATOMIC64
select GENERIC_CLOCKEVENTS select GENERIC_CLOCKEVENTS
select GENERIC_IRQ_SHOW select GENERIC_IRQ_SHOW
......
drivers/acpi/arm64/iort.c
View file @ cff22949
...@@ -1428,7 +1428,7 @@ static int __init iort_add_platform_device(struct acpi_iort_node *node, ...@@ -1428,7 +1428,7 @@ static int __init iort_add_platform_device(struct acpi_iort_node *node,
return 0; return 0;
dma_deconfigure: dma_deconfigure:
acpi_dma_deconfigure(&pdev->dev); arch_teardown_dma_ops(&pdev->dev);
dev_put: dev_put:
platform_device_put(pdev); platform_device_put(pdev);
......
drivers/acpi/scan.c
View file @ cff22949
...@@ -1469,16 +1469,6 @@ int acpi_dma_configure(struct device *dev, enum dev_dma_attr attr) ...@@ -1469,16 +1469,6 @@ int acpi_dma_configure(struct device *dev, enum dev_dma_attr attr)
} }
EXPORT_SYMBOL_GPL(acpi_dma_configure); EXPORT_SYMBOL_GPL(acpi_dma_configure);
/**
* acpi_dma_deconfigure - Tear-down DMA configuration for the device.
* @dev: The pointer to the device
*/
void acpi_dma_deconfigure(struct device *dev)
{
arch_teardown_dma_ops(dev);
}
EXPORT_SYMBOL_GPL(acpi_dma_deconfigure);
static void acpi_init_coherency(struct acpi_device *adev) static void acpi_init_coherency(struct acpi_device *adev)
{ {
unsigned long long cca = 0; unsigned long long cca = 0;
......
drivers/base/dd.c
View file @ cff22949
...@@ -480,9 +480,11 @@ static int really_probe(struct device *dev, struct device_driver *drv) ...@@ -480,9 +480,11 @@ static int really_probe(struct device *dev, struct device_driver *drv)
if (ret) if (ret)
goto pinctrl_bind_failed; goto pinctrl_bind_failed;
ret = dma_configure(dev); if (dev->bus->dma_configure) {
ret = dev->bus->dma_configure(dev);
if (ret) if (ret)
goto dma_failed; goto dma_failed;
}
if (driver_sysfs_add(dev)) { if (driver_sysfs_add(dev)) {
printk(KERN_ERR "%s: driver_sysfs_add(%s) failed\n", printk(KERN_ERR "%s: driver_sysfs_add(%s) failed\n",
...@@ -537,7 +539,7 @@ static int really_probe(struct device *dev, struct device_driver *drv) ...@@ -537,7 +539,7 @@ static int really_probe(struct device *dev, struct device_driver *drv)
goto done; goto done;
probe_failed: probe_failed:
dma_deconfigure(dev); arch_teardown_dma_ops(dev);
dma_failed: dma_failed:
if (dev->bus) if (dev->bus)
blocking_notifier_call_chain(&dev->bus->p->bus_notifier, blocking_notifier_call_chain(&dev->bus->p->bus_notifier,
...@@ -966,7 +968,7 @@ static void __device_release_driver(struct device *dev, struct device *parent) ...@@ -966,7 +968,7 @@ static void __device_release_driver(struct device *dev, struct device *parent)
drv->remove(dev); drv->remove(dev);
device_links_driver_cleanup(dev); device_links_driver_cleanup(dev);
dma_deconfigure(dev); arch_teardown_dma_ops(dev);
devres_release_all(dev); devres_release_all(dev);
dev->driver = NULL; dev->driver = NULL;
......
drivers/base/platform.c
View file @ cff22949
...@@ -1180,7 +1180,7 @@ int __init platform_bus_init(void) ...@@ -1180,7 +1180,7 @@ int __init platform_bus_init(void)
} }
#ifndef ARCH_HAS_DMA_GET_REQUIRED_MASK #ifndef ARCH_HAS_DMA_GET_REQUIRED_MASK
u64 dma_get_required_mask(struct device *dev) static u64 dma_default_get_required_mask(struct device *dev)
{ {
u32 low_totalram = ((max_pfn - 1) << PAGE_SHIFT); u32 low_totalram = ((max_pfn - 1) << PAGE_SHIFT);
u32 high_totalram = ((max_pfn - 1) >> (32 - PAGE_SHIFT)); u32 high_totalram = ((max_pfn - 1) >> (32 - PAGE_SHIFT));
...@@ -1198,6 +1198,15 @@ u64 dma_get_required_mask(struct device *dev) ...@@ -1198,6 +1198,15 @@ u64 dma_get_required_mask(struct device *dev)
} }
return mask; return mask;
} }
u64 dma_get_required_mask(struct device *dev)
{
const struct dma_map_ops *ops = get_dma_ops(dev);
if (ops->get_required_mask)
return ops->get_required_mask(dev);
return dma_default_get_required_mask(dev);
}
EXPORT_SYMBOL_GPL(dma_get_required_mask); EXPORT_SYMBOL_GPL(dma_get_required_mask);
#endif #endif
......
drivers/of/device.c
View file @ cff22949
...@@ -170,18 +170,6 @@ int of_dma_configure(struct device *dev, struct device_node *np, bool force_dma) ...@@ -170,18 +170,6 @@ int of_dma_configure(struct device *dev, struct device_node *np, bool force_dma)
} }
EXPORT_SYMBOL_GPL(of_dma_configure); EXPORT_SYMBOL_GPL(of_dma_configure);
/**
* of_dma_deconfigure - Clean up DMA configuration
* @dev: Device for which to clean up DMA configuration
*
* Clean up all configuration performed by of_dma_configure_ops() and free all
* resources that have been allocated.
*/
void of_dma_deconfigure(struct device *dev)
{
arch_teardown_dma_ops(dev);
}
int of_device_register(struct platform_device *pdev) int of_device_register(struct platform_device *pdev)
{ {
device_initialize(&pdev->dev); device_initialize(&pdev->dev);
......
drivers/pci/controller/vmd.c
View file @ cff22949
...@@ -404,12 +404,10 @@ static int vmd_dma_supported(struct device *dev, u64 mask) ...@@ -404,12 +404,10 @@ static int vmd_dma_supported(struct device *dev, u64 mask)
return vmd_dma_ops(dev)->dma_supported(to_vmd_dev(dev), mask); return vmd_dma_ops(dev)->dma_supported(to_vmd_dev(dev), mask);
} }
#ifdef ARCH_HAS_DMA_GET_REQUIRED_MASK
static u64 vmd_get_required_mask(struct device *dev) static u64 vmd_get_required_mask(struct device *dev)
{ {
return vmd_dma_ops(dev)->get_required_mask(to_vmd_dev(dev)); return vmd_dma_ops(dev)->get_required_mask(to_vmd_dev(dev));
} }
#endif
static void vmd_teardown_dma_ops(struct vmd_dev *vmd) static void vmd_teardown_dma_ops(struct vmd_dev *vmd)
{ {
...@@ -450,9 +448,7 @@ static void vmd_setup_dma_ops(struct vmd_dev *vmd) ...@@ -450,9 +448,7 @@ static void vmd_setup_dma_ops(struct vmd_dev *vmd)
ASSIGN_VMD_DMA_OPS(source, dest, sync_sg_for_device); ASSIGN_VMD_DMA_OPS(source, dest, sync_sg_for_device);
ASSIGN_VMD_DMA_OPS(source, dest, mapping_error); ASSIGN_VMD_DMA_OPS(source, dest, mapping_error);
ASSIGN_VMD_DMA_OPS(source, dest, dma_supported); ASSIGN_VMD_DMA_OPS(source, dest, dma_supported);
#ifdef ARCH_HAS_DMA_GET_REQUIRED_MASK
ASSIGN_VMD_DMA_OPS(source, dest, get_required_mask); ASSIGN_VMD_DMA_OPS(source, dest, get_required_mask);
#endif
add_dma_domain(domain); add_dma_domain(domain);
} }
#undef ASSIGN_VMD_DMA_OPS #undef ASSIGN_VMD_DMA_OPS
......
drivers/xen/swiotlb-xen.c
View file @ cff22949
...@@ -662,7 +662,7 @@ xen_swiotlb_dma_mmap(struct device *dev, struct vm_area_struct *vma, ...@@ -662,7 +662,7 @@ xen_swiotlb_dma_mmap(struct device *dev, struct vm_area_struct *vma,
return xen_get_dma_ops(dev)->mmap(dev, vma, cpu_addr, return xen_get_dma_ops(dev)->mmap(dev, vma, cpu_addr,
dma_addr, size, attrs); dma_addr, size, attrs);
#endif #endif
return dma_common_mmap(dev, vma, cpu_addr, dma_addr, size); return dma_common_mmap(dev, vma, cpu_addr, dma_addr, size, attrs);
} }
/* /*
...@@ -689,7 +689,7 @@ xen_swiotlb_get_sgtable(struct device *dev, struct sg_table *sgt, ...@@ -689,7 +689,7 @@ xen_swiotlb_get_sgtable(struct device *dev, struct sg_table *sgt,
handle, size, attrs); handle, size, attrs);
} }
#endif #endif
return dma_common_get_sgtable(dev, sgt, cpu_addr, handle, size); return dma_common_get_sgtable(dev, sgt, cpu_addr, handle, size, attrs);
} }
static int xen_swiotlb_mapping_error(struct device *dev, dma_addr_t dma_addr) static int xen_swiotlb_mapping_error(struct device *dev, dma_addr_t dma_addr)
......
include/acpi/acpi_bus.h
View file @ cff22949
...@@ -595,7 +595,6 @@ enum dev_dma_attr acpi_get_dma_attr(struct acpi_device *adev); ...@@ -595,7 +595,6 @@ enum dev_dma_attr acpi_get_dma_attr(struct acpi_device *adev);
int acpi_dma_get_range(struct device *dev, u64 *dma_addr, u64 *offset, int acpi_dma_get_range(struct device *dev, u64 *dma_addr, u64 *offset,
u64 *size); u64 *size);
int acpi_dma_configure(struct device *dev, enum dev_dma_attr attr); int acpi_dma_configure(struct device *dev, enum dev_dma_attr attr);
void acpi_dma_deconfigure(struct device *dev);
struct acpi_device *acpi_find_child_device(struct acpi_device *parent, struct acpi_device *acpi_find_child_device(struct acpi_device *parent,
u64 address, bool check_children); u64 address, bool check_children);
......
include/asm-generic/dma-mapping.h
View file @ cff22949
...@@ -4,16 +4,7 @@ ...@@ -4,16 +4,7 @@
static inline const struct dma_map_ops *get_arch_dma_ops(struct bus_type *bus) static inline const struct dma_map_ops *get_arch_dma_ops(struct bus_type *bus)
{ {
/*
* Use the non-coherent ops if available. If an architecture wants a
* more fine-grained selection of operations it will have to implement
* get_arch_dma_ops itself or use the per-device dma_ops.
*/
#ifdef CONFIG_DMA_NONCOHERENT_OPS
return &dma_noncoherent_ops;
#else
return &dma_direct_ops; return &dma_direct_ops;
#endif
} }
#endif /* _ASM_GENERIC_DMA_MAPPING_H */ #endif /* _ASM_GENERIC_DMA_MAPPING_H */
include/linux/acpi.h
View file @ cff22949
...@@ -831,8 +831,6 @@ static inline int acpi_dma_configure(struct device *dev, ...@@ -831,8 +831,6 @@ static inline int acpi_dma_configure(struct device *dev,
return 0; return 0;
} }
static inline void acpi_dma_deconfigure(struct device *dev) { }
#define ACPI_PTR(_ptr) (NULL) #define ACPI_PTR(_ptr) (NULL)
static inline void acpi_device_set_enumerated(struct acpi_device *adev) static inline void acpi_device_set_enumerated(struct acpi_device *adev)
......
include/linux/device.h
View file @ cff22949
...@@ -927,6 +927,8 @@ struct dev_links_info { ...@@ -927,6 +927,8 @@ struct dev_links_info {
* @offline: Set after successful invocation of bus type's .offline(). * @offline: Set after successful invocation of bus type's .offline().
* @of_node_reused: Set if the device-tree node is shared with an ancestor * @of_node_reused: Set if the device-tree node is shared with an ancestor
* device. * device.
* @dma_coherent: this particular device is dma coherent, even if the
* architecture supports non-coherent devices.
* *
* At the lowest level, every device in a Linux system is represented by an * At the lowest level, every device in a Linux system is represented by an
* instance of struct device. The device structure contains the information * instance of struct device. The device structure contains the information
...@@ -1016,6 +1018,11 @@ struct device { ...@@ -1016,6 +1018,11 @@ struct device {
bool offline_disabled:1; bool offline_disabled:1;
bool offline:1; bool offline:1;
bool of_node_reused:1; bool of_node_reused:1;
#if defined(CONFIG_ARCH_HAS_SYNC_DMA_FOR_DEVICE) || \
defined(CONFIG_ARCH_HAS_SYNC_DMA_FOR_CPU) || \
defined(CONFIG_ARCH_HAS_SYNC_DMA_FOR_CPU_ALL)
bool dma_coherent:1;
#endif
}; };
static inline struct device *kobj_to_dev(struct kobject *kobj) static inline struct device *kobj_to_dev(struct kobject *kobj)
......
include/linux/dma-debug.h
View file @ cff22949
...@@ -32,6 +32,9 @@ extern void dma_debug_add_bus(struct bus_type *bus); ...@@ -32,6 +32,9 @@ extern void dma_debug_add_bus(struct bus_type *bus);
extern int dma_debug_resize_entries(u32 num_entries); extern int dma_debug_resize_entries(u32 num_entries);
extern void debug_dma_map_single(struct device *dev, const void *addr,
unsigned long len);
extern void debug_dma_map_page(struct device *dev, struct page *page, extern void debug_dma_map_page(struct device *dev, struct page *page,
size_t offset, size_t size, size_t offset, size_t size,
int direction, dma_addr_t dma_addr, int direction, dma_addr_t dma_addr,
...@@ -103,6 +106,11 @@ static inline int dma_debug_resize_entries(u32 num_entries) ...@@ -103,6 +106,11 @@ static inline int dma_debug_resize_entries(u32 num_entries)
return 0; return 0;
} }
static inline void debug_dma_map_single(struct device *dev, const void *addr,
unsigned long len)
{
}
static inline void debug_dma_map_page(struct device *dev, struct page *page, static inline void debug_dma_map_page(struct device *dev, struct page *page,
size_t offset, size_t size, size_t offset, size_t size,
int direction, dma_addr_t dma_addr, int direction, dma_addr_t dma_addr,
......
include/linux/dma-direct.h
View file @ cff22949
...@@ -27,7 +27,8 @@ static inline bool dma_capable(struct device *dev, dma_addr_t addr, size_t size) ...@@ -27,7 +27,8 @@ static inline bool dma_capable(struct device *dev, dma_addr_t addr, size_t size)
if (!dev->dma_mask) if (!dev->dma_mask)
return false; return false;
return addr + size - 1 <= *dev->dma_mask; return addr + size - 1 <=
min_not_zero(*dev->dma_mask, dev->bus_dma_mask);
} }
#endif /* !CONFIG_ARCH_HAS_PHYS_TO_DMA */ #endif /* !CONFIG_ARCH_HAS_PHYS_TO_DMA */
...@@ -55,10 +56,15 @@ static inline void dma_mark_clean(void *addr, size_t size) ...@@ -55,10 +56,15 @@ static inline void dma_mark_clean(void *addr, size_t size)
} }
#endif /* CONFIG_ARCH_HAS_DMA_MARK_CLEAN */ #endif /* CONFIG_ARCH_HAS_DMA_MARK_CLEAN */
u64 dma_direct_get_required_mask(struct device *dev);
void *dma_direct_alloc(struct device *dev, size_t size, dma_addr_t *dma_handle, void *dma_direct_alloc(struct device *dev, size_t size, dma_addr_t *dma_handle,
gfp_t gfp, unsigned long attrs); gfp_t gfp, unsigned long attrs);
void dma_direct_free(struct device *dev, size_t size, void *cpu_addr, void dma_direct_free(struct device *dev, size_t size, void *cpu_addr,
dma_addr_t dma_addr, unsigned long attrs); dma_addr_t dma_addr, unsigned long attrs);
void *dma_direct_alloc_pages(struct device *dev, size_t size,
dma_addr_t *dma_handle, gfp_t gfp, unsigned long attrs);
void dma_direct_free_pages(struct device *dev, size_t size, void *cpu_addr,
dma_addr_t dma_addr, unsigned long attrs);
dma_addr_t dma_direct_map_page(struct device *dev, struct page *page, dma_addr_t dma_direct_map_page(struct device *dev, struct page *page,
unsigned long offset, size_t size, enum dma_data_direction dir, unsigned long offset, size_t size, enum dma_data_direction dir,
unsigned long attrs); unsigned long attrs);
......
include/linux/dma-mapping.h
View file @ cff22949
...@@ -130,13 +130,10 @@ struct dma_map_ops { ...@@ -130,13 +130,10 @@ struct dma_map_ops {
enum dma_data_direction direction); enum dma_data_direction direction);
int (*mapping_error)(struct device *dev, dma_addr_t dma_addr); int (*mapping_error)(struct device *dev, dma_addr_t dma_addr);
int (*dma_supported)(struct device *dev, u64 mask); int (*dma_supported)(struct device *dev, u64 mask);
#ifdef ARCH_HAS_DMA_GET_REQUIRED_MASK
u64 (*get_required_mask)(struct device *dev); u64 (*get_required_mask)(struct device *dev);
#endif
}; };
extern const struct dma_map_ops dma_direct_ops; extern const struct dma_map_ops dma_direct_ops;
extern const struct dma_map_ops dma_noncoherent_ops;
extern const struct dma_map_ops dma_virt_ops; extern const struct dma_map_ops dma_virt_ops;
#define DMA_BIT_MASK(n) (((n) == 64) ? ~0ULL : ((1ULL<<(n))-1)) #define DMA_BIT_MASK(n) (((n) == 64) ? ~0ULL : ((1ULL<<(n))-1))
...@@ -232,6 +229,7 @@ static inline dma_addr_t dma_map_single_attrs(struct device *dev, void *ptr, ...@@ -232,6 +229,7 @@ static inline dma_addr_t dma_map_single_attrs(struct device *dev, void *ptr,
dma_addr_t addr; dma_addr_t addr;
BUG_ON(!valid_dma_direction(dir)); BUG_ON(!valid_dma_direction(dir));
debug_dma_map_single(dev, ptr, size);
addr = ops->map_page(dev, virt_to_page(ptr), addr = ops->map_page(dev, virt_to_page(ptr),
offset_in_page(ptr), size, offset_in_page(ptr), size,
dir, attrs); dir, attrs);
...@@ -445,7 +443,8 @@ dma_cache_sync(struct device *dev, void *vaddr, size_t size, ...@@ -445,7 +443,8 @@ dma_cache_sync(struct device *dev, void *vaddr, size_t size,
} }
extern int dma_common_mmap(struct device *dev, struct vm_area_struct *vma, extern int dma_common_mmap(struct device *dev, struct vm_area_struct *vma,
void *cpu_addr, dma_addr_t dma_addr, size_t size); void *cpu_addr, dma_addr_t dma_addr, size_t size,
unsigned long attrs);
void *dma_common_contiguous_remap(struct page *page, size_t size, void *dma_common_contiguous_remap(struct page *page, size_t size,
unsigned long vm_flags, unsigned long vm_flags,
...@@ -477,14 +476,14 @@ dma_mmap_attrs(struct device *dev, struct vm_area_struct *vma, void *cpu_addr, ...@@ -477,14 +476,14 @@ dma_mmap_attrs(struct device *dev, struct vm_area_struct *vma, void *cpu_addr,
BUG_ON(!ops); BUG_ON(!ops);
if (ops->mmap) if (ops->mmap)
return ops->mmap(dev, vma, cpu_addr, dma_addr, size, attrs); return ops->mmap(dev, vma, cpu_addr, dma_addr, size, attrs);
return dma_common_mmap(dev, vma, cpu_addr, dma_addr, size); return dma_common_mmap(dev, vma, cpu_addr, dma_addr, size, attrs);
} }
#define dma_mmap_coherent(d, v, c, h, s) dma_mmap_attrs(d, v, c, h, s, 0) #define dma_mmap_coherent(d, v, c, h, s) dma_mmap_attrs(d, v, c, h, s, 0)
int int
dma_common_get_sgtable(struct device *dev, struct sg_table *sgt, dma_common_get_sgtable(struct device *dev, struct sg_table *sgt, void *cpu_addr,
void *cpu_addr, dma_addr_t dma_addr, size_t size); dma_addr_t dma_addr, size_t size, unsigned long attrs);
static inline int static inline int
dma_get_sgtable_attrs(struct device *dev, struct sg_table *sgt, void *cpu_addr, dma_get_sgtable_attrs(struct device *dev, struct sg_table *sgt, void *cpu_addr,
...@@ -496,7 +495,8 @@ dma_get_sgtable_attrs(struct device *dev, struct sg_table *sgt, void *cpu_addr, ...@@ -496,7 +495,8 @@ dma_get_sgtable_attrs(struct device *dev, struct sg_table *sgt, void *cpu_addr,
if (ops->get_sgtable) if (ops->get_sgtable)
return ops->get_sgtable(dev, sgt, cpu_addr, dma_addr, size, return ops->get_sgtable(dev, sgt, cpu_addr, dma_addr, size,
attrs); attrs);
return dma_common_get_sgtable(dev, sgt, cpu_addr, dma_addr, size); return dma_common_get_sgtable(dev, sgt, cpu_addr, dma_addr, size,
attrs);
} }
#define dma_get_sgtable(d, t, v, h, s) dma_get_sgtable_attrs(d, t, v, h, s, 0) #define dma_get_sgtable(d, t, v, h, s) dma_get_sgtable_attrs(d, t, v, h, s, 0)
...@@ -558,9 +558,11 @@ static inline void dma_free_attrs(struct device *dev, size_t size, ...@@ -558,9 +558,11 @@ static inline void dma_free_attrs(struct device *dev, size_t size,
} }
static inline void *dma_alloc_coherent(struct device *dev, size_t size, static inline void *dma_alloc_coherent(struct device *dev, size_t size,
dma_addr_t *dma_handle, gfp_t flag) dma_addr_t *dma_handle, gfp_t gfp)
{ {
return dma_alloc_attrs(dev, size, dma_handle, flag, 0);
return dma_alloc_attrs(dev, size, dma_handle, gfp,
(gfp & __GFP_NOWARN) ? DMA_ATTR_NO_WARN : 0);
} }
static inline void dma_free_coherent(struct device *dev, size_t size, static inline void dma_free_coherent(struct device *dev, size_t size,
...@@ -753,18 +755,6 @@ dma_mark_declared_memory_occupied(struct device *dev, ...@@ -753,18 +755,6 @@ dma_mark_declared_memory_occupied(struct device *dev,
} }
#endif /* CONFIG_HAVE_GENERIC_DMA_COHERENT */ #endif /* CONFIG_HAVE_GENERIC_DMA_COHERENT */
#ifdef CONFIG_HAS_DMA
int dma_configure(struct device *dev);
void dma_deconfigure(struct device *dev);
#else
static inline int dma_configure(struct device *dev)
{
return 0;
}
static inline void dma_deconfigure(struct device *dev) {}
#endif
/* /*
* Managed DMA API * Managed DMA API
*/ */
...@@ -806,8 +796,12 @@ static inline void dmam_release_declared_memory(struct device *dev) ...@@ -806,8 +796,12 @@ static inline void dmam_release_declared_memory(struct device *dev)
static inline void *dma_alloc_wc(struct device *dev, size_t size, static inline void *dma_alloc_wc(struct device *dev, size_t size,
dma_addr_t *dma_addr, gfp_t gfp) dma_addr_t *dma_addr, gfp_t gfp)
{ {
return dma_alloc_attrs(dev, size, dma_addr, gfp, unsigned long attrs = DMA_ATTR_NO_WARN;
DMA_ATTR_WRITE_COMBINE);
if (gfp & __GFP_NOWARN)
attrs |= DMA_ATTR_NO_WARN;
return dma_alloc_attrs(dev, size, dma_addr, gfp, attrs);
} }
#ifndef dma_alloc_writecombine #ifndef dma_alloc_writecombine
#define dma_alloc_writecombine dma_alloc_wc #define dma_alloc_writecombine dma_alloc_wc
......
include/linux/dma-noncoherent.h
View file @ cff22949
...@@ -4,18 +4,35 @@ ...@@ -4,18 +4,35 @@
#include <linux/dma-mapping.h> #include <linux/dma-mapping.h>
#ifdef CONFIG_ARCH_HAS_DMA_COHERENCE_H
#include <asm/dma-coherence.h>
#elif defined(CONFIG_ARCH_HAS_SYNC_DMA_FOR_DEVICE) || \
defined(CONFIG_ARCH_HAS_SYNC_DMA_FOR_CPU) || \
defined(CONFIG_ARCH_HAS_SYNC_DMA_FOR_CPU_ALL)
static inline bool dev_is_dma_coherent(struct device *dev)
{
return dev->dma_coherent;
}
#else
static inline bool dev_is_dma_coherent(struct device *dev)
{
return true;
}
#endif /* CONFIG_ARCH_HAS_DMA_COHERENCE_H */
void *arch_dma_alloc(struct device *dev, size_t size, dma_addr_t *dma_handle, void *arch_dma_alloc(struct device *dev, size_t size, dma_addr_t *dma_handle,
gfp_t gfp, unsigned long attrs); gfp_t gfp, unsigned long attrs);
void arch_dma_free(struct device *dev, size_t size, void *cpu_addr, void arch_dma_free(struct device *dev, size_t size, void *cpu_addr,
dma_addr_t dma_addr, unsigned long attrs); dma_addr_t dma_addr, unsigned long attrs);
long arch_dma_coherent_to_pfn(struct device *dev, void *cpu_addr,
dma_addr_t dma_addr);
#ifdef CONFIG_DMA_NONCOHERENT_MMAP #ifdef CONFIG_ARCH_HAS_DMA_MMAP_PGPROT
int arch_dma_mmap(struct device *dev, struct vm_area_struct *vma, pgprot_t arch_dma_mmap_pgprot(struct device *dev, pgprot_t prot,
void *cpu_addr, dma_addr_t dma_addr, size_t size,
unsigned long attrs); unsigned long attrs);
#else #else
#define arch_dma_mmap NULL # define arch_dma_mmap_pgprot(dev, prot, attrs) pgprot_noncached(prot)
#endif /* CONFIG_DMA_NONCOHERENT_MMAP */ #endif
#ifdef CONFIG_DMA_NONCOHERENT_CACHE_SYNC #ifdef CONFIG_DMA_NONCOHERENT_CACHE_SYNC
void arch_dma_cache_sync(struct device *dev, void *vaddr, size_t size, void arch_dma_cache_sync(struct device *dev, void *vaddr, size_t size,
......
include/linux/of_device.h
View file @ cff22949
...@@ -58,7 +58,6 @@ static inline struct device_node *of_cpu_device_node_get(int cpu) ...@@ -58,7 +58,6 @@ static inline struct device_node *of_cpu_device_node_get(int cpu)
int of_dma_configure(struct device *dev, int of_dma_configure(struct device *dev,
struct device_node *np, struct device_node *np,
bool force_dma); bool force_dma);
void of_dma_deconfigure(struct device *dev);
#else /* CONFIG_OF */ #else /* CONFIG_OF */
static inline int of_driver_match_device(struct device *dev, static inline int of_driver_match_device(struct device *dev,
...@@ -113,8 +112,6 @@ static inline int of_dma_configure(struct device *dev, ...@@ -113,8 +112,6 @@ static inline int of_dma_configure(struct device *dev,
{ {
return 0; return 0;
} }
static inline void of_dma_deconfigure(struct device *dev)
{}
#endif /* CONFIG_OF */ #endif /* CONFIG_OF */
#endif /* _LINUX_OF_DEVICE_H */ #endif /* _LINUX_OF_DEVICE_H */
kernel/dma/Kconfig
View file @ cff22949
...@@ -13,6 +13,9 @@ config NEED_DMA_MAP_STATE ...@@ -13,6 +13,9 @@ config NEED_DMA_MAP_STATE
config ARCH_DMA_ADDR_T_64BIT config ARCH_DMA_ADDR_T_64BIT
def_bool 64BIT || PHYS_ADDR_T_64BIT def_bool 64BIT || PHYS_ADDR_T_64BIT
config ARCH_HAS_DMA_COHERENCE_H
bool
config HAVE_GENERIC_DMA_COHERENT config HAVE_GENERIC_DMA_COHERENT
bool bool
...@@ -26,22 +29,19 @@ config ARCH_HAS_SYNC_DMA_FOR_CPU ...@@ -26,22 +29,19 @@ config ARCH_HAS_SYNC_DMA_FOR_CPU
config ARCH_HAS_SYNC_DMA_FOR_CPU_ALL config ARCH_HAS_SYNC_DMA_FOR_CPU_ALL
bool bool
config DMA_DIRECT_OPS config ARCH_HAS_DMA_COHERENT_TO_PFN
bool bool
depends on HAS_DMA
config DMA_NONCOHERENT_OPS config ARCH_HAS_DMA_MMAP_PGPROT
bool bool
depends on HAS_DMA
select DMA_DIRECT_OPS
config DMA_NONCOHERENT_MMAP config DMA_DIRECT_OPS
bool bool
depends on DMA_NONCOHERENT_OPS depends on HAS_DMA
config DMA_NONCOHERENT_CACHE_SYNC config DMA_NONCOHERENT_CACHE_SYNC
bool bool
depends on DMA_NONCOHERENT_OPS depends on DMA_DIRECT_OPS
config DMA_VIRT_OPS config DMA_VIRT_OPS
bool bool
......
kernel/dma/Makefile
View file @ cff22949
...@@ -4,7 +4,6 @@ obj-$(CONFIG_HAS_DMA) += mapping.o ...@@ -4,7 +4,6 @@ obj-$(CONFIG_HAS_DMA) += mapping.o
obj-$(CONFIG_DMA_CMA) += contiguous.o obj-$(CONFIG_DMA_CMA) += contiguous.o
obj-$(CONFIG_HAVE_GENERIC_DMA_COHERENT) += coherent.o obj-$(CONFIG_HAVE_GENERIC_DMA_COHERENT) += coherent.o
obj-$(CONFIG_DMA_DIRECT_OPS) += direct.o obj-$(CONFIG_DMA_DIRECT_OPS) += direct.o
obj-$(CONFIG_DMA_NONCOHERENT_OPS) += noncoherent.o
obj-$(CONFIG_DMA_VIRT_OPS) += virt.o obj-$(CONFIG_DMA_VIRT_OPS) += virt.o
obj-$(CONFIG_DMA_API_DEBUG) += debug.o obj-$(CONFIG_DMA_API_DEBUG) += debug.o
obj-$(CONFIG_SWIOTLB) += swiotlb.o obj-$(CONFIG_SWIOTLB) += swiotlb.o
......
kernel/dma/contiguous.c
View file @ cff22949
...@@ -49,7 +49,11 @@ static phys_addr_t limit_cmdline; ...@@ -49,7 +49,11 @@ static phys_addr_t limit_cmdline;
static int __init early_cma(char *p) static int __init early_cma(char *p)
{ {
pr_debug("%s(%s)\n", __func__, p); if (!p) {
pr_err("Config string not provided\n");
return -EINVAL;
}
size_cmdline = memparse(p, &p); size_cmdline = memparse(p, &p);
if (*p != '@') if (*p != '@')
return 0; return 0;
......
kernel/dma/debug.c
View file @ cff22949
...@@ -1312,6 +1312,22 @@ static void check_sg_segment(struct device *dev, struct scatterlist *sg) ...@@ -1312,6 +1312,22 @@ static void check_sg_segment(struct device *dev, struct scatterlist *sg)
#endif #endif
} }
void debug_dma_map_single(struct device *dev, const void *addr,
unsigned long len)
{
if (unlikely(dma_debug_disabled()))
return;
if (!virt_addr_valid(addr))
err_printk(dev, NULL, "DMA-API: device driver maps memory from invalid area [addr=%p] [len=%lu]\n",
addr, len);
if (is_vmalloc_addr(addr))
err_printk(dev, NULL, "DMA-API: device driver maps memory from vmalloc area [addr=%p] [len=%lu]\n",
addr, len);
}
EXPORT_SYMBOL(debug_dma_map_single);
void debug_dma_map_page(struct device *dev, struct page *page, size_t offset, void debug_dma_map_page(struct device *dev, struct page *page, size_t offset,
size_t size, int direction, dma_addr_t dma_addr, size_t size, int direction, dma_addr_t dma_addr,
bool map_single) bool map_single)
......
kernel/dma/direct.c
View file @ cff22949
// SPDX-License-Identifier: GPL-2.0 // SPDX-License-Identifier: GPL-2.0
/* /*
* DMA operations that map physical memory directly without using an IOMMU or * Copyright (C) 2018 Christoph Hellwig.
* flushing caches. *
* DMA operations that map physical memory directly without using an IOMMU.
*/ */
#include <linux/bootmem.h> /* for max_pfn */
#include <linux/export.h> #include <linux/export.h>
#include <linux/mm.h> #include <linux/mm.h>
#include <linux/dma-direct.h> #include <linux/dma-direct.h>
#include <linux/scatterlist.h> #include <linux/scatterlist.h>
#include <linux/dma-contiguous.h> #include <linux/dma-contiguous.h>
#include <linux/dma-noncoherent.h>
#include <linux/pfn.h> #include <linux/pfn.h>
#include <linux/set_memory.h> #include <linux/set_memory.h>
...@@ -41,40 +44,83 @@ check_addr(struct device *dev, dma_addr_t dma_addr, size_t size, ...@@ -41,40 +44,83 @@ check_addr(struct device *dev, dma_addr_t dma_addr, size_t size,
return false; return false;
} }
if (*dev->dma_mask >= DMA_BIT_MASK(32)) { if (*dev->dma_mask >= DMA_BIT_MASK(32) || dev->bus_dma_mask) {
dev_err(dev, dev_err(dev,
"%s: overflow %pad+%zu of device mask %llx\n", "%s: overflow %pad+%zu of device mask %llx bus mask %llx\n",
caller, &dma_addr, size, *dev->dma_mask); caller, &dma_addr, size,
*dev->dma_mask, dev->bus_dma_mask);
} }
return false; return false;
} }
return true; return true;
} }
static inline dma_addr_t phys_to_dma_direct(struct device *dev,
phys_addr_t phys)
{
if (force_dma_unencrypted())
return __phys_to_dma(dev, phys);
return phys_to_dma(dev, phys);
}
u64 dma_direct_get_required_mask(struct device *dev)
{
u64 max_dma = phys_to_dma_direct(dev, (max_pfn - 1) << PAGE_SHIFT);
if (dev->bus_dma_mask && dev->bus_dma_mask < max_dma)
max_dma = dev->bus_dma_mask;
return (1ULL << (fls64(max_dma) - 1)) * 2 - 1;
}
static gfp_t __dma_direct_optimal_gfp_mask(struct device *dev, u64 dma_mask,
u64 *phys_mask)
{
if (dev->bus_dma_mask && dev->bus_dma_mask < dma_mask)
dma_mask = dev->bus_dma_mask;
if (force_dma_unencrypted())
*phys_mask = __dma_to_phys(dev, dma_mask);
else
*phys_mask = dma_to_phys(dev, dma_mask);
/*
* Optimistically try the zone that the physical address mask falls
* into first. If that returns memory that isn't actually addressable
* we will fallback to the next lower zone and try again.
*
* Note that GFP_DMA32 and GFP_DMA are no ops without the corresponding
* zones.
*/
if (*phys_mask <= DMA_BIT_MASK(ARCH_ZONE_DMA_BITS))
return GFP_DMA;
if (*phys_mask <= DMA_BIT_MASK(32))
return GFP_DMA32;
return 0;
}
static bool dma_coherent_ok(struct device *dev, phys_addr_t phys, size_t size) static bool dma_coherent_ok(struct device *dev, phys_addr_t phys, size_t size)
{ {
dma_addr_t addr = force_dma_unencrypted() ? return phys_to_dma_direct(dev, phys) + size - 1 <=
__phys_to_dma(dev, phys) : phys_to_dma(dev, phys); min_not_zero(dev->coherent_dma_mask, dev->bus_dma_mask);
return addr + size - 1 <= dev->coherent_dma_mask;
} }
void *dma_direct_alloc(struct device *dev, size_t size, dma_addr_t *dma_handle, void *dma_direct_alloc_pages(struct device *dev, size_t size,
gfp_t gfp, unsigned long attrs) dma_addr_t *dma_handle, gfp_t gfp, unsigned long attrs)
{ {
unsigned int count = PAGE_ALIGN(size) >> PAGE_SHIFT; unsigned int count = PAGE_ALIGN(size) >> PAGE_SHIFT;
int page_order = get_order(size); int page_order = get_order(size);
struct page *page = NULL; struct page *page = NULL;
u64 phys_mask;
void *ret; void *ret;
if (attrs & DMA_ATTR_NO_WARN)
gfp |= __GFP_NOWARN;
/* we always manually zero the memory once we are done: */ /* we always manually zero the memory once we are done: */
gfp &= ~__GFP_ZERO; gfp &= ~__GFP_ZERO;
gfp |= __dma_direct_optimal_gfp_mask(dev, dev->coherent_dma_mask,
/* GFP_DMA32 and GFP_DMA are no ops without the corresponding zones: */ &phys_mask);
if (dev->coherent_dma_mask <= DMA_BIT_MASK(ARCH_ZONE_DMA_BITS))
gfp |= GFP_DMA;
if (dev->coherent_dma_mask <= DMA_BIT_MASK(32) && !(gfp & GFP_DMA))
gfp |= GFP_DMA32;
again: again:
/* CMA can be used only in the context which permits sleeping */ /* CMA can be used only in the context which permits sleeping */
if (gfpflags_allow_blocking(gfp)) { if (gfpflags_allow_blocking(gfp)) {
...@@ -93,15 +139,14 @@ void *dma_direct_alloc(struct device *dev, size_t size, dma_addr_t *dma_handle, ...@@ -93,15 +139,14 @@ void *dma_direct_alloc(struct device *dev, size_t size, dma_addr_t *dma_handle,
page = NULL; page = NULL;
if (IS_ENABLED(CONFIG_ZONE_DMA32) && if (IS_ENABLED(CONFIG_ZONE_DMA32) &&
dev->coherent_dma_mask < DMA_BIT_MASK(64) && phys_mask < DMA_BIT_MASK(64) &&
!(gfp & (GFP_DMA32 | GFP_DMA))) { !(gfp & (GFP_DMA32 | GFP_DMA))) {
gfp |= GFP_DMA32; gfp |= GFP_DMA32;
goto again; goto again;
} }
if (IS_ENABLED(CONFIG_ZONE_DMA) && if (IS_ENABLED(CONFIG_ZONE_DMA) &&
dev->coherent_dma_mask < DMA_BIT_MASK(32) && phys_mask < DMA_BIT_MASK(32) && !(gfp & GFP_DMA)) {
!(gfp & GFP_DMA)) {
gfp = (gfp & ~GFP_DMA32) | GFP_DMA; gfp = (gfp & ~GFP_DMA32) | GFP_DMA;
goto again; goto again;
} }
...@@ -124,7 +169,7 @@ void *dma_direct_alloc(struct device *dev, size_t size, dma_addr_t *dma_handle, ...@@ -124,7 +169,7 @@ void *dma_direct_alloc(struct device *dev, size_t size, dma_addr_t *dma_handle,
* NOTE: this function must never look at the dma_addr argument, because we want * NOTE: this function must never look at the dma_addr argument, because we want
* to be able to use it as a helper for iommu implementations as well. * to be able to use it as a helper for iommu implementations as well.
*/ */
void dma_direct_free(struct device *dev, size_t size, void *cpu_addr, void dma_direct_free_pages(struct device *dev, size_t size, void *cpu_addr,
dma_addr_t dma_addr, unsigned long attrs) dma_addr_t dma_addr, unsigned long attrs)
{ {
unsigned int count = PAGE_ALIGN(size) >> PAGE_SHIFT; unsigned int count = PAGE_ALIGN(size) >> PAGE_SHIFT;
...@@ -136,14 +181,96 @@ void dma_direct_free(struct device *dev, size_t size, void *cpu_addr, ...@@ -136,14 +181,96 @@ void dma_direct_free(struct device *dev, size_t size, void *cpu_addr,
free_pages((unsigned long)cpu_addr, page_order); free_pages((unsigned long)cpu_addr, page_order);
} }
void *dma_direct_alloc(struct device *dev, size_t size,
dma_addr_t *dma_handle, gfp_t gfp, unsigned long attrs)
{
if (!dev_is_dma_coherent(dev))
return arch_dma_alloc(dev, size, dma_handle, gfp, attrs);
return dma_direct_alloc_pages(dev, size, dma_handle, gfp, attrs);
}
void dma_direct_free(struct device *dev, size_t size,
void *cpu_addr, dma_addr_t dma_addr, unsigned long attrs)
{
if (!dev_is_dma_coherent(dev))
arch_dma_free(dev, size, cpu_addr, dma_addr, attrs);
else
dma_direct_free_pages(dev, size, cpu_addr, dma_addr, attrs);
}
static void dma_direct_sync_single_for_device(struct device *dev,
dma_addr_t addr, size_t size, enum dma_data_direction dir)
{
if (dev_is_dma_coherent(dev))
return;
arch_sync_dma_for_device(dev, dma_to_phys(dev, addr), size, dir);
}
static void dma_direct_sync_sg_for_device(struct device *dev,
struct scatterlist *sgl, int nents, enum dma_data_direction dir)
{
struct scatterlist *sg;
int i;
if (dev_is_dma_coherent(dev))
return;
for_each_sg(sgl, sg, nents, i)
arch_sync_dma_for_device(dev, sg_phys(sg), sg->length, dir);
}
#if defined(CONFIG_ARCH_HAS_SYNC_DMA_FOR_CPU) || \
defined(CONFIG_ARCH_HAS_SYNC_DMA_FOR_CPU_ALL)
static void dma_direct_sync_single_for_cpu(struct device *dev,
dma_addr_t addr, size_t size, enum dma_data_direction dir)
{
if (dev_is_dma_coherent(dev))
return;
arch_sync_dma_for_cpu(dev, dma_to_phys(dev, addr), size, dir);
arch_sync_dma_for_cpu_all(dev);
}
static void dma_direct_sync_sg_for_cpu(struct device *dev,
struct scatterlist *sgl, int nents, enum dma_data_direction dir)
{
struct scatterlist *sg;
int i;
if (dev_is_dma_coherent(dev))
return;
for_each_sg(sgl, sg, nents, i)
arch_sync_dma_for_cpu(dev, sg_phys(sg), sg->length, dir);
arch_sync_dma_for_cpu_all(dev);
}
static void dma_direct_unmap_page(struct device *dev, dma_addr_t addr,
size_t size, enum dma_data_direction dir, unsigned long attrs)
{
if (!(attrs & DMA_ATTR_SKIP_CPU_SYNC))
dma_direct_sync_single_for_cpu(dev, addr, size, dir);
}
static void dma_direct_unmap_sg(struct device *dev, struct scatterlist *sgl,
int nents, enum dma_data_direction dir, unsigned long attrs)
{
if (!(attrs & DMA_ATTR_SKIP_CPU_SYNC))
dma_direct_sync_sg_for_cpu(dev, sgl, nents, dir);
}
#endif
dma_addr_t dma_direct_map_page(struct device *dev, struct page *page, dma_addr_t dma_direct_map_page(struct device *dev, struct page *page,
unsigned long offset, size_t size, enum dma_data_direction dir, unsigned long offset, size_t size, enum dma_data_direction dir,
unsigned long attrs) unsigned long attrs)
{ {
dma_addr_t dma_addr = phys_to_dma(dev, page_to_phys(page)) + offset; phys_addr_t phys = page_to_phys(page) + offset;
dma_addr_t dma_addr = phys_to_dma(dev, phys);
if (!check_addr(dev, dma_addr, size, __func__)) if (!check_addr(dev, dma_addr, size, __func__))
return DIRECT_MAPPING_ERROR; return DIRECT_MAPPING_ERROR;
if (!(attrs & DMA_ATTR_SKIP_CPU_SYNC))
dma_direct_sync_single_for_device(dev, dma_addr, size, dir);
return dma_addr; return dma_addr;
} }
...@@ -162,31 +289,29 @@ int dma_direct_map_sg(struct device *dev, struct scatterlist *sgl, int nents, ...@@ -162,31 +289,29 @@ int dma_direct_map_sg(struct device *dev, struct scatterlist *sgl, int nents,
sg_dma_len(sg) = sg->length; sg_dma_len(sg) = sg->length;
} }
if (!(attrs & DMA_ATTR_SKIP_CPU_SYNC))
dma_direct_sync_sg_for_device(dev, sgl, nents, dir);
return nents; return nents;
} }
/*
* Because 32-bit DMA masks are so common we expect every architecture to be
* able to satisfy them - either by not supporting more physical memory, or by
* providing a ZONE_DMA32. If neither is the case, the architecture needs to
* use an IOMMU instead of the direct mapping.
*/
int dma_direct_supported(struct device *dev, u64 mask) int dma_direct_supported(struct device *dev, u64 mask)
{ {
#ifdef CONFIG_ZONE_DMA u64 min_mask;
if (mask < phys_to_dma(dev, DMA_BIT_MASK(ARCH_ZONE_DMA_BITS)))
return 0; if (IS_ENABLED(CONFIG_ZONE_DMA))
#else min_mask = DMA_BIT_MASK(ARCH_ZONE_DMA_BITS);
/* else
* Because 32-bit DMA masks are so common we expect every architecture min_mask = DMA_BIT_MASK(32);
* to be able to satisfy them - either by not supporting more physical
* memory, or by providing a ZONE_DMA32. If neither is the case, the min_mask = min_t(u64, min_mask, (max_pfn - 1) << PAGE_SHIFT);
* architecture needs to use an IOMMU instead of the direct mapping.
*/ return mask >= phys_to_dma(dev, min_mask);
if (mask < phys_to_dma(dev, DMA_BIT_MASK(32)))
return 0;
#endif
/*
* Upstream PCI/PCIe bridges or SoC interconnects may not carry
* as many DMA address bits as the device itself supports.
*/
if (dev->bus_dma_mask && mask > dev->bus_dma_mask)
return 0;
return 1;
} }
int dma_direct_mapping_error(struct device *dev, dma_addr_t dma_addr) int dma_direct_mapping_error(struct device *dev, dma_addr_t dma_addr)
...@@ -199,7 +324,20 @@ const struct dma_map_ops dma_direct_ops = { ...@@ -199,7 +324,20 @@ const struct dma_map_ops dma_direct_ops = {
.free = dma_direct_free, .free = dma_direct_free,
.map_page = dma_direct_map_page, .map_page = dma_direct_map_page,
.map_sg = dma_direct_map_sg, .map_sg = dma_direct_map_sg,
#if defined(CONFIG_ARCH_HAS_SYNC_DMA_FOR_DEVICE)
.sync_single_for_device = dma_direct_sync_single_for_device,
.sync_sg_for_device = dma_direct_sync_sg_for_device,
#endif
#if defined(CONFIG_ARCH_HAS_SYNC_DMA_FOR_CPU) || \
defined(CONFIG_ARCH_HAS_SYNC_DMA_FOR_CPU_ALL)
.sync_single_for_cpu = dma_direct_sync_single_for_cpu,
.sync_sg_for_cpu = dma_direct_sync_sg_for_cpu,
.unmap_page = dma_direct_unmap_page,
.unmap_sg = dma_direct_unmap_sg,
#endif
.get_required_mask = dma_direct_get_required_mask,
.dma_supported = dma_direct_supported, .dma_supported = dma_direct_supported,
.mapping_error = dma_direct_mapping_error, .mapping_error = dma_direct_mapping_error,
.cache_sync = arch_dma_cache_sync,
}; };
EXPORT_SYMBOL(dma_direct_ops); EXPORT_SYMBOL(dma_direct_ops);
kernel/dma/mapping.c
View file @ cff22949
...@@ -7,7 +7,7 @@ ...@@ -7,7 +7,7 @@
*/ */
#include <linux/acpi.h> #include <linux/acpi.h>
#include <linux/dma-mapping.h> #include <linux/dma-noncoherent.h>
#include <linux/export.h> #include <linux/export.h>
#include <linux/gfp.h> #include <linux/gfp.h>
#include <linux/of_device.h> #include <linux/of_device.h>
...@@ -202,17 +202,26 @@ EXPORT_SYMBOL(dmam_release_declared_memory); ...@@ -202,17 +202,26 @@ EXPORT_SYMBOL(dmam_release_declared_memory);
* Create scatter-list for the already allocated DMA buffer. * Create scatter-list for the already allocated DMA buffer.
*/ */
int dma_common_get_sgtable(struct device *dev, struct sg_table *sgt, int dma_common_get_sgtable(struct device *dev, struct sg_table *sgt,
void *cpu_addr, dma_addr_t handle, size_t size) void *cpu_addr, dma_addr_t dma_addr, size_t size,
unsigned long attrs)
{ {
struct page *page = virt_to_page(cpu_addr); struct page *page;
int ret; int ret;
ret = sg_alloc_table(sgt, 1, GFP_KERNEL); if (!dev_is_dma_coherent(dev)) {
if (unlikely(ret)) if (!IS_ENABLED(CONFIG_ARCH_HAS_DMA_COHERENT_TO_PFN))
return ret; return -ENXIO;
page = pfn_to_page(arch_dma_coherent_to_pfn(dev, cpu_addr,
dma_addr));
} else {
page = virt_to_page(cpu_addr);
}
ret = sg_alloc_table(sgt, 1, GFP_KERNEL);
if (!ret)
sg_set_page(sgt->sgl, page, PAGE_ALIGN(size), 0); sg_set_page(sgt->sgl, page, PAGE_ALIGN(size), 0);
return 0; return ret;
} }
EXPORT_SYMBOL(dma_common_get_sgtable); EXPORT_SYMBOL(dma_common_get_sgtable);
...@@ -220,27 +229,37 @@ EXPORT_SYMBOL(dma_common_get_sgtable); ...@@ -220,27 +229,37 @@ EXPORT_SYMBOL(dma_common_get_sgtable);
* Create userspace mapping for the DMA-coherent memory. * Create userspace mapping for the DMA-coherent memory.
*/ */
int dma_common_mmap(struct device *dev, struct vm_area_struct *vma, int dma_common_mmap(struct device *dev, struct vm_area_struct *vma,
void *cpu_addr, dma_addr_t dma_addr, size_t size) void *cpu_addr, dma_addr_t dma_addr, size_t size,
unsigned long attrs)
{ {
int ret = -ENXIO;
#ifndef CONFIG_ARCH_NO_COHERENT_DMA_MMAP #ifndef CONFIG_ARCH_NO_COHERENT_DMA_MMAP
unsigned long user_count = vma_pages(vma); unsigned long user_count = vma_pages(vma);
unsigned long count = PAGE_ALIGN(size) >> PAGE_SHIFT; unsigned long count = PAGE_ALIGN(size) >> PAGE_SHIFT;
unsigned long off = vma->vm_pgoff; unsigned long off = vma->vm_pgoff;
unsigned long pfn;
int ret = -ENXIO;
vma->vm_page_prot = pgprot_noncached(vma->vm_page_prot); vma->vm_page_prot = arch_dma_mmap_pgprot(dev, vma->vm_page_prot, attrs);
if (dma_mmap_from_dev_coherent(dev, vma, cpu_addr, size, &ret)) if (dma_mmap_from_dev_coherent(dev, vma, cpu_addr, size, &ret))
return ret; return ret;
if (off < count && user_count <= (count - off)) if (off >= count || user_count > count - off)
ret = remap_pfn_range(vma, vma->vm_start, return -ENXIO;
page_to_pfn(virt_to_page(cpu_addr)) + off,
user_count << PAGE_SHIFT,
vma->vm_page_prot);
#endif /* !CONFIG_ARCH_NO_COHERENT_DMA_MMAP */
return ret; if (!dev_is_dma_coherent(dev)) {
if (!IS_ENABLED(CONFIG_ARCH_HAS_DMA_COHERENT_TO_PFN))
return -ENXIO;
pfn = arch_dma_coherent_to_pfn(dev, cpu_addr, dma_addr);
} else {
pfn = page_to_pfn(virt_to_page(cpu_addr));
}
return remap_pfn_range(vma, vma->vm_start, pfn + vma->vm_pgoff,
user_count << PAGE_SHIFT, vma->vm_page_prot);
#else
return -ENXIO;
#endif /* !CONFIG_ARCH_NO_COHERENT_DMA_MMAP */
} }
EXPORT_SYMBOL(dma_common_mmap); EXPORT_SYMBOL(dma_common_mmap);
...@@ -327,19 +346,3 @@ void dma_common_free_remap(void *cpu_addr, size_t size, unsigned long vm_flags) ...@@ -327,19 +346,3 @@ void dma_common_free_remap(void *cpu_addr, size_t size, unsigned long vm_flags)
vunmap(cpu_addr); vunmap(cpu_addr);
} }
#endif #endif
/*
* enables DMA API use for a device
*/
int dma_configure(struct device *dev)
{
if (dev->bus->dma_configure)
return dev->bus->dma_configure(dev);
return 0;
}
void dma_deconfigure(struct device *dev)
{
of_dma_deconfigure(dev);
acpi_dma_deconfigure(dev);
}
kernel/dma/noncoherent.c deleted 100644 → 0
View file @ 13775dac
// SPDX-License-Identifier: GPL-2.0
/*
* Copyright (C) 2018 Christoph Hellwig.
*
* DMA operations that map physical memory directly without providing cache
* coherence.
*/
#include <linux/export.h>
#include <linux/mm.h>
#include <linux/dma-direct.h>
#include <linux/dma-noncoherent.h>
#include <linux/scatterlist.h>
static void dma_noncoherent_sync_single_for_device(struct device *dev,
dma_addr_t addr, size_t size, enum dma_data_direction dir)
{
arch_sync_dma_for_device(dev, dma_to_phys(dev, addr), size, dir);
}
static void dma_noncoherent_sync_sg_for_device(struct device *dev,
struct scatterlist *sgl, int nents, enum dma_data_direction dir)
{
struct scatterlist *sg;
int i;
for_each_sg(sgl, sg, nents, i)
arch_sync_dma_for_device(dev, sg_phys(sg), sg->length, dir);
}
static dma_addr_t dma_noncoherent_map_page(struct device *dev, struct page *page,
unsigned long offset, size_t size, enum dma_data_direction dir,
unsigned long attrs)
{
dma_addr_t addr;
addr = dma_direct_map_page(dev, page, offset, size, dir, attrs);
if (!dma_mapping_error(dev, addr) && !(attrs & DMA_ATTR_SKIP_CPU_SYNC))
arch_sync_dma_for_device(dev, page_to_phys(page) + offset,
size, dir);
return addr;
}
static int dma_noncoherent_map_sg(struct device *dev, struct scatterlist *sgl,
int nents, enum dma_data_direction dir, unsigned long attrs)
{
nents = dma_direct_map_sg(dev, sgl, nents, dir, attrs);
if (nents > 0 && !(attrs & DMA_ATTR_SKIP_CPU_SYNC))
dma_noncoherent_sync_sg_for_device(dev, sgl, nents, dir);
return nents;
}
#if defined(CONFIG_ARCH_HAS_SYNC_DMA_FOR_CPU) || \
defined(CONFIG_ARCH_HAS_SYNC_DMA_FOR_CPU_ALL)
static void dma_noncoherent_sync_single_for_cpu(struct device *dev,
dma_addr_t addr, size_t size, enum dma_data_direction dir)
{
arch_sync_dma_for_cpu(dev, dma_to_phys(dev, addr), size, dir);
arch_sync_dma_for_cpu_all(dev);
}
static void dma_noncoherent_sync_sg_for_cpu(struct device *dev,
struct scatterlist *sgl, int nents, enum dma_data_direction dir)
{
struct scatterlist *sg;
int i;
for_each_sg(sgl, sg, nents, i)
arch_sync_dma_for_cpu(dev, sg_phys(sg), sg->length, dir);
arch_sync_dma_for_cpu_all(dev);
}
static void dma_noncoherent_unmap_page(struct device *dev, dma_addr_t addr,
size_t size, enum dma_data_direction dir, unsigned long attrs)
{
if (!(attrs & DMA_ATTR_SKIP_CPU_SYNC))
dma_noncoherent_sync_single_for_cpu(dev, addr, size, dir);
}
static void dma_noncoherent_unmap_sg(struct device *dev, struct scatterlist *sgl,
int nents, enum dma_data_direction dir, unsigned long attrs)
{
if (!(attrs & DMA_ATTR_SKIP_CPU_SYNC))
dma_noncoherent_sync_sg_for_cpu(dev, sgl, nents, dir);
}
#endif
const struct dma_map_ops dma_noncoherent_ops = {
.alloc = arch_dma_alloc,
.free = arch_dma_free,
.mmap = arch_dma_mmap,
.sync_single_for_device = dma_noncoherent_sync_single_for_device,
.sync_sg_for_device = dma_noncoherent_sync_sg_for_device,
.map_page = dma_noncoherent_map_page,
.map_sg = dma_noncoherent_map_sg,
#if defined(CONFIG_ARCH_HAS_SYNC_DMA_FOR_CPU) || \
defined(CONFIG_ARCH_HAS_SYNC_DMA_FOR_CPU_ALL)
.sync_single_for_cpu = dma_noncoherent_sync_single_for_cpu,
.sync_sg_for_cpu = dma_noncoherent_sync_sg_for_cpu,
.unmap_page = dma_noncoherent_unmap_page,
.unmap_sg = dma_noncoherent_unmap_sg,
#endif
.dma_supported = dma_direct_supported,
.mapping_error = dma_direct_mapping_error,
.cache_sync = arch_dma_cache_sync,
};
EXPORT_SYMBOL(dma_noncoherent_ops);
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment
GitLab Nexedi Edition | About GitLab | About Nexedi | 沪ICP备2021021310号-2 | 沪ICP备2021021310号-7