Commit 25662785 authored by Linus Torvalds's avatar Linus Torvalds

Merge git://git.infradead.org/intel-iommu

Pull intel IOMMU updates from David Woodhouse:
 "This patchset improves the scalability of the Intel IOMMU code by
  resolving two spinlock bottlenecks and eliminating the linearity of
  the IOVA allocator, yielding up to ~5x performance improvement and
  approaching 'iommu=off' performance"

* git://git.infradead.org/intel-iommu:
  iommu/vt-d: Use per-cpu IOVA caching
  iommu/iova: introduce per-cpu caching to iova allocation
  iommu/vt-d: change intel-iommu to use IOVA frame numbers
  iommu/vt-d: avoid dev iotlb logic for domains with no dev iotlbs
  iommu/vt-d: only unmap mapped entries
  iommu/vt-d: correct flush_unmaps pfn usage
  iommu/vt-d: per-cpu deferred invalidation queues
  iommu/vt-d: refactoring of deferred flush entries
parents e28e909c 22e2f9fa
This diff is collapsed.
This diff is collapsed.
...@@ -19,8 +19,21 @@ ...@@ -19,8 +19,21 @@
/* iova structure */ /* iova structure */
struct iova { struct iova {
struct rb_node node; struct rb_node node;
unsigned long pfn_hi; /* IOMMU dish out addr hi */ unsigned long pfn_hi; /* Highest allocated pfn */
unsigned long pfn_lo; /* IOMMU dish out addr lo */ unsigned long pfn_lo; /* Lowest allocated pfn */
};
struct iova_magazine;
struct iova_cpu_rcache;
#define IOVA_RANGE_CACHE_MAX_SIZE 6 /* log of max cached IOVA range size (in pages) */
#define MAX_GLOBAL_MAGS 32 /* magazines per bin */
struct iova_rcache {
spinlock_t lock;
unsigned long depot_size;
struct iova_magazine *depot[MAX_GLOBAL_MAGS];
struct iova_cpu_rcache __percpu *cpu_rcaches;
}; };
/* holds all the iova translations for a domain */ /* holds all the iova translations for a domain */
...@@ -31,6 +44,7 @@ struct iova_domain { ...@@ -31,6 +44,7 @@ struct iova_domain {
unsigned long granule; /* pfn granularity for this domain */ unsigned long granule; /* pfn granularity for this domain */
unsigned long start_pfn; /* Lower limit for this domain */ unsigned long start_pfn; /* Lower limit for this domain */
unsigned long dma_32bit_pfn; unsigned long dma_32bit_pfn;
struct iova_rcache rcaches[IOVA_RANGE_CACHE_MAX_SIZE]; /* IOVA range caches */
}; };
static inline unsigned long iova_size(struct iova *iova) static inline unsigned long iova_size(struct iova *iova)
...@@ -78,6 +92,10 @@ void __free_iova(struct iova_domain *iovad, struct iova *iova); ...@@ -78,6 +92,10 @@ void __free_iova(struct iova_domain *iovad, struct iova *iova);
struct iova *alloc_iova(struct iova_domain *iovad, unsigned long size, struct iova *alloc_iova(struct iova_domain *iovad, unsigned long size,
unsigned long limit_pfn, unsigned long limit_pfn,
bool size_aligned); bool size_aligned);
void free_iova_fast(struct iova_domain *iovad, unsigned long pfn,
unsigned long size);
unsigned long alloc_iova_fast(struct iova_domain *iovad, unsigned long size,
unsigned long limit_pfn);
struct iova *reserve_iova(struct iova_domain *iovad, unsigned long pfn_lo, struct iova *reserve_iova(struct iova_domain *iovad, unsigned long pfn_lo,
unsigned long pfn_hi); unsigned long pfn_hi);
void copy_reserved_iova(struct iova_domain *from, struct iova_domain *to); void copy_reserved_iova(struct iova_domain *from, struct iova_domain *to);
...@@ -87,5 +105,6 @@ struct iova *find_iova(struct iova_domain *iovad, unsigned long pfn); ...@@ -87,5 +105,6 @@ struct iova *find_iova(struct iova_domain *iovad, unsigned long pfn);
void put_iova_domain(struct iova_domain *iovad); void put_iova_domain(struct iova_domain *iovad);
struct iova *split_and_remove_iova(struct iova_domain *iovad, struct iova *split_and_remove_iova(struct iova_domain *iovad,
struct iova *iova, unsigned long pfn_lo, unsigned long pfn_hi); struct iova *iova, unsigned long pfn_lo, unsigned long pfn_hi);
void free_cpu_cached_iovas(unsigned int cpu, struct iova_domain *iovad);
#endif #endif
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment