Commit 7cb671e7 authored by Alex Williamson's avatar Alex Williamson

vfio/type1: Reduce repetitive calls in vfio_pin_pages_remote()

vfio_pin_pages_remote() is typically called to iterate over a range
of memory.  Testing CAP_IPC_LOCK is relatively expensive, so it makes
sense to push it up to the caller, which can then repeatedly call
vfio_pin_pages_remote() using that value.  This can show nearly a 20%
improvement on the worst case path through VFIO_IOMMU_MAP_DMA with
contiguous page mapping disabled.  Testing RLIMIT_MEMLOCK is much more
lightweight, but we bring it along on the same principle and it does
seem to show a marginal improvement.
Reviewed-by: default avatarPeter Xu <peterx@redhat.com>
Reviewed-by: default avatarKirti Wankhede <kwankhede@nvidia.com>
Signed-off-by: default avatarAlex Williamson <alex.williamson@redhat.com>
parent 80dbe1fb
...@@ -380,10 +380,10 @@ static int vaddr_get_pfn(struct mm_struct *mm, unsigned long vaddr, ...@@ -380,10 +380,10 @@ static int vaddr_get_pfn(struct mm_struct *mm, unsigned long vaddr,
* first page and all consecutive pages with the same locking. * first page and all consecutive pages with the same locking.
*/ */
static long vfio_pin_pages_remote(struct vfio_dma *dma, unsigned long vaddr, static long vfio_pin_pages_remote(struct vfio_dma *dma, unsigned long vaddr,
long npage, unsigned long *pfn_base) long npage, unsigned long *pfn_base,
bool lock_cap, unsigned long limit)
{ {
unsigned long pfn = 0, limit = rlimit(RLIMIT_MEMLOCK) >> PAGE_SHIFT; unsigned long pfn = 0;
bool lock_cap = capable(CAP_IPC_LOCK);
long ret, pinned = 0, lock_acct = 0; long ret, pinned = 0, lock_acct = 0;
bool rsvd; bool rsvd;
dma_addr_t iova = vaddr - dma->vaddr + dma->iova; dma_addr_t iova = vaddr - dma->vaddr + dma->iova;
...@@ -924,13 +924,15 @@ static int vfio_pin_map_dma(struct vfio_iommu *iommu, struct vfio_dma *dma, ...@@ -924,13 +924,15 @@ static int vfio_pin_map_dma(struct vfio_iommu *iommu, struct vfio_dma *dma,
unsigned long vaddr = dma->vaddr; unsigned long vaddr = dma->vaddr;
size_t size = map_size; size_t size = map_size;
long npage; long npage;
unsigned long pfn; unsigned long pfn, limit = rlimit(RLIMIT_MEMLOCK) >> PAGE_SHIFT;
bool lock_cap = capable(CAP_IPC_LOCK);
int ret = 0; int ret = 0;
while (size) { while (size) {
/* Pin a contiguous chunk of memory */ /* Pin a contiguous chunk of memory */
npage = vfio_pin_pages_remote(dma, vaddr + dma->size, npage = vfio_pin_pages_remote(dma, vaddr + dma->size,
size >> PAGE_SHIFT, &pfn); size >> PAGE_SHIFT, &pfn,
lock_cap, limit);
if (npage <= 0) { if (npage <= 0) {
WARN_ON(!npage); WARN_ON(!npage);
ret = (int)npage; ret = (int)npage;
...@@ -1040,6 +1042,8 @@ static int vfio_iommu_replay(struct vfio_iommu *iommu, ...@@ -1040,6 +1042,8 @@ static int vfio_iommu_replay(struct vfio_iommu *iommu,
{ {
struct vfio_domain *d; struct vfio_domain *d;
struct rb_node *n; struct rb_node *n;
unsigned long limit = rlimit(RLIMIT_MEMLOCK) >> PAGE_SHIFT;
bool lock_cap = capable(CAP_IPC_LOCK);
int ret; int ret;
/* Arbitrarily pick the first domain in the list for lookups */ /* Arbitrarily pick the first domain in the list for lookups */
...@@ -1086,7 +1090,8 @@ static int vfio_iommu_replay(struct vfio_iommu *iommu, ...@@ -1086,7 +1090,8 @@ static int vfio_iommu_replay(struct vfio_iommu *iommu,
npage = vfio_pin_pages_remote(dma, vaddr, npage = vfio_pin_pages_remote(dma, vaddr,
n >> PAGE_SHIFT, n >> PAGE_SHIFT,
&pfn); &pfn, lock_cap,
limit);
if (npage <= 0) { if (npage <= 0) {
WARN_ON(!npage); WARN_ON(!npage);
ret = (int)npage; ret = (int)npage;
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment