Commit 086c83ac authored by Zhen Lei's avatar Zhen Lei Committed by Joerg Roedel

iommu/iova: Optimise the padding calculation

The mask for calculating the padding size doesn't change, so there's no
need to recalculate it every loop iteration. Furthermore, Once we've
done that, it becomes clear that we don't actually need to calculate a
padding size at all - by flipping the arithmetic around, we can just
combine the upper limit, size, and mask directly to check against the
lower limit.

For an arm64 build, this alone knocks 20% off the object code size of
the entire alloc_iova() function!
Signed-off-by: default avatarZhen Lei <thunder.leizhen@huawei.com>
Tested-by: default avatarArd Biesheuvel <ard.biesheuvel@linaro.org>
Tested-by: default avatarZhen Lei <thunder.leizhen@huawei.com>
Tested-by: default avatarNate Watterson <nwatters@codeaurora.org>
[rm: simplified more of the arithmetic, rewrote commit message]
Signed-off-by: default avatarRobin Murphy <robin.murphy@arm.com>
Signed-off-by: default avatarJoerg Roedel <jroedel@suse.de>
parent 2070f940
...@@ -182,24 +182,17 @@ iova_insert_rbtree(struct rb_root *root, struct iova *iova, ...@@ -182,24 +182,17 @@ iova_insert_rbtree(struct rb_root *root, struct iova *iova,
rb_insert_color(&iova->node, root); rb_insert_color(&iova->node, root);
} }
/*
* Computes the padding size required, to make the start address
* naturally aligned on the power-of-two order of its size
*/
static unsigned int
iova_get_pad_size(unsigned int size, unsigned int limit_pfn)
{
return (limit_pfn - size) & (__roundup_pow_of_two(size) - 1);
}
static int __alloc_and_insert_iova_range(struct iova_domain *iovad, static int __alloc_and_insert_iova_range(struct iova_domain *iovad,
unsigned long size, unsigned long limit_pfn, unsigned long size, unsigned long limit_pfn,
struct iova *new, bool size_aligned) struct iova *new, bool size_aligned)
{ {
struct rb_node *prev, *curr = NULL; struct rb_node *prev, *curr = NULL;
unsigned long flags; unsigned long flags;
unsigned long saved_pfn; unsigned long saved_pfn, new_pfn;
unsigned int pad_size = 0; unsigned long align_mask = ~0UL;
if (size_aligned)
align_mask <<= fls_long(size - 1);
/* Walk the tree backwards */ /* Walk the tree backwards */
spin_lock_irqsave(&iovad->iova_rbtree_lock, flags); spin_lock_irqsave(&iovad->iova_rbtree_lock, flags);
...@@ -209,31 +202,26 @@ static int __alloc_and_insert_iova_range(struct iova_domain *iovad, ...@@ -209,31 +202,26 @@ static int __alloc_and_insert_iova_range(struct iova_domain *iovad,
while (curr) { while (curr) {
struct iova *curr_iova = rb_entry(curr, struct iova, node); struct iova *curr_iova = rb_entry(curr, struct iova, node);
if (limit_pfn <= curr_iova->pfn_lo) { if (limit_pfn <= curr_iova->pfn_lo)
goto move_left; goto move_left;
} else if (limit_pfn > curr_iova->pfn_hi) {
if (size_aligned) if (((limit_pfn - size) & align_mask) > curr_iova->pfn_hi)
pad_size = iova_get_pad_size(size, limit_pfn);
if ((curr_iova->pfn_hi + size + pad_size) < limit_pfn)
break; /* found a free slot */ break; /* found a free slot */
}
limit_pfn = curr_iova->pfn_lo; limit_pfn = curr_iova->pfn_lo;
move_left: move_left:
prev = curr; prev = curr;
curr = rb_prev(curr); curr = rb_prev(curr);
} }
if (!curr) { new_pfn = (limit_pfn - size) & align_mask;
if (size_aligned) if (limit_pfn < size || new_pfn < iovad->start_pfn) {
pad_size = iova_get_pad_size(size, limit_pfn);
if ((iovad->start_pfn + size + pad_size) > limit_pfn) {
spin_unlock_irqrestore(&iovad->iova_rbtree_lock, flags); spin_unlock_irqrestore(&iovad->iova_rbtree_lock, flags);
return -ENOMEM; return -ENOMEM;
} }
}
/* pfn_lo will point to size aligned address if size_aligned is set */ /* pfn_lo will point to size aligned address if size_aligned is set */
new->pfn_lo = limit_pfn - (size + pad_size); new->pfn_lo = new_pfn;
new->pfn_hi = new->pfn_lo + size - 1; new->pfn_hi = new->pfn_lo + size - 1;
/* If we have 'prev', it's a valid place to start the insertion. */ /* If we have 'prev', it's a valid place to start the insertion. */
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment