Commit 305bc234 authored by Chris Wilson's avatar Chris Wilson

drm/i915: Make i915_vma_pin() small and inline

Not only is i915_vma_pin() called for every single object on every single
execbuf, it is usually a simple increment as the VMA is already bound for
execution by the GPU. Rearrange the tests for unbound and pin_count
overflow so that we can do the increment and test very cheaply and
compact enough to inline the operation into execbuf. The trick used is
to note that we can check for an overflow bit (keeping space available
for it inside the flags) at the same time as checking the binding bits.
Signed-off-by: default avatarChris Wilson <chris@chris-wilson.co.uk>
Reviewed-by: default avatarJoonas Lahtinen <joonas.lahtinen@linux.intel.com>
Link: http://patchwork.freedesktop.org/patch/msgid/1470324762-2545-17-git-send-email-chris@chris-wilson.co.uk
parent 3272db53
...@@ -3705,23 +3705,19 @@ void __i915_vma_set_map_and_fenceable(struct i915_vma *vma) ...@@ -3705,23 +3705,19 @@ void __i915_vma_set_map_and_fenceable(struct i915_vma *vma)
obj->map_and_fenceable = mappable && fenceable; obj->map_and_fenceable = mappable && fenceable;
} }
int int __i915_vma_do_pin(struct i915_vma *vma,
i915_vma_pin(struct i915_vma *vma, u64 size, u64 alignment, u64 flags) u64 size, u64 alignment, u64 flags)
{ {
unsigned int bound; unsigned int bound = vma->flags;
int ret; int ret;
GEM_BUG_ON((flags & (PIN_GLOBAL | PIN_USER)) == 0); GEM_BUG_ON((flags & (PIN_GLOBAL | PIN_USER)) == 0);
GEM_BUG_ON((flags & PIN_GLOBAL) && !i915_vma_is_ggtt(vma)); GEM_BUG_ON((flags & PIN_GLOBAL) && !i915_vma_is_ggtt(vma));
bound = vma->flags; if (WARN_ON(bound & I915_VMA_PIN_OVERFLOW)) {
if (WARN_ON((bound & I915_VMA_PIN_MASK) == I915_VMA_PIN_MASK)) ret = -EBUSY;
return -EBUSY; goto err;
}
/* Pin early to prevent the shrinker/eviction logic from destroying
* our vma as we insert and bind.
*/
__i915_vma_pin(vma);
if ((bound & (I915_VMA_GLOBAL_BIND | I915_VMA_LOCAL_BIND)) == 0) { if ((bound & (I915_VMA_GLOBAL_BIND | I915_VMA_LOCAL_BIND)) == 0) {
ret = i915_vma_insert(vma, size, alignment, flags); ret = i915_vma_insert(vma, size, alignment, flags);
......
...@@ -194,13 +194,15 @@ struct i915_vma { ...@@ -194,13 +194,15 @@ struct i915_vma {
* bits with absolutely no headroom. So use 4 bits. * bits with absolutely no headroom. So use 4 bits.
*/ */
#define I915_VMA_PIN_MASK 0xf #define I915_VMA_PIN_MASK 0xf
#define I915_VMA_PIN_OVERFLOW BIT(5)
/** Flags and address space this VMA is bound to */ /** Flags and address space this VMA is bound to */
#define I915_VMA_GLOBAL_BIND BIT(5) #define I915_VMA_GLOBAL_BIND BIT(6)
#define I915_VMA_LOCAL_BIND BIT(6) #define I915_VMA_LOCAL_BIND BIT(7)
#define I915_VMA_BIND_MASK (I915_VMA_GLOBAL_BIND | I915_VMA_LOCAL_BIND | I915_VMA_PIN_OVERFLOW)
#define I915_VMA_GGTT BIT(7) #define I915_VMA_GGTT BIT(8)
#define I915_VMA_CLOSED BIT(8) #define I915_VMA_CLOSED BIT(9)
unsigned int active; unsigned int active;
struct i915_gem_active last_read[I915_NUM_ENGINES]; struct i915_gem_active last_read[I915_NUM_ENGINES];
...@@ -620,20 +622,39 @@ i915_ggtt_view_equal(const struct i915_ggtt_view *a, ...@@ -620,20 +622,39 @@ i915_ggtt_view_equal(const struct i915_ggtt_view *a,
return true; return true;
} }
int __must_check
i915_vma_pin(struct i915_vma *vma, u64 size, u64 alignment, u64 flags);
/* Flags used by pin/bind&friends. */ /* Flags used by pin/bind&friends. */
#define PIN_MAPPABLE BIT(0) #define PIN_NONBLOCK BIT(0)
#define PIN_NONBLOCK BIT(1) #define PIN_MAPPABLE BIT(1)
#define PIN_GLOBAL BIT(2) #define PIN_ZONE_4G BIT(2)
#define PIN_OFFSET_BIAS BIT(3)
#define PIN_USER BIT(4) #define PIN_MBZ BIT(5) /* I915_VMA_PIN_OVERFLOW */
#define PIN_UPDATE BIT(5) #define PIN_GLOBAL BIT(6) /* I915_VMA_GLOBAL_BIND */
#define PIN_ZONE_4G BIT(6) #define PIN_USER BIT(7) /* I915_VMA_LOCAL_BIND */
#define PIN_HIGH BIT(7) #define PIN_UPDATE BIT(8)
#define PIN_OFFSET_FIXED BIT(8)
#define PIN_HIGH BIT(9)
#define PIN_OFFSET_BIAS BIT(10)
#define PIN_OFFSET_FIXED BIT(11)
#define PIN_OFFSET_MASK (~4095) #define PIN_OFFSET_MASK (~4095)
int __i915_vma_do_pin(struct i915_vma *vma,
u64 size, u64 alignment, u64 flags);
static inline int __must_check
i915_vma_pin(struct i915_vma *vma, u64 size, u64 alignment, u64 flags)
{
BUILD_BUG_ON(PIN_MBZ != I915_VMA_PIN_OVERFLOW);
BUILD_BUG_ON(PIN_GLOBAL != I915_VMA_GLOBAL_BIND);
BUILD_BUG_ON(PIN_USER != I915_VMA_LOCAL_BIND);
/* Pin early to prevent the shrinker/eviction logic from destroying
* our vma as we insert and bind.
*/
if (likely(((++vma->flags ^ flags) & I915_VMA_BIND_MASK) == 0))
return 0;
return __i915_vma_do_pin(vma, size, alignment, flags);
}
static inline int i915_vma_pin_count(const struct i915_vma *vma) static inline int i915_vma_pin_count(const struct i915_vma *vma)
{ {
return vma->flags & I915_VMA_PIN_MASK; return vma->flags & I915_VMA_PIN_MASK;
...@@ -647,7 +668,7 @@ static inline bool i915_vma_is_pinned(const struct i915_vma *vma) ...@@ -647,7 +668,7 @@ static inline bool i915_vma_is_pinned(const struct i915_vma *vma)
static inline void __i915_vma_pin(struct i915_vma *vma) static inline void __i915_vma_pin(struct i915_vma *vma)
{ {
vma->flags++; vma->flags++;
GEM_BUG_ON(!i915_vma_is_pinned(vma)); GEM_BUG_ON(vma->flags & I915_VMA_PIN_OVERFLOW);
} }
static inline void __i915_vma_unpin(struct i915_vma *vma) static inline void __i915_vma_unpin(struct i915_vma *vma)
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment