Commit 655250a8 authored by Chris Wilson's avatar Chris Wilson

drm/i915/execlists: Switch to rb_root_cached

The kernel recently gained an augmented rbtree with the purpose of
cacheing the leftmost element of the rbtree, a frequent optimisation to
avoid calls to rb_first() which is also employed by the
execlists->queue. Switch from our open-coded cache to the library.
Signed-off-by: default avatarChris Wilson <chris@chris-wilson.co.uk>
Reviewed-by: default avatarTvrtko Ursulin <tvrtko.ursulin@intel.com>
Link: https://patchwork.freedesktop.org/patch/msgid/20180629075348.27358-9-chris@chris-wilson.co.uk
parent cb4dc8da
...@@ -467,8 +467,7 @@ static void intel_engine_init_execlist(struct intel_engine_cs *engine) ...@@ -467,8 +467,7 @@ static void intel_engine_init_execlist(struct intel_engine_cs *engine)
GEM_BUG_ON(execlists_num_ports(execlists) > EXECLIST_MAX_PORTS); GEM_BUG_ON(execlists_num_ports(execlists) > EXECLIST_MAX_PORTS);
execlists->queue_priority = INT_MIN; execlists->queue_priority = INT_MIN;
execlists->queue = RB_ROOT; execlists->queue = RB_ROOT_CACHED;
execlists->first = NULL;
} }
/** /**
...@@ -1004,7 +1003,7 @@ bool intel_engine_is_idle(struct intel_engine_cs *engine) ...@@ -1004,7 +1003,7 @@ bool intel_engine_is_idle(struct intel_engine_cs *engine)
} }
/* ELSP is empty, but there are ready requests? E.g. after reset */ /* ELSP is empty, but there are ready requests? E.g. after reset */
if (READ_ONCE(engine->execlists.first)) if (!RB_EMPTY_ROOT(&engine->execlists.queue.rb_root))
return false; return false;
/* Ring stopped? */ /* Ring stopped? */
...@@ -1540,7 +1539,7 @@ void intel_engine_dump(struct intel_engine_cs *engine, ...@@ -1540,7 +1539,7 @@ void intel_engine_dump(struct intel_engine_cs *engine,
last = NULL; last = NULL;
count = 0; count = 0;
drm_printf(m, "\t\tQueue priority: %d\n", execlists->queue_priority); drm_printf(m, "\t\tQueue priority: %d\n", execlists->queue_priority);
for (rb = execlists->first; rb; rb = rb_next(rb)) { for (rb = rb_first_cached(&execlists->queue); rb; rb = rb_next(rb)) {
struct i915_priolist *p = struct i915_priolist *p =
rb_entry(rb, typeof(*p), node); rb_entry(rb, typeof(*p), node);
......
...@@ -695,9 +695,6 @@ static bool __guc_dequeue(struct intel_engine_cs *engine) ...@@ -695,9 +695,6 @@ static bool __guc_dequeue(struct intel_engine_cs *engine)
lockdep_assert_held(&engine->timeline.lock); lockdep_assert_held(&engine->timeline.lock);
rb = execlists->first;
GEM_BUG_ON(rb_first(&execlists->queue) != rb);
if (port_isset(port)) { if (port_isset(port)) {
if (intel_engine_has_preemption(engine)) { if (intel_engine_has_preemption(engine)) {
struct guc_preempt_work *preempt_work = struct guc_preempt_work *preempt_work =
...@@ -719,7 +716,7 @@ static bool __guc_dequeue(struct intel_engine_cs *engine) ...@@ -719,7 +716,7 @@ static bool __guc_dequeue(struct intel_engine_cs *engine)
} }
GEM_BUG_ON(port_isset(port)); GEM_BUG_ON(port_isset(port));
while (rb) { while ((rb = rb_first_cached(&execlists->queue))) {
struct i915_priolist *p = to_priolist(rb); struct i915_priolist *p = to_priolist(rb);
struct i915_request *rq, *rn; struct i915_request *rq, *rn;
...@@ -744,15 +741,13 @@ static bool __guc_dequeue(struct intel_engine_cs *engine) ...@@ -744,15 +741,13 @@ static bool __guc_dequeue(struct intel_engine_cs *engine)
submit = true; submit = true;
} }
rb = rb_next(rb); rb_erase_cached(&p->node, &execlists->queue);
rb_erase(&p->node, &execlists->queue);
INIT_LIST_HEAD(&p->requests); INIT_LIST_HEAD(&p->requests);
if (p->priority != I915_PRIORITY_NORMAL) if (p->priority != I915_PRIORITY_NORMAL)
kmem_cache_free(engine->i915->priorities, p); kmem_cache_free(engine->i915->priorities, p);
} }
done: done:
execlists->queue_priority = rb ? to_priolist(rb)->priority : INT_MIN; execlists->queue_priority = rb ? to_priolist(rb)->priority : INT_MIN;
execlists->first = rb;
if (submit) if (submit)
port_assign(port, last); port_assign(port, last);
if (last) if (last)
...@@ -761,7 +756,8 @@ static bool __guc_dequeue(struct intel_engine_cs *engine) ...@@ -761,7 +756,8 @@ static bool __guc_dequeue(struct intel_engine_cs *engine)
/* We must always keep the beast fed if we have work piled up */ /* We must always keep the beast fed if we have work piled up */
GEM_BUG_ON(port_isset(execlists->port) && GEM_BUG_ON(port_isset(execlists->port) &&
!execlists_is_active(execlists, EXECLISTS_ACTIVE_USER)); !execlists_is_active(execlists, EXECLISTS_ACTIVE_USER));
GEM_BUG_ON(execlists->first && !port_isset(execlists->port)); GEM_BUG_ON(rb_first_cached(&execlists->queue) &&
!port_isset(execlists->port));
return submit; return submit;
} }
......
...@@ -273,7 +273,7 @@ lookup_priolist(struct intel_engine_cs *engine, int prio) ...@@ -273,7 +273,7 @@ lookup_priolist(struct intel_engine_cs *engine, int prio)
find_priolist: find_priolist:
/* most positive priority is scheduled first, equal priorities fifo */ /* most positive priority is scheduled first, equal priorities fifo */
rb = NULL; rb = NULL;
parent = &execlists->queue.rb_node; parent = &execlists->queue.rb_root.rb_node;
while (*parent) { while (*parent) {
rb = *parent; rb = *parent;
p = to_priolist(rb); p = to_priolist(rb);
...@@ -311,10 +311,7 @@ lookup_priolist(struct intel_engine_cs *engine, int prio) ...@@ -311,10 +311,7 @@ lookup_priolist(struct intel_engine_cs *engine, int prio)
p->priority = prio; p->priority = prio;
INIT_LIST_HEAD(&p->requests); INIT_LIST_HEAD(&p->requests);
rb_link_node(&p->node, rb, parent); rb_link_node(&p->node, rb, parent);
rb_insert_color(&p->node, &execlists->queue); rb_insert_color_cached(&p->node, &execlists->queue, first);
if (first)
execlists->first = &p->node;
return p; return p;
} }
...@@ -602,9 +599,6 @@ static void execlists_dequeue(struct intel_engine_cs *engine) ...@@ -602,9 +599,6 @@ static void execlists_dequeue(struct intel_engine_cs *engine)
* and context switches) submission. * and context switches) submission.
*/ */
rb = execlists->first;
GEM_BUG_ON(rb_first(&execlists->queue) != rb);
if (last) { if (last) {
/* /*
* Don't resubmit or switch until all outstanding * Don't resubmit or switch until all outstanding
...@@ -666,7 +660,7 @@ static void execlists_dequeue(struct intel_engine_cs *engine) ...@@ -666,7 +660,7 @@ static void execlists_dequeue(struct intel_engine_cs *engine)
last->tail = last->wa_tail; last->tail = last->wa_tail;
} }
while (rb) { while ((rb = rb_first_cached(&execlists->queue))) {
struct i915_priolist *p = to_priolist(rb); struct i915_priolist *p = to_priolist(rb);
struct i915_request *rq, *rn; struct i915_request *rq, *rn;
...@@ -725,8 +719,7 @@ static void execlists_dequeue(struct intel_engine_cs *engine) ...@@ -725,8 +719,7 @@ static void execlists_dequeue(struct intel_engine_cs *engine)
submit = true; submit = true;
} }
rb = rb_next(rb); rb_erase_cached(&p->node, &execlists->queue);
rb_erase(&p->node, &execlists->queue);
INIT_LIST_HEAD(&p->requests); INIT_LIST_HEAD(&p->requests);
if (p->priority != I915_PRIORITY_NORMAL) if (p->priority != I915_PRIORITY_NORMAL)
kmem_cache_free(engine->i915->priorities, p); kmem_cache_free(engine->i915->priorities, p);
...@@ -752,14 +745,14 @@ static void execlists_dequeue(struct intel_engine_cs *engine) ...@@ -752,14 +745,14 @@ static void execlists_dequeue(struct intel_engine_cs *engine)
execlists->queue_priority = execlists->queue_priority =
port != execlists->port ? rq_prio(last) : INT_MIN; port != execlists->port ? rq_prio(last) : INT_MIN;
execlists->first = rb;
if (submit) { if (submit) {
port_assign(port, last); port_assign(port, last);
execlists_submit_ports(engine); execlists_submit_ports(engine);
} }
/* We must always keep the beast fed if we have work piled up */ /* We must always keep the beast fed if we have work piled up */
GEM_BUG_ON(execlists->first && !port_isset(execlists->port)); GEM_BUG_ON(rb_first_cached(&execlists->queue) &&
!port_isset(execlists->port));
/* Re-evaluate the executing context setup after each preemptive kick */ /* Re-evaluate the executing context setup after each preemptive kick */
if (last) if (last)
...@@ -922,8 +915,7 @@ static void execlists_cancel_requests(struct intel_engine_cs *engine) ...@@ -922,8 +915,7 @@ static void execlists_cancel_requests(struct intel_engine_cs *engine)
} }
/* Flush the queued requests to the timeline list (for retiring). */ /* Flush the queued requests to the timeline list (for retiring). */
rb = execlists->first; while ((rb = rb_first_cached(&execlists->queue))) {
while (rb) {
struct i915_priolist *p = to_priolist(rb); struct i915_priolist *p = to_priolist(rb);
list_for_each_entry_safe(rq, rn, &p->requests, sched.link) { list_for_each_entry_safe(rq, rn, &p->requests, sched.link) {
...@@ -933,8 +925,7 @@ static void execlists_cancel_requests(struct intel_engine_cs *engine) ...@@ -933,8 +925,7 @@ static void execlists_cancel_requests(struct intel_engine_cs *engine)
__i915_request_submit(rq); __i915_request_submit(rq);
} }
rb = rb_next(rb); rb_erase_cached(&p->node, &execlists->queue);
rb_erase(&p->node, &execlists->queue);
INIT_LIST_HEAD(&p->requests); INIT_LIST_HEAD(&p->requests);
if (p->priority != I915_PRIORITY_NORMAL) if (p->priority != I915_PRIORITY_NORMAL)
kmem_cache_free(engine->i915->priorities, p); kmem_cache_free(engine->i915->priorities, p);
...@@ -943,8 +934,7 @@ static void execlists_cancel_requests(struct intel_engine_cs *engine) ...@@ -943,8 +934,7 @@ static void execlists_cancel_requests(struct intel_engine_cs *engine)
/* Remaining _unready_ requests will be nop'ed when submitted */ /* Remaining _unready_ requests will be nop'ed when submitted */
execlists->queue_priority = INT_MIN; execlists->queue_priority = INT_MIN;
execlists->queue = RB_ROOT; execlists->queue = RB_ROOT_CACHED;
execlists->first = NULL;
GEM_BUG_ON(port_isset(execlists->port)); GEM_BUG_ON(port_isset(execlists->port));
spin_unlock_irqrestore(&engine->timeline.lock, flags); spin_unlock_irqrestore(&engine->timeline.lock, flags);
...@@ -1192,7 +1182,7 @@ static void execlists_submit_request(struct i915_request *request) ...@@ -1192,7 +1182,7 @@ static void execlists_submit_request(struct i915_request *request)
queue_request(engine, &request->sched, rq_prio(request)); queue_request(engine, &request->sched, rq_prio(request));
GEM_BUG_ON(!engine->execlists.first); GEM_BUG_ON(RB_EMPTY_ROOT(&engine->execlists.queue.rb_root));
GEM_BUG_ON(list_empty(&request->sched.link)); GEM_BUG_ON(list_empty(&request->sched.link));
submit_queue(engine, rq_prio(request)); submit_queue(engine, rq_prio(request));
...@@ -2044,7 +2034,7 @@ static void execlists_reset_finish(struct intel_engine_cs *engine) ...@@ -2044,7 +2034,7 @@ static void execlists_reset_finish(struct intel_engine_cs *engine)
struct intel_engine_execlists * const execlists = &engine->execlists; struct intel_engine_execlists * const execlists = &engine->execlists;
/* After a GPU reset, we may have requests to replay */ /* After a GPU reset, we may have requests to replay */
if (execlists->first) if (!RB_EMPTY_ROOT(&execlists->queue.rb_root))
tasklet_schedule(&execlists->tasklet); tasklet_schedule(&execlists->tasklet);
/* /*
......
...@@ -292,12 +292,7 @@ struct intel_engine_execlists { ...@@ -292,12 +292,7 @@ struct intel_engine_execlists {
/** /**
* @queue: queue of requests, in priority lists * @queue: queue of requests, in priority lists
*/ */
struct rb_root queue; struct rb_root_cached queue;
/**
* @first: leftmost level in priority @queue
*/
struct rb_node *first;
/** /**
* @csb_read: control register for Context Switch buffer * @csb_read: control register for Context Switch buffer
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment