Commit 987046ad authored by Chris Wilson's avatar Chris Wilson

drm/i915: Unify intel_ring_begin()

Combine the near identical implementations of intel_logical_ring_begin()
and intel_ring_begin() - the only difference is that the logical wait
has to check for a matching ring (which is assumed by legacy).

In the process some debug messages are culled as there were following a
WARN if we hit an actual error.

v2: Updated commentary
Signed-off-by: default avatarChris Wilson <chris@chris-wilson.co.uk>
Cc: Joonas Lahtinen <joonas.lahtinen@linux.intel.com>
Cc: Tvrtko Ursulin <tvrtko.ursulin@linux.intel.com>
Reviewed-by: default avatarJoonas Lahtinen <joonas.lahtinen@linux.intel.com>
Link: http://patchwork.freedesktop.org/patch/msgid/1461833819-3991-12-git-send-email-chris@chris-wilson.co.uk
parent f9326be5
...@@ -721,48 +721,6 @@ int intel_logical_ring_alloc_request_extras(struct drm_i915_gem_request *request ...@@ -721,48 +721,6 @@ int intel_logical_ring_alloc_request_extras(struct drm_i915_gem_request *request
return ret; return ret;
} }
static int logical_ring_wait_for_space(struct drm_i915_gem_request *req,
int bytes)
{
struct intel_ringbuffer *ringbuf = req->ringbuf;
struct intel_engine_cs *engine = req->engine;
struct drm_i915_gem_request *target;
unsigned space;
int ret;
if (intel_ring_space(ringbuf) >= bytes)
return 0;
/* The whole point of reserving space is to not wait! */
WARN_ON(ringbuf->reserved_in_use);
list_for_each_entry(target, &engine->request_list, list) {
/*
* The request queue is per-engine, so can contain requests
* from multiple ringbuffers. Here, we must ignore any that
* aren't from the ringbuffer we're considering.
*/
if (target->ringbuf != ringbuf)
continue;
/* Would completion of this request free enough space? */
space = __intel_ring_space(target->postfix, ringbuf->tail,
ringbuf->size);
if (space >= bytes)
break;
}
if (WARN_ON(&target->list == &engine->request_list))
return -ENOSPC;
ret = i915_wait_request(target);
if (ret)
return ret;
ringbuf->space = space;
return 0;
}
/* /*
* intel_logical_ring_advance_and_submit() - advance the tail and submit the workload * intel_logical_ring_advance_and_submit() - advance the tail and submit the workload
* @request: Request to advance the logical ringbuffer of. * @request: Request to advance the logical ringbuffer of.
...@@ -814,92 +772,6 @@ intel_logical_ring_advance_and_submit(struct drm_i915_gem_request *request) ...@@ -814,92 +772,6 @@ intel_logical_ring_advance_and_submit(struct drm_i915_gem_request *request)
return 0; return 0;
} }
static void __wrap_ring_buffer(struct intel_ringbuffer *ringbuf)
{
uint32_t __iomem *virt;
int rem = ringbuf->size - ringbuf->tail;
virt = ringbuf->virtual_start + ringbuf->tail;
rem /= 4;
while (rem--)
iowrite32(MI_NOOP, virt++);
ringbuf->tail = 0;
intel_ring_update_space(ringbuf);
}
static int logical_ring_prepare(struct drm_i915_gem_request *req, int bytes)
{
struct intel_ringbuffer *ringbuf = req->ringbuf;
int remain_usable = ringbuf->effective_size - ringbuf->tail;
int remain_actual = ringbuf->size - ringbuf->tail;
int ret, total_bytes, wait_bytes = 0;
bool need_wrap = false;
if (ringbuf->reserved_in_use)
total_bytes = bytes;
else
total_bytes = bytes + ringbuf->reserved_size;
if (unlikely(bytes > remain_usable)) {
/*
* Not enough space for the basic request. So need to flush
* out the remainder and then wait for base + reserved.
*/
wait_bytes = remain_actual + total_bytes;
need_wrap = true;
} else {
if (unlikely(total_bytes > remain_usable)) {
/*
* The base request will fit but the reserved space
* falls off the end. So don't need an immediate wrap
* and only need to effectively wait for the reserved
* size space from the start of ringbuffer.
*/
wait_bytes = remain_actual + ringbuf->reserved_size;
} else if (total_bytes > ringbuf->space) {
/* No wrapping required, just waiting. */
wait_bytes = total_bytes;
}
}
if (wait_bytes) {
ret = logical_ring_wait_for_space(req, wait_bytes);
if (unlikely(ret))
return ret;
if (need_wrap)
__wrap_ring_buffer(ringbuf);
}
return 0;
}
/**
* intel_logical_ring_begin() - prepare the logical ringbuffer to accept some commands
*
* @req: The request to start some new work for
* @num_dwords: number of DWORDs that we plan to write to the ringbuffer.
*
* The ringbuffer might not be ready to accept the commands right away (maybe it needs to
* be wrapped, or wait a bit for the tail to be updated). This function takes care of that
* and also preallocates a request (every workload submission is still mediated through
* requests, same as it did with legacy ringbuffer submission).
*
* Return: non-zero if the ringbuffer is not ready to be written to.
*/
int intel_logical_ring_begin(struct drm_i915_gem_request *req, int num_dwords)
{
int ret;
ret = logical_ring_prepare(req, num_dwords * sizeof(uint32_t));
if (ret)
return ret;
req->ringbuf->space -= num_dwords * sizeof(uint32_t);
return 0;
}
int intel_logical_ring_reserve_space(struct drm_i915_gem_request *request) int intel_logical_ring_reserve_space(struct drm_i915_gem_request *request)
{ {
/* /*
...@@ -912,7 +784,7 @@ int intel_logical_ring_reserve_space(struct drm_i915_gem_request *request) ...@@ -912,7 +784,7 @@ int intel_logical_ring_reserve_space(struct drm_i915_gem_request *request)
*/ */
intel_ring_reserved_space_reserve(request->ringbuf, MIN_SPACE_FOR_ADD_REQUEST); intel_ring_reserved_space_reserve(request->ringbuf, MIN_SPACE_FOR_ADD_REQUEST);
return intel_logical_ring_begin(request, 0); return intel_ring_begin(request, 0);
} }
/** /**
...@@ -982,7 +854,7 @@ int intel_execlists_submission(struct i915_execbuffer_params *params, ...@@ -982,7 +854,7 @@ int intel_execlists_submission(struct i915_execbuffer_params *params,
if (engine == &dev_priv->engine[RCS] && if (engine == &dev_priv->engine[RCS] &&
instp_mode != dev_priv->relative_constants_mode) { instp_mode != dev_priv->relative_constants_mode) {
ret = intel_logical_ring_begin(params->request, 4); ret = intel_ring_begin(params->request, 4);
if (ret) if (ret)
return ret; return ret;
...@@ -1178,7 +1050,7 @@ static int intel_logical_ring_workarounds_emit(struct drm_i915_gem_request *req) ...@@ -1178,7 +1050,7 @@ static int intel_logical_ring_workarounds_emit(struct drm_i915_gem_request *req)
if (ret) if (ret)
return ret; return ret;
ret = intel_logical_ring_begin(req, w->count * 2 + 2); ret = intel_ring_begin(req, w->count * 2 + 2);
if (ret) if (ret)
return ret; return ret;
...@@ -1671,7 +1543,7 @@ static int intel_logical_ring_emit_pdps(struct drm_i915_gem_request *req) ...@@ -1671,7 +1543,7 @@ static int intel_logical_ring_emit_pdps(struct drm_i915_gem_request *req)
const int num_lri_cmds = GEN8_LEGACY_PDPES * 2; const int num_lri_cmds = GEN8_LEGACY_PDPES * 2;
int i, ret; int i, ret;
ret = intel_logical_ring_begin(req, num_lri_cmds * 2 + 2); ret = intel_ring_begin(req, num_lri_cmds * 2 + 2);
if (ret) if (ret)
return ret; return ret;
...@@ -1718,7 +1590,7 @@ static int gen8_emit_bb_start(struct drm_i915_gem_request *req, ...@@ -1718,7 +1590,7 @@ static int gen8_emit_bb_start(struct drm_i915_gem_request *req,
req->ctx->ppgtt->pd_dirty_rings &= ~intel_engine_flag(req->engine); req->ctx->ppgtt->pd_dirty_rings &= ~intel_engine_flag(req->engine);
} }
ret = intel_logical_ring_begin(req, 4); ret = intel_ring_begin(req, 4);
if (ret) if (ret)
return ret; return ret;
...@@ -1780,7 +1652,7 @@ static int gen8_emit_flush(struct drm_i915_gem_request *request, ...@@ -1780,7 +1652,7 @@ static int gen8_emit_flush(struct drm_i915_gem_request *request,
uint32_t cmd; uint32_t cmd;
int ret; int ret;
ret = intel_logical_ring_begin(request, 4); ret = intel_ring_begin(request, 4);
if (ret) if (ret)
return ret; return ret;
...@@ -1848,7 +1720,7 @@ static int gen8_emit_flush_render(struct drm_i915_gem_request *request, ...@@ -1848,7 +1720,7 @@ static int gen8_emit_flush_render(struct drm_i915_gem_request *request,
vf_flush_wa = true; vf_flush_wa = true;
} }
ret = intel_logical_ring_begin(request, vf_flush_wa ? 12 : 6); ret = intel_ring_begin(request, vf_flush_wa ? 12 : 6);
if (ret) if (ret)
return ret; return ret;
...@@ -1922,7 +1794,7 @@ static int gen8_emit_request(struct drm_i915_gem_request *request) ...@@ -1922,7 +1794,7 @@ static int gen8_emit_request(struct drm_i915_gem_request *request)
struct intel_ringbuffer *ringbuf = request->ringbuf; struct intel_ringbuffer *ringbuf = request->ringbuf;
int ret; int ret;
ret = intel_logical_ring_begin(request, 6 + WA_TAIL_DWORDS); ret = intel_ring_begin(request, 6 + WA_TAIL_DWORDS);
if (ret) if (ret)
return ret; return ret;
...@@ -1946,7 +1818,7 @@ static int gen8_emit_request_render(struct drm_i915_gem_request *request) ...@@ -1946,7 +1818,7 @@ static int gen8_emit_request_render(struct drm_i915_gem_request *request)
struct intel_ringbuffer *ringbuf = request->ringbuf; struct intel_ringbuffer *ringbuf = request->ringbuf;
int ret; int ret;
ret = intel_logical_ring_begin(request, 8 + WA_TAIL_DWORDS); ret = intel_ring_begin(request, 8 + WA_TAIL_DWORDS);
if (ret) if (ret)
return ret; return ret;
......
...@@ -63,7 +63,6 @@ int intel_logical_ring_reserve_space(struct drm_i915_gem_request *request); ...@@ -63,7 +63,6 @@ int intel_logical_ring_reserve_space(struct drm_i915_gem_request *request);
void intel_logical_ring_stop(struct intel_engine_cs *engine); void intel_logical_ring_stop(struct intel_engine_cs *engine);
void intel_logical_ring_cleanup(struct intel_engine_cs *engine); void intel_logical_ring_cleanup(struct intel_engine_cs *engine);
int intel_logical_rings_init(struct drm_device *dev); int intel_logical_rings_init(struct drm_device *dev);
int intel_logical_ring_begin(struct drm_i915_gem_request *req, int num_dwords);
int logical_ring_flush_all_caches(struct drm_i915_gem_request *req); int logical_ring_flush_all_caches(struct drm_i915_gem_request *req);
/** /**
......
...@@ -239,11 +239,9 @@ static int emit_mocs_control_table(struct drm_i915_gem_request *req, ...@@ -239,11 +239,9 @@ static int emit_mocs_control_table(struct drm_i915_gem_request *req,
if (WARN_ON(table->size > GEN9_NUM_MOCS_ENTRIES)) if (WARN_ON(table->size > GEN9_NUM_MOCS_ENTRIES))
return -ENODEV; return -ENODEV;
ret = intel_logical_ring_begin(req, 2 + 2 * GEN9_NUM_MOCS_ENTRIES); ret = intel_ring_begin(req, 2 + 2 * GEN9_NUM_MOCS_ENTRIES);
if (ret) { if (ret)
DRM_DEBUG("intel_logical_ring_begin failed %d\n", ret);
return ret; return ret;
}
intel_logical_ring_emit(ringbuf, intel_logical_ring_emit(ringbuf,
MI_LOAD_REGISTER_IMM(GEN9_NUM_MOCS_ENTRIES)); MI_LOAD_REGISTER_IMM(GEN9_NUM_MOCS_ENTRIES));
...@@ -305,11 +303,9 @@ static int emit_mocs_l3cc_table(struct drm_i915_gem_request *req, ...@@ -305,11 +303,9 @@ static int emit_mocs_l3cc_table(struct drm_i915_gem_request *req,
if (WARN_ON(table->size > GEN9_NUM_MOCS_ENTRIES)) if (WARN_ON(table->size > GEN9_NUM_MOCS_ENTRIES))
return -ENODEV; return -ENODEV;
ret = intel_logical_ring_begin(req, 2 + GEN9_NUM_MOCS_ENTRIES); ret = intel_ring_begin(req, 2 + GEN9_NUM_MOCS_ENTRIES);
if (ret) { if (ret)
DRM_DEBUG("intel_logical_ring_begin failed %d\n", ret);
return ret; return ret;
}
intel_logical_ring_emit(ringbuf, intel_logical_ring_emit(ringbuf,
MI_LOAD_REGISTER_IMM(GEN9_NUM_MOCS_ENTRIES / 2)); MI_LOAD_REGISTER_IMM(GEN9_NUM_MOCS_ENTRIES / 2));
......
...@@ -53,12 +53,6 @@ void intel_ring_update_space(struct intel_ringbuffer *ringbuf) ...@@ -53,12 +53,6 @@ void intel_ring_update_space(struct intel_ringbuffer *ringbuf)
ringbuf->tail, ringbuf->size); ringbuf->tail, ringbuf->size);
} }
int intel_ring_space(struct intel_ringbuffer *ringbuf)
{
intel_ring_update_space(ringbuf);
return ringbuf->space;
}
bool intel_engine_stopped(struct intel_engine_cs *engine) bool intel_engine_stopped(struct intel_engine_cs *engine)
{ {
struct drm_i915_private *dev_priv = engine->dev->dev_private; struct drm_i915_private *dev_priv = engine->dev->dev_private;
...@@ -2325,51 +2319,6 @@ void intel_cleanup_engine(struct intel_engine_cs *engine) ...@@ -2325,51 +2319,6 @@ void intel_cleanup_engine(struct intel_engine_cs *engine)
engine->dev = NULL; engine->dev = NULL;
} }
static int ring_wait_for_space(struct intel_engine_cs *engine, int n)
{
struct intel_ringbuffer *ringbuf = engine->buffer;
struct drm_i915_gem_request *request;
unsigned space;
int ret;
if (intel_ring_space(ringbuf) >= n)
return 0;
/* The whole point of reserving space is to not wait! */
WARN_ON(ringbuf->reserved_in_use);
list_for_each_entry(request, &engine->request_list, list) {
space = __intel_ring_space(request->postfix, ringbuf->tail,
ringbuf->size);
if (space >= n)
break;
}
if (WARN_ON(&request->list == &engine->request_list))
return -ENOSPC;
ret = i915_wait_request(request);
if (ret)
return ret;
ringbuf->space = space;
return 0;
}
static void __wrap_ring_buffer(struct intel_ringbuffer *ringbuf)
{
uint32_t __iomem *virt;
int rem = ringbuf->size - ringbuf->tail;
virt = ringbuf->virtual_start + ringbuf->tail;
rem /= 4;
while (rem--)
iowrite32(MI_NOOP, virt++);
ringbuf->tail = 0;
intel_ring_update_space(ringbuf);
}
int intel_engine_idle(struct intel_engine_cs *engine) int intel_engine_idle(struct intel_engine_cs *engine)
{ {
struct drm_i915_gem_request *req; struct drm_i915_gem_request *req;
...@@ -2411,63 +2360,82 @@ int intel_ring_reserve_space(struct drm_i915_gem_request *request) ...@@ -2411,63 +2360,82 @@ int intel_ring_reserve_space(struct drm_i915_gem_request *request)
void intel_ring_reserved_space_reserve(struct intel_ringbuffer *ringbuf, int size) void intel_ring_reserved_space_reserve(struct intel_ringbuffer *ringbuf, int size)
{ {
WARN_ON(ringbuf->reserved_size); GEM_BUG_ON(ringbuf->reserved_size);
WARN_ON(ringbuf->reserved_in_use);
ringbuf->reserved_size = size; ringbuf->reserved_size = size;
} }
void intel_ring_reserved_space_cancel(struct intel_ringbuffer *ringbuf) void intel_ring_reserved_space_cancel(struct intel_ringbuffer *ringbuf)
{ {
WARN_ON(ringbuf->reserved_in_use); GEM_BUG_ON(!ringbuf->reserved_size);
ringbuf->reserved_size = 0; ringbuf->reserved_size = 0;
ringbuf->reserved_in_use = false;
} }
void intel_ring_reserved_space_use(struct intel_ringbuffer *ringbuf) void intel_ring_reserved_space_use(struct intel_ringbuffer *ringbuf)
{ {
WARN_ON(ringbuf->reserved_in_use); GEM_BUG_ON(!ringbuf->reserved_size);
ringbuf->reserved_size = 0;
ringbuf->reserved_in_use = true;
ringbuf->reserved_tail = ringbuf->tail;
} }
void intel_ring_reserved_space_end(struct intel_ringbuffer *ringbuf) void intel_ring_reserved_space_end(struct intel_ringbuffer *ringbuf)
{ {
WARN_ON(!ringbuf->reserved_in_use); GEM_BUG_ON(ringbuf->reserved_size);
if (ringbuf->tail > ringbuf->reserved_tail) { }
WARN(ringbuf->tail > ringbuf->reserved_tail + ringbuf->reserved_size,
"request reserved size too small: %d vs %d!\n", static int wait_for_space(struct drm_i915_gem_request *req, int bytes)
ringbuf->tail - ringbuf->reserved_tail, ringbuf->reserved_size); {
} else { struct intel_ringbuffer *ringbuf = req->ringbuf;
struct intel_engine_cs *engine = req->engine;
struct drm_i915_gem_request *target;
intel_ring_update_space(ringbuf);
if (ringbuf->space >= bytes)
return 0;
/*
* Space is reserved in the ringbuffer for finalising the request,
* as that cannot be allowed to fail. During request finalisation,
* reserved_space is set to 0 to stop the overallocation and the
* assumption is that then we never need to wait (which has the
* risk of failing with EINTR).
*
* See also i915_gem_request_alloc() and i915_add_request().
*/
GEM_BUG_ON(!ringbuf->reserved_size);
list_for_each_entry(target, &engine->request_list, list) {
unsigned space;
/* /*
* The ring was wrapped while the reserved space was in use. * The request queue is per-engine, so can contain requests
* That means that some unknown amount of the ring tail was * from multiple ringbuffers. Here, we must ignore any that
* no-op filled and skipped. Thus simply adding the ring size * aren't from the ringbuffer we're considering.
* to the tail and doing the above space check will not work.
* Rather than attempt to track how much tail was skipped,
* it is much simpler to say that also skipping the sanity
* check every once in a while is not a big issue.
*/ */
if (target->ringbuf != ringbuf)
continue;
/* Would completion of this request free enough space? */
space = __intel_ring_space(target->postfix, ringbuf->tail,
ringbuf->size);
if (space >= bytes)
break;
} }
ringbuf->reserved_size = 0; if (WARN_ON(&target->list == &engine->request_list))
ringbuf->reserved_in_use = false; return -ENOSPC;
return i915_wait_request(target);
} }
static int __intel_ring_prepare(struct intel_engine_cs *engine, int bytes) int intel_ring_begin(struct drm_i915_gem_request *req, int num_dwords)
{ {
struct intel_ringbuffer *ringbuf = engine->buffer; struct intel_ringbuffer *ringbuf = req->ringbuf;
int remain_usable = ringbuf->effective_size - ringbuf->tail;
int remain_actual = ringbuf->size - ringbuf->tail; int remain_actual = ringbuf->size - ringbuf->tail;
int ret, total_bytes, wait_bytes = 0; int remain_usable = ringbuf->effective_size - ringbuf->tail;
int bytes = num_dwords * sizeof(u32);
int total_bytes, wait_bytes;
bool need_wrap = false; bool need_wrap = false;
if (ringbuf->reserved_in_use) total_bytes = bytes + ringbuf->reserved_size;
total_bytes = bytes;
else
total_bytes = bytes + ringbuf->reserved_size;
if (unlikely(bytes > remain_usable)) { if (unlikely(bytes > remain_usable)) {
/* /*
...@@ -2476,44 +2444,40 @@ static int __intel_ring_prepare(struct intel_engine_cs *engine, int bytes) ...@@ -2476,44 +2444,40 @@ static int __intel_ring_prepare(struct intel_engine_cs *engine, int bytes)
*/ */
wait_bytes = remain_actual + total_bytes; wait_bytes = remain_actual + total_bytes;
need_wrap = true; need_wrap = true;
} else if (unlikely(total_bytes > remain_usable)) {
/*
* The base request will fit but the reserved space
* falls off the end. So we don't need an immediate wrap
* and only need to effectively wait for the reserved
* size space from the start of ringbuffer.
*/
wait_bytes = remain_actual + ringbuf->reserved_size;
} else { } else {
if (unlikely(total_bytes > remain_usable)) { /* No wrapping required, just waiting. */
/* wait_bytes = total_bytes;
* The base request will fit but the reserved space
* falls off the end. So don't need an immediate wrap
* and only need to effectively wait for the reserved
* size space from the start of ringbuffer.
*/
wait_bytes = remain_actual + ringbuf->reserved_size;
} else if (total_bytes > ringbuf->space) {
/* No wrapping required, just waiting. */
wait_bytes = total_bytes;
}
} }
if (wait_bytes) { if (wait_bytes > ringbuf->space) {
ret = ring_wait_for_space(engine, wait_bytes); int ret = wait_for_space(req, wait_bytes);
if (unlikely(ret)) if (unlikely(ret))
return ret; return ret;
if (need_wrap) intel_ring_update_space(ringbuf);
__wrap_ring_buffer(ringbuf);
} }
return 0; if (unlikely(need_wrap)) {
} GEM_BUG_ON(remain_actual > ringbuf->space);
GEM_BUG_ON(ringbuf->tail + remain_actual > ringbuf->size);
int intel_ring_begin(struct drm_i915_gem_request *req, /* Fill the tail with MI_NOOP */
int num_dwords) memset(ringbuf->virtual_start + ringbuf->tail,
{ 0, remain_actual);
struct intel_engine_cs *engine = req->engine; ringbuf->tail = 0;
int ret; ringbuf->space -= remain_actual;
}
ret = __intel_ring_prepare(engine, num_dwords * sizeof(uint32_t));
if (ret)
return ret;
engine->buffer->space -= num_dwords * sizeof(uint32_t); ringbuf->space -= bytes;
GEM_BUG_ON(ringbuf->space < 0);
return 0; return 0;
} }
......
...@@ -108,8 +108,6 @@ struct intel_ringbuffer { ...@@ -108,8 +108,6 @@ struct intel_ringbuffer {
int size; int size;
int effective_size; int effective_size;
int reserved_size; int reserved_size;
int reserved_tail;
bool reserved_in_use;
/** We track the position of the requests in the ring buffer, and /** We track the position of the requests in the ring buffer, and
* when each is retired we increment last_retired_head as the GPU * when each is retired we increment last_retired_head as the GPU
...@@ -459,7 +457,6 @@ static inline void intel_ring_advance(struct intel_engine_cs *engine) ...@@ -459,7 +457,6 @@ static inline void intel_ring_advance(struct intel_engine_cs *engine)
} }
int __intel_ring_space(int head, int tail, int size); int __intel_ring_space(int head, int tail, int size);
void intel_ring_update_space(struct intel_ringbuffer *ringbuf); void intel_ring_update_space(struct intel_ringbuffer *ringbuf);
int intel_ring_space(struct intel_ringbuffer *ringbuf);
bool intel_engine_stopped(struct intel_engine_cs *engine); bool intel_engine_stopped(struct intel_engine_cs *engine);
int __must_check intel_engine_idle(struct intel_engine_cs *engine); int __must_check intel_engine_idle(struct intel_engine_cs *engine);
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment