Commit 8a98e839 authored by Chris Wilson's avatar Chris Wilson

drm/i915/gtt: Recursive ppgtt alloc for gen8

Refactor the separate allocation routines into a single recursive
function.
Signed-off-by: default avatarChris Wilson <chris@chris-wilson.co.uk>
Reviewed-by: default avatarAbdiel Janulgue <abdiel.janulgue@linux.intel.com>
Link: https://patchwork.freedesktop.org/patch/msgid/20190712112725.2892-4-chris@chris-wilson.co.uk
parent 09975b86
...@@ -1007,199 +1007,119 @@ static void gen8_ppgtt_clear(struct i915_address_space *vm, ...@@ -1007,199 +1007,119 @@ static void gen8_ppgtt_clear(struct i915_address_space *vm,
start, start + length, vm->top); start, start + length, vm->top);
} }
static void gen8_ppgtt_clear_pd(struct i915_address_space *vm, static int __gen8_ppgtt_alloc(struct i915_address_space * const vm,
struct i915_page_directory *pd, struct i915_page_directory * const pd,
u64 start, u64 length) u64 * const start, u64 end, int lvl)
{
GEM_BUG_ON(!IS_ALIGNED(start, BIT_ULL(GEN8_PTE_SHIFT)));
GEM_BUG_ON(!IS_ALIGNED(length, BIT_ULL(GEN8_PTE_SHIFT)));
start >>= GEN8_PTE_SHIFT;
length >>= GEN8_PTE_SHIFT;
__gen8_ppgtt_clear(vm, pd, start, start + length, 1);
}
static void gen8_ppgtt_clear_pdp(struct i915_address_space *vm,
struct i915_page_directory * const pdp,
u64 start, u64 length)
{
GEM_BUG_ON(!IS_ALIGNED(start, BIT_ULL(GEN8_PTE_SHIFT)));
GEM_BUG_ON(!IS_ALIGNED(length, BIT_ULL(GEN8_PTE_SHIFT)));
start >>= GEN8_PTE_SHIFT;
length >>= GEN8_PTE_SHIFT;
__gen8_ppgtt_clear(vm, pdp, start, start + length, 2);
}
static int gen8_ppgtt_alloc_pd(struct i915_address_space *vm,
struct i915_page_directory *pd,
u64 start, u64 length)
{ {
struct i915_page_table *pt, *alloc = NULL; const struct i915_page_scratch * const scratch = &vm->scratch[lvl];
u64 from = start; struct i915_page_table *alloc = NULL;
unsigned int pde; unsigned int idx, len;
int ret = 0; int ret = 0;
len = gen8_pd_range(*start, end, lvl--, &idx);
DBG("%s(%p):{lvl:%d, start:%llx, end:%llx, idx:%d, len:%d, used:%d}\n",
__func__, vm, lvl + 1, *start, end,
idx, len, atomic_read(px_used(pd)));
GEM_BUG_ON(!len || (idx + len - 1) >> gen8_pd_shift(1));
spin_lock(&pd->lock); spin_lock(&pd->lock);
gen8_for_each_pde(pt, pd, start, length, pde) { GEM_BUG_ON(!atomic_read(px_used(pd))); /* Must be pinned! */
const int count = gen8_pte_count(start, length); do {
struct i915_page_table *pt = pd->entry[idx];
if (!pt) { if (!pt) {
spin_unlock(&pd->lock); spin_unlock(&pd->lock);
pt = fetch_and_zero(&alloc); DBG("%s(%p):{ lvl:%d, idx:%d } allocating new tree\n",
if (!pt) __func__, vm, lvl + 1, idx);
pt = alloc_pt(vm);
if (IS_ERR(pt)) {
ret = PTR_ERR(pt);
goto unwind;
}
if (count < GEN8_PTES || intel_vgpu_active(vm->i915)) pt = fetch_and_zero(&alloc);
fill_px(pt, vm->scratch[0].encode); if (lvl) {
if (!pt) {
pt = &alloc_pd(vm)->pt;
if (IS_ERR(pt)) {
ret = PTR_ERR(pt);
goto out;
}
}
spin_lock(&pd->lock); fill_px(pt, vm->scratch[lvl].encode);
if (!pd->entry[pde]) {
set_pd_entry(pd, pde, pt);
} else { } else {
alloc = pt; if (!pt) {
pt = pd->entry[pde]; pt = alloc_pt(vm);
} if (IS_ERR(pt)) {
} ret = PTR_ERR(pt);
goto out;
atomic_add(count, &pt->used); }
} }
spin_unlock(&pd->lock);
goto out;
unwind:
gen8_ppgtt_clear_pd(vm, pd, from, start - from);
out:
if (alloc)
free_px(vm, alloc);
return ret;
}
static int gen8_ppgtt_alloc_pdp(struct i915_address_space *vm,
struct i915_page_directory *pdp,
u64 start, u64 length)
{
struct i915_page_directory *pd, *alloc = NULL;
u64 from = start;
unsigned int pdpe;
int ret = 0;
spin_lock(&pdp->lock); if (intel_vgpu_active(vm->i915) ||
gen8_for_each_pdpe(pd, pdp, start, length, pdpe) { gen8_pt_count(*start, end) < I915_PDES)
if (!pd) { fill_px(pt, vm->scratch[lvl].encode);
spin_unlock(&pdp->lock);
pd = fetch_and_zero(&alloc);
if (!pd)
pd = alloc_pd(vm);
if (IS_ERR(pd)) {
ret = PTR_ERR(pd);
goto unwind;
} }
fill_px(pd, vm->scratch[1].encode); spin_lock(&pd->lock);
if (likely(!pd->entry[idx]))
set_pd_entry(pd, idx, pt);
else
alloc = pt, pt = pd->entry[idx];
}
spin_lock(&pdp->lock); if (lvl) {
if (!pdp->entry[pdpe]) { atomic_inc(&pt->used);
set_pd_entry(pdp, pdpe, pd); spin_unlock(&pd->lock);
} else {
alloc = pd; ret = __gen8_ppgtt_alloc(vm, as_pd(pt),
pd = pdp->entry[pdpe]; start, end, lvl);
if (unlikely(ret)) {
if (release_pd_entry(pd, idx, pt, scratch))
free_px(vm, pt);
goto out;
} }
}
atomic_inc(px_used(pd));
spin_unlock(&pdp->lock);
ret = gen8_ppgtt_alloc_pd(vm, pd, start, length); spin_lock(&pd->lock);
if (unlikely(ret)) atomic_dec(&pt->used);
goto unwind_pd; GEM_BUG_ON(!atomic_read(&pt->used));
} else {
unsigned int count = gen8_pt_count(*start, end);
spin_lock(&pdp->lock); DBG("%s(%p):{lvl:%d, start:%llx, end:%llx, idx:%d, len:%d, used:%d} inserting pte\n",
atomic_dec(px_used(pd)); __func__, vm, lvl, *start, end,
} gen8_pd_index(*start, 0), count,
spin_unlock(&pdp->lock); atomic_read(&pt->used));
goto out;
unwind_pd: atomic_add(count, &pt->used);
if (release_pd_entry(pdp, pdpe, &pd->pt, &vm->scratch[2])) GEM_BUG_ON(atomic_read(&pt->used) > I915_PDES);
free_px(vm, pd); *start += count;
unwind: }
gen8_ppgtt_clear_pdp(vm, pdp, from, start - from); } while (idx++, --len);
spin_unlock(&pd->lock);
out: out:
if (alloc) if (alloc)
free_px(vm, alloc); free_px(vm, alloc);
return ret; return ret;
} }
static int gen8_ppgtt_alloc_3lvl(struct i915_address_space *vm, static int gen8_ppgtt_alloc(struct i915_address_space *vm,
u64 start, u64 length) u64 start, u64 length)
{
return gen8_ppgtt_alloc_pdp(vm,
i915_vm_to_ppgtt(vm)->pd, start, length);
}
static int gen8_ppgtt_alloc_4lvl(struct i915_address_space *vm,
u64 start, u64 length)
{ {
struct i915_ppgtt *ppgtt = i915_vm_to_ppgtt(vm);
struct i915_page_directory * const pml4 = ppgtt->pd;
struct i915_page_directory *pdp, *alloc = NULL;
u64 from = start; u64 from = start;
int ret = 0; int err;
u32 pml4e;
spin_lock(&pml4->lock);
gen8_for_each_pml4e(pdp, pml4, start, length, pml4e) {
if (!pdp) {
spin_unlock(&pml4->lock);
pdp = fetch_and_zero(&alloc);
if (!pdp)
pdp = alloc_pd(vm);
if (IS_ERR(pdp)) {
ret = PTR_ERR(pdp);
goto unwind;
}
fill_px(pdp, vm->scratch[2].encode);
spin_lock(&pml4->lock); GEM_BUG_ON(!IS_ALIGNED(start, BIT_ULL(GEN8_PTE_SHIFT)));
if (!pml4->entry[pml4e]) { GEM_BUG_ON(!IS_ALIGNED(length, BIT_ULL(GEN8_PTE_SHIFT)));
set_pd_entry(pml4, pml4e, pdp);
} else {
alloc = pdp;
pdp = pml4->entry[pml4e];
}
}
atomic_inc(px_used(pdp));
spin_unlock(&pml4->lock);
ret = gen8_ppgtt_alloc_pdp(vm, pdp, start, length); start >>= GEN8_PTE_SHIFT;
if (unlikely(ret)) length >>= GEN8_PTE_SHIFT;
goto unwind_pdp; GEM_BUG_ON(length == 0);
spin_lock(&pml4->lock); err = __gen8_ppgtt_alloc(vm, i915_vm_to_ppgtt(vm)->pd,
atomic_dec(px_used(pdp)); &start, start + length, vm->top);
} if (unlikely(err))
spin_unlock(&pml4->lock); __gen8_ppgtt_clear(vm, i915_vm_to_ppgtt(vm)->pd,
goto out; from, start, vm->top);
unwind_pdp: return err;
if (release_pd_entry(pml4, pml4e, &pdp->pt, &vm->scratch[3]))
free_px(vm, pdp);
unwind:
gen8_ppgtt_clear(vm, from, start - from);
out:
if (alloc)
free_px(vm, alloc);
return ret;
} }
static inline struct sgt_dma { static inline struct sgt_dma {
...@@ -1496,19 +1416,22 @@ static int gen8_init_scratch(struct i915_address_space *vm) ...@@ -1496,19 +1416,22 @@ static int gen8_init_scratch(struct i915_address_space *vm)
static int gen8_preallocate_top_level_pdp(struct i915_ppgtt *ppgtt) static int gen8_preallocate_top_level_pdp(struct i915_ppgtt *ppgtt)
{ {
struct i915_address_space *vm = &ppgtt->vm; struct i915_address_space *vm = &ppgtt->vm;
struct i915_page_directory *pdp = ppgtt->pd; struct i915_page_directory *pd = ppgtt->pd;
struct i915_page_directory *pd; unsigned int idx;
u64 start = 0, length = ppgtt->vm.total;
unsigned int pdpe; GEM_BUG_ON(vm->top != 2);
GEM_BUG_ON((vm->total >> __gen8_pte_shift(2)) != GEN8_3LVL_PDPES);
for (idx = 0; idx < GEN8_3LVL_PDPES; idx++) {
struct i915_page_directory *pde;
gen8_for_each_pdpe(pd, pdp, start, length, pdpe) { pde = alloc_pd(vm);
pd = alloc_pd(vm); if (IS_ERR(pde))
if (IS_ERR(pd)) return PTR_ERR(pde);
return PTR_ERR(pd);
fill_px(pd, vm->scratch[1].encode); fill_px(pde, vm->scratch[1].encode);
set_pd_entry(pdp, pdpe, pd); set_pd_entry(pd, idx, pde);
atomic_inc(px_used(pd)); /* keep pinned */ atomic_inc(px_used(pde)); /* keep pinned */
} }
return 0; return 0;
...@@ -1597,7 +1520,6 @@ static struct i915_ppgtt *gen8_ppgtt_create(struct drm_i915_private *i915) ...@@ -1597,7 +1520,6 @@ static struct i915_ppgtt *gen8_ppgtt_create(struct drm_i915_private *i915)
} }
if (i915_vm_is_4lvl(&ppgtt->vm)) { if (i915_vm_is_4lvl(&ppgtt->vm)) {
ppgtt->vm.allocate_va_range = gen8_ppgtt_alloc_4lvl;
ppgtt->vm.insert_entries = gen8_ppgtt_insert_4lvl; ppgtt->vm.insert_entries = gen8_ppgtt_insert_4lvl;
} else { } else {
if (intel_vgpu_active(i915)) { if (intel_vgpu_active(i915)) {
...@@ -1606,10 +1528,10 @@ static struct i915_ppgtt *gen8_ppgtt_create(struct drm_i915_private *i915) ...@@ -1606,10 +1528,10 @@ static struct i915_ppgtt *gen8_ppgtt_create(struct drm_i915_private *i915)
goto err_free_pd; goto err_free_pd;
} }
ppgtt->vm.allocate_va_range = gen8_ppgtt_alloc_3lvl;
ppgtt->vm.insert_entries = gen8_ppgtt_insert_3lvl; ppgtt->vm.insert_entries = gen8_ppgtt_insert_3lvl;
} }
ppgtt->vm.allocate_va_range = gen8_ppgtt_alloc;
ppgtt->vm.clear_range = gen8_ppgtt_clear; ppgtt->vm.clear_range = gen8_ppgtt_clear;
if (intel_vgpu_active(i915)) if (intel_vgpu_active(i915))
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment