Commit cc662126 authored by Abdiel Janulgue's avatar Abdiel Janulgue Committed by Chris Wilson

drm/i915: Introduce DRM_I915_GEM_MMAP_OFFSET

This is really just an alias of mmap_gtt. The 'mmap offset' nomenclature
comes from the value returned by this ioctl which is the offset into the
device fd which userpace uses with mmap(2).

mmap_gtt was our initial mmap_offset implementation, this extends
our CPU mmap support to allow additional fault handlers that depends on
the object's backing pages.

Note that we multiplex mmap_gtt and mmap_offset through the same ioctl,
and use the zero extending behaviour of drm to differentiate between
them, when we inspect the flags.

To support multiple mmap types on an object we need to support multiple
mmap_offsets for an object (each offset in the global device address
space corresponding to a unique instance of the object for a file + mmap
type). As we drop the simplified drm core idea of a single mmap_offset,
we need to provide replacement hooks for the dumb mmap interface as
well.

Link: https://gitlab.freedesktop.org/mesa/mesa/merge_requests/1675
Testcase: igt/gem_mmap_offset
Signed-off-by: default avatarAbdiel Janulgue <abdiel.janulgue@linux.intel.com>
Signed-off-by: default avatarMatthew Auld <matthew.auld@intel.com>
Reviewed-by: default avatarChris Wilson <chris@chris-wilson.co.uk>
Signed-off-by: default avatarChris Wilson <chris@chris-wilson.co.uk>
Link: https://patchwork.freedesktop.org/patch/msgid/20191204120032.3682839-1-chris@chris-wilson.co.uk
parent c415ef2a
...@@ -13,6 +13,7 @@ ...@@ -13,6 +13,7 @@
#include "i915_gem_object.h" #include "i915_gem_object.h"
#include "i915_vma.h" #include "i915_vma.h"
#include "i915_gem_lmem.h" #include "i915_gem_lmem.h"
#include "i915_gem_mman.h"
static void __i915_gem_object_flush_for_display(struct drm_i915_gem_object *obj) static void __i915_gem_object_flush_for_display(struct drm_i915_gem_object *obj)
{ {
......
...@@ -28,8 +28,8 @@ int i915_gem_madvise_ioctl(struct drm_device *dev, void *data, ...@@ -28,8 +28,8 @@ int i915_gem_madvise_ioctl(struct drm_device *dev, void *data,
struct drm_file *file); struct drm_file *file);
int i915_gem_mmap_ioctl(struct drm_device *dev, void *data, int i915_gem_mmap_ioctl(struct drm_device *dev, void *data,
struct drm_file *file); struct drm_file *file);
int i915_gem_mmap_gtt_ioctl(struct drm_device *dev, void *data, int i915_gem_mmap_offset_ioctl(struct drm_device *dev, void *data,
struct drm_file *file); struct drm_file *file);
int i915_gem_pread_ioctl(struct drm_device *dev, void *data, int i915_gem_pread_ioctl(struct drm_device *dev, void *data,
struct drm_file *file); struct drm_file *file);
int i915_gem_pwrite_ioctl(struct drm_device *dev, void *data, int i915_gem_pwrite_ioctl(struct drm_device *dev, void *data,
......
This diff is collapsed.
/*
* SPDX-License-Identifier: MIT
*
* Copyright © 2019 Intel Corporation
*/
#ifndef __I915_GEM_MMAN_H__
#define __I915_GEM_MMAN_H__
#include <linux/mm_types.h>
#include <linux/types.h>
struct drm_device;
struct drm_file;
struct drm_i915_gem_object;
struct file;
struct i915_mmap_offset;
struct mutex;
int i915_gem_mmap_gtt_version(void);
int i915_gem_mmap(struct file *filp, struct vm_area_struct *vma);
int i915_gem_dumb_mmap_offset(struct drm_file *file_priv,
struct drm_device *dev,
u32 handle, u64 *offset);
void __i915_gem_object_release_mmap_gtt(struct drm_i915_gem_object *obj);
void i915_gem_object_release_mmap(struct drm_i915_gem_object *obj);
void i915_gem_object_release_mmap_offset(struct drm_i915_gem_object *obj);
#endif
...@@ -29,6 +29,7 @@ ...@@ -29,6 +29,7 @@
#include "i915_drv.h" #include "i915_drv.h"
#include "i915_gem_clflush.h" #include "i915_gem_clflush.h"
#include "i915_gem_context.h" #include "i915_gem_context.h"
#include "i915_gem_mman.h"
#include "i915_gem_object.h" #include "i915_gem_object.h"
#include "i915_globals.h" #include "i915_globals.h"
#include "i915_trace.h" #include "i915_trace.h"
...@@ -61,6 +62,9 @@ void i915_gem_object_init(struct drm_i915_gem_object *obj, ...@@ -61,6 +62,9 @@ void i915_gem_object_init(struct drm_i915_gem_object *obj,
INIT_LIST_HEAD(&obj->lut_list); INIT_LIST_HEAD(&obj->lut_list);
spin_lock_init(&obj->mmo.lock);
INIT_LIST_HEAD(&obj->mmo.offsets);
init_rcu_head(&obj->rcu); init_rcu_head(&obj->rcu);
obj->ops = ops; obj->ops = ops;
...@@ -97,6 +101,7 @@ void i915_gem_close_object(struct drm_gem_object *gem, struct drm_file *file) ...@@ -97,6 +101,7 @@ void i915_gem_close_object(struct drm_gem_object *gem, struct drm_file *file)
struct drm_i915_gem_object *obj = to_intel_bo(gem); struct drm_i915_gem_object *obj = to_intel_bo(gem);
struct drm_i915_file_private *fpriv = file->driver_priv; struct drm_i915_file_private *fpriv = file->driver_priv;
struct i915_lut_handle *lut, *ln; struct i915_lut_handle *lut, *ln;
struct i915_mmap_offset *mmo;
LIST_HEAD(close); LIST_HEAD(close);
i915_gem_object_lock(obj); i915_gem_object_lock(obj);
...@@ -111,6 +116,17 @@ void i915_gem_close_object(struct drm_gem_object *gem, struct drm_file *file) ...@@ -111,6 +116,17 @@ void i915_gem_close_object(struct drm_gem_object *gem, struct drm_file *file)
} }
i915_gem_object_unlock(obj); i915_gem_object_unlock(obj);
spin_lock(&obj->mmo.lock);
list_for_each_entry(mmo, &obj->mmo.offsets, offset) {
if (mmo->file != file)
continue;
spin_unlock(&obj->mmo.lock);
drm_vma_node_revoke(&mmo->vma_node, file);
spin_lock(&obj->mmo.lock);
}
spin_unlock(&obj->mmo.lock);
list_for_each_entry_safe(lut, ln, &close, obj_link) { list_for_each_entry_safe(lut, ln, &close, obj_link) {
struct i915_gem_context *ctx = lut->ctx; struct i915_gem_context *ctx = lut->ctx;
struct i915_vma *vma; struct i915_vma *vma;
...@@ -158,6 +174,8 @@ static void __i915_gem_free_objects(struct drm_i915_private *i915, ...@@ -158,6 +174,8 @@ static void __i915_gem_free_objects(struct drm_i915_private *i915,
wakeref = intel_runtime_pm_get(&i915->runtime_pm); wakeref = intel_runtime_pm_get(&i915->runtime_pm);
llist_for_each_entry_safe(obj, on, freed, freed) { llist_for_each_entry_safe(obj, on, freed, freed) {
struct i915_mmap_offset *mmo, *mn;
trace_i915_gem_object_destroy(obj); trace_i915_gem_object_destroy(obj);
if (!list_empty(&obj->vma.list)) { if (!list_empty(&obj->vma.list)) {
...@@ -183,6 +201,15 @@ static void __i915_gem_free_objects(struct drm_i915_private *i915, ...@@ -183,6 +201,15 @@ static void __i915_gem_free_objects(struct drm_i915_private *i915,
spin_unlock(&obj->vma.lock); spin_unlock(&obj->vma.lock);
} }
i915_gem_object_release_mmap(obj);
list_for_each_entry_safe(mmo, mn, &obj->mmo.offsets, offset) {
drm_vma_offset_remove(obj->base.dev->vma_offset_manager,
&mmo->vma_node);
kfree(mmo);
}
INIT_LIST_HEAD(&obj->mmo.offsets);
GEM_BUG_ON(atomic_read(&obj->bind_count)); GEM_BUG_ON(atomic_read(&obj->bind_count));
GEM_BUG_ON(obj->userfault_count); GEM_BUG_ON(obj->userfault_count);
GEM_BUG_ON(!list_empty(&obj->lut_list)); GEM_BUG_ON(!list_empty(&obj->lut_list));
......
...@@ -132,13 +132,13 @@ void i915_gem_object_unlock_fence(struct drm_i915_gem_object *obj, ...@@ -132,13 +132,13 @@ void i915_gem_object_unlock_fence(struct drm_i915_gem_object *obj,
static inline void static inline void
i915_gem_object_set_readonly(struct drm_i915_gem_object *obj) i915_gem_object_set_readonly(struct drm_i915_gem_object *obj)
{ {
obj->base.vma_node.readonly = true; obj->flags |= I915_BO_READONLY;
} }
static inline bool static inline bool
i915_gem_object_is_readonly(const struct drm_i915_gem_object *obj) i915_gem_object_is_readonly(const struct drm_i915_gem_object *obj)
{ {
return obj->base.vma_node.readonly; return obj->flags & I915_BO_READONLY;
} }
static inline bool static inline bool
...@@ -387,9 +387,6 @@ static inline void i915_gem_object_unpin_map(struct drm_i915_gem_object *obj) ...@@ -387,9 +387,6 @@ static inline void i915_gem_object_unpin_map(struct drm_i915_gem_object *obj)
i915_gem_object_unpin_pages(obj); i915_gem_object_unpin_pages(obj);
} }
void __i915_gem_object_release_mmap(struct drm_i915_gem_object *obj);
void i915_gem_object_release_mmap(struct drm_i915_gem_object *obj);
void void
i915_gem_object_flush_write_domain(struct drm_i915_gem_object *obj, i915_gem_object_flush_write_domain(struct drm_i915_gem_object *obj,
unsigned int flush_domains); unsigned int flush_domains);
......
...@@ -63,6 +63,23 @@ struct drm_i915_gem_object_ops { ...@@ -63,6 +63,23 @@ struct drm_i915_gem_object_ops {
void (*release)(struct drm_i915_gem_object *obj); void (*release)(struct drm_i915_gem_object *obj);
}; };
enum i915_mmap_type {
I915_MMAP_TYPE_GTT = 0,
I915_MMAP_TYPE_WC,
I915_MMAP_TYPE_WB,
I915_MMAP_TYPE_UC,
};
struct i915_mmap_offset {
struct drm_device *dev;
struct drm_vma_offset_node vma_node;
struct drm_i915_gem_object *obj;
struct drm_file *file;
enum i915_mmap_type mmap_type;
struct list_head offset;
};
struct drm_i915_gem_object { struct drm_i915_gem_object {
struct drm_gem_object base; struct drm_gem_object base;
...@@ -118,12 +135,18 @@ struct drm_i915_gem_object { ...@@ -118,12 +135,18 @@ struct drm_i915_gem_object {
unsigned int userfault_count; unsigned int userfault_count;
struct list_head userfault_link; struct list_head userfault_link;
struct {
spinlock_t lock; /* Protects access to mmo offsets */
struct list_head offsets;
} mmo;
I915_SELFTEST_DECLARE(struct list_head st_link); I915_SELFTEST_DECLARE(struct list_head st_link);
unsigned long flags; unsigned long flags;
#define I915_BO_ALLOC_CONTIGUOUS BIT(0) #define I915_BO_ALLOC_CONTIGUOUS BIT(0)
#define I915_BO_ALLOC_VOLATILE BIT(1) #define I915_BO_ALLOC_VOLATILE BIT(1)
#define I915_BO_ALLOC_FLAGS (I915_BO_ALLOC_CONTIGUOUS | I915_BO_ALLOC_VOLATILE) #define I915_BO_ALLOC_FLAGS (I915_BO_ALLOC_CONTIGUOUS | I915_BO_ALLOC_VOLATILE)
#define I915_BO_READONLY BIT(2)
/* /*
* Is the object to be mapped as read-only to the GPU * Is the object to be mapped as read-only to the GPU
......
...@@ -8,6 +8,7 @@ ...@@ -8,6 +8,7 @@
#include "i915_gem_object.h" #include "i915_gem_object.h"
#include "i915_scatterlist.h" #include "i915_scatterlist.h"
#include "i915_gem_lmem.h" #include "i915_gem_lmem.h"
#include "i915_gem_mman.h"
void __i915_gem_object_set_pages(struct drm_i915_gem_object *obj, void __i915_gem_object_set_pages(struct drm_i915_gem_object *obj,
struct sg_table *pages, struct sg_table *pages,
...@@ -207,6 +208,8 @@ int __i915_gem_object_put_pages(struct drm_i915_gem_object *obj) ...@@ -207,6 +208,8 @@ int __i915_gem_object_put_pages(struct drm_i915_gem_object *obj)
goto unlock; goto unlock;
} }
i915_gem_object_release_mmap_offset(obj);
/* /*
* ->put_pages might need to allocate memory for the bit17 swizzle * ->put_pages might need to allocate memory for the bit17 swizzle
* array, hence protect them from being reaped by removing them from gtt * array, hence protect them from being reaped by removing them from gtt
......
...@@ -11,6 +11,7 @@ ...@@ -11,6 +11,7 @@
#include "i915_drv.h" #include "i915_drv.h"
#include "i915_gem.h" #include "i915_gem.h"
#include "i915_gem_ioctls.h" #include "i915_gem_ioctls.h"
#include "i915_gem_mman.h"
#include "i915_gem_object.h" #include "i915_gem_object.h"
/** /**
......
...@@ -565,16 +565,16 @@ static bool assert_mmap_offset(struct drm_i915_private *i915, ...@@ -565,16 +565,16 @@ static bool assert_mmap_offset(struct drm_i915_private *i915,
int expected) int expected)
{ {
struct drm_i915_gem_object *obj; struct drm_i915_gem_object *obj;
int err; struct i915_mmap_offset *mmo;
obj = i915_gem_object_create_internal(i915, size); obj = i915_gem_object_create_internal(i915, size);
if (IS_ERR(obj)) if (IS_ERR(obj))
return PTR_ERR(obj); return PTR_ERR(obj);
err = create_mmap_offset(obj); mmo = mmap_offset_attach(obj, I915_MMAP_OFFSET_GTT, NULL);
i915_gem_object_put(obj); i915_gem_object_put(obj);
return err == expected; return PTR_ERR_OR_ZERO(mmo) == expected;
} }
static void disable_retire_worker(struct drm_i915_private *i915) static void disable_retire_worker(struct drm_i915_private *i915)
...@@ -609,7 +609,8 @@ static int igt_mmap_offset_exhaustion(void *arg) ...@@ -609,7 +609,8 @@ static int igt_mmap_offset_exhaustion(void *arg)
struct drm_mm *mm = &i915->drm.vma_offset_manager->vm_addr_space_mm; struct drm_mm *mm = &i915->drm.vma_offset_manager->vm_addr_space_mm;
struct drm_i915_gem_object *obj; struct drm_i915_gem_object *obj;
struct drm_mm_node *hole, *next; struct drm_mm_node *hole, *next;
int loop, err; struct i915_mmap_offset *mmo;
int loop, err = 0;
/* Disable background reaper */ /* Disable background reaper */
disable_retire_worker(i915); disable_retire_worker(i915);
...@@ -673,9 +674,10 @@ static int igt_mmap_offset_exhaustion(void *arg) ...@@ -673,9 +674,10 @@ static int igt_mmap_offset_exhaustion(void *arg)
goto out; goto out;
} }
err = create_mmap_offset(obj); mmo = mmap_offset_attach(obj, I915_MMAP_OFFSET_GTT, NULL);
if (err) { if (IS_ERR(mmo)) {
pr_err("Unable to insert object into reclaimed hole\n"); pr_err("Unable to insert object into reclaimed hole\n");
err = PTR_ERR(mmo);
goto err_obj; goto err_obj;
} }
...@@ -724,14 +726,15 @@ static int igt_mmap_offset_exhaustion(void *arg) ...@@ -724,14 +726,15 @@ static int igt_mmap_offset_exhaustion(void *arg)
} }
#define expand32(x) (((x) << 0) | ((x) << 8) | ((x) << 16) | ((x) << 24)) #define expand32(x) (((x) << 0) | ((x) << 8) | ((x) << 16) | ((x) << 24))
static int igt_mmap_gtt(void *arg) static int igt_mmap(void *arg, enum i915_mmap_type type)
{ {
struct drm_i915_private *i915 = arg; struct drm_i915_private *i915 = arg;
struct drm_i915_gem_object *obj; struct drm_i915_gem_object *obj;
struct i915_mmap_offset *mmo;
struct vm_area_struct *area; struct vm_area_struct *area;
unsigned long addr; unsigned long addr;
void *vaddr; void *vaddr;
int err, i; int err = 0, i;
if (!i915_ggtt_has_aperture(&i915->ggtt)) if (!i915_ggtt_has_aperture(&i915->ggtt))
return 0; return 0;
...@@ -749,18 +752,19 @@ static int igt_mmap_gtt(void *arg) ...@@ -749,18 +752,19 @@ static int igt_mmap_gtt(void *arg)
i915_gem_object_flush_map(obj); i915_gem_object_flush_map(obj);
i915_gem_object_unpin_map(obj); i915_gem_object_unpin_map(obj);
err = create_mmap_offset(obj); mmo = mmap_offset_attach(obj, type, NULL);
if (err) if (IS_ERR(mmo)) {
err = PTR_ERR(mmo);
goto out; goto out;
}
addr = igt_mmap_node(i915, &obj->base.vma_node, addr = igt_mmap_node(i915, &mmo->vma_node, 0, PROT_WRITE, MAP_SHARED);
0, PROT_WRITE, MAP_SHARED);
if (IS_ERR_VALUE(addr)) { if (IS_ERR_VALUE(addr)) {
err = addr; err = addr;
goto out; goto out;
} }
pr_debug("igt_mmap(obj:gtt) @ %lx\n", addr); pr_debug("igt_mmap() @ %lx\n", addr);
area = find_vma(current->mm, addr); area = find_vma(current->mm, addr);
if (!area) { if (!area) {
...@@ -769,8 +773,8 @@ static int igt_mmap_gtt(void *arg) ...@@ -769,8 +773,8 @@ static int igt_mmap_gtt(void *arg)
goto out_unmap; goto out_unmap;
} }
if (area->vm_private_data != obj) { if (area->vm_private_data != mmo) {
pr_err("vm_area_struct did not point back to our object!\n"); pr_err("vm_area_struct did not point back to our mmap_offset object!\n");
err = -EINVAL; err = -EINVAL;
goto out_unmap; goto out_unmap;
} }
...@@ -780,14 +784,14 @@ static int igt_mmap_gtt(void *arg) ...@@ -780,14 +784,14 @@ static int igt_mmap_gtt(void *arg)
u32 x; u32 x;
if (get_user(x, ux)) { if (get_user(x, ux)) {
pr_err("Unable to read from GTT mmap, offset:%zd\n", pr_err("Unable to read from mmap, offset:%zd\n",
i * sizeof(x)); i * sizeof(x));
err = -EFAULT; err = -EFAULT;
break; break;
} }
if (x != expand32(POISON_INUSE)) { if (x != expand32(POISON_INUSE)) {
pr_err("Read incorrect value from GTT mmap, offset:%zd, found:%x, expected:%x\n", pr_err("Read incorrect value from mmap, offset:%zd, found:%x, expected:%x\n",
i * sizeof(x), x, expand32(POISON_INUSE)); i * sizeof(x), x, expand32(POISON_INUSE));
err = -EINVAL; err = -EINVAL;
break; break;
...@@ -795,7 +799,7 @@ static int igt_mmap_gtt(void *arg) ...@@ -795,7 +799,7 @@ static int igt_mmap_gtt(void *arg)
x = expand32(POISON_FREE); x = expand32(POISON_FREE);
if (put_user(x, ux)) { if (put_user(x, ux)) {
pr_err("Unable to write to GTT mmap, offset:%zd\n", pr_err("Unable to write to mmap, offset:%zd\n",
i * sizeof(x)); i * sizeof(x));
err = -EFAULT; err = -EFAULT;
break; break;
...@@ -811,7 +815,7 @@ static int igt_mmap_gtt(void *arg) ...@@ -811,7 +815,7 @@ static int igt_mmap_gtt(void *arg)
goto out; goto out;
} }
if (err == 0 && memchr_inv(vaddr, POISON_FREE, PAGE_SIZE)) { if (err == 0 && memchr_inv(vaddr, POISON_FREE, PAGE_SIZE)) {
pr_err("Write via GGTT mmap did not land in backing store\n"); pr_err("Write via mmap did not land in backing store\n");
err = -EINVAL; err = -EINVAL;
} }
i915_gem_object_unpin_map(obj); i915_gem_object_unpin_map(obj);
...@@ -821,6 +825,16 @@ static int igt_mmap_gtt(void *arg) ...@@ -821,6 +825,16 @@ static int igt_mmap_gtt(void *arg)
return err; return err;
} }
static int igt_mmap_gtt(void *arg)
{
return igt_mmap(arg, I915_MMAP_TYPE_GTT);
}
static int igt_mmap_cpu(void *arg)
{
return igt_mmap(arg, I915_MMAP_TYPE_WC);
}
static int check_present_pte(pte_t *pte, unsigned long addr, void *data) static int check_present_pte(pte_t *pte, unsigned long addr, void *data)
{ {
if (!pte_present(*pte) || pte_none(*pte)) { if (!pte_present(*pte) || pte_none(*pte)) {
...@@ -873,10 +887,11 @@ static int prefault_range(u64 start, u64 len) ...@@ -873,10 +887,11 @@ static int prefault_range(u64 start, u64 len)
return __get_user(c, end - 1); return __get_user(c, end - 1);
} }
static int igt_mmap_gtt_revoke(void *arg) static int igt_mmap_revoke(void *arg, enum i915_mmap_type type)
{ {
struct drm_i915_private *i915 = arg; struct drm_i915_private *i915 = arg;
struct drm_i915_gem_object *obj; struct drm_i915_gem_object *obj;
struct i915_mmap_offset *mmo;
unsigned long addr; unsigned long addr;
int err; int err;
...@@ -887,12 +902,13 @@ static int igt_mmap_gtt_revoke(void *arg) ...@@ -887,12 +902,13 @@ static int igt_mmap_gtt_revoke(void *arg)
if (IS_ERR(obj)) if (IS_ERR(obj))
return PTR_ERR(obj); return PTR_ERR(obj);
err = create_mmap_offset(obj); mmo = mmap_offset_attach(obj, type, NULL);
if (err) if (IS_ERR(mmo)) {
err = PTR_ERR(mmo);
goto out; goto out;
}
addr = igt_mmap_node(i915, &obj->base.vma_node, addr = igt_mmap_node(i915, &mmo->vma_node, 0, PROT_WRITE, MAP_SHARED);
0, PROT_WRITE, MAP_SHARED);
if (IS_ERR_VALUE(addr)) { if (IS_ERR_VALUE(addr)) {
err = addr; err = addr;
goto out; goto out;
...@@ -902,7 +918,8 @@ static int igt_mmap_gtt_revoke(void *arg) ...@@ -902,7 +918,8 @@ static int igt_mmap_gtt_revoke(void *arg)
if (err) if (err)
goto out_unmap; goto out_unmap;
GEM_BUG_ON(!atomic_read(&obj->bind_count)); GEM_BUG_ON(mmo->mmap_type == I915_MMAP_TYPE_GTT &&
!atomic_read(&obj->bind_count));
err = check_present(addr, obj->base.size); err = check_present(addr, obj->base.size);
if (err) if (err)
...@@ -920,6 +937,15 @@ static int igt_mmap_gtt_revoke(void *arg) ...@@ -920,6 +937,15 @@ static int igt_mmap_gtt_revoke(void *arg)
} }
GEM_BUG_ON(atomic_read(&obj->bind_count)); GEM_BUG_ON(atomic_read(&obj->bind_count));
if (type != I915_MMAP_TYPE_GTT) {
__i915_gem_object_put_pages(obj);
if (i915_gem_object_has_pages(obj)) {
pr_err("Failed to put-pages object!\n");
err = -EINVAL;
goto out_unmap;
}
}
err = check_absent(addr, obj->base.size); err = check_absent(addr, obj->base.size);
if (err) if (err)
goto out_unmap; goto out_unmap;
...@@ -931,6 +957,16 @@ static int igt_mmap_gtt_revoke(void *arg) ...@@ -931,6 +957,16 @@ static int igt_mmap_gtt_revoke(void *arg)
return err; return err;
} }
static int igt_mmap_gtt_revoke(void *arg)
{
return igt_mmap_revoke(arg, I915_MMAP_TYPE_GTT);
}
static int igt_mmap_cpu_revoke(void *arg)
{
return igt_mmap_revoke(arg, I915_MMAP_TYPE_WC);
}
int i915_gem_mman_live_selftests(struct drm_i915_private *i915) int i915_gem_mman_live_selftests(struct drm_i915_private *i915)
{ {
static const struct i915_subtest tests[] = { static const struct i915_subtest tests[] = {
...@@ -938,7 +974,9 @@ int i915_gem_mman_live_selftests(struct drm_i915_private *i915) ...@@ -938,7 +974,9 @@ int i915_gem_mman_live_selftests(struct drm_i915_private *i915)
SUBTEST(igt_smoke_tiling), SUBTEST(igt_smoke_tiling),
SUBTEST(igt_mmap_offset_exhaustion), SUBTEST(igt_mmap_offset_exhaustion),
SUBTEST(igt_mmap_gtt), SUBTEST(igt_mmap_gtt),
SUBTEST(igt_mmap_cpu),
SUBTEST(igt_mmap_gtt_revoke), SUBTEST(igt_mmap_gtt_revoke),
SUBTEST(igt_mmap_cpu_revoke),
}; };
return i915_subtests(tests, i915); return i915_subtests(tests, i915);
......
...@@ -678,8 +678,13 @@ static void revoke_mmaps(struct intel_gt *gt) ...@@ -678,8 +678,13 @@ static void revoke_mmaps(struct intel_gt *gt)
continue; continue;
GEM_BUG_ON(vma->fence != &gt->ggtt->fence_regs[i]); GEM_BUG_ON(vma->fence != &gt->ggtt->fence_regs[i]);
node = &vma->obj->base.vma_node;
if (!vma->mmo)
continue;
node = &vma->mmo->vma_node;
vma_offset = vma->ggtt_view.partial.offset << PAGE_SHIFT; vma_offset = vma->ggtt_view.partial.offset << PAGE_SHIFT;
unmap_mapping_range(gt->i915->drm.anon_inode->i_mapping, unmap_mapping_range(gt->i915->drm.anon_inode->i_mapping,
drm_vma_node_offset_addr(node) + vma_offset, drm_vma_node_offset_addr(node) + vma_offset,
vma->size, vma->size,
......
...@@ -61,6 +61,7 @@ ...@@ -61,6 +61,7 @@
#include "gem/i915_gem_context.h" #include "gem/i915_gem_context.h"
#include "gem/i915_gem_ioctls.h" #include "gem/i915_gem_ioctls.h"
#include "gem/i915_gem_mman.h"
#include "gt/intel_gt.h" #include "gt/intel_gt.h"
#include "gt/intel_gt_pm.h" #include "gt/intel_gt_pm.h"
#include "gt/intel_rc6.h" #include "gt/intel_rc6.h"
...@@ -2660,18 +2661,12 @@ const struct dev_pm_ops i915_pm_ops = { ...@@ -2660,18 +2661,12 @@ const struct dev_pm_ops i915_pm_ops = {
.runtime_resume = intel_runtime_resume, .runtime_resume = intel_runtime_resume,
}; };
static const struct vm_operations_struct i915_gem_vm_ops = {
.fault = i915_gem_fault,
.open = drm_gem_vm_open,
.close = drm_gem_vm_close,
};
static const struct file_operations i915_driver_fops = { static const struct file_operations i915_driver_fops = {
.owner = THIS_MODULE, .owner = THIS_MODULE,
.open = drm_open, .open = drm_open,
.release = drm_release, .release = drm_release,
.unlocked_ioctl = drm_ioctl, .unlocked_ioctl = drm_ioctl,
.mmap = drm_gem_mmap, .mmap = i915_gem_mmap,
.poll = drm_poll, .poll = drm_poll,
.read = drm_read, .read = drm_read,
.compat_ioctl = i915_compat_ioctl, .compat_ioctl = i915_compat_ioctl,
...@@ -2718,7 +2713,7 @@ static const struct drm_ioctl_desc i915_ioctls[] = { ...@@ -2718,7 +2713,7 @@ static const struct drm_ioctl_desc i915_ioctls[] = {
DRM_IOCTL_DEF_DRV(I915_GEM_PREAD, i915_gem_pread_ioctl, DRM_RENDER_ALLOW), DRM_IOCTL_DEF_DRV(I915_GEM_PREAD, i915_gem_pread_ioctl, DRM_RENDER_ALLOW),
DRM_IOCTL_DEF_DRV(I915_GEM_PWRITE, i915_gem_pwrite_ioctl, DRM_RENDER_ALLOW), DRM_IOCTL_DEF_DRV(I915_GEM_PWRITE, i915_gem_pwrite_ioctl, DRM_RENDER_ALLOW),
DRM_IOCTL_DEF_DRV(I915_GEM_MMAP, i915_gem_mmap_ioctl, DRM_RENDER_ALLOW), DRM_IOCTL_DEF_DRV(I915_GEM_MMAP, i915_gem_mmap_ioctl, DRM_RENDER_ALLOW),
DRM_IOCTL_DEF_DRV(I915_GEM_MMAP_GTT, i915_gem_mmap_gtt_ioctl, DRM_RENDER_ALLOW), DRM_IOCTL_DEF_DRV(I915_GEM_MMAP_OFFSET, i915_gem_mmap_offset_ioctl, DRM_RENDER_ALLOW),
DRM_IOCTL_DEF_DRV(I915_GEM_SET_DOMAIN, i915_gem_set_domain_ioctl, DRM_RENDER_ALLOW), DRM_IOCTL_DEF_DRV(I915_GEM_SET_DOMAIN, i915_gem_set_domain_ioctl, DRM_RENDER_ALLOW),
DRM_IOCTL_DEF_DRV(I915_GEM_SW_FINISH, i915_gem_sw_finish_ioctl, DRM_RENDER_ALLOW), DRM_IOCTL_DEF_DRV(I915_GEM_SW_FINISH, i915_gem_sw_finish_ioctl, DRM_RENDER_ALLOW),
DRM_IOCTL_DEF_DRV(I915_GEM_SET_TILING, i915_gem_set_tiling_ioctl, DRM_RENDER_ALLOW), DRM_IOCTL_DEF_DRV(I915_GEM_SET_TILING, i915_gem_set_tiling_ioctl, DRM_RENDER_ALLOW),
...@@ -2760,7 +2755,6 @@ static struct drm_driver driver = { ...@@ -2760,7 +2755,6 @@ static struct drm_driver driver = {
.gem_close_object = i915_gem_close_object, .gem_close_object = i915_gem_close_object,
.gem_free_object_unlocked = i915_gem_free_object, .gem_free_object_unlocked = i915_gem_free_object,
.gem_vm_ops = &i915_gem_vm_ops,
.prime_handle_to_fd = drm_gem_prime_handle_to_fd, .prime_handle_to_fd = drm_gem_prime_handle_to_fd,
.prime_fd_to_handle = drm_gem_prime_fd_to_handle, .prime_fd_to_handle = drm_gem_prime_fd_to_handle,
...@@ -2771,7 +2765,8 @@ static struct drm_driver driver = { ...@@ -2771,7 +2765,8 @@ static struct drm_driver driver = {
.get_scanout_position = i915_get_crtc_scanoutpos, .get_scanout_position = i915_get_crtc_scanoutpos,
.dumb_create = i915_gem_dumb_create, .dumb_create = i915_gem_dumb_create,
.dumb_map_offset = i915_gem_mmap_gtt, .dumb_map_offset = i915_gem_dumb_mmap_offset,
.ioctls = i915_ioctls, .ioctls = i915_ioctls,
.num_ioctls = ARRAY_SIZE(i915_ioctls), .num_ioctls = ARRAY_SIZE(i915_ioctls),
.fops = &i915_driver_fops, .fops = &i915_driver_fops,
......
...@@ -1857,9 +1857,6 @@ i915_mutex_lock_interruptible(struct drm_device *dev) ...@@ -1857,9 +1857,6 @@ i915_mutex_lock_interruptible(struct drm_device *dev)
int i915_gem_dumb_create(struct drm_file *file_priv, int i915_gem_dumb_create(struct drm_file *file_priv,
struct drm_device *dev, struct drm_device *dev,
struct drm_mode_create_dumb *args); struct drm_mode_create_dumb *args);
int i915_gem_mmap_gtt(struct drm_file *file_priv, struct drm_device *dev,
u32 handle, u64 *offset);
int i915_gem_mmap_gtt_version(void);
int __must_check i915_gem_set_global_seqno(struct drm_device *dev, u32 seqno); int __must_check i915_gem_set_global_seqno(struct drm_device *dev, u32 seqno);
...@@ -1882,7 +1879,6 @@ void i915_gem_driver_release(struct drm_i915_private *dev_priv); ...@@ -1882,7 +1879,6 @@ void i915_gem_driver_release(struct drm_i915_private *dev_priv);
void i915_gem_suspend(struct drm_i915_private *dev_priv); void i915_gem_suspend(struct drm_i915_private *dev_priv);
void i915_gem_suspend_late(struct drm_i915_private *dev_priv); void i915_gem_suspend_late(struct drm_i915_private *dev_priv);
void i915_gem_resume(struct drm_i915_private *dev_priv); void i915_gem_resume(struct drm_i915_private *dev_priv);
vm_fault_t i915_gem_fault(struct vm_fault *vmf);
int i915_gem_open(struct drm_i915_private *i915, struct drm_file *file); int i915_gem_open(struct drm_i915_private *i915, struct drm_file *file);
void i915_gem_release(struct drm_device *dev, struct drm_file *file); void i915_gem_release(struct drm_device *dev, struct drm_file *file);
......
...@@ -44,6 +44,7 @@ ...@@ -44,6 +44,7 @@
#include "gem/i915_gem_clflush.h" #include "gem/i915_gem_clflush.h"
#include "gem/i915_gem_context.h" #include "gem/i915_gem_context.h"
#include "gem/i915_gem_ioctls.h" #include "gem/i915_gem_ioctls.h"
#include "gem/i915_gem_mman.h"
#include "gem/i915_gem_pm.h" #include "gem/i915_gem_pm.h"
#include "gt/intel_context.h" #include "gt/intel_context.h"
#include "gt/intel_engine_user.h" #include "gt/intel_engine_user.h"
...@@ -885,7 +886,7 @@ void i915_gem_runtime_suspend(struct drm_i915_private *i915) ...@@ -885,7 +886,7 @@ void i915_gem_runtime_suspend(struct drm_i915_private *i915)
list_for_each_entry_safe(obj, on, list_for_each_entry_safe(obj, on,
&i915->ggtt.userfault_list, userfault_link) &i915->ggtt.userfault_list, userfault_link)
__i915_gem_object_release_mmap(obj); __i915_gem_object_release_mmap_gtt(obj);
/* /*
* The fence will be lost when the device powers down. If any were * The fence will be lost when the device powers down. If any were
......
...@@ -2,6 +2,7 @@ ...@@ -2,6 +2,7 @@
* SPDX-License-Identifier: MIT * SPDX-License-Identifier: MIT
*/ */
#include "gem/i915_gem_mman.h"
#include "gt/intel_engine_user.h" #include "gt/intel_engine_user.h"
#include "i915_drv.h" #include "i915_drv.h"
......
...@@ -1054,17 +1054,16 @@ static void __i915_vma_iounmap(struct i915_vma *vma) ...@@ -1054,17 +1054,16 @@ static void __i915_vma_iounmap(struct i915_vma *vma)
void i915_vma_revoke_mmap(struct i915_vma *vma) void i915_vma_revoke_mmap(struct i915_vma *vma)
{ {
struct drm_vma_offset_node *node = &vma->obj->base.vma_node; struct drm_vma_offset_node *node;
u64 vma_offset; u64 vma_offset;
lockdep_assert_held(&vma->vm->mutex);
if (!i915_vma_has_userfault(vma)) if (!i915_vma_has_userfault(vma))
return; return;
GEM_BUG_ON(!i915_vma_is_map_and_fenceable(vma)); GEM_BUG_ON(!i915_vma_is_map_and_fenceable(vma));
GEM_BUG_ON(!vma->obj->userfault_count); GEM_BUG_ON(!vma->obj->userfault_count);
node = &vma->mmo->vma_node;
vma_offset = vma->ggtt_view.partial.offset << PAGE_SHIFT; vma_offset = vma->ggtt_view.partial.offset << PAGE_SHIFT;
unmap_mapping_range(vma->vm->i915->drm.anon_inode->i_mapping, unmap_mapping_range(vma->vm->i915->drm.anon_inode->i_mapping,
drm_vma_node_offset_addr(node) + vma_offset, drm_vma_node_offset_addr(node) + vma_offset,
......
...@@ -63,6 +63,9 @@ struct i915_vma { ...@@ -63,6 +63,9 @@ struct i915_vma {
u64 display_alignment; u64 display_alignment;
struct i915_page_sizes page_sizes; struct i915_page_sizes page_sizes;
/* mmap-offset associated with fencing for this vma */
struct i915_mmap_offset *mmo;
u32 fence_size; u32 fence_size;
u32 fence_alignment; u32 fence_alignment;
......
...@@ -395,6 +395,7 @@ typedef struct _drm_i915_sarea { ...@@ -395,6 +395,7 @@ typedef struct _drm_i915_sarea {
#define DRM_IOCTL_I915_GEM_PWRITE DRM_IOW (DRM_COMMAND_BASE + DRM_I915_GEM_PWRITE, struct drm_i915_gem_pwrite) #define DRM_IOCTL_I915_GEM_PWRITE DRM_IOW (DRM_COMMAND_BASE + DRM_I915_GEM_PWRITE, struct drm_i915_gem_pwrite)
#define DRM_IOCTL_I915_GEM_MMAP DRM_IOWR(DRM_COMMAND_BASE + DRM_I915_GEM_MMAP, struct drm_i915_gem_mmap) #define DRM_IOCTL_I915_GEM_MMAP DRM_IOWR(DRM_COMMAND_BASE + DRM_I915_GEM_MMAP, struct drm_i915_gem_mmap)
#define DRM_IOCTL_I915_GEM_MMAP_GTT DRM_IOWR(DRM_COMMAND_BASE + DRM_I915_GEM_MMAP_GTT, struct drm_i915_gem_mmap_gtt) #define DRM_IOCTL_I915_GEM_MMAP_GTT DRM_IOWR(DRM_COMMAND_BASE + DRM_I915_GEM_MMAP_GTT, struct drm_i915_gem_mmap_gtt)
#define DRM_IOCTL_I915_GEM_MMAP_OFFSET DRM_IOWR(DRM_COMMAND_BASE + DRM_I915_GEM_MMAP_GTT, struct drm_i915_gem_mmap_offset)
#define DRM_IOCTL_I915_GEM_SET_DOMAIN DRM_IOW (DRM_COMMAND_BASE + DRM_I915_GEM_SET_DOMAIN, struct drm_i915_gem_set_domain) #define DRM_IOCTL_I915_GEM_SET_DOMAIN DRM_IOW (DRM_COMMAND_BASE + DRM_I915_GEM_SET_DOMAIN, struct drm_i915_gem_set_domain)
#define DRM_IOCTL_I915_GEM_SW_FINISH DRM_IOW (DRM_COMMAND_BASE + DRM_I915_GEM_SW_FINISH, struct drm_i915_gem_sw_finish) #define DRM_IOCTL_I915_GEM_SW_FINISH DRM_IOW (DRM_COMMAND_BASE + DRM_I915_GEM_SW_FINISH, struct drm_i915_gem_sw_finish)
#define DRM_IOCTL_I915_GEM_SET_TILING DRM_IOWR (DRM_COMMAND_BASE + DRM_I915_GEM_SET_TILING, struct drm_i915_gem_set_tiling) #define DRM_IOCTL_I915_GEM_SET_TILING DRM_IOWR (DRM_COMMAND_BASE + DRM_I915_GEM_SET_TILING, struct drm_i915_gem_set_tiling)
...@@ -793,6 +794,37 @@ struct drm_i915_gem_mmap_gtt { ...@@ -793,6 +794,37 @@ struct drm_i915_gem_mmap_gtt {
__u64 offset; __u64 offset;
}; };
struct drm_i915_gem_mmap_offset {
/** Handle for the object being mapped. */
__u32 handle;
__u32 pad;
/**
* Fake offset to use for subsequent mmap call
*
* This is a fixed-size type for 32/64 compatibility.
*/
__u64 offset;
/**
* Flags for extended behaviour.
*
* It is mandatory that one of the MMAP_OFFSET types
* (GTT, WC, WB, UC, etc) should be included.
*/
__u64 flags;
#define I915_MMAP_OFFSET_GTT 0
#define I915_MMAP_OFFSET_WC 1
#define I915_MMAP_OFFSET_WB 2
#define I915_MMAP_OFFSET_UC 3
/*
* Zero-terminated chain of extensions.
*
* No current extensions defined; mbz.
*/
__u64 extensions;
};
struct drm_i915_gem_set_domain { struct drm_i915_gem_set_domain {
/** Handle for the object */ /** Handle for the object */
__u32 handle; __u32 handle;
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment