Commit 031e610a authored by Dave Airlie's avatar Dave Airlie

Merge branch 'vmwgfx-next' of git://people.freedesktop.org/~thomash/linux into drm-next

- The coherent memory changes including mm changes.
- Some vmwgfx debug fixes.
- Removal of vmwgfx legacy security checks.
Signed-off-by: default avatarDave Airlie <airlied@redhat.com>
From: Thomas Hellstrom <VMware> <thomas@shipmail.org>
Link: https://patchwork.freedesktop.org/patch/msgid/20190619072531.4026-1-thomas@shipmail.org
parents 52d2d44e 9bbfda54
...@@ -5191,6 +5191,7 @@ T: git git://people.freedesktop.org/~thomash/linux ...@@ -5191,6 +5191,7 @@ T: git git://people.freedesktop.org/~thomash/linux
S: Supported S: Supported
F: drivers/gpu/drm/vmwgfx/ F: drivers/gpu/drm/vmwgfx/
F: include/uapi/drm/vmwgfx_drm.h F: include/uapi/drm/vmwgfx_drm.h
F: mm/as_dirty_helpers.c
DRM DRIVERS DRM DRIVERS
M: David Airlie <airlied@linux.ie> M: David Airlie <airlied@linux.ie>
......
...@@ -1739,6 +1739,7 @@ int ttm_bo_device_init(struct ttm_bo_device *bdev, ...@@ -1739,6 +1739,7 @@ int ttm_bo_device_init(struct ttm_bo_device *bdev,
mutex_lock(&ttm_global_mutex); mutex_lock(&ttm_global_mutex);
list_add_tail(&bdev->device_list, &glob->device_list); list_add_tail(&bdev->device_list, &glob->device_list);
mutex_unlock(&ttm_global_mutex); mutex_unlock(&ttm_global_mutex);
bdev->vm_ops = &ttm_bo_vm_ops;
return 0; return 0;
out_no_sys: out_no_sys:
......
...@@ -42,8 +42,6 @@ ...@@ -42,8 +42,6 @@
#include <linux/uaccess.h> #include <linux/uaccess.h>
#include <linux/mem_encrypt.h> #include <linux/mem_encrypt.h>
#define TTM_BO_VM_NUM_PREFAULT 16
static vm_fault_t ttm_bo_vm_fault_idle(struct ttm_buffer_object *bo, static vm_fault_t ttm_bo_vm_fault_idle(struct ttm_buffer_object *bo,
struct vm_fault *vmf) struct vm_fault *vmf)
{ {
...@@ -106,25 +104,30 @@ static unsigned long ttm_bo_io_mem_pfn(struct ttm_buffer_object *bo, ...@@ -106,25 +104,30 @@ static unsigned long ttm_bo_io_mem_pfn(struct ttm_buffer_object *bo,
+ page_offset; + page_offset;
} }
static vm_fault_t ttm_bo_vm_fault(struct vm_fault *vmf) /**
* ttm_bo_vm_reserve - Reserve a buffer object in a retryable vm callback
* @bo: The buffer object
* @vmf: The fault structure handed to the callback
*
* vm callbacks like fault() and *_mkwrite() allow for the mm_sem to be dropped
* during long waits, and after the wait the callback will be restarted. This
* is to allow other threads using the same virtual memory space concurrent
* access to map(), unmap() completely unrelated buffer objects. TTM buffer
* object reservations sometimes wait for GPU and should therefore be
* considered long waits. This function reserves the buffer object interruptibly
* taking this into account. Starvation is avoided by the vm system not
* allowing too many repeated restarts.
* This function is intended to be used in customized fault() and _mkwrite()
* handlers.
*
* Return:
* 0 on success and the bo was reserved.
* VM_FAULT_RETRY if blocking wait.
* VM_FAULT_NOPAGE if blocking wait and retrying was not allowed.
*/
vm_fault_t ttm_bo_vm_reserve(struct ttm_buffer_object *bo,
struct vm_fault *vmf)
{ {
struct vm_area_struct *vma = vmf->vma;
struct ttm_buffer_object *bo = (struct ttm_buffer_object *)
vma->vm_private_data;
struct ttm_bo_device *bdev = bo->bdev;
unsigned long page_offset;
unsigned long page_last;
unsigned long pfn;
struct ttm_tt *ttm = NULL;
struct page *page;
int err;
int i;
vm_fault_t ret = VM_FAULT_NOPAGE;
unsigned long address = vmf->address;
struct ttm_mem_type_manager *man =
&bdev->man[bo->mem.mem_type];
struct vm_area_struct cvma;
/* /*
* Work around locking order reversal in fault / nopfn * Work around locking order reversal in fault / nopfn
* between mmap_sem and bo_reserve: Perform a trylock operation * between mmap_sem and bo_reserve: Perform a trylock operation
...@@ -151,14 +154,55 @@ static vm_fault_t ttm_bo_vm_fault(struct vm_fault *vmf) ...@@ -151,14 +154,55 @@ static vm_fault_t ttm_bo_vm_fault(struct vm_fault *vmf)
return VM_FAULT_NOPAGE; return VM_FAULT_NOPAGE;
} }
return 0;
}
EXPORT_SYMBOL(ttm_bo_vm_reserve);
/**
* ttm_bo_vm_fault_reserved - TTM fault helper
* @vmf: The struct vm_fault given as argument to the fault callback
* @prot: The page protection to be used for this memory area.
* @num_prefault: Maximum number of prefault pages. The caller may want to
* specify this based on madvice settings and the size of the GPU object
* backed by the memory.
*
* This function inserts one or more page table entries pointing to the
* memory backing the buffer object, and then returns a return code
* instructing the caller to retry the page access.
*
* Return:
* VM_FAULT_NOPAGE on success or pending signal
* VM_FAULT_SIGBUS on unspecified error
* VM_FAULT_OOM on out-of-memory
* VM_FAULT_RETRY if retryable wait
*/
vm_fault_t ttm_bo_vm_fault_reserved(struct vm_fault *vmf,
pgprot_t prot,
pgoff_t num_prefault)
{
struct vm_area_struct *vma = vmf->vma;
struct vm_area_struct cvma = *vma;
struct ttm_buffer_object *bo = (struct ttm_buffer_object *)
vma->vm_private_data;
struct ttm_bo_device *bdev = bo->bdev;
unsigned long page_offset;
unsigned long page_last;
unsigned long pfn;
struct ttm_tt *ttm = NULL;
struct page *page;
int err;
pgoff_t i;
vm_fault_t ret = VM_FAULT_NOPAGE;
unsigned long address = vmf->address;
struct ttm_mem_type_manager *man =
&bdev->man[bo->mem.mem_type];
/* /*
* Refuse to fault imported pages. This should be handled * Refuse to fault imported pages. This should be handled
* (if at all) by redirecting mmap to the exporter. * (if at all) by redirecting mmap to the exporter.
*/ */
if (bo->ttm && (bo->ttm->page_flags & TTM_PAGE_FLAG_SG)) { if (bo->ttm && (bo->ttm->page_flags & TTM_PAGE_FLAG_SG))
ret = VM_FAULT_SIGBUS; return VM_FAULT_SIGBUS;
goto out_unlock;
}
if (bdev->driver->fault_reserve_notify) { if (bdev->driver->fault_reserve_notify) {
struct dma_fence *moving = dma_fence_get(bo->moving); struct dma_fence *moving = dma_fence_get(bo->moving);
...@@ -169,11 +213,9 @@ static vm_fault_t ttm_bo_vm_fault(struct vm_fault *vmf) ...@@ -169,11 +213,9 @@ static vm_fault_t ttm_bo_vm_fault(struct vm_fault *vmf)
break; break;
case -EBUSY: case -EBUSY:
case -ERESTARTSYS: case -ERESTARTSYS:
ret = VM_FAULT_NOPAGE; return VM_FAULT_NOPAGE;
goto out_unlock;
default: default:
ret = VM_FAULT_SIGBUS; return VM_FAULT_SIGBUS;
goto out_unlock;
} }
if (bo->moving != moving) { if (bo->moving != moving) {
...@@ -189,21 +231,12 @@ static vm_fault_t ttm_bo_vm_fault(struct vm_fault *vmf) ...@@ -189,21 +231,12 @@ static vm_fault_t ttm_bo_vm_fault(struct vm_fault *vmf)
* move. * move.
*/ */
ret = ttm_bo_vm_fault_idle(bo, vmf); ret = ttm_bo_vm_fault_idle(bo, vmf);
if (unlikely(ret != 0)) { if (unlikely(ret != 0))
if (ret == VM_FAULT_RETRY && return ret;
!(vmf->flags & FAULT_FLAG_RETRY_NOWAIT)) {
/* The BO has already been unreserved. */
return ret;
}
goto out_unlock;
}
err = ttm_mem_io_lock(man, true); err = ttm_mem_io_lock(man, true);
if (unlikely(err != 0)) { if (unlikely(err != 0))
ret = VM_FAULT_NOPAGE; return VM_FAULT_NOPAGE;
goto out_unlock;
}
err = ttm_mem_io_reserve_vm(bo); err = ttm_mem_io_reserve_vm(bo);
if (unlikely(err != 0)) { if (unlikely(err != 0)) {
ret = VM_FAULT_SIGBUS; ret = VM_FAULT_SIGBUS;
...@@ -220,18 +253,8 @@ static vm_fault_t ttm_bo_vm_fault(struct vm_fault *vmf) ...@@ -220,18 +253,8 @@ static vm_fault_t ttm_bo_vm_fault(struct vm_fault *vmf)
goto out_io_unlock; goto out_io_unlock;
} }
/* cvma.vm_page_prot = ttm_io_prot(bo->mem.placement, prot);
* Make a local vma copy to modify the page_prot member if (!bo->mem.bus.is_iomem) {
* and vm_flags if necessary. The vma parameter is protected
* by mmap_sem in write mode.
*/
cvma = *vma;
cvma.vm_page_prot = vm_get_page_prot(cvma.vm_flags);
if (bo->mem.bus.is_iomem) {
cvma.vm_page_prot = ttm_io_prot(bo->mem.placement,
cvma.vm_page_prot);
} else {
struct ttm_operation_ctx ctx = { struct ttm_operation_ctx ctx = {
.interruptible = false, .interruptible = false,
.no_wait_gpu = false, .no_wait_gpu = false,
...@@ -240,24 +263,21 @@ static vm_fault_t ttm_bo_vm_fault(struct vm_fault *vmf) ...@@ -240,24 +263,21 @@ static vm_fault_t ttm_bo_vm_fault(struct vm_fault *vmf)
}; };
ttm = bo->ttm; ttm = bo->ttm;
cvma.vm_page_prot = ttm_io_prot(bo->mem.placement, if (ttm_tt_populate(bo->ttm, &ctx)) {
cvma.vm_page_prot);
/* Allocate all page at once, most common usage */
if (ttm_tt_populate(ttm, &ctx)) {
ret = VM_FAULT_OOM; ret = VM_FAULT_OOM;
goto out_io_unlock; goto out_io_unlock;
} }
} else {
/* Iomem should not be marked encrypted */
cvma.vm_page_prot = pgprot_decrypted(cvma.vm_page_prot);
} }
/* /*
* Speculatively prefault a number of pages. Only error on * Speculatively prefault a number of pages. Only error on
* first page. * first page.
*/ */
for (i = 0; i < TTM_BO_VM_NUM_PREFAULT; ++i) { for (i = 0; i < num_prefault; ++i) {
if (bo->mem.bus.is_iomem) { if (bo->mem.bus.is_iomem) {
/* Iomem should not be marked encrypted */
cvma.vm_page_prot = pgprot_decrypted(cvma.vm_page_prot);
pfn = ttm_bo_io_mem_pfn(bo, page_offset); pfn = ttm_bo_io_mem_pfn(bo, page_offset);
} else { } else {
page = ttm->pages[page_offset]; page = ttm->pages[page_offset];
...@@ -295,7 +315,26 @@ static vm_fault_t ttm_bo_vm_fault(struct vm_fault *vmf) ...@@ -295,7 +315,26 @@ static vm_fault_t ttm_bo_vm_fault(struct vm_fault *vmf)
ret = VM_FAULT_NOPAGE; ret = VM_FAULT_NOPAGE;
out_io_unlock: out_io_unlock:
ttm_mem_io_unlock(man); ttm_mem_io_unlock(man);
out_unlock: return ret;
}
EXPORT_SYMBOL(ttm_bo_vm_fault_reserved);
static vm_fault_t ttm_bo_vm_fault(struct vm_fault *vmf)
{
struct vm_area_struct *vma = vmf->vma;
pgprot_t prot;
struct ttm_buffer_object *bo = vma->vm_private_data;
vm_fault_t ret;
ret = ttm_bo_vm_reserve(bo, vmf);
if (ret)
return ret;
prot = vm_get_page_prot(vma->vm_flags);
ret = ttm_bo_vm_fault_reserved(vmf, prot, TTM_BO_VM_NUM_PREFAULT);
if (ret == VM_FAULT_RETRY && !(vmf->flags & FAULT_FLAG_RETRY_NOWAIT))
return ret;
reservation_object_unlock(bo->resv); reservation_object_unlock(bo->resv);
return ret; return ret;
} }
...@@ -395,7 +434,7 @@ static int ttm_bo_vm_access(struct vm_area_struct *vma, unsigned long addr, ...@@ -395,7 +434,7 @@ static int ttm_bo_vm_access(struct vm_area_struct *vma, unsigned long addr,
return ret; return ret;
} }
static const struct vm_operations_struct ttm_bo_vm_ops = { const struct vm_operations_struct ttm_bo_vm_ops = {
.fault = ttm_bo_vm_fault, .fault = ttm_bo_vm_fault,
.open = ttm_bo_vm_open, .open = ttm_bo_vm_open,
.close = ttm_bo_vm_close, .close = ttm_bo_vm_close,
...@@ -448,7 +487,7 @@ int ttm_bo_mmap(struct file *filp, struct vm_area_struct *vma, ...@@ -448,7 +487,7 @@ int ttm_bo_mmap(struct file *filp, struct vm_area_struct *vma,
if (unlikely(ret != 0)) if (unlikely(ret != 0))
goto out_unref; goto out_unref;
vma->vm_ops = &ttm_bo_vm_ops; vma->vm_ops = bdev->vm_ops;
/* /*
* Note: We're transferring the bo reference to * Note: We're transferring the bo reference to
...@@ -480,7 +519,7 @@ int ttm_fbdev_mmap(struct vm_area_struct *vma, struct ttm_buffer_object *bo) ...@@ -480,7 +519,7 @@ int ttm_fbdev_mmap(struct vm_area_struct *vma, struct ttm_buffer_object *bo)
ttm_bo_get(bo); ttm_bo_get(bo);
vma->vm_ops = &ttm_bo_vm_ops; vma->vm_ops = bo->bdev->vm_ops;
vma->vm_private_data = bo; vma->vm_private_data = bo;
vma->vm_flags |= VM_MIXEDMAP; vma->vm_flags |= VM_MIXEDMAP;
vma->vm_flags |= VM_IO | VM_DONTEXPAND; vma->vm_flags |= VM_IO | VM_DONTEXPAND;
......
...@@ -8,6 +8,7 @@ config DRM_VMWGFX ...@@ -8,6 +8,7 @@ config DRM_VMWGFX
select FB_CFB_IMAGEBLIT select FB_CFB_IMAGEBLIT
select DRM_TTM select DRM_TTM
select FB select FB
select AS_DIRTY_HELPERS
# Only needed for the transitional use of drm_crtc_init - can be removed # Only needed for the transitional use of drm_crtc_init - can be removed
# again once vmwgfx sets up the primary plane itself. # again once vmwgfx sets up the primary plane itself.
select DRM_KMS_HELPER select DRM_KMS_HELPER
......
...@@ -8,7 +8,7 @@ vmwgfx-y := vmwgfx_execbuf.o vmwgfx_gmr.o vmwgfx_kms.o vmwgfx_drv.o \ ...@@ -8,7 +8,7 @@ vmwgfx-y := vmwgfx_execbuf.o vmwgfx_gmr.o vmwgfx_kms.o vmwgfx_drv.o \
vmwgfx_cmdbuf_res.o vmwgfx_cmdbuf.o vmwgfx_stdu.o \ vmwgfx_cmdbuf_res.o vmwgfx_cmdbuf.o vmwgfx_stdu.o \
vmwgfx_cotable.o vmwgfx_so.o vmwgfx_binding.o vmwgfx_msg.o \ vmwgfx_cotable.o vmwgfx_so.o vmwgfx_binding.o vmwgfx_msg.o \
vmwgfx_simple_resource.o vmwgfx_va.o vmwgfx_blit.o \ vmwgfx_simple_resource.o vmwgfx_va.o vmwgfx_blit.o \
vmwgfx_validation.o \ vmwgfx_validation.o vmwgfx_page_dirty.o \
ttm_object.o ttm_lock.o ttm_object.o ttm_lock.o
obj-$(CONFIG_DRM_VMWGFX) := vmwgfx.o obj-$(CONFIG_DRM_VMWGFX) := vmwgfx.o
...@@ -1280,7 +1280,6 @@ svga3dsurface_get_pixel_offset(SVGA3dSurfaceFormat format, ...@@ -1280,7 +1280,6 @@ svga3dsurface_get_pixel_offset(SVGA3dSurfaceFormat format,
return offset; return offset;
} }
static inline u32 static inline u32
svga3dsurface_get_image_offset(SVGA3dSurfaceFormat format, svga3dsurface_get_image_offset(SVGA3dSurfaceFormat format,
surf_size_struct baseLevelSize, surf_size_struct baseLevelSize,
...@@ -1375,4 +1374,236 @@ svga3dsurface_is_screen_target_format(SVGA3dSurfaceFormat format) ...@@ -1375,4 +1374,236 @@ svga3dsurface_is_screen_target_format(SVGA3dSurfaceFormat format)
return svga3dsurface_is_dx_screen_target_format(format); return svga3dsurface_is_dx_screen_target_format(format);
} }
/**
* struct svga3dsurface_mip - Mimpmap level information
* @bytes: Bytes required in the backing store of this mipmap level.
* @img_stride: Byte stride per image.
* @row_stride: Byte stride per block row.
* @size: The size of the mipmap.
*/
struct svga3dsurface_mip {
size_t bytes;
size_t img_stride;
size_t row_stride;
struct drm_vmw_size size;
};
/**
* struct svga3dsurface_cache - Cached surface information
* @desc: Pointer to the surface descriptor
* @mip: Array of mipmap level information. Valid size is @num_mip_levels.
* @mip_chain_bytes: Bytes required in the backing store for the whole chain
* of mip levels.
* @sheet_bytes: Bytes required in the backing store for a sheet
* representing a single sample.
* @num_mip_levels: Valid size of the @mip array. Number of mipmap levels in
* a chain.
* @num_layers: Number of slices in an array texture or number of faces in
* a cubemap texture.
*/
struct svga3dsurface_cache {
const struct svga3d_surface_desc *desc;
struct svga3dsurface_mip mip[DRM_VMW_MAX_MIP_LEVELS];
size_t mip_chain_bytes;
size_t sheet_bytes;
u32 num_mip_levels;
u32 num_layers;
};
/**
* struct svga3dsurface_loc - Surface location
* @sub_resource: Surface subresource. Defined as layer * num_mip_levels +
* mip_level.
* @x: X coordinate.
* @y: Y coordinate.
* @z: Z coordinate.
*/
struct svga3dsurface_loc {
u32 sub_resource;
u32 x, y, z;
};
/**
* svga3dsurface_subres - Compute the subresource from layer and mipmap.
* @cache: Surface layout data.
* @mip_level: The mipmap level.
* @layer: The surface layer (face or array slice).
*
* Return: The subresource.
*/
static inline u32 svga3dsurface_subres(const struct svga3dsurface_cache *cache,
u32 mip_level, u32 layer)
{
return cache->num_mip_levels * layer + mip_level;
}
/**
* svga3dsurface_setup_cache - Build a surface cache entry
* @size: The surface base level dimensions.
* @format: The surface format.
* @num_mip_levels: Number of mipmap levels.
* @num_layers: Number of layers.
* @cache: Pointer to a struct svga3dsurface_cach object to be filled in.
*
* Return: Zero on success, -EINVAL on invalid surface layout.
*/
static inline int svga3dsurface_setup_cache(const struct drm_vmw_size *size,
SVGA3dSurfaceFormat format,
u32 num_mip_levels,
u32 num_layers,
u32 num_samples,
struct svga3dsurface_cache *cache)
{
const struct svga3d_surface_desc *desc;
u32 i;
memset(cache, 0, sizeof(*cache));
cache->desc = desc = svga3dsurface_get_desc(format);
cache->num_mip_levels = num_mip_levels;
cache->num_layers = num_layers;
for (i = 0; i < cache->num_mip_levels; i++) {
struct svga3dsurface_mip *mip = &cache->mip[i];
mip->size = svga3dsurface_get_mip_size(*size, i);
mip->bytes = svga3dsurface_get_image_buffer_size
(desc, &mip->size, 0);
mip->row_stride =
__KERNEL_DIV_ROUND_UP(mip->size.width,
desc->block_size.width) *
desc->bytes_per_block * num_samples;
if (!mip->row_stride)
goto invalid_dim;
mip->img_stride =
__KERNEL_DIV_ROUND_UP(mip->size.height,
desc->block_size.height) *
mip->row_stride;
if (!mip->img_stride)
goto invalid_dim;
cache->mip_chain_bytes += mip->bytes;
}
cache->sheet_bytes = cache->mip_chain_bytes * num_layers;
if (!cache->sheet_bytes)
goto invalid_dim;
return 0;
invalid_dim:
VMW_DEBUG_USER("Invalid surface layout for dirty tracking.\n");
return -EINVAL;
}
/**
* svga3dsurface_get_loc - Get a surface location from an offset into the
* backing store
* @cache: Surface layout data.
* @loc: Pointer to a struct svga3dsurface_loc to be filled in.
* @offset: Offset into the surface backing store.
*/
static inline void
svga3dsurface_get_loc(const struct svga3dsurface_cache *cache,
struct svga3dsurface_loc *loc,
size_t offset)
{
const struct svga3dsurface_mip *mip = &cache->mip[0];
const struct svga3d_surface_desc *desc = cache->desc;
u32 layer;
int i;
if (offset >= cache->sheet_bytes)
offset %= cache->sheet_bytes;
layer = offset / cache->mip_chain_bytes;
offset -= layer * cache->mip_chain_bytes;
for (i = 0; i < cache->num_mip_levels; ++i, ++mip) {
if (mip->bytes > offset)
break;
offset -= mip->bytes;
}
loc->sub_resource = svga3dsurface_subres(cache, i, layer);
loc->z = offset / mip->img_stride;
offset -= loc->z * mip->img_stride;
loc->z *= desc->block_size.depth;
loc->y = offset / mip->row_stride;
offset -= loc->y * mip->row_stride;
loc->y *= desc->block_size.height;
loc->x = offset / desc->bytes_per_block;
loc->x *= desc->block_size.width;
}
/**
* svga3dsurface_inc_loc - Clamp increment a surface location with one block
* size
* in each dimension.
* @loc: Pointer to a struct svga3dsurface_loc to be incremented.
*
* When computing the size of a range as size = end - start, the range does not
* include the end element. However a location representing the last byte
* of a touched region in the backing store *is* included in the range.
* This function modifies such a location to match the end definition
* given as start + size which is the one used in a SVGA3dBox.
*/
static inline void
svga3dsurface_inc_loc(const struct svga3dsurface_cache *cache,
struct svga3dsurface_loc *loc)
{
const struct svga3d_surface_desc *desc = cache->desc;
u32 mip = loc->sub_resource % cache->num_mip_levels;
const struct drm_vmw_size *size = &cache->mip[mip].size;
loc->sub_resource++;
loc->x += desc->block_size.width;
if (loc->x > size->width)
loc->x = size->width;
loc->y += desc->block_size.height;
if (loc->y > size->height)
loc->y = size->height;
loc->z += desc->block_size.depth;
if (loc->z > size->depth)
loc->z = size->depth;
}
/**
* svga3dsurface_min_loc - The start location in a subresource
* @cache: Surface layout data.
* @sub_resource: The subresource.
* @loc: Pointer to a struct svga3dsurface_loc to be filled in.
*/
static inline void
svga3dsurface_min_loc(const struct svga3dsurface_cache *cache,
u32 sub_resource,
struct svga3dsurface_loc *loc)
{
loc->sub_resource = sub_resource;
loc->x = loc->y = loc->z = 0;
}
/**
* svga3dsurface_min_loc - The end location in a subresource
* @cache: Surface layout data.
* @sub_resource: The subresource.
* @loc: Pointer to a struct svga3dsurface_loc to be filled in.
*
* Following the end definition given in svga3dsurface_inc_loc(),
* Compute the end location of a surface subresource.
*/
static inline void
svga3dsurface_max_loc(const struct svga3dsurface_cache *cache,
u32 sub_resource,
struct svga3dsurface_loc *loc)
{
const struct drm_vmw_size *size;
u32 mip;
loc->sub_resource = sub_resource + 1;
mip = sub_resource % cache->num_mip_levels;
size = &cache->mip[mip].size;
loc->x = size->width;
loc->y = size->height;
loc->z = size->depth;
}
#endif /* _SVGA3D_SURFACEDEFS_H_ */ #endif /* _SVGA3D_SURFACEDEFS_H_ */
...@@ -29,7 +29,6 @@ ...@@ -29,7 +29,6 @@
* Authors: Thomas Hellstrom <thellstrom-at-vmware-dot-com> * Authors: Thomas Hellstrom <thellstrom-at-vmware-dot-com>
*/ */
#include <drm/ttm/ttm_module.h>
#include <linux/atomic.h> #include <linux/atomic.h>
#include <linux/errno.h> #include <linux/errno.h>
#include <linux/wait.h> #include <linux/wait.h>
...@@ -49,8 +48,6 @@ void ttm_lock_init(struct ttm_lock *lock) ...@@ -49,8 +48,6 @@ void ttm_lock_init(struct ttm_lock *lock)
init_waitqueue_head(&lock->queue); init_waitqueue_head(&lock->queue);
lock->rw = 0; lock->rw = 0;
lock->flags = 0; lock->flags = 0;
lock->kill_takers = false;
lock->signal = SIGKILL;
} }
void ttm_read_unlock(struct ttm_lock *lock) void ttm_read_unlock(struct ttm_lock *lock)
...@@ -66,11 +63,6 @@ static bool __ttm_read_lock(struct ttm_lock *lock) ...@@ -66,11 +63,6 @@ static bool __ttm_read_lock(struct ttm_lock *lock)
bool locked = false; bool locked = false;
spin_lock(&lock->lock); spin_lock(&lock->lock);
if (unlikely(lock->kill_takers)) {
send_sig(lock->signal, current, 0);
spin_unlock(&lock->lock);
return false;
}
if (lock->rw >= 0 && lock->flags == 0) { if (lock->rw >= 0 && lock->flags == 0) {
++lock->rw; ++lock->rw;
locked = true; locked = true;
...@@ -98,11 +90,6 @@ static bool __ttm_read_trylock(struct ttm_lock *lock, bool *locked) ...@@ -98,11 +90,6 @@ static bool __ttm_read_trylock(struct ttm_lock *lock, bool *locked)
*locked = false; *locked = false;
spin_lock(&lock->lock); spin_lock(&lock->lock);
if (unlikely(lock->kill_takers)) {
send_sig(lock->signal, current, 0);
spin_unlock(&lock->lock);
return false;
}
if (lock->rw >= 0 && lock->flags == 0) { if (lock->rw >= 0 && lock->flags == 0) {
++lock->rw; ++lock->rw;
block = false; block = false;
...@@ -147,11 +134,6 @@ static bool __ttm_write_lock(struct ttm_lock *lock) ...@@ -147,11 +134,6 @@ static bool __ttm_write_lock(struct ttm_lock *lock)
bool locked = false; bool locked = false;
spin_lock(&lock->lock); spin_lock(&lock->lock);
if (unlikely(lock->kill_takers)) {
send_sig(lock->signal, current, 0);
spin_unlock(&lock->lock);
return false;
}
if (lock->rw == 0 && ((lock->flags & ~TTM_WRITE_LOCK_PENDING) == 0)) { if (lock->rw == 0 && ((lock->flags & ~TTM_WRITE_LOCK_PENDING) == 0)) {
lock->rw = -1; lock->rw = -1;
lock->flags &= ~TTM_WRITE_LOCK_PENDING; lock->flags &= ~TTM_WRITE_LOCK_PENDING;
...@@ -182,88 +164,6 @@ int ttm_write_lock(struct ttm_lock *lock, bool interruptible) ...@@ -182,88 +164,6 @@ int ttm_write_lock(struct ttm_lock *lock, bool interruptible)
return ret; return ret;
} }
static int __ttm_vt_unlock(struct ttm_lock *lock)
{
int ret = 0;
spin_lock(&lock->lock);
if (unlikely(!(lock->flags & TTM_VT_LOCK)))
ret = -EINVAL;
lock->flags &= ~TTM_VT_LOCK;
wake_up_all(&lock->queue);
spin_unlock(&lock->lock);
return ret;
}
static void ttm_vt_lock_remove(struct ttm_base_object **p_base)
{
struct ttm_base_object *base = *p_base;
struct ttm_lock *lock = container_of(base, struct ttm_lock, base);
int ret;
*p_base = NULL;
ret = __ttm_vt_unlock(lock);
BUG_ON(ret != 0);
}
static bool __ttm_vt_lock(struct ttm_lock *lock)
{
bool locked = false;
spin_lock(&lock->lock);
if (lock->rw == 0) {
lock->flags &= ~TTM_VT_LOCK_PENDING;
lock->flags |= TTM_VT_LOCK;
locked = true;
} else {
lock->flags |= TTM_VT_LOCK_PENDING;
}
spin_unlock(&lock->lock);
return locked;
}
int ttm_vt_lock(struct ttm_lock *lock,
bool interruptible,
struct ttm_object_file *tfile)
{
int ret = 0;
if (interruptible) {
ret = wait_event_interruptible(lock->queue,
__ttm_vt_lock(lock));
if (unlikely(ret != 0)) {
spin_lock(&lock->lock);
lock->flags &= ~TTM_VT_LOCK_PENDING;
wake_up_all(&lock->queue);
spin_unlock(&lock->lock);
return ret;
}
} else
wait_event(lock->queue, __ttm_vt_lock(lock));
/*
* Add a base-object, the destructor of which will
* make sure the lock is released if the client dies
* while holding it.
*/
ret = ttm_base_object_init(tfile, &lock->base, false,
ttm_lock_type, &ttm_vt_lock_remove, NULL);
if (ret)
(void)__ttm_vt_unlock(lock);
else
lock->vt_holder = tfile;
return ret;
}
int ttm_vt_unlock(struct ttm_lock *lock)
{
return ttm_ref_object_base_unref(lock->vt_holder,
lock->base.handle, TTM_REF_USAGE);
}
void ttm_suspend_unlock(struct ttm_lock *lock) void ttm_suspend_unlock(struct ttm_lock *lock)
{ {
spin_lock(&lock->lock); spin_lock(&lock->lock);
......
...@@ -63,8 +63,6 @@ ...@@ -63,8 +63,6 @@
* @lock: Spinlock protecting some lock members. * @lock: Spinlock protecting some lock members.
* @rw: Read-write lock counter. Protected by @lock. * @rw: Read-write lock counter. Protected by @lock.
* @flags: Lock state. Protected by @lock. * @flags: Lock state. Protected by @lock.
* @kill_takers: Boolean whether to kill takers of the lock.
* @signal: Signal to send when kill_takers is true.
*/ */
struct ttm_lock { struct ttm_lock {
...@@ -73,9 +71,6 @@ struct ttm_lock { ...@@ -73,9 +71,6 @@ struct ttm_lock {
spinlock_t lock; spinlock_t lock;
int32_t rw; int32_t rw;
uint32_t flags; uint32_t flags;
bool kill_takers;
int signal;
struct ttm_object_file *vt_holder;
}; };
...@@ -220,29 +215,4 @@ extern void ttm_write_unlock(struct ttm_lock *lock); ...@@ -220,29 +215,4 @@ extern void ttm_write_unlock(struct ttm_lock *lock);
*/ */
extern int ttm_write_lock(struct ttm_lock *lock, bool interruptible); extern int ttm_write_lock(struct ttm_lock *lock, bool interruptible);
/**
* ttm_lock_set_kill
*
* @lock: Pointer to a struct ttm_lock
* @val: Boolean whether to kill processes taking the lock.
* @signal: Signal to send to the process taking the lock.
*
* The kill-when-taking-lock functionality is used to kill processes that keep
* on using the TTM functionality when its resources has been taken down, for
* example when the X server exits. A typical sequence would look like this:
* - X server takes lock in write mode.
* - ttm_lock_set_kill() is called with @val set to true.
* - As part of X server exit, TTM resources are taken down.
* - X server releases the lock on file release.
* - Another dri client wants to render, takes the lock and is killed.
*
*/
static inline void ttm_lock_set_kill(struct ttm_lock *lock, bool val,
int signal)
{
lock->kill_takers = val;
if (val)
lock->signal = signal;
}
#endif #endif
...@@ -463,6 +463,8 @@ void vmw_bo_bo_free(struct ttm_buffer_object *bo) ...@@ -463,6 +463,8 @@ void vmw_bo_bo_free(struct ttm_buffer_object *bo)
{ {
struct vmw_buffer_object *vmw_bo = vmw_buffer_object(bo); struct vmw_buffer_object *vmw_bo = vmw_buffer_object(bo);
WARN_ON(vmw_bo->dirty);
WARN_ON(!RB_EMPTY_ROOT(&vmw_bo->res_tree));
vmw_bo_unmap(vmw_bo); vmw_bo_unmap(vmw_bo);
kfree(vmw_bo); kfree(vmw_bo);
} }
...@@ -476,8 +478,11 @@ void vmw_bo_bo_free(struct ttm_buffer_object *bo) ...@@ -476,8 +478,11 @@ void vmw_bo_bo_free(struct ttm_buffer_object *bo)
static void vmw_user_bo_destroy(struct ttm_buffer_object *bo) static void vmw_user_bo_destroy(struct ttm_buffer_object *bo)
{ {
struct vmw_user_buffer_object *vmw_user_bo = vmw_user_buffer_object(bo); struct vmw_user_buffer_object *vmw_user_bo = vmw_user_buffer_object(bo);
struct vmw_buffer_object *vbo = &vmw_user_bo->vbo;
vmw_bo_unmap(&vmw_user_bo->vbo); WARN_ON(vbo->dirty);
WARN_ON(!RB_EMPTY_ROOT(&vbo->res_tree));
vmw_bo_unmap(vbo);
ttm_prime_object_kfree(vmw_user_bo, prime); ttm_prime_object_kfree(vmw_user_bo, prime);
} }
...@@ -510,8 +515,9 @@ int vmw_bo_init(struct vmw_private *dev_priv, ...@@ -510,8 +515,9 @@ int vmw_bo_init(struct vmw_private *dev_priv,
acc_size = vmw_bo_acc_size(dev_priv, size, user); acc_size = vmw_bo_acc_size(dev_priv, size, user);
memset(vmw_bo, 0, sizeof(*vmw_bo)); memset(vmw_bo, 0, sizeof(*vmw_bo));
BUILD_BUG_ON(TTM_MAX_BO_PRIORITY <= 3);
INIT_LIST_HEAD(&vmw_bo->res_list); vmw_bo->base.priority = 3;
vmw_bo->res_tree = RB_ROOT;
ret = ttm_bo_init(bdev, &vmw_bo->base, size, ret = ttm_bo_init(bdev, &vmw_bo->base, size,
ttm_bo_type_device, placement, ttm_bo_type_device, placement,
......
...@@ -88,6 +88,8 @@ static const struct vmw_res_func vmw_gb_context_func = { ...@@ -88,6 +88,8 @@ static const struct vmw_res_func vmw_gb_context_func = {
.res_type = vmw_res_context, .res_type = vmw_res_context,
.needs_backup = true, .needs_backup = true,
.may_evict = true, .may_evict = true,
.prio = 3,
.dirty_prio = 3,
.type_name = "guest backed contexts", .type_name = "guest backed contexts",
.backup_placement = &vmw_mob_placement, .backup_placement = &vmw_mob_placement,
.create = vmw_gb_context_create, .create = vmw_gb_context_create,
...@@ -100,6 +102,8 @@ static const struct vmw_res_func vmw_dx_context_func = { ...@@ -100,6 +102,8 @@ static const struct vmw_res_func vmw_dx_context_func = {
.res_type = vmw_res_dx_context, .res_type = vmw_res_dx_context,
.needs_backup = true, .needs_backup = true,
.may_evict = true, .may_evict = true,
.prio = 3,
.dirty_prio = 3,
.type_name = "dx contexts", .type_name = "dx contexts",
.backup_placement = &vmw_mob_placement, .backup_placement = &vmw_mob_placement,
.create = vmw_dx_context_create, .create = vmw_dx_context_create,
......
...@@ -116,6 +116,8 @@ static const struct vmw_res_func vmw_cotable_func = { ...@@ -116,6 +116,8 @@ static const struct vmw_res_func vmw_cotable_func = {
.res_type = vmw_res_cotable, .res_type = vmw_res_cotable,
.needs_backup = true, .needs_backup = true,
.may_evict = true, .may_evict = true,
.prio = 3,
.dirty_prio = 3,
.type_name = "context guest backed object tables", .type_name = "context guest backed object tables",
.backup_placement = &vmw_mob_placement, .backup_placement = &vmw_mob_placement,
.create = vmw_cotable_create, .create = vmw_cotable_create,
...@@ -307,7 +309,7 @@ static int vmw_cotable_unbind(struct vmw_resource *res, ...@@ -307,7 +309,7 @@ static int vmw_cotable_unbind(struct vmw_resource *res,
struct ttm_buffer_object *bo = val_buf->bo; struct ttm_buffer_object *bo = val_buf->bo;
struct vmw_fence_obj *fence; struct vmw_fence_obj *fence;
if (list_empty(&res->mob_head)) if (!vmw_resource_mob_attached(res))
return 0; return 0;
WARN_ON_ONCE(bo->mem.mem_type != VMW_PL_MOB); WARN_ON_ONCE(bo->mem.mem_type != VMW_PL_MOB);
...@@ -453,6 +455,7 @@ static int vmw_cotable_resize(struct vmw_resource *res, size_t new_size) ...@@ -453,6 +455,7 @@ static int vmw_cotable_resize(struct vmw_resource *res, size_t new_size)
goto out_wait; goto out_wait;
} }
vmw_resource_mob_detach(res);
res->backup = buf; res->backup = buf;
res->backup_size = new_size; res->backup_size = new_size;
vcotbl->size_read_back = cur_size_read_back; vcotbl->size_read_back = cur_size_read_back;
...@@ -467,12 +470,12 @@ static int vmw_cotable_resize(struct vmw_resource *res, size_t new_size) ...@@ -467,12 +470,12 @@ static int vmw_cotable_resize(struct vmw_resource *res, size_t new_size)
res->backup = old_buf; res->backup = old_buf;
res->backup_size = old_size; res->backup_size = old_size;
vcotbl->size_read_back = old_size_read_back; vcotbl->size_read_back = old_size_read_back;
vmw_resource_mob_attach(res);
goto out_wait; goto out_wait;
} }
vmw_resource_mob_attach(res);
/* Let go of the old mob. */ /* Let go of the old mob. */
list_del(&res->mob_head);
list_add_tail(&res->mob_head, &buf->res_list);
vmw_bo_unreference(&old_buf); vmw_bo_unreference(&old_buf);
res->id = vcotbl->type; res->id = vcotbl->type;
...@@ -496,7 +499,7 @@ static int vmw_cotable_resize(struct vmw_resource *res, size_t new_size) ...@@ -496,7 +499,7 @@ static int vmw_cotable_resize(struct vmw_resource *res, size_t new_size)
* is called before bind() in the validation sequence is instead used for two * is called before bind() in the validation sequence is instead used for two
* things. * things.
* 1) Unscrub the cotable if it is scrubbed and still attached to a backup * 1) Unscrub the cotable if it is scrubbed and still attached to a backup
* buffer, that is, if @res->mob_head is non-empty. * buffer.
* 2) Resize the cotable if needed. * 2) Resize the cotable if needed.
*/ */
static int vmw_cotable_create(struct vmw_resource *res) static int vmw_cotable_create(struct vmw_resource *res)
...@@ -512,7 +515,7 @@ static int vmw_cotable_create(struct vmw_resource *res) ...@@ -512,7 +515,7 @@ static int vmw_cotable_create(struct vmw_resource *res)
new_size *= 2; new_size *= 2;
if (likely(new_size <= res->backup_size)) { if (likely(new_size <= res->backup_size)) {
if (vcotbl->scrubbed && !list_empty(&res->mob_head)) { if (vcotbl->scrubbed && vmw_resource_mob_attached(res)) {
ret = vmw_cotable_unscrub(res); ret = vmw_cotable_unscrub(res);
if (ret) if (ret)
return ret; return ret;
......
...@@ -254,7 +254,6 @@ static int vmw_restrict_dma_mask; ...@@ -254,7 +254,6 @@ static int vmw_restrict_dma_mask;
static int vmw_assume_16bpp; static int vmw_assume_16bpp;
static int vmw_probe(struct pci_dev *, const struct pci_device_id *); static int vmw_probe(struct pci_dev *, const struct pci_device_id *);
static void vmw_master_init(struct vmw_master *);
static int vmwgfx_pm_notifier(struct notifier_block *nb, unsigned long val, static int vmwgfx_pm_notifier(struct notifier_block *nb, unsigned long val,
void *ptr); void *ptr);
...@@ -762,10 +761,6 @@ static int vmw_driver_load(struct drm_device *dev, unsigned long chipset) ...@@ -762,10 +761,6 @@ static int vmw_driver_load(struct drm_device *dev, unsigned long chipset)
DRM_INFO("MMIO at 0x%08x size is %u kiB\n", DRM_INFO("MMIO at 0x%08x size is %u kiB\n",
dev_priv->mmio_start, dev_priv->mmio_size / 1024); dev_priv->mmio_start, dev_priv->mmio_size / 1024);
vmw_master_init(&dev_priv->fbdev_master);
ttm_lock_set_kill(&dev_priv->fbdev_master.lock, false, SIGTERM);
dev_priv->active_master = &dev_priv->fbdev_master;
dev_priv->mmio_virt = memremap(dev_priv->mmio_start, dev_priv->mmio_virt = memremap(dev_priv->mmio_start,
dev_priv->mmio_size, MEMREMAP_WB); dev_priv->mmio_size, MEMREMAP_WB);
...@@ -833,6 +828,11 @@ static int vmw_driver_load(struct drm_device *dev, unsigned long chipset) ...@@ -833,6 +828,11 @@ static int vmw_driver_load(struct drm_device *dev, unsigned long chipset)
DRM_ERROR("Failed initializing TTM buffer object driver.\n"); DRM_ERROR("Failed initializing TTM buffer object driver.\n");
goto out_no_bdev; goto out_no_bdev;
} }
dev_priv->vm_ops = *dev_priv->bdev.vm_ops;
dev_priv->vm_ops.fault = vmw_bo_vm_fault;
dev_priv->vm_ops.pfn_mkwrite = vmw_bo_vm_mkwrite;
dev_priv->vm_ops.page_mkwrite = vmw_bo_vm_mkwrite;
dev_priv->bdev.vm_ops = &dev_priv->vm_ops;
/* /*
* Enable VRAM, but initially don't use it until SVGA is enabled and * Enable VRAM, but initially don't use it until SVGA is enabled and
...@@ -1007,18 +1007,7 @@ static void vmw_driver_unload(struct drm_device *dev) ...@@ -1007,18 +1007,7 @@ static void vmw_driver_unload(struct drm_device *dev)
static void vmw_postclose(struct drm_device *dev, static void vmw_postclose(struct drm_device *dev,
struct drm_file *file_priv) struct drm_file *file_priv)
{ {
struct vmw_fpriv *vmw_fp; struct vmw_fpriv *vmw_fp = vmw_fpriv(file_priv);
vmw_fp = vmw_fpriv(file_priv);
if (vmw_fp->locked_master) {
struct vmw_master *vmaster =
vmw_master(vmw_fp->locked_master);
ttm_lock_set_kill(&vmaster->lock, true, SIGTERM);
ttm_vt_unlock(&vmaster->lock);
drm_master_put(&vmw_fp->locked_master);
}
ttm_object_file_release(&vmw_fp->tfile); ttm_object_file_release(&vmw_fp->tfile);
kfree(vmw_fp); kfree(vmw_fp);
...@@ -1047,55 +1036,6 @@ static int vmw_driver_open(struct drm_device *dev, struct drm_file *file_priv) ...@@ -1047,55 +1036,6 @@ static int vmw_driver_open(struct drm_device *dev, struct drm_file *file_priv)
return ret; return ret;
} }
static struct vmw_master *vmw_master_check(struct drm_device *dev,
struct drm_file *file_priv,
unsigned int flags)
{
int ret;
struct vmw_fpriv *vmw_fp = vmw_fpriv(file_priv);
struct vmw_master *vmaster;
if (!drm_is_primary_client(file_priv) || !(flags & DRM_AUTH))
return NULL;
ret = mutex_lock_interruptible(&dev->master_mutex);
if (unlikely(ret != 0))
return ERR_PTR(-ERESTARTSYS);
if (drm_is_current_master(file_priv)) {
mutex_unlock(&dev->master_mutex);
return NULL;
}
/*
* Check if we were previously master, but now dropped. In that
* case, allow at least render node functionality.
*/
if (vmw_fp->locked_master) {
mutex_unlock(&dev->master_mutex);
if (flags & DRM_RENDER_ALLOW)
return NULL;
DRM_ERROR("Dropped master trying to access ioctl that "
"requires authentication.\n");
return ERR_PTR(-EACCES);
}
mutex_unlock(&dev->master_mutex);
/*
* Take the TTM lock. Possibly sleep waiting for the authenticating
* master to become master again, or for a SIGTERM if the
* authenticating master exits.
*/
vmaster = vmw_master(file_priv->master);
ret = ttm_read_lock(&vmaster->lock, true);
if (unlikely(ret != 0))
vmaster = ERR_PTR(ret);
return vmaster;
}
static long vmw_generic_ioctl(struct file *filp, unsigned int cmd, static long vmw_generic_ioctl(struct file *filp, unsigned int cmd,
unsigned long arg, unsigned long arg,
long (*ioctl_func)(struct file *, unsigned int, long (*ioctl_func)(struct file *, unsigned int,
...@@ -1104,7 +1044,6 @@ static long vmw_generic_ioctl(struct file *filp, unsigned int cmd, ...@@ -1104,7 +1044,6 @@ static long vmw_generic_ioctl(struct file *filp, unsigned int cmd,
struct drm_file *file_priv = filp->private_data; struct drm_file *file_priv = filp->private_data;
struct drm_device *dev = file_priv->minor->dev; struct drm_device *dev = file_priv->minor->dev;
unsigned int nr = DRM_IOCTL_NR(cmd); unsigned int nr = DRM_IOCTL_NR(cmd);
struct vmw_master *vmaster;
unsigned int flags; unsigned int flags;
long ret; long ret;
...@@ -1140,21 +1079,7 @@ static long vmw_generic_ioctl(struct file *filp, unsigned int cmd, ...@@ -1140,21 +1079,7 @@ static long vmw_generic_ioctl(struct file *filp, unsigned int cmd,
} else if (!drm_ioctl_flags(nr, &flags)) } else if (!drm_ioctl_flags(nr, &flags))
return -EINVAL; return -EINVAL;
vmaster = vmw_master_check(dev, file_priv, flags); return ioctl_func(filp, cmd, arg);
if (IS_ERR(vmaster)) {
ret = PTR_ERR(vmaster);
if (ret != -ERESTARTSYS)
DRM_INFO("IOCTL ERROR Command %d, Error %ld.\n",
nr, ret);
return ret;
}
ret = ioctl_func(filp, cmd, arg);
if (vmaster)
ttm_read_unlock(&vmaster->lock);
return ret;
out_io_encoding: out_io_encoding:
DRM_ERROR("Invalid command format, ioctl %d\n", DRM_ERROR("Invalid command format, ioctl %d\n",
...@@ -1181,65 +1106,10 @@ static void vmw_lastclose(struct drm_device *dev) ...@@ -1181,65 +1106,10 @@ static void vmw_lastclose(struct drm_device *dev)
{ {
} }
static void vmw_master_init(struct vmw_master *vmaster)
{
ttm_lock_init(&vmaster->lock);
}
static int vmw_master_create(struct drm_device *dev,
struct drm_master *master)
{
struct vmw_master *vmaster;
vmaster = kzalloc(sizeof(*vmaster), GFP_KERNEL);
if (unlikely(!vmaster))
return -ENOMEM;
vmw_master_init(vmaster);
ttm_lock_set_kill(&vmaster->lock, true, SIGTERM);
master->driver_priv = vmaster;
return 0;
}
static void vmw_master_destroy(struct drm_device *dev,
struct drm_master *master)
{
struct vmw_master *vmaster = vmw_master(master);
master->driver_priv = NULL;
kfree(vmaster);
}
static int vmw_master_set(struct drm_device *dev, static int vmw_master_set(struct drm_device *dev,
struct drm_file *file_priv, struct drm_file *file_priv,
bool from_open) bool from_open)
{ {
struct vmw_private *dev_priv = vmw_priv(dev);
struct vmw_fpriv *vmw_fp = vmw_fpriv(file_priv);
struct vmw_master *active = dev_priv->active_master;
struct vmw_master *vmaster = vmw_master(file_priv->master);
int ret = 0;
if (active) {
BUG_ON(active != &dev_priv->fbdev_master);
ret = ttm_vt_lock(&active->lock, false, vmw_fp->tfile);
if (unlikely(ret != 0))
return ret;
ttm_lock_set_kill(&active->lock, true, SIGTERM);
dev_priv->active_master = NULL;
}
ttm_lock_set_kill(&vmaster->lock, false, SIGTERM);
if (!from_open) {
ttm_vt_unlock(&vmaster->lock);
BUG_ON(vmw_fp->locked_master != file_priv->master);
drm_master_put(&vmw_fp->locked_master);
}
dev_priv->active_master = vmaster;
/* /*
* Inform a new master that the layout may have changed while * Inform a new master that the layout may have changed while
* it was gone. * it was gone.
...@@ -1254,31 +1124,10 @@ static void vmw_master_drop(struct drm_device *dev, ...@@ -1254,31 +1124,10 @@ static void vmw_master_drop(struct drm_device *dev,
struct drm_file *file_priv) struct drm_file *file_priv)
{ {
struct vmw_private *dev_priv = vmw_priv(dev); struct vmw_private *dev_priv = vmw_priv(dev);
struct vmw_fpriv *vmw_fp = vmw_fpriv(file_priv);
struct vmw_master *vmaster = vmw_master(file_priv->master);
int ret;
/**
* Make sure the master doesn't disappear while we have
* it locked.
*/
vmw_fp->locked_master = drm_master_get(file_priv->master);
ret = ttm_vt_lock(&vmaster->lock, false, vmw_fp->tfile);
vmw_kms_legacy_hotspot_clear(dev_priv); vmw_kms_legacy_hotspot_clear(dev_priv);
if (unlikely((ret != 0))) {
DRM_ERROR("Unable to lock TTM at VT switch.\n");
drm_master_put(&vmw_fp->locked_master);
}
ttm_lock_set_kill(&vmaster->lock, false, SIGTERM);
if (!dev_priv->enable_fb) if (!dev_priv->enable_fb)
vmw_svga_disable(dev_priv); vmw_svga_disable(dev_priv);
dev_priv->active_master = &dev_priv->fbdev_master;
ttm_lock_set_kill(&dev_priv->fbdev_master.lock, false, SIGTERM);
ttm_vt_unlock(&dev_priv->fbdev_master.lock);
} }
/** /**
...@@ -1557,8 +1406,6 @@ static struct drm_driver driver = { ...@@ -1557,8 +1406,6 @@ static struct drm_driver driver = {
.disable_vblank = vmw_disable_vblank, .disable_vblank = vmw_disable_vblank,
.ioctls = vmw_ioctls, .ioctls = vmw_ioctls,
.num_ioctls = ARRAY_SIZE(vmw_ioctls), .num_ioctls = ARRAY_SIZE(vmw_ioctls),
.master_create = vmw_master_create,
.master_destroy = vmw_master_destroy,
.master_set = vmw_master_set, .master_set = vmw_master_set,
.master_drop = vmw_master_drop, .master_drop = vmw_master_drop,
.open = vmw_driver_open, .open = vmw_driver_open,
......
...@@ -44,9 +44,9 @@ ...@@ -44,9 +44,9 @@
#include <linux/sync_file.h> #include <linux/sync_file.h>
#define VMWGFX_DRIVER_NAME "vmwgfx" #define VMWGFX_DRIVER_NAME "vmwgfx"
#define VMWGFX_DRIVER_DATE "20180704" #define VMWGFX_DRIVER_DATE "20190328"
#define VMWGFX_DRIVER_MAJOR 2 #define VMWGFX_DRIVER_MAJOR 2
#define VMWGFX_DRIVER_MINOR 15 #define VMWGFX_DRIVER_MINOR 16
#define VMWGFX_DRIVER_PATCHLEVEL 0 #define VMWGFX_DRIVER_PATCHLEVEL 0
#define VMWGFX_FIFO_STATIC_SIZE (1024*1024) #define VMWGFX_FIFO_STATIC_SIZE (1024*1024)
#define VMWGFX_MAX_RELOCATIONS 2048 #define VMWGFX_MAX_RELOCATIONS 2048
...@@ -81,19 +81,30 @@ ...@@ -81,19 +81,30 @@
#define VMW_RES_SHADER ttm_driver_type4 #define VMW_RES_SHADER ttm_driver_type4
struct vmw_fpriv { struct vmw_fpriv {
struct drm_master *locked_master;
struct ttm_object_file *tfile; struct ttm_object_file *tfile;
bool gb_aware; /* user-space is guest-backed aware */ bool gb_aware; /* user-space is guest-backed aware */
}; };
/**
* struct vmw_buffer_object - TTM buffer object with vmwgfx additions
* @base: The TTM buffer object
* @res_tree: RB tree of resources using this buffer object as a backing MOB
* @pin_count: pin depth
* @dx_query_ctx: DX context if this buffer object is used as a DX query MOB
* @map: Kmap object for semi-persistent mappings
* @res_prios: Eviction priority counts for attached resources
* @dirty: structure for user-space dirty-tracking
*/
struct vmw_buffer_object { struct vmw_buffer_object {
struct ttm_buffer_object base; struct ttm_buffer_object base;
struct list_head res_list; struct rb_root res_tree;
s32 pin_count; s32 pin_count;
/* Not ref-counted. Protected by binding_mutex */ /* Not ref-counted. Protected by binding_mutex */
struct vmw_resource *dx_query_ctx; struct vmw_resource *dx_query_ctx;
/* Protected by reservation */ /* Protected by reservation */
struct ttm_bo_kmap_obj map; struct ttm_bo_kmap_obj map;
u32 res_prios[TTM_MAX_BO_PRIORITY];
struct vmw_bo_dirty *dirty;
}; };
/** /**
...@@ -124,7 +135,8 @@ struct vmw_res_func; ...@@ -124,7 +135,8 @@ struct vmw_res_func;
* @res_dirty: Resource contains data not yet in the backup buffer. Protected * @res_dirty: Resource contains data not yet in the backup buffer. Protected
* by resource reserved. * by resource reserved.
* @backup_dirty: Backup buffer contains data not yet in the HW resource. * @backup_dirty: Backup buffer contains data not yet in the HW resource.
* Protecte by resource reserved. * Protected by resource reserved.
* @coherent: Emulate coherency by tracking vm accesses.
* @backup: The backup buffer if any. Protected by resource reserved. * @backup: The backup buffer if any. Protected by resource reserved.
* @backup_offset: Offset into the backup buffer if any. Protected by resource * @backup_offset: Offset into the backup buffer if any. Protected by resource
* reserved. Note that only a few resource types can have a @backup_offset * reserved. Note that only a few resource types can have a @backup_offset
...@@ -133,28 +145,32 @@ struct vmw_res_func; ...@@ -133,28 +145,32 @@ struct vmw_res_func;
* pin-count greater than zero. It is not on the resource LRU lists and its * pin-count greater than zero. It is not on the resource LRU lists and its
* backup buffer is pinned. Hence it can't be evicted. * backup buffer is pinned. Hence it can't be evicted.
* @func: Method vtable for this resource. Immutable. * @func: Method vtable for this resource. Immutable.
* @mob_node; Node for the MOB backup rbtree. Protected by @backup reserved.
* @lru_head: List head for the LRU list. Protected by @dev_priv::resource_lock. * @lru_head: List head for the LRU list. Protected by @dev_priv::resource_lock.
* @mob_head: List head for the MOB backup list. Protected by @backup reserved.
* @binding_head: List head for the context binding list. Protected by * @binding_head: List head for the context binding list. Protected by
* the @dev_priv::binding_mutex * the @dev_priv::binding_mutex
* @res_free: The resource destructor. * @res_free: The resource destructor.
* @hw_destroy: Callback to destroy the resource on the device, as part of * @hw_destroy: Callback to destroy the resource on the device, as part of
* resource destruction. * resource destruction.
*/ */
struct vmw_resource_dirty;
struct vmw_resource { struct vmw_resource {
struct kref kref; struct kref kref;
struct vmw_private *dev_priv; struct vmw_private *dev_priv;
int id; int id;
u32 used_prio;
unsigned long backup_size; unsigned long backup_size;
bool res_dirty; u32 res_dirty : 1;
bool backup_dirty; u32 backup_dirty : 1;
u32 coherent : 1;
struct vmw_buffer_object *backup; struct vmw_buffer_object *backup;
unsigned long backup_offset; unsigned long backup_offset;
unsigned long pin_count; unsigned long pin_count;
const struct vmw_res_func *func; const struct vmw_res_func *func;
struct rb_node mob_node;
struct list_head lru_head; struct list_head lru_head;
struct list_head mob_head;
struct list_head binding_head; struct list_head binding_head;
struct vmw_resource_dirty *dirty;
void (*res_free) (struct vmw_resource *res); void (*res_free) (struct vmw_resource *res);
void (*hw_destroy) (struct vmw_resource *res); void (*hw_destroy) (struct vmw_resource *res);
}; };
...@@ -376,10 +392,6 @@ struct vmw_sw_context{ ...@@ -376,10 +392,6 @@ struct vmw_sw_context{
struct vmw_legacy_display; struct vmw_legacy_display;
struct vmw_overlay; struct vmw_overlay;
struct vmw_master {
struct ttm_lock lock;
};
struct vmw_vga_topology_state { struct vmw_vga_topology_state {
uint32_t width; uint32_t width;
uint32_t height; uint32_t height;
...@@ -542,11 +554,8 @@ struct vmw_private { ...@@ -542,11 +554,8 @@ struct vmw_private {
spinlock_t svga_lock; spinlock_t svga_lock;
/** /**
* Master management. * PM management.
*/ */
struct vmw_master *active_master;
struct vmw_master fbdev_master;
struct notifier_block pm_nb; struct notifier_block pm_nb;
bool refuse_hibernation; bool refuse_hibernation;
bool suspend_locked; bool suspend_locked;
...@@ -595,6 +604,9 @@ struct vmw_private { ...@@ -595,6 +604,9 @@ struct vmw_private {
/* Validation memory reservation */ /* Validation memory reservation */
struct vmw_validation_mem vvm; struct vmw_validation_mem vvm;
/* VM operations */
struct vm_operations_struct vm_ops;
}; };
static inline struct vmw_surface *vmw_res_to_srf(struct vmw_resource *res) static inline struct vmw_surface *vmw_res_to_srf(struct vmw_resource *res)
...@@ -612,11 +624,6 @@ static inline struct vmw_fpriv *vmw_fpriv(struct drm_file *file_priv) ...@@ -612,11 +624,6 @@ static inline struct vmw_fpriv *vmw_fpriv(struct drm_file *file_priv)
return (struct vmw_fpriv *)file_priv->driver_priv; return (struct vmw_fpriv *)file_priv->driver_priv;
} }
static inline struct vmw_master *vmw_master(struct drm_master *master)
{
return (struct vmw_master *) master->driver_priv;
}
/* /*
* The locking here is fine-grained, so that it is performed once * The locking here is fine-grained, so that it is performed once
* for every read- and write operation. This is of course costly, but we * for every read- and write operation. This is of course costly, but we
...@@ -669,7 +676,8 @@ extern void vmw_resource_unreference(struct vmw_resource **p_res); ...@@ -669,7 +676,8 @@ extern void vmw_resource_unreference(struct vmw_resource **p_res);
extern struct vmw_resource *vmw_resource_reference(struct vmw_resource *res); extern struct vmw_resource *vmw_resource_reference(struct vmw_resource *res);
extern struct vmw_resource * extern struct vmw_resource *
vmw_resource_reference_unless_doomed(struct vmw_resource *res); vmw_resource_reference_unless_doomed(struct vmw_resource *res);
extern int vmw_resource_validate(struct vmw_resource *res, bool intr); extern int vmw_resource_validate(struct vmw_resource *res, bool intr,
bool dirtying);
extern int vmw_resource_reserve(struct vmw_resource *res, bool interruptible, extern int vmw_resource_reserve(struct vmw_resource *res, bool interruptible,
bool no_backup); bool no_backup);
extern bool vmw_resource_needs_backup(const struct vmw_resource *res); extern bool vmw_resource_needs_backup(const struct vmw_resource *res);
...@@ -709,6 +717,23 @@ extern void vmw_query_move_notify(struct ttm_buffer_object *bo, ...@@ -709,6 +717,23 @@ extern void vmw_query_move_notify(struct ttm_buffer_object *bo,
extern int vmw_query_readback_all(struct vmw_buffer_object *dx_query_mob); extern int vmw_query_readback_all(struct vmw_buffer_object *dx_query_mob);
extern void vmw_resource_evict_all(struct vmw_private *dev_priv); extern void vmw_resource_evict_all(struct vmw_private *dev_priv);
extern void vmw_resource_unbind_list(struct vmw_buffer_object *vbo); extern void vmw_resource_unbind_list(struct vmw_buffer_object *vbo);
void vmw_resource_mob_attach(struct vmw_resource *res);
void vmw_resource_mob_detach(struct vmw_resource *res);
void vmw_resource_dirty_update(struct vmw_resource *res, pgoff_t start,
pgoff_t end);
int vmw_resources_clean(struct vmw_buffer_object *vbo, pgoff_t start,
pgoff_t end, pgoff_t *num_prefault);
/**
* vmw_resource_mob_attached - Whether a resource currently has a mob attached
* @res: The resource
*
* Return: true if the resource has a mob attached, false otherwise.
*/
static inline bool vmw_resource_mob_attached(const struct vmw_resource *res)
{
return !RB_EMPTY_NODE(&res->mob_node);
}
/** /**
* vmw_user_resource_noref_release - release a user resource pointer looked up * vmw_user_resource_noref_release - release a user resource pointer looked up
...@@ -787,6 +812,54 @@ static inline void vmw_user_bo_noref_release(void) ...@@ -787,6 +812,54 @@ static inline void vmw_user_bo_noref_release(void)
ttm_base_object_noref_release(); ttm_base_object_noref_release();
} }
/**
* vmw_bo_adjust_prio - Adjust the buffer object eviction priority
* according to attached resources
* @vbo: The struct vmw_buffer_object
*/
static inline void vmw_bo_prio_adjust(struct vmw_buffer_object *vbo)
{
int i = ARRAY_SIZE(vbo->res_prios);
while (i--) {
if (vbo->res_prios[i]) {
vbo->base.priority = i;
return;
}
}
vbo->base.priority = 3;
}
/**
* vmw_bo_prio_add - Notify a buffer object of a newly attached resource
* eviction priority
* @vbo: The struct vmw_buffer_object
* @prio: The resource priority
*
* After being notified, the code assigns the highest resource eviction priority
* to the backing buffer object (mob).
*/
static inline void vmw_bo_prio_add(struct vmw_buffer_object *vbo, int prio)
{
if (vbo->res_prios[prio]++ == 0)
vmw_bo_prio_adjust(vbo);
}
/**
* vmw_bo_prio_del - Notify a buffer object of a resource with a certain
* priority being removed
* @vbo: The struct vmw_buffer_object
* @prio: The resource priority
*
* After being notified, the code assigns the highest resource eviction priority
* to the backing buffer object (mob).
*/
static inline void vmw_bo_prio_del(struct vmw_buffer_object *vbo, int prio)
{
if (--vbo->res_prios[prio] == 0)
vmw_bo_prio_adjust(vbo);
}
/** /**
* Misc Ioctl functionality - vmwgfx_ioctl.c * Misc Ioctl functionality - vmwgfx_ioctl.c
...@@ -1016,7 +1089,6 @@ void vmw_kms_cursor_snoop(struct vmw_surface *srf, ...@@ -1016,7 +1089,6 @@ void vmw_kms_cursor_snoop(struct vmw_surface *srf,
int vmw_kms_write_svga(struct vmw_private *vmw_priv, int vmw_kms_write_svga(struct vmw_private *vmw_priv,
unsigned width, unsigned height, unsigned pitch, unsigned width, unsigned height, unsigned pitch,
unsigned bpp, unsigned depth); unsigned bpp, unsigned depth);
void vmw_kms_idle_workqueues(struct vmw_master *vmaster);
bool vmw_kms_validate_mode_vram(struct vmw_private *dev_priv, bool vmw_kms_validate_mode_vram(struct vmw_private *dev_priv,
uint32_t pitch, uint32_t pitch,
uint32_t height); uint32_t height);
...@@ -1338,6 +1410,25 @@ int vmw_host_log(const char *log); ...@@ -1338,6 +1410,25 @@ int vmw_host_log(const char *log);
#define VMW_DEBUG_USER(fmt, ...) \ #define VMW_DEBUG_USER(fmt, ...) \
DRM_DEBUG_DRIVER(fmt, ##__VA_ARGS__) DRM_DEBUG_DRIVER(fmt, ##__VA_ARGS__)
/**
* VMW_DEBUG_KMS - Debug output for kernel mode-setting
*
* This macro is for debugging vmwgfx mode-setting code.
*/
#define VMW_DEBUG_KMS(fmt, ...) \
DRM_DEBUG_DRIVER(fmt, ##__VA_ARGS__)
/* Resource dirtying - vmwgfx_page_dirty.c */
void vmw_bo_dirty_scan(struct vmw_buffer_object *vbo);
int vmw_bo_dirty_add(struct vmw_buffer_object *vbo);
void vmw_bo_dirty_transfer_to_res(struct vmw_resource *res);
void vmw_bo_dirty_clear_res(struct vmw_resource *res);
void vmw_bo_dirty_release(struct vmw_buffer_object *vbo);
void vmw_bo_dirty_unmap(struct vmw_buffer_object *vbo,
pgoff_t start, pgoff_t end);
vm_fault_t vmw_bo_vm_fault(struct vm_fault *vmf);
vm_fault_t vmw_bo_vm_mkwrite(struct vm_fault *vmf);
/** /**
* Inline helper functions * Inline helper functions
*/ */
......
...@@ -2560,7 +2560,6 @@ static int vmw_cmd_dx_check_subresource(struct vmw_private *dev_priv, ...@@ -2560,7 +2560,6 @@ static int vmw_cmd_dx_check_subresource(struct vmw_private *dev_priv,
offsetof(typeof(*cmd), sid)); offsetof(typeof(*cmd), sid));
cmd = container_of(header, typeof(*cmd), header); cmd = container_of(header, typeof(*cmd), header);
return vmw_cmd_res_check(dev_priv, sw_context, vmw_res_surface, return vmw_cmd_res_check(dev_priv, sw_context, vmw_res_surface,
VMW_RES_DIRTY_NONE, user_surface_converter, VMW_RES_DIRTY_NONE, user_surface_converter,
&cmd->sid, NULL); &cmd->sid, NULL);
......
...@@ -1462,7 +1462,7 @@ static int vmw_kms_check_display_memory(struct drm_device *dev, ...@@ -1462,7 +1462,7 @@ static int vmw_kms_check_display_memory(struct drm_device *dev,
if (dev_priv->active_display_unit == vmw_du_screen_target && if (dev_priv->active_display_unit == vmw_du_screen_target &&
(drm_rect_width(&rects[i]) > dev_priv->stdu_max_width || (drm_rect_width(&rects[i]) > dev_priv->stdu_max_width ||
drm_rect_height(&rects[i]) > dev_priv->stdu_max_height)) { drm_rect_height(&rects[i]) > dev_priv->stdu_max_height)) {
DRM_ERROR("Screen size not supported.\n"); VMW_DEBUG_KMS("Screen size not supported.\n");
return -EINVAL; return -EINVAL;
} }
...@@ -1486,7 +1486,7 @@ static int vmw_kms_check_display_memory(struct drm_device *dev, ...@@ -1486,7 +1486,7 @@ static int vmw_kms_check_display_memory(struct drm_device *dev,
* limit on primary bounding box * limit on primary bounding box
*/ */
if (pixel_mem > dev_priv->prim_bb_mem) { if (pixel_mem > dev_priv->prim_bb_mem) {
DRM_ERROR("Combined output size too large.\n"); VMW_DEBUG_KMS("Combined output size too large.\n");
return -EINVAL; return -EINVAL;
} }
...@@ -1496,7 +1496,7 @@ static int vmw_kms_check_display_memory(struct drm_device *dev, ...@@ -1496,7 +1496,7 @@ static int vmw_kms_check_display_memory(struct drm_device *dev,
bb_mem = (u64) bounding_box.x2 * bounding_box.y2 * 4; bb_mem = (u64) bounding_box.x2 * bounding_box.y2 * 4;
if (bb_mem > dev_priv->prim_bb_mem) { if (bb_mem > dev_priv->prim_bb_mem) {
DRM_ERROR("Topology is beyond supported limits.\n"); VMW_DEBUG_KMS("Topology is beyond supported limits.\n");
return -EINVAL; return -EINVAL;
} }
} }
...@@ -1645,6 +1645,7 @@ static int vmw_kms_check_topology(struct drm_device *dev, ...@@ -1645,6 +1645,7 @@ static int vmw_kms_check_topology(struct drm_device *dev,
struct vmw_connector_state *vmw_conn_state; struct vmw_connector_state *vmw_conn_state;
if (!du->pref_active && new_crtc_state->enable) { if (!du->pref_active && new_crtc_state->enable) {
VMW_DEBUG_KMS("Enabling a disabled display unit\n");
ret = -EINVAL; ret = -EINVAL;
goto clean; goto clean;
} }
...@@ -1701,8 +1702,10 @@ vmw_kms_atomic_check_modeset(struct drm_device *dev, ...@@ -1701,8 +1702,10 @@ vmw_kms_atomic_check_modeset(struct drm_device *dev,
return ret; return ret;
ret = vmw_kms_check_implicit(dev, state); ret = vmw_kms_check_implicit(dev, state);
if (ret) if (ret) {
VMW_DEBUG_KMS("Invalid implicit state\n");
return ret; return ret;
}
if (!state->allow_modeset) if (!state->allow_modeset)
return ret; return ret;
...@@ -2347,6 +2350,9 @@ int vmw_kms_update_layout_ioctl(struct drm_device *dev, void *data, ...@@ -2347,6 +2350,9 @@ int vmw_kms_update_layout_ioctl(struct drm_device *dev, void *data,
if (!arg->num_outputs) { if (!arg->num_outputs) {
struct drm_rect def_rect = {0, 0, 800, 600}; struct drm_rect def_rect = {0, 0, 800, 600};
VMW_DEBUG_KMS("Default layout x1 = %d y1 = %d x2 = %d y2 = %d\n",
def_rect.x1, def_rect.y1,
def_rect.x2, def_rect.y2);
vmw_du_update_layout(dev_priv, 1, &def_rect); vmw_du_update_layout(dev_priv, 1, &def_rect);
return 0; return 0;
} }
...@@ -2367,6 +2373,7 @@ int vmw_kms_update_layout_ioctl(struct drm_device *dev, void *data, ...@@ -2367,6 +2373,7 @@ int vmw_kms_update_layout_ioctl(struct drm_device *dev, void *data,
drm_rects = (struct drm_rect *)rects; drm_rects = (struct drm_rect *)rects;
VMW_DEBUG_KMS("Layout count = %u\n", arg->num_outputs);
for (i = 0; i < arg->num_outputs; i++) { for (i = 0; i < arg->num_outputs; i++) {
struct drm_vmw_rect curr_rect; struct drm_vmw_rect curr_rect;
...@@ -2383,6 +2390,10 @@ int vmw_kms_update_layout_ioctl(struct drm_device *dev, void *data, ...@@ -2383,6 +2390,10 @@ int vmw_kms_update_layout_ioctl(struct drm_device *dev, void *data,
drm_rects[i].x2 = curr_rect.x + curr_rect.w; drm_rects[i].x2 = curr_rect.x + curr_rect.w;
drm_rects[i].y2 = curr_rect.y + curr_rect.h; drm_rects[i].y2 = curr_rect.y + curr_rect.h;
VMW_DEBUG_KMS(" x1 = %d y1 = %d x2 = %d y2 = %d\n",
drm_rects[i].x1, drm_rects[i].y1,
drm_rects[i].x2, drm_rects[i].y2);
/* /*
* Currently this check is limiting the topology within * Currently this check is limiting the topology within
* mode_config->max (which actually is max texture size * mode_config->max (which actually is max texture size
...@@ -2393,7 +2404,9 @@ int vmw_kms_update_layout_ioctl(struct drm_device *dev, void *data, ...@@ -2393,7 +2404,9 @@ int vmw_kms_update_layout_ioctl(struct drm_device *dev, void *data,
if (drm_rects[i].x1 < 0 || drm_rects[i].y1 < 0 || if (drm_rects[i].x1 < 0 || drm_rects[i].y1 < 0 ||
drm_rects[i].x2 > mode_config->max_width || drm_rects[i].x2 > mode_config->max_width ||
drm_rects[i].y2 > mode_config->max_height) { drm_rects[i].y2 > mode_config->max_height) {
DRM_ERROR("Invalid GUI layout.\n"); VMW_DEBUG_KMS("Invalid layout %d %d %d %d\n",
drm_rects[i].x1, drm_rects[i].y1,
drm_rects[i].x2, drm_rects[i].y2);
ret = -EINVAL; ret = -EINVAL;
goto out_free; goto out_free;
} }
......
This diff is collapsed.
This diff is collapsed.
...@@ -71,6 +71,13 @@ struct vmw_user_resource_conv { ...@@ -71,6 +71,13 @@ struct vmw_user_resource_conv {
* @commit_notify: If the resource is a command buffer managed resource, * @commit_notify: If the resource is a command buffer managed resource,
* callback to notify that a define or remove command * callback to notify that a define or remove command
* has been committed to the device. * has been committed to the device.
* @dirty_alloc: Allocate a dirty tracker. NULL if dirty-tracking is not
* supported.
* @dirty_free: Free the dirty tracker.
* @dirty_sync: Upload the dirty mob contents to the resource.
* @dirty_add_range: Add a sequential dirty range to the resource
* dirty tracker.
* @clean: Clean the resource.
*/ */
struct vmw_res_func { struct vmw_res_func {
enum vmw_res_type res_type; enum vmw_res_type res_type;
...@@ -78,6 +85,8 @@ struct vmw_res_func { ...@@ -78,6 +85,8 @@ struct vmw_res_func {
const char *type_name; const char *type_name;
struct ttm_placement *backup_placement; struct ttm_placement *backup_placement;
bool may_evict; bool may_evict;
u32 prio;
u32 dirty_prio;
int (*create) (struct vmw_resource *res); int (*create) (struct vmw_resource *res);
int (*destroy) (struct vmw_resource *res); int (*destroy) (struct vmw_resource *res);
...@@ -88,6 +97,12 @@ struct vmw_res_func { ...@@ -88,6 +97,12 @@ struct vmw_res_func {
struct ttm_validate_buffer *val_buf); struct ttm_validate_buffer *val_buf);
void (*commit_notify)(struct vmw_resource *res, void (*commit_notify)(struct vmw_resource *res,
enum vmw_cmdbuf_res_state state); enum vmw_cmdbuf_res_state state);
int (*dirty_alloc)(struct vmw_resource *res);
void (*dirty_free)(struct vmw_resource *res);
int (*dirty_sync)(struct vmw_resource *res);
void (*dirty_range_add)(struct vmw_resource *res, size_t start,
size_t end);
int (*clean)(struct vmw_resource *res);
}; };
/** /**
......
...@@ -95,6 +95,8 @@ static const struct vmw_res_func vmw_gb_shader_func = { ...@@ -95,6 +95,8 @@ static const struct vmw_res_func vmw_gb_shader_func = {
.res_type = vmw_res_shader, .res_type = vmw_res_shader,
.needs_backup = true, .needs_backup = true,
.may_evict = true, .may_evict = true,
.prio = 3,
.dirty_prio = 3,
.type_name = "guest backed shaders", .type_name = "guest backed shaders",
.backup_placement = &vmw_mob_placement, .backup_placement = &vmw_mob_placement,
.create = vmw_gb_shader_create, .create = vmw_gb_shader_create,
...@@ -106,7 +108,9 @@ static const struct vmw_res_func vmw_gb_shader_func = { ...@@ -106,7 +108,9 @@ static const struct vmw_res_func vmw_gb_shader_func = {
static const struct vmw_res_func vmw_dx_shader_func = { static const struct vmw_res_func vmw_dx_shader_func = {
.res_type = vmw_res_shader, .res_type = vmw_res_shader,
.needs_backup = true, .needs_backup = true,
.may_evict = false, .may_evict = true,
.prio = 3,
.dirty_prio = 3,
.type_name = "dx shaders", .type_name = "dx shaders",
.backup_placement = &vmw_mob_placement, .backup_placement = &vmw_mob_placement,
.create = vmw_dx_shader_create, .create = vmw_dx_shader_create,
...@@ -423,7 +427,7 @@ static int vmw_dx_shader_create(struct vmw_resource *res) ...@@ -423,7 +427,7 @@ static int vmw_dx_shader_create(struct vmw_resource *res)
WARN_ON_ONCE(!shader->committed); WARN_ON_ONCE(!shader->committed);
if (!list_empty(&res->mob_head)) { if (vmw_resource_mob_attached(res)) {
mutex_lock(&dev_priv->binding_mutex); mutex_lock(&dev_priv->binding_mutex);
ret = vmw_dx_shader_unscrub(res); ret = vmw_dx_shader_unscrub(res);
mutex_unlock(&dev_priv->binding_mutex); mutex_unlock(&dev_priv->binding_mutex);
......
This diff is collapsed.
...@@ -33,6 +33,8 @@ ...@@ -33,6 +33,8 @@
* struct vmw_validation_bo_node - Buffer object validation metadata. * struct vmw_validation_bo_node - Buffer object validation metadata.
* @base: Metadata used for TTM reservation- and validation. * @base: Metadata used for TTM reservation- and validation.
* @hash: A hash entry used for the duplicate detection hash table. * @hash: A hash entry used for the duplicate detection hash table.
* @coherent_count: If switching backup buffers, number of new coherent
* resources that will have this buffer as a backup buffer.
* @as_mob: Validate as mob. * @as_mob: Validate as mob.
* @cpu_blit: Validate for cpu blit access. * @cpu_blit: Validate for cpu blit access.
* *
...@@ -42,6 +44,7 @@ ...@@ -42,6 +44,7 @@
struct vmw_validation_bo_node { struct vmw_validation_bo_node {
struct ttm_validate_buffer base; struct ttm_validate_buffer base;
struct drm_hash_item hash; struct drm_hash_item hash;
unsigned int coherent_count;
u32 as_mob : 1; u32 as_mob : 1;
u32 cpu_blit : 1; u32 cpu_blit : 1;
}; };
...@@ -459,6 +462,19 @@ int vmw_validation_res_reserve(struct vmw_validation_context *ctx, ...@@ -459,6 +462,19 @@ int vmw_validation_res_reserve(struct vmw_validation_context *ctx,
if (ret) if (ret)
goto out_unreserve; goto out_unreserve;
} }
if (val->switching_backup && val->new_backup &&
res->coherent) {
struct vmw_validation_bo_node *bo_node =
vmw_validation_find_bo_dup(ctx,
val->new_backup);
if (WARN_ON(!bo_node)) {
ret = -EINVAL;
goto out_unreserve;
}
bo_node->coherent_count++;
}
} }
return 0; return 0;
...@@ -562,6 +578,9 @@ int vmw_validation_bo_validate(struct vmw_validation_context *ctx, bool intr) ...@@ -562,6 +578,9 @@ int vmw_validation_bo_validate(struct vmw_validation_context *ctx, bool intr)
int ret; int ret;
list_for_each_entry(entry, &ctx->bo_list, base.head) { list_for_each_entry(entry, &ctx->bo_list, base.head) {
struct vmw_buffer_object *vbo =
container_of(entry->base.bo, typeof(*vbo), base);
if (entry->cpu_blit) { if (entry->cpu_blit) {
struct ttm_operation_ctx ctx = { struct ttm_operation_ctx ctx = {
.interruptible = intr, .interruptible = intr,
...@@ -576,6 +595,27 @@ int vmw_validation_bo_validate(struct vmw_validation_context *ctx, bool intr) ...@@ -576,6 +595,27 @@ int vmw_validation_bo_validate(struct vmw_validation_context *ctx, bool intr)
} }
if (ret) if (ret)
return ret; return ret;
/*
* Rather than having the resource code allocating the bo
* dirty tracker in resource_unreserve() where we can't fail,
* Do it here when validating the buffer object.
*/
if (entry->coherent_count) {
unsigned int coherent_count = entry->coherent_count;
while (coherent_count) {
ret = vmw_bo_dirty_add(vbo);
if (ret)
return ret;
coherent_count--;
}
entry->coherent_count -= coherent_count;
}
if (vbo->dirty)
vmw_bo_dirty_scan(vbo);
} }
return 0; return 0;
} }
...@@ -601,7 +641,8 @@ int vmw_validation_res_validate(struct vmw_validation_context *ctx, bool intr) ...@@ -601,7 +641,8 @@ int vmw_validation_res_validate(struct vmw_validation_context *ctx, bool intr)
struct vmw_resource *res = val->res; struct vmw_resource *res = val->res;
struct vmw_buffer_object *backup = res->backup; struct vmw_buffer_object *backup = res->backup;
ret = vmw_resource_validate(res, intr); ret = vmw_resource_validate(res, intr, val->dirty_set &&
val->dirty);
if (ret) { if (ret) {
if (ret != -ERESTARTSYS) if (ret != -ERESTARTSYS)
DRM_ERROR("Failed to validate resource.\n"); DRM_ERROR("Failed to validate resource.\n");
...@@ -828,3 +869,34 @@ int vmw_validation_preload_res(struct vmw_validation_context *ctx, ...@@ -828,3 +869,34 @@ int vmw_validation_preload_res(struct vmw_validation_context *ctx,
ctx->mem_size_left += size; ctx->mem_size_left += size;
return 0; return 0;
} }
/**
* vmw_validation_bo_backoff - Unreserve buffer objects registered with a
* validation context
* @ctx: The validation context
*
* This function unreserves the buffer objects previously reserved using
* vmw_validation_bo_reserve. It's typically used as part of an error path
*/
void vmw_validation_bo_backoff(struct vmw_validation_context *ctx)
{
struct vmw_validation_bo_node *entry;
/*
* Switching coherent resource backup buffers failed.
* Release corresponding buffer object dirty trackers.
*/
list_for_each_entry(entry, &ctx->bo_list, base.head) {
if (entry->coherent_count) {
unsigned int coherent_count = entry->coherent_count;
struct vmw_buffer_object *vbo =
container_of(entry->base.bo, typeof(*vbo),
base);
while (coherent_count--)
vmw_bo_dirty_release(vbo);
}
}
ttm_eu_backoff_reservation(&ctx->ticket, &ctx->bo_list);
}
...@@ -172,20 +172,6 @@ vmw_validation_bo_reserve(struct vmw_validation_context *ctx, ...@@ -172,20 +172,6 @@ vmw_validation_bo_reserve(struct vmw_validation_context *ctx,
NULL, true); NULL, true);
} }
/**
* vmw_validation_bo_backoff - Unreserve buffer objects registered with a
* validation context
* @ctx: The validation context
*
* This function unreserves the buffer objects previously reserved using
* vmw_validation_bo_reserve. It's typically used as part of an error path
*/
static inline void
vmw_validation_bo_backoff(struct vmw_validation_context *ctx)
{
ttm_eu_backoff_reservation(&ctx->ticket, &ctx->bo_list);
}
/** /**
* vmw_validation_bo_fence - Unreserve and fence buffer objects registered * vmw_validation_bo_fence - Unreserve and fence buffer objects registered
* with a validation context * with a validation context
...@@ -268,4 +254,6 @@ int vmw_validation_preload_res(struct vmw_validation_context *ctx, ...@@ -268,4 +254,6 @@ int vmw_validation_preload_res(struct vmw_validation_context *ctx,
unsigned int size); unsigned int size);
void vmw_validation_res_set_dirty(struct vmw_validation_context *ctx, void vmw_validation_res_set_dirty(struct vmw_validation_context *ctx,
void *val_private, u32 dirty); void *val_private, u32 dirty);
void vmw_validation_bo_backoff(struct vmw_validation_context *ctx);
#endif #endif
...@@ -768,4 +768,14 @@ int ttm_bo_swapout(struct ttm_bo_global *glob, ...@@ -768,4 +768,14 @@ int ttm_bo_swapout(struct ttm_bo_global *glob,
struct ttm_operation_ctx *ctx); struct ttm_operation_ctx *ctx);
void ttm_bo_swapout_all(struct ttm_bo_device *bdev); void ttm_bo_swapout_all(struct ttm_bo_device *bdev);
int ttm_bo_wait_unreserved(struct ttm_buffer_object *bo); int ttm_bo_wait_unreserved(struct ttm_buffer_object *bo);
/* Default number of pre-faulted pages in the TTM fault handler */
#define TTM_BO_VM_NUM_PREFAULT 16
vm_fault_t ttm_bo_vm_reserve(struct ttm_buffer_object *bo,
struct vm_fault *vmf);
vm_fault_t ttm_bo_vm_fault_reserved(struct vm_fault *vmf,
pgprot_t prot,
pgoff_t num_prefault);
#endif #endif
...@@ -442,6 +442,9 @@ extern struct ttm_bo_global { ...@@ -442,6 +442,9 @@ extern struct ttm_bo_global {
* @driver: Pointer to a struct ttm_bo_driver struct setup by the driver. * @driver: Pointer to a struct ttm_bo_driver struct setup by the driver.
* @man: An array of mem_type_managers. * @man: An array of mem_type_managers.
* @vma_manager: Address space manager * @vma_manager: Address space manager
* @vm_ops: Pointer to the struct vm_operations_struct used for this
* device's VM operations. The driver may override this before the first
* mmap() call.
* lru_lock: Spinlock that protects the buffer+device lru lists and * lru_lock: Spinlock that protects the buffer+device lru lists and
* ddestroy lists. * ddestroy lists.
* @dev_mapping: A pointer to the struct address_space representing the * @dev_mapping: A pointer to the struct address_space representing the
...@@ -460,6 +463,7 @@ struct ttm_bo_device { ...@@ -460,6 +463,7 @@ struct ttm_bo_device {
struct ttm_bo_global *glob; struct ttm_bo_global *glob;
struct ttm_bo_driver *driver; struct ttm_bo_driver *driver;
struct ttm_mem_type_manager man[TTM_NUM_MEM_TYPES]; struct ttm_mem_type_manager man[TTM_NUM_MEM_TYPES];
const struct vm_operations_struct *vm_ops;
/* /*
* Protected by internal locks. * Protected by internal locks.
...@@ -488,6 +492,8 @@ struct ttm_bo_device { ...@@ -488,6 +492,8 @@ struct ttm_bo_device {
bool no_retry; bool no_retry;
}; };
extern const struct vm_operations_struct ttm_bo_vm_ops;
/** /**
* struct ttm_lru_bulk_move_pos * struct ttm_lru_bulk_move_pos
* *
......
...@@ -2686,7 +2686,24 @@ typedef int (*pte_fn_t)(pte_t *pte, pgtable_t token, unsigned long addr, ...@@ -2686,7 +2686,24 @@ typedef int (*pte_fn_t)(pte_t *pte, pgtable_t token, unsigned long addr,
extern int apply_to_page_range(struct mm_struct *mm, unsigned long address, extern int apply_to_page_range(struct mm_struct *mm, unsigned long address,
unsigned long size, pte_fn_t fn, void *data); unsigned long size, pte_fn_t fn, void *data);
struct pfn_range_apply;
typedef int (*pter_fn_t)(pte_t *pte, pgtable_t token, unsigned long addr,
struct pfn_range_apply *closure);
struct pfn_range_apply {
struct mm_struct *mm;
pter_fn_t ptefn;
unsigned int alloc;
};
extern int apply_to_pfn_range(struct pfn_range_apply *closure,
unsigned long address, unsigned long size);
unsigned long apply_as_wrprotect(struct address_space *mapping,
pgoff_t first_index, pgoff_t nr);
unsigned long apply_as_clean(struct address_space *mapping,
pgoff_t first_index, pgoff_t nr,
pgoff_t bitmap_pgoff,
unsigned long *bitmap,
pgoff_t *start,
pgoff_t *end);
#ifdef CONFIG_PAGE_POISONING #ifdef CONFIG_PAGE_POISONING
extern bool page_poisoning_enabled(void); extern bool page_poisoning_enabled(void);
extern void kernel_poison_pages(struct page *page, int numpages, int enable); extern void kernel_poison_pages(struct page *page, int numpages, int enable);
......
...@@ -891,11 +891,13 @@ struct drm_vmw_shader_arg { ...@@ -891,11 +891,13 @@ struct drm_vmw_shader_arg {
* surface. * surface.
* @drm_vmw_surface_flag_create_buffer: Create a backup buffer if none is * @drm_vmw_surface_flag_create_buffer: Create a backup buffer if none is
* given. * given.
* @drm_vmw_surface_flag_coherent: Back surface with coherent memory.
*/ */
enum drm_vmw_surface_flags { enum drm_vmw_surface_flags {
drm_vmw_surface_flag_shareable = (1 << 0), drm_vmw_surface_flag_shareable = (1 << 0),
drm_vmw_surface_flag_scanout = (1 << 1), drm_vmw_surface_flag_scanout = (1 << 1),
drm_vmw_surface_flag_create_buffer = (1 << 2) drm_vmw_surface_flag_create_buffer = (1 << 2),
drm_vmw_surface_flag_coherent = (1 << 3),
}; };
/** /**
......
...@@ -765,4 +765,7 @@ config GUP_BENCHMARK ...@@ -765,4 +765,7 @@ config GUP_BENCHMARK
config ARCH_HAS_PTE_SPECIAL config ARCH_HAS_PTE_SPECIAL
bool bool
config AS_DIRTY_HELPERS
bool
endmenu endmenu
...@@ -104,3 +104,4 @@ obj-$(CONFIG_HARDENED_USERCOPY) += usercopy.o ...@@ -104,3 +104,4 @@ obj-$(CONFIG_HARDENED_USERCOPY) += usercopy.o
obj-$(CONFIG_PERCPU_STATS) += percpu-stats.o obj-$(CONFIG_PERCPU_STATS) += percpu-stats.o
obj-$(CONFIG_HMM) += hmm.o obj-$(CONFIG_HMM) += hmm.o
obj-$(CONFIG_MEMFD_CREATE) += memfd.o obj-$(CONFIG_MEMFD_CREATE) += memfd.o
obj-$(CONFIG_AS_DIRTY_HELPERS) += as_dirty_helpers.o
// SPDX-License-Identifier: GPL-2.0
#include <linux/mm.h>
#include <linux/mm_types.h>
#include <linux/hugetlb.h>
#include <linux/bitops.h>
#include <linux/mmu_notifier.h>
#include <asm/cacheflush.h>
#include <asm/tlbflush.h>
/**
* struct apply_as - Closure structure for apply_as_range
* @base: struct pfn_range_apply we derive from
* @start: Address of first modified pte
* @end: Address of last modified pte + 1
* @total: Total number of modified ptes
* @vma: Pointer to the struct vm_area_struct we're currently operating on
*/
struct apply_as {
struct pfn_range_apply base;
unsigned long start;
unsigned long end;
unsigned long total;
struct vm_area_struct *vma;
};
/**
* apply_pt_wrprotect - Leaf pte callback to write-protect a pte
* @pte: Pointer to the pte
* @token: Page table token, see apply_to_pfn_range()
* @addr: The virtual page address
* @closure: Pointer to a struct pfn_range_apply embedded in a
* struct apply_as
*
* The function write-protects a pte and records the range in
* virtual address space of touched ptes for efficient range TLB flushes.
*
* Return: Always zero.
*/
static int apply_pt_wrprotect(pte_t *pte, pgtable_t token,
unsigned long addr,
struct pfn_range_apply *closure)
{
struct apply_as *aas = container_of(closure, typeof(*aas), base);
pte_t ptent = *pte;
if (pte_write(ptent)) {
pte_t old_pte = ptep_modify_prot_start(aas->vma, addr, pte);
ptent = pte_wrprotect(old_pte);
ptep_modify_prot_commit(aas->vma, addr, pte, old_pte, ptent);
aas->total++;
aas->start = min(aas->start, addr);
aas->end = max(aas->end, addr + PAGE_SIZE);
}
return 0;
}
/**
* struct apply_as_clean - Closure structure for apply_as_clean
* @base: struct apply_as we derive from
* @bitmap_pgoff: Address_space Page offset of the first bit in @bitmap
* @bitmap: Bitmap with one bit for each page offset in the address_space range
* covered.
* @start: Address_space page offset of first modified pte relative
* to @bitmap_pgoff
* @end: Address_space page offset of last modified pte relative
* to @bitmap_pgoff
*/
struct apply_as_clean {
struct apply_as base;
pgoff_t bitmap_pgoff;
unsigned long *bitmap;
pgoff_t start;
pgoff_t end;
};
/**
* apply_pt_clean - Leaf pte callback to clean a pte
* @pte: Pointer to the pte
* @token: Page table token, see apply_to_pfn_range()
* @addr: The virtual page address
* @closure: Pointer to a struct pfn_range_apply embedded in a
* struct apply_as_clean
*
* The function cleans a pte and records the range in
* virtual address space of touched ptes for efficient TLB flushes.
* It also records dirty ptes in a bitmap representing page offsets
* in the address_space, as well as the first and last of the bits
* touched.
*
* Return: Always zero.
*/
static int apply_pt_clean(pte_t *pte, pgtable_t token,
unsigned long addr,
struct pfn_range_apply *closure)
{
struct apply_as *aas = container_of(closure, typeof(*aas), base);
struct apply_as_clean *clean = container_of(aas, typeof(*clean), base);
pte_t ptent = *pte;
if (pte_dirty(ptent)) {
pgoff_t pgoff = ((addr - aas->vma->vm_start) >> PAGE_SHIFT) +
aas->vma->vm_pgoff - clean->bitmap_pgoff;
pte_t old_pte = ptep_modify_prot_start(aas->vma, addr, pte);
ptent = pte_mkclean(old_pte);
ptep_modify_prot_commit(aas->vma, addr, pte, old_pte, ptent);
aas->total++;
aas->start = min(aas->start, addr);
aas->end = max(aas->end, addr + PAGE_SIZE);
__set_bit(pgoff, clean->bitmap);
clean->start = min(clean->start, pgoff);
clean->end = max(clean->end, pgoff + 1);
}
return 0;
}
/**
* apply_as_range - Apply a pte callback to all PTEs pointing into a range
* of an address_space.
* @mapping: Pointer to the struct address_space
* @aas: Closure structure
* @first_index: First page offset in the address_space
* @nr: Number of incremental page offsets to cover
*
* Return: Number of ptes touched. Note that this number might be larger
* than @nr if there are overlapping vmas
*/
static unsigned long apply_as_range(struct address_space *mapping,
struct apply_as *aas,
pgoff_t first_index, pgoff_t nr)
{
struct vm_area_struct *vma;
pgoff_t vba, vea, cba, cea;
unsigned long start_addr, end_addr;
struct mmu_notifier_range range;
i_mmap_lock_read(mapping);
vma_interval_tree_foreach(vma, &mapping->i_mmap, first_index,
first_index + nr - 1) {
unsigned long vm_flags = READ_ONCE(vma->vm_flags);
/*
* We can only do advisory flag tests below, since we can't
* require the vm's mmap_sem to be held to protect the flags.
* Therefore, callers that strictly depend on specific mmap
* flags to remain constant throughout the operation must
* either ensure those flags are immutable for all relevant
* vmas or can't use this function. Fixing this properly would
* require the vma::vm_flags to be protected by a separate
* lock taken after the i_mmap_lock
*/
/* Skip non-applicable VMAs */
if ((vm_flags & (VM_SHARED | VM_WRITE)) !=
(VM_SHARED | VM_WRITE))
continue;
/* Warn on and skip VMAs whose flags indicate illegal usage */
if (WARN_ON((vm_flags & (VM_HUGETLB | VM_IO)) != VM_IO))
continue;
/* Clip to the vma */
vba = vma->vm_pgoff;
vea = vba + vma_pages(vma);
cba = first_index;
cba = max(cba, vba);
cea = first_index + nr;
cea = min(cea, vea);
/* Translate to virtual address */
start_addr = ((cba - vba) << PAGE_SHIFT) + vma->vm_start;
end_addr = ((cea - vba) << PAGE_SHIFT) + vma->vm_start;
if (start_addr >= end_addr)
continue;
aas->base.mm = vma->vm_mm;
aas->vma = vma;
aas->start = end_addr;
aas->end = start_addr;
mmu_notifier_range_init(&range, MMU_NOTIFY_PROTECTION_PAGE, 0,
vma, vma->vm_mm, start_addr, end_addr);
mmu_notifier_invalidate_range_start(&range);
/* Needed when we only change protection? */
flush_cache_range(vma, start_addr, end_addr);
/*
* We're not using tlb_gather_mmu() since typically
* only a small subrange of PTEs are affected.
*/
inc_tlb_flush_pending(vma->vm_mm);
/* Should not error since aas->base.alloc == 0 */
WARN_ON(apply_to_pfn_range(&aas->base, start_addr,
end_addr - start_addr));
if (aas->end > aas->start)
flush_tlb_range(vma, aas->start, aas->end);
mmu_notifier_invalidate_range_end(&range);
dec_tlb_flush_pending(vma->vm_mm);
}
i_mmap_unlock_read(mapping);
return aas->total;
}
/**
* apply_as_wrprotect - Write-protect all ptes in an address_space range
* @mapping: The address_space we want to write protect
* @first_index: The first page offset in the range
* @nr: Number of incremental page offsets to cover
*
* WARNING: This function should only be used for address spaces whose
* vmas are marked VM_IO and that do not contain huge pages.
* To avoid interference with COW'd pages, vmas not marked VM_SHARED are
* simply skipped.
*
* Return: The number of ptes actually write-protected. Note that
* already write-protected ptes are not counted.
*/
unsigned long apply_as_wrprotect(struct address_space *mapping,
pgoff_t first_index, pgoff_t nr)
{
struct apply_as aas = {
.base = {
.alloc = 0,
.ptefn = apply_pt_wrprotect,
},
.total = 0,
};
return apply_as_range(mapping, &aas, first_index, nr);
}
EXPORT_SYMBOL_GPL(apply_as_wrprotect);
/**
* apply_as_clean - Clean all ptes in an address_space range
* @mapping: The address_space we want to clean
* @first_index: The first page offset in the range
* @nr: Number of incremental page offsets to cover
* @bitmap_pgoff: The page offset of the first bit in @bitmap
* @bitmap: Pointer to a bitmap of at least @nr bits. The bitmap needs to
* cover the whole range @first_index..@first_index + @nr.
* @start: Pointer to number of the first set bit in @bitmap.
* is modified as new bits are set by the function.
* @end: Pointer to the number of the last set bit in @bitmap.
* none set. The value is modified as new bits are set by the function.
*
* Note: When this function returns there is no guarantee that a CPU has
* not already dirtied new ptes. However it will not clean any ptes not
* reported in the bitmap.
*
* If a caller needs to make sure all dirty ptes are picked up and none
* additional are added, it first needs to write-protect the address-space
* range and make sure new writers are blocked in page_mkwrite() or
* pfn_mkwrite(). And then after a TLB flush following the write-protection
* pick up all dirty bits.
*
* WARNING: This function should only be used for address spaces whose
* vmas are marked VM_IO and that do not contain huge pages.
* To avoid interference with COW'd pages, vmas not marked VM_SHARED are
* simply skipped.
*
* Return: The number of dirty ptes actually cleaned.
*/
unsigned long apply_as_clean(struct address_space *mapping,
pgoff_t first_index, pgoff_t nr,
pgoff_t bitmap_pgoff,
unsigned long *bitmap,
pgoff_t *start,
pgoff_t *end)
{
bool none_set = (*start >= *end);
struct apply_as_clean clean = {
.base = {
.base = {
.alloc = 0,
.ptefn = apply_pt_clean,
},
.total = 0,
},
.bitmap_pgoff = bitmap_pgoff,
.bitmap = bitmap,
.start = none_set ? nr : *start,
.end = none_set ? 0 : *end,
};
unsigned long ret = apply_as_range(mapping, &clean.base, first_index,
nr);
*start = clean.start;
*end = clean.end;
return ret;
}
EXPORT_SYMBOL_GPL(apply_as_clean);
...@@ -2032,18 +2032,17 @@ int vm_iomap_memory(struct vm_area_struct *vma, phys_addr_t start, unsigned long ...@@ -2032,18 +2032,17 @@ int vm_iomap_memory(struct vm_area_struct *vma, phys_addr_t start, unsigned long
} }
EXPORT_SYMBOL(vm_iomap_memory); EXPORT_SYMBOL(vm_iomap_memory);
static int apply_to_pte_range(struct mm_struct *mm, pmd_t *pmd, static int apply_to_pte_range(struct pfn_range_apply *closure, pmd_t *pmd,
unsigned long addr, unsigned long end, unsigned long addr, unsigned long end)
pte_fn_t fn, void *data)
{ {
pte_t *pte; pte_t *pte;
int err; int err;
pgtable_t token; pgtable_t token;
spinlock_t *uninitialized_var(ptl); spinlock_t *uninitialized_var(ptl);
pte = (mm == &init_mm) ? pte = (closure->mm == &init_mm) ?
pte_alloc_kernel(pmd, addr) : pte_alloc_kernel(pmd, addr) :
pte_alloc_map_lock(mm, pmd, addr, &ptl); pte_alloc_map_lock(closure->mm, pmd, addr, &ptl);
if (!pte) if (!pte)
return -ENOMEM; return -ENOMEM;
...@@ -2054,86 +2053,109 @@ static int apply_to_pte_range(struct mm_struct *mm, pmd_t *pmd, ...@@ -2054,86 +2053,109 @@ static int apply_to_pte_range(struct mm_struct *mm, pmd_t *pmd,
token = pmd_pgtable(*pmd); token = pmd_pgtable(*pmd);
do { do {
err = fn(pte++, token, addr, data); err = closure->ptefn(pte++, token, addr, closure);
if (err) if (err)
break; break;
} while (addr += PAGE_SIZE, addr != end); } while (addr += PAGE_SIZE, addr != end);
arch_leave_lazy_mmu_mode(); arch_leave_lazy_mmu_mode();
if (mm != &init_mm) if (closure->mm != &init_mm)
pte_unmap_unlock(pte-1, ptl); pte_unmap_unlock(pte-1, ptl);
return err; return err;
} }
static int apply_to_pmd_range(struct mm_struct *mm, pud_t *pud, static int apply_to_pmd_range(struct pfn_range_apply *closure, pud_t *pud,
unsigned long addr, unsigned long end, unsigned long addr, unsigned long end)
pte_fn_t fn, void *data)
{ {
pmd_t *pmd; pmd_t *pmd;
unsigned long next; unsigned long next;
int err; int err = 0;
BUG_ON(pud_huge(*pud)); BUG_ON(pud_huge(*pud));
pmd = pmd_alloc(mm, pud, addr); pmd = pmd_alloc(closure->mm, pud, addr);
if (!pmd) if (!pmd)
return -ENOMEM; return -ENOMEM;
do { do {
next = pmd_addr_end(addr, end); next = pmd_addr_end(addr, end);
err = apply_to_pte_range(mm, pmd, addr, next, fn, data); if (!closure->alloc && pmd_none_or_clear_bad(pmd))
continue;
err = apply_to_pte_range(closure, pmd, addr, next);
if (err) if (err)
break; break;
} while (pmd++, addr = next, addr != end); } while (pmd++, addr = next, addr != end);
return err; return err;
} }
static int apply_to_pud_range(struct mm_struct *mm, p4d_t *p4d, static int apply_to_pud_range(struct pfn_range_apply *closure, p4d_t *p4d,
unsigned long addr, unsigned long end, unsigned long addr, unsigned long end)
pte_fn_t fn, void *data)
{ {
pud_t *pud; pud_t *pud;
unsigned long next; unsigned long next;
int err; int err = 0;
pud = pud_alloc(mm, p4d, addr); pud = pud_alloc(closure->mm, p4d, addr);
if (!pud) if (!pud)
return -ENOMEM; return -ENOMEM;
do { do {
next = pud_addr_end(addr, end); next = pud_addr_end(addr, end);
err = apply_to_pmd_range(mm, pud, addr, next, fn, data); if (!closure->alloc && pud_none_or_clear_bad(pud))
continue;
err = apply_to_pmd_range(closure, pud, addr, next);
if (err) if (err)
break; break;
} while (pud++, addr = next, addr != end); } while (pud++, addr = next, addr != end);
return err; return err;
} }
static int apply_to_p4d_range(struct mm_struct *mm, pgd_t *pgd, static int apply_to_p4d_range(struct pfn_range_apply *closure, pgd_t *pgd,
unsigned long addr, unsigned long end, unsigned long addr, unsigned long end)
pte_fn_t fn, void *data)
{ {
p4d_t *p4d; p4d_t *p4d;
unsigned long next; unsigned long next;
int err; int err = 0;
p4d = p4d_alloc(mm, pgd, addr); p4d = p4d_alloc(closure->mm, pgd, addr);
if (!p4d) if (!p4d)
return -ENOMEM; return -ENOMEM;
do { do {
next = p4d_addr_end(addr, end); next = p4d_addr_end(addr, end);
err = apply_to_pud_range(mm, p4d, addr, next, fn, data); if (!closure->alloc && p4d_none_or_clear_bad(p4d))
continue;
err = apply_to_pud_range(closure, p4d, addr, next);
if (err) if (err)
break; break;
} while (p4d++, addr = next, addr != end); } while (p4d++, addr = next, addr != end);
return err; return err;
} }
/* /**
* Scan a region of virtual memory, filling in page tables as necessary * apply_to_pfn_range - Scan a region of virtual memory, calling a provided
* and calling a provided function on each leaf page table. * function on each leaf page table entry
* @closure: Details about how to scan and what function to apply
* @addr: Start virtual address
* @size: Size of the region
*
* If @closure->alloc is set to 1, the function will fill in the page table
* as necessary. Otherwise it will skip non-present parts.
* Note: The caller must ensure that the range does not contain huge pages.
* The caller must also assure that the proper mmu_notifier functions are
* called before and after the call to apply_to_pfn_range.
*
* WARNING: Do not use this function unless you know exactly what you are
* doing. It is lacking support for huge pages and transparent huge pages.
*
* Return: Zero on success. If the provided function returns a non-zero status,
* the page table walk will terminate and that status will be returned.
* If @closure->alloc is set to 1, then this function may also return memory
* allocation errors arising from allocating page table memory.
*/ */
int apply_to_page_range(struct mm_struct *mm, unsigned long addr, int apply_to_pfn_range(struct pfn_range_apply *closure,
unsigned long size, pte_fn_t fn, void *data) unsigned long addr, unsigned long size)
{ {
pgd_t *pgd; pgd_t *pgd;
unsigned long next; unsigned long next;
...@@ -2143,16 +2165,65 @@ int apply_to_page_range(struct mm_struct *mm, unsigned long addr, ...@@ -2143,16 +2165,65 @@ int apply_to_page_range(struct mm_struct *mm, unsigned long addr,
if (WARN_ON(addr >= end)) if (WARN_ON(addr >= end))
return -EINVAL; return -EINVAL;
pgd = pgd_offset(mm, addr); pgd = pgd_offset(closure->mm, addr);
do { do {
next = pgd_addr_end(addr, end); next = pgd_addr_end(addr, end);
err = apply_to_p4d_range(mm, pgd, addr, next, fn, data); if (!closure->alloc && pgd_none_or_clear_bad(pgd))
continue;
err = apply_to_p4d_range(closure, pgd, addr, next);
if (err) if (err)
break; break;
} while (pgd++, addr = next, addr != end); } while (pgd++, addr = next, addr != end);
return err; return err;
} }
/**
* struct page_range_apply - Closure structure for apply_to_page_range()
* @pter: The base closure structure we derive from
* @fn: The leaf pte function to call
* @data: The leaf pte function closure
*/
struct page_range_apply {
struct pfn_range_apply pter;
pte_fn_t fn;
void *data;
};
/*
* Callback wrapper to enable use of apply_to_pfn_range for
* the apply_to_page_range interface
*/
static int apply_to_page_range_wrapper(pte_t *pte, pgtable_t token,
unsigned long addr,
struct pfn_range_apply *pter)
{
struct page_range_apply *pra =
container_of(pter, typeof(*pra), pter);
return pra->fn(pte, token, addr, pra->data);
}
/*
* Scan a region of virtual memory, filling in page tables as necessary
* and calling a provided function on each leaf page table.
*
* WARNING: Do not use this function unless you know exactly what you are
* doing. It is lacking support for huge pages and transparent huge pages.
*/
int apply_to_page_range(struct mm_struct *mm, unsigned long addr,
unsigned long size, pte_fn_t fn, void *data)
{
struct page_range_apply pra = {
.pter = {.mm = mm,
.alloc = 1,
.ptefn = apply_to_page_range_wrapper },
.fn = fn,
.data = data
};
return apply_to_pfn_range(&pra.pter, addr, size);
}
EXPORT_SYMBOL_GPL(apply_to_page_range); EXPORT_SYMBOL_GPL(apply_to_page_range);
/* /*
...@@ -2238,7 +2309,7 @@ static vm_fault_t do_page_mkwrite(struct vm_fault *vmf) ...@@ -2238,7 +2309,7 @@ static vm_fault_t do_page_mkwrite(struct vm_fault *vmf)
ret = vmf->vma->vm_ops->page_mkwrite(vmf); ret = vmf->vma->vm_ops->page_mkwrite(vmf);
/* Restore original flags so that caller is not surprised */ /* Restore original flags so that caller is not surprised */
vmf->flags = old_flags; vmf->flags = old_flags;
if (unlikely(ret & (VM_FAULT_ERROR | VM_FAULT_NOPAGE))) if (unlikely(ret & (VM_FAULT_ERROR | VM_FAULT_NOPAGE | VM_FAULT_RETRY)))
return ret; return ret;
if (unlikely(!(ret & VM_FAULT_LOCKED))) { if (unlikely(!(ret & VM_FAULT_LOCKED))) {
lock_page(page); lock_page(page);
...@@ -2515,7 +2586,7 @@ static vm_fault_t wp_pfn_shared(struct vm_fault *vmf) ...@@ -2515,7 +2586,7 @@ static vm_fault_t wp_pfn_shared(struct vm_fault *vmf)
pte_unmap_unlock(vmf->pte, vmf->ptl); pte_unmap_unlock(vmf->pte, vmf->ptl);
vmf->flags |= FAULT_FLAG_MKWRITE; vmf->flags |= FAULT_FLAG_MKWRITE;
ret = vma->vm_ops->pfn_mkwrite(vmf); ret = vma->vm_ops->pfn_mkwrite(vmf);
if (ret & (VM_FAULT_ERROR | VM_FAULT_NOPAGE)) if (ret & (VM_FAULT_ERROR | VM_FAULT_NOPAGE | VM_FAULT_RETRY))
return ret; return ret;
return finish_mkwrite_fault(vmf); return finish_mkwrite_fault(vmf);
} }
...@@ -2536,7 +2607,8 @@ static vm_fault_t wp_page_shared(struct vm_fault *vmf) ...@@ -2536,7 +2607,8 @@ static vm_fault_t wp_page_shared(struct vm_fault *vmf)
pte_unmap_unlock(vmf->pte, vmf->ptl); pte_unmap_unlock(vmf->pte, vmf->ptl);
tmp = do_page_mkwrite(vmf); tmp = do_page_mkwrite(vmf);
if (unlikely(!tmp || (tmp & if (unlikely(!tmp || (tmp &
(VM_FAULT_ERROR | VM_FAULT_NOPAGE)))) { (VM_FAULT_ERROR | VM_FAULT_NOPAGE |
VM_FAULT_RETRY)))) {
put_page(vmf->page); put_page(vmf->page);
return tmp; return tmp;
} }
...@@ -3601,7 +3673,8 @@ static vm_fault_t do_shared_fault(struct vm_fault *vmf) ...@@ -3601,7 +3673,8 @@ static vm_fault_t do_shared_fault(struct vm_fault *vmf)
unlock_page(vmf->page); unlock_page(vmf->page);
tmp = do_page_mkwrite(vmf); tmp = do_page_mkwrite(vmf);
if (unlikely(!tmp || if (unlikely(!tmp ||
(tmp & (VM_FAULT_ERROR | VM_FAULT_NOPAGE)))) { (tmp & (VM_FAULT_ERROR | VM_FAULT_NOPAGE |
VM_FAULT_RETRY)))) {
put_page(vmf->page); put_page(vmf->page);
return tmp; return tmp;
} }
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment