drm/vmwgfx: Don't send bad flags to the host

Signed-off-by: Jakob Bornecrantz <jakob@vmware.com>
Signed-off-by: Dave Airlie <airlied@redhat.com>

drivers/gpu/drm/vmwgfx/vmwgfx_drv.h

@@ -96,6 +96,8 @@ struct vmw_surface {
 	struct drm_vmw_size *sizes;
 	uint32_t num_sizes;
 
+	bool scanout;
+
 	/* TODO so far just a extra pointer */
 	struct vmw_cursor_snooper snooper;
 };

drivers/gpu/drm/vmwgfx/vmwgfx_kms.c

@@ -707,6 +707,9 @@ static struct drm_framebuffer *vmw_kms_fb_create(struct drm_device *dev,
 	if (ret)
 		goto try_dmabuf;
 
+	if (!surface->scanout)
+		goto err_not_scanout;
+
 	ret = vmw_kms_new_framebuffer_surface(dev_priv, surface, &vfb,
 					      mode_cmd->width, mode_cmd->height);
 
@@ -740,6 +743,13 @@ static struct drm_framebuffer *vmw_kms_fb_create(struct drm_device *dev,
 	}
 
 	return &vfb->base;
+
+err_not_scanout:
+	DRM_ERROR("surface not marked as scanout\n");
+	/* vmw_user_surface_lookup takes one ref */
+	vmw_surface_unreference(&surface);
+
+	return NULL;
 }
 
 static int vmw_kms_fb_changed(struct drm_device *dev)

drivers/gpu/drm/vmwgfx/vmwgfx_resource.c

@@ -35,6 +35,11 @@
 #define VMW_RES_SURFACE ttm_driver_type1
 #define VMW_RES_STREAM ttm_driver_type2
 
+/* XXX: This isn't a real hardware flag, but just a hack for kernel to
+ * know about primary surfaces. Find a better way to accomplish this.
+ */
+#define SVGA3D_SURFACE_HINT_SCANOUT (1 << 9)
+
 struct vmw_user_context {
 	struct ttm_base_object base;
 	struct vmw_resource res;
@@ -599,8 +604,17 @@ int vmw_surface_define_ioctl(struct drm_device *dev, void *data,
 	if (unlikely(ret != 0))
 		goto out_err1;
 
+	if (srf->flags & SVGA3D_SURFACE_HINT_SCANOUT) {
+		/* we should not send this flag down to hardware since
+		 * its not a official one
+		 */
+		srf->flags &= ~SVGA3D_SURFACE_HINT_SCANOUT;
+		srf->scanout = true;
+	} else {
+		srf->scanout = false;
+	}
 
-	if (srf->flags & (1 << 9) &&
+	if (srf->scanout &&
 	    srf->num_sizes == 1 &&
 	    srf->sizes[0].width == 64 &&
 	    srf->sizes[0].height == 64 &&