drm/msm/mdp5: move LM bounds check into plane->atomic_check()

The mode_config->max_{width,height} is for the maximum size of a fb, not
the max scanout limits (of the layer-mixer).  It is legal, and in fact
common, to create a larger fb, only only scan-out a smaller part of it.
For example multi-monitor configurations for x11, or android wallpaper
layer (which is created larger than the screen resolution for fast
scrolling by just changing the src x/y coordinates).
Signed-off-by: Rob Clark <robdclark@gmail.com>

drivers/gpu/drm/msm/mdp/mdp5/mdp5_kms.c

@@ -711,8 +711,8 @@ struct msm_kms *mdp5_kms_init(struct drm_device *dev)
 
 	dev->mode_config.min_width = 0;
 	dev->mode_config.min_height = 0;
-	dev->mode_config.max_width = config->hw->lm.max_width;
-	dev->mode_config.max_height = config->hw->lm.max_height;
+	dev->mode_config.max_width = 0xffff;
+	dev->mode_config.max_height = 0xffff;
 
 	dev->driver->get_vblank_timestamp = mdp5_get_vblank_timestamp;
 	dev->driver->get_scanout_position = mdp5_get_scanoutpos;

drivers/gpu/drm/msm/mdp/mdp5/mdp5_plane.c

@@ -280,7 +280,9 @@ static int mdp5_plane_atomic_check(struct drm_plane *plane,
 {
 	struct mdp5_plane_state *mdp5_state = to_mdp5_plane_state(state);
 	struct drm_plane_state *old_state = plane->state;
+	struct mdp5_cfg *config = mdp5_cfg_get_config(get_kms(plane)->cfg);
 	bool new_hwpipe = false;
+	uint32_t max_width, max_height;
 	uint32_t caps = 0;
 
 	DBG("%s: check (%d -> %d)", plane->name,
@@ -293,6 +295,17 @@ static int mdp5_plane_atomic_check(struct drm_plane *plane,
 	if (WARN_ON(to_mdp5_plane_state(old_state)->pending))
 		return -EBUSY;
 
+	max_width = config->hw->lm.max_width << 16;
+	max_height = config->hw->lm.max_height << 16;
+
+	/* Make sure source dimensions are within bounds. */
+	if ((state->src_w > max_width) || (state->src_h > max_height)) {
+		struct drm_rect src = drm_plane_state_src(state);
+		DBG("Invalid source size "DRM_RECT_FP_FMT,
+				DRM_RECT_FP_ARG(&src));
+		return -ERANGE;
+	}
+
 	if (plane_enabled(state)) {
 		unsigned int rotation;
 		const struct mdp_format *format;