drm/fb-helper: Fix single-probe color-format selection

Fix the color-format selection of the single-probe helper. Go
through all user-specified values and test each for compatibility
with the driver. If none is supported, use the driver-provided
default. This guarantees that the console is always available in
any color format at least.

Until now, the format selection of the single-probe helper tried
to either use a user-specified format or a 32-bit default format.
If the user-specified format was not supported by the driver, the
selection failed and the display remained blank.
Signed-off-by: Thomas Zimmermann <tzimmermann@suse.de>
Reviewed-by: Javier Martinez Canillas <javierm@redhat.com>
Link: https://patchwork.freedesktop.org/patch/msgid/20230102112927.26565-12-tzimmermann@suse.de

drivers/gpu/drm/drm_fb_helper.c

@@ -1726,109 +1726,125 @@ int drm_fb_helper_pan_display(struct fb_var_screeninfo *var,
 }
 EXPORT_SYMBOL(drm_fb_helper_pan_display);
 
-
-static int __drm_fb_helper_find_sizes(struct drm_fb_helper *fb_helper, int preferred_bpp,
-				      struct drm_fb_helper_surface_size *sizes)
+static uint32_t drm_fb_helper_find_format(struct drm_fb_helper *fb_helper, const uint32_t *formats,
+					  size_t format_count, uint32_t bpp, uint32_t depth)
 {
-	struct drm_client_dev *client = &fb_helper->client;
 	struct drm_device *dev = fb_helper->dev;
-	int crtc_count = 0;
-	struct drm_connector_list_iter conn_iter;
-	struct drm_connector *connector;
-	struct drm_mode_set *mode_set;
-	int best_depth = 0;
-
-	memset(sizes, 0, sizeof(struct drm_fb_helper_surface_size));
-	sizes->surface_depth = 24;
-	sizes->surface_bpp = 32;
-	sizes->fb_width = (u32)-1;
-	sizes->fb_height = (u32)-1;
+	uint32_t format;
+	size_t i;
 
 	/*
-	 * If driver picks 8 or 16 by default use that for both depth/bpp
-	 * to begin with
+	 * Do not consider YUV or other complicated formats
+	 * for framebuffers. This means only legacy formats
+	 * are supported (fmt->depth is a legacy field), but
+	 * the framebuffer emulation can only deal with such
+	 * formats, specifically RGB/BGA formats.
 	 */
-	if (preferred_bpp != sizes->surface_bpp)
-		sizes->surface_depth = sizes->surface_bpp = preferred_bpp;
+	format = drm_mode_legacy_fb_format(bpp, depth);
+	if (!format)
+		goto err;
 
-	drm_connector_list_iter_begin(fb_helper->dev, &conn_iter);
-	drm_client_for_each_connector_iter(connector, &conn_iter) {
-		struct drm_cmdline_mode *cmdline_mode;
+	for (i = 0; i < format_count; ++i) {
+		if (formats[i] == format)
+			return format;
+	}
 
-		cmdline_mode = &connector->cmdline_mode;
+err:
+	/* We found nothing. */
+	drm_warn(dev, "bpp/depth value of %u/%u not supported\n", bpp, depth);
+
+	return DRM_FORMAT_INVALID;
+}
+
+static uint32_t drm_fb_helper_find_cmdline_format(struct drm_fb_helper *fb_helper,
+						  const uint32_t *formats, size_t format_count,
+						  const struct drm_cmdline_mode *cmdline_mode)
+{
+	struct drm_device *dev = fb_helper->dev;
+	uint32_t bpp, depth;
+
+	if (!cmdline_mode->bpp_specified)
+		return DRM_FORMAT_INVALID;
 
-		if (cmdline_mode->bpp_specified) {
 	switch (cmdline_mode->bpp) {
+	case 1:
+	case 2:
+	case 4:
 	case 8:
-				sizes->surface_depth = sizes->surface_bpp = 8;
-				break;
-			case 15:
-				sizes->surface_depth = 15;
-				sizes->surface_bpp = 16;
-				break;
 	case 16:
-				sizes->surface_depth = sizes->surface_bpp = 16;
-				break;
 	case 24:
-				sizes->surface_depth = sizes->surface_bpp = 24;
+		bpp = depth = cmdline_mode->bpp;
 		break;
-			case 32:
-				sizes->surface_depth = 24;
-				sizes->surface_bpp = 32;
+	case 15:
+		bpp = 16;
+		depth = 15;
 		break;
-			}
+	case 32:
+		bpp = 32;
+		depth = 24;
 		break;
+	default:
+		drm_info(dev, "unsupported bpp value of %d\n", cmdline_mode->bpp);
+		return DRM_FORMAT_INVALID;
 	}
-	}
-	drm_connector_list_iter_end(&conn_iter);
 
-	/*
-	 * If we run into a situation where, for example, the primary plane
-	 * supports RGBA5551 (16 bpp, depth 15) but not RGB565 (16 bpp, depth
-	 * 16) we need to scale down the depth of the sizes we request.
-	 */
+	return drm_fb_helper_find_format(fb_helper, formats, format_count, bpp, depth);
+}
+
+static int __drm_fb_helper_find_sizes(struct drm_fb_helper *fb_helper, int preferred_bpp,
+				      struct drm_fb_helper_surface_size *sizes)
+{
+	struct drm_client_dev *client = &fb_helper->client;
+	struct drm_device *dev = fb_helper->dev;
+	int crtc_count = 0;
+	struct drm_connector_list_iter conn_iter;
+	struct drm_connector *connector;
+	struct drm_mode_set *mode_set;
+	uint32_t surface_format = DRM_FORMAT_INVALID;
+	const struct drm_format_info *info;
+
+	memset(sizes, 0, sizeof(*sizes));
+	sizes->fb_width = (u32)-1;
+	sizes->fb_height = (u32)-1;
+
 	drm_client_for_each_modeset(mode_set, client) {
 		struct drm_crtc *crtc = mode_set->crtc;
 		struct drm_plane *plane = crtc->primary;
-		int j;
 
 		drm_dbg_kms(dev, "test CRTC %u primary plane\n", drm_crtc_index(crtc));
 
-		for (j = 0; j < plane->format_count; j++) {
-			const struct drm_format_info *fmt;
-
-			fmt = drm_format_info(plane->format_types[j]);
-
-			/*
-			 * Do not consider YUV or other complicated formats
-			 * for framebuffers. This means only legacy formats
-			 * are supported (fmt->depth is a legacy field) but
-			 * the framebuffer emulation can only deal with such
-			 * formats, specifically RGB/BGA formats.
-			 */
-			if (fmt->depth == 0)
-				continue;
+		drm_connector_list_iter_begin(fb_helper->dev, &conn_iter);
+		drm_client_for_each_connector_iter(connector, &conn_iter) {
+			struct drm_cmdline_mode *cmdline_mode = &connector->cmdline_mode;
 
-			/* We found a perfect fit, great */
-			if (fmt->depth == sizes->surface_depth) {
-				best_depth = fmt->depth;
-				break;
+			surface_format = drm_fb_helper_find_cmdline_format(fb_helper,
+									   plane->format_types,
+									   plane->format_count,
+									   cmdline_mode);
+			if (surface_format != DRM_FORMAT_INVALID)
+				break; /* found supported format */
 		}
+		drm_connector_list_iter_end(&conn_iter);
 
-			/* Skip depths above what we're looking for */
-			if (fmt->depth > sizes->surface_depth)
-				continue;
+		if (surface_format != DRM_FORMAT_INVALID)
+			break; /* found supported format */
 
-			/* Best depth found so far */
-			if (fmt->depth > best_depth)
-				best_depth = fmt->depth;
+		/* try preferred bpp/depth */
+		surface_format = drm_fb_helper_find_format(fb_helper, plane->format_types,
+							   plane->format_count, preferred_bpp,
+							   dev->mode_config.preferred_depth);
+		if (surface_format != DRM_FORMAT_INVALID)
+			break; /* found supported format */
 	}
+
+	if (surface_format == DRM_FORMAT_INVALID) {
+		drm_warn(dev, "No compatible format found\n");
+		return -EAGAIN;
 	}
-	if (sizes->surface_depth != best_depth && best_depth) {
-		drm_info(dev, "requested bpp %d, scaled depth down to %d",
-			 sizes->surface_bpp, best_depth);
-		sizes->surface_depth = best_depth;
-	}
+
+	info = drm_format_info(surface_format);
+	sizes->surface_bpp = drm_format_info_bpp(info, 0);
+	sizes->surface_depth = info->depth;
 
 	/* first up get a count of crtcs now in use and new min/maxes width/heights */
 	crtc_count = 0;