drm/amd/display: make verified link cap not exceeding max link cap

[why]
There is a chance verified link cap can be greater than max link cap.
This causes software hang because we cannot power up PHY with link rate
that cannot handle.
The change is to guard verfieid link cap from becoming larger than max link cap
our PHy can support.
Reviewed-by: Jimmy Kizito <Jimmy.Kizito@amd.com>
Acked-by: Anson Jacob <Anson.Jacob@amd.com>
Signed-off-by: Wenjing Liu <wenjing.liu@amd.com>
Tested-by: Daniel Wheeler <daniel.wheeler@amd.com>
Signed-off-by: Alex Deucher <alexander.deucher@amd.com>

drivers/gpu/drm/amd/display/dc/core/dc_link_dp.c

@@ -2832,14 +2832,25 @@ bool dp_verify_link_cap(
 	enum link_training_result status;
 	union hpd_irq_data irq_data;
 
+	/* link training starts with the maximum common settings
+	 * supported by both sink and ASIC.
+	 */
+	max_link_cap = get_max_link_cap(link);
+	initial_link_settings = get_common_supported_link_settings(
+			*known_limit_link_setting,
+			max_link_cap);
+
 	/* Accept reported capabilities if link supports flexible encoder mapping or encoder already in use. */
 	if (link->dc->debug.skip_detection_link_training ||
 			link->is_dig_mapping_flexible) {
+		/* TODO - should we check link encoder's max link caps here?
+		 * How do we know which link encoder to check from?
+		 */
 		link->verified_link_cap = *known_limit_link_setting;
 		return true;
 	} else if (link->link_enc && link->dc->res_pool->funcs->link_encs_assign &&
 			!link_enc_cfg_is_link_enc_avail(link->ctx->dc, link->link_enc->preferred_engine)) {
-		link->verified_link_cap = *known_limit_link_setting;
+		link->verified_link_cap = initial_link_settings;
 		return true;
 	}
 
@@ -2847,8 +2858,6 @@ bool dp_verify_link_cap(
 	success = false;
 	skip_link_training = false;
 
-	max_link_cap = get_max_link_cap(link);
-
 	/* Grant extended timeout request */
 	if ((link->lttpr_mode == LTTPR_MODE_NON_TRANSPARENT) && (link->dpcd_caps.lttpr_caps.max_ext_timeout > 0)) {
 		uint8_t grant = link->dpcd_caps.lttpr_caps.max_ext_timeout & 0x80;
@@ -2870,12 +2879,6 @@ bool dp_verify_link_cap(
 
 	dp_cs_id = get_clock_source_id(link);
 
-	/* link training starts with the maximum common settings
-	 * supported by both sink and ASIC.
-	 */
-	initial_link_settings = get_common_supported_link_settings(
-			*known_limit_link_setting,
-			max_link_cap);
 	cur_link_setting = initial_link_settings;
 
 	/* Temporary Renoir-specific workaround for SWDEV-215184;
@@ -2969,7 +2972,7 @@ bool dp_verify_link_cap_with_retries(
 			link->verified_link_cap.link_spread = LINK_SPREAD_DISABLED;
 			break;
 		} else if (dp_verify_link_cap(link,
-				&link->reported_link_cap,
+				known_limit_link_setting,
 				&fail_count) && fail_count == 0) {
 			success = true;
 			break;
@@ -2984,11 +2987,21 @@ bool dp_verify_mst_link_cap(
 {
 	struct dc_link_settings max_link_cap = {0};
 
+	if (dp_get_link_encoding_format(&link->reported_link_cap) ==
+			DP_8b_10b_ENCODING) {
 		max_link_cap = get_max_link_cap(link);
 		link->verified_link_cap = get_common_supported_link_settings(
 				link->reported_link_cap,
 				max_link_cap);
-
+	}
+#if defined(CONFIG_DRM_AMD_DC_DCN)
+	else if (dp_get_link_encoding_format(&link->reported_link_cap) ==
+			DP_128b_132b_ENCODING) {
+		dp_verify_link_cap_with_retries(link,
+				&link->reported_link_cap,
+				LINK_TRAINING_MAX_VERIFY_RETRY);
+	}
+#endif
 	return true;
 }