MDEV-25981 InnoDB upgrade fails

trx_undo_mem_create_at_db_start(): Relax too strict upgrade checks
that were introduced in
commit e46f76c9 (MDEV-15912).
On commit, pages will typically be set to TRX_UNDO_CACHED state.
Having the type TRX_UNDO_INSERT in such pages is common and
unproblematic; the type would be reset in trx_undo_reuse_cached().

trx_rseg_array_init(): On failure, clean up the rollback segments
that were initialized so far, to avoid an assertion failure later
during shutdown.

storage/innobase/trx/trx0rseg.cc

@@ -411,7 +411,7 @@ trx_rseg_mem_create(ulint id, fil_space_t* space, ulint page_no)
 static dberr_t trx_undo_lists_init(trx_rseg_t *rseg, trx_id_t &max_trx_id,
                                    const trx_rsegf_t *rseg_header)
 {
-	ut_ad(srv_force_recovery < SRV_FORCE_NO_UNDO_LOG_SCAN);
+  ut_ad(srv_force_recovery < SRV_FORCE_NO_UNDO_LOG_SCAN);
 
   for (ulint i= 0; i < TRX_RSEG_N_SLOTS; i++)
   {
@@ -566,6 +566,7 @@ dberr_t trx_rseg_array_init()
 	bool wsrep_xid_in_rseg_found = false;
 #endif
 	mtr_t mtr;
+	dberr_t err = DB_SUCCESS;
 
 	for (ulint rseg_id = 0; rseg_id < TRX_SYS_N_RSEGS; rseg_id++) {
 		mtr.start();
@@ -595,10 +596,11 @@ dberr_t trx_rseg_array_init()
 				ut_ad(rseg->id == rseg_id);
 				ut_ad(!trx_sys.rseg_array[rseg_id]);
 				trx_sys.rseg_array[rseg_id] = rseg;
-				if (dberr_t err = trx_rseg_mem_restore(
-					    rseg, max_trx_id, &mtr)) {
+				if ((err = trx_rseg_mem_restore(
+					     rseg, max_trx_id, &mtr))
+				    != DB_SUCCESS) {
 					mtr.commit();
-					return err;
+					break;
 				}
 #ifdef WITH_WSREP
 				if (!wsrep_sys_xid.is_null() &&
@@ -619,6 +621,21 @@ dberr_t trx_rseg_array_init()
 		mtr.commit();
 	}
 
+	if (err != DB_SUCCESS) {
+		for (ulint rseg_id = 0; rseg_id < TRX_SYS_N_RSEGS; rseg_id++) {
+			if (trx_rseg_t*& rseg = trx_sys.rseg_array[rseg_id]) {
+				while (trx_undo_t* u= UT_LIST_GET_FIRST(
+					     rseg->undo_list)) {
+					UT_LIST_REMOVE(rseg->undo_list, u);
+					ut_free(u);
+				}
+				trx_rseg_mem_free(rseg);
+				rseg = NULL;
+			}
+		}
+		return err;
+	}
+
 #ifdef WITH_WSREP
 	if (!wsrep_sys_xid.is_null()) {
 		/* Upgrade from a version prior to 10.3.5,

storage/innobase/trx/trx0undo.cc

@@ -1078,18 +1078,11 @@ trx_undo_mem_create_at_db_start(trx_rseg_t *rseg, ulint id, uint32_t page_no,
 	const uint16_t type = mach_read_from_2(TRX_UNDO_PAGE_HDR
 					       + TRX_UNDO_PAGE_TYPE
 					       + undo_page);
-	switch (type) {
-	case 0:
-	case 2: /* TRX_UNDO_UPDATE */
-		break;
-	case 1: /* TRX_UNDO_INSERT */
-		sql_print_error("InnoDB: upgrade from older version than"
-				" MariaDB 10.3 requires clean shutdown");
-		goto corrupted;
-	default:
+	if (UNIV_UNLIKELY(type > 2)) {
+corrupted_type:
 		sql_print_error("InnoDB: unsupported undo header type %u",
 				type);
-	corrupted:
+corrupted:
 		mtr.commit();
 		return NULL;
 	}
@@ -1109,12 +1102,21 @@ trx_undo_mem_create_at_db_start(trx_rseg_t *rseg, ulint id, uint32_t page_no,
 	switch (state) {
 	case TRX_UNDO_ACTIVE:
 	case TRX_UNDO_PREPARED:
-		break;
+		if (UNIV_LIKELY(type != 1)) {
+			break;
+		}
+		sql_print_error("InnoDB: upgrade from older version than"
+				" MariaDB 10.3 requires clean shutdown");
+		goto corrupted;
 	default:
 		sql_print_error("InnoDB: unsupported undo header state %u",
 				state);
 		goto corrupted;
 	case TRX_UNDO_TO_PURGE:
+		if (UNIV_UNLIKELY(type == 1)) {
+			goto corrupted_type;
+		}
+		/* fall through */
 	case TRX_UNDO_CACHED:
 		trx_id_t id = mach_read_from_8(TRX_UNDO_TRX_NO + undo_header);
 		if (id >> 48) {