MDEV-17989 InnoDB: Failing assertion: dict_tf2_is_valid(flags, flags2)

With innodb_default_row_format=redundant, InnoDB would crash when
using table options that are incompatible with ROW_FORMAT=REDUNDANT.

create_table_info_t::m_default_row_format: Cache the value of
innodb_default_row_format.

create_table_info_t::check_table_options(): Validate ROW_TYPE_DEFAULT
with m_default_row_format.

create_table_info_t::innobase_table_flags(): Use the
cached m_default_row_format.

create_table_info_t: Never read m_form->s->row_type.
Use m_create_info->row_type instead.

dict_tf_set(): Never set invalid flags for ROW_FORMAT=REDUNDANT.

ha_innobase::truncate(): Set info.row_type based on the ROW_FORMAT
of the current table.

mysql-test/suite/innodb/r/default_row_format_create,redundant.rdiff

@@ -9,3 +9,14 @@
  DROP TABLE t1;
  CREATE TABLE t1(c1 TEXT,c2 BLOB) ENGINE=InnoDB ROW_FORMAT=DYNAMIC;
  SHOW TABLE STATUS LIKE 't1';
+@@ -31,8 +31,9 @@
+ CREATE TABLE t1 (c1 INT) ENGINE=InnoDB page_compressed=1;
+ SHOW TABLE STATUS LIKE 't1';
+ Name	Engine	Version	Row_format	Rows	Avg_row_length	Data_length	Max_data_length	Index_length	Data_free	Auto_increment	Create_time	Update_time	Check_time	Collation	Checksum	Create_options	Comment
+-t1	InnoDB	#	Dynamic	#	#	#	#	#	#	NULL	#	NULL	NULL	latin1_swedish_ci	NULL	`page_compressed`=1	
+ DROP TABLE IF EXISTS t1;
++Warnings:
++Note	1051	Unknown table 'test.t1'
+ SET @save_format = @@GLOBAL.innodb_default_row_format;
+ SET GLOBAL innodb_default_row_format = redundant;
+ CREATE TABLE t1 (c1 INT) ENGINE=InnoDB;

mysql-test/suite/innodb/r/default_row_format_create.result

@@ -19,8 +19,26 @@ Name	Engine	Version	Row_format	Rows	Avg_row_length	Data_length	Max_data_length	I
 t1	InnoDB	#	Redundant	#	#	#	#	#	#	NULL	#	NULL	NULL	latin1_swedish_ci	NULL	row_format=REDUNDANT	
 DROP TABLE t1;
 CREATE TABLE t1(c1 TEXT,c2 BLOB) ENGINE=InnoDB
-ROW_FORMAT=COMPRESSED KEY_BLOCK_SIZE=1;
+ROW_FORMAT=COMPRESSED;
 SHOW TABLE STATUS LIKE 't1';
 Name	Engine	Version	Row_format	Rows	Avg_row_length	Data_length	Max_data_length	Index_length	Data_free	Auto_increment	Create_time	Update_time	Check_time	Collation	Checksum	Create_options	Comment
-t1	InnoDB	#	Compressed	#	#	#	#	#	#	NULL	#	NULL	NULL	latin1_swedish_ci	NULL	row_format=COMPRESSED key_block_size=1	
+t1	InnoDB	#	Compressed	#	#	#	#	#	#	NULL	#	NULL	NULL	latin1_swedish_ci	NULL	row_format=COMPRESSED	
+TRUNCATE TABLE t1;
+SHOW TABLE STATUS LIKE 't1';
+Name	Engine	Version	Row_format	Rows	Avg_row_length	Data_length	Max_data_length	Index_length	Data_free	Auto_increment	Create_time	Update_time	Check_time	Collation	Checksum	Create_options	Comment
+t1	InnoDB	#	Compressed	#	#	#	#	#	#	NULL	#	NULL	NULL	latin1_swedish_ci	NULL	row_format=COMPRESSED	
+DROP TABLE t1;
+CREATE TABLE t1 (c1 INT) ENGINE=InnoDB page_compressed=1;
+SHOW TABLE STATUS LIKE 't1';
+Name	Engine	Version	Row_format	Rows	Avg_row_length	Data_length	Max_data_length	Index_length	Data_free	Auto_increment	Create_time	Update_time	Check_time	Collation	Checksum	Create_options	Comment
+t1	InnoDB	#	Dynamic	#	#	#	#	#	#	NULL	#	NULL	NULL	latin1_swedish_ci	NULL	`page_compressed`=1	
+DROP TABLE IF EXISTS t1;
+SET @save_format = @@GLOBAL.innodb_default_row_format;
+SET GLOBAL innodb_default_row_format = redundant;
+CREATE TABLE t1 (c1 INT) ENGINE=InnoDB;
+SET GLOBAL innodb_default_row_format = @save_format;
+TRUNCATE TABLE t1;
+SHOW TABLE STATUS LIKE 't1';
+Name	Engine	Version	Row_format	Rows	Avg_row_length	Data_length	Max_data_length	Index_length	Data_free	Auto_increment	Create_time	Update_time	Check_time	Collation	Checksum	Create_options	Comment
+t1	InnoDB	#	Redundant	#	#	#	#	#	#	NULL	#	NULL	NULL	latin1_swedish_ci	NULL		
 DROP TABLE t1;

mysql-test/suite/innodb/t/default_row_format_create.test

@@ -22,7 +22,25 @@ SHOW TABLE STATUS LIKE 't1';
 DROP TABLE t1;
 
 CREATE TABLE t1(c1 TEXT,c2 BLOB) ENGINE=InnoDB
-ROW_FORMAT=COMPRESSED KEY_BLOCK_SIZE=1;
+ROW_FORMAT=COMPRESSED;
+--replace_column 3 # 5 # 6 # 7 # 8 # 9 # 10 # 12 #
+SHOW TABLE STATUS LIKE 't1';
+TRUNCATE TABLE t1;
+--replace_column 3 # 5 # 6 # 7 # 8 # 9 # 10 # 12 #
+SHOW TABLE STATUS LIKE 't1';
+DROP TABLE t1;
+
+--error 0,ER_CANT_CREATE_TABLE
+CREATE TABLE t1 (c1 INT) ENGINE=InnoDB page_compressed=1;
+--replace_column 3 # 5 # 6 # 7 # 8 # 9 # 10 # 12 #
+SHOW TABLE STATUS LIKE 't1';
+DROP TABLE IF EXISTS t1;
+
+SET @save_format = @@GLOBAL.innodb_default_row_format;
+SET GLOBAL innodb_default_row_format = redundant;
+CREATE TABLE t1 (c1 INT) ENGINE=InnoDB;
+SET GLOBAL innodb_default_row_format = @save_format;
+TRUNCATE TABLE t1;
 --replace_column 3 # 5 # 6 # 7 # 8 # 9 # 10 # 12 #
 SHOW TABLE STATUS LIKE 't1';
 DROP TABLE t1;

storage/innobase/handler/ha_innodb.cc

@@ -5580,6 +5580,25 @@ normalize_table_name_c_low(
 	}
 }
 
+create_table_info_t::create_table_info_t(
+	THD*		thd,
+	TABLE*		form,
+	HA_CREATE_INFO*	create_info,
+	char*		table_name,
+	char*		remote_path,
+	bool		file_per_table,
+	trx_t*		trx)
+	: m_thd(thd),
+	  m_trx(trx),
+	  m_form(form),
+	  m_default_row_format(innodb_default_row_format),
+	  m_create_info(create_info),
+	  m_table_name(table_name), m_drop_before_rollback(false),
+	  m_remote_path(remote_path),
+	  m_innodb_file_per_table(file_per_table)
+{
+}
+
 /** Normalizes a table name string.
 A normalized name consists of the database name catenated to '/'
 and table name. For example: test/mytable.
@@ -11698,7 +11717,7 @@ Check engine specific table options not handled by SQL-parser.
 const char*
 create_table_info_t::check_table_options()
 {
-	enum row_type	row_format = m_form->s->row_type;
+	enum row_type row_format = m_create_info->row_type;
 	ha_table_option_struct *options= m_form->s->option_struct;
 	fil_encryption_t encrypt = (fil_encryption_t)options->encryption;
 	bool should_encrypt = (encrypt == FIL_ENCRYPTION_ON);
@@ -11745,7 +11764,16 @@ create_table_info_t::check_table_options()
 			return "PAGE_COMPRESSED";
 		}
 
-		if (row_format == ROW_TYPE_REDUNDANT) {
+		switch (row_format) {
+		default:
+			break;
+		case ROW_TYPE_DEFAULT:
+			if (m_default_row_format
+			    != DEFAULT_ROW_FORMAT_REDUNDANT) {
+				break;
+			}
+			/* fall through */
+		case ROW_TYPE_REDUNDANT:
 			push_warning(
 				m_thd, Sql_condition::WARN_LEVEL_WARN,
 				HA_WRONG_CREATE_OPTION,
@@ -11956,9 +11984,9 @@ create_table_info_t::parse_table_name(
 
 /** Determine InnoDB table flags.
 If strict_mode=OFF, this will adjust the flags to what should be assumed.
-@retval true if successful, false if error */
-bool
-create_table_info_t::innobase_table_flags()
+@retval true on success
+@retval false on error */
+bool create_table_info_t::innobase_table_flags()
 {
 	DBUG_ENTER("innobase_table_flags");
 
@@ -11966,7 +11994,7 @@ create_table_info_t::innobase_table_flags()
 	ulint		zip_ssize = 0;
 	enum row_type	row_type;
 	rec_format_t	innodb_row_format =
-		get_row_format(innodb_default_row_format);
+		get_row_format(m_default_row_format);
 	const bool	is_temp
 		= m_create_info->options & HA_LEX_CREATE_TMP_TABLE;
 	bool		zip_allowed
@@ -12080,7 +12108,7 @@ create_table_info_t::innobase_table_flags()
 		}
 	}
 
-	row_type = m_form->s->row_type;
+	row_type = m_create_info->row_type;
 
 	if (zip_ssize && zip_allowed) {
 		/* if ROW_FORMAT is set to default,
@@ -12420,8 +12448,6 @@ create_table_info_t::initialize()
 		DBUG_RETURN(HA_ERR_WRONG_INDEX);
 	}
 
-	ut_ad(m_form->s->row_type == m_create_info->row_type);
-
 	/* Get the transaction associated with the current thd, or create one
 	if not yet created */
 
@@ -12465,8 +12491,6 @@ int create_table_info_t::prepare_create_table(const char* name, bool strict)
 	ut_ad(m_thd != NULL);
 	ut_ad(m_create_info != NULL);
 
-	ut_ad(m_form->s->row_type == m_create_info->row_type);
-
 	set_tablespace_type(false);
 
 	normalize_table_name(m_table_name, name);
@@ -13530,6 +13554,21 @@ int ha_innobase::truncate()
 		trx_rollback_for_mysql(trx);
 		row_mysql_unlock_data_dictionary(trx);
 	} else {
+		switch (dict_tf_get_rec_format(ib_table->flags)) {
+		case REC_FORMAT_REDUNDANT:
+			info.row_type = ROW_TYPE_REDUNDANT;
+			break;
+		case REC_FORMAT_COMPACT:
+			info.row_type = ROW_TYPE_COMPACT;
+			break;
+		case REC_FORMAT_COMPRESSED:
+			info.row_type = ROW_TYPE_COMPRESSED;
+			break;
+		case REC_FORMAT_DYNAMIC:
+			info.row_type = ROW_TYPE_DYNAMIC;
+			break;
+		}
+
 		err = create(name, table, &info,
 			     ib_table->is_temporary()
 			     || dict_table_is_file_per_table(ib_table), trx);

storage/innobase/handler/ha_innodb.h

@@ -641,15 +641,7 @@ class create_table_info_t
 		char*		table_name,
 		char*		remote_path,
 		bool		file_per_table,
-		trx_t*		trx = NULL)
-	:m_thd(thd),
-	m_trx(trx),
-	m_form(form),
-	m_create_info(create_info),
-	m_table_name(table_name), m_drop_before_rollback(false),
-	m_remote_path(remote_path),
-	m_innodb_file_per_table(file_per_table)
-	{}
+		trx_t*		trx = NULL);
 
 	/** Initialize the object. */
 	int initialize();
@@ -758,6 +750,9 @@ class create_table_info_t
 	/** Information on table columns and indexes. */
 	const TABLE*	m_form;
 
+	/** Value of innodb_default_row_format */
+	const ulong	m_default_row_format;
+
 	/** Create options. */
 	HA_CREATE_INFO*	m_create_info;
 

storage/innobase/include/dict0dict.ic

@@ -727,31 +727,30 @@ dict_tf_set(
 	ulint		page_compression_level,
 	ulint		not_used)
 {
+	*flags = use_data_dir ? 1 << DICT_TF_POS_DATA_DIR : 0;
+
 	switch (format) {
 	case REC_FORMAT_REDUNDANT:
-		*flags = 0;
 		ut_ad(zip_ssize == 0);
-		break;
+		/* no other options are allowed */
+		ut_ad(!page_compressed);
+		return;
 	case REC_FORMAT_COMPACT:
-		*flags = DICT_TF_COMPACT;
+		*flags |= DICT_TF_COMPACT;
 		ut_ad(zip_ssize == 0);
 		break;
 	case REC_FORMAT_COMPRESSED:
-		*flags = DICT_TF_COMPACT
+		*flags |= DICT_TF_COMPACT
 			| (1 << DICT_TF_POS_ATOMIC_BLOBS)
 			| (zip_ssize << DICT_TF_POS_ZIP_SSIZE);
 		break;
 	case REC_FORMAT_DYNAMIC:
-		*flags = DICT_TF_COMPACT
+		*flags |= DICT_TF_COMPACT
 			| (1 << DICT_TF_POS_ATOMIC_BLOBS);
 		ut_ad(zip_ssize == 0);
 		break;
 	}
 
-	if (use_data_dir) {
-		*flags |= (1 << DICT_TF_POS_DATA_DIR);
-	}
-
 	if (page_compressed) {
 		*flags |= (1 << DICT_TF_POS_ATOMIC_BLOBS)
 		       | (1 << DICT_TF_POS_PAGE_COMPRESSION)