MDEV-14396 Assertion failed in create_option_need_rebuild

Relax a too strict debug assertion, and add a test.

mysql-test/suite/innodb/r/instant_alter,32k.rdiff

 --- instant_alter.result
 +++ instant_alter,32k.result
-@@ -1,7 +1,7 @@
- #
+@@ -2,7 +2,7 @@
  # MDEV-11369: Instant ADD COLUMN for InnoDB
  #
--CREATE TABLE t(a INT UNIQUE)ENGINE=InnoDB ROW_FORMAT=COMPRESSED;
-+CREATE TABLE t(a INT UNIQUE)ENGINE=InnoDB ROW_FORMAT=DYNAMIC;
- INSERT INTO t VALUES(1);
+ CREATE TABLE t(a INT UNIQUE)ENGINE=InnoDB ROW_FORMAT=COMPACT;
+-ALTER TABLE t ADD e INT, ROW_FORMAT=COMPRESSED;
++ALTER TABLE t ADD e INT, ROW_FORMAT=DYNAMIC;
+ INSERT INTO t SET a=1;
  SET @old_instant=
  (SELECT variable_value FROM information_schema.global_status
-@@ -31,12 +31,12 @@
+@@ -33,17 +33,17 @@
    `c` int(11) NOT NULL,
    `d` int(11) NOT NULL,
    UNIQUE KEY `a` (`a`)
@@ -20,11 +20,17 @@
  WHERE variable_name = 'innodb_instant_alter_column';
  instants
 -0
++1
+ SELECT variable_value-@old_instant instants
+ FROM information_schema.global_status
+ WHERE variable_name = 'innodb_instant_alter_column';
+ instants
+-0
 +1
  DROP TABLE t;
  connect analyze, localhost, root;
  connection default;
-@@ -355,7 +355,7 @@
+@@ -374,7 +374,7 @@
  SELECT clust_index_size FROM INFORMATION_SCHEMA.INNODB_SYS_TABLESTATS
  WHERE name = 'test/big';
  clust_index_size
@@ -33,7 +39,7 @@
  connection default;
  ALTER TABLE big ADD COLUMN
  (d1 INT DEFAULT 0, d2 VARCHAR(20) DEFAULT 'abcde',
-@@ -378,7 +378,7 @@
+@@ -397,7 +397,7 @@
  SELECT clust_index_size FROM INFORMATION_SCHEMA.INNODB_SYS_TABLESTATS
  WHERE name = 'test/big';
  clust_index_size
@@ -42,7 +48,7 @@
  connection default;
  ROLLBACK;
  CHECKSUM TABLE big;
-@@ -391,7 +391,7 @@
+@@ -410,7 +410,7 @@
  SELECT clust_index_size FROM INFORMATION_SCHEMA.INNODB_SYS_TABLESTATS
  WHERE name = 'test/big';
  clust_index_size
@@ -51,7 +57,7 @@
  connection default;
  InnoDB		0 transactions not purged
  DROP TABLE t1,t2,t3,big;
-@@ -703,7 +703,7 @@
+@@ -734,7 +734,7 @@
  SELECT clust_index_size FROM INFORMATION_SCHEMA.INNODB_SYS_TABLESTATS
  WHERE name = 'test/big';
  clust_index_size
@@ -60,7 +66,7 @@
  connection default;
  ALTER TABLE big ADD COLUMN
  (d1 INT DEFAULT 0, d2 VARCHAR(20) DEFAULT 'abcde',
-@@ -726,7 +726,7 @@
+@@ -757,7 +757,7 @@
  SELECT clust_index_size FROM INFORMATION_SCHEMA.INNODB_SYS_TABLESTATS
  WHERE name = 'test/big';
  clust_index_size
@@ -69,7 +75,7 @@
  connection default;
  ROLLBACK;
  CHECKSUM TABLE big;
-@@ -739,7 +739,7 @@
+@@ -770,7 +770,7 @@
  SELECT clust_index_size FROM INFORMATION_SCHEMA.INNODB_SYS_TABLESTATS
  WHERE name = 'test/big';
  clust_index_size
@@ -78,7 +84,7 @@
  connection default;
  InnoDB		0 transactions not purged
  DROP TABLE t1,t2,t3,big;
-@@ -1051,7 +1051,7 @@
+@@ -1094,7 +1094,7 @@
  SELECT clust_index_size FROM INFORMATION_SCHEMA.INNODB_SYS_TABLESTATS
  WHERE name = 'test/big';
  clust_index_size
@@ -87,7 +93,7 @@
  connection default;
  ALTER TABLE big ADD COLUMN
  (d1 INT DEFAULT 0, d2 VARCHAR(20) DEFAULT 'abcde',
-@@ -1074,7 +1074,7 @@
+@@ -1117,7 +1117,7 @@
  SELECT clust_index_size FROM INFORMATION_SCHEMA.INNODB_SYS_TABLESTATS
  WHERE name = 'test/big';
  clust_index_size
@@ -96,7 +102,7 @@
  connection default;
  ROLLBACK;
  CHECKSUM TABLE big;
-@@ -1087,7 +1087,7 @@
+@@ -1130,7 +1130,7 @@
  SELECT clust_index_size FROM INFORMATION_SCHEMA.INNODB_SYS_TABLESTATS
  WHERE name = 'test/big';
  clust_index_size

mysql-test/suite/innodb/r/instant_alter,64k.rdiff

 --- instant_alter.result
 +++ instant_alter,64k.result
-@@ -1,7 +1,7 @@
- #
+@@ -2,7 +2,7 @@
  # MDEV-11369: Instant ADD COLUMN for InnoDB
  #
--CREATE TABLE t(a INT UNIQUE)ENGINE=InnoDB ROW_FORMAT=COMPRESSED;
-+CREATE TABLE t(a INT UNIQUE)ENGINE=InnoDB ROW_FORMAT=DYNAMIC;
- INSERT INTO t VALUES(1);
+ CREATE TABLE t(a INT UNIQUE)ENGINE=InnoDB ROW_FORMAT=COMPACT;
+-ALTER TABLE t ADD e INT, ROW_FORMAT=COMPRESSED;
++ALTER TABLE t ADD e INT, ROW_FORMAT=DYNAMIC;
+ INSERT INTO t SET a=1;
  SET @old_instant=
  (SELECT variable_value FROM information_schema.global_status
-@@ -31,12 +31,12 @@
+@@ -33,17 +33,17 @@
    `c` int(11) NOT NULL,
    `d` int(11) NOT NULL,
    UNIQUE KEY `a` (`a`)
@@ -20,11 +20,17 @@
  WHERE variable_name = 'innodb_instant_alter_column';
  instants
 -0
++1
+ SELECT variable_value-@old_instant instants
+ FROM information_schema.global_status
+ WHERE variable_name = 'innodb_instant_alter_column';
+ instants
+-0
 +1
  DROP TABLE t;
  connect analyze, localhost, root;
  connection default;
-@@ -355,7 +355,7 @@
+@@ -374,7 +374,7 @@
  SELECT clust_index_size FROM INFORMATION_SCHEMA.INNODB_SYS_TABLESTATS
  WHERE name = 'test/big';
  clust_index_size
@@ -33,7 +39,7 @@
  connection default;
  ALTER TABLE big ADD COLUMN
  (d1 INT DEFAULT 0, d2 VARCHAR(20) DEFAULT 'abcde',
-@@ -378,7 +378,7 @@
+@@ -397,7 +397,7 @@
  SELECT clust_index_size FROM INFORMATION_SCHEMA.INNODB_SYS_TABLESTATS
  WHERE name = 'test/big';
  clust_index_size
@@ -42,7 +48,7 @@
  connection default;
  ROLLBACK;
  CHECKSUM TABLE big;
-@@ -391,7 +391,7 @@
+@@ -410,7 +410,7 @@
  SELECT clust_index_size FROM INFORMATION_SCHEMA.INNODB_SYS_TABLESTATS
  WHERE name = 'test/big';
  clust_index_size
@@ -51,7 +57,7 @@
  connection default;
  InnoDB		0 transactions not purged
  DROP TABLE t1,t2,t3,big;
-@@ -703,7 +703,7 @@
+@@ -734,7 +734,7 @@
  SELECT clust_index_size FROM INFORMATION_SCHEMA.INNODB_SYS_TABLESTATS
  WHERE name = 'test/big';
  clust_index_size
@@ -60,7 +66,7 @@
  connection default;
  ALTER TABLE big ADD COLUMN
  (d1 INT DEFAULT 0, d2 VARCHAR(20) DEFAULT 'abcde',
-@@ -726,7 +726,7 @@
+@@ -757,7 +757,7 @@
  SELECT clust_index_size FROM INFORMATION_SCHEMA.INNODB_SYS_TABLESTATS
  WHERE name = 'test/big';
  clust_index_size
@@ -69,7 +75,7 @@
  connection default;
  ROLLBACK;
  CHECKSUM TABLE big;
-@@ -739,7 +739,7 @@
+@@ -770,7 +770,7 @@
  SELECT clust_index_size FROM INFORMATION_SCHEMA.INNODB_SYS_TABLESTATS
  WHERE name = 'test/big';
  clust_index_size
@@ -78,7 +84,7 @@
  connection default;
  InnoDB		0 transactions not purged
  DROP TABLE t1,t2,t3,big;
-@@ -1051,7 +1051,7 @@
+@@ -1094,7 +1094,7 @@
  SELECT clust_index_size FROM INFORMATION_SCHEMA.INNODB_SYS_TABLESTATS
  WHERE name = 'test/big';
  clust_index_size
@@ -87,7 +93,7 @@
  connection default;
  ALTER TABLE big ADD COLUMN
  (d1 INT DEFAULT 0, d2 VARCHAR(20) DEFAULT 'abcde',
-@@ -1074,7 +1074,7 @@
+@@ -1117,7 +1117,7 @@
  SELECT clust_index_size FROM INFORMATION_SCHEMA.INNODB_SYS_TABLESTATS
  WHERE name = 'test/big';
  clust_index_size
@@ -96,7 +102,7 @@
  connection default;
  ROLLBACK;
  CHECKSUM TABLE big;
-@@ -1087,7 +1087,7 @@
+@@ -1130,7 +1130,7 @@
  SELECT clust_index_size FROM INFORMATION_SCHEMA.INNODB_SYS_TABLESTATS
  WHERE name = 'test/big';
  clust_index_size

mysql-test/suite/innodb/t/instant_alter.test

@@ -6,8 +6,9 @@
 
 let $format= `SELECT CASE WHEN @@GLOBAL.innodb_page_size>16384
 THEN 'DYNAMIC' ELSE 'COMPRESSED' END`;
-eval CREATE TABLE t(a INT UNIQUE)ENGINE=InnoDB ROW_FORMAT=$format;
-INSERT INTO t VALUES(1);
+CREATE TABLE t(a INT UNIQUE)ENGINE=InnoDB ROW_FORMAT=COMPACT;
+eval ALTER TABLE t ADD e INT, ROW_FORMAT=$format;
+INSERT INTO t SET a=1;
 SET @old_instant=
 (SELECT variable_value FROM information_schema.global_status
 WHERE variable_name = 'innodb_instant_alter_column');
@@ -26,6 +27,9 @@ SHOW CREATE TABLE t;
 SELECT variable_value-@old_instant instants
 FROM information_schema.global_status
 WHERE variable_name = 'innodb_instant_alter_column';
+SELECT variable_value-@old_instant instants
+FROM information_schema.global_status
+WHERE variable_name = 'innodb_instant_alter_column';
 DROP TABLE t;
 
 connect analyze, localhost, root;

storage/innobase/handler/handler0alter.cc

@@ -473,8 +473,8 @@ static bool create_option_need_rebuild(
 	const Alter_inplace_info*	ha_alter_info,
 	const TABLE*			table)
 {
-	DBUG_ASSERT((ha_alter_info->handler_flags & ~INNOBASE_INPLACE_IGNORE)
-		    == Alter_inplace_info::CHANGE_CREATE_OPTION);
+	DBUG_ASSERT(ha_alter_info->handler_flags
+		    & Alter_inplace_info::CHANGE_CREATE_OPTION);
 
 	if (ha_alter_info->create_info->used_fields
 	    & (HA_CREATE_USED_ROW_FORMAT