MDEV-11942 BLACKHOLE is no longer active in 10.1 by default, mysql_upgrade not...

MDEV-11942 BLACKHOLE is no longer active in 10.1 by default, mysql_upgrade not handling the situation

fix the patch. add tests

client/mysql_upgrade.c

@@ -37,7 +37,7 @@
 #endif
 
 static int phase = 0;
-static int phases_total = 6;
+static const int phases_total = 7;
 static char mysql_path[FN_REFLEN];
 static char mysqlcheck_path[FN_REFLEN];
 
@@ -69,6 +69,8 @@ static char **defaults_argv;
 
 static my_bool not_used; /* Can't use GET_BOOL without a value pointer */
 
+char upgrade_from_version[sizeof("10.20.456-MariaDB")+1];
+
 static my_bool opt_write_binlog;
 
 #define OPT_SILENT OPT_MAX_CLIENT_OPTION
@@ -675,7 +677,6 @@ static int upgrade_already_done(void)
 {
   FILE *in;
   char upgrade_info_file[FN_REFLEN]= {0};
-  char buf[sizeof(MYSQL_SERVER_VERSION)+1];
 
   if (get_upgrade_info_file_name(upgrade_info_file))
     return 0; /* Could not get filename => not sure */
@@ -683,15 +684,15 @@ static int upgrade_already_done(void)
   if (!(in= my_fopen(upgrade_info_file, O_RDONLY, MYF(0))))
     return 0; /* Could not open file => not sure */
 
-  bzero(buf, sizeof(buf));
-  if (!fgets(buf, sizeof(buf), in))
+  bzero(upgrade_from_version, sizeof(upgrade_from_version));
+  if (!fgets(upgrade_from_version, sizeof(upgrade_from_version), in))
   {
     /* Ignore, will be detected by strncmp() below */
   }
 
   my_fclose(in, MYF(0));
 
-  return (strncmp(buf, MYSQL_SERVER_VERSION,
+  return (strncmp(upgrade_from_version, MYSQL_SERVER_VERSION,
                   sizeof(MYSQL_SERVER_VERSION)-1)==0);
 }
 
@@ -924,24 +925,48 @@ static void print_line(char* line)
   fputc('\n', stderr);
 }
 
+static my_bool from_before_10_1()
+{
+  my_bool ret= TRUE;
+  DYNAMIC_STRING ds_events_struct;
+
+  if (upgrade_from_version[0])
+  {
+    return upgrade_from_version[1] == '.' ||
+           strncmp(upgrade_from_version, "10.1.", 5) < 0;
+  }
+
+  if (init_dynamic_string(&ds_events_struct, NULL, 2048, 2048))
+    die("Out of memory");
+
+  if (run_query("show create table mysql.user", &ds_events_struct, FALSE) ||
+      strstr(ds_events_struct.str, "default_role") != NULL)
+    ret= FALSE;
+  else
+    verbose("Upgrading from a version before MariaDB-10.1");
+
+  dynstr_free(&ds_events_struct);
+  return ret;
+}
+
 
 /*
   Check for entries with "Unknown storage engine" in I_S.TABLES,
   try to load plugins for these tables if available (MDEV-11942)
 */
-static int run_mysqlcheck_engines(void)
+static int install_used_engines(void)
 {
-  DYNAMIC_STRING ds_query;
+  char buf[512];
   DYNAMIC_STRING ds_result;
+  const char *query = "SELECT DISTINCT LOWER(engine) FROM information_schema.tables"
+                      " WHERE table_comment LIKE 'Unknown storage engine%'";
 
-  /* Trying to identify existing tables with unknown storage engine
-     Does not work with all engine types yet, and doesn't produce
-     results for BLACKHOLE without the dummy "WHERE row_format IS NULL"
-     condition yet. See MDEV-11943 */
-  const char *query = "SELECT DISTINCT LOWER(REPLACE(REPLACE(table_comment, 'Unknown storage engine ', ''), '\\'', '')) AS engine FROM information_schema.tables WHERE row_format IS NULL AND table_comment LIKE 'Unknown storage engine%'";
-
-  if (init_dynamic_string(&ds_query, "", 512, 512))
-    die("Out of memory");
+  if (opt_systables_only || !from_before_10_1())
+  {
+    verbose("Phase %d/%d: Installing used storage engines... Skipped", ++phase, phases_total);
+    return 0;
+  }
+  verbose("Phase %d/%d: Installing used storage engines", ++phase, phases_total);
 
   if (init_dynamic_string(&ds_result, "", 512, 512))
     die("Out of memory");
@@ -950,24 +975,28 @@ static int run_mysqlcheck_engines(void)
 
   run_query(query, &ds_result, TRUE);
 
+  if (ds_result.length)
   {
-    char *line= ds_result.str;
-    if (line && *line) {
-      do
-      {
-        line[strlen(line)-1]='\0';
-        verbose("installing missing plugin for '%s' storage engine", line);
+    char *line= ds_result.str, *next=get_line(line);
+    do
+    {
+      if (next[-1] == '\n')
+        next[-1]=0;
 
-        dynstr_set(&ds_query, "INSTALL SONAME 'ha_");
-        dynstr_append(&ds_query, line); // we simply assume SONAME=ha_ENGINENAME
-        dynstr_append(&ds_query, "'");
+      verbose("installing plugin for '%s' storage engine", line);
 
-        if (run_query(ds_query.str, NULL, TRUE)) {
-          fprintf(stderr, "... can't install plugin 'ha_%s'\n", line);
-        }
-      } while ((line= get_line(line)) && *line);
-    }
+      // we simply assume soname=ha_enginename
+      strxnmov(buf, sizeof(buf)-1, "install soname 'ha_", line, "'", NULL);
+
+
+      if (run_query(buf, NULL, TRUE))
+        fprintf(stderr, "... can't %s\n", buf);
+      line=next;
+      next=get_line(line);
+    } while (*line);
   }
+  dynstr_free(&ds_result);
+  return 0;
 }
 
 
@@ -1176,8 +1205,8 @@ int main(int argc, char **argv)
   /*
     Run "mysqlcheck" and "mysql_fix_privilege_tables.sql"
   */
-  if (run_mysqlcheck_engines() ||
-      run_mysqlcheck_upgrade(TRUE) ||
+  if (run_mysqlcheck_upgrade(TRUE) ||
+      install_used_engines() ||
       run_mysqlcheck_views() ||
       run_sql_fix_privilege_tables() ||
       run_mysqlcheck_fixnames() ||
@@ -1200,4 +1229,3 @@ int main(int argc, char **argv)
   my_end(my_end_arg);
   exit(0);
 }
-

mysql-test/r/ctype_upgrade.result

@@ -227,7 +227,7 @@ DROP TABLE mysql050614_xxx_croatian_ci;
 # Checking mysql_upgrade
 #
 # Running mysql_upgrade
-Phase 1/6: Checking and upgrading mysql database
+Phase 1/7: Checking and upgrading mysql database
 Processing databases
 mysql
 mysql.column_stats                                 OK
@@ -258,10 +258,11 @@ mysql.time_zone_name                               OK
 mysql.time_zone_transition                         OK
 mysql.time_zone_transition_type                    OK
 mysql.user                                         OK
-Phase 2/6: Fixing views
-Phase 3/6: Running 'mysql_fix_privilege_tables'
-Phase 4/6: Fixing table and database names
-Phase 5/6: Checking and upgrading tables
+Phase 2/7: Installing used storage engines... Skipped
+Phase 3/7: Fixing views
+Phase 4/7: Running 'mysql_fix_privilege_tables'
+Phase 5/7: Fixing table and database names
+Phase 6/7: Checking and upgrading tables
 Processing databases
 information_schema
 mtr
@@ -280,11 +281,11 @@ test.maria050313_ucs2_croatian_ci_def              OK
 test.maria050313_utf8_croatian_ci                  OK
 test.maria050533_xxx_croatian_ci                   OK
 test.maria100004_xxx_croatian_ci                   OK
-Phase 6/6: Running 'FLUSH PRIVILEGES'
+Phase 7/7: Running 'FLUSH PRIVILEGES'
 OK
 # Running mysql_upgrade for the second time
 # This should report OK for all tables
-Phase 1/6: Checking and upgrading mysql database
+Phase 1/7: Checking and upgrading mysql database
 Processing databases
 mysql
 mysql.column_stats                                 OK
@@ -315,10 +316,11 @@ mysql.time_zone_name                               OK
 mysql.time_zone_transition                         OK
 mysql.time_zone_transition_type                    OK
 mysql.user                                         OK
-Phase 2/6: Fixing views
-Phase 3/6: Running 'mysql_fix_privilege_tables'
-Phase 4/6: Fixing table and database names
-Phase 5/6: Checking and upgrading tables
+Phase 2/7: Installing used storage engines... Skipped
+Phase 3/7: Fixing views
+Phase 4/7: Running 'mysql_fix_privilege_tables'
+Phase 5/7: Fixing table and database names
+Phase 6/7: Checking and upgrading tables
 Processing databases
 information_schema
 mtr
@@ -331,7 +333,7 @@ test.maria050313_utf8_croatian_ci                  OK
 test.maria050533_xxx_croatian_ci                   OK
 test.maria100004_xxx_croatian_ci                   OK
 test.mysql050614_xxx_croatian_ci                   OK
-Phase 6/6: Running 'FLUSH PRIVILEGES'
+Phase 7/7: Running 'FLUSH PRIVILEGES'
 OK
 SHOW CREATE TABLE maria050313_ucs2_croatian_ci_def;
 Table	Create Table

mysql-test/r/log_tables_upgrade.result

@@ -11,7 +11,7 @@ Table	Op	Msg_type	Msg_text
 test.bug49823	repair	status	OK
 RENAME TABLE general_log TO renamed_general_log;
 RENAME TABLE test.bug49823 TO general_log;
-Phase 1/6: Checking and upgrading mysql database
+Phase 1/7: Checking and upgrading mysql database
 Processing databases
 mysql
 mysql.column_stats                                 OK
@@ -43,10 +43,11 @@ mysql.time_zone_name                               OK
 mysql.time_zone_transition                         OK
 mysql.time_zone_transition_type                    OK
 mysql.user                                         OK
-Phase 2/6: Fixing views
-Phase 3/6: Running 'mysql_fix_privilege_tables'
-Phase 4/6: Fixing table and database names
-Phase 5/6: Checking and upgrading tables
+Phase 2/7: Installing used storage engines... Skipped
+Phase 3/7: Fixing views
+Phase 4/7: Running 'mysql_fix_privilege_tables'
+Phase 5/7: Fixing table and database names
+Phase 6/7: Checking and upgrading tables
 Processing databases
 information_schema
 mtr
@@ -54,7 +55,7 @@ mtr.global_suppressions                            OK
 mtr.test_suppressions                              OK
 performance_schema
 test
-Phase 6/6: Running 'FLUSH PRIVILEGES'
+Phase 7/7: Running 'FLUSH PRIVILEGES'
 OK
 DROP TABLE general_log;
 RENAME TABLE renamed_general_log TO general_log;

mysql-test/r/mysql_upgrade-6984.result

 update mysql.user set password=password("foo") where user='root';
-Phase 1/6: Checking and upgrading mysql database
+Phase 1/7: Checking and upgrading mysql database
 Processing databases
 mysql
 mysql.column_stats                                 OK
@@ -42,10 +42,11 @@ error    : Corrupt
 mysql.innodb_table_stats
 Error    : Unknown storage engine 'InnoDB'
 error    : Corrupt
-Phase 2/6: Fixing views
-Phase 3/6: Running 'mysql_fix_privilege_tables'
-Phase 4/6: Fixing table and database names
-Phase 5/6: Checking and upgrading tables
+Phase 2/7: Installing used storage engines... Skipped
+Phase 3/7: Fixing views
+Phase 4/7: Running 'mysql_fix_privilege_tables'
+Phase 5/7: Fixing table and database names
+Phase 6/7: Checking and upgrading tables
 Processing databases
 information_schema
 mtr
@@ -53,7 +54,7 @@ mtr.global_suppressions                            OK
 mtr.test_suppressions                              OK
 performance_schema
 test
-Phase 6/6: Running 'FLUSH PRIVILEGES'
+Phase 7/7: Running 'FLUSH PRIVILEGES'
 OK
 update mysql.user set password='' where user='root';
 flush privileges;

mysql-test/r/mysql_upgrade_no_innodb.result

 The --upgrade-system-tables option was used, user tables won't be touched.
-Phase 1/6: Checking and upgrading mysql database
+Phase 1/7: Checking and upgrading mysql database
 Processing databases
 mysql
 mysql.column_stats                                 OK
@@ -42,9 +42,10 @@ error    : Corrupt
 mysql.innodb_table_stats
 Error    : Unknown storage engine 'InnoDB'
 error    : Corrupt
-Phase 2/6: Fixing views... Skipped
-Phase 3/6: Running 'mysql_fix_privilege_tables'
-Phase 4/6: Fixing table and database names ... Skipped
-Phase 5/6: Checking and upgrading tables... Skipped
-Phase 6/6: Running 'FLUSH PRIVILEGES'
+Phase 2/7: Installing used storage engines... Skipped
+Phase 3/7: Fixing views... Skipped
+Phase 4/7: Running 'mysql_fix_privilege_tables'
+Phase 5/7: Fixing table and database names ... Skipped
+Phase 6/7: Checking and upgrading tables... Skipped
+Phase 7/7: Running 'FLUSH PRIVILEGES'
 OK

mysql-test/r/mysql_upgrade_ssl.result

 #
 # Bug#55672 mysql_upgrade dies with internal error 
 #
-Phase 1/6: Checking and upgrading mysql database
+Phase 1/7: Checking and upgrading mysql database
 Processing databases
 mysql
 mysql.column_stats                                 OK
@@ -32,10 +32,11 @@ mysql.time_zone_name                               OK
 mysql.time_zone_transition                         OK
 mysql.time_zone_transition_type                    OK
 mysql.user                                         OK
-Phase 2/6: Fixing views
-Phase 3/6: Running 'mysql_fix_privilege_tables'
-Phase 4/6: Fixing table and database names
-Phase 5/6: Checking and upgrading tables
+Phase 2/7: Installing used storage engines... Skipped
+Phase 3/7: Fixing views
+Phase 4/7: Running 'mysql_fix_privilege_tables'
+Phase 5/7: Fixing table and database names
+Phase 6/7: Checking and upgrading tables
 Processing databases
 information_schema
 mtr
@@ -43,5 +44,5 @@ mtr.global_suppressions                            OK
 mtr.test_suppressions                              OK
 performance_schema
 test
-Phase 6/6: Running 'FLUSH PRIVILEGES'
+Phase 7/7: Running 'FLUSH PRIVILEGES'
 OK

mysql-test/r/mysql_upgrade_view.result

@@ -63,7 +63,7 @@ test.v2	check	error	Upgrade required. Please do "REPAIR VIEW `v2`" or dump/reloa
 check view v3 for upgrade;
 Table	Op	Msg_type	Msg_text
 test.v3	check	error	Upgrade required. Please do "REPAIR VIEW `v3`" or dump/reload to fix it!
-Phase 1/6: Checking and upgrading mysql database
+Phase 1/7: Checking and upgrading mysql database
 Processing databases
 mysql
 mysql.column_stats                                 OK
@@ -106,14 +106,15 @@ error    : Corrupt
 mysql.innodb_table_stats
 Error    : Unknown storage engine 'InnoDB'
 error    : Corrupt
-Phase 2/6: Fixing views
+Phase 2/7: Installing used storage engines... Skipped
+Phase 3/7: Fixing views
 test.v1                                            OK
 test.v1badcheck                                    OK
 test.v2                                            OK
 test.v3                                            OK
-Phase 3/6: Running 'mysql_fix_privilege_tables'
-Phase 4/6: Fixing table and database names
-Phase 5/6: Checking and upgrading tables
+Phase 4/7: Running 'mysql_fix_privilege_tables'
+Phase 5/7: Fixing table and database names
+Phase 6/7: Checking and upgrading tables
 Processing databases
 information_schema
 mtr
@@ -123,7 +124,7 @@ performance_schema
 test
 test.kv                                            OK
 test.t1                                            OK
-Phase 6/6: Running 'FLUSH PRIVILEGES'
+Phase 7/7: Running 'FLUSH PRIVILEGES'
 OK
 show create view v1;
 View	Create View	character_set_client	collation_connection
@@ -205,7 +206,7 @@ show create view v4;
 View	Create View	character_set_client	collation_connection
 v4	CREATE ALGORITHM=TEMPTABLE DEFINER=`root`@`localhost` SQL SECURITY DEFINER VIEW `v4` AS select `t1`.`a` AS `a` from `t1`	latin1	latin1_swedish_ci
 MySQL upgrade detected
-Phase 1/6: Checking and upgrading mysql database
+Phase 1/7: Checking and upgrading mysql database
 Processing databases
 mysql
 mysql.column_stats                                 OK
@@ -249,14 +250,15 @@ error    : Corrupt
 mysql.innodb_table_stats
 Error    : Unknown storage engine 'InnoDB'
 error    : Corrupt
-Phase 2/6: Fixing views from mysql
+Phase 2/7: Installing used storage engines... Skipped
+Phase 3/7: Fixing views from mysql
 test.v1                                            OK
 test.v2                                            OK
 test.v3                                            OK
 test.v4                                            OK
-Phase 3/6: Running 'mysql_fix_privilege_tables'
-Phase 4/6: Fixing table and database names
-Phase 5/6: Checking and upgrading tables
+Phase 4/7: Running 'mysql_fix_privilege_tables'
+Phase 5/7: Fixing table and database names
+Phase 6/7: Checking and upgrading tables
 Processing databases
 information_schema
 mtr
@@ -266,7 +268,7 @@ performance_schema
 test
 test.kv                                            OK
 test.t1                                            OK
-Phase 6/6: Running 'FLUSH PRIVILEGES'
+Phase 7/7: Running 'FLUSH PRIVILEGES'
 OK
 flush tables;
 show create view v1;
@@ -323,7 +325,7 @@ rename table mysql.event to mysql.ev_bk;
 flush tables;
 The --upgrade-system-tables option was used, user tables won't be touched.
 MySQL upgrade detected
-Phase 1/6: Checking and upgrading mysql database
+Phase 1/7: Checking and upgrading mysql database
 Processing databases
 mysql
 mysql.column_stats                                 OK
@@ -367,14 +369,15 @@ error    : Corrupt
 mysql.innodb_table_stats
 Error    : Unknown storage engine 'InnoDB'
 error    : Corrupt
-Phase 2/6: Fixing views from mysql
+Phase 2/7: Installing used storage engines... Skipped
+Phase 3/7: Fixing views from mysql
 test.v1                                            OK
 test.v2                                            OK
 test.v3                                            OK
-Phase 3/6: Running 'mysql_fix_privilege_tables'
-Phase 4/6: Fixing table and database names ... Skipped
-Phase 5/6: Checking and upgrading tables... Skipped
-Phase 6/6: Running 'FLUSH PRIVILEGES'
+Phase 4/7: Running 'mysql_fix_privilege_tables'
+Phase 5/7: Fixing table and database names ... Skipped
+Phase 6/7: Checking and upgrading tables... Skipped
+Phase 7/7: Running 'FLUSH PRIVILEGES'
 OK
 drop table mysql.event;
 rename table mysql.ev_bk to mysql.event;

mysql-test/t/mysql_upgrade_noengine.test

+#
+# MDEV-11942 BLACKHOLE is no longer active in 10.1 by default, mysql_upgrade not handling the situation
+#
+source include/have_innodb.inc;
+source include/not_embedded.inc;
+
+if (!$HA_BLACKHOLE_SO) {
+  skip Need blackhole plugin;
+}
+if (!$HA_ARCHIVE_SO) {
+  skip Need Archive plugin;
+}
+
+let $datadir= `select @@datadir`;
+
+install soname 'ha_blackhole';
+install soname 'ha_archive';
+
+vertical_results;
+create table t1 (a int) engine=blackhole;
+create table t2 (a int) engine=archive;
+select table_catalog, table_schema, table_name, table_type, engine, row_format, table_rows, data_length, table_comment from information_schema.tables where table_schema='test';
+
+flush tables;
+uninstall plugin blackhole;
+uninstall plugin archive;
+select table_catalog, table_schema, table_name, table_type, engine, row_format, table_rows, data_length, table_comment from information_schema.tables where table_schema='test';
+
+# upgrade from 10.1 - engines aren't enabled
+exec $MYSQL_UPGRADE 2>&1;
+select table_catalog, table_schema, table_name, table_type, engine, row_format, table_rows, data_length, table_comment from information_schema.tables where table_schema='test';
+
+# pretend it's an upgrade from 10.0
+alter table mysql.user drop column default_role, drop column max_statement_time;
+
+# but mysql_upgrade_info tells otherwise
+remove_file $datadir/mysql_upgrade_info;
+write_file $datadir/mysql_upgrade_info;
+10.1.10-MariaDB
+EOF
+
+# still upgrade from 10.1
+exec $MYSQL_UPGRADE 2>&1;
+select table_catalog, table_schema, table_name, table_type, engine, row_format, table_rows, data_length, table_comment from information_schema.tables where table_schema='test';
+
+alter table mysql.user drop column default_role, drop column max_statement_time;
+remove_file $datadir/mysql_upgrade_info;
+
+# upgrade from 10.0 - engines are enabled
+exec $MYSQL_UPGRADE 2>&1;
+select table_catalog, table_schema, table_name, table_type, engine, row_format, table_rows, data_length, table_comment from information_schema.tables where table_schema='test';
+
+drop table t1, t2;
+
+uninstall plugin blackhole;
+uninstall plugin archive;