MDEV-24093: Detect during mysql_upgrade if type_mysql_json.so is needed and load it

a. The change makes `mariadb-upgrade` detect if `MYSQL_JSON` data type is needed.
b. Install the data type if it's not installed.
c. Uninstalls the data type once finished.
d. Create `.opt` and `.inc` files `have_type_mysql_json` and adapt the
tests

Reviewed by: vicentiu@mariadb.org

client/mysql_upgrade.c

@@ -51,6 +51,7 @@ static my_bool upgrade_from_mysql;
 
 static DYNAMIC_STRING ds_args;
 static DYNAMIC_STRING conn_args;
+static DYNAMIC_STRING ds_plugin_data_types;
 
 static char *opt_password= 0;
 static char *opt_plugin_dir= 0, *opt_default_auth= 0;
@@ -187,6 +188,7 @@ static void free_used_memory(void)
 
   dynstr_free(&ds_args);
   dynstr_free(&conn_args);
+  dynstr_free(&ds_plugin_data_types);
   if (cnf_file_path)
     my_delete(cnf_file_path, MYF(MY_WME));
 }
@@ -965,6 +967,73 @@ static my_bool from_before_10_1()
 }
 
 
+static void uninstall_plugins(void)
+{
+  if (ds_plugin_data_types.length)
+  {
+    char *plugins= ds_plugin_data_types.str;
+    char *next= get_line(plugins);
+    char buff[512];
+    while(*plugins)
+    {
+      if (next[-1] == '\n')
+        next[-1]= 0;
+      verbose("uninstalling plugin for %s data type", plugins);
+      strxnmov(buff, sizeof(buff)-1, "UNINSTALL SONAME ", plugins,"", NULL);
+      run_query(buff, NULL, TRUE);
+      plugins= next;
+      next= get_line(next);
+    }
+  }
+}
+/**
+  @brief     Install plugins for missing data types
+  @details   Check for entries with "Unknown data type" in I_S.TABLES,
+             try to load plugins for these tables if available (MDEV-24093)
+
+  @return    Operation status
+    @retval  TRUE  - error
+    @retval  FALSE - success
+*/
+static int install_used_plugin_data_types(void)
+{
+  DYNAMIC_STRING ds_result;
+  const char *query = "SELECT table_comment FROM information_schema.tables"
+                      " WHERE table_comment LIKE 'Unknown data type: %'";
+  if (init_dynamic_string(&ds_result, "", 512, 512))
+    die("Out of memory");
+  run_query(query, &ds_result, TRUE);
+
+  if (ds_result.length)
+  {
+    char *line= ds_result.str;
+    char *next= get_line(line);
+    while(*line)
+    {
+      if (next[-1] == '\n')
+        next[-1]= 0;
+      if (strstr(line, "'MYSQL_JSON'"))
+      {
+        verbose("installing plugin for MYSQL_JSON data type");
+        if(!run_query("INSTALL SONAME 'type_mysql_json'", NULL, TRUE))
+        {
+          dynstr_append(&ds_plugin_data_types, "'type_mysql_json'");
+          dynstr_append(&ds_plugin_data_types, "\n");
+          break;
+        }
+        else
+        {
+          fprintf(stderr, "... can't %s\n", "INSTALL SONAME 'type_mysql_json'");
+          return 1;
+        }
+      }
+      line= next;
+      next= get_line(next);
+    }
+  }
+  dynstr_free(&ds_result);
+  return 0;
+}
 /*
   Check for entries with "Unknown storage engine" in I_S.TABLES,
   try to load plugins for these tables if available (MDEV-11942)
@@ -1218,7 +1287,8 @@ int main(int argc, char **argv)
   }
 
   if (init_dynamic_string(&ds_args, "", 512, 256) ||
-      init_dynamic_string(&conn_args, "", 512, 256))
+      init_dynamic_string(&conn_args, "", 512, 256) ||
+      init_dynamic_string(&ds_plugin_data_types, "", 512, 256))
     die("Out of memory");
 
   if (handle_options(&argc, &argv, my_long_options, get_one_option))
@@ -1281,6 +1351,7 @@ int main(int argc, char **argv)
   */
   if (run_mysqlcheck_upgrade(TRUE) ||
       install_used_engines() ||
+      install_used_plugin_data_types() ||
       run_mysqlcheck_views() ||
       run_sql_fix_privilege_tables() ||
       run_mysqlcheck_fixnames() ||
@@ -1288,6 +1359,7 @@ int main(int argc, char **argv)
       check_slave_repositories())
     die("Upgrade failed" );
 
+  uninstall_plugins();
   verbose("Phase %d/%d: Running 'FLUSH PRIVILEGES'", ++phase, phases_total);
   if (run_query("FLUSH PRIVILEGES", NULL, TRUE))
     die("Upgrade failed" );

mysql-test/include/have_type_mysql_json.inc

+if (!$TYPE_MYSQL_JSON_SO) {
+  skip Need MYSQL_JSON plugin;
+}

mysql-test/include/have_type_mysql_json.opt

+--loose-type_mysql_json
+--plugin-load-add=$TYPE_MYSQL_JSON_SO

mysql-test/main/mysql_json_mysql_upgrade.test

+--echo #
+--echo # MDEV-24093: Detect during mysql_upgrade if type_mysql_json.so 
+--echo # is needed and load it
+--echo #
+
+-- source include/have_utf8.inc
+-- source include/mysql_upgrade_preparation.inc
+
+SET NAMES utf8;
+
+let $MYSQLD_DATADIR= `select @@datadir`;
+
+--copy_file std_data/mysql_json/mysql_json_test.frm $MYSQLD_DATADIR/test/mysql_json_test.frm
+--copy_file std_data/mysql_json/mysql_json_test.MYI $MYSQLD_DATADIR/test/mysql_json_test.MYI
+--copy_file std_data/mysql_json/mysql_json_test.MYD $MYSQLD_DATADIR/test/mysql_json_test.MYD
+
+--copy_file std_data/mysql_json/mysql_json_test_big.frm $MYSQLD_DATADIR/test/mysql_json_test_big.frm
+--copy_file std_data/mysql_json/mysql_json_test_big.MYI $MYSQLD_DATADIR/test/mysql_json_test_big.MYI
+--copy_file std_data/mysql_json/mysql_json_test_big.MYD $MYSQLD_DATADIR/test/mysql_json_test_big.MYD
+
+--error ER_UNKNOWN_DATA_TYPE
+show create table mysql_json_test;
+
+--exec $MYSQL_UPGRADE --force 2>&1
+--remove_file $MYSQLD_DATADIR/mysql_upgrade_info
+
+show create table mysql_json_test;
+select * from mysql_json_test;
+show create table mysql_json_test_big;
+select * from mysql.plugin;
+drop table mysql_json_test;
+drop table mysql_json_test_big;

mysql-test/main/mysql_json_mysql_upgrade_with_plugin_loaded.result

+#
+# MDEV-24093: Detect during mysql_upgrade if type_mysql_json.so 
+# is needed and load it
+#
+SET NAMES utf8;
+call mtr.add_suppression("Table rebuild required");
+show create table mysql_json_test;
+ERROR HY000: Table rebuild required. Please do "ALTER TABLE `test.mysql_json_test` FORCE" or dump/reload to fix it!
+Phase 1/7: Checking and upgrading mysql database
+Processing databases
+mysql
+mysql.column_stats                                 OK
+mysql.columns_priv                                 OK
+mysql.db                                           OK
+mysql.event                                        OK
+mysql.func                                         OK
+mysql.global_priv                                  OK
+mysql.gtid_slave_pos                               OK
+mysql.help_category                                OK
+mysql.help_keyword                                 OK
+mysql.help_relation                                OK
+mysql.help_topic                                   OK
+mysql.index_stats                                  OK
+mysql.innodb_index_stats
+Error    : Unknown storage engine 'InnoDB'
+error    : Corrupt
+mysql.innodb_table_stats
+Error    : Unknown storage engine 'InnoDB'
+error    : Corrupt
+mysql.plugin                                       OK
+mysql.proc                                         OK
+mysql.procs_priv                                   OK
+mysql.proxies_priv                                 OK
+mysql.roles_mapping                                OK
+mysql.servers                                      OK
+mysql.table_stats                                  OK
+mysql.tables_priv                                  OK
+mysql.time_zone                                    OK
+mysql.time_zone_leap_second                        OK
+mysql.time_zone_name                               OK
+mysql.time_zone_transition                         OK
+mysql.time_zone_transition_type                    OK
+mysql.transaction_registry
+Error    : Unknown storage engine 'InnoDB'
+error    : Corrupt
+
+Repairing tables
+mysql.innodb_index_stats
+Error    : Unknown storage engine 'InnoDB'
+error    : Corrupt
+mysql.innodb_table_stats
+Error    : Unknown storage engine 'InnoDB'
+error    : Corrupt
+mysql.transaction_registry
+Error    : Unknown storage engine 'InnoDB'
+error    : Corrupt
+Phase 2/7: Installing used storage engines... Skipped
+Phase 3/7: Fixing views
+mysql.user                                         OK
+Phase 4/7: Running 'mysql_fix_privilege_tables'
+Phase 5/7: Fixing table and database names
+Phase 6/7: Checking and upgrading tables
+Processing databases
+information_schema
+mtr
+mtr.global_suppressions                            OK
+mtr.test_suppressions                              OK
+performance_schema
+test
+test.mysql_json_test                               Needs upgrade
+test.mysql_json_test_big                           Needs upgrade
+
+Repairing tables
+test.mysql_json_test                               OK
+test.mysql_json_test_big                           OK
+Phase 7/7: Running 'FLUSH PRIVILEGES'
+OK
+show create table mysql_json_test;
+Table	Create Table
+mysql_json_test	CREATE TABLE `mysql_json_test` (
+  `description` varchar(100) COLLATE utf8mb4_unicode_ci DEFAULT NULL,
+  `expected` longtext COLLATE utf8mb4_unicode_ci DEFAULT NULL,
+  `actual` longtext CHARACTER SET utf8mb4 COLLATE utf8mb4_bin DEFAULT NULL
+) ENGINE=MyISAM DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci
+show create table mysql_json_test_big;
+Table	Create Table
+mysql_json_test_big	CREATE TABLE `mysql_json_test_big` (
+  `description` varchar(100) COLLATE utf8mb4_unicode_ci DEFAULT NULL,
+  `expected` longtext COLLATE utf8mb4_unicode_ci DEFAULT NULL,
+  `actual` longtext CHARACTER SET utf8mb4 COLLATE utf8mb4_bin DEFAULT NULL
+) ENGINE=MyISAM DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci
+select * from mysql.plugin;
+name	dl
+drop table mysql_json_test;
+drop table mysql_json_test_big;

mysql-test/main/mysql_json_mysql_upgrade_with_plugin_loaded.test

+--echo #
+--echo # MDEV-24093: Detect during mysql_upgrade if type_mysql_json.so 
+--echo # is needed and load it
+--echo #
+# Let's now load plugin first
+-- source include/have_utf8.inc
+-- source include/have_type_mysql_json.inc
+-- source include/mysql_upgrade_preparation.inc
+
+SET NAMES utf8;
+call mtr.add_suppression("Table rebuild required");
+
+let $MYSQLD_DATADIR= `select @@datadir`;
+
+--copy_file std_data/mysql_json/mysql_json_test.frm $MYSQLD_DATADIR/test/mysql_json_test.frm
+--copy_file std_data/mysql_json/mysql_json_test.MYI $MYSQLD_DATADIR/test/mysql_json_test.MYI
+--copy_file std_data/mysql_json/mysql_json_test.MYD $MYSQLD_DATADIR/test/mysql_json_test.MYD
+
+--copy_file std_data/mysql_json/mysql_json_test_big.frm $MYSQLD_DATADIR/test/mysql_json_test_big.frm
+--copy_file std_data/mysql_json/mysql_json_test_big.MYI $MYSQLD_DATADIR/test/mysql_json_test_big.MYI
+--copy_file std_data/mysql_json/mysql_json_test_big.MYD $MYSQLD_DATADIR/test/mysql_json_test_big.MYD
+
+# In the previous example (mysql_json_mysql_upgrade.test) 
+# instead of ER_TABLE_NEEDS_REBUILD we had ER_UNKNOWN_DATA_TYPE
+--error ER_TABLE_NEEDS_REBUILD
+show create table mysql_json_test;
+
+--exec $MYSQL_UPGRADE --force 2>&1
+--remove_file $MYSQLD_DATADIR/mysql_upgrade_info
+
+show create table mysql_json_test;
+show create table mysql_json_test_big;
+select * from mysql.plugin;
+drop table mysql_json_test;
+drop table mysql_json_test_big;

mysql-test/main/mysql_json_table_recreate.test

 --source include/have_utf8.inc
-
-if (!$TYPE_MYSQL_JSON_SO) {
-  skip Need MYSQL_JSON plugin;
-}
+--source include/have_type_mysql_json.inc
 
 --echo #
 --echo # The following test takes 2 tables containing a JSON column and attempts

mysql-test/main/mysql_upgrade_mysql_json_datatype.result

@@ -3,7 +3,6 @@ call mtr.add_suppression("is marked as crashed");
 call mtr.add_suppression("Checking");
 SET NAMES utf8;
 set sql_mode="";
-install soname 'type_mysql_json';
 show create table tempty;
 ERROR HY000: Table rebuild required. Please do "ALTER TABLE `test.tempty` FORCE" or dump/reload to fix it!
 show create table mysql_json_test;
@@ -103,4 +102,3 @@ Total_Number_of_Tests	Succesful_Tests	String_is_valid_JSON
 drop table tempty;
 drop table mysql_json_test;
 drop table mysql_json_test_big;
-uninstall soname 'type_mysql_json';

mysql-test/main/mysql_upgrade_mysql_json_datatype.test

 -- source include/mysql_upgrade_preparation.inc
 -- source include/have_working_dns.inc
 -- source include/have_innodb.inc
-
-if (!$TYPE_MYSQL_JSON_SO) {
-  skip Need MYSQL_JSON plugin;
-}
+-- source include/have_type_mysql_json.inc
 
 call mtr.add_suppression("Table rebuild required");
 call mtr.add_suppression("is marked as crashed");
@@ -28,8 +25,6 @@ SET NAMES utf8;
 
 set sql_mode="";
 
-install soname 'type_mysql_json';
-
 --error ER_TABLE_NEEDS_REBUILD
 show create table tempty;
 --error ER_TABLE_NEEDS_REBUILD
@@ -63,5 +58,4 @@ drop table tempty;
 drop table mysql_json_test;
 drop table mysql_json_test_big;
 
-uninstall soname 'type_mysql_json';
 --remove_file $MYSQLD_DATADIR/mysql_upgrade_info