MDEV-23106 Unable to recognize/import partitioned tables from physical MySQL databases

MDEV-29253 Detect incompatible MySQL partition scheme and either convert
them or report to user and in error log.

This task is about converting in place MySQL 5.6 and 5.7 partition tables
to MariaDB as part of mariadb-upgrade.

- Update TABLE_SHARE::init_from_binary_frm_image() to be able to read
  MySQL frm files with partitions.
- Create .par file, if it do not exists, on open of partitioned table.

Executing mariadb-upgrade will create all the missing .par files.
The MySQL .frm file will be changed to MariaDB format after next
ALTER TABLE.

Other changes:
- If we are using stored mysql_version to distingush between MySQL and
  MariaDB  .frm file information, do not upgrade mysql_version in the
  .frm file as part of CHECK TABLE .. FOR UPGRADE as this would cause
  problems next time we parse the .frm file.

sql/ha_partition.cc

@@ -3106,14 +3106,15 @@ bool ha_partition::new_handlers_from_part_info(MEM_ROOT *mem_root)
   @param name  Name of table file (without extension)
 
   @return Operation status
-    @retval true   Failure
-    @retval false  Success
+    @retval 0  success
+    @retval 1  no par file
+    @retval #  other error
 
   @note On success, m_file_buffer is allocated and must be
   freed by the caller. m_name_buffer_ptr and m_tot_parts is also set.
 */
 
-bool ha_partition::read_par_file(const char *name)
+int ha_partition::read_par_file(const char *name)
 {
   char buff[FN_REFLEN];
   uchar *tot_name_len_offset;
@@ -3124,13 +3125,13 @@ bool ha_partition::read_par_file(const char *name)
   DBUG_PRINT("enter", ("table name: '%s'", name));
 
   if (m_file_buffer)
-    DBUG_RETURN(false);
+    DBUG_RETURN(0);
   fn_format(buff, name, "", ha_par_ext, MY_APPEND_EXT);
 
   /* Following could be done with mysql_file_stat to read in whole file */
   if ((file= mysql_file_open(key_file_ha_partition_par,
                              buff, O_RDONLY | O_SHARE, MYF(0))) < 0)
-    DBUG_RETURN(TRUE);
+    DBUG_RETURN(1);
   if (mysql_file_read(file, (uchar *) &buff[0], PAR_WORD_SIZE, MYF(MY_NABP)))
     goto err1;
   len_words= uint4korr(buff);
@@ -3191,12 +3192,12 @@ bool ha_partition::read_par_file(const char *name)
   }
 
   (void) mysql_file_close(file, MYF(0));
-  DBUG_RETURN(false);
+  DBUG_RETURN(0);
 
 err2:
 err1:
   (void) mysql_file_close(file, MYF(0));
-  DBUG_RETURN(true);
+  DBUG_RETURN(2);
 }
 
 
@@ -3354,14 +3355,20 @@ handlerton *ha_partition::get_def_part_engine(const char *name)
 bool ha_partition::get_from_handler_file(const char *name, MEM_ROOT *mem_root,
                                          bool is_clone)
 {
+  int error;
   DBUG_ENTER("ha_partition::get_from_handler_file");
   DBUG_PRINT("enter", ("table name: '%s'", name));
 
   if (m_file_buffer)
     DBUG_RETURN(false);
 
-  if (read_par_file(name))
-    DBUG_RETURN(true);
+  if ((error= read_par_file(name)))
+  {
+    if (error != 1 || is_clone || re_create_par_file(name))
+      DBUG_RETURN(true);
+    if (read_par_file(name))                    // Test file
+      DBUG_RETURN(true);
+  }
 
   handlerton *default_engine= get_def_part_engine(name);
   if (!default_engine)
@@ -3374,6 +3381,78 @@ bool ha_partition::get_from_handler_file(const char *name, MEM_ROOT *mem_root,
 }
 
 
+/*
+ Create .par file from SQL syntax.
+
+ This is only used with partitioned tables from MySQL 5.6 or 5.7
+ which do not have a .par file.
+*/
+
+bool ha_partition::re_create_par_file(const char *name)
+{
+  THD *thd= current_thd;
+  TABLE table;
+  TABLE_SHARE *share= table_share;
+  Query_arena *backup_stmt_arena_ptr= thd->stmt_arena;
+  Query_arena backup_arena;
+  uint8 save_context_analysis_only= thd->lex->context_analysis_only;
+  bool work_part_info_used;
+  bool tmp;
+  DBUG_ENTER("ha_partition:re_create_par_file");
+
+  /* Share can be NULL in case of delete of non existing table */
+  if (!share ||
+      !(share->mysql_version >= 50600 && share->mysql_version <= 50799))
+    DBUG_RETURN(1);
+
+  bzero((char*) &table, sizeof(table));
+  table.in_use= thd;
+  table.s= share;
+  table.file= this;
+  init_sql_alloc(key_memory_TABLE, &table.mem_root, TABLE_ALLOC_BLOCK_SIZE,
+                 0, MYF(0));
+
+  Query_arena part_func_arena(&table.mem_root,
+                              Query_arena::STMT_INITIALIZED);
+  thd->set_n_backup_active_arena(&part_func_arena, &backup_arena);
+  thd->stmt_arena= &part_func_arena;
+
+  tmp= mysql_unpack_partition(thd, share->partition_info_str,
+                              share->partition_info_str_len,
+                              &table, 0,
+                              plugin_hton(share->default_part_plugin),
+                              &work_part_info_used);
+
+  if (!tmp && m_part_info->partitions.elements == 0)
+  {
+    tmp= m_part_info->set_up_defaults_for_partitioning(thd, this,
+                                                       (HA_CREATE_INFO*) 0,
+                                                       0);
+    if (m_part_info->partitions.elements == 0)
+    {
+      /* We did not succed in creating default partitions */
+      tmp= 1;
+    }
+  }
+
+  thd->stmt_arena= backup_stmt_arena_ptr;
+  thd->restore_active_arena(&part_func_arena, &backup_arena);
+  if (!tmp)
+  {
+    tmp= create_handler_file(name);
+  }
+
+  if (table.part_info)
+    free_items(table.part_info->item_free_list);
+  thd->lex->context_analysis_only= save_context_analysis_only;
+  if (table.expr_arena)
+    table.expr_arena->free_items();
+  free_root(&table.mem_root, MYF(0));
+  DBUG_RETURN(tmp);
+}
+
+
+
 /****************************************************************************
                 MODULE open/close object
 ****************************************************************************/

sql/ha_partition.h

@@ -591,10 +591,11 @@ class ha_partition final :public handler
   */
   bool create_handler_file(const char *name);
   bool setup_engine_array(MEM_ROOT *mem_root, handlerton *first_engine);
-  bool read_par_file(const char *name);
+  int read_par_file(const char *name);
   handlerton *get_def_part_engine(const char *name);
   bool get_from_handler_file(const char *name, MEM_ROOT *mem_root,
                              bool is_clone);
+  bool re_create_par_file(const char *name);
   bool new_handlers_from_part_info(MEM_ROOT *mem_root);
   bool create_handlers(MEM_ROOT *mem_root);
   void clear_handler_file();

sql/handler.cc

@@ -4811,8 +4811,12 @@ static bool update_frm_version(TABLE *table)
     by server with the same version. This also ensures that we do not
     update frm version for temporary tables as this code doesn't support
     temporary tables.
+
+    keep_original_mysql_version is set if the table version cannot be
+    changed without rewriting the frm file.
   */
-  if (table->s->mysql_version == MYSQL_VERSION_ID)
+  if (table->s->mysql_version == MYSQL_VERSION_ID ||
+      table->s->keep_original_mysql_version)
     DBUG_RETURN(0);
 
   strxmov(path, table->s->normalized_path.str, reg_ext, NullS);

sql/table.cc

@@ -1201,7 +1201,10 @@ bool parse_vcol_defs(THD *thd, MEM_ROOT *mem_root, TABLE *table,
             ? VCOL_GENERATED_STORED : VCOL_GENERATED_VIRTUAL;
       expr_length= uint2korr(pos+1);
       if (table->s->mysql_version > 50700 && table->s->mysql_version < 100000)
+      {
+        table->s->keep_original_mysql_version= 1;
         pos+= 4;                        // MySQL from 5.7
+      }
       else
         pos+= pos[0] == 2 ? 4 : 3;      // MariaDB from 5.2 to 10.1
     }
@@ -1710,6 +1713,33 @@ class Field_data_type_info_array
 };
 
 
+/*
+  Change to use the partition storage engine
+*/
+
+#ifdef WITH_PARTITION_STORAGE_ENGINE
+static bool change_to_partiton_engine(LEX_CSTRING *name,
+                                      plugin_ref *se_plugin)
+{
+  /*
+    Use partition handler
+    tmp_plugin is locked with a local lock.
+    we unlock the old value of se_plugin before
+    replacing it with a globally locked version of tmp_plugin
+  */
+  /* Check if the partitioning engine is ready */
+  if (!plugin_is_ready(name, MYSQL_STORAGE_ENGINE_PLUGIN))
+  {
+    my_error(ER_OPTION_PREVENTS_STATEMENT, MYF(0),
+             "--skip-partition");
+    return 1;
+  }
+  plugin_unlock(NULL, *se_plugin);
+  *se_plugin= ha_lock_engine(NULL, partition_hton);
+  return 0;
+}
+#endif /* WITH_PARTITION_STORAGE_ENGINE */
+
 /**
   Read data from a binary .frm file image into a TABLE_SHARE
 
@@ -1762,6 +1792,7 @@ int TABLE_SHARE::init_from_binary_frm_image(THD *thd, bool write,
   uint vcol_screen_length;
   uchar *vcol_screen_pos;
   LEX_CUSTRING options;
+  LEX_CSTRING se_name= empty_clex_str;
   KEY first_keyinfo;
   uint len;
   uint ext_key_parts= 0;
@@ -1971,11 +2002,10 @@ int TABLE_SHARE::init_from_binary_frm_image(THD *thd, bool write,
     if (next_chunk + 2 < buff_end)
     {
       uint str_db_type_length= uint2korr(next_chunk);
-      LEX_CSTRING name;
-      name.str= (char*) next_chunk + 2;
-      name.length= str_db_type_length;
+      se_name.str= (char*) next_chunk + 2;
+      se_name.length= str_db_type_length;
 
-      plugin_ref tmp_plugin= ha_resolve_by_name(thd, &name, false);
+      plugin_ref tmp_plugin= ha_resolve_by_name(thd, &se_name, false);
       if (tmp_plugin != NULL && !plugin_equals(tmp_plugin, se_plugin) &&
           legacy_db_type != DB_TYPE_S3)
       {
@@ -1999,28 +2029,15 @@ int TABLE_SHARE::init_from_binary_frm_image(THD *thd, bool write,
       else if (str_db_type_length == 9 &&
                !strncmp((char *) next_chunk + 2, "partition", 9))
       {
-        /*
-          Use partition handler
-          tmp_plugin is locked with a local lock.
-          we unlock the old value of se_plugin before
-          replacing it with a globally locked version of tmp_plugin
-        */
-        /* Check if the partitioning engine is ready */
-        if (!plugin_is_ready(&name, MYSQL_STORAGE_ENGINE_PLUGIN))
-        {
-          my_error(ER_OPTION_PREVENTS_STATEMENT, MYF(0),
-                   "--skip-partition");
+        if (change_to_partiton_engine(&se_name, &se_plugin))
           goto err;
-        }
-        plugin_unlock(NULL, se_plugin);
-        se_plugin= ha_lock_engine(NULL, partition_hton);
       }
 #endif
       else if (!tmp_plugin)
       {
         /* purecov: begin inspected */
-        ((char*) name.str)[name.length]=0;
-        my_error(ER_UNKNOWN_STORAGE_ENGINE, MYF(0), name.str);
+        ((char*) se_name.str)[se_name.length]=0;
+        my_error(ER_UNKNOWN_STORAGE_ENGINE, MYF(0), se_name.str);
         goto err;
         /* purecov: end */
       }
@@ -2047,6 +2064,13 @@ int TABLE_SHARE::init_from_binary_frm_image(THD *thd, bool write,
         {
           goto err;
         }
+        if (plugin_data(se_plugin, handlerton*) != partition_hton &&
+            share->mysql_version >= 50600 && share->mysql_version <= 50799)
+        {
+          share->keep_original_mysql_version= 1;
+          if (change_to_partiton_engine(&se_name, &se_plugin))
+            goto err;
+        }
       }
 #else
       if (partition_info_str_len)
@@ -2314,6 +2338,7 @@ int TABLE_SHARE::init_from_binary_frm_image(THD *thd, bool write,
   if (share->mysql_version >= 50700 && share->mysql_version < 100000 &&
       vcol_screen_length)
   {
+    share->keep_original_mysql_version= 1;
     /*
       MySQL 5.7 stores the null bits for not stored fields last.
       Calculate the position for them.
@@ -2591,8 +2616,12 @@ int TABLE_SHARE::init_from_binary_frm_image(THD *thd, bool write,
     }
 
     /* Remove >32 decimals from old files */
-    if (share->mysql_version < 100200)
+    if (share->mysql_version < 100200 &&
+        (attr.pack_flag & FIELDFLAG_LONG_DECIMAL))
+    {
+      share->keep_original_mysql_version= 1;
       attr.pack_flag&= ~FIELDFLAG_LONG_DECIMAL;
+    }
 
     if (interval_nr && attr.charset->mbminlen > 1 &&
         !interval_unescaped[interval_nr - 1])
@@ -4081,7 +4110,6 @@ enum open_frm_error open_table_from_share(THD *thd, TABLE_SHARE *share,
   outparam->in_use= thd;
   outparam->s= share;
   outparam->db_stat= db_stat;
-  outparam->write_row_record= NULL;
   outparam->status= STATUS_NO_RECORD;
 
   if (share->incompatible_version &&

sql/table.h

@@ -873,6 +873,8 @@ struct TABLE_SHARE
   bool has_update_default_function;
   bool can_do_row_logging;              /* 1 if table supports RBR */
   bool long_unique_table;
+  /* 1 if frm version cannot be updated as part of upgrade */
+  bool keep_original_mysql_version;
 
   ulong table_map_id;                   /* for row-based replication */