Merge 10.3 into 10.4

cmake/cpack_rpm.cmake

@@ -195,7 +195,7 @@ MACRO(ALTERNATIVE_NAME real alt)
   SET(p "CPACK_RPM_${real}_PACKAGE_PROVIDES")
   SET(${p} "${${p}} ${alt} = ${ver} ${alt}%{?_isa} = ${ver} config(${alt}) = ${ver}")
   SET(o "CPACK_RPM_${real}_PACKAGE_OBSOLETES")
-  SET(${o} "${${o}} ${alt} ${alt}%{?_isa}")
+  SET(${o} "${${o}} ${alt}")
 ENDMACRO(ALTERNATIVE_NAME)
 
 ALTERNATIVE_NAME("devel"  "mysql-devel")
@@ -215,8 +215,9 @@ ELSEIF(RPM MATCHES "fedora" OR RPM MATCHES "(rhel|centos)7")
   ALTERNATIVE_NAME("server" "mariadb-server")
   ALTERNATIVE_NAME("server" "mysql-compat-server")
   ALTERNATIVE_NAME("test"   "mariadb-test")
-ELSEIF(RPM MATCHES "(rhel|centos)8")
-  SET(PYTHON_SHEBANG "/usr/bin/python3")
+ENDIF()
+IF(RPM MATCHES "fedora31" OR RPM MATCHES "(rhel|centos)8")
+  SET(PYTHON_SHEBANG "/usr/bin/python3" CACHE STRING "python shebang")
 ENDIF()
 
 # If we want to build build MariaDB-shared-compat,
@@ -249,6 +250,7 @@ IF(compat53 AND compat101)
 
   STRING(REPLACE "\n" " " compat_provides "${compat_provides}")
   STRING(REPLACE "\n" " " compat_obsoletes "${compat_obsoletes}")
+  STRING(REGEX REPLACE "[^ ]+\\([^ ]+ *" "" compat_obsoletes "${compat_obsoletes}")
   SETA(CPACK_RPM_compat_PACKAGE_PROVIDES "${compat_provides}")
   SETA(CPACK_RPM_compat_PACKAGE_OBSOLETES "${compat_obsoletes}")
 

include/lf.h

@@ -65,7 +65,6 @@ typedef struct {
 typedef struct {
   void * volatile pin[LF_PINBOX_PINS];
   LF_PINBOX *pinbox;
-  void  **stack_ends_here;
   void  *purgatory;
   uint32 purgatory_count;
   uint32 volatile link;

mysql-test/suite/versioning/r/partition.result

@@ -599,3 +599,17 @@ x	a
 3	bar
 4	bar
 drop table t1;
+#
+# MDEV-21011 Table corruption reported for versioned partitioned table after DELETE: "Found a misplaced row"
+#
+create table t1 (a int) with system versioning
+partition by system_time limit 3
+(partition p1 history, partition p2 history, partition pn current);
+insert into t1 values (1),(2),(3),(4);
+delete from t1;
+delete from t1;
+check table t1;
+Table	Op	Msg_type	Msg_text
+test.t1	check	note	Not supported for non-INTERVAL history partitions
+test.t1	check	note	The storage engine for the table doesn't support check
+drop table t1;

mysql-test/suite/versioning/r/update.result

@@ -276,3 +276,24 @@ update t1 set a= '2012-12-12';
 update v set a= '2000-01-01' order by b limit 1;
 drop view v;
 drop table t1, t2;
+#
+# MDEV-20441 ER_CRASHED_ON_USAGE upon update on versioned Aria table
+#
+create or replace table t1 (a varchar(8))
+engine=aria row_format=fixed
+with system versioning;
+insert into t1 (a) values ('foo');
+update t1 set a = 'bar';
+drop table t1;
+#
+# MDEV-21147 Assertion `marked_for_read()' upon UPDATE on versioned table via view
+#
+create or replace table t1 (
+pk int, a char(8), b char(8),
+primary key (pk)
+) with system versioning;
+create or replace view v1 as select * from t1;
+insert into t1 values  (1, null, 'd') ,  (2, null, 'i') ;
+update v1 set a= null where b = '';
+drop view v1;
+drop table t1;

mysql-test/suite/versioning/r/view.result

@@ -160,6 +160,7 @@ period for system_time (row_start, row_end)
 ) with system versioning;
 insert into t1 values (1), (2);
 create or replace view v1 as select * from t1 where x > 1;
+# update, delete
 update v1 set x= x + 1;
 select *, check_row(row_start, row_end) from t1 for system_time all order by x;
 x	check_row(row_start, row_end)
@@ -211,5 +212,70 @@ x	check_row(row_start, row_end)
 1	CURRENT ROW
 2	HISTORICAL ROW
 3	HISTORICAL ROW
+# replace
+create or replace table t1 (
+x int primary key, y int,
+row_start SYS_DATATYPE as row start invisible,
+row_end SYS_DATATYPE as row end invisible,
+period for system_time (row_start, row_end)
+) with system versioning;
+insert into t1 values (1, 0), (2, 0);
+create or replace view v1 as select * from t1 where x > 1;
+replace v1 values (1, 1);
+replace v1 values (2, 1);
+replace v1 values (3, 1);
+# REPLACE ignores VIEW condition because itself doesn't use WHERE
+select *, check_row(row_start, row_end) from t1 for system_time all order by x, row_end;
+x	y	check_row(row_start, row_end)
+1	0	HISTORICAL ROW
+1	1	CURRENT ROW
+2	0	HISTORICAL ROW
+2	1	CURRENT ROW
+3	1	CURRENT ROW
+# insert-select, on duplicate key
+insert v1 select * from t1 where x = 1 on duplicate key update x = v1.x - 1;
+select *, check_row(row_start, row_end) from t1 for system_time all order by x, row_end;
+x	y	check_row(row_start, row_end)
+0	1	CURRENT ROW
+1	0	HISTORICAL ROW
+1	1	HISTORICAL ROW
+2	0	HISTORICAL ROW
+2	1	CURRENT ROW
+3	1	CURRENT ROW
 drop view v1, v2;
 drop tables t1, t2;
+#
+# MDEV-21146 Assertion `m_lock_type == 2' in handler::ha_drop_table upon LOAD DATA
+#
+create table t1 (a int);
+create view v1 as select * from t1;
+create or replace table t1 (b int) with system versioning;
+load data infile 'xx' into table v1;
+ERROR HY000: View 'test.v1' references invalid table(s) or column(s) or function(s) or definer/invoker of view lack rights to use them
+drop view v1;
+drop table t1;
+#
+# MDEV-21155 Assertion with versioned table upon DELETE from view of view after replacing first view
+#
+create table t1 (a int);
+insert into t1 values (1);
+create table t2 (
+b int,
+row_start SYS_DATATYPE as row start invisible,
+row_end SYS_DATATYPE as row end invisible,
+period for system_time (row_start, row_end)
+) with system versioning;
+insert into t2 values (2);
+create view v1 as select * from t1;
+create view v2 as select * from v1;
+create or replace view v1 as select * from t2;
+delete from v2;
+select * from t1;
+a
+1
+select *, check_row(row_start, row_end) from t2 for system_time all;
+b	check_row(row_start, row_end)
+2	HISTORICAL ROW
+drop view v2;
+drop view v1;
+drop table t1, t2;

mysql-test/suite/versioning/t/partition.test

@@ -530,4 +530,19 @@ update t1 set a= 'bar' limit 4;
 select * from t1;
 drop table t1;
 
+--echo #
+--echo # MDEV-21011 Table corruption reported for versioned partitioned table after DELETE: "Found a misplaced row"
+--echo #
+create table t1 (a int) with system versioning
+partition by system_time limit 3
+(partition p1 history, partition p2 history, partition pn current);
+insert into t1 values (1),(2),(3),(4);
+delete from t1;
+delete from t1;
+check table t1;
+
+# cleanup
+drop table t1;
+
+
 --source suite/versioning/common_finish.inc

mysql-test/suite/versioning/t/update.test

@@ -157,7 +157,6 @@ replace t1 values (1,2),(1,3),(2,4);
 --echo #
 --echo # MDEV-14829 Assertion `0' failed in Protocol::end_statement upon concurrent UPDATE
 --echo #
-
 create or replace table t1 (pk int, a char(3), b char(3), primary key(pk))
   engine=innodb with system versioning;
 
@@ -192,4 +191,31 @@ drop view v;
 drop table t1, t2;
 --enable_warnings
 
+--echo #
+--echo # MDEV-20441 ER_CRASHED_ON_USAGE upon update on versioned Aria table
+--echo #
+create or replace table t1 (a varchar(8))
+engine=aria row_format=fixed
+with system versioning;
+
+insert into t1 (a) values ('foo');
+update t1 set a = 'bar';
+drop table t1;
+
+--echo #
+--echo # MDEV-21147 Assertion `marked_for_read()' upon UPDATE on versioned table via view
+--echo #
+create or replace table t1 (
+    pk int, a char(8), b char(8),
+    primary key (pk)
+) with system versioning;
+
+create or replace view v1 as select * from t1;
+insert into t1 values  (1, null, 'd') ,  (2, null, 'i') ;
+update v1 set a= null where b = '';
+
+# cleanup
+drop view v1;
+drop table t1;
+
 source suite/versioning/common_finish.inc;

mysql-test/suite/versioning/t/view.test

@@ -136,6 +136,7 @@ eval create or replace table t1 (
 ) with system versioning;
 insert into t1 values (1), (2);
 create or replace view v1 as select * from t1 where x > 1;
+--echo # update, delete
 update v1 set x= x + 1;
 select *, check_row(row_start, row_end) from t1 for system_time all order by x;
 insert v1 values (4);
@@ -153,8 +154,63 @@ select *, check_row(row_start, row_end) from t2 for system_time all order by x;
 delete v1, v2 from v1 join v2 where v1.x = v2.x + 2;
 select *, check_row(row_start, row_end) from t1 for system_time all order by x;
 select *, check_row(row_start, row_end) from t2 for system_time all order by x;
-
+--echo # replace
+--replace_result $sys_datatype_expl SYS_DATATYPE
+eval create or replace table t1 (
+  x int primary key, y int,
+  row_start $sys_datatype_expl as row start invisible,
+  row_end $sys_datatype_expl as row end invisible,
+  period for system_time (row_start, row_end)
+) with system versioning;
+insert into t1 values (1, 0), (2, 0);
+create or replace view v1 as select * from t1 where x > 1;
+replace v1 values (1, 1);
+replace v1 values (2, 1);
+replace v1 values (3, 1);
+--echo # REPLACE ignores VIEW condition because itself doesn't use WHERE
+select *, check_row(row_start, row_end) from t1 for system_time all order by x, row_end;
+--echo # insert-select, on duplicate key
+insert v1 select * from t1 where x = 1 on duplicate key update x = v1.x - 1;
+select *, check_row(row_start, row_end) from t1 for system_time all order by x, row_end;
 drop view v1, v2;
 drop tables t1, t2;
 
+--echo #
+--echo # MDEV-21146 Assertion `m_lock_type == 2' in handler::ha_drop_table upon LOAD DATA
+--echo #
+create table t1 (a int);
+create view v1 as select * from t1;
+create or replace table t1 (b int) with system versioning;
+--error ER_VIEW_INVALID
+load data infile 'xx' into table v1;
+
+# cleanup
+drop view v1;
+drop table t1;
+
+--echo #
+--echo # MDEV-21155 Assertion with versioned table upon DELETE from view of view after replacing first view
+--echo #
+create table t1 (a int);
+insert into t1 values (1);
+--replace_result $sys_datatype_expl SYS_DATATYPE
+eval create table t2 (
+  b int,
+  row_start $sys_datatype_expl as row start invisible,
+  row_end $sys_datatype_expl as row end invisible,
+  period for system_time (row_start, row_end)
+) with system versioning;
+insert into t2 values (2);
+create view v1 as select * from t1;
+create view v2 as select * from v1;
+create or replace view v1 as select * from t2;
+delete from v2;
+select * from t1;
+select *, check_row(row_start, row_end) from t2 for system_time all;
+
+# cleanup
+drop view v2;
+drop view v1;
+drop table t1, t2;
+
 --source suite/versioning/common_finish.inc

mysys/lf_alloc-pin.c

@@ -151,7 +151,6 @@ void lf_pinbox_destroy(LF_PINBOX *pinbox)
 */
 LF_PINS *lf_pinbox_get_pins(LF_PINBOX *pinbox)
 {
-  struct st_my_thread_var *var;
   uint32 pins, next, top_ver;
   LF_PINS *el;
   /*
@@ -194,12 +193,7 @@ LF_PINS *lf_pinbox_get_pins(LF_PINBOX *pinbox)
   el->link= pins;
   el->purgatory_count= 0;
   el->pinbox= pinbox;
-  var= my_thread_var;
-  /*
-    Threads that do not call my_thread_init() should still be
-    able to use the LF_HASH.
-  */
-  el->stack_ends_here= (var ? & var->stack_ends_here : NULL);
+
   return el;
 }
 
@@ -335,16 +329,18 @@ static void lf_pinbox_real_free(LF_PINS *pins)
   void *list;
   void **addr= NULL;
   void *first= NULL, *last= NULL;
+  struct st_my_thread_var *var= my_thread_var;
+  void *stack_ends_here= var ? var->stack_ends_here : NULL;
   LF_PINBOX *pinbox= pins->pinbox;
 
   npins= pinbox->pins_in_array+1;
 
 #ifdef HAVE_ALLOCA
-  if (pins->stack_ends_here != NULL)
+  if (stack_ends_here != NULL)
   {
     int alloca_size= sizeof(void *)*LF_PINBOX_PINS*npins;
     /* create a sorted list of pinned addresses, to speed up searches */
-    if (available_stack_size(&pinbox, *pins->stack_ends_here) >
+    if (available_stack_size(&pinbox, stack_ends_here) >
         alloca_size + ALLOCA_SAFETY_MARGIN)
     {
       struct st_harvester hv;

sql/ha_partition.cc

@@ -10830,8 +10830,8 @@ int ha_partition::indexes_are_disabled(void)
   @param repair        If true, move misplaced rows to correct partition.
 
   @return Operation status.
-    @retval 0     Success
-    @retval != 0  Error
+    @retval HA_ADMIN_OK     Success
+    @retval != HA_ADMIN_OK  Error
 */
 
 int ha_partition::check_misplaced_rows(uint read_part_id, bool do_repair)
@@ -10845,6 +10845,17 @@ int ha_partition::check_misplaced_rows(uint read_part_id, bool do_repair)
 
   DBUG_ASSERT(m_file);
 
+  if (m_part_info->vers_info &&
+      read_part_id != m_part_info->vers_info->now_part->id &&
+      !m_part_info->vers_info->interval.is_set())
+  {
+    print_admin_msg(ha_thd(), MYSQL_ERRMSG_SIZE, "note",
+                    table_share->db.str, table->alias,
+                    opt_op_name[CHECK_PARTS],
+                    "Not supported for non-INTERVAL history partitions");
+    DBUG_RETURN(HA_ADMIN_NOT_IMPLEMENTED);
+  }
+
   if (do_repair)
   {
     /* We must read the full row, if we need to move it! */

sql/proxy_protocol.cc

@@ -470,7 +470,7 @@ static int compare_bits(const void *s1, const void *s2, int bit_count)
   int byte_count= bit_count / 8;
   if (byte_count && (result= memcmp(s1, s2, byte_count)))
     return result;
-  int rem= byte_count % 8;
+  int rem= bit_count % 8;
   if (rem)
   {
     // compare remaining bits i.e partial bytes.

sql/sql_derived.cc

@@ -726,12 +726,27 @@ bool mysql_derived_prepare(THD *thd, LEX *lex, TABLE_LIST *derived)
   {
     /*
        System versioned tables may still require to get versioning conditions
-       (when updating view). See vers_setup_conds().
+       when modifying view (see vers_setup_conds()). Only UPDATE and DELETE are
+       affected because they use WHERE condition.
     */
     if (!unit->prepared &&
         derived->table->versioned() &&
-        (res= unit->prepare(derived, derived->derived_result, 0)))
-      goto exit;
+        derived->merge_underlying_list &&
+        /* choose only those merged views that do not select from other views */
+        !derived->merge_underlying_list->merge_underlying_list)
+    {
+      switch (thd->lex->sql_command)
+      {
+      case SQLCOM_DELETE:
+      case SQLCOM_DELETE_MULTI:
+      case SQLCOM_UPDATE:
+      case SQLCOM_UPDATE_MULTI:
+        if ((res= unit->prepare(derived, derived->derived_result, 0)))
+          goto exit;
+      default:
+        break;
+      }
+    }
     DBUG_RETURN(FALSE);
   }
 
@@ -881,7 +896,7 @@ bool mysql_derived_prepare(THD *thd, LEX *lex, TABLE_LIST *derived)
   {
     if (!derived->is_with_table_recursive_reference())
     {
-      if (derived->table)
+      if (derived->table && derived->table->s->tmp_table)
         free_tmp_table(thd, derived->table);
       delete derived->derived_result;
     }

sql/sql_insert.cc

@@ -1670,6 +1670,8 @@ static int last_uniq_key(TABLE *table,uint keynr)
 int vers_insert_history_row(TABLE *table)
 {
   DBUG_ASSERT(table->versioned(VERS_TIMESTAMP));
+  if (!table->vers_write)
+    return 0;
   restore_record(table,record[1]);
 
   // Set Sys_end to now()

sql/sql_update.cc

@@ -206,10 +206,10 @@ static bool check_fields(THD *thd, TABLE_LIST *table, List<Item> &items,
   return FALSE;
 }
 
-static bool check_has_vers_fields(TABLE *table, List<Item> &items)
+bool TABLE::vers_check_update(List<Item> &items)
 {
   List_iterator<Item> it(items);
-  if (!table->versioned())
+  if (!versioned_write())
     return false;
 
   while (Item *item= it++)
@@ -217,8 +217,11 @@ static bool check_has_vers_fields(TABLE *table, List<Item> &items)
     if (Item_field *item_field= item->field_for_view_update())
     {
       Field *field= item_field->field;
-      if (field->table == table && !field->vers_update_unversioned())
+      if (field->table == this && !field->vers_update_unversioned())
+      {
+        no_cache= true;
         return true;
+      }
     }
   }
   return false;
@@ -481,7 +484,7 @@ int mysql_update(THD *thd,
   {
     DBUG_RETURN(1);
   }
-  bool has_vers_fields= check_has_vers_fields(table, fields);
+  bool has_vers_fields= table->vers_check_update(fields);
   if (check_key_in_view(thd, table_list))
   {
     my_error(ER_NON_UPDATABLE_TABLE, MYF(0), table_list->alias.str, "UPDATE");
@@ -2254,7 +2257,7 @@ multi_update::initialize_tables(JOIN *join)
       if (safe_update_on_fly(thd, join->join_tab, table_ref, all_tables))
       {
 	table_to_update= table;			// Update table on the fly
-        has_vers_fields= check_has_vers_fields(table, *fields);
+        has_vers_fields= table->vers_check_update(*fields);
 	continue;
       }
     }
@@ -2723,7 +2726,7 @@ int multi_update::do_updates()
     if (table->vfield)
       empty_record(table);
 
-    has_vers_fields= check_has_vers_fields(table, *fields);
+    has_vers_fields= table->vers_check_update(*fields);
 
     check_opt_it.rewind();
     while(TABLE *tbl= check_opt_it++)

sql/table.cc

@@ -5110,6 +5110,7 @@ void TABLE::init(THD *thd, TABLE_LIST *tl)
   check_unique_buf= NULL;
   vers_write= s->versioned;
   quick_condition_rows=0;
+  no_cache= false;
   initialize_quick_structures();
 #ifdef HAVE_REPLICATION
   /* used in RBR Triggers */
@@ -7027,12 +7028,8 @@ void TABLE::mark_columns_needed_for_update()
     /*
       For System Versioning we have to read all columns since we store
       a copy of previous row with modified row_end back to a table.
-
-      Without write_set versioning.rpl,row is unstable until MDEV-16370 is
-      applied.
     */
     bitmap_union(read_set, &s->all_set);
-    bitmap_union(write_set, &s->all_set);
     need_signal= true;
   }
   if (check_constraints)
@@ -7195,8 +7192,16 @@ void TABLE::mark_columns_per_binlog_row_image()
           binary log will include all columns read anyway.
         */
         mark_columns_used_by_index_no_reset(s->primary_key, read_set);
-        /* Only write columns that have changed */
-        rpl_write_set= write_set;
+        if (versioned())
+        {
+          // TODO: After MDEV-18432 we don't pass history rows, so remove this:
+          rpl_write_set= &s->all_set;
+        }
+        else
+        {
+          /* Only write columns that have changed */
+          rpl_write_set= write_set;
+        }
         break;
 
       default:

sql/table.h

@@ -324,7 +324,7 @@ typedef struct st_grant_info
 
 enum tmp_table_type
 {
-  NO_TMP_TABLE, NON_TRANSACTIONAL_TMP_TABLE, TRANSACTIONAL_TMP_TABLE,
+  NO_TMP_TABLE= 0, NON_TRANSACTIONAL_TMP_TABLE, TRANSACTIONAL_TMP_TABLE,
   INTERNAL_TMP_TABLE, SYSTEM_TMP_TABLE
 };
 enum release_type { RELEASE_NORMAL, RELEASE_WAIT_FOR_DROP };
@@ -1589,8 +1589,15 @@ struct TABLE
     return s->versioned == type;
   }
 
-  bool versioned_write(vers_sys_type_t type= VERS_UNDEFINED) const
+  bool versioned_write() const
   {
+    DBUG_ASSERT(versioned() || !vers_write);
+    return versioned() ? vers_write : false;
+  }
+
+  bool versioned_write(vers_sys_type_t type) const
+  {
+    DBUG_ASSERT(type);
     DBUG_ASSERT(versioned() || !vers_write);
     return versioned(type) ? vers_write : false;
   }
@@ -1614,6 +1621,8 @@ struct TABLE
   int period_make_insert(Item *src, Field *dst);
   int insert_portion_of_time(THD *thd, const vers_select_conds_t &period_conds,
                              ha_rows *rows_inserted);
+  bool vers_check_update(List<Item> &items);
+
   int delete_row();
   void vers_update_fields();
   void vers_update_end();

storage/innobase/trx/trx0undo.cc

@@ -1617,7 +1617,7 @@ trx_undo_commit_cleanup(trx_undo_t* undo, bool is_temp)
 
 		/* Delete first the undo log segment in the file */
 		mutex_exit(&rseg->mutex);
-		trx_undo_seg_free(undo, true);
+		trx_undo_seg_free(undo, is_temp);
 		mutex_enter(&rseg->mutex);
 
 		ut_ad(rseg->curr_size > undo->size);

storage/maria/ma_check.c

@@ -6189,7 +6189,7 @@ int maria_recreate_table(HA_CHECK *param, MARIA_HA **org_info, char *filename)
 }
 
 
-	/* write suffix to data file if neaded */
+/* Write suffix to data file if needed */
 
 int maria_write_data_suffix(MARIA_SORT_INFO *sort_info, my_bool fix_datafile)
 {