Merge 10.1 into 10.2

mysql-test/r/huge_frm-6224.result

+set global max_allowed_packet=1024*1024*10;
 ERROR HY000: The definition for table `t1` is too big
+set global max_allowed_packet=default;

mysql-test/r/range_innodb.result

@@ -55,3 +55,27 @@ pk	f1	f2	f3	pk	f1	f2	f3	f4	f5
 2	6	v	NULL	14	1	q	NULL	4	q
 3	7	c	NULL	14	1	q	NULL	4	q
 drop table t1,t2;
+#
+# MDEV-14440: Server crash in in handler::ha_external_lock or Assertion `inited==RND'
+# failed in handler::ha_rnd_end upon SELECT from partitioned table
+#
+set @optimizer_switch_save= @@optimizer_switch;
+set optimizer_switch='index_merge_sort_intersection=off';
+create table t0 (a int)engine=innodb;
+insert into t0 values (0),(1),(2),(3),(4),(5),(6),(7),(8),(9);
+create table t1 (
+a int, b int, c int,
+key(a),key(b),key(c)
+)engine=innodb;
+insert into t1
+select A.a+10*B.a, A.a+10*B.a, A.a+10*B.a+100*C.a
+from t0 A, t0 B, t0 C, t0 D where D.a<5;
+set @@global.debug_dbug="+d,ha_index_init_fail";
+explain select * from t1 where a=10 and b=10;
+id	select_type	table	type	possible_keys	key	key_len	ref	rows	Extra
+1	SIMPLE	t1	index_merge	a,b	a,b	5,5	NULL	1	Using intersect(a,b); Using where
+select * from t1 where a=10 and b=10;
+ERROR HY000: Table definition has changed, please retry transaction
+DROP TABLE t0,t1;
+set @@global.debug_dbug="-d";
+set @@optimizer_switch= @optimizer_switch_save;

mysql-test/suite/innodb/r/innodb_28867993.result

+create table t1 (a int) engine=innodb;
+insert t1 values (1),(2);
+create database ib_logfile2;
+select * from t1;
+a
+1
+2
+drop table t1;
+drop database ib_logfile2;

mysql-test/suite/innodb/t/innodb_28867993.test

+#
+# Bug#28867993: POSSIBLE ISSUE WITH MYSQL SERVER RESTART
+#
+
+source include/have_innodb.inc;
+create table t1 (a int) engine=innodb;
+insert t1 values (1),(2);
+create database ib_logfile2;
+source include/restart_mysqld.inc;
+select * from t1;
+drop table t1;
+drop database ib_logfile2;

mysql-test/t/huge_frm-6224.test

@@ -4,17 +4,24 @@
 # verify that huge frms are rejected during creation, not on opening
 #
 --source include/have_partition.inc
+set global max_allowed_packet=1024*1024*10;
+connect con1,localhost,root;
 
-let $n=5646;
+let $n=8164;
 let $a=create table t1 (a int) engine=myisam partition by hash(a) partitions $n (;
 dec $n;
 while ($n)
 {
-  let $a=$a partition p01234567890123456789012345678901234567890123456789012345678$n,;
+  let $a=$a partition p01234567890123456789012345678901234567890123456789012345678$n COMMENT 'partition p01234567890123456789012345678901234567890123456789012345678$n',;
   dec $n;
 }
 
 --disable_query_log
 --error ER_TABLE_DEFINITION_TOO_BIG
 eval $a partition foo);
+--enable_query_log
+
+connection default;
+disconnect con1;
+set global max_allowed_packet=default;
 

mysql-test/t/range_innodb.test

@@ -3,6 +3,7 @@
 --echo # 
 
 --source include/have_innodb.inc
+--source include/have_debug.inc
 
 --disable_warnings
 drop table if exists t0, t1, t2;
@@ -62,3 +63,27 @@ INSERT INTO t2 VALUES (4,'q'),(NULL,'j');
 SELECT * FROM t1 AS t1_1, t1 AS t1_2, t2
 WHERE f5 = t1_2.f2 AND ( t1_1.f1 = 103 AND t1_1.f2 = 'o' OR t1_1.pk < f4 );
 drop table t1,t2;
+
+--echo #
+--echo # MDEV-14440: Server crash in in handler::ha_external_lock or Assertion `inited==RND'
+--echo # failed in handler::ha_rnd_end upon SELECT from partitioned table
+--echo #
+
+set @optimizer_switch_save= @@optimizer_switch;
+set optimizer_switch='index_merge_sort_intersection=off';
+create table t0 (a int)engine=innodb;
+insert into t0 values (0),(1),(2),(3),(4),(5),(6),(7),(8),(9);
+create table t1 (
+  a int, b int, c int,
+  key(a),key(b),key(c)
+)engine=innodb;
+insert into t1
+select A.a+10*B.a, A.a+10*B.a, A.a+10*B.a+100*C.a
+from t0 A, t0 B, t0 C, t0 D where D.a<5;
+set @@global.debug_dbug="+d,ha_index_init_fail";
+explain select * from t1 where a=10 and b=10;
+--error ER_TABLE_DEF_CHANGED
+select * from t1 where a=10 and b=10;
+DROP TABLE t0,t1;
+set @@global.debug_dbug="-d";
+set @@optimizer_switch= @optimizer_switch_save;

scripts/wsrep_sst_mariabackup.sh

@@ -91,6 +91,9 @@ DATA="${WSREP_SST_OPT_DATA}"
 INFO_FILE="xtrabackup_galera_info"
 IST_FILE="xtrabackup_ist"
 MAGIC_FILE="${DATA}/${INFO_FILE}"
+INNOAPPLYLOG="${DATA}/mariabackup.prepare.log"
+INNOMOVELOG="${DATA}/mariabackup.move.log"
+INNOBACKUPLOG="${DATA}/mariabackup.backup.log"
 
 # Setting the path for ss and ip
 export PATH="/usr/sbin:/sbin:$PATH"
@@ -358,6 +361,8 @@ read_cnf()
     ssyslog=$(parse_cnf sst sst-syslog 0)
     ssystag=$(parse_cnf mysqld_safe syslog-tag "${SST_SYSLOG_TAG:-}")
     ssystag+="-"
+    sstlogarchive=$(parse_cnf sst sst-log-archive 1)
+    sstlogarchivedir=$(parse_cnf sst sst-log-archive-dir "/tmp/sst_log_archive")
 
     if [[ $speciald -eq 0 ]];then 
         wsrep_log_error "sst-special-dirs equal to 0 is not supported, falling back to 1"
@@ -717,10 +722,68 @@ if [[ $ssyslog -eq 1 ]];then
         INNOBACKUP="${INNOBACKUPEX_BIN} --innobackupex ${WSREP_SST_OPT_CONF} $disver $iopts \$tmpopts \$INNOEXTRA --galera-info --stream=\$sfmt \$itmpdir 2> >(logger -p daemon.err -t ${ssystag}innobackupex-backup)"
     fi
 
-else 
-    INNOAPPLY="${INNOBACKUPEX_BIN} --innobackupex $disver $iapts --apply-log \$rebuildcmd \${DATA} &>\${DATA}/innobackup.prepare.log"
-    INNOMOVE="${INNOBACKUPEX_BIN} --innobackupex ${WSREP_SST_OPT_CONF} $disver $impts  --move-back --force-non-empty-directories \${DATA} &>\${DATA}/innobackup.move.log"
-    INNOBACKUP="${INNOBACKUPEX_BIN} --innobackupex ${WSREP_SST_OPT_CONF} $disver $iopts \$tmpopts \$INNOEXTRA --galera-info --stream=\$sfmt \$itmpdir 2>\${DATA}/innobackup.backup.log"
+else
+
+if [[ "$sstlogarchive" -eq 1 ]]
+then
+    ARCHIVETIMESTAMP=$(date "+%Y.%m.%d-%H.%M.%S")
+    newfile=""    
+
+    if [[ ! -z "$sstlogarchivedir" ]]
+    then
+        if [[ ! -d "$sstlogarchivedir" ]]
+        then
+            mkdir -p "$sstlogarchivedir"
+        fi
+    fi
+
+    if [ -e "${INNOAPPLYLOG}" ]
+    then
+        if [[ ! -z "$sstlogarchivedir" ]]
+        then
+            newfile=$sstlogarchivedir/$(basename "${INNOAPPLYLOG}").${ARCHIVETIMESTAMP}
+        else
+            newfile=${INNOAPPLYLOG}.${ARCHIVETIMESTAMP}
+        fi
+   
+        wsrep_log_info "Moving ${INNOAPPLYLOG} to ${newfile}"
+        mv "${INNOAPPLYLOG}" "${newfile}"
+        gzip "${newfile}"
+    fi
+
+    if [ -e "${INNOMOVELOG}" ]
+    then
+        if [[ ! -z "$sstlogarchivedir" ]]
+        then
+            newfile=$sstlogarchivedir/$(basename "${INNOMOVELOG}").${ARCHIVETIMESTAMP}
+        else
+            newfile=${INNOMOVELOG}.${ARCHIVETIMESTAMP}
+        fi
+
+        wsrep_log_info "Moving ${INNOMOVELOG} to ${newfile}"
+        mv "${INNOMOVELOG}" "${newfile}"
+        gzip "${newfile}"
+    fi
+
+    if [ -e "${INNOBACKUPLOG}" ]
+    then
+        if [[ ! -z "$sstlogarchivedir" ]]
+        then
+            newfile=$sstlogarchivedir/$(basename "${INNOBACKUPLOG}").${ARCHIVETIMESTAMP}
+        else
+            newfile=${INNOBACKUPLOG}.${ARCHIVETIMESTAMP}
+        fi
+
+        wsrep_log_info "Moving ${INNOBACKUPLOG} to ${newfile}"
+        mv "${INNOBACKUPLOG}" "${newfile}"
+        gzip "${newfile}"
+    fi
+
+fi
+ 
+    INNOAPPLY="${INNOBACKUPEX_BIN} --innobackupex $disver $iapts --apply-log \$rebuildcmd \${DATA} &> ${INNOAPPLYLOG}"
+    INNOMOVE="${INNOBACKUPEX_BIN} --innobackupex ${WSREP_SST_OPT_CONF} $disver $impts  --move-back --force-non-empty-directories \${DATA} &> ${INNOMOVELOG}"
+    INNOBACKUP="${INNOBACKUPEX_BIN} --innobackupex ${WSREP_SST_OPT_CONF} $disver $iopts \$tmpopts \$INNOEXTRA --galera-info --stream=\$sfmt \$itmpdir 2> ${INNOBACKUPLOG}"
 fi
 
 get_stream
@@ -817,7 +880,7 @@ then
 
         if [ ${RC[0]} -ne 0 ]; then
           wsrep_log_error "${INNOBACKUPEX_BIN} finished with error: ${RC[0]}. " \
-                          "Check ${DATA}/innobackup.backup.log"
+                          "Check syslog or ${INNOBACKUPLOG} for details"
           exit 22
         elif [[ ${RC[$(( ${#RC[@]}-1 ))]} -eq 1 ]];then 
           wsrep_log_error "$tcmd finished with error: ${RC[1]}"
@@ -1042,13 +1105,12 @@ then
 
         if [ $? -ne 0 ];
         then
-            wsrep_log_error "${INNOBACKUPEX_BIN} apply finished with errors. Check ${DATA}/innobackup.prepare.log" 
+            wsrep_log_error "${INNOBACKUPEX_BIN} apply finished with errors. Check syslog or ${INNOAPPLYLOG} for details" 
             exit 22
         fi
 
         MAGIC_FILE="${TDATA}/${INFO_FILE}"
         set +e
-        rm $TDATA/innobackup.prepare.log $TDATA/innobackup.move.log
         set -e
         wsrep_log_info "Moving the backup to ${TDATA}"
         timeit "Xtrabackup move stage" "$INNOMOVE"
@@ -1058,7 +1120,7 @@ then
             DATA=${TDATA}
         else 
             wsrep_log_error "Move failed, keeping ${DATA} for further diagnosis"
-            wsrep_log_error "Check ${DATA}/innobackup.move.log for details"
+            wsrep_log_error "Check syslog or ${INNOMOVELOG} for details"
             exit 22
         fi
 

sql/opt_range.cc

@@ -1558,6 +1558,7 @@ int QUICK_RANGE_SELECT::init_ror_merged_scan(bool reuse_handler,
   head->column_bitmaps_set(save_read_set, save_write_set, save_vcol_set);
   delete file;
   file= save_file;
+  free_file= false;
   DBUG_RETURN(1);
 }
 
@@ -6546,6 +6547,8 @@ TRP_ROR_INTERSECT *get_best_ror_intersect(const PARAM *param, SEL_TREE *tree,
     if (ror_intersect_add(intersect, cpk_scan, TRUE) && 
         (intersect->total_cost < min_cost))
       intersect_best= intersect; //just set pointer here
+    else
+      cpk_scan= 0; // Don't use cpk_scan
   }
   else
     cpk_scan= 0;                                // Don't use cpk_scan

sql/unireg.h

@@ -189,7 +189,7 @@ LEX_CUSTRING build_frm_image(THD *thd, const char *table,
 
 #define FRM_HEADER_SIZE 64
 #define FRM_FORMINFO_SIZE 288
-#define FRM_MAX_SIZE (512*1024)
+#define FRM_MAX_SIZE (1024*1024)
 
 static inline bool is_binary_frm_header(uchar *head)
 {

storage/innobase/dict/dict0dict.cc

@@ -2,7 +2,7 @@
 
 Copyright (c) 1996, 2016, Oracle and/or its affiliates. All Rights Reserved.
 Copyright (c) 2012, Facebook Inc.
-Copyright (c) 2013, 2018, MariaDB Corporation.
+Copyright (c) 2013, 2019, MariaDB Corporation.
 
 This program is free software; you can redistribute it and/or modify it under
 the terms of the GNU General Public License as published by the Free Software
@@ -7131,32 +7131,16 @@ UNIV_INTERN
 ulint
 dict_sys_get_size()
 {
-	ulint size = 0;
-
-	ut_ad(dict_sys);
-
-	mutex_enter(&dict_sys->mutex);
-
-	for(ulint i = 0; i < hash_get_n_cells(dict_sys->table_hash); i++) {
-		dict_table_t* table;
-
-		for (table = static_cast<dict_table_t*>(HASH_GET_FIRST(dict_sys->table_hash,i));
-		     table != NULL;
-		     table = static_cast<dict_table_t*>(HASH_GET_NEXT(name_hash, table))) {
-			dict_index_t* index;
-			size += mem_heap_get_size(table->heap) + strlen(table->name.m_name) +1;
-
-			for(index = dict_table_get_first_index(table);
-			    index != NULL;
-			    index = dict_table_get_next_index(index)) {
-				size += mem_heap_get_size(index->heap);
-			}
-		}
-	}
-
-	mutex_exit(&dict_sys->mutex);
-
-	return (size);
+	/* No mutex; this is a very crude approximation anyway */
+	ulint size = UT_LIST_GET_LEN(dict_sys->table_LRU)
+		+ UT_LIST_GET_LEN(dict_sys->table_non_LRU);
+	size *= sizeof(dict_table_t)
+		+ sizeof(dict_index_t) * 2
+		+ (sizeof(dict_col_t) + sizeof(dict_field_t)) * 10
+		+ sizeof(dict_field_t) * 5 /* total number of key fields */
+		+ 200; /* arbitrary, covering names and overhead */
+
+	return size;
 }
 
 /** Look for any dictionary objects that are found in the given tablespace.

storage/innobase/fil/fil0fil.cc

 /*****************************************************************************
 
 Copyright (c) 1995, 2017, Oracle and/or its affiliates. All Rights Reserved.
-Copyright (c) 2014, 2018, MariaDB Corporation.
+Copyright (c) 2014, 2019, MariaDB Corporation.
 
 This program is free software; you can redistribute it and/or modify it under
 the terms of the GNU General Public License as published by the Free Software
@@ -817,10 +817,9 @@ fil_try_to_close_file_in_LRU(
 }
 
 /** Flush any writes cached by the file system.
-@param[in,out]	space	tablespace */
-static
-void
-fil_flush_low(fil_space_t* space)
+@param[in,out]	space		tablespace
+@param[in]	metadata	whether to update file system metadata */
+static void fil_flush_low(fil_space_t* space, bool metadata = false)
 {
 	ut_ad(mutex_own(&fil_system->mutex));
 	ut_ad(space);
@@ -844,7 +843,7 @@ fil_flush_low(fil_space_t* space)
 		}
 #endif /* UNIV_DEBUG */
 
-		return;
+		if (!metadata) return;
 	}
 
 	/* Prevent dropping of the space while we are flushing */
@@ -1021,6 +1020,7 @@ fil_space_extend_must_retry(
 
 	os_has_said_disk_full = *success;
 	if (*success) {
+		os_file_flush(node->handle);
 		last_page_no = size;
 	} else {
 		/* Let us measure the size of the file
@@ -1052,14 +1052,14 @@ fil_space_extend_must_retry(
 	switch (space->id) {
 	case TRX_SYS_SPACE:
 		srv_sys_space.set_last_file_size(pages_in_MiB);
-		fil_flush_low(space);
+		fil_flush_low(space, true);
 		return(false);
 	default:
 		ut_ad(space->purpose == FIL_TYPE_TABLESPACE
 		      || space->purpose == FIL_TYPE_IMPORT);
 		if (space->purpose == FIL_TYPE_TABLESPACE
 		    && !space->is_being_truncated) {
-			fil_flush_low(space);
+			fil_flush_low(space, true);
 		}
 		return(false);
 	case SRV_TMP_SPACE_ID:

storage/innobase/srv/srv0srv.cc

@@ -3,7 +3,7 @@
 Copyright (c) 1995, 2017, Oracle and/or its affiliates. All Rights Reserved.
 Copyright (c) 2008, 2009 Google Inc.
 Copyright (c) 2009, Percona Inc.
-Copyright (c) 2013, 2018, MariaDB Corporation.
+Copyright (c) 2013, 2019, MariaDB Corporation.
 
 Portions of this file contain modifications contributed and copyrighted by
 Google, Inc. Those modifications are gratefully acknowledged and are described

storage/innobase/srv/srv0start.cc

@@ -1945,6 +1945,10 @@ innobase_start_or_create_for_mysql()
 				break;
 			}
 
+			if (stat_info.type != OS_FILE_TYPE_FILE) {
+				break;
+			}
+
 			if (!srv_file_check_mode(logfilename)) {
 				return(srv_init_abort(DB_ERROR));
 			}

storage/xtradb/dict/dict0dict.cc

@@ -2,7 +2,7 @@
 
 Copyright (c) 1996, 2017, Oracle and/or its affiliates. All Rights Reserved.
 Copyright (c) 2012, Facebook Inc.
-Copyright (c) 2013, 2018, MariaDB Corporation.
+Copyright (c) 2013, 2019, MariaDB Corporation.
 
 This program is free software; you can redistribute it and/or modify it under
 the terms of the GNU General Public License as published by the Free Software
@@ -7252,30 +7252,14 @@ UNIV_INTERN
 ulint
 dict_sys_get_size()
 {
-	ulint size = 0;
-
-	ut_ad(dict_sys);
-
-	mutex_enter(&dict_sys->mutex);
-
-	for(ulint i = 0; i < hash_get_n_cells(dict_sys->table_hash); i++) {
-		dict_table_t* table;
-
-		for (table = static_cast<dict_table_t*>(HASH_GET_FIRST(dict_sys->table_hash,i));
-		     table != NULL;
-		     table = static_cast<dict_table_t*>(HASH_GET_NEXT(name_hash, table))) {
-			dict_index_t* index;
-			size += mem_heap_get_size(table->heap) + strlen(table->name) +1;
-
-			for(index = dict_table_get_first_index(table);
-			    index != NULL;
-			    index = dict_table_get_next_index(index)) {
-				size += mem_heap_get_size(index->heap);
-			}
-		}
-	}
-
-	mutex_exit(&dict_sys->mutex);
-
-	return (size);
+	/* No mutex; this is a very crude approximation anyway */
+	ulint size = UT_LIST_GET_LEN(dict_sys->table_LRU)
+		+ UT_LIST_GET_LEN(dict_sys->table_non_LRU);
+	size *= sizeof(dict_table_t)
+		+ sizeof(dict_index_t) * 2
+		+ (sizeof(dict_col_t) + sizeof(dict_field_t)) * 10
+		+ sizeof(dict_field_t) * 5 /* total number of key fields */
+		+ 200; /* arbitrary, covering names and overhead */
+
+	return size;
 }

storage/xtradb/fil/fil0fil.cc

 /*****************************************************************************
 
 Copyright (c) 1995, 2017, Oracle and/or its affiliates. All Rights Reserved.
-Copyright (c) 2014, 2018, MariaDB Corporation.
+Copyright (c) 2014, 2019, MariaDB Corporation.
 
 This program is free software; you can redistribute it and/or modify it under
 the terms of the GNU General Public License as published by the Free Software
@@ -846,10 +846,9 @@ fil_try_to_close_file_in_LRU(
 }
 
 /** Flush any writes cached by the file system.
-@param[in,out]	space	tablespace */
-static
-void
-fil_flush_low(fil_space_t* space)
+@param[in,out]	space		tablespace
+@param[in]	metadata	whether to update file system metadata */
+static void fil_flush_low(fil_space_t* space, bool metadata = false)
 {
 	ut_ad(mutex_own(&fil_system->mutex));
 	ut_ad(space);
@@ -873,7 +872,7 @@ fil_flush_low(fil_space_t* space)
 		}
 #endif /* UNIV_DEBUG */
 
-		return;
+		if (!metadata) return;
 	}
 
 	/* Prevent dropping of the space while we are flushing */
@@ -1037,15 +1036,16 @@ fil_space_extend_must_retry(
 	*success = os_file_set_size(node->name, node->handle, new_size,
 		FSP_FLAGS_HAS_PAGE_COMPRESSION(space->flags));
 
-
 	DBUG_EXECUTE_IF("ib_os_aio_func_io_failure_28",
 		*success = FALSE;
 		os_has_said_disk_full = TRUE;);
 
 	if (*success) {
+		os_file_flush(node->handle);
 		os_has_said_disk_full = FALSE;
 		start_page_no = size;
 	}
+
 	mutex_enter(&fil_system->mutex);
 
 	ut_a(node->being_extended);
@@ -1069,7 +1069,7 @@ fil_space_extend_must_retry(
 			= (node->size / pages_per_mb) * pages_per_mb;
 	}
 
-	fil_flush_low(space);
+	fil_flush_low(space, true);
 	return(false);
 }
 

storage/xtradb/srv/srv0srv.cc

@@ -3,7 +3,7 @@
 Copyright (c) 1995, 2017, Oracle and/or its affiliates. All Rights Reserved.
 Copyright (c) 2008, 2009 Google Inc.
 Copyright (c) 2009, Percona Inc.
-Copyright (c) 2013, 2017, MariaDB Corporation.
+Copyright (c) 2013, 2019, MariaDB Corporation.
 
 Portions of this file contain modifications contributed and copyrighted by
 Google, Inc. Those modifications are gratefully acknowledged and are described
@@ -1637,9 +1637,7 @@ srv_printf_innodb_monitor(
 					? (recv_sys->addr_hash->n_cells * sizeof(hash_cell_t)) : 0),
 			recv_sys_subtotal);
 
-
-	fprintf(file, "Dictionary memory allocated " ULINTPF "\n",
-		dict_sys ? dict_sys_get_size() : 0);
+	fprintf(file, "Dictionary memory allocated " ULINTPF "\n", dict_size);
 
 	buf_print_io(file);
 

storage/xtradb/srv/srv0start.cc

@@ -2377,6 +2377,9 @@ innobase_start_or_create_for_mysql()
 				break;
 			}
 
+			if (stat_info.type != OS_FILE_TYPE_FILE) {
+				break;
+			}
 			if (!srv_file_check_mode(logfilename)) {
 				return(DB_ERROR);
 			}