Merge 10.2 into bb-10.2-ext

mysql-test/r/dyncol.result

@@ -1883,6 +1883,16 @@ SELECT COLUMN_JSON(COLUMN_CREATE('test','First line\nSecond line')) AS json;
 json
 {"test":"First line\u000ASecond line"}
 #
+# MDEV-15230: column_json breaks cyrillic in 10.1.31
+#
+set names utf8;
+create table t1 (b blob);
+insert into t1 values (column_create('description',column_create('title','Описание')));
+select column_json(b) from t1;
+column_json(b)
+{"description":{"title":"Описание"}}
+drop table t1;
+#
 # end of 10.0 tests
 #
 #

mysql-test/suite/innodb/r/mvcc.result

+SET @save_per_table= @@GLOBAL.innodb_file_per_table;
+SET GLOBAL innodb_file_per_table= 1;
+#
+# MDEV-15249 Crash in MVCC read after IMPORT TABLESPACE
+#
+CREATE TABLE t1 (a INT PRIMARY KEY) ENGINE=InnoDB;
+INSERT INTO t1 VALUES(0);
+FLUSH TABLES t1 WITH READ LOCK;
+UNLOCK TABLES;
+START TRANSACTION WITH CONSISTENT SNAPSHOT;
+connect  con1,localhost,root,,;
+ALTER TABLE t1 FORCE, ALGORITHM=COPY;
+connection default;
+SELECT * FROM t1;
+ERROR HY000: Table definition has changed, please retry transaction
+COMMIT;
+START TRANSACTION WITH CONSISTENT SNAPSHOT;
+connection con1;
+ALTER TABLE t1 DISCARD TABLESPACE;
+ALTER TABLE t1 IMPORT TABLESPACE;
+disconnect con1;
+connection default;
+# FIXME: Block this with ER_TABLE_DEF_CHANGED
+SELECT * FROM t1;
+a
+COMMIT;
+SELECT * FROM t1;
+a
+0
+DROP TABLE t1;
+SET GLOBAL innodb_file_per_table= @save_per_table;

mysql-test/suite/innodb/r/recovery_shutdown.result

@@ -61,4 +61,5 @@ INSERT INTO t1(a) SELECT NULL FROM t1;
 connection default;
 SET GLOBAL innodb_flush_log_at_trx_commit=1;
 CREATE TABLE u(a SERIAL) ENGINE=INNODB;
+FLUSH TABLES;
 DROP TABLE t,u;

mysql-test/suite/innodb/t/mvcc.test

+--source include/have_innodb.inc
+
+SET @save_per_table= @@GLOBAL.innodb_file_per_table;
+SET GLOBAL innodb_file_per_table= 1;
+
+let MYSQLD_DATADIR =`SELECT @@datadir`;
+
+--echo #
+--echo # MDEV-15249 Crash in MVCC read after IMPORT TABLESPACE
+--echo #
+
+CREATE TABLE t1 (a INT PRIMARY KEY) ENGINE=InnoDB;
+INSERT INTO t1 VALUES(0);
+FLUSH TABLES t1 WITH READ LOCK;
+perl;
+do "$ENV{MTR_SUITE_DIR}/include/innodb-util.pl";
+ib_backup_tablespace("test", "t1");
+EOF
+UNLOCK TABLES;
+
+START TRANSACTION WITH CONSISTENT SNAPSHOT;
+
+connect (con1,localhost,root,,);
+ALTER TABLE t1 FORCE, ALGORITHM=COPY;
+
+connection default;
+--error ER_TABLE_DEF_CHANGED
+SELECT * FROM t1;
+COMMIT;
+START TRANSACTION WITH CONSISTENT SNAPSHOT;
+
+connection con1;
+
+ALTER TABLE t1 DISCARD TABLESPACE;
+
+perl;
+do "$ENV{MTR_SUITE_DIR}/include/innodb-util.pl";
+ib_restore_tablespace("test", "t1");
+EOF
+
+ALTER TABLE t1 IMPORT TABLESPACE;
+disconnect con1;
+
+connection default;
+--echo # FIXME: Block this with ER_TABLE_DEF_CHANGED
+SELECT * FROM t1;
+COMMIT;
+SELECT * FROM t1;
+
+DROP TABLE t1;
+
+SET GLOBAL innodb_file_per_table= @save_per_table;

mysql-test/suite/innodb/t/recovery_shutdown.test

@@ -41,6 +41,8 @@ INSERT INTO t1(a) SELECT NULL FROM t1;
 SET GLOBAL innodb_flush_log_at_trx_commit=1;
 CREATE TABLE u(a SERIAL) ENGINE=INNODB;
 
+FLUSH TABLES;
+
 --let $shutdown_timeout=0
 --source include/restart_mysqld.inc
 --let $shutdown_timeout=60

mysql-test/t/dyncol.test

@@ -928,6 +928,15 @@ SELECT COLUMN_JSON(COLUMN_CREATE('a',1 AS DECIMAL,'b',1 AS DECIMAL));
 SELECT COLUMN_JSON(COLUMN_CREATE('test','"\\\t\n\Z')) AS json;
 SELECT COLUMN_JSON(COLUMN_CREATE('test','First line\nSecond line')) AS json;
 
+--echo #
+--echo # MDEV-15230: column_json breaks cyrillic in 10.1.31
+--echo #
+set names utf8;
+create table t1 (b blob);
+insert into t1 values (column_create('description',column_create('title','Описание')));
+select column_json(b) from t1;
+drop table t1;
+
 --echo #
 --echo # end of 10.0 tests
 --echo #

mysql-test/unstable-tests

@@ -58,6 +58,7 @@ main.innodb_mysql_lock             : MDEV-7861 - Wrong result
 main.join_outer                    : Modified in 10.2.12
 main.kill-2                        : MDEV-13257 - Wrong result
 main.log_slow                      : MDEV-13263 - Wrong result
+main.mdev-504                      : MDEV-15171 - warning 
 main.mysql_client_test_nonblock    : CONC-208 - Error on Power
 main.mysql_upgrade_noengine        : MDEV-14355 - Wrong result
 main.mysql_upgrade_ssl             : MDEV-13492 - Unknown SSL error

mysys/ma_dyncol.c

@@ -3833,7 +3833,7 @@ my_bool dynstr_append_json_quoted(DYNAMIC_STRING *str,
   for (i= 0; i < len; i++)
   {
     register char c= append[i];
-    if (unlikely(c <= 0x1F))
+    if (unlikely(((uchar)c) <= 0x1F))
     {
       if (lim < 5)
         {

storage/innobase/btr/btr0cur.cc

@@ -2811,7 +2811,7 @@ btr_cur_ins_lock_and_undo(
 	}
 
 	if (flags & BTR_NO_UNDO_LOG_FLAG) {
-		roll_ptr = 0;
+		roll_ptr = roll_ptr_t(1) << ROLL_PTR_INSERT_FLAG_POS;
 	} else {
 		err = trx_undo_report_row_operation(thr, index, entry,
 						    NULL, 0, NULL, NULL,
@@ -3016,7 +3016,7 @@ btr_cur_optimistic_insert(
 
 	DBUG_LOG("ib_cur",
 		 "insert " << index->name << " (" << index->id << ") by "
-		 << ib::hex(thr ? trx_get_id_for_print(thr_get_trx(thr)) : 0)
+		 << ib::hex(thr ? thr->graph->trx->id : 0)
 		 << ' ' << rec_printer(entry).str());
 	DBUG_EXECUTE_IF("do_page_reorganize",
 			btr_page_reorganize(page_cursor, index, mtr););
@@ -3033,6 +3033,29 @@ btr_cur_optimistic_insert(
 			goto fail_err;
 		}
 
+#ifdef UNIV_DEBUG
+		if (!(flags & BTR_CREATE_FLAG)
+		    && index->is_primary() && page_is_leaf(page)) {
+			const dfield_t* trx_id = dtuple_get_nth_field(
+				entry, dict_col_get_clust_pos(
+					dict_table_get_sys_col(index->table,
+							       DATA_TRX_ID),
+					index));
+
+			ut_ad(trx_id->len == DATA_TRX_ID_LEN);
+			ut_ad(trx_id[1].len == DATA_ROLL_PTR_LEN);
+			ut_ad(*static_cast<const byte*>
+			      (trx_id[1].data) & 0x80);
+			if (!(flags & BTR_NO_UNDO_LOG_FLAG)) {
+				ut_ad(thr->graph->trx->id);
+				ut_ad(thr->graph->trx->id
+				      == trx_read_trx_id(
+					      static_cast<const byte*>(
+						      trx_id->data)));
+			}
+		}
+#endif
+
 		*rec = page_cur_tuple_insert(
 			page_cursor, entry, index, offsets, heap,
 			n_ext, mtr);

storage/innobase/include/dict0mem.h

@@ -993,6 +993,13 @@ struct dict_index_t{
 			and the .ibd file is missing, or a
 			page cannot be read or decrypted */
 	inline bool is_readable() const;
+
+	/** @return whether the index is the primary key index
+	(not the clustered index of the change buffer) */
+	bool is_primary() const
+	{
+		return DICT_CLUSTERED == (type & (DICT_CLUSTERED | DICT_IBUF));
+	}
 };
 
 /** The status of online index creation */

storage/innobase/include/row0ins.h

 /*****************************************************************************
 
 Copyright (c) 1996, 2016, Oracle and/or its affiliates. All Rights Reserved.
-Copyright (c) 2017, MariaDB Corporation.
+Copyright (c) 2017, 2018, MariaDB Corporation.
 
 This program is free software; you can redistribute it and/or modify it under
 the terms of the GNU General Public License as published by the Free Software
@@ -198,10 +198,11 @@ struct ins_node_t{
 				this should be reset to NULL */
 	UT_LIST_BASE_NODE_T(dtuple_t)
 			entry_list;/* list of entries, one for each index */
-	byte*		row_id_buf;/* buffer for the row id sys field in row */
+	/** buffer for the system columns */
+	byte		sys_buf[DATA_ROW_ID_LEN
+				+ DATA_TRX_ID_LEN + DATA_ROLL_PTR_LEN];
 	trx_id_t	trx_id;	/*!< trx id or the last trx which executed the
 				node */
-	byte*		trx_id_buf;/* buffer for the trx id sys field in row */
 	mem_heap_t*	entry_sys_heap;
 				/* memory heap used as auxiliary storage;
 				entry_list and sys fields are stored here;

storage/innobase/include/row0upd.h

 /*****************************************************************************
 
 Copyright (c) 1996, 2016, Oracle and/or its affiliates. All Rights Reserved.
+Copyright (c) 2018, MariaDB Corporation.
 
 This program is free software; you can redistribute it and/or modify it under
 the terms of the GNU General Public License as published by the Free Software
@@ -126,8 +127,7 @@ row_upd_rec_sys_fields(
 	dict_index_t*	index,	/*!< in: clustered index */
 	const ulint*	offsets,/*!< in: rec_get_offsets(rec, index) */
 	const trx_t*	trx,	/*!< in: transaction */
-	roll_ptr_t	roll_ptr);/*!< in: roll ptr of the undo log record,
-				  can be 0 during IMPORT */
+	roll_ptr_t	roll_ptr);/*!< in: DB_ROLL_PTR to the undo log */
 /*********************************************************************//**
 Sets the trx id or roll ptr field of a clustered index entry. */
 void

storage/innobase/include/row0upd.ic

 /*****************************************************************************
 
 Copyright (c) 1996, 2015, Oracle and/or its affiliates. All Rights Reserved.
-Copyright (c) 2017, MariaDB Corporation.
+Copyright (c) 2017, 2018, MariaDB Corporation.
 
 This program is free software; you can redistribute it and/or modify it under
 the terms of the GNU General Public License as published by the Free Software
@@ -165,8 +165,7 @@ row_upd_rec_sys_fields(
 	dict_index_t*	index,	/*!< in: clustered index */
 	const ulint*	offsets,/*!< in: rec_get_offsets(rec, index) */
 	const trx_t*	trx,	/*!< in: transaction */
-	roll_ptr_t	roll_ptr)/*!< in: roll ptr of the undo log record,
-				 can be 0 during IMPORT */
+	roll_ptr_t	roll_ptr)/*!< in: DB_ROLL_PTR to the undo log */
 {
 	ut_ad(dict_index_is_clust(index));
 	ut_ad(rec_offs_validate(rec, index, offsets));

storage/innobase/row/row0import.cc

@@ -897,13 +897,11 @@ class PageConverter : public AbstractCallback {
 	@param index the index being converted
 	@param rec record to update
 	@param offsets column offsets for the record
-	@param deleted true if row is delete marked
 	@return DB_SUCCESS or error code. */
 	dberr_t	adjust_cluster_record(
 		const dict_index_t*	index,
 		rec_t*			rec,
-		const ulint*		offsets,
-		bool			deleted) UNIV_NOTHROW;
+		const ulint*		offsets) UNIV_NOTHROW;
 
 	/** Find an index with the matching id.
 	@return row_index_t* instance or 0 */
@@ -1675,14 +1673,12 @@ PageConverter::purge(const ulint* offsets) UNIV_NOTHROW
 /** Adjust the BLOB references and sys fields for the current record.
 @param rec record to update
 @param offsets column offsets for the record
-@param deleted true if row is delete marked
 @return DB_SUCCESS or error code. */
 dberr_t
 PageConverter::adjust_cluster_record(
 	const dict_index_t*	index,
 	rec_t*			rec,
-	const ulint*		offsets,
-	bool			deleted) UNIV_NOTHROW
+	const ulint*		offsets) UNIV_NOTHROW
 {
 	dberr_t	err;
 
@@ -1694,7 +1690,7 @@ PageConverter::adjust_cluster_record(
 
 		row_upd_rec_sys_fields(
 			rec, m_page_zip_ptr, m_cluster_index, m_offsets,
-			m_trx, 0);
+			m_trx, roll_ptr_t(1) << ROLL_PTR_INSERT_FLAG_POS);
 	}
 
 	return(err);
@@ -1737,8 +1733,7 @@ PageConverter::update_records(
 		if (clust_index) {
 
 			dberr_t err = adjust_cluster_record(
-				m_index->m_srv_index, rec, m_offsets,
-				deleted);
+				m_index->m_srv_index, rec, m_offsets);
 
 			if (err != DB_SUCCESS) {
 				return(err);

storage/innobase/row/row0ins.cc

@@ -139,49 +139,44 @@ row_ins_alloc_sys_fields(
 {
 	dtuple_t*		row;
 	dict_table_t*		table;
-	mem_heap_t*		heap;
 	const dict_col_t*	col;
 	dfield_t*		dfield;
-	byte*			ptr;
 
 	row = node->row;
 	table = node->table;
-	heap = node->entry_sys_heap;
 
-	ut_ad(row && table && heap);
 	ut_ad(dtuple_get_n_fields(row) == dict_table_get_n_cols(table));
 
 	/* allocate buffer to hold the needed system created hidden columns. */
-	const uint len = DATA_ROW_ID_LEN + DATA_TRX_ID_LEN + DATA_ROLL_PTR_LEN;
-	ptr = static_cast<byte*>(mem_heap_zalloc(heap, len));
+	compile_time_assert(DATA_ROW_ID_LEN
+			    + DATA_TRX_ID_LEN + DATA_ROLL_PTR_LEN
+			    == sizeof node->sys_buf);
+	memset(node->sys_buf, 0, sizeof node->sys_buf);
+	/* Assign DB_ROLL_PTR to 1 << ROLL_PTR_INSERT_FLAG_POS */
+	node->sys_buf[DATA_ROW_ID_LEN + DATA_TRX_ID_LEN] = 0x80;
 
 	/* 1. Populate row-id */
 	col = dict_table_get_sys_col(table, DATA_ROW_ID);
 
 	dfield = dtuple_get_nth_field(row, dict_col_get_no(col));
 
-	dfield_set_data(dfield, ptr, DATA_ROW_ID_LEN);
-
-	node->row_id_buf = ptr;
-
-	ptr += DATA_ROW_ID_LEN;
+	dfield_set_data(dfield, node->sys_buf, DATA_ROW_ID_LEN);
 
 	/* 2. Populate trx id */
 	col = dict_table_get_sys_col(table, DATA_TRX_ID);
 
 	dfield = dtuple_get_nth_field(row, dict_col_get_no(col));
 
-	dfield_set_data(dfield, ptr, DATA_TRX_ID_LEN);
-
-	node->trx_id_buf = ptr;
-
-	ptr += DATA_TRX_ID_LEN;
+	dfield_set_data(dfield, &node->sys_buf[DATA_ROW_ID_LEN],
+			DATA_TRX_ID_LEN);
 
 	col = dict_table_get_sys_col(table, DATA_ROLL_PTR);
 
 	dfield = dtuple_get_nth_field(row, dict_col_get_no(col));
 
-	dfield_set_data(dfield, ptr, DATA_ROLL_PTR_LEN);
+	dfield_set_data(dfield, &node->sys_buf[DATA_ROW_ID_LEN
+					       + DATA_TRX_ID_LEN],
+			DATA_ROLL_PTR_LEN);
 }
 
 /*********************************************************************//**
@@ -3474,7 +3469,7 @@ row_ins_alloc_row_id_step(
 
 	row_id = dict_sys_get_new_row_id();
 
-	dict_sys_write_row_id(node->row_id_buf, row_id);
+	dict_sys_write_row_id(node->sys_buf, row_id);
 }
 
 /***********************************************************//**
@@ -3765,7 +3760,7 @@ row_ins_step(
 	This happens, for example, when a row update moves it to another
 	partition. In that case, we have already set the IX lock on the
 	table during the search operation, and there is no need to set
-	it again here. But we must write trx->id to node->trx_id_buf. */
+	it again here. But we must write trx->id to node->sys_buf. */
 
 	if (node->table->no_rollback()) {
 		/* No-rollback tables should only be written to by a
@@ -3780,15 +3775,13 @@ row_ins_step(
 		restarting here. In theory, we could allow resumption
 		from the INS_NODE_INSERT_ENTRIES state here. */
 		DBUG_ASSERT(node->state == INS_NODE_SET_IX_LOCK);
-		memset(node->trx_id_buf, 0, DATA_TRX_ID_LEN);
-		memset(node->row_id_buf, 0, DATA_ROW_ID_LEN);
 		node->index = dict_table_get_first_index(node->table);
 		node->entry = UT_LIST_GET_FIRST(node->entry_list);
 		node->state = INS_NODE_INSERT_ENTRIES;
 		goto do_insert;
 	}
 
-	trx_write_trx_id(node->trx_id_buf, trx->id);
+	trx_write_trx_id(&node->sys_buf[DATA_ROW_ID_LEN], trx->id);
 
 	if (node->state == INS_NODE_SET_IX_LOCK) {
 

storage/innobase/row/row0mysql.cc

@@ -1523,7 +1523,7 @@ row_insert_for_mysql(
 
 	if (prebuilt->clust_index_was_generated) {
 		/* set row id to prebuilt */
-		ut_memcpy(prebuilt->row_id, node->row_id_buf, DATA_ROW_ID_LEN);
+		memcpy(prebuilt->row_id, node->sys_buf, DATA_ROW_ID_LEN);
 	}
 
 	dict_stats_update_if_needed(table);

storage/innobase/trx/trx0rec.cc

@@ -954,6 +954,7 @@ trx_undo_page_report_modify(
 				  dict_index_get_sys_col_pos(
 					  index, DATA_ROLL_PTR), &flen);
 	ut_ad(flen == DATA_ROLL_PTR_LEN);
+	ut_ad(memcmp(field, field_ref_zero, DATA_ROLL_PTR_LEN));
 
 	ptr += mach_u64_write_compressed(ptr, trx_read_roll_ptr(field));
 
@@ -2219,6 +2220,8 @@ trx_undo_get_undo_rec_low(
 
 	trx_undo_decode_roll_ptr(roll_ptr, &is_insert, &rseg_id, &page_no,
 				 &offset);
+	ut_ad(page_no > FSP_FIRST_INODE_PAGE_NO);
+	ut_ad(offset >= TRX_UNDO_PAGE_HDR + TRX_UNDO_PAGE_HDR_SIZE);
 	rseg = is_temp
 		? trx_sys->temp_rsegs[rseg_id]
 		: trx_sys->rseg_array[rseg_id];

storage/rocksdb/mysql-test/rocksdb/t/disabled.def

@@ -70,6 +70,7 @@ blind_delete_without_tx_api: MDEV-12286: rocksdb.blind_delete_without_tx_api tes
 unique_check: wrong error number
 autoinc_vars_thread: debug sync point wait timed out
 information_schema: MDEV-14372: unstable testcase
+bloomfilter: MDEV-14562
 
 ##
 ## Tests that fail for some other reason

storage/xtradb/include/row0upd.h

 /*****************************************************************************
 
 Copyright (c) 1996, 2016, Oracle and/or its affiliates. All Rights Reserved.
+Copyright (c) 2018, MariaDB Corporation.
 
 This program is free software; you can redistribute it and/or modify it under
 the terms of the GNU General Public License as published by the Free Software
@@ -119,8 +120,7 @@ row_upd_rec_sys_fields(
 	dict_index_t*	index,	/*!< in: clustered index */
 	const ulint*	offsets,/*!< in: rec_get_offsets(rec, index) */
 	const trx_t*	trx,	/*!< in: transaction */
-	roll_ptr_t	roll_ptr);/*!< in: roll ptr of the undo log record,
-				  can be 0 during IMPORT */
+	roll_ptr_t	roll_ptr);/*!< in: DB_ROLL_PTR to the undo log */
 /*********************************************************************//**
 Sets the trx id or roll ptr field of a clustered index entry. */
 UNIV_INTERN

storage/xtradb/include/row0upd.ic

 /*****************************************************************************
 
 Copyright (c) 1996, 2012, Oracle and/or its affiliates. All Rights Reserved.
+Copyright (c) 2018, MariaDB Corporation.
 
 This program is free software; you can redistribute it and/or modify it under
 the terms of the GNU General Public License as published by the Free Software
@@ -153,8 +154,7 @@ row_upd_rec_sys_fields(
 	dict_index_t*	index,	/*!< in: clustered index */
 	const ulint*	offsets,/*!< in: rec_get_offsets(rec, index) */
 	const trx_t*	trx,	/*!< in: transaction */
-	roll_ptr_t	roll_ptr)/*!< in: roll ptr of the undo log record,
-				 can be 0 during IMPORT */
+	roll_ptr_t	roll_ptr)/*!< in: DB_ROLL_PTR to the undo log */
 {
 	ut_ad(dict_index_is_clust(index));
 	ut_ad(rec_offs_validate(rec, index, offsets));

storage/xtradb/row/row0import.cc

@@ -1770,7 +1770,7 @@ PageConverter::adjust_cluster_record(
 
 		row_upd_rec_sys_fields(
 			rec, m_page_zip_ptr, m_cluster_index, m_offsets,
-			m_trx, 0);
+			m_trx, roll_ptr_t(1) << 55);
 	}
 
 	return(err);