Merge pull request #1142 from codership/10.2-MDEV-15740

MDEV-15740 Fixes to Galera transaction recovery

mysql-test/suite/galera/disabled.def

@@ -44,4 +44,3 @@ galera.MW-328C : MDEV-17847 Galera test failure on MW-328[A|B|C]
 galera.galera_sst_xtrabackup-v2 : MDEV-17848 Galera test failure on galera_sst_xtrabackup-v2[_data_dir]
 galera.galera_sst_xtrabackup-v2_data_dir : MDEV-17848 Galera test failure on galera_sst_xtrabackup-v2[_data_dir]
 query_cache : MDEV-18137: Galera test failure on query_cache
-galera_gcache_recover_manytrx : MDEV-15740

mysql-test/suite/galera/r/galera_gcache_recover_manytrx.result

@@ -70,28 +70,55 @@ WHILE 1 DO
 INSERT INTO t1 (f2) VALUES (REPEAT('x', 1024 * 1024 * 10));
 END WHILE;
 END|
+connect node_1_insert_simple, 127.0.0.1, root, , test, $NODE_MYPORT_1;
+connect node_1_insert_multi, 127.0.0.1, root, , test, $NODE_MYPORT_1;
+connect node_1_insert_transaction, 127.0.0.1, root, , test, $NODE_MYPORT_1;
+connect node_1_update_simple, 127.0.0.1, root, , test, $NODE_MYPORT_1;
+connect node_1_insert_1k, 127.0.0.1, root, , test, $NODE_MYPORT_1;
+connect node_1_insert_1m, 127.0.0.1, root, , test, $NODE_MYPORT_1;
+connect node_1_insert_10m, 127.0.0.1, root, , test, $NODE_MYPORT_1;
+connection node_1_insert_simple;
 CALL insert_simple();;
+connection node_1_insert_multi;
 CALL insert_multi();;
+connection node_1_insert_transaction;
 CALL insert_transaction ();;
+connection node_1_update_simple;
 CALL update_simple ();;
+connection node_1_insert_1k;
 CALL insert_1k ();;
+connection node_1_insert_1m;
 CALL insert_1m ();;
+connection node_1_insert_10m;
 CALL insert_10m ();;
+connection node_2;
 SET SESSION wsrep_sync_wait = 0;
 Killing server ...
+connection node_1;
 Killing server ...
+connection node_1_insert_simple;
 ERROR HY000: Lost connection to MySQL server during query
+connection node_1_insert_multi;
 ERROR HY000: Lost connection to MySQL server during query
+connection node_1_insert_transaction;
 ERROR HY000: Lost connection to MySQL server during query
+connection node_1_update_simple;
 ERROR HY000: Lost connection to MySQL server during query
+connection node_1_insert_1k;
 ERROR HY000: Lost connection to MySQL server during query
+connection node_1_insert_1m;
 ERROR HY000: Lost connection to MySQL server during query
+connection node_1_insert_10m;
 ERROR HY000: Lost connection to MySQL server during query
+connection node_1;
 Performing --wsrep-recover ...
 Using --wsrep-start-position when starting mysqld ...
+connection node_2;
 Performing --wsrep-recover ...
 Using --wsrep-start-position when starting mysqld ...
+connection node_1;
 include/diff_servers.inc [servers=1 2]
+connection node_1;
 DROP TABLE t1;
 DROP TABLE ten;
 DROP PROCEDURE insert_simple;
@@ -100,8 +127,10 @@ DROP PROCEDURE insert_transaction;
 DROP PROCEDURE update_simple;
 DROP PROCEDURE insert_1k;
 DROP PROCEDURE insert_1m;
+connection node_1;
 CALL mtr.add_suppression("conflict state 7 after post commit");
 CALL mtr.add_suppression("Skipped GCache ring buffer recovery");
 include/assert_grep.inc [async IST sender starting to serve]
+connection node_2;
 CALL mtr.add_suppression("Skipped GCache ring buffer recovery");
 include/assert_grep.inc [Recovering GCache ring buffer: found gapless sequence]

mysql-test/suite/wsrep/r/wsrep-recover-v25,binlogon.rdiff

+--- r/wsrep-recover-v25.result	2019-01-27 15:38:58.819204748 +0200
++++ r/wsrep-recover-v25.reject	2019-01-27 15:39:49.967358994 +0200
+@@ -18,11 +18,10 @@
+ connection default;
+ SET DEBUG_SYNC = "now WAIT_FOR after_prepare_reached";
+ # Kill the server
+-Expect seqno 3
+-3
+-Expect 5 7
++Expect seqno 2
++2
++Expect 5
+ SELECT * FROM t1;
+ f1
+ 5
+-7
+ DROP TABLE t1;

mysql-test/suite/wsrep/r/wsrep-recover-v25.result

+# Kill the server
+Expect seqno 0
+0
+CREATE TABLE t1 (f1 INT PRIMARY KEY) ENGINE=InnoDB;
+# Kill the server
+Expect seqno 1
+1
+INSERT INTO t1 VALUES (5);
+# Kill the server
+Expect seqno 2
+2
+SELECT VARIABLE_VALUE `expect 2` FROM INFORMATION_SCHEMA.GLOBAL_STATUS WHERE VARIABLE_NAME = 'wsrep_last_committed';
+expect 2
+2
+connect con1, localhost, root;
+SET DEBUG_SYNC = "ha_commit_trans_after_prepare SIGNAL after_prepare_reached WAIT_FOR continue";
+INSERT INTO t1 VALUES (7);
+connection default;
+SET DEBUG_SYNC = "now WAIT_FOR after_prepare_reached";
+# Kill the server
+Expect seqno 3
+3
+Expect 5 7
+SELECT * FROM t1;
+f1
+5
+7
+DROP TABLE t1;

mysql-test/suite/wsrep/t/wsrep-recover-step.inc

+#
+# Macro to run wsrep recovery step. This is adapted from
+# suite/galera/include/galera_wsrep_recover.inc, with additional
+# option to pass binlog argument to recovery command. The macro
+# returns recovered position split in uuid and seqno parts.
+#
+# Arguments:
+#
+# wsrep_recover_binlog_opt - Binlog options to recovery command
+#
+# Return:
+#
+# wsrep_recover_start_position_uuid - UUID corresponding to recovered position
+# wsrep_recover_start_position_seqno - seqno corresponding to recovered position
+#
+
+--exec $MYSQLD --defaults-group-suffix=.1 --defaults-file=$MYSQLTEST_VARDIR/my.cnf --log-error=$MYSQL_TMP_DIR/galera_wsrep_recover.log --innodb --wsrep-recover $wsrep_recover_binlog_opt --core-file > $MYSQL_TMP_DIR/galera_wsrep_recover.log 2>&1
+
+--perl
+	use strict;
+	my $wsrep_start_position = `grep 'WSREP: Recovered position:' $ENV{MYSQL_TMP_DIR}/galera_wsrep_recover.log | sed 's/.*WSREP\:\ Recovered\ position://' | sed 's/^[ \t]*//'`;
+	chomp($wsrep_start_position);
+        die if $wsrep_start_position eq '';
+	open(FILE, ">", "$ENV{MYSQL_TMP_DIR}/galera_wsrep_start_position.inc") or die;
+	my ($uuid, $seqno) = split /:/, $wsrep_start_position;
+	print FILE "--let \$wsrep_recover_start_position_uuid = $uuid\n";
+	print FILE "--let \$wsrep_recover_start_position_seqno = $seqno\n";
+	close FILE;
+EOF
+
+--source $MYSQL_TMP_DIR/galera_wsrep_start_position.inc
+
+if ($wsrep_recover_start_position_uuid == '') {
+   --die "Could not obtain start_position_uuid."
+}
+
+if ($wsrep_recover_start_position_seqno == '') {
+   --die "Could not obtain start_position_seqno."
+}
+
+--remove_file $MYSQL_TMP_DIR/galera_wsrep_start_position.inc

mysql-test/suite/wsrep/t/wsrep-recover-v25.cnf

+!include ../my.cnf
+
+[mysqld.1]
+wsrep-on=ON
+wsrep-cluster-address=gcomm://
+wsrep-provider=@ENV.WSREP_PROVIDER
+binlog-format=ROW

mysql-test/suite/wsrep/t/wsrep-recover-v25.combinations

+[binlogon]
+log-bin
+
+[binlogoff]

mysql-test/suite/wsrep/t/wsrep-recover-v25.test

+#
+# Verify that the wsrep XID gets updated in InnoDB rollback segment
+# properly and can be recovered with --wsrep-recover
+#
+# The test runs the following scenarios:
+#
+# 1) The server is started but no SQL is run
+# 2) DDL is executed
+# 3) INSERT is executed
+# 4) Two INSERTs are executed so that the first one in order will be
+#    blocked after certification and the second one before entering
+#    commit order critical section.
+# 5) Two DMLs are executed so that the prepare step is run out of order.
+#    Both transactions are blocked before commit order critical section.
+#
+# After each scenario server is killed and the recovered position
+# is validated.
+#
+
+--source include/have_wsrep.inc
+--source include/have_innodb.inc
+--source include/have_wsrep_provider.inc
+--source include/have_debug_sync.inc
+
+#
+# Binlog option for recovery run. This must be set in the test because
+# combinations file causes log-bin option to be set from command line,
+# not via my.cnf.
+#
+--let $log_bin = `SELECT @@log_bin`
+if ($log_bin) {
+--let $wsrep_recover_binlog_opt = --log-bin
+}
+
+#
+# Scenario 1
+# The expected recovered seqno is 1 corresponding to initial cluster
+# configuration change.
+#
+--source include/kill_mysqld.inc
+--source wsrep-recover-step.inc
+--echo Expect seqno 0
+--echo $wsrep_recover_start_position_seqno
+
+--let $restart_parameters = --wsrep-start-position=$wsrep_recover_start_position_uuid:$wsrep_recover_start_position_seqno
+--source include/start_mysqld.inc
+--source include/wait_wsrep_ready.inc
+
+#
+# Senario 2
+# The expected recovered seqno is 1 corresponding to CREATE TABLE
+#
+
+CREATE TABLE t1 (f1 INT PRIMARY KEY) ENGINE=InnoDB;
+--source include/kill_mysqld.inc
+--source wsrep-recover-step.inc
+--echo Expect seqno 1
+--echo $wsrep_recover_start_position_seqno
+
+--let $restart_parameters = --wsrep-start-position=$wsrep_recover_start_position_uuid:$wsrep_recover_start_position_seqno
+--source include/start_mysqld.inc
+--source include/wait_wsrep_ready.inc
+
+#
+# Scenario 3
+# The expected recovered seqno is 2 corresponding to CREATE TABLE and INSERT.
+#
+# The expected wsrep_last_committed after the server is restarted is 2.
+#
+
+INSERT INTO t1 VALUES (5);
+--source include/kill_mysqld.inc
+--source wsrep-recover-step.inc
+--echo Expect seqno 2
+--echo $wsrep_recover_start_position_seqno
+--let $restart_parameters = --wsrep-start-position=$wsrep_recover_start_position_uuid:$wsrep_recover_start_position_seqno
+--source include/start_mysqld.inc
+--source include/wait_wsrep_ready.inc
+
+SELECT VARIABLE_VALUE `expect 2` FROM INFORMATION_SCHEMA.GLOBAL_STATUS WHERE VARIABLE_NAME = 'wsrep_last_committed';
+
+#
+# Scenario 4
+#
+# The INSERT gets prepared but not committed.
+#
+# If binlog is off, the expected outcome is that the INSERT gets committed
+# since it is already committed in the cluster. If binlog is on, the INSERT
+# should be rolled back during recovery phase since it has not yet
+# been logged into binlog.
+#
+--connect con1, localhost, root
+SET DEBUG_SYNC = "ha_commit_trans_after_prepare SIGNAL after_prepare_reached WAIT_FOR continue";
+--send INSERT INTO t1 VALUES (7)
+
+--connection default
+SET DEBUG_SYNC = "now WAIT_FOR after_prepare_reached";
+--source include/kill_mysqld.inc
+--source wsrep-recover-step.inc
+if ($log_bin) {
+  --echo Expect seqno 2
+}
+if (!$log_bin) {
+  --echo Expect seqno 3
+}
+--echo $wsrep_recover_start_position_seqno
+--let $restart_parameters = --wsrep-start-position=$wsrep_recover_start_position_uuid:$wsrep_recover_start_position_seqno
+--source include/start_mysqld.inc
+--source include/wait_wsrep_ready.inc
+
+if ($log_bin) {
+  --echo Expect 5
+}
+if (!$log_bin) {
+   --echo Expect 5 7
+}
+SELECT * FROM t1;
+
+DROP TABLE t1;

sql/handler.cc

@@ -1827,6 +1827,35 @@ static char* xid_to_str(char *buf, XID *xid)
 }
 #endif
 
+#ifdef WITH_WSREP
+static my_xid wsrep_order_and_check_continuity(XID *list, int len)
+{
+  wsrep_sort_xid_array(list, len);
+  wsrep_uuid_t uuid;
+  wsrep_seqno_t seqno;
+  if (wsrep_get_SE_checkpoint(uuid, seqno))
+  {
+    WSREP_ERROR("Could not read wsrep SE checkpoint for recovery");
+    return 0;
+  }
+  long long cur_seqno= seqno;
+  for (int i= 0; i < len; ++i)
+  {
+    if (!wsrep_is_wsrep_xid(list + i) ||
+        wsrep_xid_seqno(*(list + i)) != cur_seqno + 1)
+    {
+      WSREP_WARN("Discovered discontinuity in recovered wsrep "
+                 "transaction XIDs. Truncating the recovery list to "
+                 "%d entries", i);
+      break;
+    }
+    ++cur_seqno;
+  }
+  WSREP_INFO("Last wsrep seqno to be recovered %lld", cur_seqno);
+  return (cur_seqno < 0 ? 0 : cur_seqno);
+}
+#endif /* WITH_WSREP */
+
 /**
   recover() step of xa.
 
@@ -1864,6 +1893,19 @@ static my_bool xarecover_handlerton(THD *unused, plugin_ref plugin,
     {
       sql_print_information("Found %d prepared transaction(s) in %s",
                             got, hton_name(hton)->str);
+#ifdef WITH_WSREP
+      /* If wsrep_on=ON, XIDs are first ordered and then the range of
+         recovered XIDs is checked for continuity. All the XIDs which
+         are in continuous range can be safely committed if binlog
+         is off since they have already ordered and certified in the
+         cluster. */
+      my_xid wsrep_limit= 0;
+      if (WSREP_ON)
+      {
+        wsrep_limit= wsrep_order_and_check_continuity(info->list, got);
+      }
+#endif /* WITH_WSREP */
+
       for (int i=0; i < got; i ++)
       {
         my_xid x= WSREP_ON && wsrep_is_wsrep_xid(&info->list[i]) ?
@@ -1879,15 +1921,21 @@ static my_bool xarecover_handlerton(THD *unused, plugin_ref plugin,
           info->found_foreign_xids++;
           continue;
         }
-        if (info->dry_run)
+        if (IF_WSREP(!(wsrep_emulate_bin_log &&
+                       wsrep_is_wsrep_xid(info->list + i) &&
+                       x <= wsrep_limit) && info->dry_run,
+                     info->dry_run))
         {
           info->found_my_xids++;
           continue;
         }
         // recovery mode
-        if (info->commit_list ?
-            my_hash_search(info->commit_list, (uchar *)&x, sizeof(x)) != 0 :
-            tc_heuristic_recover == TC_HEURISTIC_RECOVER_COMMIT)
+        if (IF_WSREP((wsrep_emulate_bin_log &&
+                      wsrep_is_wsrep_xid(info->list + i) &&
+                      x <= wsrep_limit), false) ||
+            (info->commit_list ?
+             my_hash_search(info->commit_list, (uchar *)&x, sizeof(x)) != 0 :
+             tc_heuristic_recover == TC_HEURISTIC_RECOVER_COMMIT))
         {
 #ifndef DBUG_OFF
           int rc=

sql/wsrep_xid.cc

@@ -20,6 +20,8 @@
 #include "sql_class.h"
 #include "wsrep_mysqld.h" // for logging macros
 
+#include <algorithm> /* std::sort() */
+
 /*
  * WSREPXid
  */
@@ -154,3 +156,35 @@ bool wsrep_get_SE_checkpoint(wsrep_uuid_t& uuid, wsrep_seqno_t& seqno)
 
   return false;
 }
+
+/*
+  Sort order for XIDs. Wsrep XIDs are sorted according to
+  seqno in ascending order. Non-wsrep XIDs are considered
+  equal among themselves and greater than with respect
+  to wsrep XIDs.
+ */
+struct Wsrep_xid_cmp
+{
+  bool operator()(const XID& left, const XID& right) const
+  {
+    const bool left_is_wsrep= wsrep_is_wsrep_xid(&left);
+    const bool right_is_wsrep= wsrep_is_wsrep_xid(&right);
+    if (left_is_wsrep && right_is_wsrep)
+    {
+      return (wsrep_xid_seqno(left) < wsrep_xid_seqno(right));
+    }
+    else if (left_is_wsrep)
+    {
+      return true;
+    }
+    else
+    {
+      return false;
+    }
+  }
+};
+
+void wsrep_sort_xid_array(XID *array, int len)
+{
+  std::sort(array, array + len, Wsrep_xid_cmp());
+}

sql/wsrep_xid.h

@@ -32,5 +32,7 @@ bool wsrep_get_SE_checkpoint(wsrep_uuid_t&, wsrep_seqno_t&);
 //void wsrep_set_SE_checkpoint(XID&);             /* uncomment if needed */
 bool wsrep_set_SE_checkpoint(const wsrep_uuid_t&, wsrep_seqno_t);
 
+void wsrep_sort_xid_array(XID *array, int len);
+
 #endif /* WITH_WSREP */
 #endif /* WSREP_UTILS_H */

storage/innobase/handler/ha_innodb.cc

@@ -17260,6 +17260,14 @@ innobase_rollback_by_xid(
 	}
 
 	if (trx_t* trx = trx_get_trx_by_xid(xid)) {
+#ifdef WITH_WSREP
+		/* If a wsrep transaction is being rolled back during
+		the recovery, we must clear the xid in order to avoid
+		writing serialisation history for rolled back transaction. */
+		if (wsrep_is_wsrep_xid(trx->xid)) {
+			trx->xid->null();
+		}
+#endif /* WITH_WSREP */
 		int ret = innobase_rollback_trx(trx);
 		trx_deregister_from_2pc(trx);
 		ut_ad(!trx->will_lock);