MDEV-742 XA PREPAREd transaction survive disconnect/server restart

Lifted long standing limitation to the XA of rolling it back at the
transaction's
connection close even if the XA is prepared.

Prepared XA-transaction is made to sustain connection close or server
restart.
The patch consists of

    - binary logging extension to write prepared XA part of
      transaction signified with
      its XID in a new XA_prepare_log_event. The concusion part -
      with Commit or Rollback decision - is logged separately as
      Query_log_event.
      That is in the binlog the XA consists of two separate group of
      events.

      That makes the whole XA possibly interweaving in binlog with
      other XA:s or regular transaction but with no harm to
      replication and data consistency.

      Gtid_log_event receives two more flags to identify which of the
      two XA phases of the transaction it represents. With either flag
      set also XID info is added to the event.

      When binlog is ON on the server XID::formatID is
      constrained to 4 bytes.

    - engines are made aware of the server policy to keep up user
      prepared XA:s so they (Innodb, rocksdb) don't roll them back
      anymore at their disconnect methods.

    - slave applier is refined to cope with two phase logged XA:s
      including parallel modes of execution.

This patch does not address crash-safe logging of the new events which
is being addressed by MDEV-21469.

CORNER CASES: read-only, pure myisam, binlog-*, @@skip_log_bin, etc

Are addressed along the following policies.
1. The read-only at reconnect marks XID to fail for future
   completion with ER_XA_RBROLLBACK.

2. binlog-* filtered XA when it changes engine data is regarded as
   loggable even when nothing got cached for binlog.  An empty
   XA-prepare group is recorded. Consequent Commit-or-Rollback
   succeeds in the Engine(s) as well as recorded into binlog.

3. The same applies to the non-transactional engine XA.

4. @@skip_log_bin=OFF does not record anything at XA-prepare
   (obviously), but the completion event is recorded into binlog to
   admit inconsistency with slave.

The following actions are taken by the patch.

At XA-prepare:
   when empty binlog cache - don't do anything to binlog if RO,
   otherwise write empty XA_prepare (assert(binlog-filter case)).

At Disconnect:
   when Prepared && RO (=> no binlogging was done)
     set Xid_cache_element::error := ER_XA_RBROLLBACK
     *keep* XID in the cache, and rollback the transaction.

At XA-"complete":
   Discover the error, if any don't binlog the "complete",
   return the error to the user.

Kudos
-----
Alexey Botchkov took to drive this work initially.
Sergei Golubchik, Sergei Petrunja, Marko Mäkelä provided a number of
good recommendations.
Sergei Voitovich made a magnificent review and improvements to the code.
They all deserve a bunch of thanks for making this work done!

mysql-test/include/kill_and_restart_mysqld.inc

+if (!$restart_parameters)
+{
+  let $restart_parameters = restart;
+}
+
+--let $_server_id= `SELECT @@server_id`
+--let $_expect_file_name= $MYSQLTEST_VARDIR/tmp/mysqld.$_server_id.expect
+
+--echo # Kill and $restart_parameters
+--exec echo "$restart_parameters" > $_expect_file_name
+--shutdown_server 0
+--source include/wait_until_disconnected.inc
+--enable_reconnect
+--source include/wait_until_connected_again.inc
+--disable_reconnect

mysql-test/main/flush_read_lock.result

@@ -1310,6 +1310,8 @@ unlock tables;
 # Check that XA non-COMMIT statements are not and COMMIT is
 # blocked by active FTWRL in another connection
 #
+# XA COMMIT, XA ROLLBACK and XA PREPARE does take COMMIT lock to ensure
+# that nothing is written to bin log and redo log under FTWRL mode.
 connection con1;
 flush tables with read lock;
 connection default;
@@ -1322,11 +1324,25 @@ connection con1;
 flush tables with read lock;
 connection default;
 xa end 'test1';
-xa prepare 'test1';
+xa prepare 'test1';;
+connection con1;
+unlock tables;
+# Switching to connection 'default'.
+connection default;
+# Reap XA PREPARE.
+# Switching to connection 'con1'.
+connection con1;
+flush tables with read lock;
+# Switching to connection 'default'.
+connection default;
+# Send XA ROLLBACK 'test1'
 xa rollback 'test1';
+# Switching to connection 'con1'.
 connection con1;
+# Wait until XA ROLLBACK is blocked.
 unlock tables;
 connection default;
+# Reap XA ROLLBACK
 xa start 'test1';
 insert into t3_trans values (1);
 connection con1;
@@ -1334,7 +1350,20 @@ flush tables with read lock;
 connection default;
 connection default;
 xa end 'test1';
+# Send XA PREPARE 'test1'
 xa prepare 'test1';
+# Switching to connection 'con1'.
+connection con1;
+# Wait until XA PREPARE is blocked.
+unlock tables;
+# Switching to connection 'default'.
+connection default;
+# Reap XA PREPARE.
+# Switching to connection 'con1'.
+connection con1;
+flush tables with read lock;
+# Switching to connection 'default'.
+connection default;
 # Send:
 xa commit 'test1';;
 connection con1;
@@ -1344,6 +1373,51 @@ connection default;
 # Reap XA COMMIT.
 delete from t3_trans;
 #
+# Check that XA COMMIT / ROLLBACK for prepared transaction from a
+# disconnected session is blocked by active FTWRL in another connection.
+#
+# Create temporary connection for XA transaction.
+connect  con_tmp,localhost,root,,;
+xa start 'test1';
+insert into t3_trans values (1);
+xa end 'test1';
+xa prepare 'test1';
+# Disconnect temporary connection
+disconnect con_tmp;
+# Create temporary connection for XA transaction.
+connect  con_tmp,localhost,root,,;
+xa start 'test2';
+insert into t3_trans values (2);
+xa end 'test2';
+xa prepare 'test2';
+# Disconnect temporary connection
+disconnect con_tmp;
+# Switching to connection 'con1'.
+connection con1;
+flush tables with read lock;
+# Switching to connection 'default'.
+connection default;
+# Send XA ROLLBACK 'test1'
+xa rollback 'test1';
+# Switching to connection 'con1'.
+connection con1;
+# Wait until XA ROLLBACK is blocked.
+unlock tables;
+flush tables with read lock;
+# Switching to connection 'default'.
+connection default;
+# Reap XA ROLLBACK
+# Send XA COMMIT
+xa commit 'test2';;
+# Switching to connection 'con1'.
+connection con1;
+# Wait until XA COMMIT is blocked.
+unlock tables;
+# Switching to connection 'default'.
+connection default;
+# Reap XA COMMIT.
+delete from t3_trans;
+#
 # Check that XA COMMIT blocks FTWRL in another connection.
 xa start 'test1';
 insert into t3_trans values (1);

mysql-test/main/flush_read_lock.test

@@ -1592,6 +1592,8 @@ unlock tables;
 --echo # Check that XA non-COMMIT statements are not and COMMIT is
 --echo # blocked by active FTWRL in another connection
 --echo #
+--echo # XA COMMIT, XA ROLLBACK and XA PREPARE does take COMMIT lock to ensure
+--echo # that nothing is written to bin log and redo log under FTWRL mode.
 connection $con_aux1;
 flush tables with read lock;
 connection default;
@@ -1604,11 +1606,37 @@ connection $con_aux1;
 flush tables with read lock;
 connection default;
 xa end 'test1';
-xa prepare 'test1';
-xa rollback 'test1';
+--send xa prepare 'test1';
 connection $con_aux1;
+let $wait_condition=
+  select count(*) = 1 from information_schema.processlist
+  where state = "Waiting for backup lock" and
+        info = "xa prepare 'test1'";
+--source include/wait_condition.inc
 unlock tables;
+--echo # Switching to connection 'default'.
+connection default;
+--echo # Reap XA PREPARE.
+--reap
+--echo # Switching to connection '$con_aux1'.
+connection $con_aux1;
+flush tables with read lock;
+--echo # Switching to connection 'default'.
 connection default;
+--echo # Send XA ROLLBACK 'test1'
+--send xa rollback 'test1'
+--echo # Switching to connection '$con_aux1'.
+connection $con_aux1;
+--echo # Wait until XA ROLLBACK is blocked.
+let $wait_condition=
+  select count(*) = 1 from information_schema.processlist
+  where state = "Waiting for backup lock" and
+        info = "xa rollback 'test1'";
+--source include/wait_condition.inc
+unlock tables;
+connection default;
+--echo # Reap XA ROLLBACK
+--reap
 xa start 'test1';
 insert into t3_trans values (1);
 connection $con_aux1;
@@ -1616,7 +1644,27 @@ flush tables with read lock;
 connection default;
 connection default;
 xa end 'test1';
-xa prepare 'test1';
+--echo # Send XA PREPARE 'test1'
+--send xa prepare 'test1'
+--echo # Switching to connection '$con_aux1'.
+connection $con_aux1;
+--echo # Wait until XA PREPARE is blocked.
+let $wait_condition=
+  select count(*) = 1 from information_schema.processlist
+  where state = "Waiting for backup lock" and
+        info = "xa prepare 'test1'";
+--source include/wait_condition.inc
+unlock tables;
+--echo # Switching to connection 'default'.
+connection default;
+--echo # Reap XA PREPARE.
+--reap
+--echo # Switching to connection '$con_aux1'.
+connection $con_aux1;
+flush tables with read lock;
+--echo # Switching to connection 'default'.
+connection default;
+
 --echo # Send:
 --send xa commit 'test1';
 connection $con_aux1;
@@ -1631,6 +1679,64 @@ connection default;
 --echo # Reap XA COMMIT.
 --reap
 delete from t3_trans;
+--echo #
+--echo # Check that XA COMMIT / ROLLBACK for prepared transaction from a
+--echo # disconnected session is blocked by active FTWRL in another connection.
+--echo #
+--echo # Create temporary connection for XA transaction.
+connect (con_tmp,localhost,root,,);
+xa start 'test1';
+insert into t3_trans values (1);
+xa end 'test1';
+xa prepare 'test1';
+--echo # Disconnect temporary connection
+disconnect con_tmp;
+--echo # Create temporary connection for XA transaction.
+connect (con_tmp,localhost,root,,);
+xa start 'test2';
+insert into t3_trans values (2);
+xa end 'test2';
+xa prepare 'test2';
+--echo # Disconnect temporary connection
+disconnect con_tmp;
+--echo # Switching to connection '$con_aux1'.
+connection $con_aux1;
+flush tables with read lock;
+--echo # Switching to connection 'default'.
+connection default;
+--echo # Send XA ROLLBACK 'test1'
+--send xa rollback 'test1'
+--echo # Switching to connection '$con_aux1'.
+connection $con_aux1;
+--echo # Wait until XA ROLLBACK is blocked.
+let $wait_condition=
+  select count(*) = 1 from information_schema.processlist
+  where state = "Waiting for backup lock" and
+        info = "xa rollback 'test1'";
+--source include/wait_condition.inc
+unlock tables;
+flush tables with read lock;
+--echo # Switching to connection 'default'.
+connection default;
+--echo # Reap XA ROLLBACK
+--reap
+--echo # Send XA COMMIT
+--send xa commit 'test2';
+--echo # Switching to connection '$con_aux1'.
+connection $con_aux1;
+--echo # Wait until XA COMMIT is blocked.
+let $wait_condition=
+  select count(*) = 1 from information_schema.processlist
+  where state = "Waiting for backup lock" and
+        info = "xa commit 'test2'";
+--source include/wait_condition.inc
+unlock tables;
+--echo # Switching to connection 'default'.
+connection default;
+--echo # Reap XA COMMIT.
+--reap
+delete from t3_trans;
+
 --echo #
 --echo # Check that XA COMMIT blocks FTWRL in another connection.
 xa start 'test1';

mysql-test/main/xa.result

@@ -59,6 +59,9 @@ select * from t1;
 a
 20
 disconnect con1;
+xa rollback 'testb',0x2030405060,11;
+xa recover;
+formatID	gtrid_length	bqual_length	data
 connection default;
 xa start 'tr1';
 insert t1 values (40);
@@ -400,3 +403,61 @@ XA ROLLBACK 'xid1';
 #
 XA START 'gtrid', 'bqual', 0x80000000;
 ERROR 42000: You have an error in your SQL syntax; check the manual that corresponds to your MariaDB server version for the right syntax to use near '0x80000000' at line 1
+# MDEV-7974 related
+# Check XA state when lock_wait_timeout happens
+# More tests added to flush_read_lock.test
+connect  con_tmp,localhost,root,,;
+set session lock_wait_timeout=1;
+create table asd (a int) engine=innodb;
+xa start 'test1';
+insert into asd values(1);
+xa end 'test1';
+connection default;
+flush table with read lock;
+connection con_tmp;
+# PREPARE error will do auto rollback.
+xa prepare 'test1';
+ERROR HY000: Lock wait timeout exceeded; try restarting transaction
+show errors;
+Level	Code	Message
+Error	1205	Lock wait timeout exceeded; try restarting transaction
+Error	1402	XA_RBROLLBACK: Transaction branch was rolled back
+connection default;
+unlock tables;
+connection con_tmp;
+xa start 'test1';
+insert into asd values(1);
+xa end 'test1';
+xa prepare 'test1';
+connection default;
+flush tables with read lock;
+connection con_tmp;
+# LOCK error during ROLLBACK will not alter transaction state.
+xa rollback 'test1';
+ERROR HY000: Lock wait timeout exceeded; try restarting transaction
+show errors;
+Level	Code	Message
+Error	1205	Lock wait timeout exceeded; try restarting transaction
+Error	1401	XAER_RMERR: Fatal error occurred in the transaction branch - check your data for consistency
+xa recover;
+formatID	gtrid_length	bqual_length	data
+1	5	0	test1
+# LOCK error during COMMIT will not alter transaction state.
+xa commit 'test1';
+ERROR HY000: Lock wait timeout exceeded; try restarting transaction
+show errors;
+Level	Code	Message
+Error	1205	Lock wait timeout exceeded; try restarting transaction
+Error	1401	XAER_RMERR: Fatal error occurred in the transaction branch - check your data for consistency
+xa recover;
+formatID	gtrid_length	bqual_length	data
+1	5	0	test1
+connection default;
+unlock tables;
+connection con_tmp;
+xa rollback 'test1';
+xa recover;
+formatID	gtrid_length	bqual_length	data
+drop table asd;
+disconnect con_tmp;
+connection default;

mysql-test/main/xa.test

@@ -82,6 +82,8 @@ xa start 'zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz';
 select * from t1;
 disconnect con1;
 --source include/wait_until_count_sessions.inc
+xa rollback 'testb',0x2030405060,11;
+xa recover;
 
 connection default;
 
@@ -541,3 +543,50 @@ XA ROLLBACK 'xid1';
 XA START 'gtrid', 'bqual', 0x80000000;
 
 --source include/wait_until_count_sessions.inc
+
+--echo # MDEV-7974 related
+--echo # Check XA state when lock_wait_timeout happens
+--echo # More tests added to flush_read_lock.test
+connect (con_tmp,localhost,root,,);
+set session lock_wait_timeout=1;
+create table asd (a int) engine=innodb;
+xa start 'test1';
+insert into asd values(1);
+xa end 'test1';
+connection default;
+flush table with read lock;
+connection con_tmp;
+--echo # PREPARE error will do auto rollback.
+--ERROR ER_LOCK_WAIT_TIMEOUT
+xa prepare 'test1';
+show errors;
+connection default;
+unlock tables;
+
+connection con_tmp;
+xa start 'test1';
+insert into asd values(1);
+xa end 'test1';
+xa prepare 'test1';
+connection default;
+flush tables with read lock;
+connection con_tmp;
+--echo # LOCK error during ROLLBACK will not alter transaction state.
+--ERROR ER_LOCK_WAIT_TIMEOUT
+xa rollback 'test1';
+show errors;
+xa recover;
+--echo # LOCK error during COMMIT will not alter transaction state.
+--ERROR ER_LOCK_WAIT_TIMEOUT
+xa commit 'test1';
+show errors;
+xa recover;
+connection default;
+unlock tables;
+connection con_tmp;
+xa rollback 'test1';
+xa recover;
+drop table asd;
+disconnect con_tmp;
+--source include/wait_until_disconnected.inc
+connection default;

mysql-test/main/xa_binlog.result

@@ -18,14 +18,17 @@ a
 1
 2
 3
-SHOW BINLOG EVENTS LIMIT 3,9;
+SHOW BINLOG EVENTS LIMIT 3,12;
 Log_name	Pos	Event_type	Server_id	End_log_pos	Info
-master-bin.000001	#	Gtid	1	#	BEGIN GTID #-#-#
+master-bin.000001	#	Gtid	1	#	XA START X'786174657374',X'',1 GTID #-#-#
 master-bin.000001	#	Query	1	#	use `test`; INSERT INTO t1 VALUES (1)
-master-bin.000001	#	Query	1	#	COMMIT
+master-bin.000001	#	Query	1	#	XA END X'786174657374',X'',1
+master-bin.000001	#	XA_prepare	1	#	XA PREPARE X'786174657374',X'',1
+master-bin.000001	#	Gtid	1	#	GTID #-#-#
+master-bin.000001	#	Query	1	#	XA COMMIT X'786174657374',X'',1
 master-bin.000001	#	Gtid	1	#	BEGIN GTID #-#-#
 master-bin.000001	#	Query	1	#	use `test`; INSERT INTO t1 VALUES (2)
-master-bin.000001	#	Query	1	#	COMMIT
+master-bin.000001	#	Xid	1	#	COMMIT /* xid=XX */
 master-bin.000001	#	Gtid	1	#	BEGIN GTID #-#-#
 master-bin.000001	#	Query	1	#	use `test`; INSERT INTO t1 VALUES (3)
 master-bin.000001	#	Xid	1	#	COMMIT /* xid=XX */

mysql-test/main/xa_binlog.test

@@ -27,6 +27,6 @@ SELECT * FROM t1 ORDER BY a;
 
 --replace_column 2 # 5 #
 --replace_regex /xid=[0-9]+/xid=XX/ /GTID [0-9]+-[0-9]+-[0-9]+/GTID #-#-#/
-SHOW BINLOG EVENTS LIMIT 3,9;
+SHOW BINLOG EVENTS LIMIT 3,12;
 
 DROP TABLE t1;

mysql-test/main/xa_prepared_binlog_off-master.opt

+--skip-log-bin

mysql-test/main/xa_prepared_binlog_off.test

+###############################################################################
+# MDEV-7974 (bug#12161 Xa recovery and client disconnection)
+# Testing XA behaviour with binlog turned off.
+###############################################################################
+
+--source include/not_valgrind.inc
+--source include/not_embedded.inc
+
+# Common part with XA binlogging testing
+call mtr.add_suppression("You need to use --log-bin to make --log-slave-updates work.");
+--source suite/binlog/t/binlog_xa_prepared.inc

mysql-test/main/xa_sync.result

@@ -18,6 +18,11 @@ disconnect con1;
 SET debug_sync='now SIGNAL go';
 connection con2;
 ERROR XAE04: XAER_NOTA: Unknown XID
+*** Must have 'xatest' in the list
+XA RECOVER;
+formatID	gtrid_length	bqual_length	data
+1	6	0	xatest
+XA COMMIT 'xatest';
 disconnect con2;
 connection default;
 SET debug_sync='RESET';
@@ -37,6 +42,11 @@ disconnect con1;
 SET debug_sync='now SIGNAL go';
 connection con2;
 ERROR XAE04: XAER_NOTA: Unknown XID
+*** Must have 'xatest' in the list
+XA RECOVER;
+formatID	gtrid_length	bqual_length	data
+1	6	0	xatest
+XA ROLLBACK 'xatest';
 disconnect con2;
 connection default;
 SET debug_sync='RESET';

mysql-test/main/xa_sync.test

@@ -35,6 +35,11 @@ while ($i)
   connection con2;
   --error ER_XAER_NOTA
   reap;
+  --echo *** Must have 'xatest' in the list
+  XA RECOVER;
+  # second time yields no error
+  --error 0,1402
+  --eval $op
   disconnect con2;
 
   connection default;

mysql-test/suite/binlog/include/binlog_xa_prepare_connection.inc

+#
+# This file initiate connections to run XA transactions up to
+# their prepare.
+# Connection name, transaction name and its content depends on
+# supplied parameters.
+#
+# param $type        type of transaction
+# param $index       index identifies the connection with those of type $type
+# param $sql_init1   a query to execute once connection is established
+# param $sql_init2   a query to execute once connection is established
+# param $sql_doit    a query to execute inside transaction
+#                    Note, the query may depend on tables created by caller
+#
+
+--connect (conn$index$type, 127.0.0.1,root,,test,$MASTER_MYPORT,)
+if ($sql_init1)
+{
+  --eval $sql_init1
+}
+if ($sql_init2)
+{
+  --eval $sql_init2
+}
+
+--eval XA START   'trx$index$type'
+if ($sql_doit)
+{
+  --eval $sql_doit
+}
+--eval XA END     'trx$index$type'
+--eval XA PREPARE 'trx$index$type'

mysql-test/suite/binlog/include/binlog_xa_prepare_disconnect.inc

+#
+# This file disconnects two connections. One actively and one through
+# kill. It is included by binlog_xa_prepared_do_and_restart.
+#
+# param $type              type of transaction
+# param $terminate_with    how to conclude actively disconnecte:
+#                          XA COMMIT or XA ROLLBACK
+# param $conn3_id          connection id of the being killed.
+# param $num_trx_prepared  number of transactions prepared so far
+#
+--connection default
+
+--echo *** $num_trx_prepared prepared transactions must be in the list ***
+--replace_column 2 LEN1 3 LEN2 4 TRX_N
+XA RECOVER;
+
+--connection conn1$type
+--let $conn1_id=`SELECT connection_id()`
+--disconnect conn1$type
+
+--connection default
+--let $wait_condition= SELECT count(*) = 0 FROM v_processlist WHERE PROCESSLIST_ID = $conn1_id
+--source include/wait_condition.inc
+
+# It will conclude now
+--error 0,1402
+--eval  $terminate_with 'trx1$type'
+
+--replace_result $conn3_id CONN_ID
+--eval KILL connection $conn3_id
+
+--let $wait_condition= SELECT count(*) = 0 FROM v_processlist WHERE PROCESSLIST_ID = $conn3_id
+--source include/wait_condition.inc
+
+# It will conclude now
+--error 0,1402
+--eval  $terminate_with 'trx3$type'

mysql-test/suite/binlog/include/binlog_xa_prepared_do_and_restart.inc

+#
+# This file creates various kinds of prepared XA transactions,
+# manipulates their connection state and examines how their prepared
+# status behave while the transaction is disconnected, killed or
+# the server kisses it shutdown.
+# The file can be sourced multiple times
+# param $restart_number (as the number of inclusion) adjusts
+# verification logics.
+#
+# param [in] $conn_number      Total number of connection each performing
+#                              one insert into table.
+# param [in] $commit_number    Number of commits from either.
+#                              side of the server restart.
+# param [in] $rollback_number  The same as the above just for rollback.
+# param [in] $term_number      Number of transaction that are terminated
+#                              before server restarts
+# param [in] $killed_number    Instead of disconnect make some
+#                              connections killed when their
+#                              transactions got prepared.
+# param [in] $server_disconn_number  Make some connections disconnected
+#                              by shutdown rather than actively
+# param [in] $post_restart_conn_number  Number a "warmup" connection
+#                              after server restart, they all commit
+# param [out] restart_number   Counter to be incremented at the end of the test
+#
+
+# The test consists of three sections:
+# I.   Corner cases check
+# II.  Regular case check
+# III. Post server-restart verification
+
+
+#
+# I. Corner cases of
+#
+# A. XA with an update to a temp table
+# B. XA with SELECT
+# C. XA empty
+# Demonstrate their XA status upon prepare and how they react on disconnect and
+# shutdown.
+# In each of A,B,C three prepared transactions are set up.
+# trx1 is for disconnection, trx2 for shutdown, trx3 for being killed.
+# The A case additionally contains some XA prohibited state transaction check.
+#
+# D. Prove that not prepared XA remains to be cleared out by disconnection.
+#
+
+#
+# A. The temp table only prepared XA recovers only formally to
+#    let post recovery XA COMMIT or XA ROLLBACK with no effect.
+
+--let $type = tmp
+--let $index = 1
+--let $sql_init1 = SET @@sql_log_bin = OFF
+--let $sql_init2 = CREATE TEMPORARY TABLE tmp$index (a int) ENGINE=innodb
+--let $sql_doit  = INSERT INTO tmp$index SET a=$index
+--source suite/binlog/include/binlog_xa_prepare_connection.inc
+
+--let $index = 2
+--source suite/binlog/include/binlog_xa_prepare_connection.inc
+
+--let $index = 3
+--source suite/binlog/include/binlog_xa_prepare_connection.inc
+--let $conn3_id=`SELECT connection_id()`
+
+#
+# Various prohibited XA state changes to test here:
+#
+
+--connection default
+# Stealing is not allowed
+--error ER_XAER_NOTA
+--eval  XA COMMIT 'trx1$type'
+--error ER_XAER_NOTA
+--eval  XA ROLLBACK 'trx1$type'
+
+# Before disconnect: creating a duplicate is not allowed
+--error ER_XAER_DUPID
+--eval  XA START 'trx1$type'
+
+# Manipulate now the prepared transactions.
+# Two to terminate, one to leave out.
+--let $terminate_with = XA COMMIT
+--let $num_trx_prepared = $index
+--source suite/binlog/include/binlog_xa_prepare_disconnect.inc
+
+#
+# B. "Read-only" (select) prepared XA recovers only formally to
+#    let post recovery XA COMMIT or XA ROLLBACK with no effect.
+#
+--let $type=ro
+--let $index = 1
+--let $sql_init1 =
+--let $sql_init2 =
+--let $sql_doit  = SELECT * from t ORDER BY a
+--source suite/binlog/include/binlog_xa_prepare_connection.inc
+
+--let $index = 2
+--source suite/binlog/include/binlog_xa_prepare_connection.inc
+
+--let $index = 3
+--source suite/binlog/include/binlog_xa_prepare_connection.inc
+--let $conn3_id=`SELECT connection_id()`
+
+--let $terminate_with = XA ROLLBACK
+# two three above section prepared transaction were terminated.
+--inc $num_trx_prepared
+--source suite/binlog/include/binlog_xa_prepare_disconnect.inc
+
+#
+# C. Empty prepared XA recovers only formally to
+#    let post recovery XA COMMIT or XA ROLLBACK with no effect.
+#
+--let $type=empty
+--let $index = 1
+--let $sql_init1 =
+--let $sql_init2 =
+--let $sql_doit  =
+--source suite/binlog/include/binlog_xa_prepare_connection.inc
+
+--let $index = 2
+--source suite/binlog/include/binlog_xa_prepare_connection.inc
+
+--let $index = 3
+--source suite/binlog/include/binlog_xa_prepare_connection.inc
+--let $conn3_id=`SELECT connection_id()`
+
+--let $terminate_with = XA COMMIT
+--inc $num_trx_prepared
+--source suite/binlog/include/binlog_xa_prepare_disconnect.inc
+
+#
+# D. Not prepared XA disconnects to be cleared out,
+#    no effect on data left as well.
+#    Few more prohibited XA state transactions is checked out.
+#
+--let $type=unprepared
+--let $prev_count=`SELECT count(*) from t`
+
+--connect(conn1$type, 127.0.0.1,root,,test,$MASTER_MYPORT,)
+--eval XA START   'trx1$type'
+INSERT INTO t set a=0;
+--eval XA END     'trx1$type'
+
+--error ER_XAER_RMFAIL
+INSERT INTO t set a=0;
+--error ER_XAER_RMFAIL
+--eval XA START 'trx1$type'
+--error ER_XAER_RMFAIL
+--eval XA START 'trx1$type'
+
+--disconnect conn1$type
+
+--connection default
+# No such transactions
+--error ER_XAER_NOTA
+--eval  XA COMMIT 'trx1$type'
+if (`SELECT count(*) > $prev_count from t`)
+{
+    --echo *** Unexpected commit to the table. ***
+    --die
+}
+
+#
+# II. Regular case.
+#
+# Prepared transactions get disconnected in three ways:
+# actively, being killed and by the server shutdown.
+#
+--let $i=0
+while ($i < $conn_number)
+{
+    --connect (conn$i, 127.0.0.1,root,,test,$MASTER_MYPORT,)
+    --let $conn_id=`SELECT connection_id()`
+      --disable_reconnect
+      SET @@binlog_format = STATEMENT;
+      if (`SELECT $i % 2`)
+      {
+         SET @@binlog_format = ROW;
+      }
+    --eval XA START   'trx_$i'
+    --eval INSERT INTO t SET a=$i
+    --eval XA END     'trx_$i'
+    --eval XA PREPARE 'trx_$i'
+
+    --let $disc_via_kill=`SELECT $conn_number - $i <= $killed_number`
+    if (!$disc_via_kill)
+    {
+      --let $disc_via_shutdown=`SELECT $conn_number - $i <= $killed_number + $server_disconn_number`
+      if (!$disc_via_shutdown)
+      {
+        --disconnect conn$i
+      }
+    }
+    if ($disc_via_kill)
+    {
+      --connection default
+      --replace_result $conn_id CONN_ID
+      --eval KILL CONNECTION $conn_id
+    }
+
+    if (!$disc_via_shutdown)
+    {
+      --connection default
+      --let $wait_condition= SELECT count(*) = 0 FROM v_processlist WHERE PROCESSLIST_ID = $conn_id
+      --source include/wait_condition.inc
+    }
+    --inc $i
+}
+
+# [0, $rollback_number - 1] are rolled back now
+--connection default
+
+--let $i=0
+while ($i < $rollback_number)
+{
+    --eval XA ROLLBACK 'trx_$i'
+
+    --inc $i
+}
+
+# [$rollback_number, $rollback_number + $commit_number - 1] get committed
+while ($i < $term_number)
+{
+    --eval XA COMMIT 'trx_$i'
+
+    --inc $i
+}
+
+--source include/$how_to_restart
+
+#
+# III. Post server-restart verification.
+# It concludes survived XA:s with a number of commits and rollbacks
+# as configured in the 1st part to check expected results in the end.
+# Cleanup section consists of explicit disconnect (for killed, or
+# not disconnected before shutdown).
+#
+
+# New XA can be prepared and committed
+--let $k = 0
+while ($k < $post_restart_conn_number)
+{
+    --connect (conn_restart_$k, 127.0.0.1,root,,test,$MASTER_MYPORT,)
+    --let  $conn_id=`SELECT connection_id()`
+    --eval XA START   'new_trx_$k'
+    --eval INSERT INTO t SET a=$k
+    --eval XA END     'new_trx_$k'
+    --eval XA PREPARE 'new_trx_$k'
+
+    --disconnect conn_restart_$k
+
+    --connection default
+    --let $wait_condition= SELECT count(*) = 0 FROM v_processlist WHERE PROCESSLIST_ID = $conn_id
+    --source include/wait_condition.inc
+
+    --inc $k
+}
+
+--connection default
+--let $k = 0
+while ($k < $post_restart_conn_number)
+{
+    --eval XA COMMIT  'new_trx_$k'
+    --inc $k
+}
+
+#
+# Symmetrically to the pre-restart, the resurrected trx:s are committed
+# [$term_number, $term_number + $commit_number - 1]
+# and the rest is rolled back.
+#
+--let $i = $term_number
+
+while ($i < `SELECT $term_number + $commit_number`)
+{
+    # Expected to fail
+    --error ER_XAER_DUPID
+    --eval XA START 'trx_$i'
+    --eval XA COMMIT 'trx_$i'
+    --inc $i
+}
+
+while ($i < $conn_number)
+{
+    # Expected to fail
+    --error ER_XAER_DUPID
+    --eval XA START 'trx_$i'
+    --eval XA ROLLBACK 'trx_$i'
+    --inc $i
+}
+
+#
+# Verification of correct results of recovered XA transaction handling:
+#
+SELECT * FROM t;
+
+--let $type=tmp
+--disconnect conn2$type
+--disconnect conn3$type
+--let $type=ro
+--disconnect conn2$type
+--disconnect conn3$type
+--let $type=empty
+--disconnect conn2$type
+--disconnect conn3$type
+
+--let $i= $conn_number
+--let $k= 0
+--let $expl_disconn_number = `SELECT $killed_number + $server_disconn_number`
+while ($k < $expl_disconn_number)
+{
+  --connection default
+    --error ER_XAER_NOTA
+    --eval  XA ROLLBACK 'trx_$i'
+
+  --dec $i
+  --disconnect conn$i
+
+  --inc $k
+}
+
+--inc $restart_number

mysql-test/suite/binlog/r/binlog_xa_checkpoint.result

+RESET MASTER;
+CREATE TABLE t1 (a INT PRIMARY KEY, b MEDIUMTEXT) ENGINE=Innodb;
+connect con1,localhost,root,,;
+SET DEBUG_SYNC= "at_unlog_xa_prepare SIGNAL con1_ready WAIT_FOR con1_go";
+XA START '1';
+INSERT INTO t1 SET a=1;
+XA END '1';
+XA PREPARE '1';;
+connection default;
+SET DEBUG_SYNC= "now WAIT_FOR con1_ready";
+FLUSH LOGS;
+FLUSH LOGS;
+FLUSH LOGS;
+show binary logs;
+Log_name	File_size
+master-bin.000001	#
+master-bin.000002	#
+master-bin.000003	#
+master-bin.000004	#
+include/show_binlog_events.inc
+Log_name	Pos	Event_type	Server_id	End_log_pos	Info
+master-bin.000004	#	Format_desc	#	#	SERVER_VERSION, BINLOG_VERSION
+master-bin.000004	#	Gtid_list	#	#	[#-#-#]
+master-bin.000004	#	Binlog_checkpoint	#	#	master-bin.000001
+SET DEBUG_SYNC= "now SIGNAL con1_go";
+connection con1;
+*** master-bin.000004 checkpoint must show up now ***
+connection con1;
+XA ROLLBACK '1';
+SET debug_sync = 'reset';
+connection default;
+DROP TABLE t1;
+SET debug_sync = 'reset';

mysql-test/suite/binlog/t/binlog_xa_checkpoint.test

+--source include/have_innodb.inc
+--source include/have_debug.inc
+--source include/have_debug_sync.inc
+--source include/have_binlog_format_row.inc
+
+RESET MASTER;
+
+CREATE TABLE t1 (a INT PRIMARY KEY, b MEDIUMTEXT) ENGINE=Innodb;
+
+# Test that
+# 1. XA PREPARE is binlogged before the XA has been prepared in Engine
+# 2. While XA PREPARE already binlogged in an old binlog file which has been rotated,
+#    Binlog checkpoint is not generated for the latest log until
+#    XA PREPARE returns, e.g OK to the client.
+
+
+# con1 will hang before doing commit checkpoint, blocking RESET MASTER.
+connect(con1,localhost,root,,);
+SET DEBUG_SYNC= "at_unlog_xa_prepare SIGNAL con1_ready WAIT_FOR con1_go";
+XA START '1';
+INSERT INTO t1 SET a=1;
+XA END '1';
+--send XA PREPARE '1';
+
+
+connection default;
+SET DEBUG_SYNC= "now WAIT_FOR con1_ready";
+FLUSH LOGS;
+FLUSH LOGS;
+FLUSH LOGS;
+
+--source include/show_binary_logs.inc
+--let $binlog_file= master-bin.000004
+--let $binlog_start= 4
+--source include/show_binlog_events.inc
+
+SET DEBUG_SYNC= "now SIGNAL con1_go";
+
+connection con1;
+reap;
+--echo *** master-bin.000004 checkpoint must show up now ***
+--source include/wait_for_binlog_checkpoint.inc
+
+# Todo: think about the error code returned, move to an appropriate test, or remove
+# connection default;
+#--error 1399
+# DROP TABLE t1;
+
+connection con1;
+XA ROLLBACK '1';
+SET debug_sync = 'reset';
+
+# Clean up.
+connection default;
+
+DROP TABLE t1;
+SET debug_sync = 'reset';

mysql-test/suite/binlog/t/binlog_xa_prepared.inc

+--source include/have_innodb.inc
+--source include/have_perfschema.inc
+#
+# The test verifies binlogging of XA transaction and state of prepared XA
+# as far as binlog is concerned.
+#
+# The prepared XA transactions can be disconnected from the client,
+# discovered from another connection and commited or rolled back
+# later. They also survive the server restart.  The test runs two
+# loops each consisting of prepared XA:s generation, their
+# manipulation and a server restart followed with survived XA:s
+# completion.
+#
+# Prepared XA can't get available to an external connection
+# until connection that either leaves actively or is killed
+# has completed a necessary part of its cleanup.
+# Selecting from P_S.threads provides a method to learn that.
+#
+# Total number of connection each performing one insert into table
+--let $conn_number=20
+# Number of rollbacks and commits from either side of the server restart
+--let $rollback_number=5
+--let $commit_number=5
+# Number of transactions that are terminated before server restarts
+--let $term_number=`SELECT $rollback_number + $commit_number`
+# Instead of disconnect make some connections killed when their
+# transactions got prepared.
+--let $killed_number=5
+# make some connections disconnected by shutdown rather than actively
+--let $server_disconn_number=5
+--let $prepared_at_server_restart = `SELECT $conn_number - $term_number`
+# number a "warmup" connection after server restart, they all commit
+--let $post_restart_conn_number=10
+
+# Counter to be used in GTID consistency check.
+# It's incremented per each non-XA transaction commit.
+# Local to this file variable to control one-phase commit loop
+--let $one_phase_number = 5
+
+--connection default
+
+# Remove possibly preceeding binlogs and clear initialization time
+# GTID executed info. In the following all transactions are counted
+# to conduct verification at the end of the test.
+if (`SELECT @@global.log_bin`)
+{
+  RESET MASTER;
+}
+
+# Disconected and follower threads need synchronization
+CREATE VIEW v_processlist  as SELECT * FROM performance_schema.threads where type = 'FOREGROUND';
+
+--eval call mtr.add_suppression("Found $prepared_at_server_restart prepared XA transactions")
+
+CREATE TABLE t (a INT) ENGINE=innodb;
+
+# Counter is incremented at the end of post restart to
+# reflect number of loops done in correctness computation.
+--let $restart_number = 0
+--let $how_to_restart=restart_mysqld.inc
+--source suite/binlog/include/binlog_xa_prepared_do_and_restart.inc
+
+--let $how_to_restart=kill_and_restart_mysqld.inc
+--source suite/binlog/include/binlog_xa_prepared_do_and_restart.inc
+
+--connection default
+
+# Few xs that commit in one phase, not subject to the server restart
+# nor reconnect.
+# This piece of test is related to mysqlbinlog recovery examine below.
+--let $k = 0
+while ($k < $one_phase_number)
+{
+    --eval XA START   'one_phase_trx_$k'
+    --eval INSERT INTO t SET a=$k
+    --eval XA END     'one_phase_trx_$k'
+    --eval XA COMMIT  'one_phase_trx_$k' ONE PHASE
+
+    --inc $k
+}
+
+SELECT SUM(a) FROM t;
+DROP TABLE t;
+DROP VIEW v_processlist;
+
+let $outfile= $MYSQLTEST_VARDIR/tmp/mysqlbinlog.sql;
+if (`SELECT @@global.log_bin`)
+{
+  # Recording proper samples of binlogged prepared XA:s
+  --source include/show_binlog_events.inc
+  --exec $MYSQL_BINLOG -R --to-last-log master-bin.000001 > $outfile
+}
+
+--echo All transactions must be completed, to empty-list the following:
+XA RECOVER;
+
+if (`SELECT @@global.log_bin`)
+{
+  --exec $MYSQL test < $outfile
+  --remove_file $outfile
+  XA RECOVER;
+}

mysql-test/suite/binlog/t/binlog_xa_prepared_disconnect.test

+###############################################################################
+# Bug#12161 Xa recovery and client disconnection
+# Testing new server options and binary logging prepared XA transaction.
+###############################################################################
+
+#
+# MIXED mode is chosen because formats are varied inside the sourced tests.
+#
+--source include/have_binlog_format_mixed.inc
+
+--source suite/binlog/t/binlog_xa_prepared.inc

mysql-test/suite/rpl/include/rpl_xa_mixed_engines.inc

+#
+# The test file is invoked from rpl.rpl_xa_survive_disconnect_mixed_engines
+#
+# The test file is orginized as three sections: setup, run and cleanup.
+# The main logics is resided in the run section which generates
+# three types of XA transaction: two kinds of mixed and one on non-transactional
+# table.
+#
+# param $command          one of three of: 'setup', 'run' or 'cleanup'
+# param $xa_terminate     how to conclude: 'XA COMMIT' or 'XA ROLLBACK'
+# param $one_phase        'one_phase' can be opted with XA COMMIT above
+# param $xa_prepare_opt   '1' or empty can be opted to test with and without XA PREPARE
+# param $xid              arbitrary name for xa trx, defaults to 'xa_trx'
+# Note '' is merely to underline, not a part of the value.
+#
+
+if ($command == setup)
+{
+  # Test randomizes the following variable's value:
+  SET @@session.binlog_direct_non_transactional_updates := if(floor(rand()*10)%2,'ON','OFF');
+  CREATE TABLE t (a INT) ENGINE=innodb;
+  CREATE TABLE tm (a INT) ENGINE=myisam;
+}
+if (!$xid)
+{
+  --let $xid=xa_trx
+}
+if ($command == run)
+{
+  ## Non-temporary table cases
+  # Non transactional table goes first
+  --eval XA START '$xid'
+  --disable_warnings
+  INSERT INTO tm VALUES (1);
+  INSERT INTO t VALUES (1);
+  --enable_warnings
+  --eval XA END '$xid'
+  if ($xa_prepare_opt)
+  {
+   --eval XA PREPARE '$xid'
+  }
+  --eval $xa_terminate '$xid' $one_phase
+
+  # Transactional table goes first
+  --eval XA START '$xid'
+  --disable_warnings
+  INSERT INTO t VALUES (2);
+  INSERT INTO tm VALUES (2);
+  --enable_warnings
+  --eval XA END '$xid'
+  if ($xa_prepare_opt)
+  {
+   --eval XA PREPARE '$xid'
+  }
+  --eval $xa_terminate '$xid' $one_phase
+
+  # The pure non-transactional table
+  --eval XA START '$xid'
+  --disable_warnings
+  INSERT INTO tm VALUES (3);
+  --enable_warnings
+  --eval XA END '$xid'
+  if ($xa_prepare_opt)
+  {
+   --eval XA PREPARE '$xid'
+  }
+  --eval $xa_terminate '$xid' $one_phase
+
+  ## Temporary tables
+  # create outside xa use at the tail
+  CREATE TEMPORARY TABLE tmp_i LIKE t;
+  CREATE TEMPORARY TABLE tmp_m LIKE tm;
+  --eval XA START '$xid'
+  --disable_warnings
+  INSERT INTO t VALUES (4);
+  INSERT INTO tm VALUES (4);
+  INSERT INTO tmp_i VALUES (4);
+  INSERT INTO tmp_m VALUES (4);
+  INSERT INTO t  SELECT * FROM tmp_i;
+  INSERT INTO tm SELECT * FROM tmp_m;
+  --enable_warnings
+  --eval XA END '$xid'
+  if ($xa_prepare_opt)
+  {
+   --eval XA PREPARE '$xid'
+  }
+  --eval $xa_terminate '$xid' $one_phase
+
+  # temporary tables at the head
+  --eval XA START '$xid'
+  --disable_warnings
+  INSERT INTO tmp_i VALUES (5);
+  INSERT INTO tmp_m VALUES (5);
+  INSERT INTO t  SELECT * FROM tmp_i;
+  INSERT INTO tm SELECT * FROM tmp_m;
+  INSERT INTO t VALUES (5);
+  INSERT INTO tm VALUES (5);
+  --enable_warnings
+  --eval XA END '$xid'
+  if ($xa_prepare_opt)
+  {
+   --eval XA PREPARE '$xid'
+  }
+  --eval $xa_terminate '$xid' $one_phase
+
+  # create inside xa use at the tail
+  DROP TEMPORARY TABLE tmp_i;
+  DROP TEMPORARY TABLE tmp_m;
+
+  --eval XA START '$xid'
+  --disable_warnings
+  INSERT INTO t VALUES (6);
+  INSERT INTO tm VALUES (6);
+  CREATE TEMPORARY TABLE tmp_i LIKE t;
+  CREATE TEMPORARY TABLE tmp_m LIKE tm;
+  INSERT INTO tmp_i VALUES (6);
+  INSERT INTO tmp_m VALUES (6);
+  INSERT INTO t  SELECT * FROM tmp_i;
+  INSERT INTO tm SELECT * FROM tmp_m;
+  --enable_warnings
+  --eval XA END '$xid'
+  if ($xa_prepare_opt)
+  {
+   --eval XA PREPARE '$xid'
+  }
+  --eval $xa_terminate '$xid' $one_phase
+
+  # use at the head
+  DROP TEMPORARY TABLE tmp_i;
+  DROP TEMPORARY TABLE tmp_m;
+  --eval XA START '$xid'
+  --disable_warnings
+  CREATE TEMPORARY TABLE tmp_i LIKE t;
+  CREATE TEMPORARY TABLE tmp_m LIKE tm;
+  INSERT INTO tmp_i VALUES (7);
+  INSERT INTO tmp_m VALUES (7);
+  INSERT INTO t  SELECT * FROM tmp_i;
+  INSERT INTO tm SELECT * FROM tmp_m;
+  INSERT INTO t VALUES (7);
+  INSERT INTO tm VALUES (7);
+  --enable_warnings
+  --eval XA END '$xid'
+  if ($xa_prepare_opt)
+  {
+   --eval XA PREPARE '$xid'
+  }
+  --eval $xa_terminate '$xid' $one_phase
+
+  # use at the tail and drop
+  --eval XA START '$xid'
+  --disable_warnings
+  INSERT INTO t VALUES (8);
+  INSERT INTO tm VALUES (8);
+  INSERT INTO tmp_i VALUES (8);
+  INSERT INTO tmp_m VALUES (8);
+  INSERT INTO t  SELECT * FROM tmp_i;
+  INSERT INTO tm SELECT * FROM tmp_m;
+  DROP TEMPORARY TABLE tmp_i;
+  DROP TEMPORARY TABLE tmp_m;
+  --enable_warnings
+  --eval XA END '$xid'
+  if ($xa_prepare_opt)
+  {
+   --eval XA PREPARE '$xid'
+  }
+  --eval $xa_terminate '$xid' $one_phase
+
+  ## Ineffective transactional table operation case
+
+  --eval XA START '$xid'
+  UPDATE t SET a = 99 where a = -1;
+  --eval XA END '$xid'
+  if ($xa_prepare_opt)
+  {
+   --eval XA PREPARE '$xid'
+  }
+  --eval $xa_terminate '$xid' $one_phase
+}
+
+if ($command == cleanup)
+{
+  DROP TABLE t, tm;
+}

mysql-test/suite/rpl/r/rpl_parallel_optimistic_xa.result

+include/master-slave.inc
+[connection master]
+call mtr.add_suppression("Deadlock found when trying to get lock; try restarting transaction");
+call mtr.add_suppression("WSREP: handlerton rollback failed");
+CREATE VIEW v_processlist  as SELECT * FROM performance_schema.threads where type = 'FOREGROUND';
+connection master;
+ALTER TABLE mysql.gtid_slave_pos ENGINE=InnoDB;
+connection slave;
+include/stop_slave.inc
+SET @old_parallel_threads            = @@GLOBAL.slave_parallel_threads;
+SET @@global.slave_parallel_threads  = 7;
+SET @old_parallel_mode               = @@GLOBAL.slave_parallel_mode;
+SET @@global.slave_parallel_mode     ='optimistic';
+SET @old_gtid_cleanup_batch_size     = @@GLOBAL.gtid_cleanup_batch_size;
+SET @@global.gtid_cleanup_batch_size = 1000000;
+CHANGE MASTER TO master_use_gtid=slave_pos;
+connection master;
+CREATE TABLE t0 (a int,             b INT) ENGINE=InnoDB;
+CREATE TABLE t1 (a int PRIMARY KEY, b INT) ENGINE=InnoDB;
+INSERT INTO t1 VALUES (1, 0);
+include/save_master_gtid.inc
+connection slave;
+include/start_slave.inc
+include/sync_with_master_gtid.inc
+include/stop_slave.inc
+connection master;
+include/save_master_gtid.inc
+connection slave;
+include/start_slave.inc
+include/sync_with_master_gtid.inc
+include/diff_tables.inc [master:t0, slave:t0]
+include/diff_tables.inc [master:t1, slave:t1]
+connection slave;
+include/stop_slave.inc
+set global log_warnings=default;
+SET GLOBAL slave_parallel_mode=@old_parallel_mode;
+SET GLOBAL slave_parallel_threads=@old_parallel_threads;
+include/start_slave.inc
+connection master;
+DROP VIEW v_processlist;
+DROP TABLE t0, t1;
+include/save_master_gtid.inc
+connection slave;
+include/sync_with_master_gtid.inc
+SELECT COUNT(*) <= 5*@@GLOBAL.gtid_cleanup_batch_size
+FROM mysql.gtid_slave_pos;
+COUNT(*) <= 5*@@GLOBAL.gtid_cleanup_batch_size
+1
+SET GLOBAL gtid_cleanup_batch_size= @old_gtid_cleanup_batch_size;
+connection master;
+include/rpl_end.inc

mysql-test/suite/rpl/r/rpl_parallel_optimistic_xa_lsu_off.result

+include/master-slave.inc
+[connection master]
+call mtr.add_suppression("Deadlock found when trying to get lock; try restarting transaction");
+call mtr.add_suppression("WSREP: handlerton rollback failed");
+CREATE VIEW v_processlist  as SELECT * FROM performance_schema.threads where type = 'FOREGROUND';
+connection master;
+ALTER TABLE mysql.gtid_slave_pos ENGINE=InnoDB;
+connection slave;
+include/stop_slave.inc
+SET @old_parallel_threads            = @@GLOBAL.slave_parallel_threads;
+SET @@global.slave_parallel_threads  = 7;
+SET @old_parallel_mode               = @@GLOBAL.slave_parallel_mode;
+SET @@global.slave_parallel_mode     ='optimistic';
+SET @old_gtid_cleanup_batch_size     = @@GLOBAL.gtid_cleanup_batch_size;
+SET @@global.gtid_cleanup_batch_size = 1000000;
+CHANGE MASTER TO master_use_gtid=slave_pos;
+connection master;
+CREATE TABLE t0 (a int,             b INT) ENGINE=InnoDB;
+CREATE TABLE t1 (a int PRIMARY KEY, b INT) ENGINE=InnoDB;
+INSERT INTO t1 VALUES (1, 0);
+include/save_master_gtid.inc
+connection slave;
+include/start_slave.inc
+include/sync_with_master_gtid.inc
+include/stop_slave.inc
+connection master;
+include/save_master_gtid.inc
+connection slave;
+include/start_slave.inc
+include/sync_with_master_gtid.inc
+include/diff_tables.inc [master:t0, slave:t0]
+include/diff_tables.inc [master:t1, slave:t1]
+connection slave;
+include/stop_slave.inc
+set global log_warnings=default;
+SET GLOBAL slave_parallel_mode=@old_parallel_mode;
+SET GLOBAL slave_parallel_threads=@old_parallel_threads;
+include/start_slave.inc
+connection master;
+DROP VIEW v_processlist;
+DROP TABLE t0, t1;
+include/save_master_gtid.inc
+connection slave;
+include/sync_with_master_gtid.inc
+SELECT COUNT(*) <= 5*@@GLOBAL.gtid_cleanup_batch_size
+FROM mysql.gtid_slave_pos;
+COUNT(*) <= 5*@@GLOBAL.gtid_cleanup_batch_size
+1
+SET GLOBAL gtid_cleanup_batch_size= @old_gtid_cleanup_batch_size;
+connection master;
+include/rpl_end.inc

mysql-test/suite/rpl/r/rpl_parallel_xa_same_xid.result

+include/master-slave.inc
+[connection master]
+connection slave;
+call mtr.add_suppression("WSREP: handlerton rollback failed");
+include/stop_slave.inc
+ALTER TABLE mysql.gtid_slave_pos ENGINE=InnoDB;
+SET @old_parallel_threads = @@GLOBAL.slave_parallel_threads;
+SET @old_parallel_mode               = @@GLOBAL.slave_parallel_mode;
+SET @@global.slave_parallel_mode     ='optimistic';
+include/start_slave.inc
+connection master;
+CREATE TABLE t1 (a INT, b INT) ENGINE=InnoDB;
+CREATE TABLE t2 (a INT AUTO_INCREMENT PRIMARY KEY, b INT) ENGINE=InnoDB;
+include/sync_slave_sql_with_master.inc
+include/diff_tables.inc [master:t1, slave:t1]
+connection slave;
+include/stop_slave.inc
+SET GLOBAL slave_parallel_threads=@old_parallel_threads;
+SET GLOBAL slave_parallel_mode=@old_parallel_mode;
+include/start_slave.inc
+connection master;
+DROP TABLE t1, t2;
+include/rpl_end.inc

mysql-test/suite/rpl/r/rpl_temporary_errors.result

@@ -3,7 +3,7 @@ include/master-slave.inc
 call mtr.add_suppression("Deadlock found");
 call mtr.add_suppression("Can't find record in 't.'");
 connection master;
-CREATE TABLE t1 (a INT PRIMARY KEY, b INT);
+CREATE TABLE t1 (a INT PRIMARY KEY, b INT) ENGINE=innodb;
 INSERT INTO t1 VALUES (1,1), (2,2), (3,3), (4,4);
 connection slave;
 SHOW STATUS LIKE 'Slave_retried_transactions';
@@ -11,34 +11,67 @@ Variable_name	Value
 Slave_retried_transactions	0
 set @@global.slave_exec_mode= 'IDEMPOTENT';
 UPDATE t1 SET a = 5, b = 47 WHERE a = 1;
-SELECT * FROM t1;
+SELECT * FROM t1 ORDER BY a;
 a	b
-5	47
 2	2
 3	3
 4	4
+5	47
 connection master;
 UPDATE t1 SET a = 5, b = 5 WHERE a = 1;
-SELECT * FROM t1;
+SELECT * FROM t1 ORDER BY a;
 a	b
-5	5
 2	2
 3	3
 4	4
+5	5
 connection slave;
 set @@global.slave_exec_mode= default;
 SHOW STATUS LIKE 'Slave_retried_transactions';
 Variable_name	Value
 Slave_retried_transactions	0
-SELECT * FROM t1;
+SELECT * FROM t1 ORDER BY a;
 a	b
-5	47
 2	2
 3	3
 4	4
+5	47
 include/check_slave_is_running.inc
 connection slave;
 call mtr.add_suppression("Slave SQL.*Could not execute Update_rows event on table test.t1");
+call mtr.add_suppression("Slave SQL for channel '': worker thread retried transaction");
+call mtr.add_suppression("The slave coordinator and worker threads are stopped");
+connection slave;
+set @save_innodb_lock_wait_timeout=@@global.innodb_lock_wait_timeout;
+set @save_slave_transaction_retries=@@global.slave_transaction_retries;
+set @@global.innodb_lock_wait_timeout=1;
+set @@global.slave_transaction_retries=2;
+include/restart_slave.inc
+connection slave1;
+BEGIN;
+INSERT INTO t1 SET a = 6, b = 7;
+connection master;
+INSERT INTO t1 SET a = 99, b = 99;
+XA START 'xa1';
+INSERT INTO t1 SET a = 6, b = 6;
+XA END 'xa1';
+XA PREPARE 'xa1';
+connection slave;
+include/wait_for_slave_sql_error.inc [errno=1213,1205]
+set @@global.innodb_lock_wait_timeout=1;
+set @@global.slave_transaction_retries=100;
+include/restart_slave.inc
+Warnings:
+Note	1255	Slave already has been stopped
+connection slave1;
+ROLLBACK;
+connection master;
+XA COMMIT 'xa1';
+include/sync_slave_sql_with_master.inc
+connection slave;
+include/assert.inc [XA transaction record must be in the table]
+set @@global.innodb_lock_wait_timeout=@save_innodb_lock_wait_timeout;
+set @@global.slave_transaction_retries= @save_slave_transaction_retries;
 connection master;
 DROP TABLE t1;
 connection slave;

mysql-test/suite/rpl/r/rpl_xa_gap_lock.result

+include/master-slave.inc
+[connection master]
+connection slave;
+SET @saved_innodb_limit_optimistic_insert_debug = @@GLOBAL.innodb_limit_optimistic_insert_debug;
+SET @@GLOBAL.innodb_limit_optimistic_insert_debug = 2;
+connection master;
+CREATE TABLE t1 (
+c1 INT NOT NULL,
+KEY(c1)
+) ENGINE=InnoDB;
+CREATE TABLE t2 (
+c1 INT NOT NULL,
+FOREIGN KEY(c1) REFERENCES t1(c1)
+) ENGINE=InnoDB;
+INSERT INTO t1 VALUES (1), (3), (4);
+connection master1;
+XA START 'XA1';
+INSERT INTO t1 values(2);
+XA END 'XA1';
+connection master;
+XA START 'XA2';
+INSERT INTO t2 values(3);
+XA END 'XA2';
+XA PREPARE 'XA2';
+connection master1;
+XA PREPARE 'XA1';
+XA COMMIT 'XA1';
+connection master;
+XA COMMIT 'XA2';
+include/sync_slave_sql_with_master.inc
+include/stop_slave.inc
+DROP TABLE t2, t1;
+RESET SLAVE;
+RESET MASTER;
+connection master;
+Restore binary log from the master into the slave
+include/diff_tables.inc [master:test.t1, slave:test.t1]
+include/diff_tables.inc [master:test.t2, slave:test.t2]
+DROP TABLE t2, t1;
+connection slave;
+CHANGE MASTER TO MASTER_LOG_FILE='LOG_FILE', MASTER_LOG_POS=LOG_POS;
+SET @@GLOBAL.innodb_limit_optimistic_insert_debug = @saved_innodb_limit_optimistic_insert_debug;
+include/start_slave.inc
+include/rpl_end.inc

mysql-test/suite/rpl/t/rpl_create_xa_prepared.inc

+# param $xid    to name xa and take part in the connection name
+# param $query  to execute as the xa body
+# param $db_ign the default database
+
+--connect (master_$xid, 127.0.0.1,root,,$db,$MASTER_MYPORT,)
+--eval xa start '$xid'
+--eval $query
+--eval xa end '$xid'
+--eval xa prepare '$xid';

mysql-test/suite/rpl/t/rpl_parallel_optimistic_xa_lsu_off-slave.opt

+--log-slave-updates=OFF

mysql-test/suite/rpl/t/rpl_parallel_optimistic_xa_lsu_off.test

+# --log-slave-updates OFF version of rpl_parallel_optimistic_xa
+--source rpl_parallel_optimistic_xa.test

mysql-test/suite/rpl/t/rpl_temporary_errors.test

@@ -6,7 +6,7 @@ call mtr.add_suppression("Deadlock found");
 call mtr.add_suppression("Can't find record in 't.'");
 
 connection master;
-CREATE TABLE t1 (a INT PRIMARY KEY, b INT);
+CREATE TABLE t1 (a INT PRIMARY KEY, b INT) ENGINE=innodb;
 INSERT INTO t1 VALUES (1,1), (2,2), (3,3), (4,4);
 sync_slave_with_master;
 SHOW STATUS LIKE 'Slave_retried_transactions';
@@ -14,20 +14,94 @@ SHOW STATUS LIKE 'Slave_retried_transactions';
 # the following UPDATE t1 to pass the mode is switched temprorarily
 set @@global.slave_exec_mode= 'IDEMPOTENT';
 UPDATE t1 SET a = 5, b = 47 WHERE a = 1;
-SELECT * FROM t1;
+SELECT * FROM t1 ORDER BY a;
 connection master;
 UPDATE t1 SET a = 5, b = 5 WHERE a = 1;
-SELECT * FROM t1;
+SELECT * FROM t1 ORDER BY a;
 #SHOW BINLOG EVENTS;
 sync_slave_with_master;
 set @@global.slave_exec_mode= default;
 SHOW STATUS LIKE 'Slave_retried_transactions';
-SELECT * FROM t1;
+SELECT * FROM t1 ORDER BY a;
 source include/check_slave_is_running.inc;
 
 connection slave;
 call mtr.add_suppression("Slave SQL.*Could not execute Update_rows event on table test.t1");
+call mtr.add_suppression("Slave SQL for channel '': worker thread retried transaction");
+call mtr.add_suppression("The slave coordinator and worker threads are stopped");
+#
+# Bug#24764800 REPLICATION FAILING ON SLAVE WITH XAER_RMFAIL ERROR
+#
+# Verify that a temporary failing replicated xa transaction completes
+# upon slave applier restart after previous
+# @@global.slave_transaction_retries number of retries in vain.
+#
+connection slave;
+
+set @save_innodb_lock_wait_timeout=@@global.innodb_lock_wait_timeout;
+set @save_slave_transaction_retries=@@global.slave_transaction_retries;
+
+# Slave applier parameters for the failed retry
+set @@global.innodb_lock_wait_timeout=1;
+set @@global.slave_transaction_retries=2;
+--source include/restart_slave_sql.inc
+
+# Temporary error implement: a record is blocked by slave local trx
+connection slave1;
+BEGIN;
+INSERT INTO t1 SET a = 6, b = 7;
+
+connection master;
+INSERT INTO t1 SET a = 99, b = 99; # slave applier warm up trx
+XA START 'xa1';
+INSERT INTO t1 SET a = 6, b = 6; # this record eventually must be found on slave
+XA END 'xa1';
+XA PREPARE 'xa1';
+
+connection slave;
+# convert_error(ER_LOCK_WAIT_TIMEOUT)
+--let $err_timeout= 1205
+# convert_error(ER_LOCK_DEADLOCK)
+--let $err_deadlock= 1213
+--let $slave_sql_errno=$err_deadlock,$err_timeout
+--let $show_slave_sql_error=
+--source include/wait_for_slave_sql_error.inc
+
+# b. Slave applier parameters for successful retry after restart
+set @@global.innodb_lock_wait_timeout=1;
+set @@global.slave_transaction_retries=100;
+
+--source include/restart_slave_sql.inc
+
+--let $last_retries= query_get_value(SHOW GLOBAL STATUS LIKE 'Slave_retried_transactions', Value, 1)
+--let $status_type=GLOBAL
+--let $status_var=Slave_retried_transactions
+--let $status_var_value=`SELECT 1 + $last_retries`
+--let $$status_var_comparsion= >
+--source include/wait_for_status_var.inc
+
+# Release the record after just one retry
+connection slave1;
+ROLLBACK;
+
+connection master;
+XA COMMIT 'xa1';
+
+--source include/sync_slave_sql_with_master.inc
+
+# Proof of correctness: the committed XA is on the slave
+connection slave;
+--let $assert_text=XA transaction record must be in the table
+--let $assert_cond=count(*)=1 FROM t1 WHERE a=6 AND b=6
+--source include/assert.inc
+
+# Bug#24764800 cleanup:
+set @@global.innodb_lock_wait_timeout=@save_innodb_lock_wait_timeout;
+set @@global.slave_transaction_retries= @save_slave_transaction_retries;
 
+#
+# Total cleanup:
+#
 connection master;
 DROP TABLE t1;
 --sync_slave_with_master