Bug#12352846 - TRANS_XA_START(THD*):

               ASSERTION THD->TRANSACTION.XID_STATE.XID.IS_NULL()
               FAILED

The triggered assert checks that the previous XA transaction has
done proper cleanup before a new XA transaction is started.
The bug that caused it to be triggered was that XA COMMIT did not
clean up error state if XA COMMIT discovered that the current XA
transaction had to be rolled back.

This patch fixes the problem by resetting the XA error state
before XA COMMIT calls ha_rollback_trans(). This allows following
XA transactions to be started without triggering the assert.

Test case added to xa.test.

mysql-test/r/xa.result

@@ -200,3 +200,32 @@ SELECT * FROM t1;
 a
 1
 DROP TABLE t1;
+#
+# Bug#12352846 - TRANS_XA_START(THD*):
+#                ASSERTION THD->TRANSACTION.XID_STATE.XID.IS_NULL()
+#                FAILED 
+#
+DROP TABLE IF EXISTS t1, t2;
+CREATE TABLE t1 (a INT);
+CREATE TABLE t2 (a INT);
+START TRANSACTION;
+INSERT INTO t1 VALUES (1);
+# Connection con2
+XA START 'xid1';
+# Sending:
+INSERT INTO t2 SELECT a FROM t1;
+# Connection default
+# Waiting until INSERT ... is blocked
+DELETE FROM t1;
+COMMIT;
+# Connection con2
+# Reaping: INSERT INTO t2 SELECT a FROM t1
+ERROR 40001: Deadlock found when trying to get lock; try restarting transaction
+XA COMMIT 'xid1';
+ERROR XA102: XA_RBDEADLOCK: Transaction branch was rolled back: deadlock was detected
+XA START 'xid1';
+XA END 'xid1';
+XA PREPARE 'xid1';
+XA ROLLBACK 'xid1';
+# Connection default
+DROP TABLE t1, t2;

mysql-test/t/xa.test

@@ -3,6 +3,8 @@
 #
 -- source include/have_innodb.inc
 
+--source include/not_embedded.inc
+
 # Save the initial number of concurrent sessions
 --source include/count_sessions.inc
 
@@ -326,6 +328,59 @@ SELECT * FROM t1;
 DROP TABLE t1;
 
 
+--echo #
+--echo # Bug#12352846 - TRANS_XA_START(THD*):
+--echo #                ASSERTION THD->TRANSACTION.XID_STATE.XID.IS_NULL()
+--echo #                FAILED 
+--echo #
+
+--disable_warnings
+DROP TABLE IF EXISTS t1, t2;
+--enable_warnings
+
+CREATE TABLE t1 (a INT);
+CREATE TABLE t2 (a INT);
+
+START TRANSACTION;
+INSERT INTO t1 VALUES (1);
+
+--echo # Connection con2
+--connect (con2,localhost,root)
+XA START 'xid1';
+--echo # Sending:
+--send INSERT INTO t2 SELECT a FROM t1
+
+--echo # Connection default
+--connection default
+let $wait_condition=
+  SELECT COUNT(*) = 1 FROM information_schema.processlist
+  WHERE state = "Sending data"
+  AND info = "INSERT INTO t2 SELECT a FROM t1";
+--echo # Waiting until INSERT ... is blocked
+--source include/wait_condition.inc
+DELETE FROM t1;
+COMMIT;
+
+--echo # Connection con2
+--connection con2
+--echo # Reaping: INSERT INTO t2 SELECT a FROM t1
+--error ER_LOCK_DEADLOCK
+--reap
+--error ER_XA_RBDEADLOCK
+XA COMMIT 'xid1';
+# This caused the assert to be triggered
+XA START 'xid1';
+
+XA END 'xid1';
+XA PREPARE 'xid1';
+XA ROLLBACK 'xid1';
+
+--echo # Connection default
+connection default;
+DROP TABLE t1, t2;
+disconnect con2;
+
+
 # Wait till all disconnects are completed
 --source include/wait_until_count_sessions.inc
 

sql/transaction.cc

@@ -78,6 +78,33 @@ static bool xa_trans_rolled_back(XID_STATE *xid_state)
 }
 
 
+/**
+  Rollback the active XA transaction.
+
+  @note Resets rm_error before calling ha_rollback(), so
+        the thd->transaction.xid structure gets reset
+        by ha_rollback() / THD::transaction::cleanup().
+
+  @return TRUE if the rollback failed, FALSE otherwise.
+*/
+
+static bool xa_trans_force_rollback(THD *thd)
+{
+  /*
+    We must reset rm_error before calling ha_rollback(),
+    so thd->transaction.xid structure gets reset
+    by ha_rollback()/THD::transaction::cleanup().
+  */
+  thd->transaction.xid_state.rm_error= 0;
+  if (ha_rollback_trans(thd, true))
+  {
+    my_error(ER_XAER_RMERR, MYF(0));
+    return true;
+  }
+  return false;
+}
+
+
 /**
   Begin a new transaction.
 
@@ -649,8 +676,7 @@ bool trans_xa_commit(THD *thd)
 
   if (xa_trans_rolled_back(&thd->transaction.xid_state))
   {
-    if (ha_rollback_trans(thd, TRUE))
-      my_error(ER_XAER_RMERR, MYF(0));
+    xa_trans_force_rollback(thd);
     res= thd->is_error();
   }
   else if (xa_state == XA_IDLE && thd->lex->xa_opt == XA_ONE_PHASE)
@@ -739,15 +765,7 @@ bool trans_xa_rollback(THD *thd)
     DBUG_RETURN(TRUE);
   }
 
-  /*
-    Resource Manager error is meaningless at this point, as we perform
-    explicit rollback request by user. We must reset rm_error before
-    calling ha_rollback(), so thd->transaction.xid structure gets reset
-    by ha_rollback()/THD::transaction::cleanup().
-  */
-  thd->transaction.xid_state.rm_error= 0;
-  if ((res= test(ha_rollback_trans(thd, TRUE))))
-    my_error(ER_XAER_RMERR, MYF(0));
+  res= xa_trans_force_rollback(thd);
 
   thd->variables.option_bits&= ~(OPTION_BEGIN | OPTION_KEEP_LOG);
   thd->transaction.all.modified_non_trans_table= FALSE;