BUG#51342 - more xid crashing

SET autocommit=1 while XA transaction is active may
cause various side effects, including memory corruption
and server crash.

The problem is that SET autocommit=1 and further queries
attempt to commit local transaction, whereas XA transaction
is still active.

As local and XA transactions are mutually exclusive, this
patch forbids enabling autocommit mode while XA transaction
is active.

mysql-test/r/xa.result

@@ -74,4 +74,21 @@ ERROR XA102: XA_RBDEADLOCK: Transaction branch was rolled back: deadlock was det
 xa rollback 'a','c';
 xa start 'a','c';
 drop table t1;
+#
+# BUG#51342 - more xid crashing
+#
+CREATE TABLE t1(a INT) ENGINE=InnoDB;
+XA START 'x';
+SET SESSION autocommit=0;
+INSERT INTO t1 VALUES(1);
+SET SESSION autocommit=1;
+ERROR XAE07: XAER_RMFAIL: The command cannot be executed when global transaction is in the  ACTIVE state
+SELECT @@autocommit;
+@@autocommit
+0
+INSERT INTO t1 VALUES(1);
+XA END 'x';
+XA COMMIT 'x' ONE PHASE;
+DROP TABLE t1;
+SET SESSION autocommit=DEFAULT;
 End of 5.0 tests

mysql-test/t/xa.test

@@ -122,6 +122,22 @@ xa start 'a','c';
 --connection default
 drop table t1;
 
+--echo #
+--echo # BUG#51342 - more xid crashing
+--echo #
+CREATE TABLE t1(a INT) ENGINE=InnoDB;
+XA START 'x';
+SET SESSION autocommit=0;
+INSERT INTO t1 VALUES(1);
+--error ER_XAER_RMFAIL
+SET SESSION autocommit=1;
+SELECT @@autocommit;
+INSERT INTO t1 VALUES(1);
+XA END 'x';
+XA COMMIT 'x' ONE PHASE;
+DROP TABLE t1;
+SET SESSION autocommit=DEFAULT;
+
 --echo End of 5.0 tests
 
 # Wait till all disconnects are completed

sql/set_var.cc

@@ -3065,6 +3065,13 @@ static bool set_option_autocommit(THD *thd, set_var *var)
     if ((org_options & OPTION_NOT_AUTOCOMMIT))
     {
       /* We changed to auto_commit mode */
+      if (thd->transaction.xid_state.xa_state != XA_NOTR)
+      {
+        thd->options= org_options;
+        my_error(ER_XAER_RMFAIL, MYF(0),
+                 xa_state_names[thd->transaction.xid_state.xa_state]);
+        return 1;
+      }
       thd->options&= ~OPTION_BEGIN;
       thd->transaction.all.modified_non_trans_table= FALSE;
       thd->server_status|= SERVER_STATUS_AUTOCOMMIT;