Bug#12348348 - ASSERTION IN ROW0SEL.C LINE 3698:

               TRX->CONC_STATE == 0 || TRX->CONC_STATE == 1

This bug was a different manifestation of Bug#11766752,
which was previously only fixed on mysql-trunk.

This patch backports the fix for Bug#11766752 to mysql-5.5,
which fixes the problem. The patch also adds some extra test
coverage.

mysql-test/r/xa.result

@@ -166,3 +166,37 @@ ERROR XA102: XA_RBDEADLOCK: Transaction branch was rolled back: deadlock was det
 XA END 'b';
 XA ROLLBACK 'b';
 DROP TABLE t1;
+#
+# Bug#11766752 59936: multiple xa assertions - transactional
+#              statement fuzzer
+#
+CREATE TABLE t1 (a INT) engine=InnoDB;
+XA START 'a';
+INSERT INTO t1 VALUES (1);
+SAVEPOINT savep;
+ERROR XAE07: XAER_RMFAIL: The command cannot be executed when global transaction is in the  ACTIVE state
+XA END 'a';
+SELECT * FROM t1;
+ERROR XAE07: XAER_RMFAIL: The command cannot be executed when global transaction is in the  IDLE state
+INSERT INTO t1 VALUES (2);
+ERROR XAE07: XAER_RMFAIL: The command cannot be executed when global transaction is in the  IDLE state
+SAVEPOINT savep;
+ERROR XAE07: XAER_RMFAIL: The command cannot be executed when global transaction is in the  IDLE state
+SET @a=(SELECT * FROM t1);
+ERROR XAE07: XAER_RMFAIL: The command cannot be executed when global transaction is in the  IDLE state
+XA PREPARE 'a';
+SELECT * FROM t1;
+ERROR XAE07: XAER_RMFAIL: The command cannot be executed when global transaction is in the  PREPARED state
+INSERT INTO t1 VALUES (2);
+ERROR XAE07: XAER_RMFAIL: The command cannot be executed when global transaction is in the  PREPARED state
+SAVEPOINT savep;
+ERROR XAE07: XAER_RMFAIL: The command cannot be executed when global transaction is in the  PREPARED state
+SET @a=(SELECT * FROM t1);
+ERROR XAE07: XAER_RMFAIL: The command cannot be executed when global transaction is in the  PREPARED state
+UPDATE t1 SET a=1 WHERE a=2;
+ERROR XAE07: XAER_RMFAIL: The command cannot be executed when global transaction is in the  PREPARED state
+XA COMMIT 'a';
+SELECT * FROM t1;
+a
+1
+DROP TABLE t1;

mysql-test/t/xa.test

@@ -287,6 +287,45 @@ DROP TABLE t1;
 disconnect con1;
 
 
+--echo #
+--echo # Bug#11766752 59936: multiple xa assertions - transactional
+--echo #              statement fuzzer
+--echo #
+
+CREATE TABLE t1 (a INT) engine=InnoDB;
+XA START 'a';
+INSERT INTO t1 VALUES (1);
+
+--error ER_XAER_RMFAIL
+SAVEPOINT savep;
+
+XA END 'a';
+--error ER_XAER_RMFAIL
+SELECT * FROM t1;
+--error ER_XAER_RMFAIL
+INSERT INTO t1 VALUES (2);
+--error ER_XAER_RMFAIL
+SAVEPOINT savep;
+--error ER_XAER_RMFAIL
+SET @a=(SELECT * FROM t1);
+
+XA PREPARE 'a';
+--error ER_XAER_RMFAIL
+SELECT * FROM t1;          # used to cause InnoDB assert
+--error ER_XAER_RMFAIL
+INSERT INTO t1 VALUES (2); # used to cause InnoDB assert
+--error ER_XAER_RMFAIL
+SAVEPOINT savep;
+--error ER_XAER_RMFAIL
+SET @a=(SELECT * FROM t1); # used to cause InnoDB assert
+--error ER_XAER_RMFAIL
+UPDATE t1 SET a=1 WHERE a=2;
+
+XA COMMIT 'a';
+SELECT * FROM t1;
+DROP TABLE t1;
+
+
 # Wait till all disconnects are completed
 --source include/wait_until_count_sessions.inc
 

sql/sql_base.cc

@@ -4724,6 +4724,14 @@ bool open_tables(THD *thd, TABLE_LIST **start, uint *counter, uint flags,
   bool has_prelocking_list;
   DBUG_ENTER("open_tables");
 
+  /* Accessing data in XA_IDLE or XA_PREPARED is not allowed. */
+  enum xa_states xa_state= thd->transaction.xid_state.xa_state;
+  if (*start && (xa_state == XA_IDLE || xa_state == XA_PREPARED))
+  {
+    my_error(ER_XAER_RMFAIL, MYF(0), xa_state_names[xa_state]);
+    DBUG_RETURN(true);
+  }
+
   /*
     temporary mem_root for new .frm parsing.
     TODO: variables for size

sql/transaction.cc

@@ -362,6 +362,13 @@ bool trans_savepoint(THD *thd, LEX_STRING name)
       !opt_using_transactions)
     DBUG_RETURN(FALSE);
 
+  enum xa_states xa_state= thd->transaction.xid_state.xa_state;
+  if (xa_state != XA_NOTR)
+  {
+    my_error(ER_XAER_RMFAIL, MYF(0), xa_state_names[xa_state]);
+    DBUG_RETURN(TRUE);
+  }
+
   sv= find_savepoint(thd, name);
 
   if (*sv) /* old savepoint of the same name exists */
@@ -435,6 +442,13 @@ bool trans_rollback_to_savepoint(THD *thd, LEX_STRING name)
     DBUG_RETURN(TRUE);
   }
 
+  enum xa_states xa_state= thd->transaction.xid_state.xa_state;
+  if (xa_state != XA_NOTR)
+  {
+    my_error(ER_XAER_RMFAIL, MYF(0), xa_state_names[xa_state]);
+    DBUG_RETURN(TRUE);
+  }
+
   if (ha_rollback_to_savepoint(thd, sv))
     res= TRUE;
   else if (((thd->variables.option_bits & OPTION_KEEP_LOG) ||