MDEV-21659 XA rollback foreign_xid is allowed inside active XA

MDEV-21854 xa commit `xid` one phase for already prepared transaction must always error out

Added state and one-phase option checks to XA "external" commit/rollback
branches. While the XA standard does not prohibit it,
Commit and Rollback of an XA external to the current ongoing transaction
is not allowed; after all the current transaction may rollback
to not being able to revert that decision.

mysql-test/main/xa.result

@@ -51,7 +51,7 @@ formatID	gtrid_length	bqual_length	data
 11	5	5	testb 0@P`
 1	5	5	testatestb
 xa commit 'testb',0x2030405060,11;
-ERROR XAE04: XAER_NOTA: Unknown XID
+ERROR XAE09: XAER_OUTSIDE: Some work is done outside global transaction
 xa rollback 'testa','testb';
 xa start 'zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz';
 ERROR 42000: You have an error in your SQL syntax; check the manual that corresponds to your MariaDB server version for the right syntax to use near '' at line 1
@@ -370,6 +370,32 @@ XA ROLLBACK 'Я_упaлa_c_сеновала_тормозила_головой';
 SET NAMES default;
 DROP TABLE t1;
 #
+# MDEV-21659 XA rollback foreign_xid is allowed inside active XA
+# MDEV-21854 - xa commit  one phase for already prepared transaction
+#              must always error out
+#
+BEGIN;
+XA COMMIT 'unknown';
+ERROR XAE09: XAER_OUTSIDE: Some work is done outside global transaction
+XA COMMIT 'unknown' ONE PHASE;
+ERROR XAE09: XAER_OUTSIDE: Some work is done outside global transaction
+BEGIN;
+XA ROLLBACK 'unknown';
+ERROR XAE09: XAER_OUTSIDE: Some work is done outside global transaction
+ROLLBACK;
+XA START 'xid1';
+XA COMMIT 'unknown';
+ERROR XAE09: XAER_OUTSIDE: Some work is done outside global transaction
+XA COMMIT 'unknown' ONE PHASE;
+ERROR XAE09: XAER_OUTSIDE: Some work is done outside global transaction
+XA ROLLBACK 'unknown';
+ERROR XAE09: XAER_OUTSIDE: Some work is done outside global transaction
+XA END 'xid1';
+XA PREPARE 'xid1';
+XA COMMIT 'xid1' ONE PHASE;
+ERROR XAE05: XAER_INVAL: Invalid arguments (or unsupported command)
+XA ROLLBACK 'xid1';
+#
 # MDEV-21856 - xid_t::formatID has to be constrained to 4 byte size
 #
 XA START 'gtrid', 'bqual', 0x80000000;

mysql-test/main/xa.test

@@ -72,7 +72,7 @@ xa prepare 'testa','testb';
 
 xa recover;
 
---error ER_XAER_NOTA
+--error ER_XAER_OUTSIDE
 xa commit 'testb',0x2030405060,11;
 xa rollback 'testa','testb';
 
@@ -505,6 +505,33 @@ XA ROLLBACK 'Я_упaлa_c_сеновала_тормозила_головой';
 SET NAMES default;
 
 DROP TABLE t1;
+--echo #
+--echo # MDEV-21659 XA rollback foreign_xid is allowed inside active XA
+--echo # MDEV-21854 - xa commit  one phase for already prepared transaction
+--echo #              must always error out
+--echo #
+BEGIN;
+--error ER_XAER_OUTSIDE
+XA COMMIT 'unknown';
+--error ER_XAER_OUTSIDE
+XA COMMIT 'unknown' ONE PHASE;
+BEGIN;
+--error ER_XAER_OUTSIDE
+XA ROLLBACK 'unknown';
+ROLLBACK;
+
+XA START 'xid1';
+--error ER_XAER_OUTSIDE
+XA COMMIT 'unknown';
+--error ER_XAER_OUTSIDE
+XA COMMIT 'unknown' ONE PHASE;
+--error ER_XAER_OUTSIDE
+XA ROLLBACK 'unknown';
+XA END 'xid1';
+XA PREPARE 'xid1';
+--error ER_XAER_INVAL
+XA COMMIT 'xid1' ONE PHASE;
+XA ROLLBACK 'xid1';
 
 
 --echo #

sql/xa.cc

@@ -529,6 +529,24 @@ bool trans_xa_commit(THD *thd)
   if (!thd->transaction.xid_state.is_explicit_XA() ||
       !thd->transaction.xid_state.xid_cache_element->xid.eq(thd->lex->xid))
   {
+    if (thd->in_multi_stmt_transaction_mode())
+    {
+      /*
+        Not allow to commit from inside an not-"native" to xid
+        ongoing transaction: the commit effect can't be reversed.
+      */
+      my_error(ER_XAER_OUTSIDE, MYF(0));
+      DBUG_RETURN(TRUE);
+    }
+    if (thd->lex->xa_opt != XA_NONE)
+    {
+      /*
+        Not allow to commit with one phase a prepared xa out of compatibility
+        with the native commit branch's error out.
+      */
+      my_error(ER_XAER_INVAL, MYF(0));
+      DBUG_RETURN(TRUE);
+    }
     if (thd->fix_xid_hash_pins())
     {
       my_error(ER_OUT_OF_RESOURCES, MYF(0));
@@ -558,10 +576,14 @@ bool trans_xa_commit(THD *thd)
     if ((res= MY_TEST(r)))
       my_error(r == 1 ? ER_XA_RBROLLBACK : ER_XAER_RMERR, MYF(0));
   }
-  else if (thd->transaction.xid_state.xid_cache_element->xa_state == XA_PREPARED &&
-           thd->lex->xa_opt == XA_NONE)
+  else if (thd->transaction.xid_state.xid_cache_element->xa_state == XA_PREPARED)
   {
     MDL_request mdl_request;
+    if (thd->lex->xa_opt != XA_NONE)
+    {
+      my_error(ER_XAER_INVAL, MYF(0));
+      DBUG_RETURN(TRUE);
+    }
 
     /*
       Acquire metadata lock which will ensure that COMMIT is blocked
@@ -623,6 +645,11 @@ bool trans_xa_rollback(THD *thd)
   if (!thd->transaction.xid_state.is_explicit_XA() ||
       !thd->transaction.xid_state.xid_cache_element->xid.eq(thd->lex->xid))
   {
+    if (thd->in_multi_stmt_transaction_mode())
+    {
+      my_error(ER_XAER_OUTSIDE, MYF(0));
+      DBUG_RETURN(TRUE);
+    }
     if (thd->fix_xid_hash_pins())
     {
       my_error(ER_OUT_OF_RESOURCES, MYF(0));