MDEV-21766 - Forbid XID with empty 'gtrid'

XA specification doesn't permit empty gtrid. It is now enforced by this
patch. This solution was agreed in favour of fixing InnoDB, which doesn't
expect empty XID since early 10.5.

Also fixed wrong assertion (and added a test cases) that didn't permit
64 bytes gtrid + 64 bytes bqual.

mysql-test/main/xa.result

@@ -345,3 +345,27 @@ connection default;
 XA END 'xid1';
 XA ROLLBACK 'xid1';
 DROP TABLE t1, t2, t3;
+#
+# MDEV-21766 - Forbid XID with empty 'gtrid'
+#
+CREATE TABLE t1(a INT) ENGINE=InnoDB;
+XA BEGIN '';
+ERROR XAE05: XAER_INVAL: Invalid arguments (or unsupported command)
+XA BEGIN '8bytes1x8bytes2x8bytes3x8bytes4x8bytes5x8bytes6x8bytes7x8bytes8x',
+'8bytes1x8bytes2x8bytes3x8bytes4x8bytes5x8bytes6x8bytes7x8bytes8x';
+INSERT INTO t1 VALUES(1);
+XA END '8bytes1x8bytes2x8bytes3x8bytes4x8bytes5x8bytes6x8bytes7x8bytes8x',
+'8bytes1x8bytes2x8bytes3x8bytes4x8bytes5x8bytes6x8bytes7x8bytes8x';
+XA PREPARE '8bytes1x8bytes2x8bytes3x8bytes4x8bytes5x8bytes6x8bytes7x8bytes8x',
+'8bytes1x8bytes2x8bytes3x8bytes4x8bytes5x8bytes6x8bytes7x8bytes8x';
+XA ROLLBACK '8bytes1x8bytes2x8bytes3x8bytes4x8bytes5x8bytes6x8bytes7x8bytes8x',
+'8bytes1x8bytes2x8bytes3x8bytes4x8bytes5x8bytes6x8bytes7x8bytes8x';
+SET NAMES utf8;
+XA BEGIN 'Я_упала_с_сеновала_тормозила_головой';
+ERROR 42000: You have an error in your SQL syntax; check the manual that corresponds to your MariaDB server version for the right syntax to use near '' at line 1
+XA BEGIN 'Я_упaлa_c_сеновала_тормозила_головой';
+XA END 'Я_упaлa_c_сеновала_тормозила_головой';
+XA PREPARE 'Я_упaлa_c_сеновала_тормозила_головой';
+XA ROLLBACK 'Я_упaлa_c_сеновала_тормозила_головой';
+SET NAMES default;
+DROP TABLE t1;

mysql-test/main/xa.test

@@ -476,5 +476,34 @@ XA END 'xid1';
 XA ROLLBACK 'xid1';
 DROP TABLE t1, t2, t3;
 
---source include/wait_until_count_sessions.inc
 
+--echo #
+--echo # MDEV-21766 - Forbid XID with empty 'gtrid'
+--echo #
+CREATE TABLE t1(a INT) ENGINE=InnoDB;
+
+--error ER_XAER_INVAL
+XA BEGIN '';
+
+XA BEGIN '8bytes1x8bytes2x8bytes3x8bytes4x8bytes5x8bytes6x8bytes7x8bytes8x',
+         '8bytes1x8bytes2x8bytes3x8bytes4x8bytes5x8bytes6x8bytes7x8bytes8x';
+INSERT INTO t1 VALUES(1);
+XA END '8bytes1x8bytes2x8bytes3x8bytes4x8bytes5x8bytes6x8bytes7x8bytes8x',
+       '8bytes1x8bytes2x8bytes3x8bytes4x8bytes5x8bytes6x8bytes7x8bytes8x';
+XA PREPARE '8bytes1x8bytes2x8bytes3x8bytes4x8bytes5x8bytes6x8bytes7x8bytes8x',
+           '8bytes1x8bytes2x8bytes3x8bytes4x8bytes5x8bytes6x8bytes7x8bytes8x';
+XA ROLLBACK '8bytes1x8bytes2x8bytes3x8bytes4x8bytes5x8bytes6x8bytes7x8bytes8x',
+            '8bytes1x8bytes2x8bytes3x8bytes4x8bytes5x8bytes6x8bytes7x8bytes8x';
+
+SET NAMES utf8;
+--error ER_PARSE_ERROR
+XA BEGIN 'Я_упала_с_сеновала_тормозила_головой'; # 36 characters, 67 bytes
+XA BEGIN 'Я_упaлa_c_сеновала_тормозила_головой'; # 36 characters, 64 bytes
+XA END 'Я_упaлa_c_сеновала_тормозила_головой';
+XA PREPARE 'Я_упaлa_c_сеновала_тормозила_головой';
+XA ROLLBACK 'Я_упaлa_c_сеновала_тормозила_головой';
+SET NAMES default;
+
+DROP TABLE t1;
+
+--source include/wait_until_count_sessions.inc

sql/xa.cc

@@ -432,6 +432,8 @@ bool trans_xa_start(THD *thd)
   /* TODO: JOIN is not supported yet. */
   if (thd->lex->xa_opt != XA_NONE)
     my_error(ER_XAER_INVAL, MYF(0));
+  else if (!thd->lex->xid->gtrid_length)
+    my_error(ER_XAER_INVAL, MYF(0));
   else if (thd->transaction.xid_state.is_explicit_XA())
     thd->transaction.xid_state.er_xaer_rmfail();
   else if (thd->locked_tables_mode || thd->in_active_multi_stmt_transaction())

storage/innobase/trx/trx0undo.cc

@@ -497,9 +497,12 @@ static uint16_t trx_undo_header_create(buf_block_t *undo_page, trx_id_t trx_id,
 static void trx_undo_write_xid(buf_block_t *block, uint16_t offset,
                                const XID &xid, mtr_t *mtr)
 {
-  DBUG_ASSERT(xid.gtrid_length >= 0);
+  DBUG_ASSERT(xid.gtrid_length > 0);
   DBUG_ASSERT(xid.bqual_length >= 0);
-  DBUG_ASSERT(xid.gtrid_length + xid.bqual_length < XIDDATASIZE);
+  DBUG_ASSERT(xid.gtrid_length <= MAXGTRIDSIZE);
+  DBUG_ASSERT(xid.bqual_length <= MAXBQUALSIZE);
+  static_assert(MAXGTRIDSIZE + MAXBQUALSIZE == XIDDATASIZE,
+                "gtrid and bqual don't fit xid data");
   DBUG_ASSERT(mach_read_from_2(TRX_UNDO_SEG_HDR + TRX_UNDO_LAST_LOG +
                                block->frame) == offset);