Bug #55930 Assertion `thd->transaction.stmt.is_empty() ||

           thd->in_sub_stmt || (thd->state..

OPTIMIZE TABLE is not directly supported by InnoDB. Instead,
recreate and analyze of the table is done. After recreate,
the table is closed and locks are released before the table
is reopened and locks re-acquired for the analyze phase.

This assertion was triggered if OPTIMIZE TABLE failed to
acquire thr_lock locks before starting the analyze phase.
The assertion tests (among other things) that there no
active statement transaction. However, as part of acquiring
the thr_lock lock, external_lock() is called for InnoDB
tables and this causes a statement transaction to be started.
If thr_multi_lock() later fails (e.g. due to timeout),
the failure handling code causes this assert to be triggered.

This patch fixes the problem by doing rollback of the
current statement transaction in case open_ltable (used by
OPTIMIZE TABLE) fails to acquire thr_lock locks.

Test case added to lock_sync.test.

mysql-test/r/lock_sync.result

@@ -704,3 +704,37 @@ SET DEBUG_SYNC="now SIGNAL query";
 # Connection default
 DROP EVENT e2;
 SET DEBUG_SYNC="RESET";
+#
+# Bug#55930 Assertion `thd->transaction.stmt.is_empty() ||
+#           thd->in_sub_stmt || (thd->state..
+#
+DROP TABLE IF EXISTS t1;
+CREATE TABLE t1(a INT) engine=InnoDB;
+INSERT INTO t1 VALUES (1), (2);
+# Connection con1
+SET SESSION lock_wait_timeout= 1;
+SET DEBUG_SYNC= 'ha_admin_open_ltable SIGNAL opti_recreate WAIT_FOR opti_analyze';
+# Sending:
+OPTIMIZE TABLE t1;
+# Connection con2
+SET DEBUG_SYNC= 'now WAIT_FOR opti_recreate';
+SET DEBUG_SYNC= 'after_lock_tables_takes_lock SIGNAL thrlock WAIT_FOR release_thrlock';
+# Sending:
+INSERT INTO t1 VALUES (3);
+# Connection default
+SET DEBUG_SYNC= 'now WAIT_FOR thrlock';
+SET DEBUG_SYNC= 'now SIGNAL opti_analyze';
+# Connection con1
+# Reaping: OPTIMIZE TABLE t1
+Table	Op	Msg_type	Msg_text
+test.t1	optimize	note	Table does not support optimize, doing recreate + analyze instead
+test.t1	optimize	error	Lock wait timeout exceeded; try restarting transaction
+test.t1	optimize	status	Operation failed
+Warnings:
+Error	1205	Lock wait timeout exceeded; try restarting transaction
+SET DEBUG_SYNC= 'now SIGNAL release_thrlock';
+# Connection con2
+# Reaping: INSERT INTO t1 VALUES (3)
+# Connection default
+DROP TABLE t1;
+SET DEBUG_SYNC= 'RESET';

mysql-test/t/lock_sync.test

@@ -1023,6 +1023,61 @@ DROP EVENT e2;
 SET DEBUG_SYNC="RESET";
 
 
+--echo #
+--echo # Bug#55930 Assertion `thd->transaction.stmt.is_empty() ||
+--echo #           thd->in_sub_stmt || (thd->state..
+--echo #
+
+--disable_warnings
+DROP TABLE IF EXISTS t1;
+--enable_warnings
+
+CREATE TABLE t1(a INT) engine=InnoDB;
+INSERT INTO t1 VALUES (1), (2);
+
+connect (con1, localhost, root);
+connect (con2, localhost, root);
+
+--echo # Connection con1
+connection con1;
+SET SESSION lock_wait_timeout= 1;
+SET DEBUG_SYNC= 'ha_admin_open_ltable SIGNAL opti_recreate WAIT_FOR opti_analyze';
+--echo # Sending:
+--send OPTIMIZE TABLE t1
+
+--echo # Connection con2
+connection con2;
+SET DEBUG_SYNC= 'now WAIT_FOR opti_recreate';
+SET DEBUG_SYNC= 'after_lock_tables_takes_lock SIGNAL thrlock WAIT_FOR release_thrlock';
+--echo # Sending:
+--send INSERT INTO t1 VALUES (3)
+
+--echo # Connection default
+connection default;
+SET DEBUG_SYNC= 'now WAIT_FOR thrlock';
+SET DEBUG_SYNC= 'now SIGNAL opti_analyze';
+
+--echo # Connection con1
+connection con1;
+--echo # Reaping: OPTIMIZE TABLE t1
+--reap
+SET DEBUG_SYNC= 'now SIGNAL release_thrlock';
+disconnect con1;
+--source include/wait_until_disconnected.inc
+
+--echo # Connection con2
+connection con2;
+--echo # Reaping: INSERT INTO t1 VALUES (3)
+--reap
+disconnect con2;
+--source include/wait_until_disconnected.inc
+
+--echo # Connection default
+connection default;
+DROP TABLE t1;
+SET DEBUG_SYNC= 'RESET';
+
+
 # Check that all connections opened by test cases in this file are really
 # gone so execution of other tests won't be affected by their presence.
 --source include/wait_until_count_sessions.inc

sql/sql_base.cc

@@ -5319,7 +5319,10 @@ TABLE *open_ltable(THD *thd, TABLE_LIST *table_list, thr_lock_type lock_type,
 
 end:
   if (table == NULL)
+  {
+    trans_rollback_stmt(thd);
     close_thread_tables(thd);
+  }
   thd_proc_info(thd, 0);
   DBUG_RETURN(table);
 }