MDEV-11236 Failing assertion: state == TRX_STATE_NOT_STARTED

trx_state_eq(): Add the parameter bool relaxed=false, to
allow trx->state==TRX_STATE_NOT_STARTED where a different
state is expected, if an error has been reported.

trx_release_savepoint_for_mysql(): Pass relaxed=true to
trx_state_eq(). That is, allow the transaction to be idle
when ROLLBACK TO SAVEPOINT is attempted after an error
has been reported to the client.

mysql-test/suite/innodb/r/trigger.result

+CREATE TABLE t1 (a INT) ENGINE=InnoDB;
+CREATE TABLE t2 (b INT) ENGINE=InnoDB;
+CREATE TABLE t3 (c INT) ENGINE=InnoDB;
+CREATE TRIGGER tr BEFORE INSERT ON t3 FOR EACH ROW BEGIN SAVEPOINT sv; INSERT INTO t2 VALUES (0); END $$
+START TRANSACTION;
+DELETE FROM t1;
+START TRANSACTION;
+INSERT INTO t2 VALUES (2);
+UPDATE t2 SET b = b+1;
+INSERT INTO t1 VALUES (1);
+COMMIT;
+COMMIT;
+SELECT * FROM t1;
+a
+1
+SELECT * FROM t2;
+b
+3
+SELECT * FROM t3;
+c
+DROP TABLE t1, t2, t3;
+DROP TRIGGER tr;
+ERROR HY000: Trigger does not exist

mysql-test/suite/innodb/t/trigger.test

+--source include/have_innodb.inc
+--source include/count_sessions.inc
+
+CREATE TABLE t1 (a INT) ENGINE=InnoDB;
+CREATE TABLE t2 (b INT) ENGINE=InnoDB;
+CREATE TABLE t3 (c INT) ENGINE=InnoDB;
+
+--delimiter $$
+CREATE TRIGGER tr BEFORE INSERT ON t3 FOR EACH ROW BEGIN SAVEPOINT sv; INSERT INTO t2 VALUES (0); END $$
+--delimiter ;
+
+START TRANSACTION;
+DELETE FROM t1;
+
+connect (con1,localhost,root,,test);
+START TRANSACTION;
+INSERT INTO t2 VALUES (2);
+UPDATE t2 SET b = b+1;
+
+--send
+INSERT INTO t1 VALUES (1);
+
+connection default;
+let $wait_condition=
+  select count(*) = 1 from information_schema.processlist
+  where state = 'update' and info = 'INSERT INTO t1 VALUES (1)'
+--source include/wait_condition.inc
+
+--error ER_LOCK_DEADLOCK
+INSERT INTO t3 VALUES (2);
+COMMIT;
+
+connection con1;
+reap;
+COMMIT;
+disconnect con1;
+
+connection default;
+SELECT * FROM t1;
+SELECT * FROM t2;
+SELECT * FROM t3;
+
+DROP TABLE t1, t2, t3;
+--error ER_TRG_DOES_NOT_EXIST
+DROP TRIGGER tr;
+
+--source include/wait_until_count_sessions.inc

storage/innobase/include/trx0trx.h

 /*****************************************************************************
 
 Copyright (c) 1996, 2016, Oracle and/or its affiliates. All Rights Reserved.
+Copyright (c) 2016, MariaDB Corporation. All Rights Reserved.
 
 This program is free software; you can redistribute it and/or modify it under
 the terms of the GNU General Public License as published by the Free Software
@@ -379,10 +380,14 @@ ibool
 trx_state_eq(
 /*=========*/
 	const trx_t*	trx,	/*!< in: transaction */
-	trx_state_t	state)	/*!< in: state;
+	trx_state_t	state,	/*!< in: state;
 				if state != TRX_STATE_NOT_STARTED
 				asserts that
 				trx->state != TRX_STATE_NOT_STARTED */
+	bool		relaxed = false)
+				/*!< in: whether to allow
+				trx->state == TRX_STATE_NOT_STARTED
+				after an error has been reported */
 	MY_ATTRIBUTE((nonnull, warn_unused_result));
 # ifdef UNIV_DEBUG
 /**********************************************************************//**

storage/innobase/include/trx0trx.ic

 /*****************************************************************************
 
 Copyright (c) 1996, 2011, Oracle and/or its affiliates. All Rights Reserved.
+Copyright (c) 2016, MariaDB Corporation. All Rights Reserved.
 
 This program is free software; you can redistribute it and/or modify it under
 the terms of the GNU General Public License as published by the Free Software
@@ -36,10 +37,14 @@ ibool
 trx_state_eq(
 /*=========*/
 	const trx_t*	trx,	/*!< in: transaction */
-	trx_state_t	state)	/*!< in: state;
+	trx_state_t	state,	/*!< in: state;
 				if state != TRX_STATE_NOT_STARTED
 				asserts that
 				trx->state != TRX_STATE_NOT_STARTED */
+	bool		relaxed)
+				/*!< in: whether to allow
+				trx->state == TRX_STATE_NOT_STARTED
+				after an error has been reported */
 {
 #ifdef UNIV_DEBUG
 	switch (trx->state) {
@@ -57,7 +62,9 @@ trx_state_eq(
 
 	case TRX_STATE_NOT_STARTED:
 		/* This state is not allowed for running transactions. */
-		ut_a(state == TRX_STATE_NOT_STARTED);
+		ut_a(state == TRX_STATE_NOT_STARTED
+		     || (relaxed
+			 && thd_get_error_number(trx->mysql_thd)));
 		ut_ad(!trx->in_rw_trx_list);
 		ut_ad(!trx->in_ro_trx_list);
 		return(state == trx->state);

storage/innobase/trx/trx0roll.cc

 /*****************************************************************************
 
 Copyright (c) 1996, 2016, Oracle and/or its affiliates. All Rights Reserved.
+Copyright (c) 2016, MariaDB Corporation. All Rights Reserved.
 
 This program is free software; you can redistribute it and/or modify it under
 the terms of the GNU General Public License as published by the Free Software
@@ -495,7 +496,8 @@ trx_release_savepoint_for_mysql(
 {
 	trx_named_savept_t*	savep;
 
-	ut_ad(trx_state_eq(trx, TRX_STATE_ACTIVE) || trx_state_eq(trx, TRX_STATE_PREPARED));
+	ut_ad(trx_state_eq(trx, TRX_STATE_ACTIVE, true)
+	      || trx_state_eq(trx, TRX_STATE_PREPARED, true));
 	ut_ad(trx->in_mysql_trx_list);
 
 	savep = trx_savepoint_find(trx, savepoint_name);

storage/xtradb/include/trx0trx.h

 /*****************************************************************************
 
 Copyright (c) 1996, 2016, Oracle and/or its affiliates. All Rights Reserved.
-Copyright (c) 2015, MariaDB Corporation
+Copyright (c) 2015, 2016, MariaDB Corporation. All Rights Reserved.
+
 
 This program is free software; you can redistribute it and/or modify it under
 the terms of the GNU General Public License as published by the Free Software
@@ -393,10 +394,14 @@ ibool
 trx_state_eq(
 /*=========*/
 	const trx_t*	trx,	/*!< in: transaction */
-	trx_state_t	state)	/*!< in: state;
+	trx_state_t	state,	/*!< in: state;
 				if state != TRX_STATE_NOT_STARTED
 				asserts that
 				trx->state != TRX_STATE_NOT_STARTED */
+	bool		relaxed = false)
+				/*!< in: whether to allow
+				trx->state == TRX_STATE_NOT_STARTED
+				after an error has been reported */
 	MY_ATTRIBUTE((nonnull, warn_unused_result));
 # ifdef UNIV_DEBUG
 /**********************************************************************//**

storage/xtradb/include/trx0trx.ic

 /*****************************************************************************
 
 Copyright (c) 1996, 2011, Oracle and/or its affiliates. All Rights Reserved.
+Copyright (c) 2016, MariaDB Corporation. All Rights Reserved.
 
 This program is free software; you can redistribute it and/or modify it under
 the terms of the GNU General Public License as published by the Free Software
@@ -36,10 +37,14 @@ ibool
 trx_state_eq(
 /*=========*/
 	const trx_t*	trx,	/*!< in: transaction */
-	trx_state_t	state)	/*!< in: state;
+	trx_state_t	state,	/*!< in: state;
 				if state != TRX_STATE_NOT_STARTED
 				asserts that
 				trx->state != TRX_STATE_NOT_STARTED */
+	bool		relaxed)
+				/*!< in: whether to allow
+				trx->state == TRX_STATE_NOT_STARTED
+				after an error has been reported */
 {
 #ifdef UNIV_DEBUG
 	switch (trx->state) {
@@ -57,7 +62,9 @@ trx_state_eq(
 
 	case TRX_STATE_NOT_STARTED:
 		/* This state is not allowed for running transactions. */
-		ut_a(state == TRX_STATE_NOT_STARTED);
+		ut_a(state == TRX_STATE_NOT_STARTED
+		     || (relaxed
+			 && thd_get_error_number(trx->mysql_thd)));
 		ut_ad(!trx->in_rw_trx_list);
 		ut_ad(!trx->in_ro_trx_list);
 		return(state == trx->state);

storage/xtradb/trx/trx0roll.cc

 /*****************************************************************************
 
 Copyright (c) 1996, 2016, Oracle and/or its affiliates. All Rights Reserved.
+Copyright (c) 2016, MariaDB Corporation. All Rights Reserved.
 
 This program is free software; you can redistribute it and/or modify it under
 the terms of the GNU General Public License as published by the Free Software
@@ -507,7 +508,8 @@ trx_release_savepoint_for_mysql(
 {
 	trx_named_savept_t*	savep;
 
-	ut_ad(trx_state_eq(trx, TRX_STATE_ACTIVE) || trx_state_eq(trx, TRX_STATE_PREPARED));
+	ut_ad(trx_state_eq(trx, TRX_STATE_ACTIVE, true)
+	      || trx_state_eq(trx, TRX_STATE_PREPARED, true));
 	ut_ad(trx->in_mysql_trx_list);
 
 	savep = trx_savepoint_find(trx, savepoint_name);