BUG#25032066 PREPARED TRANSACTION SHOULD NOT BE ROLLED BACK BY HIGH PRIORITY TRANSACTION

Problem :
---------
1. delete_all_rows() and rnd_init() are not returning error
after async rollback in 5.7. This results in assert in
innodb in next call.

2. High priority transaction is rolling back prepared transaction.
This is because TRX_FORCE_ROLLBACK_DISABLE is getting set only for
first entry [TrxInInnoDB].

Solution :
----------
1. return DB_FORCED_ABORT error after rollback.
2. check and disable rollback in TrxInInnodb::enter always.
Reviewed-by: Sunny Bains <sunny.bains@oracle.com>

RB: 13777

storage/innobase/handler/ha_innodb.cc

@@ -10471,12 +10471,6 @@ ha_innobase::rnd_init(
 	bool	scan)	/*!< in: true if table/index scan FALSE otherwise */
 {
 	TrxInInnoDB	trx_in_innodb(m_prebuilt->trx);
-
-	if (trx_in_innodb.is_aborted()) {
-
-		return(innobase_rollback(ht, m_user_thd, false));
-	}
-
 	int		err;
 
 	/* Store the active index value so that we can restore the original

storage/innobase/include/trx0trx.h

@@ -1468,10 +1468,30 @@ class TrxInInnoDB {
 			return;
 		}
 
-		/* Avoid excessive mutex acquire/release */
-
 		ut_ad(!is_async_rollback(trx));
 
+		/* If it hasn't already been marked for async rollback.
+		and it will be committed/rolled back. */
+		if (disable) {
+
+			trx_mutex_enter(trx);
+			if (!is_forced_rollback(trx)
+			    && is_started(trx)
+			    && !trx_is_autocommit_non_locking(trx)) {
+
+				ut_ad(trx->killed_by == 0);
+
+				/* This transaction has crossed the point of
+				no return and cannot be rolled back
+				asynchronously now. It must commit or rollback
+				synhronously. */
+
+				trx->in_innodb |= TRX_FORCE_ROLLBACK_DISABLE;
+			}
+			trx_mutex_exit(trx);
+		}
+
+		/* Avoid excessive mutex acquire/release */
 		++trx->in_depth;
 
 		/* If trx->in_depth is greater than 1 then
@@ -1489,25 +1509,7 @@ class TrxInInnoDB {
 
 		wait(trx);
 
-		ut_ad((trx->in_innodb & TRX_FORCE_ROLLBACK_MASK)
-		      < (TRX_FORCE_ROLLBACK_MASK - 1));
-
-		/* If it hasn't already been marked for async rollback.
-		and it will be committed/rolled back. */
-
-		if (!is_forced_rollback(trx)
-		    && disable
-		    && is_started(trx)
-		    && !trx_is_autocommit_non_locking(trx)) {
-
-			ut_ad(trx->killed_by == 0);
-
-			/* This transaction has crossed the point of no
-			return and cannot be rolled back asynchronously
-			now. It must commit or rollback synhronously. */
-
-			trx->in_innodb |= TRX_FORCE_ROLLBACK_DISABLE;
-		}
+		ut_ad((trx->in_innodb & TRX_FORCE_ROLLBACK_MASK) == 0);
 
 		++trx->in_innodb;