MDEV-7148 - Recurring: InnoDB: Failing assertion: !lock->recursive

On PPC64 high-loaded server may crash due to assertion failure in InnoDB
rwlocks code.

This happened because load order between "recursive" and "writer_thread"
wasn't properly enforced.

storage/innobase/include/sync0rw.ic

@@ -438,6 +438,7 @@ rw_lock_x_lock_func_nowait(
 	os_thread_id_t	curr_thread	= os_thread_get_curr_id();
 
 	ibool success;
+	ibool local_recursive= lock->recursive;
 
 #ifdef INNODB_RW_LOCKS_USE_ATOMICS
 	success = os_compare_and_swap_lint(&lock->lock_word, X_LOCK_DECR, 0);
@@ -452,10 +453,14 @@ rw_lock_x_lock_func_nowait(
 	mutex_exit(&(lock->mutex));
 
 #endif
+	/* Note: recursive must be loaded before writer_thread see
+	comment for rw_lock_set_writer_id_and_recursion_flag().
+	To achieve this we load it before os_compare_and_swap_lint(),
+	which implies full memory barrier in current implementation. */
 	if (success) {
 		rw_lock_set_writer_id_and_recursion_flag(lock, TRUE);
 
-	} else if (lock->recursive
+	} else if (local_recursive
 		   && os_thread_eq(lock->writer_thread, curr_thread)) {
 		/* Relock: this lock_word modification is safe since no other
 		threads can modify (lock, unlock, or reserve) lock_word while

storage/innobase/sync/sync0rw.c

@@ -566,6 +566,7 @@ rw_lock_x_lock_low(
 	ulint		line)	/*!< in: line where requested */
 {
 	os_thread_id_t	curr_thread	= os_thread_get_curr_id();
+	ibool local_recursive= lock->recursive;
 
 	if (rw_lock_lock_word_decr(lock, X_LOCK_DECR)) {
 
@@ -586,10 +587,12 @@ rw_lock_x_lock_low(
                                     file_name, line);
 
 	} else {
-               if (!pass)
-                 os_rmb;
-		/* Decrement failed: relock or failed lock */
-		if (!pass && lock->recursive
+		/* Decrement failed: relock or failed lock
+		Note: recursive must be loaded before writer_thread see
+		comment for rw_lock_set_writer_id_and_recursion_flag().
+		To achieve this we load it before rw_lock_lock_word_decr(),
+		which implies full memory barrier in current implementation. */
+		if (!pass && local_recursive
 		    && os_thread_eq(lock->writer_thread, curr_thread)) {
 			/* Relock */
                         lock->lock_word -= X_LOCK_DECR;

storage/xtradb/include/sync0rw.ic

@@ -438,6 +438,7 @@ rw_lock_x_lock_func_nowait(
 	os_thread_id_t	curr_thread	= os_thread_get_curr_id();
 
 	ibool success;
+	ibool local_recursive= lock->recursive;
 
 #ifdef INNODB_RW_LOCKS_USE_ATOMICS
 	success = os_compare_and_swap_lint(&lock->lock_word, X_LOCK_DECR, 0);
@@ -452,10 +453,14 @@ rw_lock_x_lock_func_nowait(
 	mutex_exit(&(lock->mutex));
 
 #endif
+	/* Note: recursive must be loaded before writer_thread see
+	comment for rw_lock_set_writer_id_and_recursion_flag().
+	To achieve this we load it before os_compare_and_swap_lint(),
+	which implies full memory barrier in current implementation. */
 	if (success) {
 		rw_lock_set_writer_id_and_recursion_flag(lock, TRUE);
 
-	} else if (lock->recursive
+	} else if (local_recursive
 		   && os_thread_eq(lock->writer_thread, curr_thread)) {
 		/* Relock: this lock_word modification is safe since no other
 		threads can modify (lock, unlock, or reserve) lock_word while

storage/xtradb/sync/sync0rw.c

@@ -563,6 +563,7 @@ rw_lock_x_lock_low(
 	ulint		line)	/*!< in: line where requested */
 {
 	os_thread_id_t	curr_thread	= os_thread_get_curr_id();
+	ibool local_recursive= lock->recursive;
 
 	if (rw_lock_lock_word_decr(lock, X_LOCK_DECR)) {
 
@@ -583,10 +584,12 @@ rw_lock_x_lock_low(
                                     file_name, line);
 
 	} else {
-               if (!pass)
-                 os_rmb;
-		/* Decrement failed: relock or failed lock */
-		if (!pass && lock->recursive
+		/* Decrement failed: relock or failed lock
+		Note: recursive must be loaded before writer_thread see
+		comment for rw_lock_set_writer_id_and_recursion_flag().
+		To achieve this we load it before rw_lock_lock_word_decr(),
+		which implies full memory barrier in current implementation. */
+		if (!pass && local_recursive
 		    && os_thread_eq(lock->writer_thread, curr_thread)) {
 			/* Relock */
                         lock->lock_word -= X_LOCK_DECR;