MDEV-31114 Assertion !...is_waiting() failed in os_aio_wait_until_no_pending_writes()

os_aio_wait_until_no_pending_reads(), os_aio_wait_until_pending_writes():
Add a Boolean parameter to indicate whether the wait should be declared
in the thread pool.

buf_flush_wait(): The callers have already declared a wait, so let us
avoid doing that again, just call os_aio_wait_until_pending_writes(false).

buf_flush_wait_flushed(): Do not declare a wait in the rare case that
the buf_flush_page_cleaner thread has been shut down already.

buf_flush_page_cleaner(), buf_flush_buffer_pool(): In the code that runs
during shutdown, do not declare waits.

buf_flush_buffer_pool(): Remove a debug assertion that might fail.
What really matters here is buf_pool.flush_list.count==0.

buf_read_recv_pages(), srv_prepare_to_delete_redo_log_file():
Do not declare waits during InnoDB startup.

storage/innobase/buf/buf0buf.cc

@@ -1317,11 +1317,11 @@ buf_tmp_buffer_t *buf_pool_t::io_buf_t::reserve()
     for (buf_tmp_buffer_t *s= slots, *e= slots + n_slots; s != e; s++)
       if (s->acquire())
         return s;
-    os_aio_wait_until_no_pending_writes();
+    os_aio_wait_until_no_pending_writes(true);
     for (buf_tmp_buffer_t *s= slots, *e= slots + n_slots; s != e; s++)
       if (s->acquire())
         return s;
-    os_aio_wait_until_no_pending_reads();
+    os_aio_wait_until_no_pending_reads(true);
   }
 }
 

storage/innobase/buf/buf0dump.cc

@@ -647,7 +647,7 @@ buf_load()
 	ut_free(dump);
 
 	if (i == dump_n) {
-		os_aio_wait_until_no_pending_reads();
+		os_aio_wait_until_no_pending_reads(true);
 	}
 
 	ut_sprintf_timestamp(now);

storage/innobase/buf/buf0flu.cc

@@ -246,7 +246,7 @@ void buf_flush_remove_pages(ulint id)
     if (!deferred)
       break;
 
-    os_aio_wait_until_no_pending_writes();
+    os_aio_wait_until_no_pending_writes(true);
   }
 }
 
@@ -1692,7 +1692,7 @@ bool buf_flush_list_space(fil_space_t *space, ulint *n_flushed)
     space->release();
 
   if (space->purpose == FIL_TYPE_IMPORT)
-    os_aio_wait_until_no_pending_writes();
+    os_aio_wait_until_no_pending_writes(true);
   else
     buf_dblwr.flush_buffered_writes();
 
@@ -1862,7 +1862,7 @@ static void buf_flush_wait(lsn_t lsn)
         break;
     }
     mysql_mutex_unlock(&buf_pool.flush_list_mutex);
-    os_aio_wait_until_no_pending_writes();
+    os_aio_wait_until_no_pending_writes(false);
     mysql_mutex_lock(&buf_pool.flush_list_mutex);
   }
 }
@@ -1898,7 +1898,7 @@ ATTRIBUTE_COLD void buf_flush_wait_flushed(lsn_t sync_lsn)
                                        MONITOR_FLUSH_SYNC_COUNT,
                                        MONITOR_FLUSH_SYNC_PAGES, n_pages);
         }
-        os_aio_wait_until_no_pending_writes();
+        os_aio_wait_until_no_pending_writes(false);
         mysql_mutex_lock(&buf_pool.flush_list_mutex);
       }
       while (buf_pool.get_oldest_modification(sync_lsn) < sync_lsn);
@@ -2421,7 +2421,7 @@ static void buf_flush_page_cleaner()
     mysql_mutex_lock(&buf_pool.flush_list_mutex);
     buf_flush_wait_LRU_batch_end();
     mysql_mutex_unlock(&buf_pool.flush_list_mutex);
-    os_aio_wait_until_no_pending_writes();
+    os_aio_wait_until_no_pending_writes(false);
   }
 
   mysql_mutex_lock(&buf_pool.flush_list_mutex);
@@ -2471,7 +2471,7 @@ ATTRIBUTE_COLD void buf_flush_buffer_pool()
   {
     mysql_mutex_unlock(&buf_pool.flush_list_mutex);
     buf_flush_list(srv_max_io_capacity);
-    os_aio_wait_until_no_pending_writes();
+    os_aio_wait_until_no_pending_writes(false);
     mysql_mutex_lock(&buf_pool.flush_list_mutex);
     service_manager_extend_timeout(INNODB_EXTEND_TIMEOUT_INTERVAL,
                                    "Waiting to flush " ULINTPF " pages",
@@ -2479,7 +2479,6 @@ ATTRIBUTE_COLD void buf_flush_buffer_pool()
   }
 
   mysql_mutex_unlock(&buf_pool.flush_list_mutex);
-  ut_ad(!os_aio_pending_writes());
   ut_ad(!os_aio_pending_reads());
 }
 

storage/innobase/buf/buf0rea.cc

@@ -687,7 +687,7 @@ void buf_read_recv_pages(ulint space_id, const uint32_t* page_nos, ulint n)
 		}
 
 		if (os_aio_pending_reads() >= limit) {
-			os_aio_wait_until_no_pending_reads();
+			os_aio_wait_until_no_pending_reads(false);
 		}
 
 		space->reacquire();

storage/innobase/include/os0file.h

@@ -1066,11 +1066,13 @@ size_t os_aio_pending_reads_approx();
 /** @return number of pending writes */
 size_t os_aio_pending_writes();
 
-/** Wait until there are no pending asynchronous writes. */
-void os_aio_wait_until_no_pending_writes();
+/** Wait until there are no pending asynchronous writes.
+@param declare  whether the wait will be declared in tpool */
+void os_aio_wait_until_no_pending_writes(bool declare);
 
-/** Wait until all pending asynchronous reads have completed. */
-void os_aio_wait_until_no_pending_reads();
+/** Wait until all pending asynchronous reads have completed.
+@param declare  whether the wait will be declared in tpool */
+void os_aio_wait_until_no_pending_reads(bool declare);
 
 /** Prints info of the aio arrays.
 @param[in/out]	file		file where to print */

storage/innobase/log/log0recv.cc

@@ -3407,7 +3407,7 @@ void recv_sys_t::apply(bool last_batch)
           else
           {
             mysql_mutex_unlock(&mutex);
-            os_aio_wait_until_no_pending_reads();
+            os_aio_wait_until_no_pending_reads(false);
             mysql_mutex_lock(&mutex);
             ut_ad(pages.empty());
           }

storage/innobase/os/os0file.cc

@@ -3644,9 +3644,9 @@ void os_aio_free()
 }
 
 /** Wait until there are no pending asynchronous writes. */
-static void os_aio_wait_until_no_pending_writes_low()
+static void os_aio_wait_until_no_pending_writes_low(bool declare)
 {
-  bool notify_wait = write_slots->pending_io_count() > 0;
+  const bool notify_wait= declare && write_slots->pending_io_count();
 
   if (notify_wait)
     tpool::tpool_wait_begin();
@@ -3657,10 +3657,11 @@ static void os_aio_wait_until_no_pending_writes_low()
      tpool::tpool_wait_end();
 }
 
-/** Wait until there are no pending asynchronous writes. */
-void os_aio_wait_until_no_pending_writes()
+/** Wait until there are no pending asynchronous writes.
+@param declare  whether the wait will be declared in tpool */
+void os_aio_wait_until_no_pending_writes(bool declare)
 {
-  os_aio_wait_until_no_pending_writes_low();
+  os_aio_wait_until_no_pending_writes_low(declare);
   buf_dblwr.wait_flush_buffered_writes();
 }
 
@@ -3688,10 +3689,11 @@ size_t os_aio_pending_writes()
   return pending;
 }
 
-/** Wait until all pending asynchronous reads have completed. */
-void os_aio_wait_until_no_pending_reads()
+/** Wait until all pending asynchronous reads have completed.
+@param declare  whether the wait will be declared in tpool */
+void os_aio_wait_until_no_pending_reads(bool declare)
 {
-  const auto notify_wait= read_slots->pending_io_count();
+  const bool notify_wait= declare && read_slots->pending_io_count();
 
   if (notify_wait)
     tpool::tpool_wait_begin();

storage/innobase/row/row0quiesce.cc

@@ -553,7 +553,7 @@ row_quiesce_table_start(
 
 	if (!trx_is_interrupted(trx)) {
 		/* Ensure that all asynchronous IO is completed. */
-		os_aio_wait_until_no_pending_writes();
+		os_aio_wait_until_no_pending_writes(true);
 		table->space->flush<false>();
 
 		if (row_quiesce_write_cfg(table, trx->mysql_thd)

storage/innobase/srv/srv0start.cc

@@ -973,7 +973,7 @@ static lsn_t srv_prepare_to_delete_redo_log_file(bool old_exists)
   ut_d(mysql_mutex_lock(&buf_pool.flush_list_mutex));
   ut_ad(!buf_pool.get_oldest_modification(0));
   ut_d(mysql_mutex_unlock(&buf_pool.flush_list_mutex));
-  ut_d(os_aio_wait_until_no_pending_writes());
+  ut_d(os_aio_wait_until_no_pending_writes(false));
 
   DBUG_RETURN(flushed_lsn);
 }