Merge 10.6 into 10.9

.github/CODEOWNERS

-/debian @ottok
-

debian/additions/innotop/innotop

@@ -20,6 +20,9 @@
 # Street, Fifth Floor, Boston, MA 02110-1335 USA
 
 use strict;
+use warnings;
+use utf8;
+use feature ':5.16';
 use warnings FATAL => 'all';
 
 our $VERSION = '1.11.4';
@@ -265,7 +268,7 @@ sub get_dbh {
             $dbh->do($sql);
             MKDEBUG && _d('Enabling charset for STDOUT');
             if ( $charset eq 'utf8' ) {
-               binmode(STDOUT, ':utf8')
+               binmode(STDOUT, ':encoding(UTF-8)')
                   or die "Can't binmode(STDOUT, ':utf8'): $OS_ERROR";
             }
             else {
@@ -612,6 +615,9 @@ sub ts_to_string {
 
 sub parse_innodb_timestamp {
    my $text = shift;
+   if ( ! defined $text ) {
+        return (0, 0, 0, 0, 0, 0);
+   }
    my ( $y, $m, $d, $h, $i, $s )
       = $text =~ m/^(\d\d)(\d\d)(\d\d) +(\d+):(\d+):(\d+)$/;
    die("Can't get timestamp from $text\n") unless $y;
@@ -803,7 +809,8 @@ sub parse_fk_transaction_error {
 # TODO: write some tests for this
 sub parse_innodb_record_dump {
    my ( $dump, $complete, $debug ) = @_;
-   return undef unless $dump;
+   # Use bare return as recommend in page 199 of PBP
+   return unless $dump;
 
    my $result = {};
 
@@ -6769,6 +6776,9 @@ sub set_precision {
    my ( $num, $precision ) = @_;
    $num = 0 unless defined $num;
    $precision = $config{num_digits}->{val} if !defined $precision;
+   if ( $num eq "" ) {
+       $num = int(0);
+   }
    sprintf("%.${precision}f", $num);
 }
 
@@ -6777,6 +6787,9 @@ sub set_precision {
 sub percent {
    my ( $num ) = @_;
    $num = 0 unless defined $num;
+   if ( $num eq "" ) {
+       $num = int(0);
+   }
    my $digits = $config{num_digits}->{val};
    return sprintf("%.${digits}f", $num * 100)
       . ($config{show_percent}->{val} ? '%' : '');
@@ -6841,7 +6854,7 @@ sub make_color_func {
       push @criteria,
          "( defined \$set->{$spec->{col}} && \$set->{$spec->{col}} $spec->{op} $val ) { return '$spec->{color}'; }";
    }
-   return undef unless @criteria;
+   return unless @criteria;
    my $sub = eval 'sub { my ( $set ) = @_; if ' . join(" elsif ", @criteria) . '}';
    die if $EVAL_ERROR;
    return $sub;
@@ -7521,10 +7534,10 @@ sub choose_connections {
 sub do_stmt {
    my ( $cxn, $stmt_name, @args ) = @_;
 
-   return undef if $file;
+   return if $file;
 
    # Test if the cxn should not even be tried
-   return undef if $dbhs{$cxn}
+   return if $dbhs{$cxn}
       && $dbhs{$cxn}->{failed}
       && ( !$dbhs{$cxn}->{dbh} || !$dbhs{$cxn}->{dbh}->{Active} || $dbhs{$cxn}->{mode} eq $config{mode}->{val} );
 
@@ -7596,10 +7609,10 @@ sub handle_cxn_error {
 sub do_query {
    my ( $cxn, $query ) = @_;
 
-   return undef if $file;
+   return if $file;
 
    # Test if the cxn should not even be tried
-   return undef if $dbhs{$cxn}
+   return if $dbhs{$cxn}
       && $dbhs{$cxn}->{failed}
       && ( !$dbhs{$cxn}->{dbh} || !$dbhs{$cxn}->{dbh}->{Active} || $dbhs{$cxn}->{mode} eq $config{mode}->{val} );
 
@@ -7781,7 +7794,7 @@ sub compile_select_stmt {
 sub compile_filter {
    my ( $text ) = @_;
    my ( $sub, $err );
-   eval "\$sub = sub { my \$set = shift; $text }";
+   eval { $sub = sub { my $set = shift; $text } };
    if ( $EVAL_ERROR ) {
       $EVAL_ERROR =~ s/at \(eval.*$//;
       $sub = sub { return $EVAL_ERROR };
@@ -8013,7 +8026,7 @@ sub load_config_plugins {
 
    # First, find a list of all plugins that exist on disk, and get information about them.
    my $dir = $config{plugin_dir}->{val};
-   foreach my $p_file ( <$dir/*.pm> ) {
+   foreach my $p_file (glob($dir."/*.pm")) {
       my ($package, $desc);
       eval {
          open my $p_in, "<", $p_file or die $OS_ERROR;
@@ -9192,7 +9205,7 @@ sub switch_var_set {
 # edit_stmt_sleep_times {{{3
 sub edit_stmt_sleep_times {
    $clear_screen_sub->();
-   my $stmt = prompt_list('Specify a statement', '', sub { return sort keys %stmt_maker_for });
+   my $stmt = prompt_list('Specify a statement', '', sub { my @tmparray = sort keys %stmt_maker_for; return @tmparray });
    return unless $stmt && exists $stmt_maker_for{$stmt};
    $clear_screen_sub->();
    my $curr_val = $stmt_sleep_time_for{$stmt} || 0;
@@ -9843,7 +9856,7 @@ sub get_slave_status {
 sub is_func {
    my ( $word ) = @_;
    return defined(&$word)
-      || eval "my \$x= sub { $word  }; 1"
+      || eval { my $x = sub { $word  }; 1 }
       || $EVAL_ERROR !~ m/^Bareword/;
 }
 

storage/innobase/buf/buf0flu.cc

@@ -966,11 +966,19 @@ uint32_t fil_space_t::flush_freed(bool writable)
   mysql_mutex_assert_not_owner(&buf_pool.flush_list_mutex);
   mysql_mutex_assert_not_owner(&buf_pool.mutex);
 
-  freed_range_mutex.lock();
-  if (freed_ranges.empty() || log_sys.get_flushed_lsn() < get_last_freed_lsn())
+  for (;;)
   {
+    freed_range_mutex.lock();
+    if (freed_ranges.empty())
+    {
+      freed_range_mutex.unlock();
+      return 0;
+    }
+    const lsn_t flush_lsn= last_freed_lsn;
+    if (log_sys.get_flushed_lsn() >= flush_lsn)
+      break;
     freed_range_mutex.unlock();
-    return 0;
+    log_write_up_to(flush_lsn, true);
   }
 
   const unsigned physical{physical_size()};

storage/innobase/buf/buf0rea.cc

@@ -265,9 +265,6 @@ buf_read_page_low(
 	buf_page_t*	bpage;
 
 	if (buf_dblwr.is_inside(page_id)) {
-		ib::error() << "Trying to read doublewrite buffer page "
-			<< page_id;
-		ut_ad(0);
 		space->release();
 		return DB_PAGE_CORRUPTED;
 	}

storage/innobase/include/fil0fil.h

@@ -638,8 +638,6 @@ struct fil_space_t final
   /** Close all tablespace files at shutdown */
   static void close_all();
 
-  /** @return last_freed_lsn */
-  lsn_t get_last_freed_lsn() { return last_freed_lsn; }
   /** Update last_freed_lsn */
   void update_last_freed_lsn(lsn_t lsn) { last_freed_lsn= lsn; }
 

storage/innobase/include/trx0sys.h

@@ -1053,7 +1053,7 @@ class trx_sys_t
   void close();
 
   /** @return total number of active (non-prepared) transactions */
-  ulint any_active_transactions();
+  size_t any_active_transactions(size_t *prepared= nullptr);
 
 
   /**

storage/innobase/srv/srv0srv.cc

@@ -1514,7 +1514,7 @@ void srv_master_callback(void*)
 }
 
 /** @return whether purge should exit due to shutdown */
-static bool srv_purge_should_exit()
+static bool srv_purge_should_exit(size_t old_history_size)
 {
   ut_ad(srv_shutdown_state <= SRV_SHUTDOWN_CLEANUP);
 
@@ -1525,8 +1525,12 @@ static bool srv_purge_should_exit()
     return true;
 
   /* Slow shutdown was requested. */
+  size_t prepared, active= trx_sys.any_active_transactions(&prepared);
   const size_t history_size= trx_sys.history_size();
-  if (history_size)
+
+  if (!history_size);
+  else if (!active && history_size == old_history_size && prepared);
+  else
   {
     static time_t progress_time;
     time_t now= time(NULL);
@@ -1543,7 +1547,7 @@ static bool srv_purge_should_exit()
     return false;
   }
 
-  return !trx_sys.any_active_transactions();
+  return !active;
 }
 
 /*********************************************************************//**
@@ -1686,7 +1690,7 @@ inline void purge_coordinator_state::do_purge()
       break;
     }
 
-    if (!srv_purge_should_exit())
+    if (!srv_purge_should_exit(history_size))
       goto loop;
   }
 
@@ -1882,15 +1886,19 @@ ulint srv_get_task_queue_length()
 /** Shut down the purge threads. */
 void srv_purge_shutdown()
 {
-	if (purge_sys.enabled()) {
-		if (!srv_fast_shutdown && !opt_bootstrap)
-			srv_update_purge_thread_count(innodb_purge_threads_MAX);
-		while(!srv_purge_should_exit()) {
-			ut_a(!purge_sys.paused());
-			srv_wake_purge_thread_if_not_active();
-			purge_coordinator_task.wait();
-		}
-		purge_sys.coordinator_shutdown();
-		srv_shutdown_purge_tasks();
-	}
+  if (purge_sys.enabled())
+  {
+    if (!srv_fast_shutdown && !opt_bootstrap)
+      srv_update_purge_thread_count(innodb_purge_threads_MAX);
+    size_t history_size= trx_sys.history_size();
+    while (!srv_purge_should_exit(history_size))
+    {
+      history_size= trx_sys.history_size();
+      ut_a(!purge_sys.paused());
+      srv_wake_purge_thread_if_not_active();
+      purge_coordinator_task.wait();
+    }
+    purge_sys.coordinator_shutdown();
+    srv_shutdown_purge_tasks();
+  }
 }

storage/innobase/trx/trx0purge.cc

@@ -369,19 +369,6 @@ trx_purge_add_undo_to_history(const trx_t* trx, trx_undo_t*& undo, mtr_t* mtr)
 	undo = NULL;
 }
 
-MY_ATTRIBUTE((nonnull, warn_unused_result))
-/** Remove undo log header from the history list.
-@param[in,out]  rseg    rollback segment header page
-@param[in]      log     undo log segment header page
-@param[in]      offset  byte offset in the undo log segment header page
-@param[in,out]  mtr     mini-transaction */
-static dberr_t trx_purge_remove_log_hdr(buf_block_t *rseg, buf_block_t* log,
-                                        uint16_t offset, mtr_t *mtr)
-{
-  return flst_remove(rseg, TRX_RSEG + TRX_RSEG_HISTORY, log,
-                     uint16_t(offset + TRX_UNDO_HISTORY_NODE), mtr);
-}
-
 /** Free an undo log segment.
 @param block     rollback segment header page
 @param mtr       mini-transaction */
@@ -391,7 +378,7 @@ static void trx_purge_free_segment(buf_block_t *block, mtr_t &mtr)
                                     block->page.frame, &mtr))
   {
     block->fix();
-    const page_id_t id{block->page.id()};
+    ut_d(const page_id_t id{block->page.id()});
     mtr.commit();
     /* NOTE: If the server is killed after the log that was produced
     up to this point was written, and before the log from the mtr.commit()
@@ -403,16 +390,8 @@ static void trx_purge_free_segment(buf_block_t *block, mtr_t &mtr)
     log_free_check();
     mtr.start();
     block->page.lock.x_lock();
-    if (UNIV_UNLIKELY(block->page.id() != id))
-    {
-      block->unfix();
-      block->page.lock.x_unlock();
-      block= buf_page_get_gen(id, 0, RW_X_LATCH, nullptr, BUF_GET, &mtr);
-      if (!block)
-        return;
-    }
-    else
-      mtr.memo_push(block, MTR_MEMO_PAGE_X_MODIFY);
+    ut_ad(block->page.id() == id);
+    mtr.memo_push(block, MTR_MEMO_PAGE_X_MODIFY);
   }
 
   while (!fseg_free_step(TRX_UNDO_SEG_HDR + TRX_UNDO_FSEG_HEADER +
@@ -434,7 +413,6 @@ trx_purge_truncate_rseg_history(trx_rseg_t& rseg,
   mtr.start();
 
   dberr_t err;
-reget:
   buf_block_t *rseg_hdr= rseg.get(&mtr, &err);
   if (!rseg_hdr)
   {
@@ -474,18 +452,16 @@ trx_purge_truncate_rseg_history(trx_rseg_t& rseg,
                        TRX_UNDO_HISTORY_NODE);
   prev_hdr_addr.boffset= static_cast<uint16_t>(prev_hdr_addr.boffset -
                                                TRX_UNDO_HISTORY_NODE);
-  err= trx_purge_remove_log_hdr(rseg_hdr, b, hdr_addr.boffset, &mtr);
+
+  err= flst_remove(rseg_hdr, TRX_RSEG + TRX_RSEG_HISTORY, b,
+                   uint16_t(hdr_addr.boffset + TRX_UNDO_HISTORY_NODE), &mtr);
   if (UNIV_UNLIKELY(err != DB_SUCCESS))
     goto func_exit;
 
   rseg_hdr->fix();
 
-  if (mach_read_from_2(b->page.frame + hdr_addr.boffset + TRX_UNDO_NEXT_LOG) ||
-      rseg.is_referenced() ||
-      rseg.needs_purge > (purge_sys.head.trx_no
-                          ? purge_sys.head.trx_no
-                          : purge_sys.tail.trx_no))
-    /* We cannot free the entire undo page. */;
+  if (mach_read_from_2(b->page.frame + hdr_addr.boffset + TRX_UNDO_NEXT_LOG))
+    /* We cannot free the entire undo log segment. */;
   else
   {
     const uint32_t seg_size=
@@ -535,12 +511,7 @@ trx_purge_truncate_rseg_history(trx_rseg_t& rseg,
   log_free_check();
   mtr.start();
   rseg_hdr->page.lock.x_lock();
-  if (UNIV_UNLIKELY(rseg_hdr->page.id() != rseg.page_id()))
-  {
-    rseg_hdr->unfix();
-    rseg_hdr->page.lock.x_unlock();
-    goto reget;
-  }
+  ut_ad(rseg_hdr->page.id() == rseg.page_id());
   mtr.memo_push(rseg_hdr, MTR_MEMO_PAGE_X_MODIFY);
 
   goto loop;
@@ -613,8 +584,9 @@ TRANSACTIONAL_TARGET static void trx_purge_truncate_history()
     {
       ut_ad(rseg.is_persistent());
       rseg.latch.wr_lock(SRW_LOCK_CALL);
-      if (dberr_t e= trx_purge_truncate_rseg_history(rseg, head))
-        err= e;
+      if (!rseg.is_referenced() && rseg.needs_purge <= head.trx_no)
+        if (dberr_t e= trx_purge_truncate_rseg_history(rseg, head))
+          err= e;
       rseg.latch.wr_unlock();
     }
 

storage/innobase/trx/trx0sys.cc

@@ -343,15 +343,29 @@ trx_sys_t::close()
 }
 
 /** @return total number of active (non-prepared) transactions */
-ulint trx_sys_t::any_active_transactions()
+size_t trx_sys_t::any_active_transactions(size_t *prepared)
 {
-  uint32_t total_trx= 0;
+  size_t total_trx= 0, prepared_trx= 0;
 
-  trx_sys.trx_list.for_each([&total_trx](const trx_t &trx) {
-    if (trx.state == TRX_STATE_COMMITTED_IN_MEMORY ||
-        (trx.state == TRX_STATE_ACTIVE && trx.id))
+  trx_sys.trx_list.for_each([&](const trx_t &trx) {
+    switch (trx.state) {
+    case TRX_STATE_NOT_STARTED:
+      break;
+    case TRX_STATE_ACTIVE:
+      if (!trx.id)
+        break;
+      /* fall through */
+    case TRX_STATE_COMMITTED_IN_MEMORY:
       total_trx++;
+      break;
+    case TRX_STATE_PREPARED:
+    case TRX_STATE_PREPARED_RECOVERED:
+      prepared_trx++;
+    }
   });
 
+  if (prepared)
+    *prepared= prepared_trx;
+
   return total_trx;
 }