MDEV-15792 Fix mtr to be able to wait for >1 exited mysqld

If a mtr test case has started two mysqld processes (replication tests),
then kills the first one and kills the second one before starting the
first (so at some point there are two mysqlds down), then the ./mtr
waiting process bricks and forgets to monitor the "expect" file of the
first mysqld, so it never gets started again, even when its contents is
changed to "restart".

A victim of this deficiency is at least galera.galera_gcache_recover.

The fix is to keep a list of all mysqlds we should wait to start, not
just one (the last one killed).

mysql-test/mysql-test-run.pl

@@ -3980,14 +3980,14 @@ sub run_testcase ($$) {
   }
 
   my $test= $tinfo->{suite}->start_test($tinfo);
-  # Set only when we have to keep waiting after expectedly died server
-  my $keep_waiting_proc = 0;
+  # Set to a list of processes we have to keep waiting (expectedly died servers)
+  my %keep_waiting_proc = ();
   my $print_timeout= start_timer($print_freq * 60);
 
   while (1)
   {
-    my $proc;
-    if ($keep_waiting_proc)
+    my $proc = 0;
+    if (scalar(keys(%keep_waiting_proc)) > 0)
     {
       # Any other process exited?
       $proc = My::SafeProcess->check_any();
@@ -3997,20 +3997,19 @@ sub run_testcase ($$) {
       }
       else
       {
-	$proc = $keep_waiting_proc;
 	# Also check if timer has expired, if so cancel waiting
 	if ( has_expired($test_timeout) )
 	{
-	  $keep_waiting_proc = 0;
+	  %keep_waiting_proc = ();
 	}
       }
     }
-    if (! $keep_waiting_proc)
+    if (scalar(keys(%keep_waiting_proc)) == 0 && !$proc)
     {
-      if($test_timeout > $print_timeout)
+      if ($test_timeout > $print_timeout)
       {
         $proc= My::SafeProcess->wait_any_timeout($print_timeout);
-         if ( $proc->{timeout} )
+        if ($proc->{timeout})
         {
           #print out that the test is still on
           mtr_print("Test still running: $tinfo->{name}");
@@ -4025,20 +4024,7 @@ sub run_testcase ($$) {
       }
     }
 
-    # Will be restored if we need to keep waiting
-    $keep_waiting_proc = 0;
-
-    unless ( defined $proc )
-    {
-      mtr_error("wait_any failed");
-    }
-    mtr_verbose("Got $proc");
-
-    mark_time_used('test');
-    # ----------------------------------------------------
-    # Was it the test program that exited
-    # ----------------------------------------------------
-    if ($proc eq $test)
+    if ($proc eq $test) # mysqltest itself exited
     {
       my $res= $test->exit_status();
 
@@ -4141,18 +4127,39 @@ sub run_testcase ($$) {
       }
 
       return ($res == 62) ? 0 : $res;
+    }
 
+    if ($proc)
+    {
+      # It was not mysqltest that exited, add to a wait-to-be-started-again list.
+      $keep_waiting_proc{$proc} = 1;
     }
 
+    mtr_verbose("Got " . join(",", keys(%keep_waiting_proc)));
+
+    mark_time_used('test');
+    my $expected_exit = 1;
+    foreach my $wait_for_proc (keys(%keep_waiting_proc)) {
       # ----------------------------------------------------
       # Check if it was an expected crash
       # ----------------------------------------------------
-    my $check_crash = check_expected_crash_and_restart($proc);
-    if ($check_crash)
+      my $check_crash = check_expected_crash_and_restart($wait_for_proc);
+      if ($check_crash == 0) # unexpected exit/crash of $wait_for_proc
+      {
+        $expected_exit = 0;
+        last;
+      }
+      elsif ($check_crash == 1) # $wait_for_proc was started again by check_expected_crash_and_restart()
       {
-      # Keep waiting if it returned 2, if 1 don't wait or stop waiting.
-      $keep_waiting_proc = 0 if $check_crash == 1;
-      $keep_waiting_proc = $proc if $check_crash == 2;
+        delete $keep_waiting_proc{$wait_for_proc};
+      }
+      elsif ($check_crash == 2) # we must keep waiting
+      {
+        # do nothing
+      }
+    }
+
+    if ($expected_exit) {
       next;
     }