Merge mysql.com:/Users/kent/mysql/bk/mysql-4.1-perl

into mysql.com:/Users/kent/mysql/bk/mysql-5.0-compile

mysql-test/lib/mtr_io.pl

@@ -8,6 +8,7 @@ use strict;
 
 sub mtr_get_pid_from_file ($);
 sub mtr_get_opts_from_file ($);
+sub mtr_fromfile ($);
 sub mtr_tofile ($@);
 sub mtr_tonewfile($@);
 
@@ -107,6 +108,8 @@ sub mtr_fromfile ($) {
   open(FILE,"<",$file) or mtr_error("can't open file \"$file\": $!");
   my $text= join('', <FILE>);
   close FILE;
+  $text =~ s/^\s+//;                    # Remove starting space, incl newlines
+  $text =~ s/\s+$//;                    # Remove ending space, incl newlines
   return $text;
 }
 

mysql-test/lib/mtr_process.pl

@@ -5,14 +5,19 @@
 # same name.
 
 #use Carp qw(cluck);
+use Socket;
+use Errno;
 use strict;
 
-use POSIX ":sys_wait_h";
+#use POSIX ":sys_wait_h";
+use POSIX 'WNOHANG';
 
 sub mtr_run ($$$$$$);
 sub mtr_spawn ($$$$$$);
-sub mtr_stop_mysqld_servers ($$);
+sub mtr_stop_mysqld_servers ($);
 sub mtr_kill_leftovers ();
+sub mtr_record_dead_children ();
+sub sleep_until_file_created ($$$);
 
 # static in C
 sub spawn_impl ($$$$$$$);
@@ -34,7 +39,18 @@ sub mtr_run ($$$$$$) {
   my $error=      shift;
   my $pid_file=   shift;
 
-  return spawn_impl($path,$arg_list_t,1,$input,$output,$error,$pid_file);
+  return spawn_impl($path,$arg_list_t,'run',$input,$output,$error,$pid_file);
+}
+
+sub mtr_run_test ($$$$$$) {
+  my $path=       shift;
+  my $arg_list_t= shift;
+  my $input=      shift;
+  my $output=     shift;
+  my $error=      shift;
+  my $pid_file=   shift;
+
+  return spawn_impl($path,$arg_list_t,'test',$input,$output,$error,$pid_file);
 }
 
 sub mtr_spawn ($$$$$$) {
@@ -45,7 +61,7 @@ sub mtr_spawn ($$$$$$) {
   my $error=      shift;
   my $pid_file=   shift;
 
-  return spawn_impl($path,$arg_list_t,0,$input,$output,$error,$pid_file);
+  return spawn_impl($path,$arg_list_t,'spawn',$input,$output,$error,$pid_file);
 }
 
 
@@ -58,7 +74,7 @@ sub mtr_spawn ($$$$$$) {
 sub spawn_impl ($$$$$$$) {
   my $path=       shift;
   my $arg_list_t= shift;
-  my $join=       shift;
+  my $mode=       shift;
   my $input=      shift;
   my $output=     shift;
   my $error=      shift;
@@ -71,107 +87,203 @@ sub spawn_impl ($$$$$$$) {
     print STDERR "#### ", "STDIN  $input\n" if $input;
     print STDERR "#### ", "STDOUT $output\n" if $output;
     print STDERR "#### ", "STDERR $error\n" if $error;
-    if ( $join )
-    {
-      print STDERR "#### ", "RUN  ";
-    }
-    else
-    {
-      print STDERR "#### ", "SPAWN ";
-    }
-    print STDERR "$path ", join(" ",@$arg_list_t), "\n";
+    print STDERR "#### ", "$mode : $path ", join(" ",@$arg_list_t), "\n";
     print STDERR "#### ", "-" x 78, "\n";
   }
 
-  my $pid= fork();
-  if ( ! defined $pid )
+ FORK:
   {
-    mtr_error("$path ($pid) can't be forked");
-  }
+    my $pid= fork();
 
-  if ( $pid )
-  {
-    # Parent, i.e. the main script
-    if ( $join )
+    if ( ! defined $pid )
     {
-      # We run a command and wait for the result
-      # FIXME this need to be improved
-      my $res= waitpid($pid,0);
-
-      if ( $res == -1 )
+      if ( $! == $!{EAGAIN} )           # See "perldoc Errno"
       {
-        mtr_error("$path ($pid) got lost somehow");
+        mtr_debug("Got EAGAIN from fork(), sleep 1 second and redo");
+        sleep(1);
+        redo FORK;
       }
-      my $exit_value=  $? >> 8;
-      my $signal_num=  $? & 127;
-      my $dumped_core= $? & 128;
-      if ( $signal_num )
-      {
-        mtr_error("$path ($pid) got signal $signal_num");
-      }
-      if ( $dumped_core )
+      else
       {
-        mtr_error("$path ($pid) dumped core");
+        mtr_error("$path ($pid) can't be forked");
       }
-      return $exit_value;
+    }
+
+    if ( $pid )
+    {
+      spawn_parent_impl($pid,$mode,$path);
     }
     else
     {
-      # We spawned a process we don't wait for
-      return $pid;
+      # Child, redirect output and exec
+      # FIXME I tried POSIX::setsid() here to detach and, I hoped,
+      # avoid zombies. But everything went wild, somehow the parent
+      # became a deamon as well, and was hard to kill ;-)
+      # Need to catch SIGCHLD and do waitpid or something instead......
+
+      $SIG{INT}= 'DEFAULT';         # Parent do some stuff, we don't
+
+      if ( $output )
+      {
+        if ( ! open(STDOUT,">",$output) )
+        {
+          mtr_error("can't redirect STDOUT to \"$output\": $!");
+        }
+      }
+      if ( $error )
+      {
+        if ( $output eq $error )
+        {
+          if ( ! open(STDERR,">&STDOUT") )
+          {
+            mtr_error("can't dup STDOUT: $!");
+          }
+        }
+        else
+        {
+          if ( ! open(STDERR,">",$error) )
+          {
+            mtr_error("can't redirect STDERR to \"$output\": $!");
+          }
+        }
+      }
+      if ( $input )
+      {
+        if ( ! open(STDIN,"<",$input) )
+        {
+          mtr_error("can't redirect STDIN to \"$input\": $!");
+        }
+      }
+      exec($path,@$arg_list_t);
     }
   }
-  else
-  {
-    # Child, redirect output and exec
-    # FIXME I tried POSIX::setsid() here to detach and, I hoped,
-    # avoid zombies. But everything went wild, somehow the parent
-    # became a deamon as well, and was hard to kill ;-)
-    # Need to catch SIGCHLD and do waitpid or something instead......
+}
+
+
+sub spawn_parent_impl {
+  my $pid=  shift;
+  my $mode= shift;
+  my $path= shift;
 
-    $SIG{INT}= 'DEFAULT';         # Parent do some stuff, we don't
+  if ( $mode eq 'run' or $mode eq 'test' )
+  {
+    my $exit_value= -1;
+    my $signal_num=  0;
+    my $dumped_core= 0;
 
-    if ( $output )
+    if ( $mode eq 'run' )
     {
-      if ( ! open(STDOUT,">",$output) )
+      # Simple run of command, we wait for it to return
+      my $ret_pid= waitpid($pid,0);
+
+      if ( $ret_pid <= 0 )
       {
-        mtr_error("can't redirect STDOUT to \"$output\": $!");
+        mtr_error("$path ($pid) got lost somehow");
       }
+
+      $exit_value=  $? >> 8;
+      $signal_num=  $? & 127;
+      $dumped_core= $? & 128;
+
+      return $exit_value;
     }
-    if ( $error )
+    else
     {
-      if ( $output eq $error )
+      # We run mysqltest and wait for it to return. But we try to
+      # catch dying mysqld processes as well.
+      #
+      # We do blocking waitpid() until we get the return from the
+      # "mysqltest" call. But if a mysqld process dies that we
+      # started, we take this as an error, and kill mysqltest.
+      #
+      # FIXME is this as it should be? Can't mysqld terminate
+      # normally from running a test case?
+
+      my $ret_pid;                      # What waitpid() returns
+
+      while ( ($ret_pid= waitpid(-1,0)) != -1 )
       {
-        if ( ! open(STDERR,">&STDOUT") )
+        # Someone terminated, don't know who. Collect
+        # status info first before $? is lost,
+        # but not $exit_value, this is flagged from
+        # 
+
+        if ( $ret_pid == $pid )
         {
-          mtr_error("can't dup STDOUT: $!");
+          # We got termination of mysqltest, we are done
+          $exit_value=  $? >> 8;
+          $signal_num=  $? & 127;
+          $dumped_core= $? & 128;
+          last;
         }
-      }
-      else
-      {
-        if ( ! open(STDERR,">",$error) )
+
+        # If one of the mysqld processes died, we want to
+        # mark this, and kill the mysqltest process.
+
+        foreach my $idx (0..1)
         {
-          mtr_error("can't redirect STDERR to \"$output\": $!");
+          if ( $::master->[$idx]->{'pid'} eq $ret_pid )
+          {
+            mtr_debug("child $ret_pid was master[$idx], " .
+                      "exit during mysqltest run");
+            $::master->[$idx]->{'pid'}= 0;
+            last;
+          }
         }
+
+        foreach my $idx (0..2)
+        {
+          if ( $::slave->[$idx]->{'pid'} eq $ret_pid )
+          {
+            mtr_debug("child $ret_pid was slave[$idx], " .
+                      "exit during mysqltest run");
+            $::slave->[$idx]->{'pid'}= 0;
+            last;
+          }
+        }
+
+        mtr_debug("waitpid() catched exit of unknown child $ret_pid, " .
+                  "exit during mysqltest run");
       }
-    }
-    if ( $input )
-    {
-      if ( ! open(STDIN,"<",$input) )
+
+      if ( $ret_pid != $pid )
       {
-        mtr_error("can't redirect STDIN to \"$input\": $!");
+        # We terminated the waiting because a "mysqld" process died.
+        # Kill the mysqltest process.
+
+        kill(9,$pid);
+
+        $ret_pid= waitpid($pid,0);
+
+        if ( $ret_pid == -1 )
+        {
+          mtr_error("$path ($pid) got lost somehow");
+        }
       }
+
+      return $exit_value;
     }
-    exec($path,@$arg_list_t);
+  }
+  else
+  {
+    # We spawned a process we don't wait for
+    return $pid;
   }
 }
 
+
+
 ##############################################################################
 #
 #  Kill processes left from previous runs
 #
 ##############################################################################
 
+# We just "ping" on the ports, and if we can't do a socket connect
+# we assume the server is dead. So we don't *really* know a server
+# is dead, we just hope that it after letting the listen port go,
+# it is dead enough for us to start a new server.
+
 sub mtr_kill_leftovers () {
 
   # First, kill all masters and slaves that would conflict with
@@ -199,10 +311,23 @@ sub mtr_kill_leftovers () {
                });
   }
 
-  mtr_stop_mysqld_servers(\@args, 1);
+  mtr_mysqladmin_shutdown(\@args);
+
+  # We now have tried to terminate nice. We have waited for the listen
+  # port to be free, but can't really tell if the mysqld process died
+  # or not. We now try to find the process PID from the PID file, and
+  # send a kill to that process. Note that Perl let kill(0,@pids) be
+  # a way to just return the numer of processes the kernel can send
+  # signals to. So this can be used (except on Cygwin) to determine
+  # if there are processes left running that we cound out might exists.
+  #
+  # But still after all this work, all we know is that we have
+  # the ports free.
 
   # We scan the "var/run/" directory for other process id's to kill
-  my $rundir= "$::glob_mysql_test_dir/var/run"; # FIXME $path_run_dir or something
+
+  # FIXME $path_run_dir or something
+  my $rundir= "$::glob_mysql_test_dir/var/run";
 
   if ( -d $rundir )
   {
@@ -218,193 +343,157 @@ sub mtr_kill_leftovers () {
       if ( -f $pidfile )
       {
         my $pid= mtr_get_pid_from_file($pidfile);
-        if ( ! unlink($pidfile) )
+
+        # Race, could have been removed between I tested with -f
+        # and the unlink() below, so I better check again with -f
+
+        if ( ! unlink($pidfile) and -f $pidfile )
         {
           mtr_error("can't remove $pidfile");
         }
-        push(@pids, $pid);
+
+        if ( $::glob_cygwin_perl or kill(0, $pid) )
+        {
+          push(@pids, $pid);            # We know (cygwin guess) it exists
+        }
       }
     }
     closedir(RUNDIR);
 
-    start_reap_all();
-
-    if ( $::glob_cygwin_perl )
+    if ( @pids )
     {
-      # We have no (easy) way of knowing the Cygwin controlling
-      # process, in the PID file we only have the Windows process id.
-      system("kill -f " . join(" ",@pids)); # Hope for the best....
-    }
-    else
-    {
-      my $retries= 10;                    # 10 seconds
-      do
+      if ( $::glob_cygwin_perl )
       {
-        kill(9, @pids);
-      } while ( $retries-- and  kill(0, @pids) );
-
-      if ( kill(0, @pids) )
+        # We have no (easy) way of knowing the Cygwin controlling
+        # process, in the PID file we only have the Windows process id.
+        system("kill -f " . join(" ",@pids)); # Hope for the best....
+        mtr_debug("Sleep 5 seconds waiting for processes to die");
+        sleep(5);
+      }
+      else
       {
-        mtr_error("can't kill processes " . join(" ", @pids));
+        my $retries= 10;                    # 10 seconds
+        do
+        {
+          kill(9, @pids);
+          mtr_debug("Sleep 1 second waiting for processes to die");
+          sleep(1)                      # Wait one second
+        } while ( $retries-- and  kill(0, @pids) );
+
+        if ( kill(0, @pids) )           # Check if some left
+        {
+          # FIXME maybe just mtr_warning() ?
+          mtr_error("can't kill process(es) " . join(" ", @pids));
+        }
       }
     }
+  }
+
+  # We may have failed everything, bug we now check again if we have
+  # the listen ports free to use, and if they are free, just go for it.
 
-    stop_reap_all();
+  foreach my $srv ( @args )
+  {
+    if ( mtr_ping_mysqld_server($srv->{'port'}, $srv->{'sockfile'}) )
+    {
+      mtr_error("can't kill old mysqld holding port $srv->{'port'}");
+    }
   }
 }
 
 ##############################################################################
 #
-#  Shut down mysqld servers
+#  Shut down mysqld servers we have started from this run of this script
 #
 ##############################################################################
 
-# To speed things we kill servers in parallel.
-# The argument is a list of 'pidfiles' and 'socketfiles'.
-# We use the pidfiles and socketfiles to try to terminate the servers.
-# This is not perfect, there could still be other server processes
-# left.
-
-# Force flag is to be set only for killing mysqld servers this script
-# didn't create in this run, i.e. initial cleanup before we start working.
-# If force flag is set, we try to kill all with mysqladmin, and
-# give up if we have no PIDs.
+# To speed things we kill servers in parallel. The argument is a list
+# of 'ports', 'pids', 'pidfiles' and 'socketfiles'.
 
-# FIXME On some operating systems, $srv->{'pid'} and $srv->{'pidfile'}
-#       will not be the same PID. We need to try to kill both I think.
+# FIXME On Cygwin, and maybe some other platforms, $srv->{'pid'} and
+# $srv->{'pidfile'} will not be the same PID. We need to try to kill
+# both I think.
 
-sub mtr_stop_mysqld_servers ($$) {
+sub mtr_stop_mysqld_servers ($) {
   my $spec=  shift;
-  my $force= shift;
 
   # ----------------------------------------------------------------------
-  # If the process was not started from this file, we got no PID,
-  # we try to find it in the PID file.
+  # First try nice normal shutdown using 'mysqladmin'
   # ----------------------------------------------------------------------
 
-  my $any_pid= 0;                     # If we have any PIDs
+  mtr_mysqladmin_shutdown($spec);
+
+  # ----------------------------------------------------------------------
+  # We loop with waitpid() nonblocking to see how many of the ones we
+  # are to kill, actually got killed by mtr_mysqladmin_shutdown().
+  # Note that we don't rely on this, the mysqld server might have stop
+  # listening to the port, but still be alive. But it is a start.
+  # ----------------------------------------------------------------------
 
   foreach my $srv ( @$spec )
   {
-    if ( ! $srv->{'pid'} and -f $srv->{'pidfile'} )
-    {
-      $srv->{'pid'}= mtr_get_pid_from_file($srv->{'pidfile'});
-    }
-    if ( $srv->{'pid'} )
+    if ( $srv->{'pid'} and (waitpid($srv->{'pid'},&WNOHANG) == $srv->{'pid'}) )
     {
-      $any_pid= 1;
+      $srv->{'pid'}= 0;
     }
   }
 
-  # If the processes where started from this script, and we know
-  # no PIDs, then we don't have to do anything.
-
-  if ( ! $any_pid and ! $force )
-  {
-    # cluck "This is how we got here!";
-    return;
-  }
-
   # ----------------------------------------------------------------------
-  # First try nice normal shutdown using 'mysqladmin'
+  # We know the process was started from this file, so there is a PID
+  # saved, or else we have nothing to do.
+  # Might be that is is recorded to be missing, but we failed to
+  # take away the PID file earlier, then we do it now.
   # ----------------------------------------------------------------------
 
-  start_reap_all();                   # Don't require waitpid() of children
+  my %mysqld_pids;
 
   foreach my $srv ( @$spec )
   {
-    if ( -e $srv->{'sockfile'} or $srv->{'port'} )
+    if ( $srv->{'pid'} )
     {
-      # FIXME wrong log.....
-      # FIXME, stderr.....
-      # Shutdown time must be high as slave may be in reconnect
-      my $args;
-
-      mtr_init_args(\$args);
-
-      mtr_add_arg($args, "--no-defaults");
-      mtr_add_arg($args, "--user=%s", $::opt_user);
-      mtr_add_arg($args, "--password=");
-      if ( -e $srv->{'sockfile'} )
-      {
-        mtr_add_arg($args, "--socket=%s", $srv->{'sockfile'});
-      }
-      if ( $srv->{'port'} )
-      {
-        mtr_add_arg($args, "--port=%s", $srv->{'port'});
-      }
-      mtr_add_arg($args, "--connect_timeout=5");
-      mtr_add_arg($args, "--shutdown_timeout=20");
-      mtr_add_arg($args, "--protocol=tcp"); # FIXME new thing, will it help?!
-      mtr_add_arg($args, "shutdown");
-      # We don't wait for termination of mysqladmin
-      mtr_spawn($::exe_mysqladmin, $args,
-                "", $::path_manager_log, $::path_manager_log, "");
+      $mysqld_pids{$srv->{'pid'}}= 1;
     }
-  }
-
-  # Wait for them all to remove their pid and socket file
-
- PIDSOCKFILEREMOVED:
-  for (my $loop= $::opt_sleep_time_for_delete; $loop; $loop--)
-  {
-    my $pidsockfiles_left= 0;
-    foreach my $srv ( @$spec )
+    else
     {
-      if ( -e $srv->{'sockfile'} or -f $srv->{'pidfile'} )
+      # Race, could have been removed between I tested with -f
+      # and the unlink() below, so I better check again with -f
+
+      if ( -f $srv->{'pidfile'} and ! unlink($srv->{'pidfile'}) and
+           -f $srv->{'pidfile'} )
       {
-        $pidsockfiles_left++;          # Could be that pidfile is left
+        mtr_error("can't remove $srv->{'pidfile'}");
       }
     }
-    if ( ! $pidsockfiles_left )
-    {
-      last PIDSOCKFILEREMOVED;
-    }
-    if ( $loop % 20 == 1 )
-    {
-      mtr_warning("Still processes alive after 10 seconds, retrying for $loop seconds...");
-    }
-    mtr_debug("Sleep for 1 second waiting for pid and socket file removal");
-    sleep(1);                          # One second
   }
 
   # ----------------------------------------------------------------------
-  # If no known PIDs, we have nothing more to try
+  # If the processes where started from this script, and we had no PIDS
+  # then we don't have to do anything.
   # ----------------------------------------------------------------------
 
-  if ( ! $any_pid )
+  if ( ! keys %mysqld_pids )
   {
-    stop_reap_all();
+    # cluck "This is how we got here!";
     return;
   }
 
   # ----------------------------------------------------------------------
-  # We may have killed all that left a socket, but we are not sure we got
-  # them all killed. If we suspect it lives, try nice kill with SIG_TERM.
-  # Note that for true Win32 processes, kill(0,$pid) will not return 1.
+  # In mtr_mysqladmin_shutdown() we only waited for the mysqld servers
+  # not to listen to the port. But we are not sure we got them all
+  # killed. If we suspect it lives, try nice kill with SIG_TERM. Note
+  # that for true Win32 processes, kill(0,$pid) will not return 1.
   # ----------------------------------------------------------------------
 
  SIGNAL:
   foreach my $sig (15,9)
   {
-    my $process_left= 0;
-    foreach my $srv ( @$spec )
+    my $retries= 10;                    # 10 seconds
+    kill($sig, keys %mysqld_pids);
+    while ( $retries-- and  kill(0, keys %mysqld_pids) )
     {
-      if ( $srv->{'pid'} and
-           ( -f $srv->{'pidfile'} or kill(0,$srv->{'pid'}) ) )
-      {
-        $process_left++;
-        mtr_warning("process $srv->{'pid'} not cooperating, " .
-                    "will send signal $sig to process");
-        kill($sig,$srv->{'pid'});       # SIG_TERM
-      }
-      if ( ! $process_left )
-      {
-        last SIGNAL;
-      }
+      mtr_debug("Sleep 1 second waiting for processes to die");
+      sleep(1)                      # Wait one second
     }
-    mtr_debug("Sleep for 5 seconds waiting for processes to die");
-    sleep(5);                           # We wait longer than usual
   }
 
   # ----------------------------------------------------------------------
@@ -437,8 +526,8 @@ sub mtr_stop_mysqld_servers ($$) {
 
           foreach my $file ($srv->{'pidfile'}, $srv->{'sockfile'})
           {
-            unlink($file);
-            if ( -e $file )
+            # Know it is dead so should be no race, careful anyway
+            if ( -f $file and ! unlink($file) and -f $file )
             {
               $errors++;
               mtr_warning("couldn't delete $file");
@@ -454,9 +543,147 @@ sub mtr_stop_mysqld_servers ($$) {
     }
   }
 
-  stop_reap_all();
+  # FIXME We just assume they are all dead, for Cygwin we are not
+  # really sure
+    
+}
+
+
+##############################################################################
+#
+#  Shut down mysqld servers using "mysqladmin ... shutdown".
+#  To speed this up, we start them in parallel and use waitpid() to
+#  catch their termination. Note that this doesn't say the servers
+#  are terminated, just that 'mysqladmin' is terminated.
+#
+#  Note that mysqladmin will ask the server about what PID file it uses,
+#  and mysqladmin will wait for it to be removed before it terminates
+#  (unless passes timeout).
+#
+#  This function will take at most about 20 seconds, and we still are not
+#  sure we killed them all. If none is responding to ping, we return 1,
+#  else we return 0.
+#
+##############################################################################
+
+sub mtr_mysqladmin_shutdown () {
+  my $spec= shift;
+
+  my @mysql_admin_pids;
+  my @to_kill_specs;
+
+  foreach my $srv ( @$spec )
+  {
+    if ( mtr_ping_mysqld_server($srv->{'port'}, $srv->{'sockfile'}) )
+    {
+      push(@to_kill_specs, $srv);
+    }
+  }
+
+
+  foreach my $srv ( @to_kill_specs )
+  {
+    # FIXME wrong log.....
+    # FIXME, stderr.....
+    # Shutdown time must be high as slave may be in reconnect
+    my $args;
+
+    mtr_init_args(\$args);
+
+    mtr_add_arg($args, "--no-defaults");
+    mtr_add_arg($args, "--user=%s", $::opt_user);
+    mtr_add_arg($args, "--password=");
+    if ( -e $srv->{'sockfile'} )
+    {
+      mtr_add_arg($args, "--socket=%s", $srv->{'sockfile'});
+    }
+    if ( $srv->{'port'} )
+    {
+      mtr_add_arg($args, "--port=%s", $srv->{'port'});
+    }
+    if ( $srv->{'port'} and ! -e $srv->{'sockfile'} )
+    {
+      mtr_add_arg($args, "--protocol=tcp"); # Needed if no --socket
+    }
+    mtr_add_arg($args, "--connect_timeout=5");
+    mtr_add_arg($args, "--shutdown_timeout=20");
+    mtr_add_arg($args, "shutdown");
+    # We don't wait for termination of mysqladmin
+    my $pid= mtr_spawn($::exe_mysqladmin, $args,
+                       "", $::path_manager_log, $::path_manager_log, "");
+    push(@mysql_admin_pids, $pid);
+  }
+
+  # We wait blocking, we wait for the last one anyway
+  foreach my $pid (@mysql_admin_pids)
+  {
+    waitpid($pid,0);                    # FIXME no need to check -1 or 0?
+  }
 
-  # FIXME We just assume they are all dead, we don't know....
+  # If we trusted "mysqladmin --shutdown_timeout= ..." we could just
+  # terminate now, but we don't (FIXME should be debugged).
+  # So we try again to ping and at least wait the same amount of time
+  # mysqladmin would for all to die.
+
+  my $timeout= 20;                      # 20 seconds max
+  my $res= 1;                           # If we just fall through, we are done
+ 
+ TIME:
+  while ( $timeout-- )
+  {
+    foreach my $srv ( @to_kill_specs )
+    {
+      $res= 1;                          # We are optimistic
+      if ( mtr_ping_mysqld_server($srv->{'port'}, $srv->{'sockfile'}) )
+      {
+        mtr_debug("Sleep 1 second waiting for processes to stop using port");
+        sleep(1);                       # One second
+        $res= 0;
+        next TIME;
+      }
+    }
+    last;                               # If we got here, we are done
+  }
+
+  return $res;
+}
+
+##############################################################################
+#
+#  The operating system will keep information about dead children, 
+#  we read this information here, and if we have records the process
+#  is alive, we mark it as dead.
+#
+##############################################################################
+
+sub mtr_record_dead_children () {
+
+  my $ret_pid;
+
+  # FIXME the man page says to wait for -1 to terminate,
+  # but on OS X we get '0' all the time...
+  while ( ($ret_pid= waitpid(-1,&WNOHANG)) > 0 )
+  {
+    mtr_debug("waitpid() catched exit of child $ret_pid");
+    foreach my $idx (0..1)
+    {
+      if ( $::master->[$idx]->{'pid'} eq $ret_pid )
+      {
+        mtr_debug("child $ret_pid was master[$idx]");
+        $::master->[$idx]->{'pid'}= 0;
+      }
+    }
+
+    foreach my $idx (0..2)
+    {
+      if ( $::slave->[$idx]->{'pid'} eq $ret_pid )
+      {
+        mtr_debug("child $ret_pid was slave[$idx]");
+        $::slave->[$idx]->{'pid'}= 0;
+        last;
+      }
+    }
+  }
 }
 
 sub start_reap_all {
@@ -467,6 +694,32 @@ sub stop_reap_all {
   $SIG{CHLD}= 'DEFAULT';
 }
 
+sub mtr_ping_mysqld_server () {
+  my $port= shift;
+
+  my $remote= "localhost";
+  my $iaddr=  inet_aton($remote);
+  if ( ! $iaddr )
+  {
+    mtr_error("can't find IP number for $remote");
+  }
+  my $paddr=  sockaddr_in($port, $iaddr);
+  my $proto=  getprotobyname('tcp');
+  if ( ! socket(SOCK, PF_INET, SOCK_STREAM, $proto) )
+  {
+    mtr_error("can't create socket: $!");
+  }
+  if ( connect(SOCK, $paddr) )
+  {
+    close(SOCK);                        # FIXME check error?
+    return 1;
+  }
+  else
+  {
+    return 0;
+  }
+}
+
 ##############################################################################
 #
 #  Wait for a file to be created
@@ -474,33 +727,38 @@ sub stop_reap_all {
 ##############################################################################
 
 
-sub sleep_until_file_created ($$) {
+sub sleep_until_file_created ($$$) {
   my $pidfile= shift;
   my $timeout= shift;
+  my $pid=     shift;
 
-  my $loop=  $timeout;
-  while ( $loop-- )
+  for ( my $loop= 1; $loop <= $timeout; $loop++ )
   {
     if ( -r $pidfile )
     {
-      return;
+      return 1;
     }
-    mtr_debug("Sleep for 1 second waiting for creation of $pidfile");
 
-    if ( $loop % 20 == 1 )
+    # Check if it died after the fork() was successful 
+    if ( waitpid($pid,&WNOHANG) == $pid )
     {
-      mtr_warning("Waiting for $pidfile to be created, still trying for $loop seconds...");
+      return 0;
+    }
+
+    mtr_debug("Sleep 1 second waiting for creation of $pidfile");
+
+    if ( $loop % 60 == 0 )
+    {
+      my $left= $timeout - $loop;
+      mtr_warning("Waited $loop seconds for $pidfile to be created, " .
+                  "still waiting for $left seconds...");
     }
 
     sleep(1);
   }
 
-  if ( ! -r $pidfile )
-  {
-    mtr_error("No $pidfile was created");
-  }
+  return 0;
 }
 
 
-
 1;

mysql-test/lib/mtr_report.pl

@@ -10,6 +10,7 @@ sub mtr_report_test_name($);
 sub mtr_report_test_passed($);
 sub mtr_report_test_failed($);
 sub mtr_report_test_skipped($);
+sub mtr_report_test_disabled($);
 
 sub mtr_show_failed_diff ($);
 sub mtr_report_stats ($);
@@ -72,7 +73,14 @@ sub mtr_report_test_skipped ($) {
   my $tinfo= shift;
 
   $tinfo->{'result'}= 'MTR_RES_SKIPPED';
-  print "[ skipped ]\n";
+  if ( $tinfo->{'disable'} )
+  {
+    print "[ disabled ]  $tinfo->{'comment'}\n";
+  }
+  else
+  {
+    print "[ skipped ]\n";
+  }
 }
 
 sub mtr_report_test_passed ($) {
@@ -95,9 +103,18 @@ sub mtr_report_test_failed ($) {
   $tinfo->{'result'}= 'MTR_RES_FAILED';
   print "[ fail ]\n";
 
-  print "Errors are (from $::path_timefile) :\n";
-  print mtr_fromfile($::path_timefile); # FIXME print_file() instead
-  print "\n(the last lines may be the most important ones)\n";
+  # FIXME Instead of this test, and meaningless error message in 'else'
+  # we should write out into $::path_timefile when the error occurs.
+  if ( -f $::path_timefile )
+  {
+    print "Errors are (from $::path_timefile) :\n";
+    print mtr_fromfile($::path_timefile); # FIXME print_file() instead
+    print "\n(the last lines may be the most important ones)\n";
+  }
+  else
+  {
+    print "Unexpected termination, probably when starting mysqld\n";
+  }
 }
 
 sub mtr_report_stats ($) {

mysql-test/mysql-test-run.pl

@@ -84,10 +84,11 @@ use Sys::Hostname;
 #use Carp;
 use IO::Socket;
 use IO::Socket::INET;
-use Data::Dumper;
+#use Data::Dumper;
 use strict;
 #use diagnostics;
 
+require "lib/mtr_cases.pl";
 require "lib/mtr_process.pl";
 require "lib/mtr_io.pl";
 require "lib/mtr_gcov.pl";
@@ -165,14 +166,12 @@ our $glob_user=                   'test';
 our $glob_use_embedded_server=    0;
 
 our $glob_basedir;
-our $glob_do_test;
 
 # The total result
 
 our $path_charsetsdir;
 our $path_client_bindir;
 our $path_language;
-our $path_tests_bindir;
 our $path_timefile;
 our $path_manager_log;           # Used by mysqldadmin
 our $path_slave_load_tmpdir;     # What is this?!
@@ -192,8 +191,10 @@ our $exe_master_mysqld;
 our $exe_mysql;
 our $exe_mysqladmin;
 our $exe_mysqlbinlog;
+our $exe_mysql_client_test;
 our $exe_mysqld;
 our $exe_mysqldump;              # Called from test case
+our $exe_mysql_fix_system_tables;
 our $exe_mysqltest;
 our $exe_slave_mysqld;
 
@@ -208,6 +209,7 @@ our $opt_current_test;
 our $opt_ddd;
 our $opt_debug;
 our $opt_do_test;
+our @opt_cases;                  # The test cases names in argv
 our $opt_embedded_server;
 our $opt_extern;
 our $opt_fast;
@@ -232,8 +234,6 @@ our $opt_local_master;
 our $master;                    # Will be struct in C
 our $slave;
 
-our $opt_master_myport;
-our $opt_slave_myport;
 our $opt_ndbcluster_port;
 our $opt_ndbconnectstring;
 
@@ -297,8 +297,6 @@ sub command_line_setup ();
 sub executable_setup ();
 sub environment_setup ();
 sub kill_and_cleanup ();
-sub collect_test_cases ($);
-sub sleep_until_file_created ($$);
 sub ndbcluster_start ();
 sub ndbcluster_stop ();
 sub run_benchmarks ($);
@@ -306,6 +304,7 @@ sub run_tests ();
 sub mysql_install_db ();
 sub install_db ($$);
 sub run_testcase ($);
+sub report_failure_and_restart ($);
 sub do_before_start_master ($$);
 sub do_before_start_slave ($$);
 sub mysqld_start ($$$$);
@@ -358,7 +357,15 @@ sub main () {
 
   if ( $opt_start_and_exit )
   {
-    mtr_report("Servers started, exiting");
+    # FIXME what about ndb?
+    if ( mysqld_start('master',0,[],[]) )
+    {
+      mtr_report("Servers started, exiting");
+    }
+    else
+    {
+      mtr_error("Can't start the mysqld server");
+    }
   }
   else
   {
@@ -447,8 +454,8 @@ sub command_line_setup () {
   $path_manager_log= "$glob_mysql_test_dir/var/log/manager.log";
   $opt_current_test= "$glob_mysql_test_dir/var/log/current_test";
 
-  $opt_master_myport=   9306;
-  $opt_slave_myport=    9308;
+  my $opt_master_myport=   9306;
+  my $opt_slave_myport=    9308;
   $opt_ndbcluster_port= 9350;
 
   # Read the command line
@@ -532,6 +539,8 @@ sub command_line_setup () {
     usage("");
   }
 
+  @opt_cases= @ARGV;
+
   # Put this into a hash, will be a C struct
 
   $master->[0]->{'path_myddir'}=  "$glob_mysql_test_dir/var/master-data";
@@ -598,7 +607,7 @@ sub command_line_setup () {
   # Look at the command line options and set script flags
   # --------------------------------------------------------------------------
 
-  if ( $opt_record and ! @ARGV)
+  if ( $opt_record and ! @opt_cases )
   {
     mtr_error("Will not run in record mode without a specific test case");
   }
@@ -733,7 +742,8 @@ sub executable_setup () {
       {
         mtr_error("Can't find embedded server 'mysqltest'");
       }
-      $path_tests_bindir= "$glob_basedir/libmysqld/examples";
+      $exe_mysql_client_test=
+        "$glob_basedir/libmysqld/examples/mysql_client_test_embedded";
     }
     else
     {
@@ -749,7 +759,8 @@ sub executable_setup () {
       {
         $exe_mysqltest=  "$glob_basedir/client/mysqltest";
       }
-      $path_tests_bindir= "$glob_basedir/tests";
+      $exe_mysql_client_test=
+        "$glob_basedir/tests/mysql_client_test";
     }
     if ( -f "$glob_basedir/client/.libs/mysqldump" )
     {
@@ -768,22 +779,26 @@ sub executable_setup () {
       $exe_mysqlbinlog=   "$glob_basedir/client/mysqlbinlog";
     }
 
-    $exe_mysqld= "$glob_basedir/sql/mysqld";
-    $path_client_bindir= "$glob_basedir/client";
-    $exe_mysqladmin=    "$path_client_bindir/mysqladmin";
-    $exe_mysql=         "$path_client_bindir/mysql";
-    $path_language=      "$glob_basedir/sql/share/english/";
-    $path_charsetsdir=   "$glob_basedir/sql/share/charsets";
+    $path_client_bindir=  "$glob_basedir/client";
+    $exe_mysqld=          "$glob_basedir/sql/mysqld";
+    $exe_mysqladmin=      "$path_client_bindir/mysqladmin";
+    $exe_mysql=           "$path_client_bindir/mysql";
+    $exe_mysql_fix_system_tables= "$glob_basedir/scripts/mysql_fix_privilege_tables";
+    $path_language=       "$glob_basedir/sql/share/english/";
+    $path_charsetsdir=    "$glob_basedir/sql/share/charsets";
   }
   else
   {
-    $path_client_bindir= "$glob_basedir/bin";
-    $path_tests_bindir=  "$glob_basedir/tests";
-    $exe_mysqltest=     "$path_client_bindir/mysqltest";
-    $exe_mysqldump=     "$path_client_bindir/mysqldump";
-    $exe_mysqlbinlog=  "$path_client_bindir/mysqlbinlog";
-    $exe_mysqladmin=    "$path_client_bindir/mysqladmin";
-    $exe_mysql=         "$path_client_bindir/mysql";
+    my $path_tests_bindir=  "$glob_basedir/tests";
+
+    $path_client_bindir=    "$glob_basedir/bin";
+    $exe_mysqltest=         "$path_client_bindir/mysqltest";
+    $exe_mysqldump=         "$path_client_bindir/mysqldump";
+    $exe_mysqlbinlog=       "$path_client_bindir/mysqlbinlog";
+    $exe_mysqladmin=        "$path_client_bindir/mysqladmin";
+    $exe_mysql=             "$path_client_bindir/mysql";
+    $exe_mysql_fix_system_tables= "$path_client_bindir/scripts/mysql_fix_privilege_tables";
+
     if ( -d "$glob_basedir/share/mysql/english" )
     {
       $path_language    ="$glob_basedir/share/mysql/english/";
@@ -804,6 +819,33 @@ sub executable_setup () {
       $exe_mysqld= "$glob_basedir/bin/mysqld";
     }
 
+    if ( $glob_use_embedded_server )
+    {
+      if ( -f "$path_client_bindir/mysqltest_embedded" )
+      {
+        # FIXME valgrind?
+        $exe_mysqltest="$path_client_bindir/mysqltest_embedded";
+      }
+      else
+      {
+        error("Cannot find embedded server 'mysqltest_embedded'");
+      }
+      if ( -d "$path_tests_bindir/mysql_client_test_embedded" )
+      {
+        $exe_mysql_client_test=
+          "$path_tests_bindir/mysql_client_test_embedded";
+      }
+      else
+      {
+        $exe_mysql_client_test=
+          "$path_client_bindir/mysql_client_test_embedded";
+      }
+    }
+    else
+    {
+      $exe_mysqltest="$path_client_bindir/mysqltest";
+      $exe_mysql_client_test="$path_client_bindir/mysql_client_test";
+    }
   }
 
   # FIXME special $exe_master_mysqld and $exe_slave_mysqld
@@ -846,13 +888,18 @@ sub environment_setup () {
   # Also command lines in .opt files may contain env vars
   # --------------------------------------------------------------------------
 
-  $ENV{'LC_COLLATE'}=     "C";
-  $ENV{'MYSQL_TEST_DIR'}= $glob_mysql_test_dir;
-  $ENV{'MASTER_MYPORT'}=  $opt_master_myport;
-  $ENV{'SLAVE_MYPORT'}=   $opt_slave_myport;
-# $ENV{'MYSQL_TCP_PORT'}= '@MYSQL_TCP_PORT@'; # FIXME
-  $ENV{'MYSQL_TCP_PORT'}= 3306;
-  $ENV{'MASTER_MYSOCK'}=  $master->[0]->{'path_mysock'};
+  $ENV{'UMASK'}=              "0660"; # The octal *string*
+  $ENV{'UMASK_DIR'}=          "0770"; # The octal *string*
+  $ENV{'LC_COLLATE'}=         "C";
+  $ENV{'USE_RUNNING_SERVER'}= $glob_use_running_server;
+  $ENV{'MYSQL_TEST_DIR'}=     $glob_mysql_test_dir;
+  $ENV{'MASTER_MYSOCK'}=      $master->[0]->{'path_mysock'};
+  $ENV{'MASTER_MYSOCK1'}=     $master->[1]->{'path_mysock'};
+  $ENV{'MASTER_MYPORT'}=      $master->[0]->{'path_myport'};
+  $ENV{'MASTER_MYPORT1'}=     $master->[1]->{'path_myport'};
+  $ENV{'SLAVE_MYPORT'}=       $slave->[0]->{'path_myport'};
+# $ENV{'MYSQL_TCP_PORT'}=     '@MYSQL_TCP_PORT@'; # FIXME
+  $ENV{'MYSQL_TCP_PORT'}=     3306;
 }
 
 
@@ -875,203 +922,6 @@ sub handle_int_signal () {
 }
 
 
-##############################################################################
-#
-#  Collect information about test cases we are to run
-#
-##############################################################################
-
-sub collect_test_cases ($) {
-  my $suite= shift;             # Test suite name
-
-  my $testdir;
-  my $resdir;
-
-  if ( $suite eq "main" )
-  {
-    $testdir= "$glob_mysql_test_dir/t";
-    $resdir=  "$glob_mysql_test_dir/r";
-  }
-  else
-  {
-    $testdir= "$glob_mysql_test_dir/suite/$suite/t";
-    $resdir=  "$glob_mysql_test_dir/suite/$suite/r";
-  }
-
-  my @tests;               # Array of hash, will be array of C struct
-
-  opendir(TESTDIR, $testdir) or mtr_error("Can't open dir \"$testdir\": $!");
-
-  foreach my $elem ( sort readdir(TESTDIR) ) {
-    my $tname= mtr_match_extension($elem,"test");
-    next if ! defined $tname;
-    next if $opt_do_test and ! defined mtr_match_prefix($elem,$opt_do_test);
-    my $path= "$testdir/$elem";
-
-    # ----------------------------------------------------------------------
-    # Skip some tests silently
-    # ----------------------------------------------------------------------
-
-    if ( $opt_start_from and $tname lt $opt_start_from )
-    {
-      next;
-    }
-
-    # ----------------------------------------------------------------------
-    # Skip some tests but include in list, just mark them to skip
-    # ----------------------------------------------------------------------
-
-    my $tinfo= {};
-    $tinfo->{'name'}= $tname;
-    $tinfo->{'result_file'}= "$resdir/$tname.result";
-    push(@tests, $tinfo);
-
-    if ( $opt_skip_test and defined mtr_match_prefix($tname,$opt_skip_test) )
-    {
-      $tinfo->{'skip'}= 1;
-      next;
-    }
-
-    # FIXME temporary solution, we have a hard coded list of test cases to
-    # skip if we are using the embedded server
-
-    if ( $glob_use_embedded_server and
-         mtr_match_any_exact($tname,\@skip_if_embedded_server) )
-    {
-      $tinfo->{'skip'}= 1;
-      next;
-    }
-
-    # ----------------------------------------------------------------------
-    # Collect information about test case
-    # ----------------------------------------------------------------------
-
-    $tinfo->{'path'}= $path;
-    $tinfo->{'timezone'}= "GMT-3"; # for UNIX_TIMESTAMP tests to work
-
-    if ( defined mtr_match_prefix($tname,"rpl") )
-    {
-      if ( $opt_skip_rpl )
-      {
-        $tinfo->{'skip'}= 1;
-        next;
-      }
-
-      # FIXME currently we always restart slaves
-      $tinfo->{'slave_restart'}= 1;
-
-      if ( $tname eq 'rpl_failsafe' or $tname eq 'rpl_chain_temp_table' )
-      {
-        $tinfo->{'slave_num'}= 3;
-      }
-      else
-      {
-        $tinfo->{'slave_num'}= 1;
-      }
-    }
-
-    # FIXME what about embedded_server + ndbcluster, skip ?!
-
-    my $master_opt_file= "$testdir/$tname-master.opt";
-    my $slave_opt_file=  "$testdir/$tname-slave.opt";
-    my $slave_mi_file=   "$testdir/$tname.slave-mi";
-    my $master_sh=       "$testdir/$tname-master.sh";
-    my $slave_sh=        "$testdir/$tname-slave.sh";
-
-    if ( -f $master_opt_file )
-    {
-      $tinfo->{'master_restart'}= 1;    # We think so for now
-      # This is a dirty hack from old mysql-test-run, we use the opt file
-      # to flag other things as well, it is not a opt list at all
-      my $extra_master_opt= mtr_get_opts_from_file($master_opt_file);
-
-      foreach my $opt (@$extra_master_opt)
-      {
-        my $value;
-
-        $value= mtr_match_prefix($opt, "--timezone=");
-
-        if ( defined $value )
-        {
-          $tinfo->{'timezone'}= $value;
-          $extra_master_opt= [];
-          $tinfo->{'master_restart'}= 0;
-          last;
-        }
-
-        $value= mtr_match_prefix($opt, "--result-file=");
-
-        if ( defined $value )
-        {
-          $tinfo->{'result_file'}= "r/$value.result";
-          if ( $opt_result_ext and $opt_record or
-               -f "$tinfo->{'result_file'}$opt_result_ext")
-          {
-            $tinfo->{'result_file'}.= $opt_result_ext;
-          }
-          $extra_master_opt= [];
-          $tinfo->{'master_restart'}= 0;
-          last;
-        }
-      }
-
-      $tinfo->{'master_opt'}= $extra_master_opt;
-    }
-
-    if ( -f $slave_opt_file )
-    {
-      $tinfo->{'slave_opt'}= mtr_get_opts_from_file($slave_opt_file);
-      $tinfo->{'slave_restart'}= 1;
-    }
-
-    if ( -f $slave_mi_file )
-    {
-      $tinfo->{'slave_mi'}= mtr_get_opts_from_file($slave_mi_file);
-      $tinfo->{'slave_restart'}= 1;
-    }
-
-    if ( -f $master_sh )
-    {
-      if ( $glob_win32_perl )
-      {
-        $tinfo->{'skip'}= 1;
-      }
-      else
-      {
-        $tinfo->{'master_sh'}= $master_sh;
-        $tinfo->{'master_restart'}= 1;
-      }
-    }
-
-    if ( -f $slave_sh )
-    {
-      if ( $glob_win32_perl )
-      {
-        $tinfo->{'skip'}= 1;
-      }
-      else
-      {
-        $tinfo->{'slave_sh'}= $slave_sh;
-        $tinfo->{'slave_restart'}= 1;
-      }
-    }
-
-    # We can't restart a running server that may be in use
-
-    if ( $glob_use_running_server and
-         ( $tinfo->{'master_restart'} or $tinfo->{'slave_restart'} ) )
-    {
-      $tinfo->{'skip'}= 1;
-    }
-
-  }
-
-  closedir TESTDIR;
-
-  return \@tests;
-}
-
-
 ##############################################################################
 #
 #  Handle left overs from previous runs
@@ -1189,6 +1039,10 @@ sub run_benchmarks ($) {
   if ( ! $glob_use_embedded_server and ! $opt_local_master )
   {
     $master->[0]->{'pid'}= mysqld_start('master',0,[],[]);
+    if ( ! $master->[0]->{'pid'} )
+    {
+      mtr_error("Can't start the mysqld server");
+    }
   }
 
   mtr_init_args(\$args);
@@ -1254,7 +1108,7 @@ sub run_suite () {
 
   mtr_print_thick_line();
 
-  mtr_report("Finding Tests in the '$suite' suite");
+  mtr_report("Finding  Tests in the '$suite' suite");
 
   my $tests= collect_test_cases($suite);
 
@@ -1301,10 +1155,12 @@ sub run_suite () {
 
 sub mysql_install_db () {
 
-  mtr_report("Installing Test Databases");
-
+  # FIXME not exactly true I think, needs improvements
   install_db('master', $master->[0]->{'path_myddir'});
+  install_db('master', $master->[1]->{'path_myddir'});
   install_db('slave',  $slave->[0]->{'path_myddir'});
+  install_db('slave',  $slave->[1]->{'path_myddir'});
+  install_db('slave',  $slave->[2]->{'path_myddir'});
 
   return 0;
 }
@@ -1422,6 +1278,12 @@ sub run_testcase ($) {
   mtr_tofile($master->[0]->{'path_myerr'},"CURRENT_TEST: $tname\n");
   do_before_start_master($tname,$tinfo->{'master_sh'});
 
+  # ----------------------------------------------------------------------
+  # If any mysqld servers running died, we have to know
+  # ----------------------------------------------------------------------
+
+  mtr_record_dead_children();
+
   # ----------------------------------------------------------------------
   # Start masters
   # ----------------------------------------------------------------------
@@ -1439,14 +1301,24 @@ sub run_testcase ($) {
       {
         $master->[0]->{'pid'}=
           mysqld_start('master',0,$tinfo->{'master_opt'},[]);
+        if ( ! $master->[0]->{'pid'} )
+        {
+          report_failure_and_restart($tinfo);
+          return;
+        }
       }
       if ( $opt_with_ndbcluster and ! $master->[1]->{'pid'} )
       {
         $master->[1]->{'pid'}=
           mysqld_start('master',1,$tinfo->{'master_opt'},[]);
+        if ( ! $master->[1]->{'pid'} )
+        {
+          report_failure_and_restart($tinfo);
+          return;
+        }
       }
 
-      if ( $tinfo->{'master_opt'} )
+      if ( @{$tinfo->{'master_opt'}} )
       {
         $master->[0]->{'uses_special_flags'}= 1;
       }
@@ -1469,6 +1341,11 @@ sub run_testcase ($) {
           $slave->[$idx]->{'pid'}=
             mysqld_start('slave',$idx,
                          $tinfo->{'slave_opt'}, $tinfo->{'slave_mi'});
+          if ( ! $slave->[$idx]->{'pid'} )
+          {
+            report_failure_and_restart($tinfo);
+            return;
+          }
         }
       }
     }
@@ -1502,30 +1379,37 @@ sub run_testcase ($) {
                    "mysqltest returned unexpected code $res, " .
                    "it has probably crashed");
       }
-      mtr_report_test_failed($tinfo);
-      mtr_show_failed_diff($tname);
-      print "\n";
-      if ( ! $opt_force )
-      {
-        print "Aborting: $tname failed. To continue, re-run with '--force'.";
-        print "\n";
-        if ( ! $opt_gdb and ! $glob_use_running_server and
-             ! $opt_ddd and ! $glob_use_embedded_server )
-        {
-          stop_masters_slaves();
-        }
-        exit(1);
-      }
+      report_failure_and_restart($tinfo);
+    }
+  }
+}
 
-      # FIXME always terminate on failure?!
-      if ( ! $opt_gdb and ! $glob_use_running_server and
-           ! $opt_ddd and ! $glob_use_embedded_server )
-      {
-        stop_masters_slaves();
-      }
-      print "Resuming Tests\n\n";
+
+sub report_failure_and_restart ($) {
+  my $tinfo= shift;
+
+  mtr_report_test_failed($tinfo);
+  mtr_show_failed_diff($tinfo->{'name'});
+  print "\n";
+  if ( ! $opt_force )
+  {
+    print "Aborting: $tinfo->{'name'} failed. To continue, re-run with '--force'.";
+    print "\n";
+    if ( ! $opt_gdb and ! $glob_use_running_server and
+         ! $opt_ddd and ! $glob_use_embedded_server )
+    {
+      stop_masters_slaves();
     }
+    exit(1);
+  }
+
+  # FIXME always terminate on failure?!
+  if ( ! $opt_gdb and ! $glob_use_running_server and
+       ! $opt_ddd and ! $glob_use_embedded_server )
+  {
+    stop_masters_slaves();
   }
+  print "Resuming Tests\n\n";
 }
 
 
@@ -1603,11 +1487,13 @@ sub do_before_start_slave ($$) {
 }
 
 sub mysqld_arguments ($$$$$) {
-  my $args=       shift;
-  my $type=       shift;        # master/slave/bootstrap
-  my $idx=        shift;
-  my $extra_opt=  shift;
-  my $slave_master_info=  shift;
+  my $args=              shift;
+  my $type=              shift;        # master/slave/bootstrap
+  my $idx=               shift;
+  my $extra_opt=         shift;
+  my $slave_master_info= shift;
+
+#  print STDERR Dumper($extra_opt);
 
   my $sidx= "";                 # Index as string, 0 is empty string
   if ( $idx > 0 )
@@ -1835,10 +1721,10 @@ sub mysqld_arguments ($$$$$) {
 ##############################################################################
 
 sub mysqld_start ($$$$) {
-  my $type=       shift;        # master/slave/bootstrap
-  my $idx=        shift;
-  my $extra_opt=  shift;
-  my $slave_master_info=  shift;
+  my $type=              shift;        # master/slave/bootstrap
+  my $idx=               shift;
+  my $extra_opt=         shift;
+  my $slave_master_info= shift;
 
   my $args;                             # Arg vector
   my $exe;
@@ -1893,9 +1779,8 @@ sub mysqld_start ($$$$) {
                          $master->[$idx]->{'path_myerr'},
                          $master->[$idx]->{'path_myerr'}, "") )
     {
-      sleep_until_file_created($master->[$idx]->{'path_mypid'},
-                               $master->[$idx]->{'start_timeout'});
-      return $pid;
+      return sleep_until_file_created($master->[$idx]->{'path_mypid'},
+                                      $master->[$idx]->{'start_timeout'}, $pid);
     }
   }
 
@@ -1905,13 +1790,12 @@ sub mysqld_start ($$$$) {
                          $slave->[$idx]->{'path_myerr'},
                          $slave->[$idx]->{'path_myerr'}, "") )
     {
-      sleep_until_file_created($slave->[$idx]->{'path_mypid'},
-                               $master->[$idx]->{'start_timeout'});
-      return $pid;
+      return sleep_until_file_created($slave->[$idx]->{'path_mypid'},
+                                      $master->[$idx]->{'start_timeout'}, $pid);
     }
   }
 
-  mtr_error("Can't start mysqld FIXME");
+  return 0;
 }
 
 sub stop_masters_slaves () {
@@ -1944,7 +1828,7 @@ sub stop_masters () {
     }
   }
 
-  mtr_stop_mysqld_servers(\@args, 0);
+  mtr_stop_mysqld_servers(\@args);
 }
 
 sub stop_slaves () {
@@ -1966,7 +1850,7 @@ sub stop_slaves () {
     }
   }
 
-  mtr_stop_mysqld_servers(\@args, 0);
+  mtr_stop_mysqld_servers(\@args);
 }
 
 
@@ -1992,17 +1876,32 @@ sub run_mysqltest ($$) {
   }
 
   my $cmdline_mysql=
-    "$exe_mysql --host=localhost --port=$master->[0]->{'path_myport'} " .
-    "--socket=$master->[0]->{'path_mysock'} --user=root --password=";
+    "$exe_mysql --host=localhost  --user=root --password= " .
+    "--port=$master->[0]->{'path_myport'} " .
+    "--socket=$master->[0]->{'path_mysock'}";
+
+  my $cmdline_mysql_client_test=
+    "$exe_mysql_client_test --no-defaults --testcase --user=root --silent " .
+    "--port=$master->[0]->{'path_myport'} " .
+    "--socket=$master->[0]->{'path_mysock'}";
+
+  my $cmdline_mysql_fix_system_tables=
+    "$exe_mysql_fix_system_tables --no-defaults --host=localhost --user=root --password= " .
+    "--basedir=$glob_basedir --bindir=$path_client_bindir --verbose " .
+    "--port=$master->[0]->{'path_myport'} " .
+    "--socket=$master->[0]->{'path_mysock'}";
+
+
 
   # FIXME really needing a PATH???
   # $ENV{'PATH'}= "/bin:/usr/bin:/usr/local/bin:/usr/bsd:/usr/X11R6/bin:/usr/openwin/bin:/usr/bin/X11:$ENV{'PATH'}";
 
-  $ENV{'MYSQL'}=                    $exe_mysql;
+  $ENV{'MYSQL'}=                    $cmdline_mysql;
   $ENV{'MYSQL_DUMP'}=               $cmdline_mysqldump;
-  $ENV{'MYSQL_BINLOG'}=             $exe_mysqlbinlog;
-  $ENV{'CLIENT_BINDIR'}=            $path_client_bindir;
-  $ENV{'TESTS_BINDIR'}=             $path_tests_bindir;
+  $ENV{'MYSQL_BINLOG'}=             $cmdline_mysqlbinlog;
+  $ENV{'MYSQL_FIX_SYSTEM_TABLES'}=  $cmdline_mysql_fix_system_tables;
+  $ENV{'MYSQL_CLIENT_TEST'}=        $cmdline_mysql_client_test;
+  $ENV{'CHARSETSDIR'}=              $path_charsetsdir;
 
   my $exe= $exe_mysqltest;
   my $args;