Partial fix for BUG#14106: IM: im_life_cycle and im_utils

tests fail on FreeBSD.

The patch contains of the following:
  - make Instance Manager, running in the daemon mode, dump
    the pid of angel-process in the special file;
  - default value of angel-pid-file-name is 'mysqlmanager.angel.pid';
  - if ordinary (IM) pid-file-name is specified in the configuration,
    angel-pid-file-name is updated according to the following
    rule: extension of the basename of pid-file-name is replaced by
    '.angel.pid.
    For example:
    - pid-file-name: /tmp/im.pid
      => angel-pid-file-name: /tmp/im.angel.pid
    - pid-file-name: /tmp/im.txt
      => angel-pid-file-name: /tmp/im.angel.pid
    - pid-file-name: /tmp/5.0/im
      => angel-pid-file-name: /tmp/5.0/im.angel.pid
  - add support for configuration option to customize angel
    pid file name;
  - fix test suite to use angel pid to kill Instance Manager
    by all means if something went wrong.

Background
----------

The problem is that on some OSes (FreeBSD for one) Instance
Manager does not get SIGTERM, so can not shutdown gracefully.
Test suite wasn't able to cope with it, so this leads to the
mess in test results.

The problem should be split into two:
  - fix signal handling;
  - fix test suite.

This patch fixes test suite so that it will be able to kill
uncooperative Instance Manager. In order to achieve this,
test suite needs to know PID of IM Angel process.


mysql-test/lib/mtr_process.pl:
  Added a function to send a signal to a process.
mysql-test/mysql-test-run.pl:
  Changed procedure of stopping Instance Manager.
  1. Try to stop IM normally (by sending SIGTERM);
  2. If one of IM-related processes is still alive,
  kill them all by SIGKILL and complain in the log.
server-tools/instance-manager/manager.cc:
  Made create_pid_file() available for the whole project.
server-tools/instance-manager/manager.h:
  Made create_pid_file() available for the whole project.
server-tools/instance-manager/mysqlmanager.cc:
  Dump PID of angel process into file.
server-tools/instance-manager/options.cc:
  Added an option to allow to customize angel pid file name.
server-tools/instance-manager/options.h:
  Added an option to allow to customize angel pid file name.

mysql-test/lib/mtr_process.pl

@@ -20,6 +20,7 @@ sub mtr_record_dead_children ();
 sub mtr_exit ($);
 sub sleep_until_file_created ($$$);
 sub mtr_kill_processes ($);
+sub mtr_kill_process ($$$$);
 
 # static in C
 sub spawn_impl ($$$$$$$$);
@@ -885,6 +886,25 @@ sub mtr_kill_processes ($) {
   }
 }
 
+
+sub mtr_kill_process ($$$$) {
+  my $pid= shift;
+  my $signal= shift;
+  my $retries= shift;
+  my $timeout= shift;
+
+  while (1)
+  {
+    kill($signal, $pid);
+
+    last unless kill (0, $pid) and $retries--;
+
+    mtr_debug("Sleep $timeout second waiting for processes to die");
+
+    sleep($timeout);
+  }
+}
+
 ##############################################################################
 #
 #  When we exit, we kill off all children

mysql-test/mysql-test-run.pl

@@ -924,6 +924,7 @@ sub command_line_setup () {
    path_err =>        "$opt_vardir/log/im.err",
    path_log =>        "$opt_vardir/log/im.log",
    path_pid =>        "$opt_vardir/run/im.pid",
+   path_angel_pid =>  "$opt_vardir/run/im.angel.pid",
    path_sock =>       "$sockdir/im.sock",
    port =>            $im_port,
    start_timeout =>   $master->[0]->{'start_timeout'},
@@ -1179,6 +1180,7 @@ sub environment_setup () {
   $ENV{'NDB_STATUS_OK'}=      "YES";
 
   $ENV{'IM_PATH_PID'}=        $instance_manager->{path_pid};
+  $ENV{'IM_PATH_ANGEL_PID'}=  $instance_manager->{path_angel_pid};
   $ENV{'IM_PORT'}=            $instance_manager->{port};
 
   $ENV{'IM_MYSQLD1_SOCK'}=    $instance_manager->{instances}->[0]->{path_sock};
@@ -1803,6 +1805,7 @@ sub im_create_defaults_file($) {
 
 [manager]
 pid-file            = $instance_manager->{path_pid}
+angel-pid-file      = $instance_manager->{path_angel_pid}
 socket              = $instance_manager->{path_sock}
 port                = $instance_manager->{port}
 password-file       = $instance_manager->{password_file}
@@ -1827,7 +1830,7 @@ log-slow-queries    = $instance->{path_datadir}/mysqld$server_id.slow.log
 language            = $path_language
 character-sets-dir  = $path_charsetsdir
 basedir             = $path_my_basedir
-server_id           =$server_id
+server_id           = $server_id
 skip-stack-trace
 skip-innodb
 skip-bdb
@@ -2786,6 +2789,18 @@ sub im_start($$) {
 sub im_stop($) {
   my $instance_manager = shift;
 
+  # Obtain mysqld-process pids before we start stopping IM (it can delete pid
+  # files).
+
+  my @mysqld_pids = ();
+  my $instances = $instance_manager->{'instances'};
+
+  push(@mysqld_pids, mtr_get_pid_from_file($instances->[0]->{'path_pid'}))
+    if -r $instances->[0]->{'path_pid'};
+
+  push(@mysqld_pids, mtr_get_pid_from_file($instances->[1]->{'path_pid'}))
+    if -r $instances->[1]->{'path_pid'};
+
   # Re-read pid from the file, since during tests Instance Manager could have
   # been restarted, so its pid could have been changed.
 
@@ -2793,34 +2808,79 @@ sub im_stop($) {
     mtr_get_pid_from_file($instance_manager->{'path_pid'})
       if -f $instance_manager->{'path_pid'};
 
+  if (-f $instance_manager->{'path_angel_pid'})
+  {
+    $instance_manager->{'angel_pid'} =
+      mtr_get_pid_from_file($instance_manager->{'path_angel_pid'})
+  }
+  else
+  {
+    $instance_manager->{'angel_pid'} = undef;
+  }
+
   # Inspired from mtr_stop_mysqld_servers().
 
   start_reap_all();
 
-  # Create list of pids. We should stop Instance Manager and all started
-  # mysqld-instances. Some of them may be nonguarded, so IM will not stop them
-  # on shutdown.
+  # Try graceful shutdown.
 
-  my @pids = ( $instance_manager->{'pid'} );
-  my $instances = $instance_manager->{'instances'};
+  mtr_kill_process($instance_manager->{'pid'}, 'TERM', 10, 1);
+
+  # Check that all processes died.
+
+  my $clean_shutdown= 0;
 
-  if ( -r $instances->[0]->{'path_pid'} )
+  while (1)
   {
-    push(@pids, mtr_get_pid_from_file($instances->[0]->{'path_pid'}));
+    last if kill (0, $instance_manager->{'pid'});
+
+    last if (defined $instance_manager->{'angel_pid'}) &&
+            kill (0, $instance_manager->{'angel_pid'});
+
+    foreach my $pid (@mysqld_pids)
+    {
+      last if kill (0, $pid);
+    }
+
+    $clean_shutdown= 1;
+    last;
   }
 
-  if ( -r $instances->[1]->{'path_pid'} )
+  # Kill leftovers (the order is important).
+
+  unless ($clean_shutdown)
   {
-    push(@pids, mtr_get_pid_from_file($instances->[1]->{'path_pid'}));
-  }
+    mtr_kill_process($instance_manager->{'angel_pid'}, 'KILL', 10, 1)
+      if defined $instance_manager->{'angel_pid'};
+    
+    mtr_kill_process($instance_manager->{'pid'}, 'KILL', 10, 1);
 
-  # Kill processes.
+    # Shutdown managed mysqld-processes. Some of them may be nonguarded, so IM
+    # will not stop them on shutdown. So, we should firstly try to end them
+    # legally.
+
+    mtr_kill_processes(\@mysqld_pids);
+
+    # Complain in error log so that a warning will be shown.
+
+    my $errlog= "$opt_vardir/log/mysql-test-run.pl.err";
+
+    open (ERRLOG, ">>$errlog") ||
+      mtr_error("Can not open error log ($errlog)");
+
+    my $ts= localtime();
+    print ERRLOG
+      "Warning: [$ts] Instance Manager did not shutdown gracefully.\n";
+
+    close ERRLOG;
+  }
 
-  mtr_kill_processes(\@pids);
+  # That's all.
 
   stop_reap_all();
 
   $instance_manager->{'pid'} = undef;
+  $instance_manager->{'angel_pid'} = undef;
 }
 
 

server-tools/instance-manager/manager.cc

@@ -35,12 +35,12 @@
 #endif
 
 
-static int create_pid_file(const char *pid_file_name)
+int create_pid_file(const char *pid_file_name, int pid)
 {
   if (FILE *pid_file= my_fopen(pid_file_name,
                                O_WRONLY | O_CREAT | O_BINARY, MYF(0)))
   {
-    fprintf(pid_file, "%d\n", (int) getpid());
+    fprintf(pid_file, "%d\n", (int) pid);
     my_fclose(pid_file, MYF(0));
     return 0;
   }
@@ -138,8 +138,13 @@ void manager(const Options &options)
   if (user_map.load(options.password_file_name))
     return;
 
-  /* write pid file */
-  if (create_pid_file(options.pid_file_name))
+  /* write Instance Manager pid file */
+
+  log_info("IM pid file: '%s'; PID: %d.",
+           (const char *) options.pid_file_name,
+           (int) manager_pid);
+
+  if (create_pid_file(options.pid_file_name, manager_pid))
     return;
 
   sigset_t mask;

server-tools/instance-manager/manager.h

@@ -20,4 +20,6 @@ struct Options;
 
 void manager(const Options &options);
 
+int create_pid_file(const char *pid_file_name, int pid);
+
 #endif // INCLUDES_MYSQL_INSTANCE_MANAGER_MANAGER_H

server-tools/instance-manager/mysqlmanager.cc

@@ -338,6 +338,14 @@ static void angel(const Options &options)
     /* Here we return to main, and fall into manager */
     break;
   default:                                    // parent, success
+    pid= getpid(); /* Get our pid. */
+
+    log_info("Angel pid file: '%s'; PID: %d.",
+             (const char *) options.angel_pid_file_name,
+             (int) pid);
+
+    create_pid_file(Options::angel_pid_file_name, pid);
+
     while (child_status == CHILD_OK && is_terminated == 0)
       sigsuspend(&zeromask);
 

server-tools/instance-manager/options.cc

@@ -44,6 +44,7 @@ const char *Options::user= 0;                   /* No default value */
 const char *default_password_file_name= QUOTE(DEFAULT_PASSWORD_FILE_NAME);
 const char *default_log_file_name= QUOTE(DEFAULT_LOG_FILE_NAME);
 const char *Options::config_file= QUOTE(DEFAULT_CONFIG_FILE);
+const char *Options::angel_pid_file_name= NULL;
 #endif
 const char *Options::log_file_name= default_log_file_name;
 const char *Options::pid_file_name= QUOTE(DEFAULT_PID_FILE_NAME);
@@ -58,6 +59,9 @@ char **Options::saved_argv= NULL;
 /* Remember if the config file was forced */
 bool Options::is_forced_default_file= 0;
 
+static const char * const ANGEL_PID_FILE_SUFFIX= ".angel.pid";
+static const int ANGEL_PID_FILE_SUFFIX_LEN= strlen(ANGEL_PID_FILE_SUFFIX);
+
 /*
   List of options, accepted by the instance manager.
   List must be closed with empty option.
@@ -72,6 +76,7 @@ enum options {
 #ifndef __WIN__
   OPT_RUN_AS_SERVICE,
   OPT_USER,
+  OPT_ANGEL_PID_FILE,
 #else
   OPT_INSTALL_SERVICE,
   OPT_REMOVE_SERVICE,
@@ -94,7 +99,14 @@ static struct my_option my_long_options[] =
 
   { "pid-file", OPT_PID_FILE, "Pid file to use.",
     (gptr *) &Options::pid_file_name, (gptr *) &Options::pid_file_name,
-   0, GET_STR, REQUIRED_ARG, 0, 0, 0, 0, 0, 0 },
+    0, GET_STR, REQUIRED_ARG, 0, 0, 0, 0, 0, 0 },
+
+#ifndef __WIN__
+  { "angel-pid-file", OPT_ANGEL_PID_FILE, "Pid file for angel process.",
+    (gptr *) &Options::angel_pid_file_name,
+    (gptr *) &Options::angel_pid_file_name,
+    0, GET_STR, REQUIRED_ARG, 0, 0, 0, 0, 0, 0 },
+#endif
 
   { "socket", OPT_SOCKET, "Socket file to use for connection.",
     (gptr *) &Options::socket_file_name, (gptr *) &Options::socket_file_name,
@@ -290,6 +302,46 @@ int Options::load(int argc, char **argv)
                       get_one_option)) != 0)
     goto err;
 
+#ifndef __WIN__
+  if (Options::run_as_service)
+  {
+    if (Options::angel_pid_file_name == NULL)
+    {
+      /*
+        Calculate angel pid file on the IM pid file basis: replace the
+        extension (everything after the last dot) of the pid file basename to
+        '.angel.pid'.
+      */
+
+      char *angel_pid_file_name;
+      char *base_name_ptr;
+      char *ext_ptr;
+
+      angel_pid_file_name= (char *) malloc(strlen(Options::pid_file_name) +
+                                           ANGEL_PID_FILE_SUFFIX_LEN);
+
+      strcpy(angel_pid_file_name, Options::pid_file_name);
+
+      base_name_ptr= strrchr(angel_pid_file_name, '/');
+
+      if (!base_name_ptr)
+        base_name_ptr= angel_pid_file_name + 1;
+
+      ext_ptr= strrchr(base_name_ptr, '.');
+      if (ext_ptr)
+        *ext_ptr= 0;
+
+      strcat(angel_pid_file_name, ANGEL_PID_FILE_SUFFIX);
+
+      Options::angel_pid_file_name= angel_pid_file_name;
+    }
+    else
+    {
+      Options::angel_pid_file_name= strdup(Options::angel_pid_file_name);
+    }
+  }
+#endif
+
   return 0;
 
 err:
@@ -301,6 +353,9 @@ void Options::cleanup()
   /* free_defaults returns nothing */
   if (Options::saved_argv != NULL)
     free_defaults(Options::saved_argv);
+
+  if (Options::run_as_service)
+    free((void *) Options::angel_pid_file_name);
 }
 
 #ifdef __WIN__

server-tools/instance-manager/options.h

@@ -35,6 +35,7 @@ struct Options
 #else
   static char run_as_service;        /* handle_options doesn't support bool */
   static const char *user;
+  static const char *angel_pid_file_name;
 #endif
   static bool is_forced_default_file;
   static const char *log_file_name;