mtr_process.pl 29.9 KB
Newer Older
unknown's avatar
unknown committed
1 2 3 4 5 6
# -*- cperl -*-

# This is a library file used by the Perl version of mysql-test-run,
# and is part of the translation of the Bourne shell script with the
# same name.

unknown's avatar
unknown committed
7 8
use Socket;
use Errno;
unknown's avatar
unknown committed
9 10
use strict;

unknown's avatar
unknown committed
11 12
#use POSIX ":sys_wait_h";
use POSIX 'WNOHANG';
unknown's avatar
unknown committed
13

14 15
sub mtr_run ($$$$$$;$);
sub mtr_spawn ($$$$$$;$);
unknown's avatar
unknown committed
16
sub mtr_check_stop_servers ($);
unknown's avatar
unknown committed
17
sub mtr_kill_leftovers ();
unknown's avatar
unknown committed
18
sub mtr_wait_blocking ($);
unknown's avatar
unknown committed
19
sub mtr_record_dead_children ();
unknown's avatar
unknown committed
20 21
sub mtr_ndbmgm_start($$);
sub mtr_mysqladmin_start($$$);
unknown's avatar
unknown committed
22
sub mtr_exit ($);
unknown's avatar
unknown committed
23
sub sleep_until_file_created ($$$);
24
sub mtr_kill_processes ($);
unknown's avatar
unknown committed
25 26
sub mtr_ping_with_timeout($);
sub mtr_ping_port ($);
unknown's avatar
unknown committed
27 28

# static in C
29
sub spawn_impl ($$$$$$$$);
unknown's avatar
unknown committed
30 31 32 33 34 35 36 37 38

##############################################################################
#
#  Execute an external command
#
##############################################################################

# This function try to mimic the C version used in "netware/mysql_test_run.c"

39
sub mtr_run ($$$$$$;$) {
unknown's avatar
unknown committed
40 41 42 43 44 45
  my $path=       shift;
  my $arg_list_t= shift;
  my $input=      shift;
  my $output=     shift;
  my $error=      shift;
  my $pid_file=   shift;
46
  my $spawn_opts= shift;
unknown's avatar
unknown committed
47

48 49
  return spawn_impl($path,$arg_list_t,'run',$input,$output,$error,$pid_file,
    $spawn_opts);
unknown's avatar
unknown committed
50 51
}

52
sub mtr_run_test ($$$$$$;$) {
unknown's avatar
unknown committed
53 54 55 56 57 58
  my $path=       shift;
  my $arg_list_t= shift;
  my $input=      shift;
  my $output=     shift;
  my $error=      shift;
  my $pid_file=   shift;
59
  my $spawn_opts= shift;
unknown's avatar
unknown committed
60

61 62
  return spawn_impl($path,$arg_list_t,'test',$input,$output,$error,$pid_file,
    $spawn_opts);
unknown's avatar
unknown committed
63 64
}

65
sub mtr_spawn ($$$$$$;$) {
unknown's avatar
unknown committed
66 67 68 69 70 71
  my $path=       shift;
  my $arg_list_t= shift;
  my $input=      shift;
  my $output=     shift;
  my $error=      shift;
  my $pid_file=   shift;
72
  my $spawn_opts= shift;
unknown's avatar
unknown committed
73

74 75
  return spawn_impl($path,$arg_list_t,'spawn',$input,$output,$error,$pid_file,
    $spawn_opts);
unknown's avatar
unknown committed
76 77 78 79 80 81 82 83 84
}


##############################################################################
#
#  If $join is set, we return the error code, else we return the PID
#
##############################################################################

85
sub spawn_impl ($$$$$$$$) {
unknown's avatar
unknown committed
86 87
  my $path=       shift;
  my $arg_list_t= shift;
unknown's avatar
unknown committed
88
  my $mode=       shift;
unknown's avatar
unknown committed
89 90 91 92
  my $input=      shift;
  my $output=     shift;
  my $error=      shift;
  my $pid_file=   shift;                 # FIXME
93
  my $spawn_opts= shift;
unknown's avatar
unknown committed
94 95 96

  if ( $::opt_script_debug )
  {
unknown's avatar
unknown committed
97 98 99 100 101
    print STDERR "\n";
    print STDERR "#### ", "-" x 78, "\n";
    print STDERR "#### ", "STDIN  $input\n" if $input;
    print STDERR "#### ", "STDOUT $output\n" if $output;
    print STDERR "#### ", "STDERR $error\n" if $error;
unknown's avatar
unknown committed
102
    print STDERR "#### ", "$mode : $path ", join(" ",@$arg_list_t), "\n";
103 104 105 106 107 108 109 110 111 112 113 114
    print STDERR "#### ", "spawn options:\n";
    if ($spawn_opts)
    {
      foreach my $key (sort keys %{$spawn_opts})
      {
        print STDERR "#### ", "  - $key: $spawn_opts->{$key}\n";
      }
    }
    else
    {
      print STDERR "#### ", "  none\n";
    }
unknown's avatar
unknown committed
115
    print STDERR "#### ", "-" x 78, "\n";
unknown's avatar
unknown committed
116 117
  }

unknown's avatar
unknown committed
118 119 120
  mtr_error("Can't spawn with empty \"path\"") unless defined $path;


unknown's avatar
unknown committed
121
 FORK:
unknown's avatar
unknown committed
122
  {
unknown's avatar
unknown committed
123
    my $pid= fork();
unknown's avatar
unknown committed
124

unknown's avatar
unknown committed
125
    if ( ! defined $pid )
unknown's avatar
unknown committed
126
    {
unknown's avatar
unknown committed
127
      if ( $! == $!{EAGAIN} )           # See "perldoc Errno"
unknown's avatar
unknown committed
128
      {
unknown's avatar
unknown committed
129 130 131
        mtr_debug("Got EAGAIN from fork(), sleep 1 second and redo");
        sleep(1);
        redo FORK;
unknown's avatar
unknown committed
132
      }
unknown's avatar
unknown committed
133
      else
unknown's avatar
unknown committed
134
      {
unknown's avatar
unknown committed
135
        mtr_error("$path ($pid) can't be forked");
unknown's avatar
unknown committed
136
      }
unknown's avatar
unknown committed
137 138 139 140 141
    }

    if ( $pid )
    {
      spawn_parent_impl($pid,$mode,$path);
unknown's avatar
unknown committed
142 143 144
    }
    else
    {
unknown's avatar
unknown committed
145 146 147 148 149 150 151 152
      # Child, redirect output and exec
      # FIXME I tried POSIX::setsid() here to detach and, I hoped,
      # avoid zombies. But everything went wild, somehow the parent
      # became a deamon as well, and was hard to kill ;-)
      # Need to catch SIGCHLD and do waitpid or something instead......

      $SIG{INT}= 'DEFAULT';         # Parent do some stuff, we don't

153 154 155 156 157 158 159
      my $log_file_open_mode = '>';

      if ($spawn_opts and $spawn_opts->{'append_log_file'})
      {
        $log_file_open_mode = '>>';
      }

unknown's avatar
unknown committed
160 161
      if ( $output )
      {
unknown's avatar
unknown committed
162 163 164 165 166 167 168 169 170
	if ( $::glob_win32_perl )
	{
	  # Don't redirect stdout on ActiveState perl since this is
          # just another thread in the same process.
          # Should be fixed so that the thread that is created with fork
          # executes the exe in another process and wait's for it to return.
          # In the meanwhile, we get all the output from mysqld's to screen
	}
        elsif ( ! open(STDOUT,$log_file_open_mode,$output) )
unknown's avatar
unknown committed
171
        {
172
          mtr_child_error("can't redirect STDOUT to \"$output\": $!");
unknown's avatar
unknown committed
173 174
        }
      }
175

unknown's avatar
unknown committed
176 177 178 179 180 181
      if ( $error )
      {
        if ( $output eq $error )
        {
          if ( ! open(STDERR,">&STDOUT") )
          {
182
            mtr_child_error("can't dup STDOUT: $!");
unknown's avatar
unknown committed
183 184 185 186
          }
        }
        else
        {
187
          if ( ! open(STDERR,$log_file_open_mode,$error) )
unknown's avatar
unknown committed
188
          {
189
            mtr_child_error("can't redirect STDERR to \"$error\": $!");
unknown's avatar
unknown committed
190 191 192
          }
        }
      }
193

unknown's avatar
unknown committed
194 195 196 197
      if ( $input )
      {
        if ( ! open(STDIN,"<",$input) )
        {
198
          mtr_child_error("can't redirect STDIN to \"$input\": $!");
unknown's avatar
unknown committed
199 200
        }
      }
201 202 203

      if ( ! exec($path,@$arg_list_t) )
      {
204
        mtr_child_error("failed to execute \"$path\": $!");
205
      }
unknown's avatar
unknown committed
206 207
    }
  }
unknown's avatar
unknown committed
208 209 210 211 212 213 214
}


sub spawn_parent_impl {
  my $pid=  shift;
  my $mode= shift;
  my $path= shift;
unknown's avatar
unknown committed
215

unknown's avatar
unknown committed
216 217 218
  if ( $mode eq 'run' or $mode eq 'test' )
  {
    if ( $mode eq 'run' )
unknown's avatar
unknown committed
219
    {
unknown's avatar
unknown committed
220 221
      # Simple run of command, we wait for it to return
      my $ret_pid= waitpid($pid,0);
unknown's avatar
unknown committed
222
      if ( $ret_pid != $pid )
unknown's avatar
unknown committed
223
      {
unknown's avatar
unknown committed
224
        mtr_error("$path ($pid) got lost somehow");
unknown's avatar
unknown committed
225
      }
unknown's avatar
unknown committed
226

unknown's avatar
unknown committed
227
      return mtr_process_exit_status($?);
unknown's avatar
unknown committed
228
    }
unknown's avatar
unknown committed
229
    else
unknown's avatar
unknown committed
230
    {
unknown's avatar
unknown committed
231 232 233 234 235 236 237 238 239 240
      # We run mysqltest and wait for it to return. But we try to
      # catch dying mysqld processes as well.
      #
      # We do blocking waitpid() until we get the return from the
      # "mysqltest" call. But if a mysqld process dies that we
      # started, we take this as an error, and kill mysqltest.
      #
      # FIXME is this as it should be? Can't mysqld terminate
      # normally from running a test case?

unknown's avatar
unknown committed
241
      my $exit_value= -1;
242
      my $saved_exit_value;
unknown's avatar
unknown committed
243 244 245
      my $ret_pid;                      # What waitpid() returns

      while ( ($ret_pid= waitpid(-1,0)) != -1 )
unknown's avatar
unknown committed
246
      {
unknown's avatar
unknown committed
247 248 249 250
        # Someone terminated, don't know who. Collect
        # status info first before $? is lost,
        # but not $exit_value, this is flagged from

251 252 253 254 255 256 257 258 259 260 261 262 263 264 265 266 267 268
        my $timer_name= mtr_timer_timeout($::glob_timers, $ret_pid);
        if ( $timer_name )
        {
          if ( $timer_name eq "suite" )
          {
            # We give up here
            # FIXME we should only give up the suite, not all of the run?
            print STDERR "\n";
            mtr_error("Test suite timeout");
          }
          elsif ( $timer_name eq "testcase" )
          {
            $saved_exit_value=  63;       # Mark as timeout
            kill(9, $pid);                # Kill mysqltest
            next;                         # Go on and catch the termination
          }
        }

unknown's avatar
unknown committed
269
        if ( $ret_pid == $pid )
unknown's avatar
unknown committed
270
        {
unknown's avatar
unknown committed
271
          # We got termination of mysqltest, we are done
unknown's avatar
unknown committed
272
          $exit_value= mtr_process_exit_status($?);
unknown's avatar
unknown committed
273
          last;
unknown's avatar
unknown committed
274
        }
unknown's avatar
unknown committed
275

unknown's avatar
unknown committed
276 277
        # One of the child processes died, unless this was expected
	# mysqltest should be killed and test aborted
unknown's avatar
unknown committed
278

unknown's avatar
unknown committed
279
	check_expected_crash_and_restart($ret_pid);
unknown's avatar
unknown committed
280
      }
unknown's avatar
unknown committed
281 282

      if ( $ret_pid != $pid )
unknown's avatar
unknown committed
283
      {
unknown's avatar
unknown committed
284 285
        # We terminated the waiting because a "mysqld" process died.
        # Kill the mysqltest process.
unknown's avatar
unknown committed
286
	mtr_verbose("Kill mysqltest because another process died");
unknown's avatar
unknown committed
287 288 289 290
        kill(9,$pid);

        $ret_pid= waitpid($pid,0);

unknown's avatar
unknown committed
291
        if ( $ret_pid != $pid )
unknown's avatar
unknown committed
292 293 294
        {
          mtr_error("$path ($pid) got lost somehow");
        }
unknown's avatar
unknown committed
295
      }
unknown's avatar
unknown committed
296

297
      return $saved_exit_value || $exit_value;
unknown's avatar
unknown committed
298
    }
unknown's avatar
unknown committed
299 300 301 302 303
  }
  else
  {
    # We spawned a process we don't wait for
    return $pid;
unknown's avatar
unknown committed
304 305 306
  }
}

unknown's avatar
unknown committed
307

unknown's avatar
unknown committed
308 309 310 311 312 313 314 315 316 317 318 319 320 321 322 323 324
# ----------------------------------------------------------------------
# We try to emulate how an Unix shell calculates the exit code
# ----------------------------------------------------------------------

sub mtr_process_exit_status {
  my $raw_status= shift;

  if ( $raw_status & 127 )
  {
    return ($raw_status & 127) + 128;  # Signal num + 128
  }
  else
  {
    return $raw_status >> 8;           # Exit code
  }
}

unknown's avatar
unknown committed
325

unknown's avatar
unknown committed
326 327 328 329 330 331
##############################################################################
#
#  Kill processes left from previous runs
#
##############################################################################

unknown's avatar
unknown committed
332

unknown's avatar
unknown committed
333 334 335 336
# Kill all processes(mysqld, ndbd, ndb_mgmd and im) that would conflict with
# this run
# Make sure to remove the PID file, if any.
# kill IM manager first, else it will restart the servers
unknown's avatar
unknown committed
337 338
sub mtr_kill_leftovers () {

unknown's avatar
unknown committed
339 340
  mtr_report("Killing Possible Leftover Processes");
  mtr_debug("mtr_kill_leftovers(): started.");
unknown's avatar
unknown committed
341

unknown's avatar
unknown committed
342
  mkpath("$::opt_vardir/log"); # Needed for mysqladmin log
unknown's avatar
unknown committed
343

unknown's avatar
unknown committed
344 345 346 347 348 349 350 351 352 353 354 355 356 357 358
  # Stop or kill Instance Manager and all its children. If we failed to do
  # that, we can only abort -- there is nothing left to do.

#  mtr_error("Failed to stop Instance Manager.")
#    unless mtr_im_stop($::instance_manager);

  # Start shutdown of masters and slaves. Don't touch IM-managed mysqld
  # instances -- they should be stopped by mtr_im_stop().

  mtr_debug("Shutting down mysqld-instances...");

  my @kill_pids;
  my %admin_pids;

  foreach my $srv (@{$::master}, @{$::slave})
unknown's avatar
unknown committed
359
  {
unknown's avatar
unknown committed
360 361 362 363 364 365 366 367 368 369 370 371 372 373 374 375 376 377
    mtr_debug("  - mysqld " .
              "(pid: $srv->{pid}; " .
              "pid file: '$srv->{path_pid}'; " .
              "socket: '$srv->{path_sock}'; ".
              "port: $srv->{port})");

    my $pid= mtr_mysqladmin_start($srv, "shutdown", 70);

    # Save the pid of the mysqladmin process
    $admin_pids{$pid}= 1;

    push(@kill_pids,{
		     pid      => $srv->{'pid'},
		     pidfile  => $srv->{'path_pid'},
		     sockfile => $srv->{'path_sock'},
		     port     => $srv->{'port'},
		    });
    $srv->{'pid'}= 0; # Assume we are done with it
unknown's avatar
unknown committed
378 379
  }

unknown's avatar
unknown committed
380
  if ( ! $::opt_skip_ndbcluster )
unknown's avatar
unknown committed
381
  {
unknown's avatar
unknown committed
382 383 384 385 386 387 388 389 390 391 392 393 394 395 396 397 398 399 400 401 402 403 404 405 406 407 408 409 410 411 412 413 414 415
    # Start shutdown of clusters.
    mtr_debug("Shutting down cluster...");

    foreach my $cluster (@{$::clusters})
    {
      mtr_debug("  - cluster " .
		"(pid: $cluster->{pid}; " .
		"pid file: '$cluster->{path_pid})");

      my $pid= mtr_ndbmgm_start($cluster, "shutdown");

      # Save the pid of the ndb_mgm process
      $admin_pids{$pid}= 1;

      push(@kill_pids,{
		       pid      => $cluster->{'pid'},
		       pidfile  => $cluster->{'path_pid'}
		      });

      $cluster->{'pid'}= 0; # Assume we are done with it

      foreach my $ndbd (@{$cluster->{'ndbds'}})
      {
	mtr_debug("    - ndbd " .
		  "(pid: $ndbd->{pid}; " .
		  "pid file: '$ndbd->{path_pid})");

	push(@kill_pids,{
			 pid      => $ndbd->{'pid'},
			 pidfile  => $ndbd->{'path_pid'},
			});
	$ndbd->{'pid'}= 0; # Assume we are done with it
      }
    }
unknown's avatar
unknown committed
416 417
  }

unknown's avatar
unknown committed
418 419 420 421 422 423 424 425 426
  # Wait for all the admin processes to complete
  mtr_wait_blocking(\%admin_pids);

  # If we trusted "mysqladmin --shutdown_timeout= ..." we could just
  # terminate now, but we don't (FIXME should be debugged).
  # So we try again to ping and at least wait the same amount of time
  # mysqladmin would for all to die.

  mtr_ping_with_timeout(\@kill_pids);
unknown's avatar
unknown committed
427 428 429 430 431 432 433 434 435 436 437

  # We now have tried to terminate nice. We have waited for the listen
  # port to be free, but can't really tell if the mysqld process died
  # or not. We now try to find the process PID from the PID file, and
  # send a kill to that process. Note that Perl let kill(0,@pids) be
  # a way to just return the numer of processes the kernel can send
  # signals to. So this can be used (except on Cygwin) to determine
  # if there are processes left running that we cound out might exists.
  #
  # But still after all this work, all we know is that we have
  # the ports free.
unknown's avatar
unknown committed
438 439

  # We scan the "var/run/" directory for other process id's to kill
unknown's avatar
unknown committed
440 441

  # FIXME $path_run_dir or something
442
  my $rundir= "$::opt_vardir/run";
unknown's avatar
unknown committed
443

unknown's avatar
unknown committed
444 445
  mtr_debug("Processing PID files in directory '$rundir'...");

unknown's avatar
unknown committed
446 447 448 449 450 451 452 453 454 455 456 457 458
  if ( -d $rundir )
  {
    opendir(RUNDIR, $rundir)
      or mtr_error("can't open directory \"$rundir\": $!");

    my @pids;

    while ( my $elem= readdir(RUNDIR) )
    {
      my $pidfile= "$rundir/$elem";

      if ( -f $pidfile )
      {
unknown's avatar
unknown committed
459
        mtr_debug("Processing PID file: '$pidfile'...");
unknown's avatar
unknown committed
460

unknown's avatar
unknown committed
461
        my $pid= mtr_get_pid_from_file($pidfile);
unknown's avatar
unknown committed
462

unknown's avatar
unknown committed
463
        mtr_debug("Got pid: $pid from file '$pidfile'");
unknown's avatar
unknown committed
464 465 466

        if ( $::glob_cygwin_perl or kill(0, $pid) )
        {
unknown's avatar
unknown committed
467
          mtr_debug("There is process with pid $pid -- scheduling for kill.");
unknown's avatar
unknown committed
468 469
          push(@pids, $pid);            # We know (cygwin guess) it exists
        }
unknown's avatar
unknown committed
470 471 472 473
        else
        {
          mtr_debug("There is no process with pid $pid -- skipping.");
        }
unknown's avatar
unknown committed
474 475 476 477
      }
    }
    closedir(RUNDIR);

unknown's avatar
unknown committed
478
    if ( @pids )
unknown's avatar
unknown committed
479
    {
unknown's avatar
unknown committed
480 481 482 483 484
      mtr_debug("Killing the following processes with PID files: " .
                join(' ', @pids) . "...");

      start_reap_all();

unknown's avatar
unknown committed
485
      if ( $::glob_cygwin_perl )
unknown's avatar
unknown committed
486
      {
unknown's avatar
unknown committed
487 488 489 490 491 492 493
        # We have no (easy) way of knowing the Cygwin controlling
        # process, in the PID file we only have the Windows process id.
        system("kill -f " . join(" ",@pids)); # Hope for the best....
        mtr_debug("Sleep 5 seconds waiting for processes to die");
        sleep(5);
      }
      else
unknown's avatar
unknown committed
494
      {
unknown's avatar
unknown committed
495 496 497
        my $retries= 10;                    # 10 seconds
        do
        {
unknown's avatar
unknown committed
498
          mtr_debug("Sending SIGKILL to pids: " . join(' ', @pids));
unknown's avatar
unknown committed
499
          kill(9, @pids);
unknown's avatar
unknown committed
500
          mtr_report("Sleep 1 second waiting for processes to die");
unknown's avatar
unknown committed
501 502 503 504 505
          sleep(1)                      # Wait one second
        } while ( $retries-- and  kill(0, @pids) );

        if ( kill(0, @pids) )           # Check if some left
        {
506
          mtr_warning("can't kill process(es) " . join(" ", @pids));
unknown's avatar
unknown committed
507
        }
unknown's avatar
unknown committed
508
      }
unknown's avatar
unknown committed
509 510

      stop_reap_all();
unknown's avatar
unknown committed
511
    }
unknown's avatar
unknown committed
512
  }
unknown's avatar
unknown committed
513 514 515 516
  else
  {
    mtr_debug("Directory for PID files ($rundir) does not exist.");
  }
unknown's avatar
unknown committed
517

unknown's avatar
unknown committed
518
  # We may have failed everything, but we now check again if we have
unknown's avatar
unknown committed
519
  # the listen ports free to use, and if they are free, just go for it.
unknown's avatar
unknown committed
520

unknown's avatar
unknown committed
521 522 523
  mtr_debug("Checking known mysqld servers...");

  foreach my $srv ( @kill_pids )
unknown's avatar
unknown committed
524
  {
unknown's avatar
unknown committed
525
    if ( defined $srv->{'port'} and mtr_ping_port($srv->{'port'}) )
unknown's avatar
unknown committed
526
    {
unknown's avatar
unknown committed
527
      mtr_warning("can't kill old process holding port $srv->{'port'}");
unknown's avatar
unknown committed
528
    }
unknown's avatar
unknown committed
529 530
  }

unknown's avatar
unknown committed
531 532
  mtr_debug("mtr_kill_leftovers(): finished.");
}
unknown's avatar
unknown committed
533 534


unknown's avatar
unknown committed
535 536 537 538 539
# Check that all processes in list are killed
# The argument is a list of 'ports', 'pids', 'pidfiles' and 'socketfiles'
# for which shutdown has been started. Make sure they all get killed
# in one way or the other.
#
unknown's avatar
unknown committed
540
# FIXME On Cygwin, and maybe some other platforms, $srv->{'pid'} and
unknown's avatar
unknown committed
541
# the pid in $srv->{'pidfile'} will not be the same PID. We need to try to kill
unknown's avatar
unknown committed
542
# both I think.
unknown's avatar
unknown committed
543

unknown's avatar
unknown committed
544
sub mtr_check_stop_servers ($) {
unknown's avatar
unknown committed
545
  my $spec=  shift;
unknown's avatar
unknown committed
546

unknown's avatar
unknown committed
547 548
  # Return if no processes are defined
  return if ! @$spec;
unknown's avatar
unknown committed
549

unknown's avatar
unknown committed
550 551 552
  #mtr_report("mtr_check_stop_servers");

  mtr_ping_with_timeout(\@$spec);
unknown's avatar
unknown committed
553 554 555

  # ----------------------------------------------------------------------
  # We loop with waitpid() nonblocking to see how many of the ones we
unknown's avatar
unknown committed
556 557 558
  # are to kill, actually got killed by mysqladmin or ndb_mgm
  #
  # Note that we don't rely on this, the mysqld server might have stopped
unknown's avatar
unknown committed
559 560
  # listening to the port, but still be alive. But it is a start.
  # ----------------------------------------------------------------------
unknown's avatar
unknown committed
561

unknown's avatar
unknown committed
562
  foreach my $srv ( @$spec )
unknown's avatar
unknown committed
563
  {
unknown's avatar
unknown committed
564 565
    my $ret_pid;
    if ( $srv->{'pid'} )
unknown's avatar
unknown committed
566
    {
unknown's avatar
unknown committed
567 568 569 570 571 572 573 574 575 576
      $ret_pid= waitpid($srv->{'pid'},&WNOHANG);
      if ($ret_pid == $srv->{'pid'})
      {
	mtr_verbose("Caught exit of process $ret_pid");
	$srv->{'pid'}= 0;
      }
      else
      {
	# mtr_warning("caught exit of unknown child $ret_pid");
      }
unknown's avatar
unknown committed
577 578 579
    }
  }

unknown's avatar
unknown committed
580
  # ----------------------------------------------------------------------
unknown's avatar
unknown committed
581 582 583 584
  # We know the process was started from this file, so there is a PID
  # saved, or else we have nothing to do.
  # Might be that is is recorded to be missing, but we failed to
  # take away the PID file earlier, then we do it now.
unknown's avatar
unknown committed
585
  # ----------------------------------------------------------------------
unknown's avatar
unknown committed
586

unknown's avatar
unknown committed
587
  my %mysqld_pids;
unknown's avatar
unknown committed
588 589

  foreach my $srv ( @$spec )
unknown's avatar
unknown committed
590
  {
unknown's avatar
unknown committed
591
    if ( $srv->{'pid'} )
unknown's avatar
unknown committed
592
    {
unknown's avatar
unknown committed
593
      $mysqld_pids{$srv->{'pid'}}= 1;
unknown's avatar
unknown committed
594
    }
unknown's avatar
unknown committed
595
    else
unknown's avatar
unknown committed
596
    {
unknown's avatar
unknown committed
597
      # Server is dead, we remove the pidfile if any
unknown's avatar
unknown committed
598 599 600 601 602
      # Race, could have been removed between I tested with -f
      # and the unlink() below, so I better check again with -f

      if ( -f $srv->{'pidfile'} and ! unlink($srv->{'pidfile'}) and
           -f $srv->{'pidfile'} )
unknown's avatar
unknown committed
603
      {
unknown's avatar
unknown committed
604
        mtr_error("can't remove $srv->{'pidfile'}");
unknown's avatar
unknown committed
605 606 607 608
      }
    }
  }

unknown's avatar
unknown committed
609
  # ----------------------------------------------------------------------
unknown's avatar
unknown committed
610
  # If all the processes in list already have been killed,
unknown's avatar
unknown committed
611
  # then we don't have to do anything.
unknown's avatar
unknown committed
612
  # ----------------------------------------------------------------------
unknown's avatar
unknown committed
613

unknown's avatar
unknown committed
614
  if ( ! keys %mysqld_pids )
unknown's avatar
unknown committed
615
  {
unknown's avatar
unknown committed
616
    return;
unknown's avatar
unknown committed
617 618
  }

unknown's avatar
unknown committed
619
  # ----------------------------------------------------------------------
unknown's avatar
unknown committed
620 621 622 623
  # In mtr_mysqladmin_shutdown() we only waited for the mysqld servers
  # not to listen to the port. But we are not sure we got them all
  # killed. If we suspect it lives, try nice kill with SIG_TERM. Note
  # that for true Win32 processes, kill(0,$pid) will not return 1.
unknown's avatar
unknown committed
624
  # ----------------------------------------------------------------------
unknown's avatar
unknown committed
625

unknown's avatar
unknown committed
626 627
  start_reap_all();                     # Avoid zombies

unknown's avatar
unknown committed
628 629
  my @mysqld_pids= keys %mysqld_pids;
  mtr_kill_processes(\@mysqld_pids);
unknown's avatar
unknown committed
630

unknown's avatar
unknown committed
631 632
  stop_reap_all();                      # Get into control again

unknown's avatar
unknown committed
633 634 635 636 637
  # ----------------------------------------------------------------------
  # Now, we check if all we can find using kill(0,$pid) are dead,
  # and just assume the rest are. We cleanup socket and PID files.
  # ----------------------------------------------------------------------

unknown's avatar
unknown committed
638
  {
unknown's avatar
unknown committed
639 640
    my $errors= 0;
    foreach my $srv ( @$spec )
unknown's avatar
unknown committed
641
    {
unknown's avatar
unknown committed
642
      if ( $srv->{'pid'} )
unknown's avatar
unknown committed
643
      {
unknown's avatar
unknown committed
644 645 646 647 648 649 650 651
        if ( kill(0,$srv->{'pid'}) )
        {
          # FIXME In Cygwin there seem to be some fast reuse
          # of PIDs, so dying may not be the right thing to do.
          $errors++;
          mtr_warning("can't kill process $srv->{'pid'}");
        }
        else
unknown's avatar
unknown committed
652
        {
unknown's avatar
unknown committed
653 654 655 656 657 658 659 660 661 662
          # We managed to kill it at last
          # FIXME In Cygwin, we will get here even if the process lives.

          # Not needed as we know the process is dead, but to be safe
          # we unlink and check success in two steps. We first unlink
          # without checking the error code, and then check if the
          # file still exists.

          foreach my $file ($srv->{'pidfile'}, $srv->{'sockfile'})
          {
unknown's avatar
unknown committed
663
            # Know it is dead so should be no race, careful anyway
unknown's avatar
unknown committed
664
            if ( defined $file and -f $file and ! unlink($file) and -f $file )
unknown's avatar
unknown committed
665 666 667 668 669
            {
              $errors++;
              mtr_warning("couldn't delete $file");
            }
          }
unknown's avatar
unknown committed
670
	  $srv->{'pid'}= 0;
unknown's avatar
unknown committed
671 672 673
        }
      }
    }
unknown's avatar
unknown committed
674
    if ( $errors )
unknown's avatar
unknown committed
675
    {
unknown's avatar
unknown committed
676 677 678 679 680 681 682 683 684 685 686
      # There where errors killing processes
      # do one last attempt to ping the servers
      # and if they can't be pinged, assume they are dead
      if ( ! mtr_ping_with_timeout( \@$spec ) )
      {
	mtr_error("we could not kill or clean up all processes");
      }
      else
      {
	mtr_verbose("All ports were free, continuing");
      }
unknown's avatar
unknown committed
687 688 689
    }
  }

unknown's avatar
unknown committed
690 691
  # FIXME We just assume they are all dead, for Cygwin we are not
  # really sure
unknown's avatar
unknown committed
692

unknown's avatar
unknown committed
693 694
}

unknown's avatar
unknown committed
695 696 697
# Wait for all the process in the list to terminate
sub mtr_wait_blocking($) {
  my $admin_pids= shift;
unknown's avatar
unknown committed
698 699


unknown's avatar
unknown committed
700 701 702 703 704 705 706 707 708 709 710 711 712 713 714 715 716 717 718
  # Return if no processes defined
  return if ! %$admin_pids;

  mtr_verbose("mtr_wait_blocking");

  # Wait for all the started processes to exit
  # As mysqladmin is such a simple program, we trust it to terminate itself.
  # I.e. we wait blocking, and wait for them all before we go on.
  foreach my $pid (keys %{$admin_pids})
  {
    my $ret_pid= waitpid($pid,0);

  }
}

# Start "mysqladmin shutdown" for a specific mysqld
sub mtr_mysqladmin_start($$$) {
  my $srv= shift;
  my $command= shift;
unknown's avatar
unknown committed
719
  my $adm_shutdown_tmo= shift;
unknown's avatar
unknown committed
720

unknown's avatar
unknown committed
721 722
  my $args;
  mtr_init_args(\$args);
unknown's avatar
unknown committed
723

unknown's avatar
unknown committed
724 725 726 727 728 729 730 731 732
  mtr_add_arg($args, "--no-defaults");
  mtr_add_arg($args, "--user=%s", $::opt_user);
  mtr_add_arg($args, "--password=");
  mtr_add_arg($args, "--silent");
  if ( -e $srv->{'path_sock'} )
  {
    mtr_add_arg($args, "--socket=%s", $srv->{'path_sock'});
  }
  if ( $srv->{'port'} )
unknown's avatar
unknown committed
733
  {
unknown's avatar
unknown committed
734 735 736 737 738 739 740 741 742 743 744 745 746 747 748 749 750 751 752 753 754 755 756 757 758 759 760 761 762 763 764 765 766 767 768 769 770 771 772 773 774 775 776 777 778 779 780 781 782 783 784 785 786 787 788 789 790 791 792 793 794
    mtr_add_arg($args, "--port=%s", $srv->{'port'});
  }
  if ( $srv->{'port'} and ! -e $srv->{'path_sock'} )
  {
    mtr_add_arg($args, "--protocol=tcp"); # Needed if no --socket
  }
  mtr_add_arg($args, "--connect_timeout=5");

  # Shutdown time must be high as slave may be in reconnect
  mtr_add_arg($args, "--shutdown_timeout=$adm_shutdown_tmo");
  mtr_add_arg($args, "$command");
  my $path_mysqladmin_log= "$::opt_vardir/log/mysqladmin.log";
  my $pid= mtr_spawn($::exe_mysqladmin, $args,
		     "", $path_mysqladmin_log, $path_mysqladmin_log, "",
		     { append_log_file => 1 });
  mtr_verbose("mtr_mysqladmin_start, pid: $pid");
  return $pid;

}

# Start "ndb_mgm shutdown" for a specific cluster, it will
# shutdown all data nodes and leave the ndb_mgmd running
sub mtr_ndbmgm_start($$) {
  my $cluster= shift;
  my $command= shift;

  my $args;

  mtr_init_args(\$args);

  mtr_add_arg($args, "--no-defaults");
  mtr_add_arg($args, "--core");
  mtr_add_arg($args, "--try-reconnect=1");
  mtr_add_arg($args, "--ndb_connectstring=%s", $cluster->{'connect_string'});
  mtr_add_arg($args, "-e");
  mtr_add_arg($args, "$command");

  my $pid= mtr_spawn($::exe_ndb_mgm, $args,
		     "", "/dev/null", "/dev/null", "",
		     {});
  mtr_verbose("mtr_ndbmgm_start, pid: $pid");
  return $pid;

}


# Ping all servers in list, exit when none of them answers
# or when timeout has passed
sub mtr_ping_with_timeout($) {
  my $spec= shift;
  my $timeout= 200;                     # 20 seconds max
  my $res= 1;                           # If we just fall through, we are done
                                        # in the sense that the servers don't
                                        # listen to their ports any longer

  mtr_debug("Waiting for mysqld servers to stop...");

 TIME:
  while ( $timeout-- )
  {
    foreach my $srv ( @$spec )
unknown's avatar
unknown committed
795
    {
unknown's avatar
unknown committed
796 797 798 799 800 801 802 803 804 805 806 807 808 809 810 811 812 813
      $res= 1;                          # We are optimistic
      if ( $srv->{'pid'} and defined $srv->{'port'} )
      {
	if ( mtr_ping_port($srv->{'port'}) )
	{
	  mtr_verbose("waiting for process $srv->{'pid'} to stop ".
		      "using port $srv->{'port'}");

	  # Millisceond sleep emulated with select
	  select(undef, undef, undef, (0.1));
	  $res= 0;
	  next TIME;
	}
	else
	{
	  # Process was not using port
	}
      }
unknown's avatar
unknown committed
814
    }
unknown's avatar
unknown committed
815
    last;                               # If we got here, we are done
unknown's avatar
unknown committed
816 817
  }

unknown's avatar
unknown committed
818 819 820 821 822
  if ($res)
  {
    mtr_debug("mtr_ping_with_timeout(): All mysqld instances are down.");
  }
  else
unknown's avatar
unknown committed
823
  {
unknown's avatar
unknown committed
824 825
    mtr_report("mtr_ping_with_timeout(): At least one server is alive.");
  }
unknown's avatar
unknown committed
826

unknown's avatar
unknown committed
827 828 829 830 831 832 833 834 835 836 837 838
  return $res;
}


#
# Loop through our list of processes and look for and entry
# with the provided pid
# Set the pid of that process to 0 if found
#
sub mark_process_dead($)
{
  my $ret_pid= shift;
unknown's avatar
unknown committed
839

unknown's avatar
unknown committed
840 841 842
  foreach my $mysqld (@{$::master}, @{$::slave})
  {
    if ( $mysqld->{'pid'} eq $ret_pid )
unknown's avatar
unknown committed
843
    {
unknown's avatar
unknown committed
844 845 846
      mtr_verbose("$mysqld->{'type'} $mysqld->{'idx'} exited, pid: $ret_pid");
      $mysqld->{'pid'}= 0;
      return;
unknown's avatar
unknown committed
847
    }
unknown's avatar
unknown committed
848 849 850 851 852
  }

  foreach my $cluster (@{$::clusters})
  {
    if ( $cluster->{'pid'} eq $ret_pid )
unknown's avatar
unknown committed
853
    {
unknown's avatar
unknown committed
854 855 856
      mtr_verbose("$cluster->{'name'} cluster ndb_mgmd exited, pid: $ret_pid");
      $cluster->{'pid'}= 0;
      return;
unknown's avatar
unknown committed
857
    }
unknown's avatar
unknown committed
858 859

    foreach my $ndbd (@{$cluster->{'ndbds'}})
unknown's avatar
unknown committed
860
    {
unknown's avatar
unknown committed
861 862 863 864 865 866
      if ( $ndbd->{'pid'} eq $ret_pid )
      {
	mtr_verbose("$cluster->{'name'} cluster ndbd exited, pid: $ret_pid");
	$ndbd->{'pid'}= 0;
	return;
      }
unknown's avatar
unknown committed
867 868
    }
  }
unknown's avatar
unknown committed
869 870 871 872 873 874 875 876 877 878 879 880
  mtr_warning("mark_process_dead couldn't find an entry for pid: $ret_pid");

}

#
# Loop through our list of processes and look for and entry
# with the provided pid, if found check for the file indicating
# expected crash and restart it.
#
sub check_expected_crash_and_restart($)
{
  my $ret_pid= shift;
unknown's avatar
unknown committed
881

unknown's avatar
unknown committed
882
  foreach my $mysqld (@{$::master}, @{$::slave})
unknown's avatar
unknown committed
883
  {
unknown's avatar
unknown committed
884
    if ( $mysqld->{'pid'} eq $ret_pid )
885
    {
unknown's avatar
unknown committed
886 887 888 889 890 891 892
      mtr_verbose("$mysqld->{'type'} $mysqld->{'idx'} exited, pid: $ret_pid");
      $mysqld->{'pid'}= 0;

      # Check if crash expected and restart if it was
      my $expect_file= "$::opt_vardir/tmp/" . "$mysqld->{'type'}" .
	"$mysqld->{'idx'}" . ".expect";
      if ( -f $expect_file )
893
      {
unknown's avatar
unknown committed
894 895 896 897
	mtr_verbose("Crash was expected, file $expect_file exists");
	mysqld_start($mysqld, $mysqld->{'start_opts'},
		     $mysqld->{'start_slave_master_info'});
	unlink($expect_file);
898
      }
unknown's avatar
unknown committed
899 900

      return;
901
    }
unknown's avatar
unknown committed
902
  }
unknown's avatar
unknown committed
903

unknown's avatar
unknown committed
904
  foreach my $cluster (@{$::clusters})
unknown's avatar
unknown committed
905
  {
unknown's avatar
unknown committed
906
    if ( $cluster->{'pid'} eq $ret_pid )
unknown's avatar
unknown committed
907
    {
unknown's avatar
unknown committed
908 909 910 911 912 913 914
      mtr_verbose("$cluster->{'name'} cluster ndb_mgmd exited, pid: $ret_pid");
      $cluster->{'pid'}= 0;

      # Check if crash expected and restart if it was
      my $expect_file= "$::opt_vardir/tmp/ndb_mgmd_" . "$cluster->{'type'}" .
	".expect";
      if ( -f $expect_file )
unknown's avatar
unknown committed
915
      {
unknown's avatar
unknown committed
916 917 918
	mtr_verbose("Crash was expected, file $expect_file exists");
	ndbmgmd_start($cluster);
	unlink($expect_file);
unknown's avatar
unknown committed
919
      }
unknown's avatar
unknown committed
920
      return;
unknown's avatar
unknown committed
921
    }
unknown's avatar
unknown committed
922

unknown's avatar
unknown committed
923 924 925 926 927 928 929 930 931 932 933 934 935 936 937 938 939 940 941 942 943 944
    foreach my $ndbd (@{$cluster->{'ndbds'}})
    {
      if ( $ndbd->{'pid'} eq $ret_pid )
      {
	mtr_verbose("$cluster->{'name'} cluster ndbd exited, pid: $ret_pid");
	$ndbd->{'pid'}= 0;

	# Check if crash expected and restart if it was
	my $expect_file= "$::opt_vardir/tmp/ndbd_" . "$cluster->{'type'}" .
	  "$ndbd->{'idx'}" . ".expect";
	if ( -f $expect_file )
	{
	  mtr_verbose("Crash was expected, file $expect_file exists");
	  ndbd_start($cluster, $ndbd->{'idx'},
		     $ndbd->{'start_extra_args'});
	  unlink($expect_file);
	}
	return;
      }
    }
  }
  mtr_warning("check_expected_crash_and_restart couldn't find an entry for pid: $ret_pid");
unknown's avatar
unknown committed
945

unknown's avatar
unknown committed
946 947 948 949 950 951 952 953 954 955 956 957
}

##############################################################################
#
#  The operating system will keep information about dead children, 
#  we read this information here, and if we have records the process
#  is alive, we mark it as dead.
#
##############################################################################

sub mtr_record_dead_children () {

unknown's avatar
unknown committed
958
  my $process_died= 0;
unknown's avatar
unknown committed
959 960
  my $ret_pid;

unknown's avatar
unknown committed
961 962 963
  # Wait without blockinng to see if any processes had died
  # -1 or 0 means there are no more procesess to wait for
  while ( ($ret_pid= waitpid(-1,&WNOHANG)) != 0 and $ret_pid != -1)
unknown's avatar
unknown committed
964
  {
unknown's avatar
unknown committed
965 966 967
    mtr_warning("mtr_record_dead_children: $ret_pid");
    mark_process_dead($ret_pid);
    $process_died= 1;
unknown's avatar
unknown committed
968
  }
unknown's avatar
unknown committed
969
  return $process_died;
unknown's avatar
unknown committed
970 971
}

unknown's avatar
unknown committed
972
sub start_reap_all {
973 974 975 976 977 978 979 980
  # This causes terminating processes to not become zombies, avoiding
  # the need for (or possibility of) explicit waitpid().
  $SIG{CHLD}= 'IGNORE';

  # On some platforms (Linux, QNX, OSX, ...) there is potential race
  # here. If a process terminated before setting $SIG{CHLD} (but after
  # any attempt to waitpid() it), it will still be a zombie. So we
  # have to handle any such process here.
unknown's avatar
unknown committed
981 982 983 984 985 986
  my $pid;
  while(($pid= waitpid(-1, &WNOHANG)) != 0 and $pid != -1)
  {
    mtr_warning("start_reap_all pid: $pid");
    mark_process_dead($pid);
  };
unknown's avatar
unknown committed
987 988 989 990 991
}

sub stop_reap_all {
  $SIG{CHLD}= 'DEFAULT';
}
unknown's avatar
unknown committed
992

unknown's avatar
unknown committed
993 994

sub mtr_ping_port ($) {
unknown's avatar
unknown committed
995 996
  my $port= shift;

unknown's avatar
unknown committed
997 998
  mtr_verbose("mtr_ping_port: $port");

unknown's avatar
unknown committed
999 1000 1001 1002 1003 1004 1005 1006 1007 1008 1009 1010
  my $remote= "localhost";
  my $iaddr=  inet_aton($remote);
  if ( ! $iaddr )
  {
    mtr_error("can't find IP number for $remote");
  }
  my $paddr=  sockaddr_in($port, $iaddr);
  my $proto=  getprotobyname('tcp');
  if ( ! socket(SOCK, PF_INET, SOCK_STREAM, $proto) )
  {
    mtr_error("can't create socket: $!");
  }
unknown's avatar
unknown committed
1011 1012 1013

  mtr_debug("Pinging server (port: $port)...");

unknown's avatar
unknown committed
1014 1015 1016
  if ( connect(SOCK, $paddr) )
  {
    close(SOCK);                        # FIXME check error?
unknown's avatar
unknown committed
1017
    mtr_verbose("USED");
unknown's avatar
unknown committed
1018 1019 1020 1021
    return 1;
  }
  else
  {
unknown's avatar
unknown committed
1022
    mtr_verbose("FREE");
unknown's avatar
unknown committed
1023 1024 1025 1026
    return 0;
  }
}

unknown's avatar
unknown committed
1027 1028 1029 1030 1031 1032
##############################################################################
#
#  Wait for a file to be created
#
##############################################################################

unknown's avatar
unknown committed
1033
# FIXME check that the pidfile contains the expected pid!
unknown's avatar
unknown committed
1034

unknown's avatar
unknown committed
1035
sub sleep_until_file_created ($$$) {
unknown's avatar
unknown committed
1036 1037
  my $pidfile= shift;
  my $timeout= shift;
unknown's avatar
unknown committed
1038
  my $pid=     shift;
unknown's avatar
unknown committed
1039 1040
  my $sleeptime= 100; # Milliseconds
  my $loops= ($timeout * 1000) / $sleeptime;
unknown's avatar
unknown committed
1041

unknown's avatar
unknown committed
1042
  for ( my $loop= 1; $loop <= $loops; $loop++ )
unknown's avatar
unknown committed
1043 1044 1045
  {
    if ( -r $pidfile )
    {
unknown's avatar
unknown committed
1046
      return $pid;
unknown's avatar
unknown committed
1047 1048
    }

unknown's avatar
unknown committed
1049 1050
    # Check if it died after the fork() was successful
    if ( $pid != 0 && waitpid($pid,&WNOHANG) == $pid )
unknown's avatar
unknown committed
1051
    {
unknown's avatar
unknown committed
1052
      mtr_warning("Process $pid died");
unknown's avatar
unknown committed
1053 1054 1055
      return 0;
    }

unknown's avatar
unknown committed
1056
    mtr_debug("Sleep $sleeptime milliseconds waiting for $pidfile");
unknown's avatar
unknown committed
1057

unknown's avatar
unknown committed
1058 1059 1060
    # Print extra message every 60 seconds
    my $seconds= ($loop * $sleeptime) / 1000;
    if ( $seconds > 1 and int($seconds) % 60 == 0 )
unknown's avatar
unknown committed
1061
    {
unknown's avatar
unknown committed
1062 1063
      my $left= $timeout - $seconds;
      mtr_warning("Waited $seconds seconds for $pidfile to be created, " .
unknown's avatar
unknown committed
1064
                  "still waiting for $left seconds...");
unknown's avatar
unknown committed
1065 1066
    }

unknown's avatar
unknown committed
1067 1068
    # Millisceond sleep emulated with select
    select(undef, undef, undef, ($sleeptime/1000));
unknown's avatar
unknown committed
1069 1070
  }

unknown's avatar
unknown committed
1071
  return 0;
unknown's avatar
unknown committed
1072 1073 1074
}


1075 1076 1077
sub mtr_kill_processes ($) {
  my $pids = shift;

unknown's avatar
unknown committed
1078 1079 1080
  mtr_verbose("mtr_kill_processes " . join(" ", @$pids));

  foreach my $pid (@$pids)
1081
  {
unknown's avatar
unknown committed
1082
    foreach my $sig (15, 9)
1083
    {
unknown's avatar
unknown committed
1084
      last if mtr_im_kill_process([ $pid ], $sig, 10, 1);
1085 1086 1087 1088
    }
  }
}

unknown's avatar
unknown committed
1089

unknown's avatar
unknown committed
1090 1091 1092 1093 1094 1095
##############################################################################
#
#  When we exit, we kill off all children
#
##############################################################################

unknown's avatar
unknown committed
1096 1097 1098
# FIXME something is wrong, we sometimes terminate with "Hangup" written
# to tty, and no STDERR output telling us why.

unknown's avatar
unknown committed
1099
# FIXME for some reason, setting HUP to 'IGNORE' will cause exit() to
unknown's avatar
unknown committed
1100 1101
# write out "Hangup", and maybe loose some output. We insert a sleep...

unknown's avatar
unknown committed
1102 1103
sub mtr_exit ($) {
  my $code= shift;
unknown's avatar
unknown committed
1104
#  cluck("Called mtr_exit()");
1105
  mtr_timer_stop_all($::glob_timers);
unknown's avatar
unknown committed
1106
  local $SIG{HUP} = 'IGNORE';
unknown's avatar
unknown committed
1107 1108 1109 1110 1111 1112 1113 1114 1115
  # ToDo: Signalling -$$ will only work if we are the process group
  # leader (in fact on QNX it will signal our session group leader,
  # which might be Do-compile or Pushbuild, causing tests to be
  # aborted). So we only do it if we are the group leader. We might
  # set ourselves as the group leader at startup (with
  # POSIX::setpgrp(0,0)), but then care must be needed to always do
  # proper child process cleanup.
  kill('HUP', -$$) if !$::glob_win32_perl and $$ == getpgrp();

unknown's avatar
unknown committed
1116 1117 1118
  exit($code);
}

unknown's avatar
unknown committed
1119 1120
###########################################################################

unknown's avatar
unknown committed
1121
1;