mysqlhotcopy.sh 27 KB
Newer Older
1
#!@PERL@ -w
bk@work.mysql.com's avatar
bk@work.mysql.com committed
2 3 4 5 6 7 8

use strict;
use Getopt::Long;
use Data::Dumper;
use File::Basename;
use File::Path;
use DBI;
9
use Sys::Hostname;
bk@work.mysql.com's avatar
bk@work.mysql.com committed
10 11 12

=head1 NAME

13
mysqlhotcopy - fast on-line hot-backup utility for local MySQL databases and tables
bk@work.mysql.com's avatar
bk@work.mysql.com committed
14 15 16 17 18 19 20 21 22

=head1 SYNOPSIS

  mysqlhotcopy db_name

  mysqlhotcopy --suffix=_copy db_name_1 ... db_name_n

  mysqlhotcopy db_name_1 ... db_name_n /path/to/new_directory

monty@donna.mysql.com's avatar
monty@donna.mysql.com committed
23 24 25 26 27 28 29 30 31 32 33
  mysqlhotcopy db_name./regex/

  mysqlhotcopy db_name./^\(foo\|bar\)/

  mysqlhotcopy db_name./~regex/

  mysqlhotcopy db_name_1./regex_1/ db_name_1./regex_2/ ... db_name_n./regex_n/ /path/to/new_directory

  mysqlhotcopy --method='scp -Bq -i /usr/home/foo/.ssh/identity' --user=root --password=secretpassword \
         db_1./^nice_table/ user@some.system.dom:~/path/to/new_directory

34
WARNING: THIS PROGRAM IS STILL IN BETA. Comments/patches welcome.
bk@work.mysql.com's avatar
bk@work.mysql.com committed
35 36 37 38 39

=cut

# Documentation continued at end of file

40
my $VERSION = "1.16";
41 42

my $opt_tmpdir = $ENV{TMPDIR} || "/tmp";
bk@work.mysql.com's avatar
bk@work.mysql.com committed
43 44 45

my $OPTIONS = <<"_OPTIONS";

46 47
$0 Ver $VERSION

48
Usage: $0 db_name[./table_regex/] [new_db_name | directory]
bk@work.mysql.com's avatar
bk@work.mysql.com committed
49 50 51 52 53 54 55

  -?, --help           display this helpscreen and exit
  -u, --user=#         user for database login if not current user
  -p, --password=#     password to use when connecting to server
  -P, --port=#         port to use when connecting to local server
  -S, --socket=#       socket to use when connecting to local server

56 57 58
  --allowold           don\'t abort if target already exists (rename it _old)
  --keepold            don\'t delete previous (now renamed) target when done
  --noindices          don\'t include full index files in copy
bk@work.mysql.com's avatar
bk@work.mysql.com committed
59 60 61 62 63 64 65 66 67 68
  --method=#           method for copy (only "cp" currently supported)

  -q, --quiet          be silent except for errors
  --debug              enable debug
  -n, --dryrun         report actions without doing them

  --regexp=#           copy all databases with names matching regexp
  --suffix=#           suffix for names of copied databases
  --checkpoint=#       insert checkpoint entry into specified db.table
  --flushlog           flush logs once all tables are locked 
monty@tik.mysql.fi's avatar
monty@tik.mysql.fi committed
69 70
  --resetmaster        reset the binlog once all tables are locked
  --resetslave         reset the master.info once all tables are locked
71
  --tmpdir=#	       temporary directory (instead of $opt_tmpdir)
72
  --record_log_pos=#   record slave and master status in specified db.table
monty@donna.mysql.com's avatar
monty@donna.mysql.com committed
73

74
  Try \'perldoc $0 for more complete documentation\'
bk@work.mysql.com's avatar
bk@work.mysql.com committed
75 76 77 78 79 80 81
_OPTIONS

sub usage {
    die @_, $OPTIONS;
}

my %opt = (
82
    user	=> scalar getpwuid($>),
83
    noindices	=> 0,
bk@work.mysql.com's avatar
bk@work.mysql.com committed
84 85 86 87 88 89 90 91 92 93 94 95 96 97
    allowold	=> 0,	# for safety
    keepold	=> 0,
    method	=> "cp",
    flushlog    => 0,
);
Getopt::Long::Configure(qw(no_ignore_case)); # disambuguate -p and -P
GetOptions( \%opt,
    "help",
    "user|u=s",
    "password|p=s",
    "port|P=s",
    "socket|S=s",
    "allowold!",
    "keepold!",
98
    "noindices!",
bk@work.mysql.com's avatar
bk@work.mysql.com committed
99 100 101 102 103 104 105
    "method=s",
    "debug",
    "quiet|q",
    "mv!",
    "regexp=s",
    "suffix=s",
    "checkpoint=s",
106
    "record_log_pos=s",
bk@work.mysql.com's avatar
bk@work.mysql.com committed
107
    "flushlog",
monty@tik.mysql.fi's avatar
monty@tik.mysql.fi committed
108 109
    "resetmaster",
    "resetslave",
110
    "tmpdir|t=s",
bk@work.mysql.com's avatar
bk@work.mysql.com committed
111 112 113 114 115 116 117
    "dryrun|n",
) or usage("Invalid option");

# @db_desc
# ==========
# a list of hash-refs containing:
#
monty@donna.mysql.com's avatar
monty@donna.mysql.com committed
118 119 120 121 122
#   'src'     - name of the db to copy
#   't_regex' - regex describing tables in src
#   'target'  - destination directory of the copy
#   'tables'  - array-ref to list of tables in the db
#   'files'   - array-ref to list of files to be copied
123
#               (RAID files look like 'nn/name.MYD')
124
#   'index'   - array-ref to list of indexes to be copied
bk@work.mysql.com's avatar
bk@work.mysql.com committed
125 126 127 128
#

my @db_desc = ();
my $tgt_name = undef;
monty@donna.mysql.com's avatar
monty@donna.mysql.com committed
129

130 131
usage("") if ($opt{help});

bk@work.mysql.com's avatar
bk@work.mysql.com committed
132 133
if ( $opt{regexp} || $opt{suffix} || @ARGV > 2 ) {
    $tgt_name   = pop @ARGV unless ( exists $opt{suffix} );
monty@donna.mysql.com's avatar
monty@donna.mysql.com committed
134
    @db_desc = map { s{^([^\.]+)\./(.+)/$}{$1}; { 'src' => $_, 't_regex' => ( $2 ? $2 : '.*' ) } } @ARGV;
bk@work.mysql.com's avatar
bk@work.mysql.com committed
135 136 137 138
}
else {
    usage("Database name to hotcopy not specified") unless ( @ARGV );

monty@donna.mysql.com's avatar
monty@donna.mysql.com committed
139 140 141
    $ARGV[0] =~ s{^([^\.]+)\./(.+)/$}{$1};
    @db_desc = ( { 'src' => $ARGV[0], 't_regex' => ( $2 ? $2 : '.*' ) } );

bk@work.mysql.com's avatar
bk@work.mysql.com committed
142 143 144 145 146 147 148 149 150 151
    if ( @ARGV == 2 ) {
	$tgt_name   = $ARGV[1];
    }
    else {
	$opt{suffix} = "_copy";
    }
}

my %mysqld_vars;
my $start_time = time;
152
$opt_tmpdir= $opt{tmpdir} if $opt{tmpdir};
bk@work.mysql.com's avatar
bk@work.mysql.com committed
153 154 155 156 157 158 159 160 161
$0 = $1 if $0 =~ m:/([^/]+)$:;
$opt{quiet} = 0 if $opt{debug};
$opt{allowold} = 1 if $opt{keepold};

# --- connect to the database ---
my $dsn = ";host=localhost";
$dsn .= ";port=$opt{port}" if $opt{port};
$dsn .= ";mysql_socket=$opt{socket}" if $opt{socket};

monty@donna.mysql.com's avatar
monty@donna.mysql.com committed
162 163 164
my $dbh = DBI->connect("dbi:mysql:$dsn;mysql_read_default_group=mysqlhotcopy",
                        $opt{user}, $opt{password},
{
bk@work.mysql.com's avatar
bk@work.mysql.com committed
165 166 167 168 169 170 171 172 173 174 175 176 177 178 179
    RaiseError => 1,
    PrintError => 0,
    AutoCommit => 1,
});

# --- check that checkpoint table exists if specified ---
if ( $opt{checkpoint} ) {
    eval { $dbh->do( qq{ select time_stamp, src, dest, msg 
			 from $opt{checkpoint} where 1 != 1} );
       };

    die "Error accessing Checkpoint table ($opt{checkpoint}): $@"
      if ( $@ );
}

180 181 182 183 184 185 186 187 188 189
# --- check that log_pos table exists if specified ---
if ( $opt{record_log_pos} ) {
    eval { $dbh->do( qq{ select host, time_stamp, log_file, log_pos, master_host, master_log_file, master_log_pos
			 from $opt{record_log_pos} where 1 != 1} );
       };

    die "Error accessing log_pos table ($opt{record_log_pos}): $@"
      if ( $@ );
}

bk@work.mysql.com's avatar
bk@work.mysql.com committed
190
# --- get variables from database ---
monty@donna.mysql.com's avatar
monty@donna.mysql.com committed
191
my $sth_vars = $dbh->prepare("show variables like 'datadir'");
bk@work.mysql.com's avatar
bk@work.mysql.com committed
192 193 194 195
$sth_vars->execute;
while ( my ($var,$value) = $sth_vars->fetchrow_array ) {
    $mysqld_vars{ $var } = $value;
}
monty@donna.mysql.com's avatar
monty@donna.mysql.com committed
196
my $datadir = $mysqld_vars{'datadir'}
bk@work.mysql.com's avatar
bk@work.mysql.com committed
197 198 199 200 201
    || die "datadir not in mysqld variables";
$datadir =~ s:/$::;


# --- get target path ---
monty@donna.mysql.com's avatar
monty@donna.mysql.com committed
202 203
my ($tgt_dirname, $to_other_database);
$to_other_database=0;
204
if (defined($tgt_name) && $tgt_name =~ m:^\w+$: && @db_desc <= 1)
monty@donna.mysql.com's avatar
monty@donna.mysql.com committed
205
{
bk@work.mysql.com's avatar
bk@work.mysql.com committed
206
    $tgt_dirname = "$datadir/$tgt_name";
monty@donna.mysql.com's avatar
monty@donna.mysql.com committed
207
    $to_other_database=1;
bk@work.mysql.com's avatar
bk@work.mysql.com committed
208
}
209
elsif (defined($tgt_name) && ($tgt_name =~ m:/: || $tgt_name eq '.')) {
bk@work.mysql.com's avatar
bk@work.mysql.com committed
210 211 212
    $tgt_dirname = $tgt_name;
}
elsif ( $opt{suffix} ) {
213
    print "Using copy suffix '$opt{suffix}'\n" unless $opt{quiet};
bk@work.mysql.com's avatar
bk@work.mysql.com committed
214
}
215 216 217 218
else
{
  $tgt_name="" if (!defined($tgt_name));
  die "Target '$tgt_name' doesn't look like a database name or directory path.\n";
bk@work.mysql.com's avatar
bk@work.mysql.com committed
219 220 221 222 223 224 225 226 227 228 229 230 231 232
}

# --- resolve database names from regexp ---
if ( defined $opt{regexp} ) {
    my $sth_dbs = $dbh->prepare("show databases");
    $sth_dbs->execute;
    while ( my ($db_name) = $sth_dbs->fetchrow_array ) {
	push @db_desc, { 'src' => $db_name } if ( $db_name =~ m/$opt{regexp}/o );
    }
}

# --- get list of tables to hotcopy ---

my $hc_locks = "";
233
my $hc_tables = "";
bk@work.mysql.com's avatar
bk@work.mysql.com committed
234 235 236 237 238
my $num_tables = 0;
my $num_files = 0;

foreach my $rdb ( @db_desc ) {
    my $db = $rdb->{src};
239
    my @dbh_tables = get_list_of_tables( $db );
bk@work.mysql.com's avatar
bk@work.mysql.com committed
240

monty@donna.mysql.com's avatar
monty@donna.mysql.com committed
241
    ## generate regex for tables/files
242 243 244 245 246 247 248 249 250 251 252 253 254 255 256 257 258 259
    my $t_regex;
    my $negated;
    if ($rdb->{t_regex}) {
        $t_regex = $rdb->{t_regex};        ## assign temporary regex
        $negated = $t_regex =~ tr/~//d;    ## remove and count
                                           ## negation operator: we
                                           ## don't allow ~ in table
                                           ## names

        $t_regex = qr/$t_regex/;           ## make regex string from
                                           ## user regex

        ## filter (out) tables specified in t_regex
        print "Filtering tables with '$t_regex'\n" if $opt{debug};
        @dbh_tables = ( $negated 
                        ? grep { $_ !~ $t_regex } @dbh_tables
                        : grep { $_ =~ $t_regex } @dbh_tables );
    }
monty@donna.mysql.com's avatar
monty@donna.mysql.com committed
260 261

    ## get list of files to copy
bk@work.mysql.com's avatar
bk@work.mysql.com committed
262 263 264 265
    my $db_dir = "$datadir/$db";
    opendir(DBDIR, $db_dir ) 
      or die "Cannot open dir '$db_dir': $!";

monty@donna.mysql.com's avatar
monty@donna.mysql.com committed
266
    my %db_files;
267 268 269 270 271 272 273 274 275 276 277 278 279 280
    my @raid_dir = ();

    while ( defined( my $name = readdir DBDIR ) ) {
	if ( $name =~ /^\d\d$/ && -d "$db_dir/$name" ) {
	    push @raid_dir, $name;
	}
	else {
	    $db_files{$name} = $1 if ( $name =~ /(.+)\.\w+$/ );
        }
    }
    closedir( DBDIR );

    scan_raid_dir( \%db_files, $db_dir, @raid_dir );

monty@donna.mysql.com's avatar
monty@donna.mysql.com committed
281 282 283
    unless( keys %db_files ) {
	warn "'$db' is an empty database\n";
    }
bk@work.mysql.com's avatar
bk@work.mysql.com committed
284

monty@donna.mysql.com's avatar
monty@donna.mysql.com committed
285
    ## filter (out) files specified in t_regex
286 287 288 289 290 291 292 293 294 295
    my @db_files;
    if ($rdb->{t_regex}) {
        @db_files = ($negated
                     ? grep { $db_files{$_} !~ $t_regex } keys %db_files
                     : grep { $db_files{$_} =~ $t_regex } keys %db_files );
    }
    else {
        @db_files = keys %db_files;
    }

296
    @db_files = sort @db_files;
297

298
    my @index_files=();
monty@donna.mysql.com's avatar
monty@donna.mysql.com committed
299 300

    ## remove indices unless we're told to keep them
301 302
    if ($opt{noindices}) {
        @index_files= grep { /\.(ISM|MYI)$/ } @db_files;
bk@work.mysql.com's avatar
bk@work.mysql.com committed
303 304 305 306
	@db_files = grep { not /\.(ISM|MYI)$/ } @db_files;
    }

    $rdb->{files}  = [ @db_files ];
307
    $rdb->{index}  = [ @index_files ];
308
    my @hc_tables = map { "`$db`.`$_`" } @dbh_tables;
bk@work.mysql.com's avatar
bk@work.mysql.com committed
309 310
    $rdb->{tables} = [ @hc_tables ];

311 312
    $rdb->{raid_dirs} = [ get_raid_dirs( $rdb->{files} ) ];

bk@work.mysql.com's avatar
bk@work.mysql.com committed
313 314
    $hc_locks .= ", "  if ( length $hc_locks && @hc_tables );
    $hc_locks .= join ", ", map { "$_ READ" } @hc_tables;
monty@donna.mysql.com's avatar
monty@donna.mysql.com committed
315
    $hc_tables .= ", "  if ( length $hc_tables && @hc_tables );
316
    $hc_tables .= join ", ", @hc_tables;
bk@work.mysql.com's avatar
bk@work.mysql.com committed
317 318 319 320 321 322 323

    $num_tables += scalar @hc_tables;
    $num_files  += scalar @{$rdb->{files}};
}

# --- resolve targets for copies ---

324
if (defined($tgt_name) && length $tgt_name ) {
bk@work.mysql.com's avatar
bk@work.mysql.com committed
325 326 327
    # explicit destination directory specified

    # GNU `cp -r` error message
monty@donna.mysql.com's avatar
monty@donna.mysql.com committed
328 329 330 331 332 333 334 335
    die "copying multiple databases, but last argument ($tgt_dirname) is not a directory\n"
      if ( @db_desc > 1 && !(-e $tgt_dirname && -d $tgt_dirname ) );

    if ($to_other_database)
    {
      foreach my $rdb ( @db_desc ) {
	$rdb->{target} = "$tgt_dirname";
      }
bk@work.mysql.com's avatar
bk@work.mysql.com committed
336
    }
monty@donna.mysql.com's avatar
monty@donna.mysql.com committed
337 338 339 340 341 342
    elsif ($opt{method} =~ /^scp\b/) 
    {   # we have to trust scp to hit the target
	foreach my $rdb ( @db_desc ) {
	    $rdb->{target} = "$tgt_dirname/$rdb->{src}";
	}
    }
monty@donna.mysql.com's avatar
monty@donna.mysql.com committed
343 344 345 346 347 348 349
    else
    {
      die "Last argument ($tgt_dirname) is not a directory\n"
	if (!(-e $tgt_dirname && -d $tgt_dirname ) );
      foreach my $rdb ( @db_desc ) {
	$rdb->{target} = "$tgt_dirname/$rdb->{src}";
      }
bk@work.mysql.com's avatar
bk@work.mysql.com committed
350
    }
monty@donna.mysql.com's avatar
monty@donna.mysql.com committed
351 352 353
  }
else {
  die "Error: expected \$opt{suffix} to exist" unless ( exists $opt{suffix} );
monty@donna.mysql.com's avatar
monty@donna.mysql.com committed
354

monty@donna.mysql.com's avatar
monty@donna.mysql.com committed
355 356 357
  foreach my $rdb ( @db_desc ) {
    $rdb->{target} = "$datadir/$rdb->{src}$opt{suffix}";
  }
bk@work.mysql.com's avatar
bk@work.mysql.com committed
358 359 360 361 362 363 364 365
}

print Dumper( \@db_desc ) if ( $opt{debug} );

# --- bail out if all specified databases are empty ---

die "No tables to hot-copy" unless ( length $hc_locks );

366
# --- create target directories if we are using 'cp' ---
bk@work.mysql.com's avatar
bk@work.mysql.com committed
367 368

my @existing = ();
369 370 371 372

if ($opt{method} =~ /^cp\b/)
{
  foreach my $rdb ( @db_desc ) {
bk@work.mysql.com's avatar
bk@work.mysql.com committed
373
    push @existing, $rdb->{target} if ( -d  $rdb->{target} );
374
  }
bk@work.mysql.com's avatar
bk@work.mysql.com committed
375

376 377 378 379 380
  if ( @existing && !$opt{allowold} )
  {
    $dbh->disconnect();
    die "Can't hotcopy to '", join( "','", @existing ), "' because directory\nalready exist and the --allowold option was not given.\n"
  }
381
}
bk@work.mysql.com's avatar
bk@work.mysql.com committed
382 383 384 385

retire_directory( @existing ) if ( @existing );

foreach my $rdb ( @db_desc ) {
386 387 388 389 390 391 392 393 394 395 396 397 398 399
    foreach my $td ( '', @{$rdb->{raid_dirs}} ) {

	my $tgt_dirpath = "$rdb->{target}/$td";
	if ( $opt{dryrun} ) {
	    print "mkdir $tgt_dirpath, 0750\n";
	}
	elsif ($opt{method} =~ /^scp\b/) {
	    ## assume it's there?
	    ## ...
	}
	else {
	    mkdir($tgt_dirpath, 0750)
		or die "Can't create '$tgt_dirpath': $!\n";
	}
bk@work.mysql.com's avatar
bk@work.mysql.com committed
400 401 402 403 404 405 406 407 408 409 410 411 412
    }
}

##############################
# --- PERFORM THE HOT-COPY ---
#
# Note that we try to keep the time between the LOCK and the UNLOCK
# as short as possible, and only start when we know that we should
# be able to complete without error.

# read lock all the tables we'll be copying
# in order to get a consistent snapshot of the database

413 414 415 416 417 418
if ( $opt{checkpoint} || $opt{record_log_pos} ) {
  # convert existing READ lock on checkpoint and/or log_pos table into WRITE lock
  foreach my $table ( grep { defined } ( $opt{checkpoint}, $opt{record_log_pos} ) ) {
    $hc_locks .= ", $table WRITE" 
	unless ( $hc_locks =~ s/$table\s+READ/$table WRITE/ );
  }
bk@work.mysql.com's avatar
bk@work.mysql.com committed
419 420 421 422 423 424
}

my $hc_started = time;	# count from time lock is granted

if ( $opt{dryrun} ) {
    print "LOCK TABLES $hc_locks\n";
425
    print "FLUSH TABLES /*!32323 $hc_tables */\n";
bk@work.mysql.com's avatar
bk@work.mysql.com committed
426
    print "FLUSH LOGS\n" if ( $opt{flushlog} );
monty@tik.mysql.fi's avatar
monty@tik.mysql.fi committed
427 428
    print "RESET MASTER\n" if ( $opt{resetmaster} );
    print "RESET SLAVE\n" if ( $opt{resetslave} );
bk@work.mysql.com's avatar
bk@work.mysql.com committed
429 430 431 432 433 434 435 436 437
}
else {
    my $start = time;
    $dbh->do("LOCK TABLES $hc_locks");
    printf "Locked $num_tables tables in %d seconds.\n", time-$start unless $opt{quiet};
    $hc_started = time;	# count from time lock is granted

    # flush tables to make on-disk copy uptodate
    $start = time;
438 439
    $dbh->do("FLUSH TABLES /*!32323 $hc_tables */");
    printf "Flushed tables ($hc_tables) in %d seconds.\n", time-$start unless $opt{quiet};
bk@work.mysql.com's avatar
bk@work.mysql.com committed
440
    $dbh->do( "FLUSH LOGS" ) if ( $opt{flushlog} );
monty@tik.mysql.fi's avatar
monty@tik.mysql.fi committed
441 442
    $dbh->do( "RESET MASTER" ) if ( $opt{resetmaster} );
    $dbh->do( "RESET SLAVE" ) if ( $opt{resetslave} );
443 444 445 446 447

    if ( $opt{record_log_pos} ) {
	record_log_pos( $dbh, $opt{record_log_pos} );
	$dbh->do("FLUSH TABLES /*!32323 $hc_tables */");
    }
bk@work.mysql.com's avatar
bk@work.mysql.com committed
448
}
monty@donna.mysql.com's avatar
monty@donna.mysql.com committed
449

bk@work.mysql.com's avatar
bk@work.mysql.com committed
450 451
my @failed = ();

452 453 454 455 456
foreach my $rdb ( @db_desc )
{
  my @files = map { "$datadir/$rdb->{src}/$_" } @{$rdb->{files}};
  next unless @files;
  
457
  eval { copy_files($opt{method}, \@files, $rdb->{target}, $rdb->{raid_dirs} ); };
458 459 460
  push @failed, "$rdb->{src} -> $rdb->{target} failed: $@"
    if ( $@ );
  
461
  @files = @{$rdb->{index}};
462 463
  if ($rdb->{index})
  {
464 465
    copy_index($opt{method}, \@files,
	       "$datadir/$rdb->{src}", $rdb->{target} );
466 467 468 469 470 471 472 473 474 475 476 477 478
  }
  
  if ( $opt{checkpoint} ) {
    my $msg = ( $@ ) ? "Failed: $@" : "Succeeded";
    
    eval {
      $dbh->do( qq{ insert into $opt{checkpoint} (src, dest, msg) 
		      VALUES ( '$rdb->{src}', '$rdb->{target}', '$msg' )
		    } ); 
    };
    
    if ( $@ ) {
      warn "Failed to update checkpoint table: $@\n";
bk@work.mysql.com's avatar
bk@work.mysql.com committed
479
    }
480
  }
bk@work.mysql.com's avatar
bk@work.mysql.com committed
481 482 483 484 485 486 487 488 489 490 491 492 493 494 495 496 497 498 499 500 501 502 503 504 505 506 507 508 509
}

if ( $opt{dryrun} ) {
    print "UNLOCK TABLES\n";
    if ( @existing && !$opt{keepold} ) {
	my @oldies = map { $_ . '_old' } @existing;
	print "rm -rf @oldies\n" 
    }
    $dbh->disconnect();
    exit(0);
}
else {
    $dbh->do("UNLOCK TABLES");
}

my $hc_dur = time - $hc_started;
printf "Unlocked tables.\n" unless $opt{quiet};

#
# --- HOT-COPY COMPLETE ---
###########################

$dbh->disconnect;

if ( @failed ) {
    # hotcopy failed - cleanup
    # delete any @targets 
    # rename _old copy back to original

510 511 512 513 514 515
    my @targets = ();
    foreach my $rdb ( @db_desc ) {
        push @targets, $rdb->{target} if ( -d  $rdb->{target} );
    }
    print "Deleting @targets \n" if $opt{debug};

bk@work.mysql.com's avatar
bk@work.mysql.com committed
516 517 518 519 520 521 522 523 524 525 526 527 528 529 530 531 532 533 534 535 536 537 538 539 540 541 542 543 544 545 546 547 548 549
    print "Deleting @targets \n" if $opt{debug};
    rmtree([@targets]);
    if (@existing) {
	print "Restoring @existing from back-up\n" if $opt{debug};
        foreach my $dir ( @existing ) {
	    rename("${dir}_old", $dir )
	      or warn "Can't rename ${dir}_old to $dir: $!\n";
	}
    }

    die join( "\n", @failed );
}
else {
    # hotcopy worked
    # delete _old unless $opt{keepold}

    if ( @existing && !$opt{keepold} ) {
	my @oldies = map { $_ . '_old' } @existing;
	print "Deleting previous copy in @oldies\n" if $opt{debug};
	rmtree([@oldies]);
    }

    printf "$0 copied %d tables (%d files) in %d second%s (%d seconds overall).\n",
	    $num_tables, $num_files,
	    $hc_dur, ($hc_dur==1)?"":"s", time - $start_time
	unless $opt{quiet};
}

exit 0;


# ---

sub copy_files {
550
    my ($method, $files, $target, $raid_dirs) = @_;
bk@work.mysql.com's avatar
bk@work.mysql.com committed
551 552
    my @cmd;
    print "Copying ".@$files." files...\n" unless $opt{quiet};
monty@donna.mysql.com's avatar
monty@donna.mysql.com committed
553

bk@work.mysql.com's avatar
bk@work.mysql.com committed
554
    if ($method =~ /^s?cp\b/) { # cp or scp with optional flags
555
	my @cp = ($method);
bk@work.mysql.com's avatar
bk@work.mysql.com committed
556 557
	# add option to preserve mod time etc of copied files
	# not critical, but nice to have
558
	push @cp, "-p" if $^O =~ m/^(solaris|linux|freebsd)$/;
monty@donna.mysql.com's avatar
monty@donna.mysql.com committed
559 560

	# add recursive option for scp
561 562
	push @cp, "-r" if $^O =~ /m^(solaris|linux|freebsd)$/ && $method =~ /^scp\b/;

563
	my @non_raid = map { "'$_'" } grep { ! m:/\d{2}/[^/]+$: } @$files;
monty@donna.mysql.com's avatar
monty@donna.mysql.com committed
564

bk@work.mysql.com's avatar
bk@work.mysql.com committed
565
	# add files to copy and the destination directory
566
+ 	safe_system( @cp, @non_raid, "'$target'" );
567 568
	
	foreach my $rd ( @$raid_dirs ) {
569 570
	    my @raid = map { "'$_'" } grep { m:$rd/: } @$files;
	    safe_system( @cp, @raid, "'$target'/$rd" ) if ( @raid );
571
	}
bk@work.mysql.com's avatar
bk@work.mysql.com committed
572
    }
573 574
    else
    {
bk@work.mysql.com's avatar
bk@work.mysql.com committed
575 576
	die "Can't use unsupported method '$method'\n";
    }
577
}
bk@work.mysql.com's avatar
bk@work.mysql.com committed
578

579 580 581 582 583 584 585 586 587 588 589 590 591 592 593 594 595 596 597 598 599 600 601 602 603 604 605 606 607 608 609 610 611 612 613 614 615 616 617 618 619 620 621 622 623 624 625 626 627 628 629 630 631
#
# Copy only the header of the index file
#

sub copy_index
{
  my ($method, $files, $source, $target) = @_;
  my $tmpfile="$opt_tmpdir/mysqlhotcopy$$";
  
  print "Copying indices for ".@$files." files...\n" unless $opt{quiet};  
  foreach my $file (@$files)
  {
    my $from="$source/$file";
    my $to="$target/$file";
    my $buff;
    open(INPUT, "<$from") || die "Can't open file $from: $!\n";
    my $length=read INPUT, $buff, 2048;
    die "Can't read index header from $from\n" if ($length < 1024);
    close INPUT;
    
    if ( $opt{dryrun} )
    {
      print "$opt{method}-header $from $to\n";
    }
    elsif ($opt{method} eq 'cp')
    {
      open(OUTPUT,">$to")   || die "Can\'t create file $to: $!\n";
      if (syswrite(OUTPUT,$buff) != length($buff))
      {
	die "Error when writing data to $to: $!\n";
      }
      close OUTPUT	   || die "Error on close of $to: $!\n";
    }
    elsif ($opt{method} eq 'scp')
    {
      my $tmp=$tmpfile;
      open(OUTPUT,">$tmp") || die "Can\'t create file $tmp: $!\n";
      if (syswrite(OUTPUT,$buff) != length($buff))
      {
	die "Error when writing data to $tmp: $!\n";
      }
      close OUTPUT	     || die "Error on close of $tmp: $!\n";
      safe_system("scp $tmp $to");
    }
    else
    {
      die "Can't use unsupported method '$opt{method}'\n";
    }
  }
  unlink "$tmpfile" if  ($opt{method} eq 'scp');
}


632 633
sub safe_system
{
634
  my @cmd= @_;
bk@work.mysql.com's avatar
bk@work.mysql.com committed
635

636 637 638 639 640 641 642 643
  if ( $opt{dryrun} )
  {
    print "@cmd\n";
    return;
  }

  ## for some reason system fails but backticks works ok for scp...
  print "Executing '@cmd'\n" if $opt{debug};
644
  my $cp_status = system "@cmd > /dev/null";
645 646 647 648 649
  if ($cp_status != 0) {
    warn "Burp ('scuse me). Trying backtick execution...\n" if $opt{debug}; #'
    ## try something else
    `@cmd` && die "Error: @cmd failed ($cp_status) while copying files.\n";
  }
bk@work.mysql.com's avatar
bk@work.mysql.com committed
650 651 652 653 654 655 656 657 658 659 660 661 662 663 664 665 666 667 668 669 670 671 672
}

sub retire_directory {
    my ( @dir ) = @_;

    foreach my $dir ( @dir ) {
	my $tgt_oldpath = $dir . '_old';
	if ( $opt{dryrun} ) {
	    print "rmtree $tgt_oldpath\n" if ( -d $tgt_oldpath );
	    print "rename $dir, $tgt_oldpath\n";
	    next;
	}

	if ( -d $tgt_oldpath ) {
	    print "Deleting previous 'old' hotcopy directory ('$tgt_oldpath')\n" unless $opt{quiet};
	    rmtree([$tgt_oldpath])
	}
	rename($dir, $tgt_oldpath)
	  or die "Can't rename $dir=>$tgt_oldpath: $!\n";
	print "Existing hotcopy directory renamed to '$tgt_oldpath'\n" unless $opt{quiet};
    }
}

673 674 675 676 677 678 679 680 681 682 683 684 685 686 687 688 689 690 691 692 693 694 695 696 697 698 699 700 701 702 703 704 705 706
sub record_log_pos {
    my ( $dbh, $table_name ) = @_;

    eval {
	my ($file,$position) = get_row( $dbh, "show master status" );
	die "master status is undefined" if !defined $file || !defined $position;
	
	my ($master_host, undef, undef, undef, $log_file, $log_pos ) 
	    = get_row( $dbh, "show slave status" );
	
	my $hostname = hostname();
	
	$dbh->do( qq{ replace into $table_name 
			  set host=?, log_file=?, log_pos=?, 
                          master_host=?, master_log_file=?, master_log_pos=? }, 
		  undef, 
		  $hostname, $file, $position, 
		  $master_host, $log_file, $log_pos  );
	
    };
    
    if ( $@ ) {
	warn "Failed to store master position: $@\n";
    }
}

sub get_row {
  my ( $dbh, $sql ) = @_;

  my $sth = $dbh->prepare($sql);
  $sth->execute;
  return $sth->fetchrow_array();
}

707 708 709 710 711 712 713 714 715 716 717 718 719 720 721 722 723 724 725 726 727 728 729 730 731 732 733 734 735
sub scan_raid_dir {
    my ( $r_db_files, $data_dir, @raid_dir ) = @_;

    local(*RAID_DIR);
    
    foreach my $rd ( @raid_dir ) {

	opendir(RAID_DIR, "$data_dir/$rd" ) 
	    or die "Cannot open dir '$data_dir/$rd': $!";

	while ( defined( my $name = readdir RAID_DIR ) ) {
	    $r_db_files->{"$rd/$name"} = $1 if ( $name =~ /(.+)\.\w+$/ );
	}
	closedir( RAID_DIR );
    }
}

sub get_raid_dirs {
    my ( $r_files ) = @_;

    my %dirs = ();
    foreach my $f ( @$r_files ) {
	if ( $f =~ m:^(\d\d)/: ) {
	    $dirs{$1} = 1;
	}
    }
    return sort keys %dirs;
}

736 737 738 739 740 741 742 743 744 745 746 747 748 749 750 751 752 753 754
sub get_list_of_tables {
    my ( $db ) = @_;

    # "use database" cannot cope with database names containing spaces
    # so create a new connection 

    my $dbh = DBI->connect("dbi:mysql:${db}${dsn};mysql_read_default_group=mysqlhotcopy",
			    $opt{user}, $opt{password},
    {
	RaiseError => 1,
	PrintError => 0,
	AutoCommit => 1,
    });

    my @dbh_tables = eval { $dbh->tables() };
    $dbh->disconnect();
    return @dbh_tables;
}

bk@work.mysql.com's avatar
bk@work.mysql.com committed
755 756 757 758 759 760 761 762 763 764 765 766 767 768 769 770 771 772 773 774 775 776 777 778 779 780 781 782 783 784 785 786 787 788
__END__

=head1 DESCRIPTION

mysqlhotcopy is designed to make stable copies of live MySQL databases.

Here "live" means that the database server is running and the database
may be in active use. And "stable" means that the copy will not have
any corruptions that could occur if the table files were simply copied
without first being locked and flushed from within the server.

=head1 OPTIONS

=over 4

=item --checkpoint checkpoint-table

As each database is copied, an entry is written to the specified
checkpoint-table.  This has the happy side-effect of updating the
MySQL update-log (if it is switched on) giving a good indication of
where roll-forward should begin for backup+rollforward schemes.

The name of the checkpoint table should be supplied in database.table format.
The checkpoint-table must contain at least the following fields:

=over 4

  time_stamp timestamp not null
  src varchar(32)
  dest varchar(60)
  msg varchar(255)

=back

789 790 791 792 793 794 795 796 797 798 799 800 801 802 803 804 805 806 807 808 809 810 811 812 813 814 815 816 817 818 819 820
=item --record_log_pos log-pos-table

Just before the database files are copied, update the record in the
log-pos-table from the values returned from "show master status" and
"show slave status". The master status values are stored in the
log_file and log_pos columns, and establish the position in the binary
logs that any slaves of this host should adopt if initialised from
this dump.  The slave status values are stored in master_host,
master_log_file, and master_log_pos, and these are useful if the host
performing the dump is a slave and other sibling slaves are to be
initialised from this dump.

The name of the log-pos table should be supplied in database.table format.
A sample log-pos table definition:

=over 4

CREATE TABLE log_pos (
  host            varchar(60) NOT null,
  time_stamp      timestamp(14) NOT NULL,
  log_file        varchar(32) default NULL,
  log_pos         int(11)     default NULL,
  master_host     varchar(60) NULL,
  master_log_file varchar(32) NULL,
  master_log_pos  int NULL,

  PRIMARY KEY  (host) 
);

=back


bk@work.mysql.com's avatar
bk@work.mysql.com committed
821 822 823 824 825 826 827 828 829 830 831 832 833 834 835 836 837 838 839 840 841 842 843 844 845 846 847 848 849
=item --suffix suffix

Each database is copied back into the originating datadir under
a new name. The new name is the original name with the suffix
appended. 

If only a single db_name is supplied and the --suffix flag is not
supplied, then "--suffix=_copy" is assumed.

=item --allowold

Move any existing version of the destination to a backup directory for
the duration of the copy. If the copy successfully completes, the backup 
directory is deleted - unless the --keepold flag is set.  If the copy fails,
the backup directory is restored.

The backup directory name is the original name with "_old" appended.
Any existing versions of the backup directory are deleted.

=item --keepold

Behaves as for the --allowold, with the additional feature 
of keeping the backup directory after the copy successfully completes.

=item --flushlog

Rotate the log files by executing "FLUSH LOGS" after all tables are
locked, and before they are copied.

monty@tik.mysql.fi's avatar
monty@tik.mysql.fi committed
850 851 852 853 854 855 856 857 858 859 860 861
=item --resetmaster

Reset the bin-log by executing "RESET MASTER" after all tables are
locked, and before they are copied. Usefull if you are recovering a
slave in a replication setup.

=item --resetslave

Reset the master.info by executing "RESET SLAVE" after all tables are
locked, and before they are copied. Usefull if you are recovering a
server in a mutual replication setup.

bk@work.mysql.com's avatar
bk@work.mysql.com committed
862 863
=item --regexp pattern

monty@donna.mysql.com's avatar
monty@donna.mysql.com committed
864 865 866 867 868 869 870 871 872 873 874 875 876 877 878 879
Copy all databases with names matching the pattern

=item db_name./pattern/

Copy only tables matching pattern. Shell metacharacters ( (, ), |, !,
etc.) have to be escaped (e.g. \). For example, to select all tables
in database db1 whose names begin with 'foo' or 'bar':

    mysqlhotcopy --indices --method=cp db1./^\(foo\|bar\)/

=item db_name./~pattern/

Copy only tables not matching pattern. For example, to copy tables
that do not begin with foo nor bar:

    mysqlhotcopy --indices --method=cp db1./~^\(foo\|bar\)/
bk@work.mysql.com's avatar
bk@work.mysql.com committed
880 881 882 883 884 885 886 887 888 889 890 891 892 893 894 895 896 897 898 899 900

=item -?, --help

Display helpscreen and exit

=item -u, --user=#         

user for database login if not current user

=item -p, --password=#     

password to use when connecting to server

=item -P, --port=#         

port to use when connecting to local server

=item -S, --socket=#         

UNIX domain socket to use when connecting to local server

901
=item  --noindices          
bk@work.mysql.com's avatar
bk@work.mysql.com committed
902

903 904 905
Don\'t include index files in copy. Only up to the first 2048 bytes
are copied;  You can restore the indexes with isamchk -r or myisamchk -r
on the backup.
bk@work.mysql.com's avatar
bk@work.mysql.com committed
906 907 908

=item  --method=#           

monty@donna.mysql.com's avatar
monty@donna.mysql.com committed
909 910 911 912 913
method for copy (only "cp" currently supported). Alpha support for
"scp" was added in November 2000. Your experience with the scp method
will vary with your ability to understand how scp works. 'man scp'
and 'man ssh' are your friends.

914 915 916 917
The destination directory _must exist_ on the target machine using the
scp method. --keepold and --allowold are meeningless with scp.
Liberal use of the --debug option will help you figure out what\'s
really going on when you do an scp.
monty@donna.mysql.com's avatar
monty@donna.mysql.com committed
918 919 920 921 922

Note that using scp will lock your tables for a _long_ time unless
your network connection is _fast_. If this is unacceptable to you,
use the 'cp' method to copy the tables to some temporary area and then
scp or rsync the files at your leisure.
bk@work.mysql.com's avatar
bk@work.mysql.com committed
923 924 925 926 927 928 929 930 931 932 933 934 935 936 937 938 939 940 941 942 943 944 945

=item -q, --quiet              

be silent except for errors

=item  --debug

Debug messages are displayed 

=item -n, --dryrun

Display commands without actually doing them

=back

=head1 WARRANTY

This software is free and comes without warranty of any kind. You
should never trust backup software without studying the code yourself.
Study the code inside this script and only rely on it if I<you> believe
that it does the right thing for you.

Patches adding bug fixes, documentation and new features are welcome.
946
Please send these to internals@lists.mysql.com.
bk@work.mysql.com's avatar
bk@work.mysql.com committed
947 948 949

=head1 TO DO

monty@donna.mysql.com's avatar
monty@donna.mysql.com committed
950 951
Extend the individual table copy to allow multiple subsets of tables
to be specified on the command line:
bk@work.mysql.com's avatar
bk@work.mysql.com committed
952 953 954 955 956 957 958

  mysqlhotcopy db newdb  t1 t2 /^foo_/ : t3 /^bar_/ : +

where ":" delimits the subsets, the /^foo_/ indicates all tables
with names begining with "foo_" and the "+" indicates all tables
not copied by the previous subsets.

monty@donna.mysql.com's avatar
monty@donna.mysql.com committed
959 960 961
newdb is either another not existing database or a full path to a directory
where we can create a directory 'db'

962
Add option to lock each table in turn for people who don\'t need
bk@work.mysql.com's avatar
bk@work.mysql.com committed
963 964 965 966 967 968 969 970 971 972 973 974 975
cross-table integrity.

Add option to FLUSH STATUS just before UNLOCK TABLES.

Add support for other copy methods (eg tar to single file?).

Add support for forthcoming MySQL ``RAID'' table subdirectory layouts.

=head1 AUTHOR

Tim Bunce

Martin Waite - added checkpoint, flushlog, regexp and dryrun options
976 977
               Fixed cleanup of targets when hotcopy fails. 
	       Added --record_log_pos.
978
               RAID tables are now copied (don't know if this works over scp).
bk@work.mysql.com's avatar
bk@work.mysql.com committed
979

monty@donna.mysql.com's avatar
monty@donna.mysql.com committed
980 981 982
Ralph Corderoy - added synonyms for commands

Scott Wiersdorf - added table regex and scp support
983 984

Monty - working --noindex (copy only first 2048 bytes of index file)
985
        Fixes for --method=scp
986 987

Ask Bjoern Hansen - Cleanup code to fix a few bugs and enable -w again.
monty@tik.mysql.fi's avatar
monty@tik.mysql.fi committed
988 989 990

Emil S. Hansen - Added resetslave and resetmaster.

991 992 993 994
Jeremy D. Zawodny - Removed depricated DBI calls.  Fixed bug which
resulted in nothing being copied when a regexp was specified but no
database name(s).

995
Martin Waite - Fix to handle database name that contains space.