mysqlhotcopy.sh 31 KB
Newer Older
1
#!@PERL@ -w
bk@work.mysql.com's avatar
bk@work.mysql.com committed
2 3 4 5 6 7 8

use strict;
use Getopt::Long;
use Data::Dumper;
use File::Basename;
use File::Path;
use DBI;
9
use Sys::Hostname;
bk@work.mysql.com's avatar
bk@work.mysql.com committed
10 11 12

=head1 NAME

13
mysqlhotcopy - fast on-line hot-backup utility for local MySQL databases and tables
bk@work.mysql.com's avatar
bk@work.mysql.com committed
14 15 16 17 18 19 20 21 22

=head1 SYNOPSIS

  mysqlhotcopy db_name

  mysqlhotcopy --suffix=_copy db_name_1 ... db_name_n

  mysqlhotcopy db_name_1 ... db_name_n /path/to/new_directory

monty@donna.mysql.com's avatar
monty@donna.mysql.com committed
23 24 25 26 27 28 29 30 31 32 33
  mysqlhotcopy db_name./regex/

  mysqlhotcopy db_name./^\(foo\|bar\)/

  mysqlhotcopy db_name./~regex/

  mysqlhotcopy db_name_1./regex_1/ db_name_1./regex_2/ ... db_name_n./regex_n/ /path/to/new_directory

  mysqlhotcopy --method='scp -Bq -i /usr/home/foo/.ssh/identity' --user=root --password=secretpassword \
         db_1./^nice_table/ user@some.system.dom:~/path/to/new_directory

34
WARNING: THIS PROGRAM IS STILL IN BETA. Comments/patches welcome.
bk@work.mysql.com's avatar
bk@work.mysql.com committed
35 36 37 38 39

=cut

# Documentation continued at end of file

40
my $VERSION = "1.20";
41 42

my $opt_tmpdir = $ENV{TMPDIR} || "/tmp";
bk@work.mysql.com's avatar
bk@work.mysql.com committed
43 44 45

my $OPTIONS = <<"_OPTIONS";

46 47
$0 Ver $VERSION

48
Usage: $0 db_name[./table_regex/] [new_db_name | directory]
bk@work.mysql.com's avatar
bk@work.mysql.com committed
49 50 51

  -?, --help           display this helpscreen and exit
  -u, --user=#         user for database login if not current user
52 53 54
  -p, --password=#     password to use when connecting to server (if not set
                       in my.cnf, which is recommended)
  -h, --host=#         Hostname for local server when connecting over TCP/IP
55
  -P, --port=#         port to use when connecting to local server with TCP/IP
bk@work.mysql.com's avatar
bk@work.mysql.com committed
56 57
  -S, --socket=#       socket to use when connecting to local server

58 59
  --allowold           don\'t abort if target dir already exists (rename it _old)
  --addtodest          don\'t rename target dir if it exists, just add files to it
60 61
  --keepold            don\'t delete previous (now renamed) target when done
  --noindices          don\'t include full index files in copy
bk@work.mysql.com's avatar
bk@work.mysql.com committed
62 63 64 65 66 67 68 69 70 71
  --method=#           method for copy (only "cp" currently supported)

  -q, --quiet          be silent except for errors
  --debug              enable debug
  -n, --dryrun         report actions without doing them

  --regexp=#           copy all databases with names matching regexp
  --suffix=#           suffix for names of copied databases
  --checkpoint=#       insert checkpoint entry into specified db.table
  --flushlog           flush logs once all tables are locked 
monty@tik.mysql.fi's avatar
monty@tik.mysql.fi committed
72 73
  --resetmaster        reset the binlog once all tables are locked
  --resetslave         reset the master.info once all tables are locked
74
  --tmpdir=#	       temporary directory (instead of $opt_tmpdir)
75
  --record_log_pos=#   record slave and master status in specified db.table
monty@donna.mysql.com's avatar
monty@donna.mysql.com committed
76

77
  Try \'perldoc $0 for more complete documentation\'
bk@work.mysql.com's avatar
bk@work.mysql.com committed
78 79 80 81 82 83
_OPTIONS

sub usage {
    die @_, $OPTIONS;
}

paul@teton.kitebird.com's avatar
paul@teton.kitebird.com committed
84 85 86
# Do not initialize user or password options; that way, any user/password
# options specified in option files will be used.  If no values are specified
# all, the defaults will be used (login name, no password).
87

bk@work.mysql.com's avatar
bk@work.mysql.com committed
88
my %opt = (
89
    noindices	=> 0,
bk@work.mysql.com's avatar
bk@work.mysql.com committed
90 91 92 93 94 95 96 97
    allowold	=> 0,	# for safety
    keepold	=> 0,
    method	=> "cp",
    flushlog    => 0,
);
Getopt::Long::Configure(qw(no_ignore_case)); # disambuguate -p and -P
GetOptions( \%opt,
    "help",
98
    "host|h=s",
bk@work.mysql.com's avatar
bk@work.mysql.com committed
99 100 101 102 103 104
    "user|u=s",
    "password|p=s",
    "port|P=s",
    "socket|S=s",
    "allowold!",
    "keepold!",
105
    "addtodest!",
106
    "noindices!",
bk@work.mysql.com's avatar
bk@work.mysql.com committed
107 108 109 110 111 112 113
    "method=s",
    "debug",
    "quiet|q",
    "mv!",
    "regexp=s",
    "suffix=s",
    "checkpoint=s",
114
    "record_log_pos=s",
bk@work.mysql.com's avatar
bk@work.mysql.com committed
115
    "flushlog",
monty@tik.mysql.fi's avatar
monty@tik.mysql.fi committed
116 117
    "resetmaster",
    "resetslave",
118
    "tmpdir|t=s",
bk@work.mysql.com's avatar
bk@work.mysql.com committed
119 120 121 122 123 124 125
    "dryrun|n",
) or usage("Invalid option");

# @db_desc
# ==========
# a list of hash-refs containing:
#
monty@donna.mysql.com's avatar
monty@donna.mysql.com committed
126 127 128 129 130
#   'src'     - name of the db to copy
#   't_regex' - regex describing tables in src
#   'target'  - destination directory of the copy
#   'tables'  - array-ref to list of tables in the db
#   'files'   - array-ref to list of files to be copied
131
#               (RAID files look like 'nn/name.MYD')
132
#   'index'   - array-ref to list of indexes to be copied
bk@work.mysql.com's avatar
bk@work.mysql.com committed
133 134 135 136
#

my @db_desc = ();
my $tgt_name = undef;
monty@donna.mysql.com's avatar
monty@donna.mysql.com committed
137

138 139
usage("") if ($opt{help});

bk@work.mysql.com's avatar
bk@work.mysql.com committed
140 141
if ( $opt{regexp} || $opt{suffix} || @ARGV > 2 ) {
    $tgt_name   = pop @ARGV unless ( exists $opt{suffix} );
monty@donna.mysql.com's avatar
monty@donna.mysql.com committed
142
    @db_desc = map { s{^([^\.]+)\./(.+)/$}{$1}; { 'src' => $_, 't_regex' => ( $2 ? $2 : '.*' ) } } @ARGV;
bk@work.mysql.com's avatar
bk@work.mysql.com committed
143 144 145 146
}
else {
    usage("Database name to hotcopy not specified") unless ( @ARGV );

monty@donna.mysql.com's avatar
monty@donna.mysql.com committed
147 148 149
    $ARGV[0] =~ s{^([^\.]+)\./(.+)/$}{$1};
    @db_desc = ( { 'src' => $ARGV[0], 't_regex' => ( $2 ? $2 : '.*' ) } );

bk@work.mysql.com's avatar
bk@work.mysql.com committed
150 151 152 153 154 155 156 157 158 159
    if ( @ARGV == 2 ) {
	$tgt_name   = $ARGV[1];
    }
    else {
	$opt{suffix} = "_copy";
    }
}

my %mysqld_vars;
my $start_time = time;
160
$opt_tmpdir= $opt{tmpdir} if $opt{tmpdir};
bk@work.mysql.com's avatar
bk@work.mysql.com committed
161 162 163 164 165
$0 = $1 if $0 =~ m:/([^/]+)$:;
$opt{quiet} = 0 if $opt{debug};
$opt{allowold} = 1 if $opt{keepold};

# --- connect to the database ---
166 167
my $dsn;
$dsn  = ";host=" . (defined($opt{host}) ? $opt{host} : "localhost");
bk@work.mysql.com's avatar
bk@work.mysql.com committed
168 169 170
$dsn .= ";port=$opt{port}" if $opt{port};
$dsn .= ";mysql_socket=$opt{socket}" if $opt{socket};

paul@teton.kitebird.com's avatar
paul@teton.kitebird.com committed
171 172 173
# use mysql_read_default_group=mysqlhotcopy so that [client] and
# [mysqlhotcopy] groups will be read from standard options files.

monty@donna.mysql.com's avatar
monty@donna.mysql.com committed
174 175 176
my $dbh = DBI->connect("dbi:mysql:$dsn;mysql_read_default_group=mysqlhotcopy",
                        $opt{user}, $opt{password},
{
bk@work.mysql.com's avatar
bk@work.mysql.com committed
177 178 179 180 181 182 183
    RaiseError => 1,
    PrintError => 0,
    AutoCommit => 1,
});

# --- check that checkpoint table exists if specified ---
if ( $opt{checkpoint} ) {
184
    $opt{checkpoint} = quote_names( $opt{checkpoint} );
bk@work.mysql.com's avatar
bk@work.mysql.com committed
185 186 187 188 189 190 191 192
    eval { $dbh->do( qq{ select time_stamp, src, dest, msg 
			 from $opt{checkpoint} where 1 != 1} );
       };

    die "Error accessing Checkpoint table ($opt{checkpoint}): $@"
      if ( $@ );
}

193 194
# --- check that log_pos table exists if specified ---
if ( $opt{record_log_pos} ) {
195 196
    $opt{record_log_pos} = quote_names( $opt{record_log_pos} );

197 198 199 200 201 202 203 204
    eval { $dbh->do( qq{ select host, time_stamp, log_file, log_pos, master_host, master_log_file, master_log_pos
			 from $opt{record_log_pos} where 1 != 1} );
       };

    die "Error accessing log_pos table ($opt{record_log_pos}): $@"
      if ( $@ );
}

bk@work.mysql.com's avatar
bk@work.mysql.com committed
205
# --- get variables from database ---
monty@donna.mysql.com's avatar
monty@donna.mysql.com committed
206
my $sth_vars = $dbh->prepare("show variables like 'datadir'");
bk@work.mysql.com's avatar
bk@work.mysql.com committed
207 208 209 210
$sth_vars->execute;
while ( my ($var,$value) = $sth_vars->fetchrow_array ) {
    $mysqld_vars{ $var } = $value;
}
monty@donna.mysql.com's avatar
monty@donna.mysql.com committed
211
my $datadir = $mysqld_vars{'datadir'}
bk@work.mysql.com's avatar
bk@work.mysql.com committed
212 213 214 215 216
    || die "datadir not in mysqld variables";
$datadir =~ s:/$::;


# --- get target path ---
monty@donna.mysql.com's avatar
monty@donna.mysql.com committed
217 218
my ($tgt_dirname, $to_other_database);
$to_other_database=0;
219
if (defined($tgt_name) && $tgt_name =~ m:^\w+$: && @db_desc <= 1)
monty@donna.mysql.com's avatar
monty@donna.mysql.com committed
220
{
bk@work.mysql.com's avatar
bk@work.mysql.com committed
221
    $tgt_dirname = "$datadir/$tgt_name";
monty@donna.mysql.com's avatar
monty@donna.mysql.com committed
222
    $to_other_database=1;
bk@work.mysql.com's avatar
bk@work.mysql.com committed
223
}
224
elsif (defined($tgt_name) && ($tgt_name =~ m:/: || $tgt_name eq '.')) {
bk@work.mysql.com's avatar
bk@work.mysql.com committed
225 226 227
    $tgt_dirname = $tgt_name;
}
elsif ( $opt{suffix} ) {
228
    print "Using copy suffix '$opt{suffix}'\n" unless $opt{quiet};
bk@work.mysql.com's avatar
bk@work.mysql.com committed
229
}
230 231 232 233
else
{
  $tgt_name="" if (!defined($tgt_name));
  die "Target '$tgt_name' doesn't look like a database name or directory path.\n";
bk@work.mysql.com's avatar
bk@work.mysql.com committed
234 235 236 237
}

# --- resolve database names from regexp ---
if ( defined $opt{regexp} ) {
238 239 240 241 242
    my $t_regex = '.*';
    if ( $opt{regexp} =~ s{^/(.+)/\./(.+)/$}{$1} ) {
        $t_regex = $2;
    }

bk@work.mysql.com's avatar
bk@work.mysql.com committed
243 244 245
    my $sth_dbs = $dbh->prepare("show databases");
    $sth_dbs->execute;
    while ( my ($db_name) = $sth_dbs->fetchrow_array ) {
246
	push @db_desc, { 'src' => $db_name, 't_regex' => $t_regex } if ( $db_name =~ m/$opt{regexp}/o );
bk@work.mysql.com's avatar
bk@work.mysql.com committed
247 248 249 250 251 252
    }
}

# --- get list of tables to hotcopy ---

my $hc_locks = "";
253
my $hc_tables = "";
bk@work.mysql.com's avatar
bk@work.mysql.com committed
254 255 256 257 258
my $num_tables = 0;
my $num_files = 0;

foreach my $rdb ( @db_desc ) {
    my $db = $rdb->{src};
259
    my @dbh_tables = get_list_of_tables( $db );
bk@work.mysql.com's avatar
bk@work.mysql.com committed
260

monty@donna.mysql.com's avatar
monty@donna.mysql.com committed
261
    ## generate regex for tables/files
262 263 264 265 266 267 268 269 270 271 272 273 274 275 276 277 278 279
    my $t_regex;
    my $negated;
    if ($rdb->{t_regex}) {
        $t_regex = $rdb->{t_regex};        ## assign temporary regex
        $negated = $t_regex =~ tr/~//d;    ## remove and count
                                           ## negation operator: we
                                           ## don't allow ~ in table
                                           ## names

        $t_regex = qr/$t_regex/;           ## make regex string from
                                           ## user regex

        ## filter (out) tables specified in t_regex
        print "Filtering tables with '$t_regex'\n" if $opt{debug};
        @dbh_tables = ( $negated 
                        ? grep { $_ !~ $t_regex } @dbh_tables
                        : grep { $_ =~ $t_regex } @dbh_tables );
    }
monty@donna.mysql.com's avatar
monty@donna.mysql.com committed
280 281

    ## get list of files to copy
bk@work.mysql.com's avatar
bk@work.mysql.com committed
282 283 284 285
    my $db_dir = "$datadir/$db";
    opendir(DBDIR, $db_dir ) 
      or die "Cannot open dir '$db_dir': $!";

monty@donna.mysql.com's avatar
monty@donna.mysql.com committed
286
    my %db_files;
287 288 289 290 291 292 293 294 295 296 297 298 299 300
    my @raid_dir = ();

    while ( defined( my $name = readdir DBDIR ) ) {
	if ( $name =~ /^\d\d$/ && -d "$db_dir/$name" ) {
	    push @raid_dir, $name;
	}
	else {
	    $db_files{$name} = $1 if ( $name =~ /(.+)\.\w+$/ );
        }
    }
    closedir( DBDIR );

    scan_raid_dir( \%db_files, $db_dir, @raid_dir );

monty@donna.mysql.com's avatar
monty@donna.mysql.com committed
301 302 303
    unless( keys %db_files ) {
	warn "'$db' is an empty database\n";
    }
bk@work.mysql.com's avatar
bk@work.mysql.com committed
304

monty@donna.mysql.com's avatar
monty@donna.mysql.com committed
305
    ## filter (out) files specified in t_regex
306 307 308 309 310 311 312 313 314 315
    my @db_files;
    if ($rdb->{t_regex}) {
        @db_files = ($negated
                     ? grep { $db_files{$_} !~ $t_regex } keys %db_files
                     : grep { $db_files{$_} =~ $t_regex } keys %db_files );
    }
    else {
        @db_files = keys %db_files;
    }

316
    @db_files = sort @db_files;
317

318
    my @index_files=();
monty@donna.mysql.com's avatar
monty@donna.mysql.com committed
319 320

    ## remove indices unless we're told to keep them
321 322
    if ($opt{noindices}) {
        @index_files= grep { /\.(ISM|MYI)$/ } @db_files;
bk@work.mysql.com's avatar
bk@work.mysql.com committed
323 324 325 326
	@db_files = grep { not /\.(ISM|MYI)$/ } @db_files;
    }

    $rdb->{files}  = [ @db_files ];
327
    $rdb->{index}  = [ @index_files ];
328
    my @hc_tables = map { quote_names("$db.$_") } @dbh_tables;
bk@work.mysql.com's avatar
bk@work.mysql.com committed
329 330
    $rdb->{tables} = [ @hc_tables ];

331 332
    $rdb->{raid_dirs} = [ get_raid_dirs( $rdb->{files} ) ];

bk@work.mysql.com's avatar
bk@work.mysql.com committed
333 334
    $hc_locks .= ", "  if ( length $hc_locks && @hc_tables );
    $hc_locks .= join ", ", map { "$_ READ" } @hc_tables;
monty@donna.mysql.com's avatar
monty@donna.mysql.com committed
335
    $hc_tables .= ", "  if ( length $hc_tables && @hc_tables );
336
    $hc_tables .= join ", ", @hc_tables;
bk@work.mysql.com's avatar
bk@work.mysql.com committed
337 338 339 340 341 342 343

    $num_tables += scalar @hc_tables;
    $num_files  += scalar @{$rdb->{files}};
}

# --- resolve targets for copies ---

344
if (defined($tgt_name) && length $tgt_name ) {
bk@work.mysql.com's avatar
bk@work.mysql.com committed
345 346 347
    # explicit destination directory specified

    # GNU `cp -r` error message
monty@donna.mysql.com's avatar
monty@donna.mysql.com committed
348 349 350 351 352 353 354 355
    die "copying multiple databases, but last argument ($tgt_dirname) is not a directory\n"
      if ( @db_desc > 1 && !(-e $tgt_dirname && -d $tgt_dirname ) );

    if ($to_other_database)
    {
      foreach my $rdb ( @db_desc ) {
	$rdb->{target} = "$tgt_dirname";
      }
bk@work.mysql.com's avatar
bk@work.mysql.com committed
356
    }
monty@donna.mysql.com's avatar
monty@donna.mysql.com committed
357 358 359 360 361 362
    elsif ($opt{method} =~ /^scp\b/) 
    {   # we have to trust scp to hit the target
	foreach my $rdb ( @db_desc ) {
	    $rdb->{target} = "$tgt_dirname/$rdb->{src}";
	}
    }
monty@donna.mysql.com's avatar
monty@donna.mysql.com committed
363 364 365 366 367 368 369
    else
    {
      die "Last argument ($tgt_dirname) is not a directory\n"
	if (!(-e $tgt_dirname && -d $tgt_dirname ) );
      foreach my $rdb ( @db_desc ) {
	$rdb->{target} = "$tgt_dirname/$rdb->{src}";
      }
bk@work.mysql.com's avatar
bk@work.mysql.com committed
370
    }
monty@donna.mysql.com's avatar
monty@donna.mysql.com committed
371 372 373
  }
else {
  die "Error: expected \$opt{suffix} to exist" unless ( exists $opt{suffix} );
monty@donna.mysql.com's avatar
monty@donna.mysql.com committed
374

monty@donna.mysql.com's avatar
monty@donna.mysql.com committed
375 376 377
  foreach my $rdb ( @db_desc ) {
    $rdb->{target} = "$datadir/$rdb->{src}$opt{suffix}";
  }
bk@work.mysql.com's avatar
bk@work.mysql.com committed
378 379 380 381 382 383 384 385
}

print Dumper( \@db_desc ) if ( $opt{debug} );

# --- bail out if all specified databases are empty ---

die "No tables to hot-copy" unless ( length $hc_locks );

386
# --- create target directories if we are using 'cp' ---
bk@work.mysql.com's avatar
bk@work.mysql.com committed
387 388

my @existing = ();
389 390 391 392

if ($opt{method} =~ /^cp\b/)
{
  foreach my $rdb ( @db_desc ) {
bk@work.mysql.com's avatar
bk@work.mysql.com committed
393
    push @existing, $rdb->{target} if ( -d  $rdb->{target} );
394
  }
bk@work.mysql.com's avatar
bk@work.mysql.com committed
395

396
  if ( @existing && !($opt{allowold} || $opt{addtodest}) )
397 398
  {
    $dbh->disconnect();
399
    die "Can't hotcopy to '", join( "','", @existing ), "' because directory\nalready exist and the --allowold or --addtodest options were not given.\n"
400
  }
401
}
bk@work.mysql.com's avatar
bk@work.mysql.com committed
402

403
retire_directory( @existing ) if @existing && !$opt{addtodest};
bk@work.mysql.com's avatar
bk@work.mysql.com committed
404 405

foreach my $rdb ( @db_desc ) {
406 407 408
    foreach my $td ( '', @{$rdb->{raid_dirs}} ) {

	my $tgt_dirpath = "$rdb->{target}/$td";
409 410
	# Remove trailing slashes (needed for Mac OS X)
    	substr($tgt_dirpath, 1) =~ s|/+$||;
411 412 413 414 415 416 417 418
	if ( $opt{dryrun} ) {
	    print "mkdir $tgt_dirpath, 0750\n";
	}
	elsif ($opt{method} =~ /^scp\b/) {
	    ## assume it's there?
	    ## ...
	}
	else {
419 420
	    mkdir($tgt_dirpath, 0750) or die "Can't create '$tgt_dirpath': $!\n"
		unless -d $tgt_dirpath;
421 422
	    my @f_info= stat "$datadir/$rdb->{src}";
	    chown $f_info[4], $f_info[5], $tgt_dirpath;
423
	}
bk@work.mysql.com's avatar
bk@work.mysql.com committed
424 425 426 427 428 429 430 431 432 433 434 435 436
    }
}

##############################
# --- PERFORM THE HOT-COPY ---
#
# Note that we try to keep the time between the LOCK and the UNLOCK
# as short as possible, and only start when we know that we should
# be able to complete without error.

# read lock all the tables we'll be copying
# in order to get a consistent snapshot of the database

437 438 439 440 441 442
if ( $opt{checkpoint} || $opt{record_log_pos} ) {
  # convert existing READ lock on checkpoint and/or log_pos table into WRITE lock
  foreach my $table ( grep { defined } ( $opt{checkpoint}, $opt{record_log_pos} ) ) {
    $hc_locks .= ", $table WRITE" 
	unless ( $hc_locks =~ s/$table\s+READ/$table WRITE/ );
  }
bk@work.mysql.com's avatar
bk@work.mysql.com committed
443 444 445 446 447 448
}

my $hc_started = time;	# count from time lock is granted

if ( $opt{dryrun} ) {
    print "LOCK TABLES $hc_locks\n";
449
    print "FLUSH TABLES /*!32323 $hc_tables */\n";
bk@work.mysql.com's avatar
bk@work.mysql.com committed
450
    print "FLUSH LOGS\n" if ( $opt{flushlog} );
monty@tik.mysql.fi's avatar
monty@tik.mysql.fi committed
451 452
    print "RESET MASTER\n" if ( $opt{resetmaster} );
    print "RESET SLAVE\n" if ( $opt{resetslave} );
bk@work.mysql.com's avatar
bk@work.mysql.com committed
453 454 455 456 457 458 459 460 461
}
else {
    my $start = time;
    $dbh->do("LOCK TABLES $hc_locks");
    printf "Locked $num_tables tables in %d seconds.\n", time-$start unless $opt{quiet};
    $hc_started = time;	# count from time lock is granted

    # flush tables to make on-disk copy uptodate
    $start = time;
462 463
    $dbh->do("FLUSH TABLES /*!32323 $hc_tables */");
    printf "Flushed tables ($hc_tables) in %d seconds.\n", time-$start unless $opt{quiet};
bk@work.mysql.com's avatar
bk@work.mysql.com committed
464
    $dbh->do( "FLUSH LOGS" ) if ( $opt{flushlog} );
monty@tik.mysql.fi's avatar
monty@tik.mysql.fi committed
465 466
    $dbh->do( "RESET MASTER" ) if ( $opt{resetmaster} );
    $dbh->do( "RESET SLAVE" ) if ( $opt{resetslave} );
467 468 469 470 471

    if ( $opt{record_log_pos} ) {
	record_log_pos( $dbh, $opt{record_log_pos} );
	$dbh->do("FLUSH TABLES /*!32323 $hc_tables */");
    }
bk@work.mysql.com's avatar
bk@work.mysql.com committed
472
}
monty@donna.mysql.com's avatar
monty@donna.mysql.com committed
473

bk@work.mysql.com's avatar
bk@work.mysql.com committed
474 475
my @failed = ();

476 477 478 479 480
foreach my $rdb ( @db_desc )
{
  my @files = map { "$datadir/$rdb->{src}/$_" } @{$rdb->{files}};
  next unless @files;
  
481
  eval { copy_files($opt{method}, \@files, $rdb->{target}, $rdb->{raid_dirs} ); };
482 483 484
  push @failed, "$rdb->{src} -> $rdb->{target} failed: $@"
    if ( $@ );
  
485
  @files = @{$rdb->{index}};
486 487
  if ($rdb->{index})
  {
488 489
    copy_index($opt{method}, \@files,
	       "$datadir/$rdb->{src}", $rdb->{target} );
490 491 492 493 494 495 496 497 498 499 500 501 502
  }
  
  if ( $opt{checkpoint} ) {
    my $msg = ( $@ ) ? "Failed: $@" : "Succeeded";
    
    eval {
      $dbh->do( qq{ insert into $opt{checkpoint} (src, dest, msg) 
		      VALUES ( '$rdb->{src}', '$rdb->{target}', '$msg' )
		    } ); 
    };
    
    if ( $@ ) {
      warn "Failed to update checkpoint table: $@\n";
bk@work.mysql.com's avatar
bk@work.mysql.com committed
503
    }
504
  }
bk@work.mysql.com's avatar
bk@work.mysql.com committed
505 506 507 508 509 510 511 512 513 514 515 516 517 518 519 520 521 522 523 524 525 526 527 528 529 530 531 532 533
}

if ( $opt{dryrun} ) {
    print "UNLOCK TABLES\n";
    if ( @existing && !$opt{keepold} ) {
	my @oldies = map { $_ . '_old' } @existing;
	print "rm -rf @oldies\n" 
    }
    $dbh->disconnect();
    exit(0);
}
else {
    $dbh->do("UNLOCK TABLES");
}

my $hc_dur = time - $hc_started;
printf "Unlocked tables.\n" unless $opt{quiet};

#
# --- HOT-COPY COMPLETE ---
###########################

$dbh->disconnect;

if ( @failed ) {
    # hotcopy failed - cleanup
    # delete any @targets 
    # rename _old copy back to original

534 535 536 537 538 539
    my @targets = ();
    foreach my $rdb ( @db_desc ) {
        push @targets, $rdb->{target} if ( -d  $rdb->{target} );
    }
    print "Deleting @targets \n" if $opt{debug};

bk@work.mysql.com's avatar
bk@work.mysql.com committed
540 541 542 543 544 545 546 547 548 549 550 551 552 553 554 555 556 557 558 559 560 561 562 563 564 565 566 567 568 569 570 571 572 573
    print "Deleting @targets \n" if $opt{debug};
    rmtree([@targets]);
    if (@existing) {
	print "Restoring @existing from back-up\n" if $opt{debug};
        foreach my $dir ( @existing ) {
	    rename("${dir}_old", $dir )
	      or warn "Can't rename ${dir}_old to $dir: $!\n";
	}
    }

    die join( "\n", @failed );
}
else {
    # hotcopy worked
    # delete _old unless $opt{keepold}

    if ( @existing && !$opt{keepold} ) {
	my @oldies = map { $_ . '_old' } @existing;
	print "Deleting previous copy in @oldies\n" if $opt{debug};
	rmtree([@oldies]);
    }

    printf "$0 copied %d tables (%d files) in %d second%s (%d seconds overall).\n",
	    $num_tables, $num_files,
	    $hc_dur, ($hc_dur==1)?"":"s", time - $start_time
	unless $opt{quiet};
}

exit 0;


# ---

sub copy_files {
574
    my ($method, $files, $target, $raid_dirs) = @_;
bk@work.mysql.com's avatar
bk@work.mysql.com committed
575 576
    my @cmd;
    print "Copying ".@$files." files...\n" unless $opt{quiet};
monty@donna.mysql.com's avatar
monty@donna.mysql.com committed
577

bk@work.mysql.com's avatar
bk@work.mysql.com committed
578
    if ($method =~ /^s?cp\b/) { # cp or scp with optional flags
guilhem@mysql.com's avatar
guilhem@mysql.com committed
579
	my $cp = $method;
bk@work.mysql.com's avatar
bk@work.mysql.com committed
580 581
	# add option to preserve mod time etc of copied files
	# not critical, but nice to have
guilhem@mysql.com's avatar
guilhem@mysql.com committed
582
	$cp.= " -p" if $^O =~ m/^(solaris|linux|freebsd|darwin)$/;
monty@donna.mysql.com's avatar
monty@donna.mysql.com committed
583 584

	# add recursive option for scp
guilhem@mysql.com's avatar
guilhem@mysql.com committed
585
	$cp.= " -r" if $^O =~ /m^(solaris|linux|freebsd|darwin)$/ && $method =~ /^scp\b/;
586

587
	my @non_raid = map { "'$_'" } grep { ! m:/\d{2}/[^/]+$: } @$files;
monty@donna.mysql.com's avatar
monty@donna.mysql.com committed
588

bk@work.mysql.com's avatar
bk@work.mysql.com committed
589
	# add files to copy and the destination directory
guilhem@mysql.com's avatar
guilhem@mysql.com committed
590
	safe_system( $cp, @non_raid, "'$target'" ) if (@non_raid);
591 592
	
	foreach my $rd ( @$raid_dirs ) {
593
	    my @raid = map { "'$_'" } grep { m:$rd/: } @$files;
guilhem@mysql.com's avatar
guilhem@mysql.com committed
594
	    safe_system( $cp, @raid, "'$target'/$rd" ) if ( @raid );
595
	}
bk@work.mysql.com's avatar
bk@work.mysql.com committed
596
    }
597 598
    else
    {
bk@work.mysql.com's avatar
bk@work.mysql.com committed
599 600
	die "Can't use unsupported method '$method'\n";
    }
601
}
bk@work.mysql.com's avatar
bk@work.mysql.com committed
602

603 604 605 606 607 608 609 610 611 612 613 614 615 616 617 618 619 620 621 622 623 624 625 626 627 628 629 630 631 632 633 634 635 636 637 638 639 640 641 642 643 644 645 646 647 648 649 650 651 652 653 654 655
#
# Copy only the header of the index file
#

sub copy_index
{
  my ($method, $files, $source, $target) = @_;
  my $tmpfile="$opt_tmpdir/mysqlhotcopy$$";
  
  print "Copying indices for ".@$files." files...\n" unless $opt{quiet};  
  foreach my $file (@$files)
  {
    my $from="$source/$file";
    my $to="$target/$file";
    my $buff;
    open(INPUT, "<$from") || die "Can't open file $from: $!\n";
    my $length=read INPUT, $buff, 2048;
    die "Can't read index header from $from\n" if ($length < 1024);
    close INPUT;
    
    if ( $opt{dryrun} )
    {
      print "$opt{method}-header $from $to\n";
    }
    elsif ($opt{method} eq 'cp')
    {
      open(OUTPUT,">$to")   || die "Can\'t create file $to: $!\n";
      if (syswrite(OUTPUT,$buff) != length($buff))
      {
	die "Error when writing data to $to: $!\n";
      }
      close OUTPUT	   || die "Error on close of $to: $!\n";
    }
    elsif ($opt{method} eq 'scp')
    {
      my $tmp=$tmpfile;
      open(OUTPUT,">$tmp") || die "Can\'t create file $tmp: $!\n";
      if (syswrite(OUTPUT,$buff) != length($buff))
      {
	die "Error when writing data to $tmp: $!\n";
      }
      close OUTPUT	     || die "Error on close of $tmp: $!\n";
      safe_system("scp $tmp $to");
    }
    else
    {
      die "Can't use unsupported method '$opt{method}'\n";
    }
  }
  unlink "$tmpfile" if  ($opt{method} eq 'scp');
}


guilhem@mysql.com's avatar
guilhem@mysql.com committed
656 657 658 659 660 661 662 663 664 665 666 667 668 669 670 671 672 673 674 675 676 677 678 679
sub safe_system {
  my @sources= @_;
  my $method= shift @sources;
  my $target= pop @sources;
  ## @sources = list of source file names

  ## We have to deal with very long command lines, otherwise they may generate 
  ## "Argument list too long".
  ## With 10000 tables the command line can be around 1MB, much more than 128kB
  ## which is the common limit on Linux (can be read from
  ## /usr/src/linux/include/linux/binfmts.h
  ## see http://www.linuxjournal.com/article.php?sid=6060).
 
  my $chunk_limit= 100 * 1024; # 100 kB
  my @chunk= (); 
  my $chunk_length= 0;
  foreach (@sources) {
      push @chunk, $_;
      $chunk_length+= length($_);
      if ($chunk_length > $chunk_limit) {
          safe_simple_system($method, @chunk, $target);
          @chunk=();
          $chunk_length= 0;
      }
680
  }
guilhem@mysql.com's avatar
guilhem@mysql.com committed
681 682
  if ($chunk_length > 0) { # do not forget last small chunk
      safe_simple_system($method, @chunk, $target); 
683
  }
bk@work.mysql.com's avatar
bk@work.mysql.com committed
684 685
}

guilhem@mysql.com's avatar
guilhem@mysql.com committed
686 687 688 689 690 691 692 693 694 695 696 697 698 699 700 701 702 703
sub safe_simple_system {
    my @cmd= @_;

    if ( $opt{dryrun} ) {
        print "@cmd\n";
    }
    else {
        ## for some reason system fails but backticks works ok for scp...
        print "Executing '@cmd'\n" if $opt{debug};
        my $cp_status = system "@cmd > /dev/null";
        if ($cp_status != 0) {
            warn "Executing command failed ($cp_status). Trying backtick execution...\n";
            ## try something else
            `@cmd` || die "Error: @cmd failed ($?) while copying files.\n";
        }
    }
}

bk@work.mysql.com's avatar
bk@work.mysql.com committed
704 705 706 707 708 709 710 711 712 713 714 715 716 717 718 719 720 721 722 723 724
sub retire_directory {
    my ( @dir ) = @_;

    foreach my $dir ( @dir ) {
	my $tgt_oldpath = $dir . '_old';
	if ( $opt{dryrun} ) {
	    print "rmtree $tgt_oldpath\n" if ( -d $tgt_oldpath );
	    print "rename $dir, $tgt_oldpath\n";
	    next;
	}

	if ( -d $tgt_oldpath ) {
	    print "Deleting previous 'old' hotcopy directory ('$tgt_oldpath')\n" unless $opt{quiet};
	    rmtree([$tgt_oldpath])
	}
	rename($dir, $tgt_oldpath)
	  or die "Can't rename $dir=>$tgt_oldpath: $!\n";
	print "Existing hotcopy directory renamed to '$tgt_oldpath'\n" unless $opt{quiet};
    }
}

725 726 727 728 729 730 731 732 733 734 735 736 737 738 739 740 741 742 743 744 745 746 747 748 749 750 751 752 753 754 755 756 757 758
sub record_log_pos {
    my ( $dbh, $table_name ) = @_;

    eval {
	my ($file,$position) = get_row( $dbh, "show master status" );
	die "master status is undefined" if !defined $file || !defined $position;
	
	my ($master_host, undef, undef, undef, $log_file, $log_pos ) 
	    = get_row( $dbh, "show slave status" );
	
	my $hostname = hostname();
	
	$dbh->do( qq{ replace into $table_name 
			  set host=?, log_file=?, log_pos=?, 
                          master_host=?, master_log_file=?, master_log_pos=? }, 
		  undef, 
		  $hostname, $file, $position, 
		  $master_host, $log_file, $log_pos  );
	
    };
    
    if ( $@ ) {
	warn "Failed to store master position: $@\n";
    }
}

sub get_row {
  my ( $dbh, $sql ) = @_;

  my $sth = $dbh->prepare($sql);
  $sth->execute;
  return $sth->fetchrow_array();
}

759 760 761 762 763 764 765 766 767 768 769 770 771 772 773 774 775 776 777 778 779 780 781 782 783 784 785 786 787
sub scan_raid_dir {
    my ( $r_db_files, $data_dir, @raid_dir ) = @_;

    local(*RAID_DIR);
    
    foreach my $rd ( @raid_dir ) {

	opendir(RAID_DIR, "$data_dir/$rd" ) 
	    or die "Cannot open dir '$data_dir/$rd': $!";

	while ( defined( my $name = readdir RAID_DIR ) ) {
	    $r_db_files->{"$rd/$name"} = $1 if ( $name =~ /(.+)\.\w+$/ );
	}
	closedir( RAID_DIR );
    }
}

sub get_raid_dirs {
    my ( $r_files ) = @_;

    my %dirs = ();
    foreach my $f ( @$r_files ) {
	if ( $f =~ m:^(\d\d)/: ) {
	    $dirs{$1} = 1;
	}
    }
    return sort keys %dirs;
}

788 789 790 791 792 793 794 795 796 797 798 799 800 801 802 803 804 805 806
sub get_list_of_tables {
    my ( $db ) = @_;

    # "use database" cannot cope with database names containing spaces
    # so create a new connection 

    my $dbh = DBI->connect("dbi:mysql:${db}${dsn};mysql_read_default_group=mysqlhotcopy",
			    $opt{user}, $opt{password},
    {
	RaiseError => 1,
	PrintError => 0,
	AutoCommit => 1,
    });

    my @dbh_tables = eval { $dbh->tables() };
    $dbh->disconnect();
    return @dbh_tables;
}

807 808 809 810 811 812 813
sub quote_names {
  my ( $name ) = @_;
  # given a db.table name, add quotes

  my ($db, $table, @cruft) = split( /\./, $name );
  die "Invalid db.table name '$name'" if (@cruft || !defined $db || !defined $table );

814 815 816 817
  # Earlier versions of DBD return table name non-quoted,
  # such as DBD-2.1012 and the newer ones, such as DBD-2.9002
  # returns it quoted. Let's have a support for both.
  $table=~ s/\`//g;
818 819 820
  return "`$db`.`$table`";
}

bk@work.mysql.com's avatar
bk@work.mysql.com committed
821 822 823 824 825 826 827 828 829 830 831 832 833 834 835 836 837 838 839 840 841 842 843 844 845 846 847 848 849 850 851 852 853 854
__END__

=head1 DESCRIPTION

mysqlhotcopy is designed to make stable copies of live MySQL databases.

Here "live" means that the database server is running and the database
may be in active use. And "stable" means that the copy will not have
any corruptions that could occur if the table files were simply copied
without first being locked and flushed from within the server.

=head1 OPTIONS

=over 4

=item --checkpoint checkpoint-table

As each database is copied, an entry is written to the specified
checkpoint-table.  This has the happy side-effect of updating the
MySQL update-log (if it is switched on) giving a good indication of
where roll-forward should begin for backup+rollforward schemes.

The name of the checkpoint table should be supplied in database.table format.
The checkpoint-table must contain at least the following fields:

=over 4

  time_stamp timestamp not null
  src varchar(32)
  dest varchar(60)
  msg varchar(255)

=back

855 856 857 858 859 860 861 862 863 864 865 866 867 868 869 870 871 872 873 874 875 876 877 878 879 880 881 882 883 884 885 886
=item --record_log_pos log-pos-table

Just before the database files are copied, update the record in the
log-pos-table from the values returned from "show master status" and
"show slave status". The master status values are stored in the
log_file and log_pos columns, and establish the position in the binary
logs that any slaves of this host should adopt if initialised from
this dump.  The slave status values are stored in master_host,
master_log_file, and master_log_pos, and these are useful if the host
performing the dump is a slave and other sibling slaves are to be
initialised from this dump.

The name of the log-pos table should be supplied in database.table format.
A sample log-pos table definition:

=over 4

CREATE TABLE log_pos (
  host            varchar(60) NOT null,
  time_stamp      timestamp(14) NOT NULL,
  log_file        varchar(32) default NULL,
  log_pos         int(11)     default NULL,
  master_host     varchar(60) NULL,
  master_log_file varchar(32) NULL,
  master_log_pos  int NULL,

  PRIMARY KEY  (host) 
);

=back


bk@work.mysql.com's avatar
bk@work.mysql.com committed
887 888 889 890 891 892 893 894 895 896 897 898 899 900 901 902 903 904 905 906 907 908 909 910
=item --suffix suffix

Each database is copied back into the originating datadir under
a new name. The new name is the original name with the suffix
appended. 

If only a single db_name is supplied and the --suffix flag is not
supplied, then "--suffix=_copy" is assumed.

=item --allowold

Move any existing version of the destination to a backup directory for
the duration of the copy. If the copy successfully completes, the backup 
directory is deleted - unless the --keepold flag is set.  If the copy fails,
the backup directory is restored.

The backup directory name is the original name with "_old" appended.
Any existing versions of the backup directory are deleted.

=item --keepold

Behaves as for the --allowold, with the additional feature 
of keeping the backup directory after the copy successfully completes.

911 912 913 914 915 916 917 918 919 920 921 922 923 924 925 926
=item --addtodest

Don't rename target directory if it already exists, just add the
copied files into it.

This is most useful when backing up a database with many large
tables and you don't want to have all the tables locked for the
whole duration.

In this situation, I<if> you are happy for groups of tables to be
backed up separately (and thus possibly not be logically consistant
with one another) then you can run mysqlhotcopy several times on
the same database each with different db_name./table_regex/.
All but the first should use the --addtodest option so the tables
all end up in the same directory.

bk@work.mysql.com's avatar
bk@work.mysql.com committed
927 928 929 930 931
=item --flushlog

Rotate the log files by executing "FLUSH LOGS" after all tables are
locked, and before they are copied.

monty@tik.mysql.fi's avatar
monty@tik.mysql.fi committed
932 933 934
=item --resetmaster

Reset the bin-log by executing "RESET MASTER" after all tables are
935
locked, and before they are copied. Useful if you are recovering a
monty@tik.mysql.fi's avatar
monty@tik.mysql.fi committed
936 937 938 939 940
slave in a replication setup.

=item --resetslave

Reset the master.info by executing "RESET SLAVE" after all tables are
941
locked, and before they are copied. Useful if you are recovering a
monty@tik.mysql.fi's avatar
monty@tik.mysql.fi committed
942 943
server in a mutual replication setup.

bk@work.mysql.com's avatar
bk@work.mysql.com committed
944 945
=item --regexp pattern

monty@donna.mysql.com's avatar
monty@donna.mysql.com committed
946 947
Copy all databases with names matching the pattern

948 949 950 951 952 953 954 955
=item --regexp /pattern1/./pattern2/

Copy all tables with names matching pattern2 from all databases with
names matching pattern1. For example, to select all tables which
names begin with 'bar' from all databases which names end with 'foo':

   mysqlhotcopy --indices --method=cp --regexp /foo$/./^bar/

monty@donna.mysql.com's avatar
monty@donna.mysql.com committed
956 957 958 959 960 961 962 963 964 965 966 967 968 969
=item db_name./pattern/

Copy only tables matching pattern. Shell metacharacters ( (, ), |, !,
etc.) have to be escaped (e.g. \). For example, to select all tables
in database db1 whose names begin with 'foo' or 'bar':

    mysqlhotcopy --indices --method=cp db1./^\(foo\|bar\)/

=item db_name./~pattern/

Copy only tables not matching pattern. For example, to copy tables
that do not begin with foo nor bar:

    mysqlhotcopy --indices --method=cp db1./~^\(foo\|bar\)/
bk@work.mysql.com's avatar
bk@work.mysql.com committed
970 971 972 973 974 975 976 977 978 979 980

=item -?, --help

Display helpscreen and exit

=item -u, --user=#         

user for database login if not current user

=item -p, --password=#     

981 982 983 984 985
password to use when connecting to the server. Note that you are strongly
encouraged *not* to use this option as every user would be able to see the
password in the process list. Instead use the '[mysqlhotcopy]' section in
one of the config files, normally /etc/my.cnf or your personal ~/.my.cnf.
(See the chapter 'my.cnf Option Files' in the manual)
bk@work.mysql.com's avatar
bk@work.mysql.com committed
986

987 988 989 990 991
=item -h, -h, --host=#

Hostname for local server when connecting over TCP/IP.  By specifying this
different from 'localhost' will trigger mysqlhotcopy to use TCP/IP connection.

bk@work.mysql.com's avatar
bk@work.mysql.com committed
992 993
=item -P, --port=#         

994 995
port to use when connecting to MySQL server with TCP/IP.  This is only used
when using the --host option.
bk@work.mysql.com's avatar
bk@work.mysql.com committed
996 997 998 999 1000

=item -S, --socket=#         

UNIX domain socket to use when connecting to local server

1001
=item  --noindices          
bk@work.mysql.com's avatar
bk@work.mysql.com committed
1002

1003 1004 1005
Don\'t include index files in copy. Only up to the first 2048 bytes
are copied;  You can restore the indexes with isamchk -r or myisamchk -r
on the backup.
bk@work.mysql.com's avatar
bk@work.mysql.com committed
1006 1007 1008

=item  --method=#           

monty@donna.mysql.com's avatar
monty@donna.mysql.com committed
1009 1010 1011 1012 1013
method for copy (only "cp" currently supported). Alpha support for
"scp" was added in November 2000. Your experience with the scp method
will vary with your ability to understand how scp works. 'man scp'
and 'man ssh' are your friends.

1014
The destination directory _must exist_ on the target machine using the
1015
scp method. --keepold and --allowold are meaningless with scp.
1016 1017
Liberal use of the --debug option will help you figure out what\'s
really going on when you do an scp.
monty@donna.mysql.com's avatar
monty@donna.mysql.com committed
1018 1019 1020 1021 1022

Note that using scp will lock your tables for a _long_ time unless
your network connection is _fast_. If this is unacceptable to you,
use the 'cp' method to copy the tables to some temporary area and then
scp or rsync the files at your leisure.
bk@work.mysql.com's avatar
bk@work.mysql.com committed
1023 1024 1025 1026 1027 1028 1029 1030 1031 1032 1033 1034 1035 1036 1037 1038 1039 1040 1041 1042 1043 1044 1045

=item -q, --quiet              

be silent except for errors

=item  --debug

Debug messages are displayed 

=item -n, --dryrun

Display commands without actually doing them

=back

=head1 WARRANTY

This software is free and comes without warranty of any kind. You
should never trust backup software without studying the code yourself.
Study the code inside this script and only rely on it if I<you> believe
that it does the right thing for you.

Patches adding bug fixes, documentation and new features are welcome.
1046
Please send these to internals@lists.mysql.com.
bk@work.mysql.com's avatar
bk@work.mysql.com committed
1047 1048 1049

=head1 TO DO

monty@donna.mysql.com's avatar
monty@donna.mysql.com committed
1050 1051
Extend the individual table copy to allow multiple subsets of tables
to be specified on the command line:
bk@work.mysql.com's avatar
bk@work.mysql.com committed
1052 1053 1054 1055 1056 1057 1058

  mysqlhotcopy db newdb  t1 t2 /^foo_/ : t3 /^bar_/ : +

where ":" delimits the subsets, the /^foo_/ indicates all tables
with names begining with "foo_" and the "+" indicates all tables
not copied by the previous subsets.

monty@donna.mysql.com's avatar
monty@donna.mysql.com committed
1059 1060 1061
newdb is either another not existing database or a full path to a directory
where we can create a directory 'db'

1062
Add option to lock each table in turn for people who don\'t need
bk@work.mysql.com's avatar
bk@work.mysql.com committed
1063 1064 1065 1066 1067 1068 1069 1070 1071 1072 1073 1074 1075
cross-table integrity.

Add option to FLUSH STATUS just before UNLOCK TABLES.

Add support for other copy methods (eg tar to single file?).

Add support for forthcoming MySQL ``RAID'' table subdirectory layouts.

=head1 AUTHOR

Tim Bunce

Martin Waite - added checkpoint, flushlog, regexp and dryrun options
1076 1077
               Fixed cleanup of targets when hotcopy fails. 
	       Added --record_log_pos.
1078
               RAID tables are now copied (don't know if this works over scp).
bk@work.mysql.com's avatar
bk@work.mysql.com committed
1079

monty@donna.mysql.com's avatar
monty@donna.mysql.com committed
1080 1081 1082
Ralph Corderoy - added synonyms for commands

Scott Wiersdorf - added table regex and scp support
1083 1084

Monty - working --noindex (copy only first 2048 bytes of index file)
1085
        Fixes for --method=scp
1086 1087

Ask Bjoern Hansen - Cleanup code to fix a few bugs and enable -w again.
monty@tik.mysql.fi's avatar
monty@tik.mysql.fi committed
1088 1089 1090

Emil S. Hansen - Added resetslave and resetmaster.

1091 1092 1093 1094
Jeremy D. Zawodny - Removed depricated DBI calls.  Fixed bug which
resulted in nothing being copied when a regexp was specified but no
database name(s).

1095
Martin Waite - Fix to handle database name that contains space.
1096 1097

Paul DuBois - Remove end '/' from directory names