xtrabackup.cc 175 KB
Newer Older
1
/******************************************************
2
MariaBackup: hot backup tool for InnoDB
Vladislav Vaintroub's avatar
Vladislav Vaintroub committed
3
(c) 2009-2017 Percona LLC and/or its affiliates
4 5 6
Originally Created 3/3/2009 Yasufumi Kinoshita
Written by Alexey Kopytov, Aleksandr Kuzminsky, Stewart Smith, Vadim Tkachenko,
Yasufumi Kinoshita, Ignacio Nin and Baron Schwartz.
7
(c) 2017, 2018, MariaDB Corporation.
8
Portions written by Marko Mäkelä.
9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38

This program is free software; you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation; version 2 of the License.

This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
GNU General Public License for more details.

You should have received a copy of the GNU General Public License
along with this program; if not, write to the Free Software
Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA

*******************************************************

This file incorporates work covered by the following copyright and
permission notice:

Copyright (c) 2000, 2011, MySQL AB & Innobase Oy. All Rights Reserved.

This program is free software; you can redistribute it and/or modify it under
the terms of the GNU General Public License as published by the Free Software
Foundation; version 2 of the License.

This program is distributed in the hope that it will be useful, but WITHOUT
ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details.

You should have received a copy of the GNU General Public License along with
39 40
this program; if not, write to the Free Software Foundation, Inc.,
51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA
41 42 43 44 45

*******************************************************/

//#define XTRABACKUP_TARGET_IS_PLUGIN

Michael Widenius's avatar
Michael Widenius committed
46
#include <my_global.h>
47 48
#include <my_config.h>
#include <unireg.h>
49 50 51 52 53 54 55 56 57 58 59 60
#include <mysql_version.h>
#include <my_base.h>
#include <my_getopt.h>
#include <mysql_com.h>
#include <my_default.h>
#include <mysqld.h>

#include <fcntl.h>
#include <string.h>

#ifdef __linux__
# include <sys/prctl.h>
61
#include <sys/resource.h>
62 63 64 65 66 67 68
#endif


#include <btr0sea.h>
#include <dict0priv.h>
#include <lock0lock.h>
#include <log0recv.h>
69
#include <log0crypt.h>
70 71 72
#include <row0mysql.h>
#include <row0quiesce.h>
#include <srv0start.h>
73
#include "trx0sys.h"
74 75
#include <buf0dblwr.h>

Vladislav Vaintroub's avatar
Vladislav Vaintroub committed
76
#include <list>
77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94
#include <sstream>
#include <set>
#include <mysql.h>

#define G_PTR uchar*

#include "common.h"
#include "datasink.h"

#include "xb_regex.h"
#include "fil_cur.h"
#include "write_filt.h"
#include "xtrabackup.h"
#include "ds_buffer.h"
#include "ds_tmpfile.h"
#include "xbstream.h"
#include "changed_page_bitmap.h"
#include "read_filt.h"
95
#include "backup_wsrep.h"
96 97 98 99
#include "innobackupex.h"
#include "backup_mysql.h"
#include "backup_copy.h"
#include "backup_mysql.h"
100 101 102
#include "encryption_plugin.h"
#include <sql_plugin.h>
#include <srv0srv.h>
Vladislav Vaintroub's avatar
Vladislav Vaintroub committed
103
#include <crc_glue.h>
104
#include <log.h>
105 106 107 108 109 110

int sys_var_init();

/* === xtrabackup specific options === */
char xtrabackup_real_target_dir[FN_REFLEN] = "./xtrabackup_backupfiles/";
char *xtrabackup_target_dir= xtrabackup_real_target_dir;
111
static my_bool xtrabackup_version;
112
static my_bool verbose;
113 114 115 116 117 118 119 120 121 122 123
my_bool xtrabackup_backup;
my_bool xtrabackup_prepare;
my_bool xtrabackup_copy_back;
my_bool xtrabackup_move_back;
my_bool xtrabackup_decrypt_decompress;
my_bool xtrabackup_print_param;

my_bool xtrabackup_export;

longlong xtrabackup_use_memory;

124
uint opt_protocol;
125 126 127 128 129 130
long xtrabackup_throttle; /* 0:unlimited */
static lint io_ticket;
static os_event_t wait_throttle;
static os_event_t log_copying_stop;

char *xtrabackup_incremental;
131 132 133
lsn_t incremental_lsn;
lsn_t incremental_to_lsn;
lsn_t incremental_last_lsn;
134
xb_page_bitmap *changed_page_bitmap;
135

136 137 138
char *xtrabackup_incremental_basedir; /* for --backup */
char *xtrabackup_extra_lsndir; /* for --backup with --extra-lsndir */
char *xtrabackup_incremental_dir; /* for --prepare */
139 140 141 142 143

char xtrabackup_real_incremental_basedir[FN_REFLEN];
char xtrabackup_real_extra_lsndir[FN_REFLEN];
char xtrabackup_real_incremental_dir[FN_REFLEN];

144

145 146
char *xtrabackup_tmpdir;

147 148 149
char *xtrabackup_tables;
char *xtrabackup_tables_file;
char *xtrabackup_tables_exclude;
150 151
char *xb_rocksdb_datadir;
my_bool xb_backup_rocksdb = 1;
Vladislav Vaintroub's avatar
Vladislav Vaintroub committed
152 153 154 155 156 157 158

typedef std::list<regex_t> regex_list_t;
static regex_list_t regex_include_list;
static regex_list_t regex_exclude_list;

static hash_table_t* tables_include_hash = NULL;
static hash_table_t* tables_exclude_hash = NULL;
159 160 161

char *xtrabackup_databases = NULL;
char *xtrabackup_databases_file = NULL;
Vladislav Vaintroub's avatar
Vladislav Vaintroub committed
162 163 164
char *xtrabackup_databases_exclude = NULL;
static hash_table_t* databases_include_hash = NULL;
static hash_table_t* databases_exclude_hash = NULL;
165 166 167 168 169 170 171 172 173 174 175 176

static hash_table_t* inc_dir_tables_hash;

struct xb_filter_entry_struct{
	char*		name;
	ibool		has_tables;
	hash_node_t	name_hash;
};
typedef struct xb_filter_entry_struct	xb_filter_entry_t;

lsn_t checkpoint_lsn_start;
lsn_t checkpoint_no_start;
177 178 179
static lsn_t log_copy_scanned_lsn;
static bool log_copying_running;
static bool io_watching_thread_running;
180 181 182 183 184 185 186 187

int xtrabackup_parallel;

char *xtrabackup_stream_str = NULL;
xb_stream_fmt_t xtrabackup_stream_fmt = XB_STREAM_FMT_NONE;
ibool xtrabackup_stream = FALSE;

const char *xtrabackup_compress_alg = NULL;
188
uint xtrabackup_compress = FALSE;
189 190 191 192 193 194
uint xtrabackup_compress_threads;
ulonglong xtrabackup_compress_chunk_size = 0;

/* sleep interval beetween log copy iterations in log copying thread
in milliseconds (default is 1 second) */
ulint xtrabackup_log_copy_interval = 1000;
195
static ulong max_buf_pool_modified_pct;
196 197

/* Ignored option (--log) for MySQL option compatibility */
198
static char*	log_ignored_opt;
199

200 201 202

extern my_bool opt_use_ssl;
my_bool opt_ssl_verify_server_cert;
203
my_bool opt_extended_validation;
Vladislav Vaintroub's avatar
Vladislav Vaintroub committed
204
my_bool opt_encrypted_backup;
205

206 207
/* === metadata of backup === */
#define XTRABACKUP_METADATA_FILENAME "xtrabackup_checkpoints"
208
char metadata_type[30] = ""; /*[full-backuped|log-applied|incremental]*/
209
static lsn_t metadata_from_lsn;
210
lsn_t metadata_to_lsn;
211
static lsn_t metadata_last_lsn;
212

213
static ds_file_t*	dst_log_file;
214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239

static char mysql_data_home_buff[2];

const char *defaults_group = "mysqld";

/* === static parameters in ha_innodb.cc */

#define HA_INNOBASE_ROWS_IN_TABLE 10000 /* to get optimization right */
#define HA_INNOBASE_RANGE_COUNT	  100

ulong 	innobase_large_page_size = 0;

/* The default values for the following, type long or longlong, start-up
parameters are declared in mysqld.cc: */

long innobase_buffer_pool_awe_mem_mb = 0;
long innobase_file_io_threads = 4;
long innobase_read_io_threads = 4;
long innobase_write_io_threads = 4;

longlong innobase_page_size = (1LL << 14); /* 16KB */
char*	innobase_buffer_pool_filename = NULL;

/* The default values for the following char* start-up parameters
are determined in innobase_init below: */

240 241 242
static char*	innobase_ignored_opt;
char*	innobase_data_home_dir;
char*	innobase_data_file_path;
243

244 245 246 247 248 249
my_bool innobase_use_doublewrite;
my_bool innobase_use_large_pages;
my_bool	innobase_file_per_table;
my_bool innobase_locks_unsafe_for_binlog;
my_bool innobase_rollback_on_timeout;
my_bool innobase_create_status_file;
250 251 252 253 254 255 256 257 258

/* The following counter is used to convey information to InnoDB
about server activity: in selects it is not sensible to call
srv_active_wake_master_thread after each fetch or search, we only do
it every INNOBASE_WAKE_INTERVAL'th step. */

#define INNOBASE_WAKE_INTERVAL	32
ulong	innobase_active_counter	= 0;

259
#ifndef _WIN32
260
static char *xtrabackup_debug_sync = NULL;
261
#endif
262 263 264 265

my_bool xtrabackup_incremental_force_scan = FALSE;

/* The flushed lsn which is read from data files */
266
lsn_t	flushed_lsn= 0;
267 268

ulong xb_open_files_limit= 0;
269 270
char *xb_plugin_dir;
char *xb_plugin_load;
271
my_bool xb_close_files;
272 273 274 275 276 277 278 279 280 281 282 283 284 285 286 287 288 289

/* Datasinks */
ds_ctxt_t       *ds_data     = NULL;
ds_ctxt_t       *ds_meta     = NULL;
ds_ctxt_t       *ds_redo     = NULL;

static bool	innobackupex_mode = false;

/* String buffer used by --print-param to accumulate server options as they are
parsed from the defaults file */
static std::ostringstream print_param_str;

/* Set of specified parameters */
std::set<std::string> param_set;

static ulonglong global_max_value;

extern "C" sig_handler handle_fatal_signal(int sig);
290
extern LOGGER logger;
291 292 293 294 295 296 297 298 299 300

my_bool opt_galera_info = FALSE;
my_bool opt_slave_info = FALSE;
my_bool opt_no_lock = FALSE;
my_bool opt_safe_slave_backup = FALSE;
my_bool opt_rsync = FALSE;
my_bool opt_force_non_empty_dirs = FALSE;
my_bool opt_noversioncheck = FALSE;
my_bool opt_no_backup_locks = FALSE;
my_bool opt_decompress = FALSE;
Marko Mäkelä's avatar
Marko Mäkelä committed
301
my_bool opt_remove_original;
302

303
my_bool opt_lock_ddl_per_table = FALSE;
304
static my_bool opt_check_privileges;
305

306 307 308 309 310
extern const char *innodb_checksum_algorithm_names[];
extern TYPELIB innodb_checksum_algorithm_typelib;
extern const char *innodb_flush_method_names[];
extern TYPELIB innodb_flush_method_typelib;

311 312 313 314 315 316
static const char *binlog_info_values[] = {"off", "lockless", "on", "auto",
					   NullS};
static TYPELIB binlog_info_typelib = {array_elements(binlog_info_values)-1, "",
				      binlog_info_values, NULL};
ulong opt_binlog_info;

317 318
char *opt_incremental_history_name;
char *opt_incremental_history_uuid;
319

320 321 322 323 324 325 326
char *opt_user;
char *opt_password;
char *opt_host;
char *opt_defaults_group;
char *opt_socket;
uint opt_port;
char *opt_log_bin;
327 328 329 330 331 332 333 334 335 336 337 338 339 340 341 342 343 344

const char *query_type_names[] = { "ALL", "UPDATE", "SELECT", NullS};

TYPELIB query_type_typelib= {array_elements(query_type_names) - 1, "",
	query_type_names, NULL};

ulong opt_lock_wait_query_type;
ulong opt_kill_long_query_type;

uint opt_kill_long_queries_timeout = 0;
uint opt_lock_wait_timeout = 0;
uint opt_lock_wait_threshold = 0;
uint opt_debug_sleep_before_unlock = 0;
uint opt_safe_slave_backup_timeout = 0;

const char *opt_history = NULL;


345 346 347
char mariabackup_exe[FN_REFLEN];
char orig_argv1[FN_REFLEN];

348 349 350 351 352 353 354 355 356 357 358 359 360 361 362 363
pthread_mutex_t backup_mutex;
pthread_cond_t  scanned_lsn_cond;

typedef std::map<space_id_t,std::string> space_id_to_name_t;

struct ddl_tracker_t {
	/** Tablspaces with their ID and name, as they were copied to backup.*/
	space_id_to_name_t tables_in_backup;
	/** Tablespaces for that optimized DDL without redo log was found.*/
	std::set<space_id_t> optimized_ddl;
	/** Drop operations found in redo log. */
	std::set<space_id_t> drops;
	/* For DDL operation found in redo log,  */
	space_id_to_name_t id_to_name;
};

364
static ddl_tracker_t ddl_tracker;
365

366 367 368 369 370 371 372 373 374 375 376 377 378
/* Simple datasink creation tracking...add datasinks in the reverse order you
want them destroyed. */
#define XTRABACKUP_MAX_DATASINKS	10
static	ds_ctxt_t	*datasinks[XTRABACKUP_MAX_DATASINKS];
static	uint		actual_datasinks = 0;
static inline
void
xtrabackup_add_datasink(ds_ctxt_t *ds)
{
	xb_ad(actual_datasinks < XTRABACKUP_MAX_DATASINKS);
	datasinks[actual_datasinks] = ds; actual_datasinks++;
}

379 380 381 382 383

typedef void (*process_single_tablespace_func_t)(const char *dirname, const char *filname, bool is_remote);
static dberr_t enumerate_ibd_files(process_single_tablespace_func_t callback);


384
/* ======== Datafiles iterator ======== */
385 386 387 388 389 390 391 392 393
struct datafiles_iter_t {
	fil_space_t	*space;
	fil_node_t	*node;
	ibool		started;
	pthread_mutex_t	mutex;
};

/* ======== Datafiles iterator ======== */
static
394
datafiles_iter_t *
395
datafiles_iter_new()
396 397 398
{
	datafiles_iter_t *it;

399 400
	it = static_cast<datafiles_iter_t *>(malloc(sizeof(datafiles_iter_t)));
	pthread_mutex_init(&it->mutex, NULL);
401 402 403 404 405 406 407 408

	it->space = NULL;
	it->node = NULL;
	it->started = FALSE;

	return it;
}

409
static
410 411 412 413 414
fil_node_t *
datafiles_iter_next(datafiles_iter_t *it)
{
	fil_node_t *new_node;

415
	pthread_mutex_lock(&it->mutex);
416 417 418 419 420 421 422 423 424 425 426 427

	if (it->node == NULL) {
		if (it->started)
			goto end;
		it->started = TRUE;
	} else {
		it->node = UT_LIST_GET_NEXT(chain, it->node);
		if (it->node != NULL)
			goto end;
	}

	it->space = (it->space == NULL) ?
428
		UT_LIST_GET_FIRST(fil_system.space_list) :
429 430 431
		UT_LIST_GET_NEXT(space_list, it->space);

	while (it->space != NULL &&
432
	       (it->space->purpose != FIL_TYPE_TABLESPACE ||
433 434 435 436 437 438 439 440 441
		UT_LIST_GET_LEN(it->space->chain) == 0))
		it->space = UT_LIST_GET_NEXT(space_list, it->space);
	if (it->space == NULL)
		goto end;

	it->node = UT_LIST_GET_FIRST(it->space->chain);

end:
	new_node = it->node;
442
	pthread_mutex_unlock(&it->mutex);
443 444 445 446

	return new_node;
}

447
static
448 449 450
void
datafiles_iter_free(datafiles_iter_t *it)
{
451 452
	pthread_mutex_destroy(&it->mutex);
	free(it);
453 454
}

455 456 457 458 459 460 461 462 463 464 465 466 467 468 469 470 471 472 473 474 475 476 477 478 479 480 481 482 483 484 485 486 487 488 489 490 491 492 493 494 495 496 497 498 499 500 501 502 503 504 505 506 507 508 509 510 511 512 513 514 515 516 517 518 519 520 521 522
#ifndef DBUG_OFF
struct dbug_thread_param_t
{
	MYSQL *con;
	const char *query;
	int expect_err;
	int expect_errno;
	os_event_t done_event;
};


/* Thread procedure used in dbug_start_query_thread. */
extern "C"
os_thread_ret_t
DECLARE_THREAD(dbug_execute_in_new_connection)(void *arg)
{
	mysql_thread_init();
	dbug_thread_param_t *par= (dbug_thread_param_t *)arg;
	int err = mysql_query(par->con, par->query);
	int err_no = mysql_errno(par->con);
	DBUG_ASSERT(par->expect_err == err);
	if (err && par->expect_errno)
		DBUG_ASSERT(err_no == par->expect_errno);
	mysql_close(par->con);
	mysql_thread_end();
	os_event_t done = par->done_event;
	delete par;
	os_event_set(done);
	os_thread_exit();
	return os_thread_ret_t(0);
}

/*
Execute query from a new connection, in own thread.

@param query - query to be executed
@param wait_state - if not NULL, wait until query from new connection
	reaches this state (value of column State in I_S.PROCESSLIST)
@param expected_err - if 0, query is supposed to finish successfully,
	otherwise query should return error.
@param expected_errno - if not 0, and query finished with error,
	expected mysql_errno()
*/
static os_event_t dbug_start_query_thread(
	const char *query,
	const char *wait_state,
	int expected_err,
	int expected_errno)

{
	dbug_thread_param_t *par = new dbug_thread_param_t;
	par->query = query;
	par->expect_err = expected_err;
	par->expect_errno = expected_errno;
	par->done_event = os_event_create(0);
	par->con =  xb_mysql_connect();
	os_thread_create(dbug_execute_in_new_connection, par, 0);

	if (!wait_state)
		return par->done_event;

	char q[256];
	snprintf(q, sizeof(q),
		"SELECT 1 FROM INFORMATION_SCHEMA.PROCESSLIST where ID=%lu"
		" AND Command='Query' AND State='%s'",
		mysql_thread_id(par->con), wait_state);
	for (;;) {
		MYSQL_RES *result = xb_mysql_query(mysql_connection,q, true, true);
523 524 525
		bool exists = mysql_fetch_row(result) != NULL;
		mysql_free_result(result);
		if (exists) {
526 527
			goto end;
		}
528
		msg("Waiting for query '%s' on connection %lu to "
529 530 531 532 533
			" reach state '%s'", query, mysql_thread_id(par->con),
			wait_state);
		my_sleep(1000);
	}
end:
534
	msg("query '%s' on connection %lu reached state '%s'", query,
535 536 537 538 539 540 541
	mysql_thread_id(par->con), wait_state);
	return par->done_event;
}

os_event_t dbug_alter_thread_done;
#endif

542 543 544
void mdl_lock_all()
{
	mdl_lock_init();
545
	datafiles_iter_t *it = datafiles_iter_new();
546 547 548 549 550 551 552 553 554 555 556 557
	if (!it)
		return;

	while (fil_node_t *node = datafiles_iter_next(it)){
		if (fil_is_user_tablespace_id(node->space->id)
			&& check_if_skip_table(node->space->name))
			continue;

		mdl_lock_table(node->space->id);
	}
	datafiles_iter_free(it);
}
558 559


560 561 562 563 564 565 566 567 568 569 570 571 572 573 574 575 576 577 578 579
// Convert non-null terminated filename to space name
std::string filename_to_spacename(const byte *filename, size_t len)
{
	// null- terminate filename
	char *f = (char *)malloc(len + 1);
	ut_a(f);
	memcpy(f, filename, len);
	f[len] = 0;
	for (size_t i = 0; i < len; i++)
		if (f[i] == '\\')
			f[i] = '/';
	char *p = strrchr(f, '.');
	ut_a(p);
	*p = 0;
	char *table = strrchr(f, '/');
	ut_a(table);
	*table = 0;
	char *db = strrchr(f, '/');
	ut_a(db);
	*table = '/';
580 581 582
	std::string s(db+1);
	free(f);
	return s;
583
}
584

585 586 587 588 589 590 591
/** Report an operation to create, delete, or rename a file during backup.
@param[in]	space_id	tablespace identifier
@param[in]	flags		tablespace flags (NULL if not create)
@param[in]	name		file name (not NUL-terminated)
@param[in]	len		length of name, in bytes
@param[in]	new_name	new file name (NULL if not rename)
@param[in]	new_len		length of new_name, in bytes (0 if NULL) */
592
static void backup_file_op(ulint space_id, const byte* flags,
593 594 595
	const byte* name, ulint len,
	const byte* new_name, ulint new_len)
{
596

597 598 599 600 601 602 603 604
	ut_ad(!flags || !new_name);
	ut_ad(name);
	ut_ad(len);
	ut_ad(!new_name == !new_len);
	pthread_mutex_lock(&backup_mutex);

	if (flags) {
		ddl_tracker.id_to_name[space_id] = filename_to_spacename(name, len);
605
		msg("DDL tracking :  create %zu \"%.*s\": %x",
606 607 608 609
			space_id, int(len), name, mach_read_from_4(flags));
	}
	else if (new_name) {
		ddl_tracker.id_to_name[space_id] = filename_to_spacename(new_name, new_len);
610
		msg("DDL tracking : rename %zu \"%.*s\",\"%.*s\"",
611 612 613
			space_id, int(len), name, int(new_len), new_name);
	} else {
		ddl_tracker.drops.insert(space_id);
614
		msg("DDL tracking : delete %zu \"%.*s\"", space_id, int(len), name);
615 616 617
	}
	pthread_mutex_unlock(&backup_mutex);
}
618 619


620 621 622 623 624 625 626 627 628 629 630 631 632 633 634 635
/*
 This callback is called if DDL operation is detected,
 at the end of backup

 Normally, DDL operations are blocked due to FTWRL,
 but in rare cases of --no-lock, they are not.

 We will abort backup in this case.
*/
static void backup_file_op_fail(ulint space_id, const byte* flags,
	const byte* name, ulint len,
	const byte* new_name, ulint new_len)
{
	ut_a(opt_no_lock);
	bool fail;
	if (flags) {
636
		msg("DDL tracking :  create %zu \"%.*s\": %x",
637 638 639 640 641
			space_id, int(len), name, mach_read_from_4(flags));
		std::string  spacename = filename_to_spacename(name, len);
		fail = !check_if_skip_table(spacename.c_str());
	}
	else if (new_name) {
642
		msg("DDL tracking : rename %zu \"%.*s\",\"%.*s\"",
643 644 645 646 647 648 649 650
			space_id, int(len), name, int(new_len), new_name);
		std::string  spacename = filename_to_spacename(name, len);
		std::string  new_spacename = filename_to_spacename(new_name, new_len);
		fail = !check_if_skip_table(spacename.c_str()) || !check_if_skip_table(new_spacename.c_str());
	}
	else {
		std::string  spacename = filename_to_spacename(name, len);
		fail = !check_if_skip_table(spacename.c_str());
651
		msg("DDL tracking : delete %zu \"%.*s\"", space_id, int(len), name);
652 653
	}
	if (fail) {
654
		die("DDL operation detected in the late phase of backup."
655
			"Backup is inconsistent. Remove --no-lock option to fix.");
656 657 658 659
	}
}


660
/** Callback whenever MLOG_INDEX_LOAD happens.
661 662
@param[in]	space_id	space id to check */
static void backup_optimized_ddl_op(ulint space_id)
663 664 665 666
{
	pthread_mutex_lock(&backup_mutex);
	ddl_tracker.optimized_ddl.insert(space_id);
	pthread_mutex_unlock(&backup_mutex);
667 668
}

669 670 671 672 673 674
/*
  Optimized DDL callback at the end of backup that
  run with --no-lock. Usually aborts the backup.
*/
static void backup_optimized_ddl_op_fail(ulint space_id) {
	ut_a(opt_no_lock);
675
	msg("DDL tracking : optimized DDL on space %zu", space_id);
676 677
	if (ddl_tracker.tables_in_backup.find(space_id) != ddl_tracker.tables_in_backup.end()) {
		msg("ERROR : Optimized DDL operation detected in the late phase of backup."
678
			"Backup is inconsistent. Remove --no-lock option to fix.");
679 680 681 682 683
		exit(EXIT_FAILURE);
	}
}


684 685 686
/** Callback whenever MLOG_TRUNCATE happens. */
static void backup_truncate_fail()
{
687
	msg("mariabackup: Incompatible TRUNCATE operation detected.%s",
688 689 690 691 692
	    opt_lock_ddl_per_table
	    ? ""
	    : " Use --lock-ddl-per-table to lock all tables before backup.");
}

693 694 695 696 697 698 699 700 701 702 703 704 705 706 707 708 709 710 711 712 713 714 715 716 717 718 719 720 721 722 723 724

/*
  Retrieve default data directory, to be used with --copy-back.

  On Windows, default datadir is ..\data, relative to the
  directory where mariabackup.exe is located(usually "bin")

  Elsewhere, the compiled-in constant MYSQL_DATADIR is used.
*/
static char *get_default_datadir() {
	static char ddir[] = MYSQL_DATADIR;
#ifdef _WIN32
	static char buf[MAX_PATH];
	DWORD size = (DWORD)sizeof(buf) - 1;
	if (GetModuleFileName(NULL, buf, size) <= size)
	{
		char *p;
		if ((p = strrchr(buf, '\\')))
		{
			*p = 0;
			if ((p = strrchr(buf, '\\')))
			{
				strncpy(p + 1, "data", buf + MAX_PATH - p);
				return buf;
			}
		}
	}
#endif
	return ddir;
}


725 726 727 728 729 730
/* ======== Date copying thread context ======== */

typedef struct {
	datafiles_iter_t 	*it;
	uint			num;
	uint			*count;
731
	pthread_mutex_t*	count_mutex;
732 733 734 735
	os_thread_id_t		id;
} data_thread_ctxt_t;

/* ======== for option and variables ======== */
736
#include <../../client/client_priv.h>
737 738 739 740 741 742 743 744 745 746 747 748 749 750 751 752 753 754 755 756 757

enum options_xtrabackup
{
  OPT_XTRA_TARGET_DIR = 1000,     /* make sure it is larger
                                     than OPT_MAX_CLIENT_OPTION */
  OPT_XTRA_BACKUP,
  OPT_XTRA_PREPARE,
  OPT_XTRA_EXPORT,
  OPT_XTRA_PRINT_PARAM,
  OPT_XTRA_USE_MEMORY,
  OPT_XTRA_THROTTLE,
  OPT_XTRA_LOG_COPY_INTERVAL,
  OPT_XTRA_INCREMENTAL,
  OPT_XTRA_INCREMENTAL_BASEDIR,
  OPT_XTRA_EXTRA_LSNDIR,
  OPT_XTRA_INCREMENTAL_DIR,
  OPT_XTRA_TABLES,
  OPT_XTRA_TABLES_FILE,
  OPT_XTRA_DATABASES,
  OPT_XTRA_DATABASES_FILE,
  OPT_XTRA_PARALLEL,
758
  OPT_XTRA_EXTENDED_VALIDATION,
Vladislav Vaintroub's avatar
Vladislav Vaintroub committed
759
  OPT_XTRA_ENCRYPTED_BACKUP,
760 761 762 763 764 765 766 767 768 769 770
  OPT_XTRA_STREAM,
  OPT_XTRA_COMPRESS,
  OPT_XTRA_COMPRESS_THREADS,
  OPT_XTRA_COMPRESS_CHUNK_SIZE,
  OPT_LOG,
  OPT_INNODB,
  OPT_INNODB_DATA_FILE_PATH,
  OPT_INNODB_DATA_HOME_DIR,
  OPT_INNODB_ADAPTIVE_HASH_INDEX,
  OPT_INNODB_DOUBLEWRITE,
  OPT_INNODB_FILE_PER_TABLE,
771
  OPT_INNODB_FLUSH_METHOD,
772 773 774 775 776 777 778 779 780 781 782 783 784 785 786 787 788 789 790 791 792 793 794 795
  OPT_INNODB_LOG_GROUP_HOME_DIR,
  OPT_INNODB_MAX_DIRTY_PAGES_PCT,
  OPT_INNODB_MAX_PURGE_LAG,
  OPT_INNODB_STATUS_FILE,
  OPT_INNODB_AUTOEXTEND_INCREMENT,
  OPT_INNODB_BUFFER_POOL_SIZE,
  OPT_INNODB_COMMIT_CONCURRENCY,
  OPT_INNODB_CONCURRENCY_TICKETS,
  OPT_INNODB_FILE_IO_THREADS,
  OPT_INNODB_IO_CAPACITY,
  OPT_INNODB_READ_IO_THREADS,
  OPT_INNODB_WRITE_IO_THREADS,
  OPT_INNODB_USE_NATIVE_AIO,
  OPT_INNODB_PAGE_SIZE,
  OPT_INNODB_BUFFER_POOL_FILENAME,
  OPT_INNODB_LOCK_WAIT_TIMEOUT,
  OPT_INNODB_LOG_BUFFER_SIZE,
  OPT_INNODB_LOG_FILE_SIZE,
  OPT_INNODB_LOG_FILES_IN_GROUP,
  OPT_INNODB_OPEN_FILES,
  OPT_XTRA_DEBUG_SYNC,
  OPT_INNODB_CHECKSUM_ALGORITHM,
  OPT_INNODB_UNDO_DIRECTORY,
  OPT_INNODB_UNDO_TABLESPACES,
796
  OPT_INNODB_LOG_CHECKSUMS,
797 798 799 800 801 802 803 804 805 806 807 808 809 810 811 812 813 814 815 816 817 818 819 820 821 822 823 824 825 826
  OPT_XTRA_INCREMENTAL_FORCE_SCAN,
  OPT_DEFAULTS_GROUP,
  OPT_CLOSE_FILES,
  OPT_CORE_FILE,

  OPT_COPY_BACK,
  OPT_MOVE_BACK,
  OPT_GALERA_INFO,
  OPT_SLAVE_INFO,
  OPT_NO_LOCK,
  OPT_SAFE_SLAVE_BACKUP,
  OPT_RSYNC,
  OPT_FORCE_NON_EMPTY_DIRS,
  OPT_NO_VERSION_CHECK,
  OPT_NO_BACKUP_LOCKS,
  OPT_DECOMPRESS,
  OPT_INCREMENTAL_HISTORY_NAME,
  OPT_INCREMENTAL_HISTORY_UUID,
  OPT_REMOVE_ORIGINAL,
  OPT_LOCK_WAIT_QUERY_TYPE,
  OPT_KILL_LONG_QUERY_TYPE,
  OPT_HISTORY,
  OPT_KILL_LONG_QUERIES_TIMEOUT,
  OPT_LOCK_WAIT_TIMEOUT,
  OPT_LOCK_WAIT_THRESHOLD,
  OPT_DEBUG_SLEEP_BEFORE_UNLOCK,
  OPT_SAFE_SLAVE_BACKUP_TIMEOUT,
  OPT_BINLOG_INFO,
  OPT_XB_SECURE_AUTH,

Vladislav Vaintroub's avatar
Vladislav Vaintroub committed
827 828
  OPT_XTRA_TABLES_EXCLUDE,
  OPT_XTRA_DATABASES_EXCLUDE,
829
  OPT_PROTOCOL,
Marko Mäkelä's avatar
Marko Mäkelä committed
830
  OPT_INNODB_COMPRESSION_LEVEL,
831 832
  OPT_LOCK_DDL_PER_TABLE,
  OPT_ROCKSDB_DATADIR,
833 834
  OPT_BACKUP_ROCKSDB,
  OPT_XTRA_CHECK_PRIVILEGES
835 836 837 838
};

struct my_option xb_client_options[] =
{
839 840 841
  {"verbose", 'V', "display verbose output",
   (G_PTR*) &verbose, (G_PTR*) &verbose, 0, GET_BOOL, NO_ARG,
   FALSE, 0, 0, 0, 0, 0},
842 843 844 845 846 847 848 849 850 851 852 853 854 855 856 857 858 859 860 861 862 863 864 865 866 867 868 869 870 871 872 873 874 875 876 877 878 879 880 881 882 883 884 885 886 887 888 889 890 891 892
  {"version", 'v', "print xtrabackup version information",
   (G_PTR *) &xtrabackup_version, (G_PTR *) &xtrabackup_version, 0, GET_BOOL,
   NO_ARG, 0, 0, 0, 0, 0, 0},
  {"target-dir", OPT_XTRA_TARGET_DIR, "destination directory", (G_PTR*) &xtrabackup_target_dir,
   (G_PTR*) &xtrabackup_target_dir, 0, GET_STR, REQUIRED_ARG, 0, 0, 0, 0, 0, 0},
  {"backup", OPT_XTRA_BACKUP, "take backup to target-dir",
   (G_PTR*) &xtrabackup_backup, (G_PTR*) &xtrabackup_backup,
   0, GET_BOOL, NO_ARG, 0, 0, 0, 0, 0, 0},
  {"prepare", OPT_XTRA_PREPARE, "prepare a backup for starting mysql server on the backup.",
   (G_PTR*) &xtrabackup_prepare, (G_PTR*) &xtrabackup_prepare,
   0, GET_BOOL, NO_ARG, 0, 0, 0, 0, 0, 0},
  {"export", OPT_XTRA_EXPORT, "create files to import to another database when prepare.",
   (G_PTR*) &xtrabackup_export, (G_PTR*) &xtrabackup_export,
   0, GET_BOOL, NO_ARG, 0, 0, 0, 0, 0, 0},
  {"print-param", OPT_XTRA_PRINT_PARAM, "print parameter of mysqld needed for copyback.",
   (G_PTR*) &xtrabackup_print_param, (G_PTR*) &xtrabackup_print_param,
   0, GET_BOOL, NO_ARG, 0, 0, 0, 0, 0, 0},
  {"use-memory", OPT_XTRA_USE_MEMORY, "The value is used instead of buffer_pool_size",
   (G_PTR*) &xtrabackup_use_memory, (G_PTR*) &xtrabackup_use_memory,
   0, GET_LL, REQUIRED_ARG, 100*1024*1024L, 1024*1024L, LONGLONG_MAX, 0,
   1024*1024L, 0},
  {"throttle", OPT_XTRA_THROTTLE, "limit count of IO operations (pairs of read&write) per second to IOS values (for '--backup')",
   (G_PTR*) &xtrabackup_throttle, (G_PTR*) &xtrabackup_throttle,
   0, GET_LONG, REQUIRED_ARG, 0, 0, LONG_MAX, 0, 1, 0},
  {"log", OPT_LOG, "Ignored option for MySQL option compatibility",
   (G_PTR*) &log_ignored_opt, (G_PTR*) &log_ignored_opt, 0,
   GET_STR, OPT_ARG, 0, 0, 0, 0, 0, 0},
  {"log-copy-interval", OPT_XTRA_LOG_COPY_INTERVAL, "time interval between checks done by log copying thread in milliseconds (default is 1 second).",
   (G_PTR*) &xtrabackup_log_copy_interval, (G_PTR*) &xtrabackup_log_copy_interval,
   0, GET_LONG, REQUIRED_ARG, 1000, 0, LONG_MAX, 0, 1, 0},
  {"extra-lsndir", OPT_XTRA_EXTRA_LSNDIR, "(for --backup): save an extra copy of the xtrabackup_checkpoints file in this directory.",
   (G_PTR*) &xtrabackup_extra_lsndir, (G_PTR*) &xtrabackup_extra_lsndir,
   0, GET_STR, REQUIRED_ARG, 0, 0, 0, 0, 0, 0},
  {"incremental-lsn", OPT_XTRA_INCREMENTAL, "(for --backup): copy only .ibd pages newer than specified LSN 'high:low'. ##ATTENTION##: If a wrong LSN value is specified, it is impossible to diagnose this, causing the backup to be unusable. Be careful!",
   (G_PTR*) &xtrabackup_incremental, (G_PTR*) &xtrabackup_incremental,
   0, GET_STR, REQUIRED_ARG, 0, 0, 0, 0, 0, 0},
  {"incremental-basedir", OPT_XTRA_INCREMENTAL_BASEDIR, "(for --backup): copy only .ibd pages newer than backup at specified directory.",
   (G_PTR*) &xtrabackup_incremental_basedir, (G_PTR*) &xtrabackup_incremental_basedir,
   0, GET_STR, REQUIRED_ARG, 0, 0, 0, 0, 0, 0},
  {"incremental-dir", OPT_XTRA_INCREMENTAL_DIR, "(for --prepare): apply .delta files and logfile in the specified directory.",
   (G_PTR*) &xtrabackup_incremental_dir, (G_PTR*) &xtrabackup_incremental_dir,
   0, GET_STR, REQUIRED_ARG, 0, 0, 0, 0, 0, 0},
  {"tables", OPT_XTRA_TABLES, "filtering by regexp for table names.",
   (G_PTR*) &xtrabackup_tables, (G_PTR*) &xtrabackup_tables,
   0, GET_STR, REQUIRED_ARG, 0, 0, 0, 0, 0, 0},
  {"tables_file", OPT_XTRA_TABLES_FILE, "filtering by list of the exact database.table name in the file.",
   (G_PTR*) &xtrabackup_tables_file, (G_PTR*) &xtrabackup_tables_file,
   0, GET_STR, REQUIRED_ARG, 0, 0, 0, 0, 0, 0},
  {"databases", OPT_XTRA_DATABASES, "filtering by list of databases.",
   (G_PTR*) &xtrabackup_databases, (G_PTR*) &xtrabackup_databases,
   0, GET_STR, REQUIRED_ARG, 0, 0, 0, 0, 0, 0},
893
  {"databases_file", OPT_XTRA_DATABASES_FILE,
894 895 896
   "filtering by list of databases in the file.",
   (G_PTR*) &xtrabackup_databases_file, (G_PTR*) &xtrabackup_databases_file,
   0, GET_STR, REQUIRED_ARG, 0, 0, 0, 0, 0, 0},
Vladislav Vaintroub's avatar
Vladislav Vaintroub committed
897 898 899 900 901 902 903 904 905 906
  {"tables-exclude", OPT_XTRA_TABLES_EXCLUDE, "filtering by regexp for table names. "
  "Operates the same way as --tables, but matched names are excluded from backup. "
  "Note that this option has a higher priority than --tables.",
    (G_PTR*) &xtrabackup_tables_exclude, (G_PTR*) &xtrabackup_tables_exclude,
    0, GET_STR, REQUIRED_ARG, 0, 0, 0, 0, 0, 0},
  {"databases-exclude", OPT_XTRA_DATABASES_EXCLUDE, "Excluding databases based on name, "
  "Operates the same way as --databases, but matched names are excluded from backup. "
  "Note that this option has a higher priority than --databases.",
    (G_PTR*) &xtrabackup_databases_exclude, (G_PTR*) &xtrabackup_databases_exclude,
    0, GET_STR, REQUIRED_ARG, 0, 0, 0, 0, 0, 0},
907 908

  {"stream", OPT_XTRA_STREAM, "Stream all backup files to the standard output "
909 910 911
   "in the specified format." 
   "Supported format is 'xbstream'."
   ,
912 913 914 915 916 917 918 919 920 921 922 923 924 925 926 927 928 929 930 931 932 933 934 935 936 937 938 939 940 941 942 943 944 945 946 947 948 949 950 951 952 953 954 955 956 957 958 959 960 961
   (G_PTR*) &xtrabackup_stream_str, (G_PTR*) &xtrabackup_stream_str, 0, GET_STR,
   REQUIRED_ARG, 0, 0, 0, 0, 0, 0},

  {"compress", OPT_XTRA_COMPRESS, "Compress individual backup files using the "
   "specified compression algorithm. Currently the only supported algorithm "
   "is 'quicklz'. It is also the default algorithm, i.e. the one used when "
   "--compress is used without an argument.",
   (G_PTR*) &xtrabackup_compress_alg, (G_PTR*) &xtrabackup_compress_alg, 0,
   GET_STR, OPT_ARG, 0, 0, 0, 0, 0, 0},

  {"compress-threads", OPT_XTRA_COMPRESS_THREADS,
   "Number of threads for parallel data compression. The default value is 1.",
   (G_PTR*) &xtrabackup_compress_threads, (G_PTR*) &xtrabackup_compress_threads,
   0, GET_UINT, REQUIRED_ARG, 1, 1, UINT_MAX, 0, 0, 0},

  {"compress-chunk-size", OPT_XTRA_COMPRESS_CHUNK_SIZE,
   "Size of working buffer(s) for compression threads in bytes. The default value is 64K.",
   (G_PTR*) &xtrabackup_compress_chunk_size, (G_PTR*) &xtrabackup_compress_chunk_size,
   0, GET_ULL, REQUIRED_ARG, (1 << 16), 1024, ULONGLONG_MAX, 0, 0, 0},

  {"incremental-force-scan", OPT_XTRA_INCREMENTAL_FORCE_SCAN,
   "Perform a full-scan incremental backup even in the presence of changed "
   "page bitmap data",
   (G_PTR*)&xtrabackup_incremental_force_scan,
   (G_PTR*)&xtrabackup_incremental_force_scan, 0, GET_BOOL, NO_ARG,
   0, 0, 0, 0, 0, 0},


  {"close_files", OPT_CLOSE_FILES, "do not keep files opened. Use at your own "
   "risk.", (G_PTR*) &xb_close_files, (G_PTR*) &xb_close_files, 0, GET_BOOL,
   NO_ARG, 0, 0, 0, 0, 0, 0},

  {"core-file", OPT_CORE_FILE, "Write core on fatal signals", 0, 0, 0,
   GET_NO_ARG, NO_ARG, 0, 0, 0, 0, 0, 0},


  {"copy-back", OPT_COPY_BACK, "Copy all the files in a previously made "
   "backup from the backup directory to their original locations.",
   (uchar *) &xtrabackup_copy_back, (uchar *) &xtrabackup_copy_back, 0,
   GET_BOOL, NO_ARG, 0, 0, 0, 0, 0, 0},

  {"move-back", OPT_MOVE_BACK, "Move all the files in a previously made "
   "backup from the backup directory to the actual datadir location. "
   "Use with caution, as it removes backup files.",
   (uchar *) &xtrabackup_move_back, (uchar *) &xtrabackup_move_back, 0,
   GET_BOOL, NO_ARG, 0, 0, 0, 0, 0, 0},

  {"galera-info", OPT_GALERA_INFO, "This options creates the "
   "xtrabackup_galera_info file which contains the local node state at "
   "the time of the backup. Option should be used when performing the "
962
   "backup of MariaDB Galera Cluster. Has no effect when backup locks "
963 964 965 966 967 968 969 970 971 972 973 974 975 976 977 978 979 980 981 982 983 984 985 986 987 988 989 990 991 992 993 994 995 996 997 998 999 1000 1001 1002 1003 1004 1005 1006 1007 1008 1009 1010 1011 1012 1013 1014 1015 1016 1017 1018 1019 1020 1021 1022 1023 1024 1025 1026 1027 1028 1029 1030 1031 1032 1033 1034 1035 1036 1037 1038 1039 1040 1041 1042 1043 1044 1045 1046 1047 1048 1049 1050 1051 1052 1053 1054 1055 1056 1057 1058 1059 1060 1061 1062 1063 1064 1065 1066 1067 1068 1069
   "are used to create the backup.",
   (uchar *) &opt_galera_info, (uchar *) &opt_galera_info, 0,
   GET_BOOL, NO_ARG, 0, 0, 0, 0, 0, 0},

  {"slave-info", OPT_SLAVE_INFO, "This option is useful when backing "
   "up a replication slave server. It prints the binary log position "
   "and name of the master server. It also writes this information to "
   "the \"xtrabackup_slave_info\" file as a \"CHANGE MASTER\" command. "
   "A new slave for this master can be set up by starting a slave server "
   "on this backup and issuing a \"CHANGE MASTER\" command with the "
   "binary log position saved in the \"xtrabackup_slave_info\" file.",
   (uchar *) &opt_slave_info, (uchar *) &opt_slave_info, 0,
   GET_BOOL, NO_ARG, 0, 0, 0, 0, 0, 0},

  {"no-lock", OPT_NO_LOCK, "Use this option to disable table lock "
   "with \"FLUSH TABLES WITH READ LOCK\". Use it only if ALL your "
   "tables are InnoDB and you DO NOT CARE about the binary log "
   "position of the backup. This option shouldn't be used if there "
   "are any DDL statements being executed or if any updates are "
   "happening on non-InnoDB tables (this includes the system MyISAM "
   "tables in the mysql database), otherwise it could lead to an "
   "inconsistent backup. If you are considering to use --no-lock "
   "because your backups are failing to acquire the lock, this could "
   "be because of incoming replication events preventing the lock "
   "from succeeding. Please try using --safe-slave-backup to "
   "momentarily stop the replication slave thread, this may help "
   "the backup to succeed and you then don't need to resort to "
   "using this option.",
   (uchar *) &opt_no_lock, (uchar *) &opt_no_lock, 0,
   GET_BOOL, NO_ARG, 0, 0, 0, 0, 0, 0},

  {"safe-slave-backup", OPT_SAFE_SLAVE_BACKUP, "Stop slave SQL thread "
   "and wait to start backup until Slave_open_temp_tables in "
   "\"SHOW STATUS\" is zero. If there are no open temporary tables, "
   "the backup will take place, otherwise the SQL thread will be "
   "started and stopped until there are no open temporary tables. "
   "The backup will fail if Slave_open_temp_tables does not become "
   "zero after --safe-slave-backup-timeout seconds. The slave SQL "
   "thread will be restarted when the backup finishes.",
   (uchar *) &opt_safe_slave_backup,
   (uchar *) &opt_safe_slave_backup,
   0, GET_BOOL, NO_ARG, 0, 0, 0, 0, 0, 0},

  {"rsync", OPT_RSYNC, "Uses the rsync utility to optimize local file "
   "transfers. When this option is specified, innobackupex uses rsync "
   "to copy all non-InnoDB files instead of spawning a separate cp for "
   "each file, which can be much faster for servers with a large number "
   "of databases or tables.  This option cannot be used together with "
   "--stream.",
   (uchar *) &opt_rsync, (uchar *) &opt_rsync,
   0, GET_BOOL, NO_ARG, 0, 0, 0, 0, 0, 0},

  {"force-non-empty-directories", OPT_FORCE_NON_EMPTY_DIRS, "This "
   "option, when specified, makes --copy-back or --move-back transfer "
   "files to non-empty directories. Note that no existing files will be "
   "overwritten. If --copy-back or --nove-back has to copy a file from "
   "the backup directory which already exists in the destination "
   "directory, it will still fail with an error.",
   (uchar *) &opt_force_non_empty_dirs,
   (uchar *) &opt_force_non_empty_dirs,
   0, GET_BOOL, NO_ARG, 0, 0, 0, 0, 0, 0},

  {"no-version-check", OPT_NO_VERSION_CHECK, "This option disables the "
   "version check which is enabled by the --version-check option.",
   (uchar *) &opt_noversioncheck,
   (uchar *) &opt_noversioncheck,
   0, GET_BOOL, NO_ARG, 0, 0, 0, 0, 0, 0},

  {"no-backup-locks", OPT_NO_BACKUP_LOCKS, "This option controls if "
   "backup locks should be used instead of FLUSH TABLES WITH READ LOCK "
   "on the backup stage. The option has no effect when backup locks are "
   "not supported by the server. This option is enabled by default, "
   "disable with --no-backup-locks.",
   (uchar *) &opt_no_backup_locks,
   (uchar *) &opt_no_backup_locks,
   0, GET_BOOL, NO_ARG, 0, 0, 0, 0, 0, 0},

  {"decompress", OPT_DECOMPRESS, "Decompresses all files with the .qp "
   "extension in a backup previously made with the --compress option.",
   (uchar *) &opt_decompress,
   (uchar *) &opt_decompress,
   0, GET_BOOL, NO_ARG, 0, 0, 0, 0, 0, 0},

  {"user", 'u', "This option specifies the MySQL username used "
   "when connecting to the server, if that's not the current user. "
   "The option accepts a string argument. See mysql --help for details.",
   (uchar*) &opt_user, (uchar*) &opt_user, 0, GET_STR,
   REQUIRED_ARG, 0, 0, 0, 0, 0, 0},

  {"host", 'H', "This option specifies the host to use when "
   "connecting to the database server with TCP/IP.  The option accepts "
   "a string argument. See mysql --help for details.",
   (uchar*) &opt_host, (uchar*) &opt_host, 0, GET_STR,
   REQUIRED_ARG, 0, 0, 0, 0, 0, 0},

  {"port", 'P', "This option specifies the port to use when "
   "connecting to the database server with TCP/IP.  The option accepts "
   "a string argument. See mysql --help for details.",
   &opt_port, &opt_port, 0, GET_UINT, REQUIRED_ARG,
   0, 0, 0, 0, 0, 0},

  {"password", 'p', "This option specifies the password to use "
   "when connecting to the database. It accepts a string argument.  "
   "See mysql --help for details.",
   0, 0, 0, GET_STR,
   REQUIRED_ARG, 0, 0, 0, 0, 0, 0},

1070 1071 1072
  {"protocol", OPT_PROTOCOL, "The protocol to use for connection (tcp, socket, pipe, memory).",
   0, 0, 0, GET_STR, REQUIRED_ARG, 0, 0, 0, 0, 0, 0},

1073 1074 1075 1076 1077 1078 1079 1080 1081 1082 1083 1084 1085 1086 1087 1088 1089 1090 1091 1092 1093 1094 1095 1096 1097 1098 1099 1100 1101 1102 1103 1104
  {"socket", 'S', "This option specifies the socket to use when "
   "connecting to the local database server with a UNIX domain socket.  "
   "The option accepts a string argument. See mysql --help for details.",
   (uchar*) &opt_socket, (uchar*) &opt_socket, 0, GET_STR,
   REQUIRED_ARG, 0, 0, 0, 0, 0, 0},

  {"incremental-history-name", OPT_INCREMENTAL_HISTORY_NAME,
   "This option specifies the name of the backup series stored in the "
   "PERCONA_SCHEMA.xtrabackup_history history record to base an "
   "incremental backup on. Xtrabackup will search the history table "
   "looking for the most recent (highest innodb_to_lsn), successful "
   "backup in the series and take the to_lsn value to use as the "
   "starting lsn for the incremental backup. This will be mutually "
   "exclusive with --incremental-history-uuid, --incremental-basedir "
   "and --incremental-lsn. If no valid lsn can be found (no series by "
   "that name, no successful backups by that name) xtrabackup will "
   "return with an error. It is used with the --incremental option.",
   (uchar*) &opt_incremental_history_name,
   (uchar*) &opt_incremental_history_name, 0, GET_STR,
   REQUIRED_ARG, 0, 0, 0, 0, 0, 0},

  {"incremental-history-uuid", OPT_INCREMENTAL_HISTORY_UUID,
   "This option specifies the UUID of the specific history record "
   "stored in the PERCONA_SCHEMA.xtrabackup_history to base an "
   "incremental backup on. --incremental-history-name, "
   "--incremental-basedir and --incremental-lsn. If no valid lsn can be "
   "found (no success record with that uuid) xtrabackup will return "
   "with an error. It is used with the --incremental option.",
   (uchar*) &opt_incremental_history_uuid,
   (uchar*) &opt_incremental_history_uuid, 0, GET_STR,
   REQUIRED_ARG, 0, 0, 0, 0, 0, 0},

1105 1106 1107 1108 1109
  {"remove-original", OPT_REMOVE_ORIGINAL, "Remove .qp files after decompression.",
   (uchar *) &opt_remove_original,
   (uchar *) &opt_remove_original,
   0, GET_BOOL, NO_ARG, 0, 0, 0, 0, 0, 0},

1110 1111 1112 1113 1114 1115 1116 1117 1118 1119 1120 1121 1122 1123 1124 1125 1126 1127 1128 1129 1130 1131 1132 1133 1134 1135 1136 1137 1138 1139 1140 1141 1142 1143 1144 1145 1146 1147 1148 1149 1150 1151 1152 1153 1154 1155 1156 1157 1158 1159 1160 1161 1162 1163 1164 1165 1166 1167 1168 1169 1170 1171 1172 1173 1174 1175 1176 1177 1178 1179 1180 1181 1182
  {"ftwrl-wait-query-type", OPT_LOCK_WAIT_QUERY_TYPE,
   "This option specifies which types of queries are allowed to complete "
   "before innobackupex will issue the global lock. Default is all.",
   (uchar*) &opt_lock_wait_query_type,
   (uchar*) &opt_lock_wait_query_type, &query_type_typelib,
   GET_ENUM, REQUIRED_ARG, QUERY_TYPE_ALL, 0, 0, 0, 0, 0},

  {"kill-long-query-type", OPT_KILL_LONG_QUERY_TYPE,
   "This option specifies which types of queries should be killed to "
   "unblock the global lock. Default is \"all\".",
   (uchar*) &opt_kill_long_query_type,
   (uchar*) &opt_kill_long_query_type, &query_type_typelib,
   GET_ENUM, REQUIRED_ARG, QUERY_TYPE_SELECT, 0, 0, 0, 0, 0},

  {"history", OPT_HISTORY,
   "This option enables the tracking of backup history in the "
   "PERCONA_SCHEMA.xtrabackup_history table. An optional history "
   "series name may be specified that will be placed with the history "
   "record for the current backup being taken.",
   NULL, NULL, 0, GET_STR, OPT_ARG, 0, 0, 0, 0, 0, 0},

  {"kill-long-queries-timeout", OPT_KILL_LONG_QUERIES_TIMEOUT,
   "This option specifies the number of seconds innobackupex waits "
   "between starting FLUSH TABLES WITH READ LOCK and killing those "
   "queries that block it. Default is 0 seconds, which means "
   "innobackupex will not attempt to kill any queries.",
   (uchar*) &opt_kill_long_queries_timeout,
   (uchar*) &opt_kill_long_queries_timeout, 0, GET_UINT,
   REQUIRED_ARG, 0, 0, 0, 0, 0, 0},

  {"ftwrl-wait-timeout", OPT_LOCK_WAIT_TIMEOUT,
   "This option specifies time in seconds that innobackupex should wait "
   "for queries that would block FTWRL before running it. If there are "
   "still such queries when the timeout expires, innobackupex terminates "
   "with an error. Default is 0, in which case innobackupex does not "
   "wait for queries to complete and starts FTWRL immediately.",
   (uchar*) &opt_lock_wait_timeout,
   (uchar*) &opt_lock_wait_timeout, 0, GET_UINT,
   REQUIRED_ARG, 0, 0, 0, 0, 0, 0},

  {"ftwrl-wait-threshold", OPT_LOCK_WAIT_THRESHOLD,
   "This option specifies the query run time threshold which is used by "
   "innobackupex to detect long-running queries with a non-zero value "
   "of --ftwrl-wait-timeout. FTWRL is not started until such "
   "long-running queries exist. This option has no effect if "
   "--ftwrl-wait-timeout is 0. Default value is 60 seconds.",
   (uchar*) &opt_lock_wait_threshold,
   (uchar*) &opt_lock_wait_threshold, 0, GET_UINT,
   REQUIRED_ARG, 60, 0, 0, 0, 0, 0},

  {"debug-sleep-before-unlock", OPT_DEBUG_SLEEP_BEFORE_UNLOCK,
   "This is a debug-only option used by the XtraBackup test suite.",
   (uchar*) &opt_debug_sleep_before_unlock,
   (uchar*) &opt_debug_sleep_before_unlock, 0, GET_UINT,
   REQUIRED_ARG, 0, 0, 0, 0, 0, 0},

  {"safe-slave-backup-timeout", OPT_SAFE_SLAVE_BACKUP_TIMEOUT,
   "How many seconds --safe-slave-backup should wait for "
   "Slave_open_temp_tables to become zero. (default 300)",
   (uchar*) &opt_safe_slave_backup_timeout,
   (uchar*) &opt_safe_slave_backup_timeout, 0, GET_UINT,
   REQUIRED_ARG, 300, 0, 0, 0, 0, 0},

  {"binlog-info", OPT_BINLOG_INFO,
   "This option controls how XtraBackup should retrieve server's binary log "
   "coordinates corresponding to the backup. Possible values are OFF, ON, "
   "LOCKLESS and AUTO. See the XtraBackup manual for more information",
   &opt_binlog_info, &opt_binlog_info,
   &binlog_info_typelib, GET_ENUM, OPT_ARG, BINLOG_INFO_AUTO, 0, 0, 0, 0, 0},

  {"secure-auth", OPT_XB_SECURE_AUTH, "Refuse client connecting to server if it"
    " uses old (pre-4.1.1) protocol.", &opt_secure_auth,
    &opt_secure_auth, 0, GET_BOOL, NO_ARG, 1, 0, 0, 0, 0, 0},
1183
#define MYSQL_CLIENT
1184
#include "sslopt-longopts.h"
1185
#undef MYSQL_CLIENT
1186 1187 1188 1189 1190 1191

  { 0, 0, 0, 0, 0, 0, GET_NO_ARG, NO_ARG, 0, 0, 0, 0, 0, 0}
};

uint xb_client_options_count = array_elements(xb_client_options);

1192 1193 1194 1195 1196
#ifndef DBUG_OFF
/** Parameters to DBUG */
static const char *dbug_option;
#endif

1197 1198 1199 1200 1201 1202 1203 1204 1205 1206 1207 1208 1209 1210 1211
struct my_option xb_server_options[] =
{
  {"datadir", 'h', "Path to the database root.", (G_PTR*) &mysql_data_home,
   (G_PTR*) &mysql_data_home, 0, GET_STR, REQUIRED_ARG, 0, 0, 0, 0, 0, 0},
  {"tmpdir", 't',
   "Path for temporary files. Several paths may be specified, separated by a "
#if defined(__WIN__) || defined(OS2) || defined(__NETWARE__)
   "semicolon (;)"
#else
   "colon (:)"
#endif
   ", in this case they are used in a round-robin fashion.",
   (G_PTR*) &opt_mysql_tmpdir,
   (G_PTR*) &opt_mysql_tmpdir, 0, GET_STR, REQUIRED_ARG, 0, 0, 0, 0, 0, 0},
  {"parallel", OPT_XTRA_PARALLEL,
Vladislav Vaintroub's avatar
Vladislav Vaintroub committed
1212 1213
   "Number of threads to use for parallel datafiles transfer. "
   "The default value is 1.",
1214 1215 1216
   (G_PTR*) &xtrabackup_parallel, (G_PTR*) &xtrabackup_parallel, 0, GET_INT,
   REQUIRED_ARG, 1, 1, INT_MAX, 0, 0, 0},

1217
  {"extended_validation", OPT_XTRA_EXTENDED_VALIDATION,
1218 1219 1220
   "Enable extended validation for Innodb data pages during backup phase. "
   "Will slow down backup considerably, in case encryption is used. "
   "May fail if tables are created during the backup.",
1221 1222 1223 1224
   (G_PTR*)&opt_extended_validation,
   (G_PTR*)&opt_extended_validation,
   0, GET_BOOL, NO_ARG, FALSE, 0, 0, 0, 0, 0},

Vladislav Vaintroub's avatar
Vladislav Vaintroub committed
1225
  {"encrypted_backup", OPT_XTRA_ENCRYPTED_BACKUP,
1226
   "In --backup, assume that nonzero key_version implies that the page"
Vladislav Vaintroub's avatar
Vladislav Vaintroub committed
1227
   " is encrypted. Use --backup --skip-encrypted-backup to allow"
1228
   " copying unencrypted that were originally created before MySQL 5.1.48.",
Vladislav Vaintroub's avatar
Vladislav Vaintroub committed
1229 1230
   (G_PTR*)&opt_encrypted_backup,
   (G_PTR*)&opt_encrypted_backup,
1231
   0, GET_BOOL, NO_ARG, TRUE, 0, 0, 0, 0, 0},
1232

1233 1234 1235 1236 1237
   {"log", OPT_LOG, "Ignored option for MySQL option compatibility",
   (G_PTR*) &log_ignored_opt, (G_PTR*) &log_ignored_opt, 0,
   GET_STR, OPT_ARG, 0, 0, 0, 0, 0, 0},

   {"log_bin", OPT_LOG, "Base name for the log sequence",
1238
   &opt_log_bin, &opt_log_bin, 0, GET_STR, OPT_ARG, 0, 0, 0, 0, 0, 0},
1239 1240 1241 1242

   {"innodb", OPT_INNODB, "Ignored option for MySQL option compatibility",
   (G_PTR*) &innobase_ignored_opt, (G_PTR*) &innobase_ignored_opt, 0,
   GET_STR, OPT_ARG, 0, 0, 0, 0, 0, 0},
1243
#ifdef BTR_CUR_HASH_ADAPT
1244 1245 1246
  {"innodb_adaptive_hash_index", OPT_INNODB_ADAPTIVE_HASH_INDEX,
   "Enable InnoDB adaptive hash index (enabled by default).  "
   "Disable with --skip-innodb-adaptive-hash-index.",
1247 1248
   &btr_search_enabled,
   &btr_search_enabled,
1249
   0, GET_BOOL, NO_ARG, 1, 0, 0, 0, 0, 0},
1250
#endif /* BTR_CUR_HASH_ADAPT */
1251 1252
  {"innodb_autoextend_increment", OPT_INNODB_AUTOEXTEND_INCREMENT,
   "Data file autoextend increment in megabytes",
1253 1254
   (G_PTR*) &sys_tablespace_auto_extend_increment,
   (G_PTR*) &sys_tablespace_auto_extend_increment,
1255 1256 1257
   0, GET_ULONG, REQUIRED_ARG, 8L, 1L, 1000L, 0, 1L, 0},
  {"innodb_data_file_path", OPT_INNODB_DATA_FILE_PATH,
   "Path to individual files and their sizes.", &innobase_data_file_path,
1258
   &innobase_data_file_path, 0, GET_STR, REQUIRED_ARG, 0, 0, 0, 0, 0, 0},
1259 1260
  {"innodb_data_home_dir", OPT_INNODB_DATA_HOME_DIR,
   "The common part for InnoDB table spaces.", &innobase_data_home_dir,
1261
   &innobase_data_home_dir, 0, GET_STR, REQUIRED_ARG, 0, 0, 0, 0, 0, 0},
1262 1263 1264 1265
  {"innodb_doublewrite", OPT_INNODB_DOUBLEWRITE,
   "Enable InnoDB doublewrite buffer during --prepare.",
   (G_PTR*) &innobase_use_doublewrite,
   (G_PTR*) &innobase_use_doublewrite, 0, GET_BOOL, NO_ARG, 0, 0, 0, 0, 0, 0},
1266 1267 1268 1269 1270 1271 1272 1273 1274 1275 1276 1277 1278 1279 1280 1281 1282 1283 1284 1285 1286
  {"innodb_io_capacity", OPT_INNODB_IO_CAPACITY,
   "Number of IOPs the server can do. Tunes the background IO rate",
   (G_PTR*) &srv_io_capacity, (G_PTR*) &srv_io_capacity,
   0, GET_ULONG, OPT_ARG, 200, 100, ~0UL, 0, 0, 0},
  {"innodb_file_io_threads", OPT_INNODB_FILE_IO_THREADS,
   "Number of file I/O threads in InnoDB.", (G_PTR*) &innobase_file_io_threads,
   (G_PTR*) &innobase_file_io_threads, 0, GET_LONG, REQUIRED_ARG, 4, 4, 64, 0,
   1, 0},
  {"innodb_read_io_threads", OPT_INNODB_READ_IO_THREADS,
   "Number of background read I/O threads in InnoDB.", (G_PTR*) &innobase_read_io_threads,
   (G_PTR*) &innobase_read_io_threads, 0, GET_LONG, REQUIRED_ARG, 4, 1, 64, 0,
   1, 0},
  {"innodb_write_io_threads", OPT_INNODB_WRITE_IO_THREADS,
   "Number of background write I/O threads in InnoDB.", (G_PTR*) &innobase_write_io_threads,
   (G_PTR*) &innobase_write_io_threads, 0, GET_LONG, REQUIRED_ARG, 4, 1, 64, 0,
   1, 0},
  {"innodb_file_per_table", OPT_INNODB_FILE_PER_TABLE,
   "Stores each InnoDB table to an .ibd file in the database dir.",
   (G_PTR*) &innobase_file_per_table,
   (G_PTR*) &innobase_file_per_table, 0, GET_BOOL, NO_ARG,
   FALSE, 0, 0, 0, 0, 0},
1287

1288 1289 1290 1291 1292 1293
  {"innodb_flush_method", OPT_INNODB_FLUSH_METHOD,
   "With which method to flush data.",
   &srv_file_flush_method, &srv_file_flush_method,
   &innodb_flush_method_typelib, GET_ENUM, REQUIRED_ARG,
   IF_WIN(SRV_ALL_O_DIRECT_FSYNC, SRV_FSYNC), 0, 0, 0, 0, 0},

1294 1295
  {"innodb_log_buffer_size", OPT_INNODB_LOG_BUFFER_SIZE,
   "The size of the buffer which InnoDB uses to write log to the log files on disk.",
1296 1297
   (G_PTR*) &srv_log_buffer_size, (G_PTR*) &srv_log_buffer_size, 0,
   GET_ULONG, REQUIRED_ARG, 1024*1024L, 256*1024L, LONG_MAX, 0, 1024, 0},
1298
  {"innodb_log_file_size", OPT_INNODB_LOG_FILE_SIZE,
1299 1300 1301 1302
   "Ignored for mysqld option compatibility",
   (G_PTR*) &srv_log_file_size, (G_PTR*) &srv_log_file_size, 0,
   GET_ULL, REQUIRED_ARG, 48 << 20, 1 << 20, 512ULL << 30, 0,
   UNIV_PAGE_SIZE_MAX, 0},
1303
  {"innodb_log_files_in_group", OPT_INNODB_LOG_FILES_IN_GROUP,
1304 1305 1306
   "Ignored for mysqld option compatibility",
   &srv_n_log_files, &srv_n_log_files,
   0, GET_LONG, REQUIRED_ARG, 1, 1, 100, 0, 1, 0},
1307 1308
  {"innodb_log_group_home_dir", OPT_INNODB_LOG_GROUP_HOME_DIR,
   "Path to InnoDB log files.", &srv_log_group_home_dir,
1309
   &srv_log_group_home_dir, 0, GET_STR, REQUIRED_ARG, 0, 0, 0, 0, 0, 0},
1310 1311 1312 1313 1314 1315 1316
  {"innodb_max_dirty_pages_pct", OPT_INNODB_MAX_DIRTY_PAGES_PCT,
   "Percentage of dirty pages allowed in bufferpool.", (G_PTR*) &srv_max_buf_pool_modified_pct,
   (G_PTR*) &srv_max_buf_pool_modified_pct, 0, GET_ULONG, REQUIRED_ARG, 90, 0, 100, 0, 0, 0},
  {"innodb_use_native_aio", OPT_INNODB_USE_NATIVE_AIO,
   "Use native AIO if supported on this platform.",
   (G_PTR*) &srv_use_native_aio,
   (G_PTR*) &srv_use_native_aio, 0, GET_BOOL, NO_ARG,
1317
   TRUE, 0, 0, 0, 0, 0},
1318 1319 1320 1321 1322 1323 1324
  {"innodb_page_size", OPT_INNODB_PAGE_SIZE,
   "The universal page size of the database.",
   (G_PTR*) &innobase_page_size, (G_PTR*) &innobase_page_size, 0,
   /* Use GET_LL to support numeric suffixes in 5.6 */
   GET_LL, REQUIRED_ARG,
   (1LL << 14), (1LL << 12), (1LL << UNIV_PAGE_SIZE_SHIFT_MAX), 0, 1L, 0},
  {"innodb_buffer_pool_filename", OPT_INNODB_BUFFER_POOL_FILENAME,
1325
   "Ignored for mysqld option compatibility",
1326 1327 1328 1329
   (G_PTR*) &innobase_buffer_pool_filename,
   (G_PTR*) &innobase_buffer_pool_filename,
   0, GET_STR, REQUIRED_ARG, 0, 0, 0, 0, 0, 0},

1330 1331 1332 1333 1334
#ifndef DBUG_OFF /* unfortunately "debug" collides with existing options */
  {"dbug", '#', "Built in DBUG debugger.",
   &dbug_option, &dbug_option, 0, GET_STR, OPT_ARG,
   0, 0, 0, 0, 0, 0},
#endif
1335 1336 1337 1338 1339 1340 1341 1342 1343 1344 1345 1346
#ifndef __WIN__
  {"debug-sync", OPT_XTRA_DEBUG_SYNC,
   "Debug sync point. This is only used by the xtrabackup test suite",
   (G_PTR*) &xtrabackup_debug_sync,
   (G_PTR*) &xtrabackup_debug_sync,
   0, GET_STR, REQUIRED_ARG, 0, 0, 0, 0, 0, 0},
#endif

  {"innodb_checksum_algorithm", OPT_INNODB_CHECKSUM_ALGORITHM,
  "The algorithm InnoDB uses for page checksumming. [CRC32, STRICT_CRC32, "
   "INNODB, STRICT_INNODB, NONE, STRICT_NONE]", &srv_checksum_algorithm,
   &srv_checksum_algorithm, &innodb_checksum_algorithm_typelib, GET_ENUM,
1347
   REQUIRED_ARG, SRV_CHECKSUM_ALGORITHM_CRC32, 0, 0, 0, 0, 0},
1348

1349 1350
  {"innodb_undo_directory", OPT_INNODB_UNDO_DIRECTORY,
   "Directory where undo tablespace files live, this path can be absolute.",
1351
   &srv_undo_dir, &srv_undo_dir, 0, GET_STR, REQUIRED_ARG, 0, 0, 0, 0, 0,
1352 1353 1354 1355 1356 1357 1358
   0},

  {"innodb_undo_tablespaces", OPT_INNODB_UNDO_TABLESPACES,
   "Number of undo tablespaces to use.",
   (G_PTR*)&srv_undo_tablespaces, (G_PTR*)&srv_undo_tablespaces,
   0, GET_ULONG, REQUIRED_ARG, 0, 0, 126, 0, 1, 0},

1359 1360 1361 1362 1363
  {"innodb_compression_level", OPT_INNODB_COMPRESSION_LEVEL,
   "Compression level used for zlib compression.",
   (G_PTR*)&page_zip_level, (G_PTR*)&page_zip_level,
   0, GET_UINT, REQUIRED_ARG, 6, 0, 9, 0, 0, 0},

1364 1365 1366 1367
  {"defaults_group", OPT_DEFAULTS_GROUP, "defaults group in config file (default \"mysqld\").",
   (G_PTR*) &defaults_group, (G_PTR*) &defaults_group,
   0, GET_STR, REQUIRED_ARG, 0, 0, 0, 0, 0, 0},

1368 1369 1370
  {"plugin-dir", OPT_PLUGIN_DIR,
  "Server plugin directory. Used to load encryption plugin during 'prepare' phase."
  "Has no effect in the 'backup' phase (plugin directory during backup is the same as server's)",
1371 1372 1373
  &xb_plugin_dir, &xb_plugin_dir,
  0, GET_STR, REQUIRED_ARG, 0, 0, 0, 0, 0, 0 },

1374 1375 1376 1377 1378
  {"innodb-log-checksums", OPT_INNODB_LOG_CHECKSUMS,
   "Whether to require checksums for InnoDB redo log blocks",
   &innodb_log_checksums, &innodb_log_checksums,
   0, GET_BOOL, REQUIRED_ARG, 1, 0, 0, 0, 0, 0 },

1379 1380 1381 1382 1383
  {"open_files_limit", OPT_OPEN_FILES_LIMIT, "the maximum number of file "
   "descriptors to reserve with setrlimit().",
   (G_PTR*) &xb_open_files_limit, (G_PTR*) &xb_open_files_limit, 0, GET_ULONG,
   REQUIRED_ARG, 0, 0, UINT_MAX, 0, 1, 0},

1384 1385 1386 1387 1388
  {"lock-ddl-per-table", OPT_LOCK_DDL_PER_TABLE, "Lock DDL for each table "
   "before xtrabackup starts to copy it and until the backup is completed.",
   (uchar*) &opt_lock_ddl_per_table, (uchar*) &opt_lock_ddl_per_table, 0,
   GET_BOOL, NO_ARG, 0, 0, 0, 0, 0, 0},

1389 1390 1391 1392 1393
  {"rocksdb-datadir", OPT_ROCKSDB_DATADIR, "RocksDB data directory."
   "This option is only  used with --copy-back or --move-back option",
  &xb_rocksdb_datadir, &xb_rocksdb_datadir,
  0, GET_STR, REQUIRED_ARG, 0, 0, 0, 0, 0, 0 },

1394
  { "rocksdb-backup", OPT_BACKUP_ROCKSDB, "Backup rocksdb data, if rocksdb plugin is installed."
1395 1396 1397 1398
   "Used only with --backup option. Can be useful for partial backups, to exclude all rocksdb data",
   &xb_backup_rocksdb, &xb_backup_rocksdb,
   0, GET_BOOL, NO_ARG, 1, 0, 0, 0, 0, 0 },

1399 1400 1401 1402
   {"check-privileges", OPT_XTRA_CHECK_PRIVILEGES, "Check database user "
   "privileges fro the backup user",
   &opt_check_privileges, &opt_check_privileges,
   0, GET_BOOL, NO_ARG, 1, 0, 0, 0, 0, 0 },
1403

1404 1405 1406 1407 1408 1409 1410 1411 1412 1413 1414 1415 1416 1417 1418 1419 1420 1421 1422 1423 1424 1425 1426 1427 1428 1429 1430 1431 1432 1433 1434 1435 1436 1437 1438 1439 1440 1441 1442
  { 0, 0, 0, 0, 0, 0, GET_NO_ARG, NO_ARG, 0, 0, 0, 0, 0, 0}
};

uint xb_server_options_count = array_elements(xb_server_options);

#ifndef __WIN__
static int debug_sync_resumed;

static void sigcont_handler(int sig);

static void sigcont_handler(int sig __attribute__((unused)))
{
	debug_sync_resumed= 1;
}
#endif

static inline
void
debug_sync_point(const char *name)
{
#ifndef __WIN__
	FILE	*fp;
	pid_t	pid;
	char	pid_path[FN_REFLEN];

	if (xtrabackup_debug_sync == NULL) {
		return;
	}

	if (strcmp(xtrabackup_debug_sync, name)) {
		return;
	}

	pid = getpid();

	snprintf(pid_path, sizeof(pid_path), "%s/xtrabackup_debug_sync",
		 xtrabackup_target_dir);
	fp = fopen(pid_path, "w");
	if (fp == NULL) {
1443
		die("Can't open open %s", pid_path);
1444 1445 1446 1447
	}
	fprintf(fp, "%u\n", (uint) pid);
	fclose(fp);

1448
	msg("mariabackup: DEBUG: Suspending at debug sync point '%s'. "
1449
	    "Resume with 'kill -SIGCONT %u'.", name, (uint) pid);
1450 1451 1452 1453 1454 1455 1456 1457

	debug_sync_resumed= 0;
	kill(pid, SIGSTOP);
	while (!debug_sync_resumed) {
		sleep(1);
	}

	/* On resume */
1458
	msg("mariabackup: DEBUG: removing the pid file.");
1459 1460 1461 1462
	my_delete(pid_path, MYF(MY_WME));
#endif
}

1463

1464
static std::set<std::string> tables_for_export;
1465 1466 1467 1468 1469 1470 1471 1472 1473 1474 1475

static void append_export_table(const char *dbname, const char *tablename, bool is_remote)
{
  if(dbname && tablename && !is_remote)
  {
    char buf[3*FN_REFLEN];
    snprintf(buf,sizeof(buf),"%s/%s",dbname, tablename);
    // trim .ibd
    char *p=strrchr(buf, '.');
    if (p) *p=0;

1476 1477 1478 1479 1480 1481 1482 1483 1484
    std::string name=ut_get_name(0, buf);
    /* Strip partition name comment from table name, if any */
    if (ends_with(name.c_str(), "*/"))
    {
      size_t pos= name.rfind("/*");
      if (pos != std::string::npos)
         name.resize(pos);
    }
    tables_for_export.insert(name);
1485 1486 1487 1488 1489 1490 1491 1492 1493 1494 1495 1496 1497 1498
  }
}


#define BOOTSTRAP_FILENAME "mariabackup_prepare_for_export.sql"

static int create_bootstrap_file()
{
  FILE *f= fopen(BOOTSTRAP_FILENAME,"wb");
  if(!f)
   return -1;

  fputs("SET NAMES UTF8;\n",f);
  enumerate_ibd_files(append_export_table);
1499 1500
  for (std::set<std::string>::iterator it = tables_for_export.begin();
       it != tables_for_export.end(); it++)
1501
  {
1502
     const char *tab = it->c_str();
1503 1504 1505 1506 1507 1508 1509 1510 1511 1512 1513 1514 1515 1516 1517 1518 1519 1520 1521 1522 1523 1524 1525 1526 1527 1528
     fprintf(f,
     "BEGIN NOT ATOMIC "
       "DECLARE CONTINUE HANDLER FOR NOT FOUND,SQLEXCEPTION BEGIN END;"
       "FLUSH TABLES %s FOR EXPORT;"
     "END;\n"
     "UNLOCK TABLES;\n",
      tab);
  }
  fclose(f);
  return 0;
}

static int prepare_export()
{
  int err= -1;

  char cmdline[2*FN_REFLEN];
  FILE *outf;

  if (create_bootstrap_file())
    return -1;

  // Process defaults-file , it can have some --lc-language stuff,
  // which is* unfortunately* still necessary to get mysqld up
  if (strncmp(orig_argv1,"--defaults-file=",16) == 0)
  {
1529
    snprintf(cmdline, sizeof cmdline,
1530 1531
     IF_WIN("\"","") "\"%s\" --mysqld \"%s\" "
      " --defaults-extra-file=./backup-my.cnf --defaults-group-suffix=%s --datadir=."
1532
      " --innodb --innodb-fast-shutdown=0 --loose-partition"
1533
      " --innodb_purge_rseg_truncate_frequency=1 --innodb-buffer-pool-size=%llu"
1534
      " --console  --skip-log-error --bootstrap  < "  BOOTSTRAP_FILENAME IF_WIN("\"",""),
1535 1536 1537 1538 1539 1540 1541 1542
      mariabackup_exe, 
      orig_argv1, (my_defaults_group_suffix?my_defaults_group_suffix:""),
      xtrabackup_use_memory);
  }
  else
  {
    sprintf(cmdline,
     IF_WIN("\"","") "\"%s\" --mysqld"
1543
      " --defaults-file=./backup-my.cnf --defaults-group-suffix=%s --datadir=."
1544
      " --innodb --innodb-fast-shutdown=0 --loose-partition"
1545 1546 1547
      " --innodb_purge_rseg_truncate_frequency=1 --innodb-buffer-pool-size=%llu"
      " --console  --log-error= --bootstrap  < "  BOOTSTRAP_FILENAME IF_WIN("\"",""),
      mariabackup_exe,
1548
      (my_defaults_group_suffix?my_defaults_group_suffix:""),
1549 1550 1551 1552 1553 1554 1555 1556 1557 1558 1559 1560 1561 1562 1563 1564 1565 1566 1567 1568 1569
      xtrabackup_use_memory);
  }

  msg("Prepare export : executing %s\n", cmdline);
  fflush(stderr);

  outf= popen(cmdline,"r");
  if (!outf)
    goto end;
  
  char outline[FN_REFLEN];
  while(fgets(outline, sizeof(outline)-1, outf))
    fprintf(stderr,"%s",outline);

  err = pclose(outf);
end:
  unlink(BOOTSTRAP_FILENAME);
  return err;
}


Marko Mäkelä's avatar
Marko Mäkelä committed
1570 1571 1572 1573 1574
static const char *xb_client_default_groups[]={
   "xtrabackup", "mariabackup",
   "client", "client-server",
   "client-mariadb",
   0, 0, 0
1575
};
1576

1577 1578 1579 1580 1581 1582 1583 1584 1585 1586
static const char *xb_server_default_groups[]={
   "xtrabackup", "mariabackup",
   "mysqld", "server", MYSQL_BASE_VERSION,
   "mariadb", MARIADB_BASE_VERSION,
   "client-server",
   #ifdef WITH_WSREP
   "galera",
   #endif
   0, 0, 0
};
1587 1588 1589

static void print_version(void)
{
1590
  fprintf(stderr, "%s based on MariaDB server %s %s (%s)\n",
1591
      my_progname, MYSQL_SERVER_VERSION, SYSTEM_TYPE, MACHINE_TYPE);
1592 1593 1594 1595 1596 1597 1598 1599 1600 1601 1602 1603 1604 1605 1606 1607 1608 1609 1610 1611 1612
}

static void usage(void)
{
  puts("Open source backup tool for InnoDB and XtraDB\n\
\n\
Copyright (C) 2009-2015 Percona LLC and/or its affiliates.\n\
Portions Copyright (C) 2000, 2011, MySQL AB & Innobase Oy. All Rights Reserved.\n\
\n\
This program is free software; you can redistribute it and/or\n\
modify it under the terms of the GNU General Public License\n\
as published by the Free Software Foundation version 2\n\
of the License.\n\
\n\
This program is distributed in the hope that it will be useful,\n\
but WITHOUT ANY WARRANTY; without even the implied warranty of\n\
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the\n\
GNU General Public License for more details.\n\
\n\
You can download full text of the license on http://www.gnu.org/licenses/gpl-2.0.txt\n");

1613
  printf("Usage: %s [--defaults-file=#] [--backup | --prepare | --copy-back | --move-back] [OPTIONS]\n",my_progname);
1614 1615 1616 1617 1618 1619 1620 1621 1622 1623 1624 1625 1626 1627 1628 1629 1630 1631 1632 1633 1634 1635 1636 1637 1638 1639 1640 1641 1642 1643 1644 1645 1646 1647 1648 1649 1650 1651 1652 1653 1654 1655 1656 1657 1658 1659 1660 1661 1662 1663 1664 1665 1666 1667 1668 1669 1670 1671 1672
  print_defaults("my", xb_server_default_groups);
  my_print_help(xb_client_options);
  my_print_help(xb_server_options);
  my_print_variables(xb_server_options);
  my_print_variables(xb_client_options);
}

#define ADD_PRINT_PARAM_OPT(value)              \
  { \
    print_param_str << opt->name << "=" << value << "\n"; \
    param_set.insert(opt->name); \
  }

/************************************************************************
Check if parameter is set in defaults file or via command line argument
@return true if parameter is set. */
bool
check_if_param_set(const char *param)
{
	return param_set.find(param) != param_set.end();
}

my_bool
xb_get_one_option(int optid,
		  const struct my_option *opt __attribute__((unused)),
		  char *argument)
{
  switch(optid) {
  case 'h':
    strmake(mysql_real_data_home,argument, FN_REFLEN - 1);
    mysql_data_home= mysql_real_data_home;

    ADD_PRINT_PARAM_OPT(mysql_real_data_home);
    break;

  case 't':

    ADD_PRINT_PARAM_OPT(opt_mysql_tmpdir);
    break;

  case OPT_INNODB_DATA_HOME_DIR:

    ADD_PRINT_PARAM_OPT(innobase_data_home_dir);
    break;

  case OPT_INNODB_DATA_FILE_PATH:

    ADD_PRINT_PARAM_OPT(innobase_data_file_path);
    break;

  case OPT_INNODB_LOG_GROUP_HOME_DIR:

    ADD_PRINT_PARAM_OPT(srv_log_group_home_dir);
    break;

  case OPT_INNODB_LOG_FILES_IN_GROUP:
  case OPT_INNODB_LOG_FILE_SIZE:
    break;

1673 1674 1675 1676 1677 1678
  case OPT_INNODB_FLUSH_METHOD:
    ut_a(srv_file_flush_method
	 <= IF_WIN(SRV_ALL_O_DIRECT_FSYNC, SRV_O_DIRECT_NO_FSYNC));
    ADD_PRINT_PARAM_OPT(innodb_flush_method_names[srv_file_flush_method]);
    break;

1679 1680 1681 1682 1683 1684 1685 1686 1687 1688 1689 1690 1691 1692 1693 1694 1695 1696 1697 1698 1699 1700
  case OPT_INNODB_PAGE_SIZE:

    ADD_PRINT_PARAM_OPT(innobase_page_size);
    break;

  case OPT_INNODB_UNDO_DIRECTORY:

    ADD_PRINT_PARAM_OPT(srv_undo_dir);
    break;

  case OPT_INNODB_UNDO_TABLESPACES:

    ADD_PRINT_PARAM_OPT(srv_undo_tablespaces);
    break;

  case OPT_INNODB_CHECKSUM_ALGORITHM:

    ut_a(srv_checksum_algorithm <= SRV_CHECKSUM_ALGORITHM_STRICT_NONE);

    ADD_PRINT_PARAM_OPT(innodb_checksum_algorithm_names[srv_checksum_algorithm]);
    break;

1701 1702 1703 1704
  case OPT_INNODB_COMPRESSION_LEVEL:
    ADD_PRINT_PARAM_OPT(page_zip_level);
    break;

1705 1706 1707 1708 1709 1710 1711 1712 1713 1714
  case OPT_INNODB_BUFFER_POOL_FILENAME:

    ADD_PRINT_PARAM_OPT(innobase_buffer_pool_filename);
    break;

  case OPT_XTRA_TARGET_DIR:
    strmake(xtrabackup_real_target_dir,argument, sizeof(xtrabackup_real_target_dir)-1);
    xtrabackup_target_dir= xtrabackup_real_target_dir;
    break;
  case OPT_XTRA_STREAM:
1715
    if (!strcasecmp(argument, "xbstream"))
1716 1717 1718
      xtrabackup_stream_fmt = XB_STREAM_FMT_XBSTREAM;
    else
    {
1719
      msg("Invalid --stream argument: %s", argument);
1720 1721 1722 1723 1724 1725 1726 1727 1728
      return 1;
    }
    xtrabackup_stream = TRUE;
    break;
  case OPT_XTRA_COMPRESS:
    if (argument == NULL)
      xtrabackup_compress_alg = "quicklz";
    else if (strcasecmp(argument, "quicklz"))
    {
1729
      msg("Invalid --compress argument: %s", argument);
1730 1731 1732 1733 1734 1735 1736 1737 1738 1739 1740 1741 1742 1743 1744 1745 1746 1747 1748 1749 1750 1751 1752 1753 1754 1755 1756 1757 1758
      return 1;
    }
    xtrabackup_compress = TRUE;
    break;
  case OPT_DECOMPRESS:
    opt_decompress = TRUE;
    xtrabackup_decrypt_decompress = true;
    break;
  case (int) OPT_CORE_FILE:
    test_flags |= TEST_CORE_ON_SIGNAL;
    break;
  case OPT_HISTORY:
    if (argument) {
      opt_history = argument;
    } else {
      opt_history = "";
    }
    break;
  case 'p':
    if (argument)
    {
      char *start= argument;
      my_free(opt_password);
      opt_password= my_strdup(argument, MYF(MY_FAE));
      while (*argument) *argument++= 'x';               // Destroy argument
      if (*start)
        start[1]=0 ;
    }
    break;
1759 1760 1761
  case OPT_PROTOCOL:
    if (argument)
    {
1762 1763 1764 1765 1766 1767
      if ((opt_protocol= find_type_with_warning(argument, &sql_protocol_typelib,
                                                opt->name)) <= 0)
      {
        sf_leaking_memory= 1; /* no memory leak reports here */
        exit(1);
      }
1768 1769
    }
    break;
1770
#define MYSQL_CLIENT
1771
#include "sslopt-case.h"
1772
#undef MYSQL_CLIENT
1773 1774 1775 1776 1777 1778 1779 1780 1781 1782 1783 1784 1785 1786 1787

  case '?':
    usage();
    exit(EXIT_SUCCESS);
    break;
  case 'v':
    print_version();
    exit(EXIT_SUCCESS);
    break;
  default:
    break;
  }
  return 0;
}

1788
static bool innodb_init_param()
1789
{
1790
	srv_is_being_started = TRUE;
1791 1792 1793
	/* === some variables from mysqld === */
	memset((G_PTR) &mysql_tmpdir_list, 0, sizeof(mysql_tmpdir_list));

Marko Mäkelä's avatar
Marko Mäkelä committed
1794 1795
	if (init_tmpdir(&mysql_tmpdir_list, opt_mysql_tmpdir)) {
		msg("init_tmpdir() failed");
1796
		return true;
Marko Mäkelä's avatar
Marko Mäkelä committed
1797
	}
1798
	xtrabackup_tmpdir = my_tmpdir(&mysql_tmpdir_list);
1799 1800 1801 1802 1803
	/* dummy for initialize all_charsets[] */
	get_charset_name(0);

	srv_page_size = 0;
	srv_page_size_shift = 0;
1804 1805 1806
#ifdef BTR_CUR_HASH_ADAPT
	btr_ahi_parts = 1;
#endif /* BTR_CUR_HASH_ADAPT */
1807 1808

	if (innobase_page_size != (1LL << 14)) {
1809
		size_t n_shift = get_bit_shift(size_t(innobase_page_size));
1810 1811

		if (n_shift >= 12 && n_shift <= UNIV_PAGE_SIZE_SHIFT_MAX) {
1812 1813
			srv_page_size_shift = ulong(n_shift);
			srv_page_size = 1U << n_shift;
1814
			msg("InnoDB: The universal page size of the "
1815
			    "database is set to %lu.", srv_page_size);
1816
		} else {
Marko Mäkelä's avatar
Marko Mäkelä committed
1817
			msg("invalid value of "
1818
			    "innobase_page_size: %lld", innobase_page_size);
1819
			goto error;
1820 1821 1822
		}
	} else {
		srv_page_size_shift = 14;
1823
		srv_page_size = 1U << 14;
1824 1825 1826 1827 1828
	}

	/* Check that values don't overflow on 32-bit systems. */
	if (sizeof(ulint) == 4) {
		if (xtrabackup_use_memory > UINT_MAX32) {
1829
			msg("mariabackup: use-memory can't be over 4GB"
1830
			    " on 32-bit systems");
1831 1832 1833
		}
	}

1834 1835
	static char default_path[2] = { FN_CURLIB, 0 };
	fil_path_to_mysql_datadir = default_path;
1836 1837 1838 1839

	/* Set InnoDB initialization parameters according to the values
	read from MySQL .cnf file */

1840
	if (xtrabackup_backup) {
1841
		msg("mariabackup: using the following InnoDB configuration:");
1842
	} else {
1843
		msg("mariabackup: using the following InnoDB configuration "
1844
		    "for recovery:");
1845 1846 1847 1848 1849 1850
	}

	/*--------------- Data files -------------------------*/

	/* The default dir for data files is the datadir of MySQL */

1851
	srv_data_home = (xtrabackup_backup && innobase_data_home_dir
1852
			 ? innobase_data_home_dir : default_path);
1853
	msg("innodb_data_home_dir = %s", srv_data_home);
1854 1855 1856 1857 1858 1859 1860 1861

	/* Set default InnoDB data file size to 10 MB and let it be
  	auto-extending. Thus users can use InnoDB in >= 4.0 without having
	to specify any startup options. */

	if (!innobase_data_file_path) {
  		innobase_data_file_path = (char*) "ibdata1:10M:autoextend";
	}
1862
	msg("innodb_data_file_path = %s",
1863 1864
	    innobase_data_file_path);

1865 1866 1867 1868
	/* This is the first time univ_page_size is used.
	It was initialized to 16k pages before srv_page_size was set */
	univ_page_size.copy_from(
		page_size_t(srv_page_size, srv_page_size, false));
1869

1870 1871 1872 1873
	srv_sys_space.set_space_id(TRX_SYS_SPACE);
	srv_sys_space.set_name("innodb_system");
	srv_sys_space.set_path(srv_data_home);
	srv_sys_space.set_flags(FSP_FLAGS_PAGE_SSIZE());
1874

1875
	if (!srv_sys_space.parse_params(innobase_data_file_path, true)) {
1876 1877 1878
		goto error;
	}

1879 1880 1881
	srv_sys_space.normalize_size();
	srv_lock_table_size = 5 * (srv_buf_pool_size >> srv_page_size_shift);

1882 1883 1884 1885
	/* -------------- Log files ---------------------------*/

	/* The default dir for log files is the datadir of MySQL */

1886
	if (!(xtrabackup_backup && srv_log_group_home_dir)) {
1887 1888 1889 1890 1891
		srv_log_group_home_dir = default_path;
	}
	if (xtrabackup_prepare && xtrabackup_incremental_dir) {
		srv_log_group_home_dir = xtrabackup_incremental_dir;
	}
1892
	msg("innodb_log_group_home_dir = %s",
1893 1894
	    srv_log_group_home_dir);

1895
	os_normalize_path(srv_log_group_home_dir);
1896 1897 1898

	if (strchr(srv_log_group_home_dir, ';')) {
		msg("syntax error in innodb_log_group_home_dir, ");
1899
		goto error;
1900 1901 1902 1903 1904 1905 1906 1907
	}

	srv_adaptive_flushing = FALSE;

        /* We set srv_pool_size here in units of 1 kB. InnoDB internally
        changes the value so that it becomes the number of database pages. */

	srv_buf_pool_size = (ulint) xtrabackup_use_memory;
1908
	srv_buf_pool_chunk_unit = (ulong)srv_buf_pool_size;
1909
	srv_buf_pool_instances = 1;
1910
	srv_n_page_cleaners = 1;
1911 1912 1913 1914 1915 1916 1917 1918 1919 1920 1921 1922 1923 1924 1925

	srv_n_file_io_threads = (ulint) innobase_file_io_threads;
	srv_n_read_io_threads = (ulint) innobase_read_io_threads;
	srv_n_write_io_threads = (ulint) innobase_write_io_threads;

	srv_use_doublewrite_buf = (ibool) innobase_use_doublewrite;

	os_use_large_pages = (ibool) innobase_use_large_pages;
	os_large_page_size = (ulint) innobase_large_page_size;
	row_rollback_on_timeout = (ibool) innobase_rollback_on_timeout;

	srv_file_per_table = (my_bool) innobase_file_per_table;

        srv_locks_unsafe_for_binlog = (ibool) innobase_locks_unsafe_for_binlog;

1926
	srv_max_n_open_files = ULINT_UNDEFINED - 5;
1927 1928
	srv_innodb_status = (ibool) innobase_create_status_file;

1929
	srv_print_verbose_log = verbose ? 2 : 1;
1930 1931 1932 1933 1934 1935 1936

	/* Store the default charset-collation number of this MySQL
	installation */

	/* We cannot treat characterset here for now!! */
	data_mysql_default_charset_coll = (ulint)default_charset_info->number;

1937
	ut_ad(DATA_MYSQL_BINARY_CHARSET_COLL == my_charset_bin.number);
1938

1939 1940
#ifdef _WIN32
	srv_use_native_aio = TRUE;
1941 1942 1943 1944

#elif defined(LINUX_NATIVE_AIO)

	if (srv_use_native_aio) {
1945
		msg("InnoDB: Using Linux native AIO");
1946 1947 1948 1949 1950 1951 1952 1953 1954 1955 1956 1957 1958 1959 1960 1961
	}
#else
	/* Currently native AIO is supported only on windows and linux
	and that also when the support is compiled in. In all other
	cases, we ignore the setting of innodb_use_native_aio. */
	srv_use_native_aio = FALSE;

#endif

	/* Assign the default value to srv_undo_dir if it's not specified, as
	my_getopt does not support default values for string options. We also
	ignore the option and override innodb_undo_directory on --prepare,
	because separate undo tablespaces are copied to the root backup
	directory. */

	if (!srv_undo_dir || !xtrabackup_backup) {
1962
		srv_undo_dir = (char*) ".";
1963 1964
	}

1965 1966 1967
	log_checksum_algorithm_ptr = innodb_log_checksums || srv_encrypt_log
		? log_block_calc_checksum_crc32
		: log_block_calc_checksum_none;
1968

1969 1970 1971 1972
#ifdef _WIN32
	srv_use_native_aio = TRUE;
#endif
	return false;
1973 1974

error:
1975
	msg("mariabackup: innodb_init_param(): Error occured.\n");
1976
	return true;
1977 1978
}

1979
static bool innodb_init()
1980
{
1981 1982 1983 1984 1985 1986 1987 1988
	bool create_new_db = false;
	/* Check if the data files exist or not. */
	dberr_t err = srv_sys_space.check_file_spec(&create_new_db, 5U << 20);

	if (err == DB_SUCCESS) {
		err = srv_start(create_new_db);
	}

1989
	if (err != DB_SUCCESS) {
1990
		msg("mariabackup: innodb_init() returned %d (%s).",
1991 1992 1993
		    err, ut_strerr(err));
		innodb_shutdown();
		return(TRUE);
1994 1995 1996 1997 1998 1999 2000 2001 2002 2003 2004 2005 2006 2007 2008 2009 2010 2011 2012
	}

	return(FALSE);
}

/* ================= common ================= */

/***********************************************************************
Read backup meta info.
@return TRUE on success, FALSE on failure. */
static
my_bool
xtrabackup_read_metadata(char *filename)
{
	FILE	*fp;
	my_bool	 r = TRUE;

	fp = fopen(filename,"r");
	if(!fp) {
2013
		msg("Error: cannot open %s", filename);
2014 2015 2016 2017 2018 2019 2020 2021 2022 2023 2024 2025 2026 2027 2028 2029 2030 2031 2032 2033 2034 2035 2036 2037 2038 2039 2040 2041 2042 2043 2044 2045 2046 2047 2048 2049 2050 2051 2052 2053 2054 2055 2056 2057
		return(FALSE);
	}

	if (fscanf(fp, "backup_type = %29s\n", metadata_type)
	    != 1) {
		r = FALSE;
		goto end;
	}
	/* Use UINT64PF instead of LSN_PF here, as we have to maintain the file
	format. */
	if (fscanf(fp, "from_lsn = " UINT64PF "\n", &metadata_from_lsn)
			!= 1) {
		r = FALSE;
		goto end;
	}
	if (fscanf(fp, "to_lsn = " UINT64PF "\n", &metadata_to_lsn)
			!= 1) {
		r = FALSE;
		goto end;
	}
	if (fscanf(fp, "last_lsn = " UINT64PF "\n", &metadata_last_lsn)
			!= 1) {
		metadata_last_lsn = 0;
	}
	/* Optional fields */

end:
	fclose(fp);

	return(r);
}

/***********************************************************************
Print backup meta info to a specified buffer. */
static
void
xtrabackup_print_metadata(char *buf, size_t buf_len)
{
	/* Use UINT64PF instead of LSN_PF here, as we have to maintain the file
	format. */
	snprintf(buf, buf_len,
		 "backup_type = %s\n"
		 "from_lsn = " UINT64PF "\n"
		 "to_lsn = " UINT64PF "\n"
2058
		 "last_lsn = " UINT64PF "\n",
2059 2060 2061
		 metadata_type,
		 metadata_from_lsn,
		 metadata_to_lsn,
2062
		 metadata_last_lsn);
2063 2064 2065 2066 2067 2068 2069 2070 2071 2072 2073 2074 2075 2076 2077 2078 2079 2080 2081 2082 2083 2084 2085 2086
}

/***********************************************************************
Stream backup meta info to a specified datasink.
@return TRUE on success, FALSE on failure. */
static
my_bool
xtrabackup_stream_metadata(ds_ctxt_t *ds_ctxt)
{
	char		buf[1024];
	size_t		len;
	ds_file_t	*stream;
	MY_STAT		mystat;
	my_bool		rc = TRUE;

	xtrabackup_print_metadata(buf, sizeof(buf));

	len = strlen(buf);

	mystat.st_size = len;
	mystat.st_mtime = my_time(0);

	stream = ds_open(ds_ctxt, XTRABACKUP_METADATA_FILENAME, &mystat);
	if (stream == NULL) {
2087
		msg("Error: cannot open output stream for %s", XTRABACKUP_METADATA_FILENAME);
2088 2089 2090 2091 2092 2093 2094 2095 2096 2097 2098 2099 2100 2101 2102 2103 2104 2105 2106 2107 2108 2109 2110 2111 2112 2113 2114 2115 2116 2117 2118
		return(FALSE);
	}

	if (ds_write(stream, buf, len)) {
		rc = FALSE;
	}

	if (ds_close(stream)) {
		rc = FALSE;
	}

	return(rc);
}

/***********************************************************************
Write backup meta info to a specified file.
@return TRUE on success, FALSE on failure. */
static
my_bool
xtrabackup_write_metadata(const char *filepath)
{
	char		buf[1024];
	size_t		len;
	FILE		*fp;

	xtrabackup_print_metadata(buf, sizeof(buf));

	len = strlen(buf);

	fp = fopen(filepath, "w");
	if(!fp) {
2119
		msg("Error: cannot open %s", filepath);
2120 2121 2122 2123 2124 2125 2126 2127 2128 2129 2130 2131 2132 2133 2134 2135 2136 2137 2138 2139 2140 2141 2142 2143
		return(FALSE);
	}
	if (fwrite(buf, len, 1, fp) < 1) {
		fclose(fp);
		return(FALSE);
	}

	fclose(fp);

	return(TRUE);
}

/***********************************************************************
Read meta info for an incremental delta.
@return TRUE on success, FALSE on failure. */
static my_bool
xb_read_delta_metadata(const char *filepath, xb_delta_info_t *info)
{
	FILE*	fp;
	char	key[51];
	char	value[51];
	my_bool	r			= TRUE;

	/* set defaults */
2144
	ulint page_size = ULINT_UNDEFINED, zip_size = 0;
2145 2146 2147 2148 2149 2150 2151 2152 2153 2154 2155
	info->space_id = ULINT_UNDEFINED;

	fp = fopen(filepath, "r");
	if (!fp) {
		/* Meta files for incremental deltas are optional */
		return(TRUE);
	}

	while (!feof(fp)) {
		if (fscanf(fp, "%50s = %50s\n", key, value) == 2) {
			if (strcmp(key, "page_size") == 0) {
2156
				page_size = strtoul(value, NULL, 10);
2157
			} else if (strcmp(key, "zip_size") == 0) {
2158
				zip_size = strtoul(value, NULL, 10);
2159 2160 2161 2162 2163 2164 2165 2166
			} else if (strcmp(key, "space_id") == 0) {
				info->space_id = strtoul(value, NULL, 10);
			}
		}
	}

	fclose(fp);

2167
	if (page_size == ULINT_UNDEFINED) {
2168
		msg("page_size is required in %s", filepath);
2169
		r = FALSE;
2170 2171 2172
	} else {
		info->page_size = page_size_t(zip_size ? zip_size : page_size,
					      page_size, zip_size != 0);
2173
	}
2174

2175
	if (info->space_id == ULINT_UNDEFINED) {
2176
		msg("mariabackup: Warning: This backup was taken with XtraBackup 2.0.1 "
2177
			"or earlier, some DDL operations between full and incremental "
2178
			"backups may be handled incorrectly");
2179 2180 2181 2182 2183 2184 2185 2186 2187 2188 2189 2190 2191 2192 2193 2194 2195 2196
	}

	return(r);
}

/***********************************************************************
Write meta info for an incremental delta.
@return TRUE on success, FALSE on failure. */
my_bool
xb_write_delta_metadata(const char *filename, const xb_delta_info_t *info)
{
	ds_file_t	*f;
	char		buf[64];
	my_bool		ret;
	size_t		len;
	MY_STAT		mystat;

	snprintf(buf, sizeof(buf),
2197 2198 2199
		 "page_size = " ULINTPF "\n"
		 "zip_size = " ULINTPF " \n"
		 "space_id = " ULINTPF "\n",
2200 2201 2202 2203
		 info->page_size.logical(),
		 info->page_size.is_compressed()
		 ? info->page_size.physical() : 0,
		 info->space_id);
2204 2205 2206 2207 2208 2209 2210
	len = strlen(buf);

	mystat.st_size = len;
	mystat.st_mtime = my_time(0);

	f = ds_open(ds_meta, filename, &mystat);
	if (f == NULL) {
2211
		msg("Error: Can't open output stream for %s",filename);
2212 2213 2214 2215 2216 2217 2218 2219 2220 2221 2222 2223 2224 2225 2226 2227
		return(FALSE);
	}

	ret = (ds_write(f, buf, len) == 0);

	if (ds_close(f)) {
		ret = FALSE;
	}

	return(ret);
}

/* ================= backup ================= */
void
xtrabackup_io_throttling(void)
{
2228
	if (xtrabackup_backup && xtrabackup_throttle && (io_ticket--) < 0) {
2229 2230 2231 2232 2233
		os_event_reset(wait_throttle);
		os_event_wait(wait_throttle);
	}
}

Vladislav Vaintroub's avatar
Vladislav Vaintroub committed
2234 2235 2236 2237 2238 2239 2240 2241 2242 2243 2244 2245 2246 2247 2248 2249 2250 2251 2252 2253 2254 2255 2256 2257 2258 2259 2260 2261 2262 2263 2264 2265 2266 2267 2268 2269 2270 2271
static
my_bool regex_list_check_match(
	const regex_list_t& list,
	const char* name)
{
	regmatch_t tables_regmatch[1];
	for (regex_list_t::const_iterator i = list.begin(), end = list.end();
	     i != end; ++i) {
		const regex_t& regex = *i;
		int regres = regexec(&regex, name, 1, tables_regmatch, 0);

		if (regres != REG_NOMATCH) {
			return(TRUE);
		}
	}
	return(FALSE);
}

static
my_bool
find_filter_in_hashtable(
	const char* name,
	hash_table_t* table,
	xb_filter_entry_t** result
)
{
	xb_filter_entry_t* found = NULL;
	HASH_SEARCH(name_hash, table, ut_fold_string(name),
		    xb_filter_entry_t*,
		    found, (void) 0,
		    !strcmp(found->name, name));

	if (found && result) {
		*result = found;
	}
	return (found != NULL);
}

2272
/************************************************************************
Vladislav Vaintroub's avatar
Vladislav Vaintroub committed
2273 2274
Checks if a given table name matches any of specifications given in
regex_list or tables_hash.
2275

Vladislav Vaintroub's avatar
Vladislav Vaintroub committed
2276
@return TRUE on match or both regex_list and tables_hash are empty.*/
2277
static my_bool
Vladislav Vaintroub's avatar
Vladislav Vaintroub committed
2278 2279 2280
check_if_table_matches_filters(const char *name,
	const regex_list_t& regex_list,
	hash_table_t* tables_hash)
2281
{
Vladislav Vaintroub's avatar
Vladislav Vaintroub committed
2282 2283 2284
	if (regex_list.empty() && !tables_hash) {
		return(FALSE);
	}
2285

Vladislav Vaintroub's avatar
Vladislav Vaintroub committed
2286 2287 2288
	if (regex_list_check_match(regex_list, name)) {
		return(TRUE);
	}
2289

Vladislav Vaintroub's avatar
Vladislav Vaintroub committed
2290 2291
	if (tables_hash && find_filter_in_hashtable(name, tables_hash, NULL)) {
		return(TRUE);
2292 2293
	}

Vladislav Vaintroub's avatar
Vladislav Vaintroub committed
2294 2295 2296 2297 2298 2299 2300 2301 2302
	return FALSE;
}

enum skip_database_check_result {
	DATABASE_SKIP,
	DATABASE_SKIP_SOME_TABLES,
	DATABASE_DONT_SKIP,
	DATABASE_DONT_SKIP_UNLESS_EXPLICITLY_EXCLUDED,
};
2303

Vladislav Vaintroub's avatar
Vladislav Vaintroub committed
2304 2305 2306 2307 2308 2309 2310 2311 2312 2313 2314 2315 2316 2317 2318 2319 2320 2321 2322 2323 2324 2325 2326 2327 2328 2329 2330 2331 2332 2333 2334 2335 2336 2337
/************************************************************************
Checks if a database specified by name should be skipped from backup based on
the --databases, --databases_file or --databases_exclude options.

@return TRUE if entire database should be skipped,
	FALSE otherwise.
*/
static
skip_database_check_result
check_if_skip_database(
	const char* name  /*!< in: path to the database */
)
{
	/* There are some filters for databases, check them */
	xb_filter_entry_t*	database = NULL;

	if (databases_exclude_hash &&
		find_filter_in_hashtable(name, databases_exclude_hash,
					 &database) &&
		!database->has_tables) {
		/* Database is found and there are no tables specified,
		   skip entire db. */
		return DATABASE_SKIP;
	}

	if (databases_include_hash) {
		if (!find_filter_in_hashtable(name, databases_include_hash,
					      &database)) {
		/* Database isn't found, skip the database */
			return DATABASE_SKIP;
		} else if (database->has_tables) {
			return DATABASE_SKIP_SOME_TABLES;
		} else {
			return DATABASE_DONT_SKIP_UNLESS_EXPLICITLY_EXCLUDED;
2338 2339 2340
		}
	}

Vladislav Vaintroub's avatar
Vladislav Vaintroub committed
2341 2342 2343 2344 2345 2346 2347 2348 2349 2350 2351 2352 2353 2354 2355 2356 2357 2358
	return DATABASE_DONT_SKIP;
}

/************************************************************************
Checks if a database specified by path should be skipped from backup based on
the --databases, --databases_file or --databases_exclude options.

@return TRUE if the table should be skipped. */
my_bool
check_if_skip_database_by_path(
	const char* path /*!< in: path to the db directory. */
)
{
	if (databases_include_hash == NULL &&
		databases_exclude_hash == NULL) {
		return(FALSE);
	}

2359
	const char* db_name = strrchr(path, OS_PATH_SEPARATOR);
Vladislav Vaintroub's avatar
Vladislav Vaintroub committed
2360 2361 2362 2363 2364 2365 2366
	if (db_name == NULL) {
		db_name = path;
	} else {
		++db_name;
	}

	return check_if_skip_database(db_name) == DATABASE_SKIP;
2367 2368 2369 2370 2371 2372 2373 2374 2375 2376 2377 2378 2379 2380 2381 2382 2383 2384
}

/************************************************************************
Checks if a table specified as a name in the form "database/name" (InnoDB 5.6)
or "./database/name.ibd" (InnoDB 5.5-) should be skipped from backup based on
the --tables or --tables-file options.

@return TRUE if the table should be skipped. */
my_bool
check_if_skip_table(
/******************/
	const char*	name)	/*!< in: path to the table */
{
	char buf[FN_REFLEN];
	const char *dbname, *tbname;
	const char *ptr;
	char *eptr;

Vladislav Vaintroub's avatar
Vladislav Vaintroub committed
2385 2386 2387 2388 2389 2390
	if (regex_exclude_list.empty() &&
		regex_include_list.empty() &&
		tables_include_hash == NULL &&
		tables_exclude_hash == NULL &&
		databases_include_hash == NULL &&
		databases_exclude_hash == NULL) {
2391 2392 2393 2394 2395
		return(FALSE);
	}

	dbname = NULL;
	tbname = name;
2396
	while ((ptr = strchr(tbname, '/')) != NULL) {
2397 2398 2399 2400 2401 2402 2403 2404
		dbname = tbname;
		tbname = ptr + 1;
	}

	if (dbname == NULL) {
		return(FALSE);
	}

2405 2406 2407
	strncpy(buf, dbname, FN_REFLEN - 1);
	buf[FN_REFLEN - 1] = '\0';
	buf[tbname - 1 - dbname] = '\0';
2408

Vladislav Vaintroub's avatar
Vladislav Vaintroub committed
2409 2410 2411 2412
	const skip_database_check_result skip_database =
			check_if_skip_database(buf);
	if (skip_database == DATABASE_SKIP) {
		return (TRUE);
2413 2414 2415 2416 2417 2418 2419 2420 2421 2422 2423 2424 2425 2426 2427
	}

	buf[tbname - 1 - dbname] = '.';

	/* Check if there's a suffix in the table name. If so, truncate it. We
	rely on the fact that a dot cannot be a part of a table name (it is
	encoded by the server with the @NNNN syntax). */
	if ((eptr = strchr(&buf[tbname - dbname], '.')) != NULL) {

		*eptr = '\0';
	}

	/* For partitioned tables first try to match against the regexp
	without truncating the #P#... suffix so we can backup individual
	partitions with regexps like '^test[.]t#P#p5' */
Vladislav Vaintroub's avatar
Vladislav Vaintroub committed
2428 2429 2430 2431 2432 2433
	if (check_if_table_matches_filters(buf, regex_exclude_list,
					   tables_exclude_hash)) {
		return(TRUE);
	}
	if (check_if_table_matches_filters(buf, regex_include_list,
					   tables_include_hash)) {
2434 2435 2436 2437 2438
		return(FALSE);
	}
	if ((eptr = strstr(buf, "#P#")) != NULL) {
		*eptr = 0;

Vladislav Vaintroub's avatar
Vladislav Vaintroub committed
2439 2440 2441 2442 2443 2444
		if (check_if_table_matches_filters(buf, regex_exclude_list,
						   tables_exclude_hash)) {
			return (TRUE);
		}
		if (check_if_table_matches_filters(buf, regex_include_list,
						   tables_include_hash)) {
2445 2446 2447 2448
			return(FALSE);
		}
	}

Vladislav Vaintroub's avatar
Vladislav Vaintroub committed
2449 2450 2451 2452 2453 2454 2455 2456 2457 2458 2459 2460 2461 2462 2463 2464
	if (skip_database == DATABASE_DONT_SKIP_UNLESS_EXPLICITLY_EXCLUDED) {
		/* Database is in include-list, and qualified name wasn't
		   found in any of exclusion filters.*/
		return (FALSE);
	}

	if (skip_database == DATABASE_SKIP_SOME_TABLES ||
		!regex_include_list.empty() ||
		tables_include_hash) {

		/* Include lists are present, but qualified name
		   failed to match any.*/
		return(TRUE);
	}

	return(FALSE);
2465 2466 2467 2468 2469 2470 2471 2472 2473
}

const char*
xb_get_copy_action(const char *dflt)
{
	const char *action;

	if (xtrabackup_stream) {
		if (xtrabackup_compress) {
2474
			action = "Compressing and streaming";
2475 2476 2477 2478 2479
		} else {
			action = "Streaming";
		}
	} else {
		if (xtrabackup_compress) {
2480
			action = "Compressing";
2481 2482 2483 2484 2485 2486 2487 2488 2489 2490 2491 2492
		} else {
			action = dflt;
		}
	}

	return(action);
}

/* TODO: We may tune the behavior (e.g. by fil_aio)*/

static
my_bool
2493
xtrabackup_copy_datafile(fil_node_t* node, uint thread_n, const char *dest_name=0, ulonglong max_size=ULLONG_MAX)
2494 2495 2496 2497 2498 2499 2500 2501 2502 2503 2504 2505 2506 2507 2508 2509 2510 2511 2512 2513 2514 2515 2516
{
	char			 dst_name[FN_REFLEN];
	ds_file_t		*dstfile = NULL;
	xb_fil_cur_t		 cursor;
	xb_fil_cur_result_t	 res;
	xb_write_filt_t		*write_filter = NULL;
	xb_write_filt_ctxt_t	 write_filt_ctxt;
	const char		*action;
	xb_read_filt_t		*read_filter;
	my_bool			rc = FALSE;

	/* Get the name and the path for the tablespace. node->name always
	contains the path (which may be absolute for remote tablespaces in
	5.6+). space->name contains the tablespace name in the form
	"./database/table.ibd" (in 5.5-) or "database/table" (in 5.6+). For a
	multi-node shared tablespace, space->name contains the name of the first
	node, but that's irrelevant, since we only need node_name to match them
	against filters, and the shared tablespace is always copied regardless
	of the filters value. */

	const char* const node_name = node->space->name;
	const char* const node_path = node->name;

2517 2518
	if (fil_is_user_tablespace_id(node->space->id)
	    && check_if_skip_table(node_name)) {
2519
		msg(thread_n, "Skipping %s.", node_name);
2520 2521 2522
		return(FALSE);
	}

2523 2524 2525 2526 2527
	bool was_dropped;
	pthread_mutex_lock(&backup_mutex);
	was_dropped = (ddl_tracker.drops.find(node->space->id) != ddl_tracker.drops.end());
	pthread_mutex_unlock(&backup_mutex);
	if (was_dropped) {
Marko Mäkelä's avatar
Marko Mäkelä committed
2528 2529 2530 2531 2532
		if (node->is_open()) {
			mutex_enter(&fil_system.mutex);
			node->close();
			mutex_exit(&fil_system.mutex);
		}
2533 2534 2535
		goto skip;
	}

2536 2537 2538 2539 2540 2541
	if (!changed_page_bitmap) {
		read_filter = &rf_pass_through;
	}
	else {
		read_filter = &rf_bitmap;
	}
2542 2543

	res = xb_fil_cur_open(&cursor, read_filter, node, thread_n,max_size);
2544 2545 2546 2547 2548 2549
	if (res == XB_FIL_CUR_SKIP) {
		goto skip;
	} else if (res == XB_FIL_CUR_ERROR) {
		goto error;
	}

2550 2551 2552
	strncpy(dst_name, dest_name ? dest_name : cursor.rel_path,
		sizeof dst_name - 1);
	dst_name[sizeof dst_name - 1] = '\0';
2553 2554 2555 2556 2557 2558 2559 2560 2561 2562 2563 2564 2565

	/* Setup the page write filter */
	if (xtrabackup_incremental) {
		write_filter = &wf_incremental;
	} else {
		write_filter = &wf_write_through;
	}

	memset(&write_filt_ctxt, 0, sizeof(xb_write_filt_ctxt_t));
	ut_a(write_filter->process != NULL);

	if (write_filter->init != NULL &&
	    !write_filter->init(&write_filt_ctxt, dst_name, &cursor)) {
2566
		msg (thread_n, "mariabackup: error: failed to initialize page write filter.");
2567 2568 2569 2570 2571
		goto error;
	}

	dstfile = ds_open(ds_data, dst_name, &cursor.statinfo);
	if (dstfile == NULL) {
2572
		msg(thread_n,"mariabackup: error: can't open the destination stream for %s", dst_name);
2573 2574 2575 2576 2577 2578
		goto error;
	}

	action = xb_get_copy_action();

	if (xtrabackup_stream) {
2579
		msg(thread_n, "%s %s", action, node_path);
2580
	} else {
2581
		msg(thread_n, "%s %s to %s", action, node_path, dstfile->path);
2582 2583 2584 2585 2586 2587 2588 2589 2590 2591 2592 2593 2594 2595 2596 2597 2598 2599
	}

	/* The main copy loop */
	while ((res = xb_fil_cur_read(&cursor)) == XB_FIL_CUR_SUCCESS) {
		if (!write_filter->process(&write_filt_ctxt, dstfile)) {
			goto error;
		}
	}

	if (res == XB_FIL_CUR_ERROR) {
		goto error;
	}

	if (write_filter->finalize
	    && !write_filter->finalize(&write_filt_ctxt, dstfile)) {
		goto error;
	}

2600 2601 2602 2603
	pthread_mutex_lock(&backup_mutex);
	ddl_tracker.tables_in_backup[node->space->id] = node_name;
	pthread_mutex_unlock(&backup_mutex);

2604
	/* close */
2605
	msg(thread_n,"        ...done");
2606 2607 2608 2609 2610 2611 2612 2613 2614 2615 2616 2617 2618 2619 2620 2621 2622
	xb_fil_cur_close(&cursor);
	if (ds_close(dstfile)) {
		rc = TRUE;
	}
	if (write_filter && write_filter->deinit) {
		write_filter->deinit(&write_filt_ctxt);
	}
	return(rc);

error:
	xb_fil_cur_close(&cursor);
	if (dstfile != NULL) {
		ds_close(dstfile);
	}
	if (write_filter && write_filter->deinit) {
		write_filter->deinit(&write_filt_ctxt);;
	}
2623
	msg(thread_n, "mariabackup: xtrabackup_copy_datafile() failed.");
2624 2625 2626 2627 2628 2629 2630 2631 2632 2633
	return(TRUE); /*ERROR*/

skip:

	if (dstfile != NULL) {
		ds_close(dstfile);
	}
	if (write_filter && write_filter->deinit) {
		write_filter->deinit(&write_filt_ctxt);
	}
2634
	msg(thread_n,"Warning: We assume the  table was dropped during xtrabackup execution and ignore the tablespace %s", node_name);
2635 2636 2637
	return(FALSE);
}

2638
/** Copy redo log blocks to the data sink.
2639 2640
@param start_lsn	buffer start LSN
@param end_lsn		buffer end LSN
2641
@param last		whether we are copying the final part of the log
2642
@return	last scanned LSN
2643
@retval	0	on failure */
2644
static lsn_t xtrabackup_copy_log(lsn_t start_lsn, lsn_t end_lsn, bool last)
2645
{
2646
	lsn_t	scanned_lsn	= start_lsn;
2647
	const byte* log_block = log_sys.buf;
2648
	bool more_data = false;
2649

2650 2651 2652 2653
	for (ulint scanned_checkpoint = 0;
	     scanned_lsn < end_lsn;
	     log_block += OS_FILE_LOG_BLOCK_SIZE) {
		ulint checkpoint = log_block_get_checkpoint_no(log_block);
2654

2655 2656
		if (scanned_checkpoint > checkpoint
		    && scanned_checkpoint - checkpoint >= 0x80000000UL) {
2657 2658
			/* Garbage from a log buffer flush which was made
			before the most recent database recovery */
2659 2660
			msg(0,"checkpoint wrap: " LSN_PF ",%zx,%zx",
				scanned_lsn, scanned_checkpoint, checkpoint);
2661 2662 2663
			break;
		}

2664
		scanned_checkpoint = checkpoint;
2665

2666
		ulint	data_len = log_block_get_data_len(log_block);
2667

2668 2669 2670 2671 2672 2673
		more_data = recv_sys_add_to_parsing_buf(
				log_block,
				scanned_lsn + data_len);

		recv_sys->scanned_lsn = scanned_lsn + data_len;

2674 2675 2676 2677 2678 2679 2680
		if (data_len == OS_FILE_LOG_BLOCK_SIZE) {
			/* We got a full log block. */
			scanned_lsn += data_len;
		} else if (data_len
			   >= OS_FILE_LOG_BLOCK_SIZE - LOG_BLOCK_TRL_SIZE
			   || data_len <= LOG_BLOCK_HDR_SIZE) {
			/* We got a garbage block (abrupt end of the log). */
2681
			msg(0,"garbage block: " LSN_PF ",%zu",scanned_lsn, data_len);
2682 2683 2684 2685
			break;
		} else {
			/* We got a partial block (abrupt end of the log). */
			scanned_lsn += data_len;
2686
			break;
2687 2688 2689
		}
	}

2690 2691
	if (more_data && recv_parse_log_recs(0, STORE_NO, false)) {

2692
		msg("Error: copying the log failed");
2693 2694 2695 2696 2697 2698

		return(0);
	}

	recv_sys_justify_left_parsing_buf();

2699
	log_sys.log.scanned_lsn = scanned_lsn;
2700

2701
	end_lsn = last
2702
		? ut_uint64_align_up(scanned_lsn, OS_FILE_LOG_BLOCK_SIZE)
2703
		: scanned_lsn & ~lsn_t(OS_FILE_LOG_BLOCK_SIZE - 1);
2704

2705
	if (ulint write_size = ulint(end_lsn - start_lsn)) {
2706
		if (srv_encrypt_log) {
2707
			log_crypt(log_sys.buf, start_lsn, write_size);
2708
		}
2709

2710
		if (ds_write(dst_log_file, log_sys.buf, write_size)) {
Marko Mäkelä's avatar
Marko Mäkelä committed
2711
			msg("Error: write to logfile failed");
2712 2713
			return(0);
		}
2714 2715
	}

2716
	return(scanned_lsn);
2717 2718
}

2719
/** Copy redo log until the current end of the log is reached
2720
@param last	whether we are copying the final part of the log
2721
@return	whether the operation failed */
2722
static bool xtrabackup_copy_logfile(bool last = false)
2723 2724
{
	ut_a(dst_log_file != NULL);
2725
	ut_ad(recv_sys != NULL);
2726

2727 2728
	lsn_t	start_lsn;
	lsn_t	end_lsn;
2729

2730 2731 2732
	recv_sys->parse_start_lsn = log_copy_scanned_lsn;
	recv_sys->scanned_lsn = log_copy_scanned_lsn;

2733 2734
	start_lsn = ut_uint64_align_down(log_copy_scanned_lsn,
					 OS_FILE_LOG_BLOCK_SIZE);
2735 2736
	do {
		end_lsn = start_lsn + RECV_SCAN_SIZE;
2737

2738
		xtrabackup_io_throttling();
2739

2740
		log_mutex_enter();
2741
		lsn_t lsn= start_lsn;
2742
		for (int retries= 0; retries < 100; retries++) {
Marko Mäkelä's avatar
Marko Mäkelä committed
2743
			if (log_sys.log.read_log_seg(&lsn, end_lsn)
2744
			    || lsn != start_lsn) {
2745 2746
				break;
			}
2747
			msg("Retrying read of log at LSN=" LSN_PF, lsn);
2748 2749
			my_sleep(1000);
		}
2750

2751
		start_lsn = (lsn == start_lsn)
2752
			? 0 : xtrabackup_copy_log(start_lsn, lsn, last);
2753

2754
		log_mutex_exit();
2755

2756
		if (!start_lsn) {
2757 2758 2759 2760
			msg(recv_sys->found_corrupt_log
			    ? "xtrabackup_copy_logfile() failed: corrupt log."
			    : "xtrabackup_copy_logfile() failed.");
			return true;
2761
		}
2762
	} while (start_lsn == end_lsn);
2763

2764
	ut_ad(start_lsn == log_sys.log.scanned_lsn);
2765

2766
	msg(">> log scanned up to (" LSN_PF ")", start_lsn);
2767

2768
	/* update global variable*/
2769
	pthread_mutex_lock(&backup_mutex);
2770
	log_copy_scanned_lsn = start_lsn;
2771 2772
	pthread_cond_broadcast(&scanned_lsn_cond);
	pthread_mutex_unlock(&backup_mutex);
2773

2774
	debug_sync_point("xtrabackup_copy_logfile_pause");
2775
	return(false);
2776 2777
}

2778 2779 2780 2781 2782 2783 2784 2785 2786 2787 2788 2789 2790 2791 2792
/**
Wait until redo log copying thread processes given lsn
*/
void backup_wait_for_lsn(lsn_t lsn) {
	bool completed = false;
	pthread_mutex_lock(&backup_mutex);
	do {
		pthread_cond_wait(&scanned_lsn_cond, &backup_mutex);
		completed = log_copy_scanned_lsn >= lsn;
	} while (!completed);
	pthread_mutex_unlock(&backup_mutex);
}

extern lsn_t server_lsn_after_lock;

2793
static os_thread_ret_t DECLARE_THREAD(log_copying_thread)(void*)
2794 2795 2796 2797 2798 2799 2800
{
	/*
	  Initialize mysys thread-specific memory so we can
	  use mysys functions in this thread.
	*/
	my_thread_init();

2801
	for (;;) {
2802 2803 2804 2805
		os_event_reset(log_copying_stop);
		os_event_wait_time_low(log_copying_stop,
				       xtrabackup_log_copy_interval * 1000ULL,
				       0);
2806 2807 2808 2809 2810 2811
		if (xtrabackup_copy_logfile()) {
			break;
		}

		log_mutex_enter();
		bool completed = metadata_to_lsn
2812
			&& metadata_to_lsn <= log_copy_scanned_lsn;
2813 2814 2815 2816 2817
		log_mutex_exit();
		if (completed) {
			break;
		}
	}
2818

2819
	log_copying_running = false;
2820
	my_thread_end();
2821
	os_thread_exit();
2822 2823 2824 2825 2826

	return(0);
}

/* io throttle watching (rough) */
2827
static os_thread_ret_t DECLARE_THREAD(io_watching_thread)(void*)
2828 2829 2830 2831
{
	/* currently, for --backup only */
	ut_a(xtrabackup_backup);

2832
	while (log_copying_running && !metadata_to_lsn) {
2833 2834 2835 2836 2837 2838 2839 2840 2841
		os_thread_sleep(1000000); /*1 sec*/
		io_ticket = xtrabackup_throttle;
		os_event_set(wait_throttle);
	}

	/* stop io throttle */
	xtrabackup_throttle = 0;
	os_event_set(wait_throttle);

2842
	io_watching_thread_running = false;
2843

2844
	os_thread_exit();
2845 2846 2847 2848

	return(0);
}

2849 2850 2851 2852 2853 2854 2855 2856 2857 2858 2859 2860 2861 2862 2863 2864 2865 2866 2867 2868 2869 2870
#ifndef DBUG_OFF
/*
In debug mode,  execute SQL statement that was passed via environment.
To use this facility, you need to

1. Add code DBUG_EXECUTE_MARIABACKUP_EVENT("my_event_name", key););
  to the code. key is usually a table name
2. Set environment variable my_event_name_$key SQL statement you want to execute
   when event occurs, in DBUG_EXECUTE_IF from above.
   In mtr , you can set environment via 'let' statement (do not use $ as the first char
   for the variable)
3. start mariabackup with --dbug=+d,debug_mariabackup_events
*/
static void dbug_mariabackup_event(const char *event,const char *key)
{
	char envvar[FN_REFLEN];
	if (key) {
		snprintf(envvar, sizeof(envvar), "%s_%s", event, key);
		char *slash = strchr(envvar, '/');
		if (slash)
			*slash = '_';
	} else {
2871 2872
		strncpy(envvar, event, sizeof envvar - 1);
		envvar[sizeof envvar - 1] = '\0';
2873 2874 2875
	}
	char *sql = getenv(envvar);
	if (sql) {
2876
		msg("dbug_mariabackup_event : executing '%s'", sql);
2877 2878 2879 2880 2881 2882 2883 2884 2885
		xb_mysql_query(mysql_connection, sql, false, true);
	}

}
#define DBUG_MARIABACKUP_EVENT(A, B) DBUG_EXECUTE_IF("mariabackup_events", dbug_mariabackup_event(A,B););
#else
#define DBUG_MARIABACKUP_EVENT(A,B)
#endif

2886 2887 2888 2889
/**************************************************************************
Datafiles copying thread.*/
static
os_thread_ret_t
2890
DECLARE_THREAD(data_copy_thread_func)(
2891 2892 2893 2894 2895 2896 2897 2898 2899 2900 2901 2902 2903 2904 2905 2906
/*==================*/
	void *arg) /* thread context */
{
	data_thread_ctxt_t	*ctxt = (data_thread_ctxt_t *) arg;
	uint			num = ctxt->num;
	fil_node_t*		node;

	/*
	  Initialize mysys thread-specific memory so we can
	  use mysys functions in this thread.
	*/
	my_thread_init();

	debug_sync_point("data_copy_thread_func");

	while ((node = datafiles_iter_next(ctxt->it)) != NULL) {
2907
		DBUG_MARIABACKUP_EVENT("before_copy", node->space->name);
2908 2909
		/* copy the datafile */
		if(xtrabackup_copy_datafile(node, num)) {
2910
			die("failed to copy datafile.");
2911
		}
2912 2913 2914

		DBUG_MARIABACKUP_EVENT("after_copy", node->space->name);

2915 2916
	}

2917
	pthread_mutex_lock(ctxt->count_mutex);
2918
	(*ctxt->count)--;
2919
	pthread_mutex_unlock(ctxt->count_mutex);
2920 2921

	my_thread_end();
2922
	os_thread_exit();
2923 2924 2925 2926 2927 2928 2929 2930 2931
	OS_THREAD_DUMMY_RETURN;
}

/************************************************************************
Initialize the appropriate datasink(s). Both local backups and streaming in the
'xbstream' format allow parallel writes so we can write directly.

Otherwise (i.e. when streaming in the 'tar' format) we need 2 separate datasinks
for the data stream (and don't allow parallel data copying) and for metainfo
2932
files (including ib_logfile0). The second datasink writes to temporary
2933 2934 2935 2936 2937 2938 2939 2940 2941 2942 2943 2944 2945 2946 2947 2948 2949 2950 2951 2952 2953
files first, and then streams them in a serialized way when closed. */
static void
xtrabackup_init_datasinks(void)
{
	/* Start building out the pipelines from the terminus back */
	if (xtrabackup_stream) {
		/* All streaming goes to stdout */
		ds_data = ds_meta = ds_redo = ds_create(xtrabackup_target_dir,
						        DS_TYPE_STDOUT);
	} else {
		/* Local filesystem */
		ds_data = ds_meta = ds_redo = ds_create(xtrabackup_target_dir,
						        DS_TYPE_LOCAL);
	}

	/* Track it for destruction */
	xtrabackup_add_datasink(ds_data);

	/* Stream formatting */
	if (xtrabackup_stream) {
		ds_ctxt_t	*ds;
2954 2955 2956

	 ut_a(xtrabackup_stream_fmt == XB_STREAM_FMT_XBSTREAM);
	 ds = ds_create(xtrabackup_target_dir, DS_TYPE_XBSTREAM);
2957 2958 2959 2960 2961 2962 2963

		xtrabackup_add_datasink(ds);

		ds_set_pipe(ds, ds_data);
		ds_data = ds;


2964
		ds_redo = ds_meta = ds_data;
2965 2966 2967 2968 2969 2970 2971 2972 2973 2974 2975 2976 2977 2978 2979 2980 2981 2982 2983 2984 2985 2986 2987 2988 2989 2990 2991 2992 2993 2994 2995 2996 2997 2998 2999 3000 3001 3002 3003 3004 3005 3006 3007 3008 3009 3010 3011 3012 3013 3014 3015 3016 3017 3018 3019
	}

	/* Compression for ds_data and ds_redo */
	if (xtrabackup_compress) {
		ds_ctxt_t	*ds;

		/* Use a 1 MB buffer for compressed output stream */
		ds = ds_create(xtrabackup_target_dir, DS_TYPE_BUFFER);
		ds_buffer_set_size(ds, 1024 * 1024);
		xtrabackup_add_datasink(ds);
		ds_set_pipe(ds, ds_data);
		if (ds_data != ds_redo) {
			ds_data = ds;
			ds = ds_create(xtrabackup_target_dir, DS_TYPE_BUFFER);
			ds_buffer_set_size(ds, 1024 * 1024);
			xtrabackup_add_datasink(ds);
			ds_set_pipe(ds, ds_redo);
			ds_redo = ds;
		} else {
			ds_redo = ds_data = ds;
		}

		ds = ds_create(xtrabackup_target_dir, DS_TYPE_COMPRESS);
		xtrabackup_add_datasink(ds);
		ds_set_pipe(ds, ds_data);
		if (ds_data != ds_redo) {
			ds_data = ds;
			ds = ds_create(xtrabackup_target_dir, DS_TYPE_COMPRESS);
			xtrabackup_add_datasink(ds);
			ds_set_pipe(ds, ds_redo);
			ds_redo = ds;
		} else {
			ds_redo = ds_data = ds;
		}
	}
}

/************************************************************************
Destroy datasinks.

Destruction is done in the specific order to not violate their order in the
pipeline so that each datasink is able to flush data down the pipeline. */
static void xtrabackup_destroy_datasinks(void)
{
	for (uint i = actual_datasinks; i > 0; i--) {
		ds_destroy(datasinks[i-1]);
		datasinks[i-1] = NULL;
	}
	ds_data = NULL;
	ds_meta = NULL;
	ds_redo = NULL;
}

#define SRV_MAX_N_PENDING_SYNC_IOS	100

3020
/** Initialize the tablespace cache subsystem. */
3021
static
3022 3023
void
xb_fil_io_init()
3024
{
3025
	fil_system.create(srv_file_per_table ? 50000 : 5000);
3026
}
3027

3028 3029 3030 3031 3032 3033 3034 3035 3036 3037 3038 3039 3040 3041
static
Datafile*
xb_new_datafile(const char *name, bool is_remote)
{
	if (is_remote) {
		RemoteDatafile *remote_file = new RemoteDatafile();
		remote_file->set_name(name);
		return(remote_file);
	} else {
		Datafile *file = new Datafile();
		file->set_name(name);
		file->make_filepath(".", name, IBD);
		return(file);
	}
3042 3043
}

3044

3045 3046
static
void
3047 3048 3049 3050
xb_load_single_table_tablespace(
	const char *dirname,
	const char *filname,
	bool is_remote)
3051
{
3052 3053
	ut_ad(srv_operation == SRV_OPERATION_BACKUP
	      || srv_operation == SRV_OPERATION_RESTORE_DELTA);
3054 3055
	/* Ignore .isl files on XtraBackup recovery. All tablespaces must be
	local. */
3056
	if (is_remote && srv_operation == SRV_OPERATION_RESTORE_DELTA) {
3057 3058 3059 3060 3061
		return;
	}
	if (check_if_skip_table(filname)) {
		return;
	}
3062

3063 3064 3065 3066 3067 3068 3069 3070 3071
	/* The name ends in .ibd or .isl;
	try opening the file */
	char*	name;
	size_t	dirlen		= dirname == NULL ? 0 : strlen(dirname);
	size_t	namelen		= strlen(filname);
	ulint	pathlen		= dirname == NULL ? namelen + 1: dirlen + namelen + 2;
	lsn_t	flush_lsn;
	dberr_t	err;
	fil_space_t	*space;
3072

3073
	name = static_cast<char*>(ut_malloc_nokey(pathlen));
3074

3075
	if (dirname != NULL) {
3076
		snprintf(name, pathlen, "%s/%s", dirname, filname);
3077 3078
		name[pathlen - 5] = 0;
	} else {
3079
		snprintf(name, pathlen, "%s", filname);
3080 3081
		name[pathlen - 5] = 0;
	}
3082

3083
	Datafile *file = xb_new_datafile(name, is_remote);
3084

3085
	if (file->open_read_only(true) != DB_SUCCESS) {
3086
		die("Can't open datafile %s", name);
3087
	}
3088

3089 3090 3091 3092 3093 3094 3095 3096 3097 3098
	for (int i = 0; i < 10; i++) {
		err = file->validate_first_page(&flush_lsn);
		if (err != DB_CORRUPTION) {
			break;
		}

		my_sleep(1000);
	}

	bool is_empty_file = file->exists() && file->is_empty_file();
3099

3100 3101 3102
	if (err == DB_SUCCESS && file->space_id() != SRV_TMP_SPACE_ID) {
		os_offset_t	node_size = os_file_get_size(file->handle());
		os_offset_t	n_pages;
3103

3104 3105 3106 3107 3108 3109 3110 3111 3112 3113
		ut_a(node_size != (os_offset_t) -1);

		n_pages = node_size / page_size_t(file->flags()).physical();

		space = fil_space_create(
			name, file->space_id(), file->flags(),
			FIL_TYPE_TABLESPACE, NULL/* TODO: crypt_data */);

		ut_a(space != NULL);

3114 3115
		space->add(file->filepath(), OS_FILE_CLOSED, ulint(n_pages),
			   false, false);
3116 3117
		/* by opening the tablespace we forcing node and space objects
		in the cache to be populated with fields from space header */
3118
		space->open();
3119

3120 3121
		if (srv_operation == SRV_OPERATION_RESTORE_DELTA
		    || xb_close_files) {
3122
			space->close();
3123 3124 3125 3126 3127 3128 3129
		}
	}

	ut_free(name);

	delete file;

3130
	if (err != DB_SUCCESS && xtrabackup_backup && !is_empty_file) {
3131
		die("Failed to not validate first page of the file %s, error %d",name, (int)err);
3132 3133 3134 3135 3136 3137
	}
}

/** Scan the database directories under the MySQL datadir, looking for
.ibd files and determining the space id in each of them.
@return	DB_SUCCESS or error number */
3138 3139

static dberr_t enumerate_ibd_files(process_single_tablespace_func_t callback)
3140 3141 3142 3143 3144 3145 3146 3147 3148
{
	int		ret;
	char*		dbpath		= NULL;
	ulint		dbpath_len	= 100;
	os_file_dir_t	dir;
	os_file_dir_t	dbdir;
	os_file_stat_t	dbinfo;
	os_file_stat_t	fileinfo;
	dberr_t		err		= DB_SUCCESS;
3149
	size_t len;
3150 3151 3152 3153 3154 3155 3156 3157 3158 3159 3160 3161 3162 3163 3164 3165 3166 3167 3168 3169 3170

	/* The datadir of MySQL is always the default directory of mysqld */

	dir = os_file_opendir(fil_path_to_mysql_datadir, true);

	if (dir == NULL) {

		return(DB_ERROR);
	}

	dbpath = static_cast<char*>(ut_malloc_nokey(dbpath_len));

	/* Scan all directories under the datadir. They are the database
	directories of MySQL. */

	ret = fil_file_readdir_next_file(&err, fil_path_to_mysql_datadir, dir,
					 &dbinfo);
	while (ret == 0) {

		/* General tablespaces are always at the first level of the
		data home dir */
3171 3172 3173
		if (dbinfo.type == OS_FILE_TYPE_FILE) {
			bool is_isl = ends_with(dbinfo.name, ".isl");
			bool is_ibd = !is_isl && ends_with(dbinfo.name,".ibd");
3174 3175

			if (is_isl || is_ibd) {
3176
				(*callback)(NULL, dbinfo.name, is_isl);
3177 3178 3179 3180 3181 3182 3183 3184 3185 3186 3187 3188 3189 3190 3191 3192 3193 3194 3195 3196 3197 3198 3199
			}
		}

		if (dbinfo.type == OS_FILE_TYPE_FILE
		    || dbinfo.type == OS_FILE_TYPE_UNKNOWN) {

			goto next_datadir_item;
		}

		/* We found a symlink or a directory; try opening it to see
		if a symlink is a directory */

		len = strlen(fil_path_to_mysql_datadir)
			+ strlen (dbinfo.name) + 2;
		if (len > dbpath_len) {
			dbpath_len = len;

			if (dbpath) {
				ut_free(dbpath);
			}

			dbpath = static_cast<char*>(ut_malloc_nokey(dbpath_len));
		}
3200 3201
		snprintf(dbpath, dbpath_len,
			 "%s/%s", fil_path_to_mysql_datadir, dbinfo.name);
3202 3203 3204 3205 3206 3207 3208 3209 3210 3211 3212 3213 3214 3215 3216 3217 3218 3219 3220 3221 3222 3223 3224 3225 3226 3227 3228 3229
		os_normalize_path(dbpath);

		if (check_if_skip_database_by_path(dbpath)) {
			fprintf(stderr, "Skipping db: %s\n", dbpath);
			goto next_datadir_item;
		}

		/* We want wrong directory permissions to be a fatal error for
		XtraBackup. */
		dbdir = os_file_opendir(dbpath, true);

		if (dbdir != NULL) {

			/* We found a database directory; loop through it,
			looking for possible .ibd files in it */

			for (ret = fil_file_readdir_next_file(&err, dbpath,
							      dbdir,
							      &fileinfo);
			     ret == 0;
			     ret = fil_file_readdir_next_file(&err, dbpath,
							      dbdir,
							      &fileinfo)) {
				if (fileinfo.type == OS_FILE_TYPE_DIR) {
					continue;
				}

				/* We found a symlink or a file */
3230
				if (strlen(fileinfo.name) > 4) {
3231
					bool is_isl= false;
Marko Mäkelä's avatar
Marko Mäkelä committed
3232
					if (ends_with(fileinfo.name, ".ibd") || ((is_isl = ends_with(fileinfo.name, ".isl"))))
3233
						(*callback)(dbinfo.name, fileinfo.name, is_isl);
3234 3235 3236 3237 3238
				}
			}

			if (0 != os_file_closedir(dbdir)) {
				fprintf(stderr, "InnoDB: Warning: could not"
3239
				 " close database directory %s\n",
3240 3241 3242 3243 3244 3245 3246 3247 3248 3249 3250 3251 3252 3253 3254 3255 3256 3257 3258 3259 3260 3261 3262 3263 3264
					dbpath);

				err = DB_ERROR;
			}

		} else {

			err = DB_ERROR;
			break;

		}

next_datadir_item:
		ret = fil_file_readdir_next_file(&err,
						 fil_path_to_mysql_datadir,
						 dir, &dbinfo);
	}

	ut_free(dbpath);

	if (0 != os_file_closedir(dir)) {
		fprintf(stderr,
			"InnoDB: Error: could not close MySQL datadir\n");

		return(DB_ERROR);
3265 3266
	}

3267 3268 3269
	return(err);
}

3270 3271 3272 3273 3274 3275 3276
/** Assign srv_undo_space_id_start variable if there are undo tablespace present.
Read the TRX_SYS page from ibdata1 file and get the minimum space id from
the first slot rollback segments of TRX_SYS_PAGE_NO.
@retval DB_ERROR if file open or page read failed.
@retval DB_SUCCESS if srv_undo_space_id assigned successfully. */
static dberr_t xb_assign_undo_space_start()
{
3277

3278 3279 3280
	pfs_os_file_t	file;
	byte*		buf;
	byte*		page;
Marko Mäkelä's avatar
Marko Mäkelä committed
3281
	bool		ret;
3282
	dberr_t		error = DB_SUCCESS;
3283
	ulint		space;
3284
	int n_retries = 5;
3285 3286 3287 3288 3289

	if (srv_undo_tablespaces == 0) {
		return error;
	}

3290 3291
	file = os_file_create(0, srv_sys_space.first_datafile()->filepath(),
		OS_FILE_OPEN, OS_FILE_NORMAL, OS_DATA_FILE, true, &ret);
3292

Marko Mäkelä's avatar
Marko Mäkelä committed
3293
	if (!ret) {
3294
		msg("Error opening %s", srv_sys_space.first_datafile()->filepath());
3295 3296 3297
		return DB_ERROR;
	}

3298
	buf = static_cast<byte*>(ut_malloc_nokey(2U << srv_page_size_shift));
3299
	page = static_cast<byte*>(ut_align(buf, srv_page_size));
3300 3301

retry:
3302
	if (!os_file_read(IORequestRead, file, page,
3303
			  TRX_SYS_PAGE_NO << srv_page_size_shift,
3304
			  srv_page_size)) {
Marko Mäkelä's avatar
Marko Mäkelä committed
3305
		msg("Reading TRX_SYS page failed.\n");
3306 3307 3308 3309 3310
		error = DB_ERROR;
		goto func_exit;
	}

	/* TRX_SYS page can't be compressed or encrypted. */
Marko Mäkelä's avatar
Marko Mäkelä committed
3311
	if (buf_page_is_corrupted(false, page, univ_page_size)) {
3312 3313 3314 3315 3316 3317 3318 3319
		if (n_retries--) {
			os_thread_sleep(1000);
			goto retry;
		} else {
			msg("mariabackup: TRX_SYS page corrupted.\n");
			error = DB_ERROR;
			goto func_exit;
		}
3320 3321 3322 3323 3324
	}

	/* 0th slot always points to system tablespace.
	1st slot should point to first undotablespace which is minimum. */

3325 3326 3327 3328
	ut_ad(mach_read_from_4(TRX_SYS + TRX_SYS_RSEGS
			       + TRX_SYS_RSEG_SLOT_SIZE
			       + TRX_SYS_RSEG_PAGE_NO + page)
	      != FIL_NULL);
3329 3330 3331 3332 3333 3334 3335 3336 3337 3338 3339 3340 3341 3342 3343

	space = mach_read_ulint(TRX_SYS + TRX_SYS_RSEGS
				+ TRX_SYS_RSEG_SLOT_SIZE
				+ TRX_SYS_RSEG_SPACE + page, MLOG_4BYTES);

	srv_undo_space_id_start = space;

func_exit:
	ut_free(buf);
	ret = os_file_close(file);
	ut_a(ret);

	return error;
}

3344 3345 3346 3347 3348 3349 3350 3351 3352 3353 3354 3355 3356
/****************************************************************************
Populates the tablespace memory cache by scanning for and opening data files.
@returns DB_SUCCESS or error code.*/
static
dberr_t
xb_load_tablespaces()
{
	bool	create_new_db;
	dberr_t	err;
	ulint   sum_of_new_sizes;
        lsn_t	flush_lsn;

	ut_ad(srv_operation == SRV_OPERATION_BACKUP
3357
	      || srv_operation == SRV_OPERATION_RESTORE_DELTA);
3358 3359 3360 3361 3362

	err = srv_sys_space.check_file_spec(&create_new_db, 0);

	/* create_new_db must not be true. */
	if (err != DB_SUCCESS || create_new_db) {
3363
		msg("Could not find data files at the specified datadir");
3364 3365 3366
		return(DB_ERROR);
	}

3367 3368 3369 3370 3371 3372 3373 3374 3375
	for (int i= 0; i < 10; i++) {
		err = srv_sys_space.open_or_create(false, false, &sum_of_new_sizes,
						 &flush_lsn);
		if (err == DB_PAGE_CORRUPTED || err == DB_CORRUPTION) {
			my_sleep(1000);
		}
		else
		 break;
	}
3376 3377

	if (err != DB_SUCCESS) {
3378
		msg("Could not open data files.\n");
3379 3380 3381 3382 3383
		return(err);
	}

	/* Add separate undo tablespaces to fil_system */

3384 3385 3386 3387 3388 3389
	err = xb_assign_undo_space_start();

	if (err != DB_SUCCESS) {
		return err;
	}

3390 3391
	err = srv_undo_tablespaces_init(false);

3392 3393 3394 3395
	if (err != DB_SUCCESS) {
		return(err);
	}

3396
	/* It is important to call xb_load_single_table_tablespaces() after
3397 3398 3399
	srv_undo_tablespaces_init(), because fil_is_user_tablespace_id() *
	relies on srv_undo_tablespaces_open to be properly initialized */

3400
	msg("mariabackup: Generating a list of tablespaces");
3401

3402
	err = enumerate_ibd_files(xb_load_single_table_tablespace);
3403 3404 3405 3406 3407
	if (err != DB_SUCCESS) {
		return(err);
	}

	debug_sync_point("xtrabackup_load_tablespaces_pause");
3408
	DBUG_MARIABACKUP_EVENT("after_load_tablespaces", 0);
3409 3410 3411 3412 3413 3414 3415
	return(DB_SUCCESS);
}

/************************************************************************
Initialize the tablespace memory cache and populate it by scanning for and
opening data files.
@returns DB_SUCCESS or error code.*/
3416 3417 3418
static
dberr_t
xb_data_files_init()
3419 3420 3421 3422 3423 3424 3425 3426
{
	xb_fil_io_init();

	return(xb_load_tablespaces());
}

/************************************************************************
Destroy the tablespace memory cache. */
3427
static
3428
void
3429
xb_data_files_close()
3430
{
3431
	ut_ad(!os_thread_count);
3432 3433 3434 3435 3436 3437 3438 3439 3440 3441 3442 3443 3444 3445 3446 3447 3448 3449 3450 3451 3452 3453
	fil_close_all_files();
	if (buf_dblwr) {
		buf_dblwr_free();
	}
}

/***********************************************************************
Allocate and initialize the entry for databases and tables filtering
hash tables. If memory allocation is not successful, terminate program.
@return pointer to the created entry.  */
static
xb_filter_entry_t *
xb_new_filter_entry(
/*================*/
	const char*	name)	/*!< in: name of table/database */
{
	xb_filter_entry_t	*entry;
	ulint namelen = strlen(name);

	ut_a(namelen <= NAME_LEN * 2 + 1);

	entry = static_cast<xb_filter_entry_t *>
3454
		(malloc(sizeof(xb_filter_entry_t) + namelen + 1));
3455 3456 3457 3458 3459 3460 3461 3462 3463 3464 3465 3466 3467 3468 3469 3470 3471 3472 3473 3474 3475 3476 3477 3478 3479 3480 3481 3482 3483 3484 3485 3486 3487 3488 3489 3490 3491 3492 3493 3494 3495 3496 3497 3498 3499 3500 3501 3502
	memset(entry, '\0', sizeof(xb_filter_entry_t) + namelen + 1);
	entry->name = ((char*)entry) + sizeof(xb_filter_entry_t);
	strcpy(entry->name, name);
	entry->has_tables = FALSE;

	return entry;
}

/***********************************************************************
Add entry to hash table. If hash table is NULL, allocate and initialize
new hash table */
static
xb_filter_entry_t*
xb_add_filter(
/*========================*/
	const char*	name,	/*!< in: name of table/database */
	hash_table_t**	hash)	/*!< in/out: hash to insert into */
{
	xb_filter_entry_t*	entry;

	entry = xb_new_filter_entry(name);

	if (UNIV_UNLIKELY(*hash == NULL)) {
		*hash = hash_create(1000);
	}
	HASH_INSERT(xb_filter_entry_t,
		name_hash, *hash,
		ut_fold_string(entry->name),
		entry);

	return entry;
}

/***********************************************************************
Validate name of table or database. If name is invalid, program will
be finished with error code */
static
void
xb_validate_name(
/*=============*/
	const char*	name,	/*!< in: name */
	size_t		len)	/*!< in: length of name */
{
	const char*	p;

	/* perform only basic validation. validate length and
	path symbols */
	if (len > NAME_LEN) {
3503
		die("name `%s` is too long.", name);
3504 3505
	}
	p = strpbrk(name, "/\\~");
Monty's avatar
Monty committed
3506
	if (p && (uint) (p - name) < NAME_LEN) {
3507
		die("name `%s` is not valid.", name);
3508 3509 3510 3511 3512 3513 3514 3515 3516 3517
	}
}

/***********************************************************************
Register new filter entry which can be either database
or table name.  */
static
void
xb_register_filter_entry(
/*=====================*/
Vladislav Vaintroub's avatar
Vladislav Vaintroub committed
3518 3519 3520 3521
	const char*	name,	/*!< in: name */
	hash_table_t** databases_hash,
	hash_table_t** tables_hash
	)
3522 3523 3524 3525 3526 3527 3528 3529 3530 3531 3532 3533 3534 3535 3536
{
	const char*		p;
	size_t			namelen;
	xb_filter_entry_t*	db_entry = NULL;

	namelen = strlen(name);
	if ((p = strchr(name, '.')) != NULL) {
		char dbname[NAME_LEN + 1];

		xb_validate_name(name, p - name);
		xb_validate_name(p + 1, namelen - (p - name));

		strncpy(dbname, name, p - name);
		dbname[p - name] = 0;

Vladislav Vaintroub's avatar
Vladislav Vaintroub committed
3537 3538
		if (*databases_hash) {
			HASH_SEARCH(name_hash, (*databases_hash),
3539 3540 3541 3542 3543 3544
					ut_fold_string(dbname),
					xb_filter_entry_t*,
					db_entry, (void) 0,
					!strcmp(db_entry->name, dbname));
		}
		if (!db_entry) {
Vladislav Vaintroub's avatar
Vladislav Vaintroub committed
3545
			db_entry = xb_add_filter(dbname, databases_hash);
3546 3547
		}
		db_entry->has_tables = TRUE;
Vladislav Vaintroub's avatar
Vladislav Vaintroub committed
3548
		xb_add_filter(name, tables_hash);
3549 3550 3551
	} else {
		xb_validate_name(name, namelen);

Vladislav Vaintroub's avatar
Vladislav Vaintroub committed
3552
		xb_add_filter(name, databases_hash);
3553 3554 3555
	}
}

Vladislav Vaintroub's avatar
Vladislav Vaintroub committed
3556 3557 3558 3559 3560 3561 3562 3563 3564 3565 3566 3567 3568 3569 3570 3571 3572 3573 3574 3575
static
void
xb_register_include_filter_entry(
	const char* name
)
{
	xb_register_filter_entry(name, &databases_include_hash,
				 &tables_include_hash);
}

static
void
xb_register_exclude_filter_entry(
	const char* name
)
{
	xb_register_filter_entry(name, &databases_exclude_hash,
				 &tables_exclude_hash);
}

3576 3577 3578 3579 3580 3581 3582 3583 3584
/***********************************************************************
Register new table for the filter.  */
static
void
xb_register_table(
/*==============*/
	const char* name)	/*!< in: name of table */
{
	if (strchr(name, '.') == NULL) {
3585
		die("`%s` is not fully qualified name.", name);
3586 3587
	}

Vladislav Vaintroub's avatar
Vladislav Vaintroub committed
3588
	xb_register_include_filter_entry(name);
3589 3590 3591 3592
}

static
void
Vladislav Vaintroub's avatar
Vladislav Vaintroub committed
3593 3594 3595 3596
xb_add_regex_to_list(
	const char* regex,  /*!< in: regex */
	const char* error_context,  /*!< in: context to error message */
	regex_list_t* list) /*! in: list to put new regex to */
3597 3598 3599 3600
{
	char			errbuf[100];
	int			ret;

Vladislav Vaintroub's avatar
Vladislav Vaintroub committed
3601 3602
	regex_t compiled_regex;
	ret = regcomp(&compiled_regex, regex, REG_EXTENDED);
3603 3604

	if (ret != 0) {
Vladislav Vaintroub's avatar
Vladislav Vaintroub committed
3605
		regerror(ret, &compiled_regex, errbuf, sizeof(errbuf));
3606
		msg("mariabackup: error: %s regcomp(%s): %s",
Vladislav Vaintroub's avatar
Vladislav Vaintroub committed
3607
			error_context, regex, errbuf);
3608 3609 3610
		exit(EXIT_FAILURE);
	}

Vladislav Vaintroub's avatar
Vladislav Vaintroub committed
3611 3612 3613 3614 3615 3616 3617 3618 3619 3620 3621 3622 3623 3624 3625 3626 3627 3628 3629 3630 3631 3632 3633
	list->push_back(compiled_regex);
}

/***********************************************************************
Register new regex for the include filter.  */
static
void
xb_register_include_regex(
/*==============*/
	const char* regex)	/*!< in: regex */
{
	xb_add_regex_to_list(regex, "tables", &regex_include_list);
}

/***********************************************************************
Register new regex for the exclude filter.  */
static
void
xb_register_exclude_regex(
/*==============*/
	const char* regex)	/*!< in: regex */
{
	xb_add_regex_to_list(regex, "tables-exclude", &regex_exclude_list);
3634 3635 3636 3637 3638 3639 3640 3641 3642 3643 3644 3645 3646 3647 3648 3649 3650 3651 3652 3653 3654 3655 3656 3657 3658 3659 3660 3661 3662 3663 3664 3665 3666 3667 3668 3669 3670 3671 3672 3673 3674
}

typedef void (*insert_entry_func_t)(const char*);

/***********************************************************************
Scan string and load filter entries from it.  */
static
void
xb_load_list_string(
/*================*/
	char* list,			/*!< in: string representing a list */
	const char* delimiters,		/*!< in: delimiters of entries */
	insert_entry_func_t ins)	/*!< in: callback to add entry */
{
	char*	p;
	char*	saveptr;

	p = strtok_r(list, delimiters, &saveptr);
	while (p) {

		ins(p);

		p = strtok_r(NULL, delimiters, &saveptr);
	}
}

/***********************************************************************
Scan file and load filter entries from it.  */
static
void
xb_load_list_file(
/*==============*/
	const char* filename,		/*!< in: name of file */
	insert_entry_func_t ins)	/*!< in: callback to add entry */
{
	char	name_buf[NAME_LEN*2+2];
	FILE*	fp;

	/* read and store the filenames */
	fp = fopen(filename, "r");
	if (!fp) {
3675
		die("Can't open %s",
3676 3677 3678 3679 3680 3681 3682
		    filename);
	}
	while (fgets(name_buf, sizeof(name_buf), fp) != NULL) {
		char*	p = strchr(name_buf, '\n');
		if (p) {
			*p = '\0';
		} else {
3683
			die("`%s...` name is too long", name_buf);
3684 3685 3686 3687 3688 3689 3690 3691 3692 3693 3694 3695 3696 3697 3698
		}

		ins(name_buf);
	}

	fclose(fp);
}


static
void
xb_filters_init()
{
	if (xtrabackup_databases) {
		xb_load_list_string(xtrabackup_databases, " \t",
Vladislav Vaintroub's avatar
Vladislav Vaintroub committed
3699
				    xb_register_include_filter_entry);
3700 3701 3702 3703
	}

	if (xtrabackup_databases_file) {
		xb_load_list_file(xtrabackup_databases_file,
Vladislav Vaintroub's avatar
Vladislav Vaintroub committed
3704 3705 3706 3707 3708 3709
				  xb_register_include_filter_entry);
	}

	if (xtrabackup_databases_exclude) {
		xb_load_list_string(xtrabackup_databases_exclude, " \t",
				    xb_register_exclude_filter_entry);
3710 3711 3712 3713
	}

	if (xtrabackup_tables) {
		xb_load_list_string(xtrabackup_tables, ",",
Vladislav Vaintroub's avatar
Vladislav Vaintroub committed
3714
				    xb_register_include_regex);
3715 3716 3717 3718 3719
	}

	if (xtrabackup_tables_file) {
		xb_load_list_file(xtrabackup_tables_file, xb_register_table);
	}
Vladislav Vaintroub's avatar
Vladislav Vaintroub committed
3720 3721 3722 3723 3724

	if (xtrabackup_tables_exclude) {
		xb_load_list_string(xtrabackup_tables_exclude, ",",
				    xb_register_exclude_regex);
	}
3725 3726 3727 3728 3729 3730 3731 3732 3733 3734 3735 3736 3737 3738 3739 3740 3741 3742 3743 3744 3745 3746 3747
}

static
void
xb_filter_hash_free(hash_table_t* hash)
{
	ulint	i;

	/* free the hash elements */
	for (i = 0; i < hash_get_n_cells(hash); i++) {
		xb_filter_entry_t*	table;

		table = static_cast<xb_filter_entry_t *>
			(HASH_GET_FIRST(hash, i));

		while (table) {
			xb_filter_entry_t*	prev_table = table;

			table = static_cast<xb_filter_entry_t *>
				(HASH_GET_NEXT(name_hash, prev_table));

			HASH_DELETE(xb_filter_entry_t, name_hash, hash,
				ut_fold_string(prev_table->name), prev_table);
3748
			free(prev_table);
3749 3750 3751 3752 3753 3754 3755
		}
	}

	/* free hash */
	hash_table_free(hash);
}

Vladislav Vaintroub's avatar
Vladislav Vaintroub committed
3756 3757 3758 3759 3760 3761 3762 3763
static void xb_regex_list_free(regex_list_t* list)
{
	while (list->size() > 0) {
		xb_regfree(&list->front());
		list->pop_front();
	}
}

3764 3765 3766 3767 3768 3769
/************************************************************************
Destroy table filters for partial backup. */
static
void
xb_filters_free()
{
Vladislav Vaintroub's avatar
Vladislav Vaintroub committed
3770 3771 3772 3773 3774
	xb_regex_list_free(&regex_include_list);
	xb_regex_list_free(&regex_exclude_list);

	if (tables_include_hash) {
		xb_filter_hash_free(tables_include_hash);
3775 3776
	}

Vladislav Vaintroub's avatar
Vladislav Vaintroub committed
3777 3778
	if (tables_exclude_hash) {
		xb_filter_hash_free(tables_exclude_hash);
3779 3780
	}

Vladislav Vaintroub's avatar
Vladislav Vaintroub committed
3781 3782 3783 3784 3785 3786
	if (databases_include_hash) {
		xb_filter_hash_free(databases_include_hash);
	}

	if (databases_exclude_hash) {
		xb_filter_hash_free(databases_exclude_hash);
3787 3788 3789 3790
	}
}

/*********************************************************************//**
3791
Create log file metadata. */
3792
static
3793
void
3794 3795
open_or_create_log_file(
/*====================*/
3796
	fil_space_t* space,
3797 3798 3799 3800 3801
	ulint	i)			/*!< in: log file number in group */
{
	char	name[10000];
	ulint	dirnamelen;

3802
	os_normalize_path(srv_log_group_home_dir);
3803 3804 3805 3806 3807 3808

	dirnamelen = strlen(srv_log_group_home_dir);
	ut_a(dirnamelen < (sizeof name) - 10 - sizeof "ib_logfile");
	memcpy(name, srv_log_group_home_dir, dirnamelen);

	/* Add a path separator if needed. */
3809 3810
	if (dirnamelen && name[dirnamelen - 1] != OS_PATH_SEPARATOR) {
		name[dirnamelen++] = OS_PATH_SEPARATOR;
3811 3812
	}

3813
	sprintf(name + dirnamelen, "%s%zu", "ib_logfile", i);
3814 3815 3816

	ut_a(fil_validate());

3817 3818 3819
	space->add(name, OS_FILE_CLOSED,
		   ulint(srv_log_file_size >> srv_page_size_shift),
		   false, false);
3820 3821 3822 3823 3824 3825 3826 3827 3828 3829 3830 3831 3832 3833 3834 3835 3836 3837 3838 3839 3840 3841 3842 3843 3844 3845 3846 3847 3848 3849 3850 3851 3852 3853 3854 3855 3856 3857 3858 3859 3860 3861 3862 3863 3864 3865 3866 3867 3868 3869 3870 3871 3872 3873 3874 3875 3876 3877 3878
}

/***********************************************************************
Set the open files limit. Based on set_max_open_files().

@return the resulting open files limit. May be less or more than the requested
value.  */
static uint
xb_set_max_open_files(
/*==================*/
	uint max_file_limit)	/*!<in: open files limit */
{
#if defined(RLIMIT_NOFILE)
	struct rlimit rlimit;
	uint old_cur;

	if (getrlimit(RLIMIT_NOFILE, &rlimit)) {

		goto end;
	}

	old_cur = (uint) rlimit.rlim_cur;

	if (rlimit.rlim_cur == RLIM_INFINITY) {

		rlimit.rlim_cur = max_file_limit;
	}

	if (rlimit.rlim_cur >= max_file_limit) {

		max_file_limit = rlimit.rlim_cur;
		goto end;
	}

	rlimit.rlim_cur = rlimit.rlim_max = max_file_limit;

	if (setrlimit(RLIMIT_NOFILE, &rlimit)) {

		max_file_limit = old_cur;	/* Use original value */
	} else {

		rlimit.rlim_cur = 0;	/* Safety if next call fails */

		(void) getrlimit(RLIMIT_NOFILE, &rlimit);

		if (rlimit.rlim_cur) {

			/* If call didn't fail */
			max_file_limit = (uint) rlimit.rlim_cur;
		}
	}

end:
	return(max_file_limit);
#else
	return(0);
#endif
}

3879 3880
static void stop_backup_threads()
{
3881
	if (log_copying_stop && log_copying_running) {
3882
		os_event_set(log_copying_stop);
3883 3884
		fputs("mariabackup: Stopping log copying thread", stderr);
		fflush(stderr);
3885
		while (log_copying_running) {
3886 3887
			putc('.', stderr);
			fflush(stderr);
3888 3889
			os_thread_sleep(200000); /*0.2 sec*/
		}
3890
		putc('\n', stderr);
3891 3892 3893 3894 3895 3896 3897 3898 3899 3900 3901 3902
		os_event_destroy(log_copying_stop);
	}

	if (wait_throttle) {
		/* wait for io_watching_thread completion */
		while (io_watching_thread_running) {
			os_thread_sleep(1000000);
		}
		os_event_destroy(wait_throttle);
	}
}

3903 3904
/** Implement the core of --backup
@return	whether the operation succeeded */
3905
static bool xtrabackup_backup_low()
3906
{
3907 3908
	ut_ad(!metadata_to_lsn);

3909 3910 3911 3912 3913 3914 3915
	/* read the latest checkpoint lsn */
	{
		ulint	max_cp_field;

		log_mutex_enter();

		if (recv_find_max_checkpoint(&max_cp_field) == DB_SUCCESS
3916
		    && log_sys.log.format != 0) {
3917
			if (max_cp_field == LOG_CHECKPOINT_1) {
Marko Mäkelä's avatar
Marko Mäkelä committed
3918
				log_header_read(max_cp_field);
3919
			}
3920
			metadata_to_lsn = mach_read_from_8(
3921
				log_sys.checkpoint_buf + LOG_CHECKPOINT_LSN);
Marko Mäkelä's avatar
Marko Mäkelä committed
3922
			msg("mariabackup: The latest check point"
3923
			    " (for incremental): '" LSN_PF "'",
3924 3925
			    metadata_to_lsn);
		} else {
3926
			msg("Error: recv_find_max_checkpoint() failed.");
3927 3928 3929 3930 3931 3932
		}
		log_mutex_exit();
	}

	stop_backup_threads();

3933 3934
	if (metadata_to_lsn && xtrabackup_copy_logfile(true)) {
		ds_close(dst_log_file);
3935
		dst_log_file = NULL;
3936 3937 3938
		return false;
	}

3939
	if (ds_close(dst_log_file) || !metadata_to_lsn) {
3940 3941 3942 3943 3944 3945 3946 3947 3948 3949 3950 3951 3952 3953 3954 3955
		dst_log_file = NULL;
		return false;
	}

	dst_log_file = NULL;

	if(!xtrabackup_incremental) {
		strcpy(metadata_type, "full-backuped");
		metadata_from_lsn = 0;
	} else {
		strcpy(metadata_type, "incremental");
		metadata_from_lsn = incremental_lsn;
	}
	metadata_last_lsn = log_copy_scanned_lsn;

	if (!xtrabackup_stream_metadata(ds_meta)) {
3956
		msg("Error: failed to stream metadata.");
3957 3958 3959 3960 3961 3962 3963 3964
		return false;
	}
	if (xtrabackup_extra_lsndir) {
		char	filename[FN_REFLEN];

		sprintf(filename, "%s/%s", xtrabackup_extra_lsndir,
			XTRABACKUP_METADATA_FILENAME);
		if (!xtrabackup_write_metadata(filename)) {
3965 3966
			msg("Error: failed to write metadata "
			    "to '%s'.", filename);
3967 3968
			return false;
		}
3969 3970 3971
		sprintf(filename, "%s/%s", xtrabackup_extra_lsndir,
			XTRABACKUP_INFO);
		if (!write_xtrabackup_info(mysql_connection, filename, false)) {
3972 3973
			msg("Error: failed to write info "
			 "to '%s'.", filename);
3974 3975
			return false;
		}
3976 3977 3978 3979 3980
	}

	return true;
}

3981 3982 3983 3984 3985
/** Implement --backup
@return	whether the operation succeeded */
static
bool
xtrabackup_backup_func()
3986 3987 3988 3989
{
	MY_STAT			 stat_info;
	uint			 i;
	uint			 count;
3990
	pthread_mutex_t		 count_mutex;
3991
	data_thread_ctxt_t 	*data_threads;
3992 3993
	pthread_mutex_init(&backup_mutex, NULL);
	pthread_cond_init(&scanned_lsn_cond, NULL);
3994 3995

#ifdef USE_POSIX_FADVISE
3996
	msg("uses posix_fadvise().");
3997 3998 3999 4000 4001 4002
#endif

	/* cd to datadir */

	if (my_setwd(mysql_real_data_home,MYF(MY_WME)))
	{
4003
		msg("my_setwd() failed , %s", mysql_real_data_home);
4004
		return(false);
4005
	}
4006
	msg("cd to %s", mysql_real_data_home);
4007
	encryption_plugin_backup_init(mysql_connection);
4008
	msg("open files limit requested %u, set to %u",
4009 4010 4011 4012 4013 4014 4015
	    (uint) xb_open_files_limit,
	    xb_set_max_open_files(xb_open_files_limit));

	mysql_data_home= mysql_data_home_buff;
	mysql_data_home[0]=FN_CURLIB;		// all paths are relative from here
	mysql_data_home[1]=0;

Marko Mäkelä's avatar
Marko Mäkelä committed
4016
	srv_n_purge_threads = 1;
4017 4018
	srv_read_only_mode = TRUE;

4019
	srv_operation = SRV_OPERATION_BACKUP;
4020
	log_file_op = backup_file_op;
4021
	metadata_to_lsn = 0;
4022 4023

	/* initialize components */
4024 4025
        if(innodb_init_param()) {
fail:
4026
		metadata_to_lsn = log_copying_running;
4027
		stop_backup_threads();
4028
		log_file_op = NULL;
4029 4030 4031 4032
		if (dst_log_file) {
			ds_close(dst_log_file);
			dst_log_file = NULL;
		}
4033
		if (fil_system.is_initialised()) {
4034 4035
			innodb_shutdown();
		}
4036 4037
		return(false);
	}
4038 4039 4040 4041 4042 4043 4044 4045 4046 4047 4048 4049 4050 4051 4052 4053 4054 4055

	if (srv_buf_pool_size >= 1000 * 1024 * 1024) {
                                  /* Here we still have srv_pool_size counted
                                  in kilobytes (in 4.0 this was in bytes)
				  srv_boot() converts the value to
                                  pages; if buffer pool is less than 1000 MB,
                                  assume fewer threads. */
                srv_max_n_threads = 50000;

	} else if (srv_buf_pool_size >= 8 * 1024 * 1024) {

                srv_max_n_threads = 10000;
        } else {
		srv_max_n_threads = 1000;       /* saves several MB of memory,
                                                especially in 64-bit
                                                computers */
        }

4056 4057 4058 4059 4060 4061
	sync_check_init();
	ut_d(sync_check_enable());
	/* Reset the system variables in the recovery module. */
	recv_sys_var_init();
	trx_pool_init();

4062
	ut_crc32_init();
Vladislav Vaintroub's avatar
Vladislav Vaintroub committed
4063
	crc_init();
4064
	recv_sys_init();
4065

4066
#ifdef WITH_INNODB_DISALLOW_WRITES
4067
	srv_allow_writes_event = os_event_create(0);
4068 4069 4070
	os_event_set(srv_allow_writes_event);
#endif

4071 4072 4073
	xb_filters_init();

	xb_fil_io_init();
4074 4075 4076 4077
	srv_n_file_io_threads = srv_n_read_io_threads;

	os_aio_init(srv_n_read_io_threads, srv_n_write_io_threads,
		    SRV_MAX_N_PENDING_SYNC_IOS);
4078

4079
	log_sys.create();
4080
	log_sys.log.create(srv_n_log_files);
4081 4082 4083
	fil_space_t*	space = fil_space_create(
		"innodb_redo_log", SRV_LOG_SPACE_FIRST_ID, 0,
		FIL_TYPE_LOG, NULL);
4084

4085 4086
	for (ulint i = 0; i < srv_n_log_files; i++) {
		open_or_create_log_file(space, i);
4087 4088 4089 4090 4091 4092
	}

	/* create extra LSN dir if it does not exist. */
	if (xtrabackup_extra_lsndir
		&&!my_stat(xtrabackup_extra_lsndir,&stat_info,MYF(0))
		&& (my_mkdir(xtrabackup_extra_lsndir,0777,MYF(0)) < 0)) {
4093
		msg("Error: cannot mkdir %d: %s\n",
4094
		    my_errno, xtrabackup_extra_lsndir);
4095
		goto fail;
4096 4097 4098
	}

	/* create target dir if not exist */
4099
	if (!xtrabackup_stream_str && !my_stat(xtrabackup_target_dir,&stat_info,MYF(0))
4100
		&& (my_mkdir(xtrabackup_target_dir,0777,MYF(0)) < 0)){
4101
		msg("Error: cannot mkdir %d: %s\n",
4102
		    my_errno, xtrabackup_target_dir);
4103
		goto fail;
4104 4105 4106 4107 4108 4109 4110 4111 4112 4113 4114 4115
	}

        {
	/* definition from recv_recovery_from_checkpoint_start() */
	ulint		max_cp_field;

	/* start back ground thread to copy newer log */
	os_thread_id_t log_copying_thread_id;

	/* get current checkpoint_lsn */
	/* Look for the latest checkpoint from any of the log groups */

4116
	log_mutex_enter();
4117

4118
	dberr_t err = recv_find_max_checkpoint(&max_cp_field);
4119 4120

	if (err != DB_SUCCESS) {
4121 4122 4123
log_fail:
		log_mutex_exit();
		goto fail;
4124
	}
4125

4126
	if (log_sys.log.format == 0) {
4127
old_format:
4128 4129
		msg("Error: cannot process redo log"
		    " before MariaDB 10.2.2");
4130 4131
		log_mutex_exit();
		goto log_fail;
4132 4133
	}

4134
	const byte* buf = log_sys.checkpoint_buf;
4135 4136

reread_log_header:
4137 4138
	checkpoint_lsn_start = log_sys.log.lsn;
	checkpoint_no_start = log_sys.next_checkpoint_no;
4139

4140
	err = recv_find_max_checkpoint(&max_cp_field);
4141

4142
	if (err != DB_SUCCESS) {
4143
		goto log_fail;
4144
	}
4145

4146
	if (log_sys.log.format == 0) {
4147 4148
		goto old_format;
	}
4149

4150
	log_header_read(max_cp_field);
4151

4152
	if (checkpoint_no_start != mach_read_from_8(buf + LOG_CHECKPOINT_NO)) {
4153 4154 4155
		goto reread_log_header;
	}

4156
	log_mutex_exit();
4157 4158 4159 4160

	xtrabackup_init_datasinks();

	if (!select_history()) {
4161
		goto fail;
4162 4163 4164 4165
	}

	/* open the log file */
	memset(&stat_info, 0, sizeof(MY_STAT));
4166
	dst_log_file = ds_open(ds_redo, "ib_logfile0", &stat_info);
4167
	if (dst_log_file == NULL) {
4168 4169
		msg("§rror: failed to open the target stream for "
		    "'ib_logfile0'.");
4170
		goto fail;
4171 4172 4173
	}

	/* label it */
4174 4175
	byte MY_ALIGNED(OS_FILE_LOG_BLOCK_SIZE) log_hdr[OS_FILE_LOG_BLOCK_SIZE];
	memset(log_hdr, 0, sizeof log_hdr);
4176
	mach_write_to_4(LOG_HEADER_FORMAT + log_hdr, log_sys.log.format);
4177
	mach_write_to_4(LOG_HEADER_SUBFORMAT + log_hdr, log_sys.log.subformat);
4178 4179 4180 4181 4182 4183 4184 4185 4186
	mach_write_to_8(LOG_HEADER_START_LSN + log_hdr, checkpoint_lsn_start);
	strcpy(reinterpret_cast<char*>(LOG_HEADER_CREATOR + log_hdr),
	       "Backup " MYSQL_SERVER_VERSION);
	log_block_set_checksum(log_hdr,
			       log_block_calc_checksum_crc32(log_hdr));

	/* Write the log header. */
	if (ds_write(dst_log_file, log_hdr, sizeof log_hdr)) {
	log_write_fail:
4187
		msg("error: write to logfile failed");
4188 4189 4190 4191 4192 4193 4194 4195 4196 4197 4198 4199 4200 4201 4202 4203
		goto fail;
	}
	/* Adjust the checkpoint page. */
	memcpy(log_hdr, buf, OS_FILE_LOG_BLOCK_SIZE);
	mach_write_to_8(log_hdr + LOG_CHECKPOINT_OFFSET,
			(checkpoint_lsn_start & (OS_FILE_LOG_BLOCK_SIZE - 1))
			| LOG_FILE_HDR_SIZE);
	log_block_set_checksum(log_hdr,
			       log_block_calc_checksum_crc32(log_hdr));
	/* Write checkpoint page 1 and two empty log pages before the
	payload. */
	if (ds_write(dst_log_file, log_hdr, OS_FILE_LOG_BLOCK_SIZE)
	    || !memset(log_hdr, 0, sizeof log_hdr)
	    || ds_write(dst_log_file, log_hdr, sizeof log_hdr)
	    || ds_write(dst_log_file, log_hdr, sizeof log_hdr)) {
		goto log_write_fail;
4204 4205
	}

4206
	log_copying_running = true;
4207 4208 4209 4210 4211
	/* start io throttle */
	if(xtrabackup_throttle) {
		os_thread_id_t io_watching_thread_id;

		io_ticket = xtrabackup_throttle;
4212 4213
		wait_throttle = os_event_create(0);
		io_watching_thread_running = true;
4214 4215 4216 4217 4218

		os_thread_create(io_watching_thread, NULL,
				 &io_watching_thread_id);
	}

4219 4220 4221
	/* Populate fil_system with tablespaces to copy */
	err = xb_load_tablespaces();
	if (err != DB_SUCCESS) {
4222 4223
		msg("merror: xb_load_tablespaces() failed with"
		    " error %s.", ut_strerr(err));
4224 4225
fail_before_log_copying_thread_start:
		log_copying_running = false;
4226 4227 4228
		goto fail;
	}

4229
	/* copy log file by current position */
4230
	log_copy_scanned_lsn = checkpoint_lsn_start;
4231
	recv_sys->recovered_lsn = log_copy_scanned_lsn;
4232 4233
	log_optimized_ddl_op = backup_optimized_ddl_op;
	log_truncate = backup_truncate_fail;
4234

4235 4236
	if (xtrabackup_copy_logfile())
		goto fail_before_log_copying_thread_start;
4237

4238
	log_copying_stop = os_event_create(0);
4239 4240 4241 4242
	os_thread_create(log_copying_thread, NULL, &log_copying_thread_id);

	/* FLUSH CHANGED_PAGE_BITMAPS call */
	if (!flush_changed_page_bitmaps()) {
4243
		goto fail;
4244 4245 4246 4247 4248 4249 4250
	}
	debug_sync_point("xtrabackup_suspend_at_start");


	ut_a(xtrabackup_parallel > 0);

	if (xtrabackup_parallel > 1) {
4251
		msg("mariabackup: Starting %u threads for parallel data "
4252
		    "files transfer", xtrabackup_parallel);
4253 4254
	}

4255 4256
	if (opt_lock_ddl_per_table) {
		mdl_lock_all();
4257 4258 4259 4260 4261

		DBUG_EXECUTE_IF("check_mdl_lock_works",
			dbug_alter_thread_done =
			dbug_start_query_thread("ALTER TABLE test.t ADD COLUMN mdl_lock_column int",
				"Waiting for table metadata lock", 1, ER_QUERY_INTERRUPTED););
4262 4263
	}

4264
	datafiles_iter_t *it = datafiles_iter_new();
4265
	if (it == NULL) {
4266
		msg("mariabackup: Error: datafiles_iter_new() failed.");
4267
		goto fail;
4268 4269 4270 4271
	}

	/* Create data copying threads */
	data_threads = (data_thread_ctxt_t *)
4272
		malloc(sizeof(data_thread_ctxt_t) * xtrabackup_parallel);
4273
	count = xtrabackup_parallel;
4274
	pthread_mutex_init(&count_mutex, NULL);
4275 4276 4277 4278 4279

	for (i = 0; i < (uint) xtrabackup_parallel; i++) {
		data_threads[i].it = it;
		data_threads[i].num = i+1;
		data_threads[i].count = &count;
4280
		data_threads[i].count_mutex = &count_mutex;
4281 4282 4283 4284 4285 4286 4287
		os_thread_create(data_copy_thread_func, data_threads + i,
				 &data_threads[i].id);
	}

	/* Wait for threads to exit */
	while (1) {
		os_thread_sleep(1000000);
4288 4289 4290 4291
		pthread_mutex_lock(&count_mutex);
		bool stop = count == 0;
		pthread_mutex_unlock(&count_mutex);
		if (stop) {
4292 4293 4294 4295
			break;
		}
	}

4296 4297
	pthread_mutex_destroy(&count_mutex);
	free(data_threads);
4298 4299 4300
	datafiles_iter_free(it);
	}

4301
	bool ok = backup_start();
4302

4303 4304
	if (ok) {
		ok = xtrabackup_backup_low();
4305

4306
		backup_release();
4307

4308 4309 4310 4311 4312
		DBUG_EXECUTE_IF("check_mdl_lock_works",
			os_event_wait(dbug_alter_thread_done);
			os_event_destroy(dbug_alter_thread_done);
		);

4313 4314
		if (ok) {
			backup_finish();
4315 4316 4317
		}
	}

4318
	if (!ok) {
4319
		goto fail;
4320 4321
	}

4322 4323 4324
	if (changed_page_bitmap) {
		xb_page_bitmap_deinit(changed_page_bitmap);
	}
4325 4326
	xtrabackup_destroy_datasinks();

4327 4328
	msg("Redo log (from LSN " LSN_PF " to " LSN_PF
	    ") was copied.", checkpoint_lsn_start, log_copy_scanned_lsn);
4329 4330 4331 4332
	xb_filters_free();

	xb_data_files_close();

4333
	/* Make sure that the latest checkpoint was included */
4334
	if (metadata_to_lsn > log_copy_scanned_lsn) {
4335 4336
		msg("Error: failed to copy enough redo log ("
		    "LSN=" LSN_PF "; checkpoint LSN=" LSN_PF ").",
4337
		    log_copy_scanned_lsn, metadata_to_lsn);
4338
		goto fail;
4339
	}
4340 4341

	innodb_shutdown();
4342 4343 4344
	log_file_op = NULL;
	pthread_mutex_destroy(&backup_mutex);
	pthread_cond_destroy(&scanned_lsn_cond);
4345
	return(true);
4346 4347
}

4348 4349 4350 4351 4352 4353 4354 4355 4356 4357 4358 4359 4360 4361 4362 4363 4364 4365 4366 4367 4368 4369 4370 4371 4372

/**
This function handles DDL changes at the end of backup, under protection of
FTWRL.  This ensures consistent backup in presence of DDL.

- New tables, that were created during backup, are now copied into backup.
  Also, tablespaces with optimized (no redo loggin DDL) are re-copied into 
  backup. This tablespaces will get the extension ".new" in the backup

- Tables that were renamed during backup, are marked as renamed
  For these, file <old_name>.ren will be created.
  The content of the file is the new tablespace name.

- Tables that were deleted during backup, are marked as deleted
  For these , an empty file <name>.del will be created

  It is the responsibility of the prepare phase to deal with .new, .ren, and .del
  files.
*/
void backup_fix_ddl(void)
{
	std::set<std::string> new_tables;
	std::set<std::string> dropped_tables;
	std::map<std::string, std::string> renamed_tables;

4373 4374 4375 4376 4377 4378 4379 4380
	/* Disable further DDL on backed up tables (only needed for --no-lock).*/
	pthread_mutex_lock(&backup_mutex);
	log_file_op = backup_file_op_fail;
	log_optimized_ddl_op = backup_optimized_ddl_op_fail;
	pthread_mutex_unlock(&backup_mutex);

	DBUG_MARIABACKUP_EVENT("backup_fix_ddl",0);

4381 4382 4383 4384 4385 4386 4387 4388 4389 4390 4391 4392 4393 4394 4395 4396 4397 4398 4399 4400 4401 4402 4403 4404 4405 4406 4407 4408 4409 4410 4411 4412 4413 4414 4415 4416 4417 4418 4419 4420 4421 4422 4423 4424 4425 4426 4427 4428 4429 4430 4431 4432 4433 4434 4435 4436 4437 4438 4439 4440 4441 4442 4443 4444 4445 4446 4447 4448 4449 4450 4451 4452 4453 4454 4455 4456 4457 4458 4459
	for (space_id_to_name_t::iterator iter = ddl_tracker.tables_in_backup.begin();
		iter != ddl_tracker.tables_in_backup.end();
		iter++) {

		const std::string name = iter->second;
		ulint id = iter->first;

		if (ddl_tracker.drops.find(id) != ddl_tracker.drops.end()) {
			dropped_tables.insert(name);
			continue;
		}

		bool has_optimized_ddl =
			ddl_tracker.optimized_ddl.find(id) != ddl_tracker.optimized_ddl.end();

		if (ddl_tracker.id_to_name.find(id) == ddl_tracker.id_to_name.end()) {
			if (has_optimized_ddl) {
				new_tables.insert(name);
			}
			continue;
		}

		/* tablespace was affected by DDL. */
		const std::string new_name = ddl_tracker.id_to_name[id];
		if (new_name != name) {
			if (has_optimized_ddl) {
				/* table was renamed, but we need a full copy
				of it because of optimized DDL. We emulate a drop/create.*/
				dropped_tables.insert(name);
				new_tables.insert(new_name);
			} else {
				/* Renamed, and no optimized DDL*/
				renamed_tables[name] = new_name;
			}
		} else if (has_optimized_ddl) {
			/* Table was recreated, or optimized DDL ran.
			In both cases we need a full copy in the backup.*/
			new_tables.insert(name);
		}
	}

	/* Find tables that were created during backup (and not removed).*/
	for(space_id_to_name_t::iterator iter = ddl_tracker.id_to_name.begin();
		iter != ddl_tracker.id_to_name.end();
		iter++) {

		ulint id = iter->first;
		std::string name = iter->second;

		if (ddl_tracker.tables_in_backup.find(id) != ddl_tracker.tables_in_backup.end()) {
			/* already processed above */
			continue;
		}

		if (ddl_tracker.drops.find(id) == ddl_tracker.drops.end()) {
			dropped_tables.erase(name);
			new_tables.insert(name);
		}
	}

	// Mark tablespaces for rename
	for (std::map<std::string, std::string>::iterator iter = renamed_tables.begin();
		iter != renamed_tables.end(); ++iter) {
		const std::string old_name = iter->first;
		std::string new_name = iter->second;
		backup_file_printf((old_name + ".ren").c_str(), "%s", new_name.c_str());
	}

	// Mark tablespaces for drop
	for (std::set<std::string>::iterator iter = dropped_tables.begin();
		iter != dropped_tables.end();
		iter++) {
		const std::string name(*iter);
		backup_file_printf((name + ".del").c_str(), "%s", "");
	}

	//  Load and copy new tables.
	//  Close all datanodes first, reload only new tables.
	std::vector<fil_node_t *> all_nodes;
Marko Mäkelä's avatar
Marko Mäkelä committed
4460
	datafiles_iter_t *it = datafiles_iter_new();
4461 4462 4463 4464 4465 4466 4467 4468 4469
	if (!it)
		return;
	while (fil_node_t *node = datafiles_iter_next(it)) {
		all_nodes.push_back(node);
	}
	for (size_t i = 0; i < all_nodes.size(); i++) {
		fil_node_t *n = all_nodes[i];
		if (n->space->id == 0)
			continue;
Marko Mäkelä's avatar
Marko Mäkelä committed
4470 4471 4472 4473 4474
		if (n->is_open()) {
			mutex_enter(&fil_system.mutex);
			n->close();
			mutex_exit(&fil_system.mutex);
		}
4475 4476
		fil_space_free(n->space->id, false);
	}
4477
	datafiles_iter_free(it);
4478 4479 4480 4481 4482 4483 4484 4485 4486 4487 4488

	for (std::set<std::string>::iterator iter = new_tables.begin();
		iter != new_tables.end(); iter++) {
		const char *space_name = iter->c_str();
		if (check_if_skip_table(space_name))
			continue;
		std::string name(*iter);
		bool is_remote = access((name + ".ibd").c_str(), R_OK) != 0;
		const char *extension = is_remote ? ".isl" : ".ibd";
		name.append(extension);
		char buf[FN_REFLEN];
4489 4490
		strncpy(buf, name.c_str(), sizeof buf - 1);
		buf[sizeof buf - 1] = '\0';
4491 4492 4493
		const char *dbname = buf;
		char *p = strchr(buf, '/');
		if (p == 0) {
4494
			msg("Unexpected tablespace %s filename %s", space_name, name.c_str());
4495 4496 4497 4498 4499 4500 4501 4502
			ut_a(0);
		}
		ut_a(p);
		*p = 0;
		const char *tablename = p + 1;
		xb_load_single_table_tablespace(dbname, tablename, is_remote);
	}

Marko Mäkelä's avatar
Marko Mäkelä committed
4503
	it = datafiles_iter_new();
4504 4505 4506 4507 4508 4509 4510 4511 4512 4513 4514
	if (!it)
		return;

	while (fil_node_t *node = datafiles_iter_next(it)) {
		fil_space_t * space = node->space;
		if (!fil_is_user_tablespace_id(space->id))
			continue;
		std::string dest_name(node->space->name);
		dest_name.append(".new");
		xtrabackup_copy_datafile(node, 0, dest_name.c_str()/*, do_full_copy ? ULONGLONG_MAX:UNIV_PAGE_SIZE */);
	}
4515 4516

	datafiles_iter_free(it);
4517 4518
}

4519 4520 4521 4522 4523 4524 4525 4526 4527 4528 4529 4530 4531 4532 4533 4534 4535 4536 4537 4538 4539 4540 4541 4542 4543 4544 4545 4546 4547 4548 4549
/* ================= prepare ================= */

/***********************************************************************
Generates path to the meta file path from a given path to an incremental .delta
by replacing trailing ".delta" with ".meta", or returns error if 'delta_path'
does not end with the ".delta" character sequence.
@return TRUE on success, FALSE on error. */
static
ibool
get_meta_path(
	const char	*delta_path,	/* in: path to a .delta file */
	char 		*meta_path)	/* out: path to the corresponding .meta
					file */
{
	size_t		len = strlen(delta_path);

	if (len <= 6 || strcmp(delta_path + len - 6, ".delta")) {
		return FALSE;
	}
	memcpy(meta_path, delta_path, len - 6);
	strcpy(meta_path + len - 6, XB_DELTA_INFO_SUFFIX);

	return TRUE;
}

/****************************************************************//**
Create a new tablespace on disk and return the handle to its opened
file. Code adopted from fil_create_new_single_table_tablespace with
the main difference that only disk file is created without updating
the InnoDB in-memory dictionary data structures.

4550
@return true on success, false on error.  */
4551
static
4552
bool
4553 4554 4555 4556
xb_space_create_file(
/*==================*/
	const char*	path,		/*!<in: path to tablespace */
	ulint		space_id,	/*!<in: space id */
4557
	ulint		flags,		/*!<in: tablespace flags */
Marko Mäkelä's avatar
Marko Mäkelä committed
4558
	pfs_os_file_t*	file)		/*!<out: file handle */
4559
{
4560
	bool		ret;
4561 4562 4563
	byte*		buf;
	byte*		page;

4564 4565
	*file = os_file_create_simple_no_error_handling(
		0, path, OS_FILE_CREATE, OS_FILE_READ_WRITE, false, &ret);
4566
	if (!ret) {
4567
		msg("Can't create file %s", path);
4568 4569 4570 4571
		return ret;
	}

	ret = os_file_set_size(path, *file,
4572 4573
			       FIL_IBD_FILE_INITIAL_SIZE
			       << srv_page_size_shift);
4574
	if (!ret) {
4575
		msg("mariabackup: cannot set size for file %s", path);
4576 4577 4578 4579 4580
		os_file_close(*file);
		os_file_delete(0, path);
		return ret;
	}

4581
	buf = static_cast<byte *>(malloc(3U << srv_page_size_shift));
4582
	/* Align the memory for file i/o if we might have O_DIRECT set */
4583
	page = static_cast<byte *>(ut_align(buf, srv_page_size));
4584

4585
	memset(page, '\0', srv_page_size);
4586 4587 4588 4589

	fsp_header_init_fields(page, space_id, flags);
	mach_write_to_4(page + FIL_PAGE_ARCH_LOG_NO_OR_SPACE_ID, space_id);

4590
	const page_size_t page_size(flags);
4591

4592
	if (!page_size.is_compressed()) {
4593
		buf_flush_init_for_writing(NULL, page, NULL, 0);
4594

4595
		ret = os_file_write(IORequestWrite, path, *file, page, 0,
4596
				    srv_page_size);
4597 4598 4599
	} else {
		page_zip_des_t	page_zip;
		ulint zip_size = page_size.physical();
4600
		page_zip_set_size(&page_zip, zip_size);
4601
		page_zip.data = page + srv_page_size;
4602
		fprintf(stderr, "zip_size = " ULINTPF "\n", zip_size);
4603 4604 4605 4606 4607 4608 4609

#ifdef UNIV_DEBUG
		page_zip.m_start =
#endif /* UNIV_DEBUG */
			page_zip.m_end = page_zip.m_nonempty =
			page_zip.n_blobs = 0;

4610
		buf_flush_init_for_writing(NULL, page, &page_zip, 0);
4611

4612 4613
		ret = os_file_write(IORequestWrite, path, *file,
				    page_zip.data, 0, zip_size);
4614 4615
	}

4616
	free(buf);
4617 4618

	if (!ret) {
4619
		msg("mariabackup: could not write the first page to %s",
4620 4621 4622 4623 4624 4625 4626 4627 4628
		    path);
		os_file_close(*file);
		os_file_delete(0, path);
		return ret;
	}

	return TRUE;
}

4629 4630 4631 4632 4633 4634 4635 4636 4637 4638
static fil_space_t* fil_space_get_by_name(const char* name)
{
	ut_ad(mutex_own(&fil_system.mutex));
	for (fil_space_t* space = UT_LIST_GET_FIRST(fil_system.space_list);
	     space != NULL;
	     space = UT_LIST_GET_NEXT(space_list, space))
		if (!strcmp(space->name, name)) return space;
	return NULL;
}

4639 4640 4641 4642 4643 4644 4645 4646
/***********************************************************************
Searches for matching tablespace file for given .delta file and space_id
in given directory. When matching tablespace found, renames it to match the
name of .delta file. If there was a tablespace with matching name and
mismatching ID, renames it to xtrabackup_tmp_#ID.ibd. If there was no
matching file, creates a new tablespace.
@return file handle of matched or created file */
static
Marko Mäkelä's avatar
Marko Mäkelä committed
4647
pfs_os_file_t
4648 4649 4650
xb_delta_open_matching_space(
	const char*	dbname,		/* in: path to destination database dir */
	const char*	name,		/* in: name of delta file (without .delta) */
4651
	const xb_delta_info_t& info,
4652 4653
	char*		real_name,	/* out: full path of destination file */
	size_t		real_name_len,	/* out: buffer size for real_name */
4654
	bool* 		success)	/* out: indicates error. true = success */
4655 4656 4657 4658
{
	char			dest_dir[FN_REFLEN];
	char			dest_space_name[FN_REFLEN];
	fil_space_t*		fil_space;
Marko Mäkelä's avatar
Marko Mäkelä committed
4659
	pfs_os_file_t		file;
4660 4661 4662
	xb_filter_entry_t*	table;

	ut_a(dbname != NULL ||
4663 4664
	     !fil_is_user_tablespace_id(info.space_id) ||
	     info.space_id == ULINT_UNDEFINED);
4665

4666
	*success = false;
4667 4668 4669 4670

	if (dbname) {
		snprintf(dest_dir, FN_REFLEN, "%s/%s",
			xtrabackup_target_dir, dbname);
4671
		os_normalize_path(dest_dir);
4672 4673 4674 4675

		snprintf(dest_space_name, FN_REFLEN, "%s/%s", dbname, name);
	} else {
		snprintf(dest_dir, FN_REFLEN, "%s", xtrabackup_target_dir);
4676
		os_normalize_path(dest_dir);
4677 4678 4679 4680 4681 4682 4683

		snprintf(dest_space_name, FN_REFLEN, "%s", name);
	}

	snprintf(real_name, real_name_len,
		 "%s/%s",
		 xtrabackup_target_dir, dest_space_name);
4684
	os_normalize_path(real_name);
4685 4686 4687 4688 4689
	/* Truncate ".ibd" */
	dest_space_name[strlen(dest_space_name) - 4] = '\0';

	/* Create the database directory if it doesn't exist yet */
	if (!os_file_create_directory(dest_dir, FALSE)) {
4690
		msg("mariabackup: error: cannot create dir %s", dest_dir);
4691 4692 4693
		return file;
	}

4694 4695 4696 4697 4698 4699 4700 4701 4702 4703
	log_mutex_enter();
	if (!fil_is_user_tablespace_id(info.space_id)) {
found:
		/* open the file and return its handle */

		file = os_file_create_simple_no_error_handling(
			0, real_name,
			OS_FILE_OPEN, OS_FILE_READ_WRITE, false, success);

		if (!*success) {
Marko Mäkelä's avatar
Marko Mäkelä committed
4704
			msg("mariabackup: Cannot open file %s\n", real_name);
4705 4706 4707 4708
		}
exit:
		log_mutex_exit();
		return file;
4709 4710 4711 4712
	}

	/* remember space name for further reference */
	table = static_cast<xb_filter_entry_t *>
4713
		(malloc(sizeof(xb_filter_entry_t) +
4714 4715 4716 4717 4718 4719 4720
			strlen(dest_space_name) + 1));

	table->name = ((char*)table) + sizeof(xb_filter_entry_t);
	strcpy(table->name, dest_space_name);
	HASH_INSERT(xb_filter_entry_t, name_hash, inc_dir_tables_hash,
			ut_fold_string(table->name), table);

4721
	mutex_enter(&fil_system.mutex);
4722
	fil_space = fil_space_get_by_name(dest_space_name);
4723
	mutex_exit(&fil_system.mutex);
4724 4725

	if (fil_space != NULL) {
4726 4727
		if (fil_space->id == info.space_id
		    || info.space_id == ULINT_UNDEFINED) {
4728 4729 4730 4731 4732 4733
			/* we found matching space */
			goto found;
		} else {

			char	tmpname[FN_REFLEN];

4734
			snprintf(tmpname, FN_REFLEN, "%s/xtrabackup_tmp_#" ULINTPF,
4735 4736
				 dbname, fil_space->id);

4737
			msg("mariabackup: Renaming %s to %s.ibd",
4738 4739
				fil_space->name, tmpname);

4740 4741
			if (fil_space->rename(tmpname, NULL, false)
			    != DB_SUCCESS) {
4742
				msg("mariabackup: Cannot rename %s to %s",
4743 4744 4745 4746 4747 4748
					fil_space->name, tmpname);
				goto exit;
			}
		}
	}

4749
	if (info.space_id == ULINT_UNDEFINED)
4750
	{
4751
		die("Can't handle DDL operation on tablespace "
4752 4753
		    "%s\n", dest_space_name);
	}
4754
	mutex_enter(&fil_system.mutex);
4755
	fil_space = fil_space_get_by_id(info.space_id);
4756
	mutex_exit(&fil_system.mutex);
4757 4758 4759 4760 4761
	if (fil_space != NULL) {
		char	tmpname[FN_REFLEN];

		strncpy(tmpname, dest_space_name, FN_REFLEN);

4762
		msg("mariabackup: Renaming %s to %s",
4763 4764
		    fil_space->name, dest_space_name);

4765
		if (fil_space->rename(tmpname, NULL, false) != DB_SUCCESS)
4766
		{
4767
			msg("mariabackup: Cannot rename %s to %s",
4768 4769 4770 4771 4772 4773 4774 4775
				fil_space->name, dest_space_name);
			goto exit;
		}

		goto found;
	}

	/* No matching space found. create the new one.  */
4776 4777 4778 4779 4780 4781 4782 4783 4784 4785 4786 4787 4788 4789 4790 4791 4792 4793 4794
	const ulint flags = info.page_size.is_compressed()
		? get_bit_shift(info.page_size.physical()
				>> (UNIV_ZIP_SIZE_SHIFT_MIN - 1))
		<< FSP_FLAGS_POS_ZIP_SSIZE
		| FSP_FLAGS_MASK_POST_ANTELOPE
		| FSP_FLAGS_MASK_ATOMIC_BLOBS
		| (info.page_size.logical() == UNIV_PAGE_SIZE_ORIG
		   ? 0
		   : get_bit_shift(info.page_size.logical()
				   >> (UNIV_ZIP_SIZE_SHIFT_MIN - 1))
		   << FSP_FLAGS_POS_PAGE_SSIZE)
		: FSP_FLAGS_PAGE_SSIZE();
	ut_ad(page_size_t(flags).equals_to(info.page_size));

	if (fil_space_create(dest_space_name, info.space_id, flags,
			      FIL_TYPE_TABLESPACE, 0)) {
		*success = xb_space_create_file(real_name, info.space_id,
						flags, &file);
	} else {
4795
		msg("Can't create tablespace %s\n", dest_space_name);
4796 4797 4798 4799 4800 4801 4802 4803 4804 4805 4806 4807 4808 4809 4810 4811 4812
	}

	goto exit;
}

/************************************************************************
Applies a given .delta file to the corresponding data file.
@return TRUE on success */
static
ibool
xtrabackup_apply_delta(
	const char*	dirname,	/* in: dir name of incremental */
	const char*	dbname,		/* in: database name (ibdata: NULL) */
	const char*	filename,	/* in: file name (not a path),
					including the .delta extension */
	void*		/*data*/)
{
Marko Mäkelä's avatar
Marko Mäkelä committed
4813 4814
	pfs_os_file_t	src_file;
	pfs_os_file_t	dst_file;
4815 4816 4817 4818
	char	src_path[FN_REFLEN];
	char	dst_path[FN_REFLEN];
	char	meta_path[FN_REFLEN];
	char	space_name[FN_REFLEN];
4819
	bool	success;
4820 4821 4822 4823 4824

	ibool	last_buffer = FALSE;
	ulint	page_in_buffer;
	ulint	incremental_buffers = 0;

4825
	xb_delta_info_t info(univ_page_size, SRV_TMP_SPACE_ID);
4826 4827 4828 4829 4830 4831 4832 4833 4834 4835 4836 4837 4838 4839 4840 4841 4842 4843 4844 4845 4846 4847
	ulint		page_size;
	ulint		page_size_shift;
	byte*		incremental_buffer_base = NULL;
	byte*		incremental_buffer;

	size_t		offset;

	ut_a(xtrabackup_incremental);

	if (dbname) {
		snprintf(src_path, sizeof(src_path), "%s/%s/%s",
			 dirname, dbname, filename);
		snprintf(dst_path, sizeof(dst_path), "%s/%s/%s",
			 xtrabackup_real_target_dir, dbname, filename);
	} else {
		snprintf(src_path, sizeof(src_path), "%s/%s",
			 dirname, filename);
		snprintf(dst_path, sizeof(dst_path), "%s/%s",
			 xtrabackup_real_target_dir, filename);
	}
	dst_path[strlen(dst_path) - 6] = '\0';

4848 4849
	strncpy(space_name, filename, FN_REFLEN - 1);
	space_name[FN_REFLEN - 1] = '\0';
4850 4851 4852 4853 4854 4855
	space_name[strlen(space_name) -  6] = 0;

	if (!get_meta_path(src_path, meta_path)) {
		goto error;
	}

4856 4857 4858
	os_normalize_path(dst_path);
	os_normalize_path(src_path);
	os_normalize_path(meta_path);
4859 4860 4861 4862 4863

	if (!xb_read_delta_metadata(meta_path, &info)) {
		goto error;
	}

4864
	page_size = info.page_size.physical();
4865
	page_size_shift = get_bit_shift(page_size);
4866
	msg("page size for %s is %zu bytes",
4867 4868 4869
	    src_path, page_size);
	if (page_size_shift < 10 ||
	    page_size_shift > UNIV_PAGE_SIZE_SHIFT_MAX) {
4870 4871
		msg("error: invalid value of page_size "
		    "(%zu bytes) read from %s", page_size, meta_path);
4872 4873 4874
		goto error;
	}

4875 4876 4877
	src_file = os_file_create_simple_no_error_handling(
		0, src_path,
		OS_FILE_OPEN, OS_FILE_READ_WRITE, false, &success);
4878 4879
	if (!success) {
		os_file_get_last_error(TRUE);
4880
		msg("error: can't open %s", src_path);
4881 4882 4883 4884 4885 4886
		goto error;
	}

	posix_fadvise(src_file, 0, 0, POSIX_FADV_SEQUENTIAL);

	dst_file = xb_delta_open_matching_space(
4887
			dbname, space_name, info,
4888 4889
			dst_path, sizeof(dst_path), &success);
	if (!success) {
4890
		msg("error: can't open %s", dst_path);
4891 4892 4893 4894 4895 4896 4897
		goto error;
	}

	posix_fadvise(dst_file, 0, 0, POSIX_FADV_DONTNEED);

	/* allocate buffer for incremental backup (4096 pages) */
	incremental_buffer_base = static_cast<byte *>
4898
		(malloc((page_size / 4 + 1) * page_size));
4899 4900
	incremental_buffer = static_cast<byte *>
		(ut_align(incremental_buffer_base,
4901
			  page_size));
4902

4903
	msg("Applying %s to %s...", src_path, dst_path);
4904 4905 4906 4907 4908 4909 4910 4911

	while (!last_buffer) {
		ulint cluster_header;

		/* read to buffer */
		/* first block of block cluster */
		offset = ((incremental_buffers * (page_size / 4))
			 << page_size_shift);
4912 4913
		success = os_file_read(IORequestRead, src_file,
				       incremental_buffer, offset, page_size);
4914 4915 4916 4917 4918 4919 4920 4921 4922 4923 4924 4925
		if (!success) {
			goto error;
		}

		cluster_header = mach_read_from_4(incremental_buffer);
		switch(cluster_header) {
			case 0x78747261UL: /*"xtra"*/
				break;
			case 0x58545241UL: /*"XTRA"*/
				last_buffer = TRUE;
				break;
			default:
4926 4927
				msg("error: %s seems not "
				    ".delta file.", src_path);
4928 4929 4930
				goto error;
		}

4931 4932 4933
		/* FIXME: If the .delta modifies FSP_SIZE on page 0,
		extend the file to that size. */

4934 4935 4936 4937 4938 4939 4940 4941 4942 4943
		for (page_in_buffer = 1; page_in_buffer < page_size / 4;
		     page_in_buffer++) {
			if (mach_read_from_4(incremental_buffer + page_in_buffer * 4)
			    == 0xFFFFFFFFUL)
				break;
		}

		ut_a(last_buffer || page_in_buffer == page_size / 4);

		/* read whole of the cluster */
4944 4945 4946
		success = os_file_read(IORequestRead, src_file,
				       incremental_buffer,
				       offset, page_in_buffer * page_size);
4947 4948 4949 4950 4951 4952 4953 4954 4955 4956 4957 4958 4959 4960 4961 4962
		if (!success) {
			goto error;
		}

		posix_fadvise(src_file, offset, page_in_buffer * page_size,
			      POSIX_FADV_DONTNEED);

		for (page_in_buffer = 1; page_in_buffer < page_size / 4;
		     page_in_buffer++) {
			ulint offset_on_page;

			offset_on_page = mach_read_from_4(incremental_buffer + page_in_buffer * 4);

			if (offset_on_page == 0xFFFFFFFFUL)
				break;

4963 4964 4965 4966 4967 4968
			uchar *buf = incremental_buffer + page_in_buffer * page_size;
			const os_offset_t off = os_offset_t(offset_on_page)*page_size;

			if (off == 0) {
				/* Read tablespace size from page 0,
				and extend the file to specified size.*/
Marko Mäkelä's avatar
Marko Mäkelä committed
4969 4970 4971 4972 4973 4974 4975
				os_offset_t n_pages = mach_read_from_4(
					buf + FSP_HEADER_OFFSET + FSP_SIZE);
				if (mach_read_from_4(buf
						     + FIL_PAGE_SPACE_ID)) {
					if (!os_file_set_size(
						    dst_path, dst_file,
						    n_pages * page_size))
4976
						goto error;
Marko Mäkelä's avatar
Marko Mäkelä committed
4977
				} else if (fil_space_t* space
4978
					   = fil_system.sys_space) {
Marko Mäkelä's avatar
Marko Mäkelä committed
4979 4980 4981 4982 4983 4984 4985 4986 4987
					/* The system tablespace can
					consist of multiple files. The
					first one has full tablespace
					size in page 0, but only the last
					file should be extended. */
					fil_node_t* n = UT_LIST_GET_FIRST(
						space->chain);
					bool fail = !strcmp(n->name, dst_path)
						&& !fil_space_extend(
Vladislav Vaintroub's avatar
Vladislav Vaintroub committed
4988
							space, (ulint)n_pages);
Marko Mäkelä's avatar
Marko Mäkelä committed
4989
					if (fail) goto error;
4990
				}
4991 4992
			}

4993
			success = os_file_write(IORequestWrite,
4994
						dst_path, dst_file, buf, off, page_size);
4995 4996 4997 4998 4999
			if (!success) {
				goto error;
			}
		}

5000 5001 5002 5003 5004 5005 5006
		/* Free file system buffer cache after the batch was written. */
#ifdef __linux__
		os_file_flush_func(dst_file);
#endif
		posix_fadvise(dst_file, 0, 0, POSIX_FADV_DONTNEED);


5007 5008 5009
		incremental_buffers++;
	}

5010
	free(incremental_buffer_base);
5011
	if (src_file != OS_FILE_CLOSED) {
5012
		os_file_close(src_file);
5013 5014
		os_file_delete(0,src_path);
	}
5015
	if (dst_file != OS_FILE_CLOSED)
5016 5017 5018 5019
		os_file_close(dst_file);
	return TRUE;

error:
5020 5021
	free(incremental_buffer_base);
	if (src_file != OS_FILE_CLOSED)
5022
		os_file_close(src_file);
5023
	if (dst_file != OS_FILE_CLOSED)
5024
		os_file_close(dst_file);
5025
	msg("Error: xtrabackup_apply_delta(): "
5026 5027 5028 5029
	    "failed to apply %s to %s.\n", src_path, dst_path);
	return FALSE;
}

5030 5031 5032 5033 5034 5035 5036 5037 5038 5039 5040 5041 5042

std::string change_extension(std::string filename, std::string new_ext) {
	DBUG_ASSERT(new_ext.size() == 3);
	std::string new_name(filename);
	new_name.resize(new_name.size() - new_ext.size());
	new_name.append(new_ext);
	return new_name;
}


static void rename_file(const char *from,const char *to) {
	msg("Renaming %s to %s\n", from, to);
	if (my_rename(from, to, MY_WME)) {
5043
		die("Can't rename %s to %s errno %d", from, to, errno);
5044 5045 5046 5047 5048 5049
	}
}

static void rename_file(const std::string& from, const std::string &to) {
	rename_file(from.c_str(), to.c_str());
}
5050 5051 5052 5053 5054 5055 5056 5057 5058 5059 5060
/************************************************************************
Callback to handle datadir entry. Function of this type will be called
for each entry which matches the mask by xb_process_datadir.
@return should return TRUE on success */
typedef ibool (*handle_datadir_entry_func_t)(
/*=========================================*/
	const char*	data_home_dir,		/*!<in: path to datadir */
	const char*	db_name,		/*!<in: database name */
	const char*	file_name,		/*!<in: file name with suffix */
	void*		arg);			/*!<in: caller-provided data */

5061 5062 5063
/** Rename, and replace destination file, if exists */
static void rename_force(const char *from, const char *to) {
	if (access(to, R_OK) == 0) {
5064
		msg("Removing %s", to);
5065 5066 5067 5068 5069 5070 5071 5072 5073 5074 5075 5076 5077 5078 5079 5080 5081 5082 5083 5084 5085 5086 5087 5088 5089 5090 5091
		if (my_delete(to, MYF(MY_WME))) {
			msg("Can't remove %s, errno %d", to, errno);
			exit(EXIT_FAILURE);
		}
	}
	rename_file(from,to);
}

/* During prepare phase, rename ".new" files , that were created in backup_fix_ddl(),
  to ".ibd".*/
static ibool prepare_handle_new_files(
	const char*	data_home_dir,		/*!<in: path to datadir */
	const char*	db_name,		/*!<in: database name */
	const char*	file_name,		/*!<in: file name with suffix */
	void *)
{

	std::string src_path = std::string(data_home_dir) + '/' + std::string(db_name) + '/' + file_name;
	std::string dest_path = src_path;

	size_t index = dest_path.find(".new");
	DBUG_ASSERT(index != std::string::npos);
	dest_path.replace(index, 4, ".ibd");
	rename_force(src_path.c_str(),dest_path.c_str());
	return TRUE;
}

5092 5093 5094 5095 5096 5097 5098 5099 5100 5101 5102 5103 5104 5105 5106 5107 5108 5109 5110 5111 5112 5113 5114 5115 5116 5117 5118 5119 5120 5121 5122 5123 5124 5125 5126 5127 5128 5129 5130 5131 5132 5133 5134
/************************************************************************
Callback to handle datadir entry. Deletes entry if it has no matching
fil_space in fil_system directory.
@return FALSE if delete attempt was unsuccessful */
static
ibool
rm_if_not_found(
	const char*	data_home_dir,		/*!<in: path to datadir */
	const char*	db_name,		/*!<in: database name */
	const char*	file_name,		/*!<in: file name with suffix */
	void*		arg __attribute__((unused)))
{
	char			name[FN_REFLEN];
	xb_filter_entry_t*	table;

	snprintf(name, FN_REFLEN, "%s/%s", db_name, file_name);
	/* Truncate ".ibd" */
	name[strlen(name) - 4] = '\0';

	HASH_SEARCH(name_hash, inc_dir_tables_hash, ut_fold_string(name),
		    xb_filter_entry_t*,
		    table, (void) 0,
		    !strcmp(table->name, name));

	if (!table) {
		snprintf(name, FN_REFLEN, "%s/%s/%s", data_home_dir,
						      db_name, file_name);
		return os_file_delete(0, name);
	}

	return(TRUE);
}

/************************************************************************
Function enumerates files in datadir (provided by path) which are matched
by provided suffix. For each entry callback is called.
@return FALSE if callback for some entry returned FALSE */
static
ibool
xb_process_datadir(
	const char*			path,	/*!<in: datadir path */
	const char*			suffix,	/*!<in: suffix to match
						against */
5135
	handle_datadir_entry_func_t	func)	/*!<in: callback */
5136 5137
{
	ulint		ret;
5138
	char		dbpath[OS_FILE_MAX_PATH+1];
5139 5140 5141 5142 5143 5144 5145 5146 5147 5148 5149 5150 5151 5152 5153 5154 5155 5156 5157 5158 5159 5160 5161 5162 5163 5164 5165 5166 5167 5168 5169
	os_file_dir_t	dir;
	os_file_dir_t	dbdir;
	os_file_stat_t	dbinfo;
	os_file_stat_t	fileinfo;
	ulint		suffix_len;
	dberr_t		err 		= DB_SUCCESS;
	static char	current_dir[2];

	current_dir[0] = FN_CURLIB;
	current_dir[1] = 0;
	srv_data_home = current_dir;

	suffix_len = strlen(suffix);

	/* datafile */
	dbdir = os_file_opendir(path, FALSE);

	if (dbdir != NULL) {
		ret = fil_file_readdir_next_file(&err, path, dbdir,
							&fileinfo);
		while (ret == 0) {
			if (fileinfo.type == OS_FILE_TYPE_DIR) {
				goto next_file_item_1;
			}

			if (strlen(fileinfo.name) > suffix_len
			    && 0 == strcmp(fileinfo.name + 
					strlen(fileinfo.name) - suffix_len,
					suffix)) {
				if (!func(
					    path, NULL,
5170
					    fileinfo.name, NULL))
5171
				{
5172
					os_file_closedir(dbdir);
5173 5174 5175 5176 5177 5178 5179 5180 5181 5182 5183
					return(FALSE);
				}
			}
next_file_item_1:
			ret = fil_file_readdir_next_file(&err,
							path, dbdir,
							&fileinfo);
		}

		os_file_closedir(dbdir);
	} else {
5184
		msg("Can't open dir %s", path);
5185 5186 5187 5188 5189 5190
	}

	/* single table tablespaces */
	dir = os_file_opendir(path, FALSE);

	if (dir == NULL) {
5191
		msg("Can't open dir %s", path);
5192 5193 5194 5195 5196 5197 5198 5199 5200 5201 5202
	}

		ret = fil_file_readdir_next_file(&err, path, dir,
								&dbinfo);
	while (ret == 0) {
		if (dbinfo.type == OS_FILE_TYPE_FILE
		    || dbinfo.type == OS_FILE_TYPE_UNKNOWN) {

		        goto next_datadir_item;
		}

5203
		snprintf(dbpath, sizeof(dbpath)-1, "%s/%s", path, dbinfo.name);
5204

5205
		os_normalize_path(dbpath);
5206 5207 5208 5209 5210 5211 5212 5213 5214 5215 5216 5217 5218 5219 5220 5221 5222 5223 5224 5225 5226 5227 5228 5229

		dbdir = os_file_opendir(dbpath, FALSE);

		if (dbdir != NULL) {

			ret = fil_file_readdir_next_file(&err, dbpath, dbdir,
								&fileinfo);
			while (ret == 0) {

			        if (fileinfo.type == OS_FILE_TYPE_DIR) {

				        goto next_file_item_2;
				}

				if (strlen(fileinfo.name) > suffix_len
				    && 0 == strcmp(fileinfo.name + 
						strlen(fileinfo.name) -
								suffix_len,
						suffix)) {
					/* The name ends in suffix; process
					the file */
					if (!func(
						    path,
						    dbinfo.name,
5230
						    fileinfo.name, NULL))
5231
					{
5232
						os_file_closedir(dbdir);
5233 5234 5235 5236 5237 5238 5239 5240 5241 5242 5243 5244 5245 5246 5247 5248 5249 5250 5251 5252 5253 5254 5255 5256 5257 5258 5259 5260 5261 5262
						return(FALSE);
					}
				}
next_file_item_2:
				ret = fil_file_readdir_next_file(&err,
								dbpath, dbdir,
								&fileinfo);
			}

			os_file_closedir(dbdir);
		}
next_datadir_item:
		ret = fil_file_readdir_next_file(&err,
						path,
								dir, &dbinfo);
	}

	os_file_closedir(dir);

	return(TRUE);
}

/************************************************************************
Applies all .delta files from incremental_dir to the full backup.
@return TRUE on success. */
static
ibool
xtrabackup_apply_deltas()
{
	return xb_process_datadir(xtrabackup_incremental_dir, ".delta",
5263
		xtrabackup_apply_delta);
5264 5265 5266 5267 5268 5269 5270
}


static
void
innodb_free_param()
{
5271
	srv_sys_space.shutdown();
5272 5273 5274 5275
	free_tmpdir(&mysql_tmpdir_list);
}


5276 5277 5278 5279 5280 5281 5282 5283 5284 5285 5286
/** Check if file exists*/
static bool file_exists(std::string name)
{
	return access(name.c_str(), R_OK) == 0 ;
}

/** Read file content into STL string */
static std::string read_file_as_string(const std::string file) {
	char content[FN_REFLEN];
	FILE *f = fopen(file.c_str(), "r");
	if (!f) {
5287
		msg("Can not open %s", file.c_str());
5288 5289 5290 5291 5292 5293 5294 5295 5296 5297 5298
	}
	size_t len = fread(content, 1, FN_REFLEN, f);
	fclose(f);
	return std::string(content, len);
}

/** Delete file- Provide verbose diagnostics and exit, if operation fails. */
static void delete_file(const std::string& file, bool if_exists = false) {
	if (if_exists && !file_exists(file))
		return;
	if (my_delete(file.c_str(), MYF(MY_WME))) {
5299
		die("Can't remove %s, errno %d", file.c_str(), errno);
5300 5301 5302 5303 5304 5305 5306 5307 5308 5309 5310 5311 5312 5313 5314 5315 5316 5317 5318 5319 5320 5321 5322 5323 5324 5325 5326 5327 5328 5329 5330 5331 5332 5333 5334 5335 5336 5337 5338 5339 5340 5341 5342 5343 5344 5345 5346 5347 5348 5349 5350 5351 5352 5353 5354 5355 5356 5357 5358 5359 5360 5361 5362 5363 5364 5365 5366 5367 5368 5369 5370 5371
	}
}

/**
Rename tablespace during prepare.
Backup in its end phase may generate some .ren files, recording
tablespaces that should be renamed in --prepare.
*/
static void rename_table_in_prepare(const std::string &datadir, const std::string& from , const std::string& to,
	const char *extension=0) {
	if (!extension) {
		static const char *extensions_nonincremental[] = { ".ibd", 0 };
		static const char *extensions_incremental[] = { ".ibd.delta", ".ibd.meta", 0 };
		const char **extensions = xtrabackup_incremental_dir ?
			extensions_incremental : extensions_nonincremental;
		for (size_t i = 0; extensions[i]; i++) {
			rename_table_in_prepare(datadir, from, to, extensions[i]);
		}
		return;
	}
	std::string src = std::string(datadir) + "/" + from + extension;
	std::string dest = std::string(datadir) + "/" + to + extension;
	std::string ren2, tmp;
	if (file_exists(dest)) {
		ren2= std::string(datadir) + "/" + to + ".ren";
		if (!file_exists(ren2)) {
			msg("ERROR : File %s was not found, but expected during rename processing\n", ren2.c_str());
			ut_a(0);
		}
		tmp = to + "#";
		rename_table_in_prepare(datadir, to, tmp);
	}
	rename_file(src, dest);
	if (ren2.size()) {
		// Make sure the temp. renamed file is processed.
		std::string to2 = read_file_as_string(ren2);
		rename_table_in_prepare(datadir, tmp, to2);
		delete_file(ren2);
	}
}

static ibool prepare_handle_ren_files(const char *datadir, const char *db, const char *filename, void *) {

	std::string ren_file = std::string(datadir) + "/" + db + "/" + filename;
	if (!file_exists(ren_file))
		return TRUE;

	std::string to = read_file_as_string(ren_file);
	std::string source_space_name = std::string(db) + "/" + filename;
	source_space_name.resize(source_space_name.size() - 4); // remove extension

	rename_table_in_prepare(datadir, source_space_name.c_str(), to.c_str());
	delete_file(ren_file);
	return TRUE;
}

/* Remove tablespaces during backup, based on */
static ibool prepare_handle_del_files(const char *datadir, const char *db, const char *filename, void *) {
	std::string del_file = std::string(datadir) + "/" + db + "/" + filename;
	std::string path(del_file);
	path.resize(path.size() - 4); // remove extension;
	if (xtrabackup_incremental) {
		delete_file(path + ".ibd.delta", true);
		delete_file(path + ".ibd.meta", true);
	}
	else {
		delete_file(path + ".ibd", true);
	}
	delete_file(del_file);
	return TRUE;
}

5372 5373 5374 5375
/** Implement --prepare
@return	whether the operation succeeded */
static bool
xtrabackup_prepare_func(char** argv)
5376 5377 5378 5379 5380 5381 5382
{
	char			 metadata_path[FN_REFLEN];

	/* cd to target-dir */

	if (my_setwd(xtrabackup_real_target_dir,MYF(MY_WME)))
	{
5383
		msg("can't my_setwd %s", xtrabackup_real_target_dir);
5384
		return(false);
5385
	}
5386
	msg("cd to %s", xtrabackup_real_target_dir);
5387

5388 5389
	fil_path_to_mysql_datadir = ".";

5390 5391 5392 5393 5394 5395 5396 5397
	/* Fix DDL for prepare. Process .del,.ren, and .new files.
	The order in which files are processed, is important
	(see MDEV-18185, MDEV-18201)
	*/
	xb_process_datadir(xtrabackup_incremental_dir ? xtrabackup_incremental_dir : ".",
		".del", prepare_handle_del_files);
	xb_process_datadir(xtrabackup_incremental_dir? xtrabackup_incremental_dir:".",
		".ren", prepare_handle_ren_files);
5398 5399 5400 5401 5402 5403 5404 5405
	if (xtrabackup_incremental_dir) {
		xb_process_datadir(xtrabackup_incremental_dir, ".new.meta", prepare_handle_new_files);
		xb_process_datadir(xtrabackup_incremental_dir, ".new.delta", prepare_handle_new_files);
	}
	else {
		xb_process_datadir(".", ".new", prepare_handle_new_files);
	}

5406
	int argc; for (argc = 0; argv[argc]; argc++) {}
5407 5408
	encryption_plugin_prepare_init(argc, argv);

5409 5410 5411
	xtrabackup_target_dir= mysql_data_home_buff;
	xtrabackup_target_dir[0]=FN_CURLIB;		// all paths are relative from here
	xtrabackup_target_dir[1]=0;
5412 5413
	const lsn_t target_lsn = xtrabackup_incremental
		? incremental_to_lsn : metadata_to_lsn;
5414 5415

	/*
5416
	  read metadata of target
5417 5418 5419 5420 5421
	*/
	sprintf(metadata_path, "%s/%s", xtrabackup_target_dir,
		XTRABACKUP_METADATA_FILENAME);

	if (!xtrabackup_read_metadata(metadata_path)) {
5422
		msg("Error: failed to read metadata from '%s'\n",
5423
		    metadata_path);
5424
		return(false);
5425 5426
	}

5427
	if (!strcmp(metadata_type, "full-backuped")) {
5428
		if (xtrabackup_incremental) {
5429 5430
			msg("error: applying incremental backup "
			    "needs a prepared target.");
5431 5432
			return(false);
		}
5433
		msg("This target seems to be not prepared yet.");
5434
	} else if (!strcmp(metadata_type, "log-applied")) {
5435
		msg("This target seems to be already prepared.");
5436
	} else {
5437
		msg("This target does not have correct metadata.");
5438
		return(false);
5439
	}
5440

5441 5442 5443
	bool ok = !xtrabackup_incremental
		|| metadata_to_lsn == incremental_lsn;
	if (!ok) {
5444 5445 5446
		msg("error: This incremental backup seems "
		    "not to be proper for the target. Check 'to_lsn' of the target and "
		    "'from_lsn' of the incremental.");
5447
		return(false);
5448 5449 5450
	}

	srv_max_n_threads = 1000;
5451
	srv_undo_logs = 1;
Marko Mäkelä's avatar
Marko Mäkelä committed
5452
	srv_n_purge_threads = 1;
5453

5454 5455
	xb_filters_init();

5456 5457
	srv_log_group_home_dir = NULL;
	srv_thread_concurrency = 1;
5458

5459
	if (xtrabackup_incremental) {
5460 5461
		srv_operation = SRV_OPERATION_RESTORE_DELTA;

5462
		if (innodb_init_param()) {
5463
			goto error_cleanup;
5464
		}
5465

5466 5467 5468 5469
		sync_check_init();
		ut_d(sync_check_enable());
		ut_crc32_init();
		recv_sys_init();
5470
		log_sys.create();
5471
		recv_recovery_on = true;
5472

5473 5474 5475 5476 5477
#ifdef WITH_INNODB_DISALLOW_WRITES
		srv_allow_writes_event = os_event_create(0);
		os_event_set(srv_allow_writes_event);
#endif
		dberr_t err = xb_data_files_init();
5478
		if (err != DB_SUCCESS) {
5479
			msg("mariabackup: error: xb_data_files_init() failed "
5480
			    "with error %s\n", ut_strerr(err));
5481 5482
			goto error_cleanup;
		}
5483

5484 5485
		inc_dir_tables_hash = hash_create(1000);

5486 5487
		ok = xtrabackup_apply_deltas();

5488 5489
		xb_data_files_close();

5490 5491 5492 5493 5494 5495
		if (ok) {
			/* Cleanup datadir from tablespaces deleted
			between full and incremental backups */

			xb_process_datadir("./", ".ibd", rm_if_not_found);
		}
5496 5497

		xb_filter_hash_free(inc_dir_tables_hash);
5498

5499
		fil_system.close();
5500 5501 5502 5503
#ifdef WITH_INNODB_DISALLOW_WRITES
		os_event_destroy(srv_allow_writes_event);
#endif
		innodb_free_param();
5504
		log_sys.close();
5505 5506
		sync_check_close();
		if (!ok) goto error_cleanup;
5507
	}
5508

5509 5510
	srv_operation = xtrabackup_export
		? SRV_OPERATION_RESTORE_EXPORT : SRV_OPERATION_RESTORE;
5511

5512
	if (innodb_init_param()) {
5513 5514 5515 5516
		goto error_cleanup;
	}

	/* increase IO threads */
5517
	if (srv_n_file_io_threads < 10) {
5518 5519 5520 5521
		srv_n_read_io_threads = 4;
		srv_n_write_io_threads = 4;
	}

5522 5523 5524 5525
	msg("Starting InnoDB instance for recovery.");

	msg("mariabackup: Using %lld bytes for buffer pool "
	    "(set by --use-memory parameter)", xtrabackup_use_memory);
5526

5527 5528 5529 5530 5531 5532
	srv_max_buf_pool_modified_pct = (double)max_buf_pool_modified_pct;

	if (srv_max_dirty_pages_pct_lwm > srv_max_buf_pool_modified_pct) {
		srv_max_dirty_pages_pct_lwm = srv_max_buf_pool_modified_pct;
	}

5533
	if (innodb_init()) {
5534 5535 5536
		goto error_cleanup;
	}

5537
	if (ok) {
Marko Mäkelä's avatar
Marko Mäkelä committed
5538
		msg("Last binlog file %s, position %lld",
5539 5540
		    trx_sys.recovered_binlog_filename,
		    longlong(trx_sys.recovered_binlog_offset));
5541 5542 5543
	}

	/* Check whether the log is applied enough or not. */
5544 5545
	if ((srv_start_lsn || fil_space_get(SRV_LOG_SPACE_FIRST_ID))
	    && srv_start_lsn < target_lsn) {
5546
		msg("mariabackup: error: "
5547
		    "The log was only applied up to LSN " LSN_PF
5548
		    ", instead of " LSN_PF,
5549 5550
		    srv_start_lsn, target_lsn);
		ok = false;
5551
	}
5552
#ifdef WITH_WSREP
5553
	else if (ok) xb_write_galera_info(xtrabackup_incremental);
5554
#endif
5555

5556 5557
	innodb_shutdown();
	innodb_free_param();
5558 5559

	/* output to metadata file */
5560
	if (ok) {
5561 5562
		char	filename[FN_REFLEN];

5563
		strcpy(metadata_type, "log-applied");
5564 5565 5566 5567 5568 5569 5570 5571 5572 5573 5574

		if(xtrabackup_incremental
		   && metadata_to_lsn < incremental_to_lsn)
		{
			metadata_to_lsn = incremental_to_lsn;
			metadata_last_lsn = incremental_last_lsn;
		}

		sprintf(filename, "%s/%s", xtrabackup_target_dir, XTRABACKUP_METADATA_FILENAME);
		if (!xtrabackup_write_metadata(filename)) {

5575
			msg("mariabackup: Error: failed to write metadata "
5576
			    "to '%s'", filename);
5577 5578
			ok = false;
		} else if (xtrabackup_extra_lsndir) {
5579 5580
			sprintf(filename, "%s/%s", xtrabackup_extra_lsndir, XTRABACKUP_METADATA_FILENAME);
			if (!xtrabackup_write_metadata(filename)) {
5581
				msg("mariabackup: Error: failed to write "
5582
				    "metadata to '%s'", filename);
5583
				ok = false;
5584 5585 5586 5587
			}
		}
	}

5588
	if (ok) ok = apply_log_finish();
5589

5590 5591 5592 5593
	if (ok && xtrabackup_export)
		ok= (prepare_export() == 0);

error_cleanup:
5594
	xb_filters_free();
5595
	return ok;
5596 5597 5598 5599 5600 5601 5602 5603 5604 5605 5606 5607 5608 5609 5610 5611 5612 5613 5614 5615 5616
}

/**************************************************************************
Append group name to xb_load_default_groups list. */
static
void
append_defaults_group(const char *group, const char *default_groups[],
		      size_t default_groups_size)
{
	uint i;
	bool appended = false;
	for (i = 0; i < default_groups_size - 1; i++) {
		if (default_groups[i] == NULL) {
			default_groups[i] = group;
			appended = true;
			break;
		}
	}
	ut_a(appended);
}

5617 5618 5619 5620 5621 5622 5623 5624 5625 5626 5627 5628 5629 5630 5631 5632 5633 5634 5635 5636 5637 5638 5639 5640 5641 5642 5643 5644 5645 5646 5647 5648 5649 5650 5651 5652 5653 5654
static const char*
normalize_privilege_target_name(const char* name)
{
	if (strcmp(name, "*") == 0) {
		return "\\*";
	}
	else {
		/* should have no regex special characters. */
		ut_ad(strpbrk(name, ".()[]*+?") == 0);
	}
	return name;
}

/******************************************************************//**
Check if specific privilege is granted.
Uses regexp magic to check if requested privilege is granted for given
database.table or database.* or *.*
or if user has 'ALL PRIVILEGES' granted.
@return true if requested privilege is granted, false otherwise. */
static bool
has_privilege(const std::list<std::string> &granted,
	const char* required,
	const char* db_name,
	const char* table_name)
{
	char buffer[1000];
	regex_t priv_re;
	regmatch_t tables_regmatch[1];
	bool result = false;

	db_name = normalize_privilege_target_name(db_name);
	table_name = normalize_privilege_target_name(table_name);

	int written = snprintf(buffer, sizeof(buffer),
		"GRANT .*(%s)|(ALL PRIVILEGES).* ON (\\*|`%s`)\\.(\\*|`%s`)",
		required, db_name, table_name);
	if (written < 0 || written == sizeof(buffer)
		|| regcomp(&priv_re, buffer, REG_EXTENDED)) {
5655
		die("regcomp() failed for '%s'", buffer);
5656 5657 5658 5659 5660 5661 5662 5663 5664 5665 5666 5667 5668 5669 5670 5671 5672 5673 5674 5675 5676 5677 5678 5679 5680 5681 5682 5683 5684 5685 5686 5687 5688 5689 5690 5691 5692 5693 5694 5695 5696
	}

	typedef std::list<std::string>::const_iterator string_iter;
	for (string_iter i = granted.begin(), e = granted.end(); i != e; ++i) {
		int res = regexec(&priv_re, i->c_str(),
			1, tables_regmatch, 0);

		if (res != REG_NOMATCH) {
			result = true;
			break;
		}
	}

	xb_regfree(&priv_re);
	return result;
}

enum {
	PRIVILEGE_OK = 0,
	PRIVILEGE_WARNING = 1,
	PRIVILEGE_ERROR = 2,
};

/******************************************************************//**
Check if specific privilege is granted.
Prints error message if required privilege is missing.
@return PRIVILEGE_OK if requested privilege is granted, error otherwise. */
static
int check_privilege(
	const std::list<std::string> &granted_priv, /* in: list of
							granted privileges*/
	const char* required,		/* in: required privilege name */
	const char* target_database,	/* in: required privilege target
						database name */
	const char* target_table,	/* in: required privilege target
						table name */
	int error = PRIVILEGE_ERROR)	/* in: return value if privilege
						is not granted */
{
	if (!has_privilege(granted_priv,
		required, target_database, target_table)) {
5697
		msg("%s: missing required privilege %s on %s.%s",
5698 5699 5700 5701 5702 5703 5704 5705 5706 5707 5708 5709 5710 5711 5712 5713 5714 5715 5716 5717 5718 5719 5720 5721 5722 5723 5724 5725 5726 5727 5728 5729 5730 5731 5732 5733 5734 5735 5736 5737 5738 5739 5740 5741 5742 5743 5744 5745 5746 5747 5748 5749 5750 5751 5752 5753 5754 5755 5756 5757 5758 5759 5760 5761 5762 5763 5764 5765 5766 5767 5768
			(error == PRIVILEGE_ERROR ? "Error" : "Warning"),
			required, target_database, target_table);
		return error;
	}
	return PRIVILEGE_OK;
}


/******************************************************************//**
Check DB user privileges according to the intended actions.

Fetches DB user privileges, determines intended actions based on
command-line arguments and prints missing privileges.
May terminate application with EXIT_FAILURE exit code.*/
static void
check_all_privileges()
{
	if (!mysql_connection) {
		/* Not connected, no queries is going to be executed. */
		return;
	}

	/* Fetch effective privileges. */
	std::list<std::string> granted_privileges;
	MYSQL_ROW row = 0;
	MYSQL_RES* result = xb_mysql_query(mysql_connection, "SHOW GRANTS",
		true);
	while ((row = mysql_fetch_row(result))) {
		granted_privileges.push_back(*row);
	}
	mysql_free_result(result);

	int check_result = PRIVILEGE_OK;

	/* FLUSH TABLES WITH READ LOCK */
	if (!opt_no_lock)
	{
		check_result |= check_privilege(
			granted_privileges,
			"RELOAD", "*", "*");
	}

	if (!opt_no_lock)
	{
		check_result |= check_privilege(
			granted_privileges,
		"PROCESS", "*", "*");
	}

	/* KILL ... */
	if ((!opt_no_lock && (opt_kill_long_queries_timeout || opt_lock_ddl_per_table))
		/* START SLAVE SQL_THREAD */
		/* STOP SLAVE SQL_THREAD */
		|| opt_safe_slave_backup) {
		check_result |= check_privilege(
			granted_privileges,
			"SUPER", "*", "*",
			PRIVILEGE_WARNING);
	}

	/* SHOW MASTER STATUS */
	/* SHOW SLAVE STATUS */
	if (opt_galera_info || opt_slave_info
		|| (opt_no_lock && opt_safe_slave_backup)) {
		check_result |= check_privilege(granted_privileges,
			"REPLICATION CLIENT", "*", "*",
			PRIVILEGE_WARNING);
	}

	if (check_result & PRIVILEGE_ERROR) {
		mysql_close(mysql_connection);
5769
		die("Insufficient privileges");
5770 5771 5772
	}
}

5773 5774 5775 5776 5777 5778 5779 5780 5781 5782 5783 5784 5785
bool
xb_init()
{
	const char *mixed_options[4] = {NULL, NULL, NULL, NULL};
	int n_mixed_options;

	/* sanity checks */

	if (opt_slave_info
		&& opt_no_lock
		&& !opt_safe_slave_backup) {
		msg("Error: --slave-info is used with --no-lock but "
			"without --safe-slave-backup. The binlog position "
5786
			"cannot be consistent with the backup data.");
5787 5788 5789
		return(false);
	}

5790 5791 5792 5793 5794 5795 5796 5797 5798 5799 5800 5801 5802
	if (xtrabackup_backup && opt_rsync)
	{
		if (xtrabackup_stream_fmt)
		{
			msg("Error: --rsync doesn't work with --stream\n");
			return(false);
		}
		bool have_rsync = IF_WIN(false, (system("rsync --version > /dev/null 2>&1") == 0));
		if (!have_rsync)
		{
			msg("Error: rsync executable not found, cannot run backup with --rsync\n");
			return false;
		}
5803 5804 5805 5806 5807 5808 5809 5810 5811 5812 5813 5814 5815 5816 5817 5818 5819 5820 5821 5822 5823 5824 5825 5826 5827 5828 5829 5830 5831 5832 5833 5834 5835 5836
	}

	n_mixed_options = 0;

	if (opt_decompress) {
		mixed_options[n_mixed_options++] = "--decompress";
	}

	if (xtrabackup_copy_back) {
		mixed_options[n_mixed_options++] = "--copy-back";
	}

	if (xtrabackup_move_back) {
		mixed_options[n_mixed_options++] = "--move-back";
	}

	if (xtrabackup_prepare) {
		mixed_options[n_mixed_options++] = "--apply-log";
	}

	if (n_mixed_options > 1) {
		msg("Error: %s and %s are mutually exclusive\n",
			mixed_options[0], mixed_options[1]);
		return(false);
	}

	if (xtrabackup_backup) {
		if ((mysql_connection = xb_mysql_connect()) == NULL) {
			return(false);
		}

		if (!get_mysql_vars(mysql_connection)) {
			return(false);
		}
5837 5838 5839
		if (opt_check_privileges) {
			check_all_privileges();
		}
5840 5841 5842 5843 5844 5845 5846
		history_start_time = time(NULL);

	}

	return(true);
}

5847 5848 5849 5850 5851 5852 5853 5854

extern void init_signals(void);

#include <sql_locale.h>

/* Messages . Avoid loading errmsg.sys file */
void setup_error_messages()
{
5855 5856
  static const char *my_msgs[ERRORS_PER_RANGE];
  static const char **all_msgs[] = { my_msgs, my_msgs, my_msgs, my_msgs };
5857 5858 5859 5860 5861 5862 5863 5864 5865 5866 5867 5868 5869 5870 5871 5872 5873 5874 5875 5876 5877 5878 5879 5880 5881 5882
  my_default_lc_messages = &my_locale_en_US;
  my_default_lc_messages->errmsgs->errmsgs = all_msgs;

  /* Populate the necessary error messages */
  struct {
    int id;
    const char *fmt;
  }
  xb_msgs[] =
  {
  { ER_DATABASE_NAME,"Database" },
  { ER_TABLE_NAME,"Table"},
  { ER_PARTITION_NAME, "Partition" },
  { ER_SUBPARTITION_NAME, "Subpartition" },
  { ER_TEMPORARY_NAME, "Temporary"},
  { ER_RENAMED_NAME, "Renamed"},
  { ER_CANT_FIND_DL_ENTRY, "Can't find symbol '%-.128s' in library"},
  { ER_CANT_OPEN_LIBRARY, "Can't open shared library '%-.192s' (errno: %d, %-.128s)" },
  { ER_OUTOFMEMORY, "Out of memory; restart server and try again (needed %d bytes)" },
  { ER_CANT_OPEN_LIBRARY, "Can't open shared library '%-.192s' (errno: %d, %-.128s)" },
  { ER_UDF_NO_PATHS, "No paths allowed for shared library" },
  { ER_CANT_INITIALIZE_UDF,"Can't initialize function '%-.192s'; %-.80s"},
  { ER_PLUGIN_IS_NOT_LOADED,"Plugin '%-.192s' is not loaded" }
  };

  for (int i = 0; i < (int)array_elements(all_msgs); i++)
5883
    all_msgs[0][i] = "Unknown error";
5884 5885

  for (int i = 0; i < (int)array_elements(xb_msgs); i++)
5886
    all_msgs[0][xb_msgs[i].id - ER_ERROR_FIRST] = xb_msgs[i].fmt;
5887 5888
}

5889 5890 5891
void
handle_options(int argc, char **argv, char ***argv_client, char ***argv_server)
{
5892 5893
	/* Setup some variables for Innodb.*/

5894
	srv_operation = SRV_OPERATION_RESTORE;
5895 5896

	files_charset_info = &my_charset_utf8_general_ci;
5897

5898 5899 5900 5901

	setup_error_messages();
	sys_var_init();
	plugin_mutex_init();
5902
	mysql_prlock_init(key_rwlock_LOCK_system_variables_hash, &LOCK_system_variables_hash);
5903 5904 5905 5906 5907 5908 5909 5910
	opt_stack_trace = 1;
	test_flags |=  TEST_SIGINT;
	init_signals();
#ifndef _WIN32
	/* Exit process on SIGINT. */
	my_sigset(SIGINT, SIG_DFL);
#endif

5911
	sf_leaking_memory = 1; /* don't report memory leaks on early exist */
5912

5913 5914 5915 5916 5917 5918 5919 5920 5921 5922 5923 5924 5925 5926 5927 5928 5929 5930 5931 5932 5933 5934 5935 5936 5937 5938 5939 5940 5941 5942 5943 5944 5945 5946 5947 5948 5949 5950 5951 5952 5953 5954 5955 5956 5957 5958 5959 5960 5961 5962 5963 5964 5965 5966 5967
	int i;
	int ho_error;

	char*	target_dir = NULL;
	bool	prepare = false;

	char	conf_file[FN_REFLEN];
	int	argc_client = argc;
	int	argc_server = argc;

	/* scan options for group and config file to load defaults from */
	for (i = 1; i < argc; i++) {

		char *optend = strcend(argv[i], '=');

		if (strncmp(argv[i], "--defaults-group",
			    optend - argv[i]) == 0) {
			defaults_group = optend + 1;
			append_defaults_group(defaults_group,
				xb_server_default_groups,
				array_elements(xb_server_default_groups));
		}

		if (strncmp(argv[i], "--login-path",
			    optend - argv[i]) == 0) {
			append_defaults_group(optend + 1,
				xb_client_default_groups,
				array_elements(xb_client_default_groups));
		}

		if (!strncmp(argv[i], "--prepare",
			     optend - argv[i])) {
			prepare = true;
		}

		if (!strncmp(argv[i], "--apply-log",
			     optend - argv[i])) {
			prepare = true;
		}

		if (!strncmp(argv[i], "--target-dir",
			     optend - argv[i]) && *optend) {
			target_dir = optend + 1;
		}

		if (!*optend && argv[i][0] != '-') {
			target_dir = argv[i];
		}
	}

	snprintf(conf_file, sizeof(conf_file), "my");

	if (prepare && target_dir) {
		snprintf(conf_file, sizeof(conf_file),
			 "%s/backup-my.cnf", target_dir);
5968 5969 5970 5971 5972 5973 5974 5975
			if (!strncmp(argv[1], "--defaults-file=", 16)) {
				/* Remove defaults-file*/
				for (int i = 2; ; i++) {
					if ((argv[i-1]= argv[i]) == 0)
						break;
				}
				argc--;
			}
5976
	}
5977 5978 5979

	*argv_client = argv;
	*argv_server = argv;
5980 5981
	load_defaults_or_exit(conf_file, xb_server_default_groups,
			      &argc_server, argv_server);
5982

5983 5984 5985 5986
	int n;
	for (n = 0; (*argv_server)[n]; n++) {};
	argc_server = n;

5987 5988 5989 5990 5991 5992 5993 5994 5995 5996 5997 5998 5999 6000
	print_param_str <<
		"# This MySQL options file was generated by XtraBackup.\n"
		"[" << defaults_group << "]\n";

	/* We want xtrabackup to ignore unknown options, because it only
	recognizes a small subset of server variables */
	my_getopt_skip_unknown = TRUE;

	/* Reset u_max_value for all options, as we don't want the
	--maximum-... modifier to set the actual option values */
	for (my_option *optp= xb_server_options; optp->name; optp++) {
		optp->u_max_value = (G_PTR *) &global_max_value;
	}

6001

6002 6003 6004 6005 6006 6007
	/* Throw a descriptive error if --defaults-file or --defaults-extra-file
	is not the first command line argument */
	for (int i = 2 ; i < argc ; i++) {
		char *optend = strcend((argv)[i], '=');

		if (optend - argv[i] == 15 &&
6008 6009
			!strncmp(argv[i], "--defaults-file", optend - argv[i])) {
			die("--defaults-file must be specified first on the command line");
6010
		}
6011 6012 6013 6014
		if (optend - argv[i] == 21 &&
			!strncmp(argv[i], "--defaults-extra-file",
				optend - argv[i])) {
			die("--defaults-extra-file must be specified first on the command line");
6015 6016 6017 6018 6019 6020 6021 6022
		}
	}

	if (argc_server > 0
	    && (ho_error=handle_options(&argc_server, argv_server,
					xb_server_options, xb_get_one_option)))
		exit(ho_error);

6023 6024
	load_defaults_or_exit(conf_file, xb_client_default_groups,
			      &argc_client, argv_client);
6025

6026 6027 6028
	for (n = 0; (*argv_client)[n]; n++) {};
 	argc_client = n;

6029
	if (innobackupex_mode && argc_client > 0) {
6030 6031 6032 6033 6034 6035 6036 6037 6038 6039 6040 6041 6042 6043 6044 6045 6046 6047 6048 6049 6050 6051 6052 6053 6054 6055 6056 6057 6058 6059
		/* emulate innobackupex script */
		innobackupex_mode = true;
		if (!ibx_handle_options(&argc_client, argv_client)) {
			exit(EXIT_FAILURE);
		}
	}

	if (argc_client > 0
	    && (ho_error=handle_options(&argc_client, argv_client,
					xb_client_options, xb_get_one_option)))
		exit(ho_error);

	/* Reject command line arguments that don't look like options, i.e. are
	not of the form '-X' (single-character options) or '--option' (long
	options) */
	for (int i = 0 ; i < argc_client ; i++) {
		const char * const opt = (*argv_client)[i];

		if (strncmp(opt, "--", 2) &&
		    !(strlen(opt) == 2 && opt[0] == '-')) {
			bool server_option = true;

			for (int j = 0; j < argc_server; j++) {
				if (opt == (*argv_server)[j]) {
					server_option = false;
					break;
				}
			}

			if (!server_option) {
6060
				msg("mariabackup: Error:"
6061
				    " unknown argument: '%s'", opt);
6062 6063 6064 6065 6066 6067
				exit(EXIT_FAILURE);
			}
		}
	}
}

6068
static int main_low(char** argv);
6069
static int get_exepath(char *buf, size_t size, const char *argv0);
6070

6071
/* ================= main =================== */
6072 6073 6074
int main(int argc, char **argv)
{
	char **client_defaults, **server_defaults;
6075 6076 6077 6078 6079 6080

	if (get_exepath(mariabackup_exe,FN_REFLEN, argv[0]))
    strncpy(mariabackup_exe,argv[0], FN_REFLEN-1);


	if (argc > 1 )
6081
	{
6082 6083 6084 6085 6086 6087 6088 6089 6090 6091 6092 6093 6094 6095 6096 6097 6098 6099
		/* In "prepare export", we need  to start mysqld 
		Since it is not always be installed on the machine,
		we start "mariabackup --mysqld", which acts as mysqld
		*/
		if (strcmp(argv[1], "--mysqld") == 0)
		{
			extern int mysqld_main(int argc, char **argv);
			argc--;
			argv++;
			argv[0]+=2;
			return mysqld_main(argc, argv);
		}
		if(strcmp(argv[1], "--innobackupex") == 0)
		{
			argv++;
			argc--;
			innobackupex_mode = true;
		}
6100
	}
6101 6102 6103
  
	if (argc > 1)
		strncpy(orig_argv1,argv[1],sizeof(orig_argv1) -1);
6104

6105
	init_signals();
6106 6107 6108 6109 6110 6111 6112 6113 6114 6115 6116
	MY_INIT(argv[0]);

	pthread_key_create(&THR_THD, NULL);
	my_pthread_setspecific_ptr(THR_THD, NULL);

	xb_regex_init();

	capture_tool_command(argc, argv);

	if (mysql_server_init(-1, NULL, NULL))
	{
6117
		die("mysql_server_init() failed");
6118 6119
	}

6120
	system_charset_info = &my_charset_utf8_general_ci;
6121 6122
	key_map_full.set_all();

6123 6124 6125 6126 6127
	logger.init_base();
	logger.set_handlers(LOG_FILE, LOG_NONE, LOG_NONE);
	mysql_mutex_init(key_LOCK_error_log, &LOCK_error_log,
			 MY_MUTEX_INIT_FAST);

6128 6129
	handle_options(argc, argv, &client_defaults, &server_defaults);

6130 6131 6132 6133 6134 6135 6136 6137 6138 6139 6140 6141 6142 6143 6144 6145 6146 6147 6148 6149 6150 6151 6152 6153 6154 6155 6156 6157 6158
#ifndef DBUG_OFF
	if (dbug_option) {
		DBUG_SET_INITIAL(dbug_option);
		DBUG_SET(dbug_option);
	}
#endif

	int status = main_low(server_defaults);

	backup_cleanup();

	if (innobackupex_mode) {
		ibx_cleanup();
	}

	free_defaults(client_defaults);
	free_defaults(server_defaults);

#ifndef DBUG_OFF
	if (dbug_option) {
		DBUG_END();
	}
#endif

	if (THR_THD)
		(void) pthread_key_delete(THR_THD);

	logger.cleanup_base();
	mysql_mutex_destroy(&LOCK_error_log);
6159

6160
	if (status == EXIT_SUCCESS) {
6161
		msg("completed OK!");
6162
	}
6163

6164 6165
	return status;
}
6166

6167 6168
static int main_low(char** argv)
{
6169 6170
	if (innobackupex_mode) {
		if (!ibx_init()) {
6171
			return(EXIT_FAILURE);
6172 6173 6174
		}
	}

6175 6176
	if (!xtrabackup_print_param && !xtrabackup_prepare
	    && !strcmp(mysql_data_home, "./")) {
6177 6178
		if (!xtrabackup_print_param)
			usage();
6179
		msg("mariabackup: Error: Please set parameter 'datadir'");
6180
		return(EXIT_FAILURE);
6181 6182 6183 6184
	}

	/* Expand target-dir, incremental-basedir, etc. */

6185
	char cwd[FN_REFLEN];
6186 6187 6188 6189 6190
	my_getwd(cwd, sizeof(cwd), MYF(0));

	my_load_path(xtrabackup_real_target_dir,
		     xtrabackup_target_dir, cwd);
	unpack_dirname(xtrabackup_real_target_dir,
6191
		       xtrabackup_real_target_dir);
6192 6193 6194 6195 6196 6197
	xtrabackup_target_dir= xtrabackup_real_target_dir;

	if (xtrabackup_incremental_basedir) {
		my_load_path(xtrabackup_real_incremental_basedir,
			     xtrabackup_incremental_basedir, cwd);
		unpack_dirname(xtrabackup_real_incremental_basedir,
6198
			       xtrabackup_real_incremental_basedir);
6199 6200 6201 6202 6203 6204 6205 6206
		xtrabackup_incremental_basedir =
			xtrabackup_real_incremental_basedir;
	}

	if (xtrabackup_incremental_dir) {
		my_load_path(xtrabackup_real_incremental_dir,
			     xtrabackup_incremental_dir, cwd);
		unpack_dirname(xtrabackup_real_incremental_dir,
6207
			       xtrabackup_real_incremental_dir);
6208 6209 6210 6211 6212 6213 6214
		xtrabackup_incremental_dir = xtrabackup_real_incremental_dir;
	}

	if (xtrabackup_extra_lsndir) {
		my_load_path(xtrabackup_real_extra_lsndir,
			     xtrabackup_extra_lsndir, cwd);
		unpack_dirname(xtrabackup_real_extra_lsndir,
6215
			       xtrabackup_real_extra_lsndir);
6216 6217 6218 6219 6220 6221 6222 6223 6224 6225 6226 6227 6228 6229 6230 6231 6232 6233 6234 6235 6236 6237 6238 6239 6240 6241 6242 6243 6244 6245 6246 6247 6248
		xtrabackup_extra_lsndir = xtrabackup_real_extra_lsndir;
	}

	/* get default temporary directory */
	if (!opt_mysql_tmpdir || !opt_mysql_tmpdir[0]) {
		opt_mysql_tmpdir = getenv("TMPDIR");
#if defined(__WIN__)
		if (!opt_mysql_tmpdir) {
			opt_mysql_tmpdir = getenv("TEMP");
		}
		if (!opt_mysql_tmpdir) {
			opt_mysql_tmpdir = getenv("TMP");
		}
#endif
		if (!opt_mysql_tmpdir || !opt_mysql_tmpdir[0]) {
			opt_mysql_tmpdir = const_cast<char*>(DEFAULT_TMPDIR);
		}
	}

	/* temporary setting of enough size */
	srv_page_size_shift = UNIV_PAGE_SIZE_SHIFT_MAX;
	srv_page_size = UNIV_PAGE_SIZE_MAX;
	if (xtrabackup_backup && xtrabackup_incremental) {
		/* direct specification is only for --backup */
		/* and the lsn is prior to the other option */

		char* endchar;
		int error = 0;
		incremental_lsn = strtoll(xtrabackup_incremental, &endchar, 10);
		if (*endchar != '\0')
			error = 1;

		if (error) {
6249
			msg("mariabackup: value '%s' may be wrong format for "
6250
			    "incremental option.", xtrabackup_incremental);
6251
			return(EXIT_FAILURE);
6252 6253 6254 6255 6256 6257 6258
		}
	} else if (xtrabackup_backup && xtrabackup_incremental_basedir) {
		char	filename[FN_REFLEN];

		sprintf(filename, "%s/%s", xtrabackup_incremental_basedir, XTRABACKUP_METADATA_FILENAME);

		if (!xtrabackup_read_metadata(filename)) {
6259
			msg("mariabackup: error: failed to read metadata from "
6260
			    "%s", filename);
6261
			return(EXIT_FAILURE);
6262 6263 6264 6265 6266 6267 6268 6269 6270 6271
		}

		incremental_lsn = metadata_to_lsn;
		xtrabackup_incremental = xtrabackup_incremental_basedir; //dummy
	} else if (xtrabackup_prepare && xtrabackup_incremental_dir) {
		char	filename[FN_REFLEN];

		sprintf(filename, "%s/%s", xtrabackup_incremental_dir, XTRABACKUP_METADATA_FILENAME);

		if (!xtrabackup_read_metadata(filename)) {
6272
			msg("mariabackup: error: failed to read metadata from "
6273
			    "%s", filename);
6274
			return(EXIT_FAILURE);
6275 6276 6277 6278 6279 6280 6281 6282 6283 6284 6285 6286 6287 6288 6289
		}

		incremental_lsn = metadata_from_lsn;
		incremental_to_lsn = metadata_to_lsn;
		incremental_last_lsn = metadata_last_lsn;
		xtrabackup_incremental = xtrabackup_incremental_dir; //dummy

	} else if (opt_incremental_history_name) {
		xtrabackup_incremental = opt_incremental_history_name;
	} else if (opt_incremental_history_uuid) {
		xtrabackup_incremental = opt_incremental_history_uuid;
	} else {
		xtrabackup_incremental = NULL;
	}

6290
	if (xtrabackup_stream && !xtrabackup_backup) {
6291
		msg("Warning: --stream parameter is ignored, it only works together with --backup.");
6292 6293
	}

6294
	if (!xb_init()) {
6295
		return(EXIT_FAILURE);
6296 6297 6298 6299 6300
	}

	/* --print-param */
	if (xtrabackup_print_param) {
		printf("%s", print_param_str.str().c_str());
6301
		return(EXIT_SUCCESS);
6302 6303 6304 6305
	}

	print_version();
	if (xtrabackup_incremental) {
6306
		msg("incremental backup from " LSN_PF " is enabled.",
6307 6308 6309 6310
		    incremental_lsn);
	}

	if (xtrabackup_export && innobase_file_per_table == FALSE) {
6311
		msg("mariabackup: auto-enabling --innodb-file-per-table due to "
6312
		    "the --export option");
6313 6314 6315 6316 6317 6318 6319 6320 6321 6322 6323 6324 6325 6326
		innobase_file_per_table = TRUE;
	}

	/* cannot execute both for now */
	{
		int num = 0;

		if (xtrabackup_backup) num++;
		if (xtrabackup_prepare) num++;
		if (xtrabackup_copy_back) num++;
		if (xtrabackup_move_back) num++;
		if (xtrabackup_decrypt_decompress) num++;
		if (num != 1) { /* !XOR (for now) */
			usage();
6327
			return(EXIT_FAILURE);
6328 6329 6330 6331 6332 6333 6334 6335 6336 6337
		}
	}

#ifndef __WIN__
	if (xtrabackup_debug_sync) {
		signal(SIGCONT, sigcont_handler);
	}
#endif

	/* --backup */
6338 6339 6340
	if (xtrabackup_backup && !xtrabackup_backup_func()) {
		return(EXIT_FAILURE);
	}
6341 6342

	/* --prepare */
6343 6344 6345
	if (xtrabackup_prepare
	    && !xtrabackup_prepare_func(argv)) {
		return(EXIT_FAILURE);
6346
	}
6347 6348 6349

	if (xtrabackup_copy_back || xtrabackup_move_back) {
		if (!check_if_param_set("datadir")) {
6350
			mysql_data_home = get_default_datadir();
6351 6352
		}
		if (!copy_back())
6353
			return(EXIT_FAILURE);
6354 6355 6356
	}

	if (xtrabackup_decrypt_decompress && !decrypt_decompress()) {
6357
		return(EXIT_FAILURE);
6358 6359
	}

6360
	return(EXIT_SUCCESS);
6361
}
6362 6363 6364 6365 6366


static int get_exepath(char *buf, size_t size, const char *argv0)
{
#ifdef _WIN32
6367
  DWORD ret = GetModuleFileNameA(NULL, buf, (DWORD)size);
6368 6369 6370 6371 6372 6373 6374 6375 6376 6377
  if (ret > 0)
    return 0;
#elif defined(__linux__)
  ssize_t ret = readlink("/proc/self/exe", buf, size-1);
  if(ret > 0)
    return 0;
#endif

  return my_realpath(buf, argv0, 0);
}
6378 6379 6380 6381 6382 6383 6384 6385 6386


#if defined (__SANITIZE_ADDRESS__) && defined (__linux__)
/* Avoid LeakSanitizer's false positives. */
const char* __asan_default_options()
{
  return "detect_leaks=0";
}
#endif