Skip to content

M
MariaDB
Commit ce4c56db authored by Vladislav Vaintroub's avatar Vladislav Vaintroub Committed by Sergei Golubchik
Browse files
Options

MDEV-9566 Port Percona Xtrabackup to MariaDB as mariabackup 

- Modify  backup code to work with MariaDB's 10.1 xtradb
- Remove  crypt encryption, version_check.pl, "compact backup"
- Add encryption plugin
parent d7714308
Hide whitespace changes
Inline Side-by-side
Showing with 949 additions and 3163 deletions
extra/mariabackup/CMakeLists.txt
View file @ ce4c56db
......@@ -13,65 +13,138 @@
# along with this program; if not, write to the Free Software
# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA
INCLUDE(gcrypt)
INCLUDE(curl)
INCLUDE(libev)
ADD_SUBDIRECTORY(libarchive)
ADD_SUBDIRECTORY(jsmn)
FIND_GCRYPT()
FIND_CURL()
FIND_EV()
# xxd is needed to embed version_check script
FIND_PROGRAM(XXD_PATH xxd)
IF(NOT XXD_PATH)
MESSAGE(FATAL_ERROR "xxd not found. Try to install vim-common.")
ENDIF(NOT XXD_PATH)
OPTION(WITH_MARIABACKUP "Include mariabackup" ON)
IF(NOT WITH_MARIABACKUP)
RETURN()
ENDIF()
IF(NOT WIN32)
CHECK_SYMBOL_EXISTS(regcomp regex.h HAVE_SYSTEM_REGEX)
IF(HAVE_SYSTEM_REGEX)
ADD_DEFINITIONS(-DHAVE_SYSTEM_REGEX)
ENDIF()
ENDIF()
IF(WITH_LIBARCHIVE STREQUAL "STATIC")
SET(CMAKE_FIND_LIBRARY_SUFFIXES .a .lib)
ENDIF()
FIND_PACKAGE(LibArchive)
IF(NOT DEFINED WITH_LIBARCHIVE)
IF(LibArchive_FOUND)
SET(WITH_LIBARCHIVE_DEFAULT ON)
ELSE()
SET(WITH_LIBARCHIVE_DEFAULT OFF)
ENDIF()
SET(WITH_LIBARCHIVE ${WITH_LIBARCHIVE_DEFAULT} CACHE STRING "Use libarchive for streaming features (ON, OFF or STATIC)" )
ENDIF()
IF(NOT WITH_LIBARCHIVE MATCHES "^(ON|OFF|STATIC)$")
MESSAGE(FATAL_ERROR "Invalid value for WITH_LIBARCHIVE: '${WITH_LIBARCHIVE}'. Use one of ON, OFF or STATIC")
ENDIF()
IF(UNIX)
SET(PIC_FLAG -fPIC)
ENDIF()
IF((NOT WITH_LIBARCHIVE STREQUAL "OFF") AND (NOT LibArchive_FOUND))
IF(CMAKE_VERSION VERSION_LESS "2.8.12")
MESSAGE("libarchive can't be built, old cmake")
ELSE()
# Build a local version
INCLUDE(ExternalProject)
SET(libarchive_PREFIX ${CMAKE_CURRENT_BINARY_DIR}/libarchive)
SET(libarchive_CMAKE_ARGS
-DCMAKE_INSTALL_PREFIX:PATH=<INSTALL_DIR>
-DCMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE}
-DENABLE_ICONV=OFF
-DENABLE_TAR=ON
-DENABLE_OPENSSL=OFF
-DENABLE_TEST=OFF
"-DCMAKE_C_FLAGS_DEBUG=${CMAKE_C_FLAGS_DEBUG} ${PIC_FLAG}"
"-DCMAKE_C_FLAGS_RELWITHDEBINFO=${CMAKE_C_FLAGS_RELWITHDEBINFO} ${PIC_FLAG}"
"-DCMAKE_C_FLAGS_RELEASE=${CMAKE_C_FLAGS_RELEASE} ${PIC_FLAG}"
"-DCMAKE_C_FLAGS_MINSIZEREL=${CMAKE_C_FLAGS_MINSIZEREL} ${PIC_FLAG}"
)
IF(WIN32)
SET(libarchive_CMAKE_ARGS ${libarchive_CMAKE_ARGS} -DWINDOWS_VERSION=WIN7 -DCMAKE_DEBUG_POSTFIX=d)
ENDIF()
SET(LIBARCHIVE_DIR ${CMAKE_CURRENT_BINARY_DIR}/libarchive)
ExternalProject_Add(libarchive
PREFIX ${libarchive_PREFIX}
DOWNLOAD_DIR ${LIBARCHIVE_DIR}
URL http://www.libarchive.org/downloads/libarchive-3.2.2.tar.gz
INSTALL_DIR ${LIBARCHIVE_DIR}
CMAKE_ARGS ${libarchive_CMAKE_ARGS}
)
ADD_LIBRARY(archive_static STATIC IMPORTED)
ADD_DEPENDENCIES(archive_static libarchive)
IF(WIN32)
SET(LIBARCHIVE_RELEASE_LIB ${LIBARCHIVE_DIR}/lib/${CMAKE_STATIC_LIBRARY_PREFIX}archive_static${CMAKE_STATIC_LIBRARY_SUFFIX})
SET(LIBARCHIVE_DEBUG_LIB ${LIBARCHIVE_DIR}/lib/${CMAKE_STATIC_LIBRARY_PREFIX}archive_staticd${CMAKE_STATIC_LIBRARY_SUFFIX})
SET_PROPERTY(TARGET archive_static PROPERTY IMPORTED_LOCATION_RELWITHDEBINFO ${LIBARCHIVE_RELEASE_LIB})
SET_PROPERTY(TARGET archive_static PROPERTY IMPORTED_LOCATION_RELEASE ${LIBARCHIVE_RELEASE_LIB})
SET_PROPERTY(TARGET archive_static PROPERTY IMPORTED_LOCATION_DEBUG ${LIBARCHIVE_DEBUG_LIB})
SET_PROPERTY(TARGET archive_static PROPERTY IMPORTED_LOCATION_MINSIZEREL ${LIBARCHIVE_RELEASE_LIB})
ELSE()
SET_PROPERTY(TARGET archive_static PROPERTY IMPORTED_LOCATION ${LIBARCHIVE_DIR}/lib/${CMAKE_STATIC_LIBRARY_PREFIX}archive${CMAKE_STATIC_LIBRARY_SUFFIX})
ENDIF()
SET(LibArchive_FOUND ON )
SET(LibArchive_INCLUDE_DIRS ${LIBARCHIVE_DIR}/include )
SET(LibArchive_LIBRARIES archive_static)
IF(WIN32)
SET(LIBARCHIVE_STATIC 1)
ENDIF()
ENDIF()
ENDIF()
IF(WITH_LIBARCHIVE AND LibArchive_FOUND)
ADD_DEFINITIONS(-DHAVE_LIBARCHIVE)
IF(LIBARCHIVE_STATIC)
ADD_DEFINITIONS(-DLIBARCHIVE_STATIC)
ENDIF()
INCLUDE_DIRECTORIES(${LibArchive_INCLUDE_DIRS})
LINK_LIBRARIES(${LibArchive_LIBRARIES})
SET(DS_ARCHIVE_SOURCE ds_archive.c)
ENDIF()
INCLUDE_DIRECTORIES(
${CMAKE_SOURCE_DIR}/include
${CMAKE_SOURCE_DIR}/storage/innobase/include
${CMAKE_SOURCE_DIR}/storage/xtradb/include
${CMAKE_SOURCE_DIR}/sql
${CMAKE_SOURCE_DIR}/storage/innobase/xtrabackup/src/libarchive/libarchive
${CMAKE_SOURCE_DIR}/storage/innobase/xtrabackup/src/quicklz
${CMAKE_SOURCE_DIR}/storage/innobase/xtrabackup/src/jsmn
${GCRYPT_INCLUDE_DIR}
${CURL_INCLUDE_DIRS}
${LIBEV_INCLUDE_DIRS}
${CMAKE_CURRENT_BINARY_DIR}
${CMAKE_CURRENT_SOURCE_DIR}/quicklz
${CMAKE_CURRENT_SOURCE_DIR}
)
ADD_DEFINITIONS(${SSL_DEFINES})
IF(NOT HAVE_SYSTEM_REGEX)
INCLUDE_DIRECTORIES(${CMAKE_SOURCE_DIR}/pcre)
ENDIF()
ADD_DEFINITIONS(-UMYSQL_SERVER)
########################################################################
# xtrabackup binary
########################################################################
CONFIGURE_FILE(${CMAKE_CURRENT_SOURCE_DIR}/xtrabackup_version.h.in
${CMAKE_CURRENT_BINARY_DIR}/xtrabackup_version.h )
ADD_CUSTOM_COMMAND(OUTPUT ${CMAKE_CURRENT_BINARY_DIR}/version_check_pl.h
COMMAND ${XXD_PATH} --include version_check.pl
${CMAKE_CURRENT_BINARY_DIR}/version_check_pl.h
WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR})
IF(WIN32)
SET(NT_SERVICE_SOURCE ${PROJECT_SOURCE_DIR}/sql/nt_servc.cc)
ELSE()
SET(NT_SERVICE_SOURCE)
ENDIF()
ADD_CUSTOM_TARGET(GenVersionCheck
DEPENDS ${CMAKE_CURRENT_BINARY_DIR}/version_check_pl.h)
ADD_DEFINITIONS(-DPCRE_STATIC=1)
SET_SOURCE_FILES_PROPERTIES(
xtrabackup.cc
backup_mysql.cc
PROPERTIES COMPILE_FLAGS -DMYSQL_CLIENT)
MYSQL_ADD_EXECUTABLE(xtrabackup
MYSQL_ADD_EXECUTABLE(mariabackup
xtrabackup.cc
innobackupex.cc
changed_page_bitmap.cc
compact.cc
datasink.c
ds_archive.c
${DS_ARCHIVE_SOURCE}
ds_buffer.c
ds_compress.c
ds_encrypt.c
......@@ -89,31 +162,29 @@ MYSQL_ADD_EXECUTABLE(xtrabackup
xbstream_write.c
backup_mysql.cc
backup_copy.cc
../../../../sql-common/client_authentication.cc
encryption_plugin.cc
${PROJECT_SOURCE_DIR}/libmysql/libmysql.c
${PROJECT_SOURCE_DIR}/sql/net_serv.cc
${NT_SERVICE_SOURCE}
COMPONENT backup
)
SET_TARGET_PROPERTIES(xtrabackup PROPERTIES ENABLE_EXPORTS TRUE)
TARGET_LINK_LIBRARIES(xtrabackup
mysqlserver
${GCRYPT_LIBS}
archive_static
)
# Export all symbols on Unix, for better crash callstacks
SET_TARGET_PROPERTIES(mariabackup PROPERTIES ENABLE_EXPORTS TRUE)
ADD_DEPENDENCIES(xtrabackup GenVersionCheck)
########################################################################
# innobackupex symlink
########################################################################
ADD_CUSTOM_COMMAND(TARGET xtrabackup
COMMAND ${CMAKE_COMMAND} ARGS -E create_symlink
xtrabackup innobackupex)
INSTALL(FILES ${CMAKE_CURRENT_BINARY_DIR}/innobackupex DESTINATION bin)
TARGET_LINK_LIBRARIES(mariabackup sql)
IF(NOT HAVE_SYSTEM_REGEX)
TARGET_LINK_LIBRARIES(mariabackup pcreposix)
ENDIF()
########################################################################
# xbstream binary
########################################################################
MYSQL_ADD_EXECUTABLE(xbstream
MYSQL_ADD_EXECUTABLE(mbstream
ds_buffer.c
ds_local.c
ds_stdout.c
......@@ -121,53 +192,15 @@ MYSQL_ADD_EXECUTABLE(xbstream
xbstream.c
xbstream_read.c
xbstream_write.c
)
SET_TARGET_PROPERTIES(xbstream
PROPERTIES LINKER_LANGUAGE CXX
)
TARGET_LINK_LIBRARIES(xbstream
mysys
mysys_ssl
)
########################################################################
# xbcrypt binary
########################################################################
MYSQL_ADD_EXECUTABLE(xbcrypt
xbcrypt.c
xbcrypt_common.c
xbcrypt_read.c
xbcrypt_write.c
COMPONENT backup
)
SET_TARGET_PROPERTIES(xbcrypt
PROPERTIES LINKER_LANGUAGE CXX
)
TARGET_LINK_LIBRARIES(xbcrypt
${GCRYPT_LIBS}
TARGET_LINK_LIBRARIES(mbstream
mysys
mysys_ssl
)
)
########################################################################
# xbcloud binary
########################################################################
MYSQL_ADD_EXECUTABLE(xbcloud
xbcloud.cc
)
SET_TARGET_PROPERTIES(xbcloud
PROPERTIES LINKER_LANGUAGE CXX
)
TARGET_LINK_LIBRARIES(xbcloud
${GCRYPT_LIBS}
${LIBEV_LIBRARIES}
${CURL_LIBRARIES}
mysys
mysys_ssl
jsmn
)
IF(MSVC)
SET_TARGET_PROPERTIES(mbstream PROPERTIES LINK_FLAGS setargv.obj)
ENDIF()
extra/mariabackup/backup_copy.cc
View file @ ce4c56db
/******************************************************
hot backup tool for InnoDB
(c) 2009-2015 Percona LLC and/or its affiliates
(c) 2017 MariaDB
Originally Created 3/3/2009 Yasufumi Kinoshita
Written by Alexey Kopytov, Aleksandr Kuzminsky, Stewart Smith, Vadim Tkachenko,
Yasufumi Kinoshita, Ignacio Nin and Baron Schwartz.
......@@ -48,13 +49,14 @@ Place, Suite 330, Boston, MA 02111-1307 USA
#include <set>
#include <string>
#include <mysqld.h>
#include <version_check_pl.h>
#include <sstream>
#include "fil_cur.h"
#include "xtrabackup.h"
#include "common.h"
#include "backup_copy.h"
#include "backup_mysql.h"
#include <btr0btr.h>
#include "xb0xb.h"
/* list of files to sync for --rsync mode */
......@@ -110,6 +112,7 @@ struct datadir_thread_ctxt_t {
bool ret;
};
static bool backup_files_from_datadir(const char *dir_path);
/************************************************************************
Retirn true if character if file separator */
......@@ -225,7 +228,7 @@ datadir_iter_next_database(datadir_iter_t *it)
it->dbdir = NULL;
}
while (fil_file_readdir_next_file(&it->err, it->datadir_path,
while (os_file_readdir_next_file(it->datadir_path,
it->dir, &it->dbinfo) == 0) {
ulint len;
......@@ -340,7 +343,7 @@ datadir_iter_next_file(datadir_iter_t *it)
return(false);
}
while (fil_file_readdir_next_file(&it->err, it->dbpath, it->dbdir,
while (os_file_readdir_next_file(it->dbpath, it->dbdir,
&it->fileinfo) == 0) {
if (it->fileinfo.type == OS_FILE_TYPE_DIR) {
......@@ -449,9 +452,9 @@ struct datafile_cur_t {
uint thread_n;
byte* orig_buf;
byte* buf;
ib_int64_t buf_size;
ib_int64_t buf_read;
ib_int64_t buf_offset;
size_t buf_size;
size_t buf_read;
size_t buf_offset;
};
static
......@@ -486,7 +489,7 @@ datafile_open(const char *file, datafile_cur_t *cursor, uint thread_n)
cursor->abs_path,
OS_FILE_OPEN,
OS_FILE_READ_ONLY,
&success);
&success, 0);
if (!success) {
/* The following call prints an error message */
os_file_get_last_error(TRUE);
......@@ -498,7 +501,7 @@ datafile_open(const char *file, datafile_cur_t *cursor, uint thread_n)
return(false);
}
if (my_fstat(cursor->file, &cursor->statinfo, MYF(MY_WME))) {
if (!my_stat(cursor->abs_path, &cursor->statinfo, 0)) {
msg("[%02u] error: cannot stat %s\n",
thread_n, cursor->abs_path);
......@@ -510,7 +513,7 @@ datafile_open(const char *file, datafile_cur_t *cursor, uint thread_n)
posix_fadvise(cursor->file, 0, 0, POSIX_FADV_SEQUENTIAL);
cursor->buf_size = 10 * 1024 * 1024;
cursor->buf = static_cast<byte *>(ut_malloc(cursor->buf_size));
cursor->buf = static_cast<byte *>(ut_malloc((ulint)cursor->buf_size));
return(true);
}
......@@ -525,7 +528,7 @@ datafile_read(datafile_cur_t *cursor)
xtrabackup_io_throttling();
to_read = min(cursor->statinfo.st_size - cursor->buf_offset,
to_read = (ulint)MY_MIN(cursor->statinfo.st_size - cursor->buf_offset,
cursor->buf_size);
if (to_read == 0) {
......@@ -603,6 +606,11 @@ ends_with(const char *str, const char *suffix)
&& strcmp(str + str_len - suffix_len, suffix) == 0);
}
static bool starts_with(const char *str, const char *prefix)
{
return strncmp(str, prefix, strlen(prefix)) == 0;
}
/************************************************************************
Create directories recursively.
@return 0 if directories created successfully. */
......@@ -644,6 +652,7 @@ Return true if first and second arguments are the same path. */
bool
equal_paths(const char *first, const char *second)
{
#ifdef HAVE_REALPATH
char real_first[PATH_MAX];
char real_second[PATH_MAX];
......@@ -655,6 +664,9 @@ equal_paths(const char *first, const char *second)
}
return (strcmp(real_first, real_second) == 0);
#else
return strcmp(first, second) == 0;
#endif
}
/************************************************************************
......@@ -675,10 +687,8 @@ directory_exists(const char *dir, bool create)
}
if (mkdirp(dir, 0777, MYF(0)) < 0) {
msg("Can not create directory %s: %s\n", dir,
my_strerror(errbuf, sizeof(errbuf), my_errno));
my_strerror(errbuf, sizeof(errbuf), my_errno);
msg("Can not create directory %s: %s\n", dir, errbuf);
return(false);
}
......@@ -688,9 +698,9 @@ directory_exists(const char *dir, bool create)
os_dir = os_file_opendir(dir, FALSE);
if (os_dir == NULL) {
my_strerror(errbuf, sizeof(errbuf), my_errno);
msg("Can not open directory %s: %s\n", dir,
my_strerror(errbuf, sizeof(errbuf), my_errno));
errbuf);
return(false);
}
......@@ -1054,16 +1064,17 @@ move_file(ds_ctxt_t *datasink,
dst_file_path, thread_n);
msg_ts("[%02u] Removing %s\n", thread_n, src_file_path);
if (unlink(src_file_path) != 0) {
my_strerror(errbuf, sizeof(errbuf), errno);
msg("Error: unlink %s failed: %s\n",
src_file_path,
my_strerror(errbuf,
sizeof(errbuf), errno));
errbuf);
}
return(ret);
}
my_strerror(errbuf, sizeof(errbuf), my_errno);
msg("Can not move file %s to %s: %s\n",
src_file_path, dst_file_path_abs,
my_strerror(errbuf, sizeof(errbuf), my_errno));
errbuf);
return(false);
}
......@@ -1360,6 +1371,10 @@ backup_start()
return(false);
}
if (!backup_files_from_datadir(fil_path_to_mysql_datadir)) {
return false;
}
// There is no need to stop slave thread before coping non-Innodb data when
// --no-lock option is used because --no-lock option requires that no DDL or
// DML to non-transaction tables can occur.
......@@ -1578,7 +1593,11 @@ ibx_cleanup_full_backup()
while (datadir_iter_next(it, &node)) {
if (node.is_empty_dir) {
#ifdef _WIN32
DeleteFile(node.filepath);
#else
rmdir(node.filepath);
#endif
}
if (xtrabackup_incremental && !node.is_empty_dir
......@@ -1605,6 +1624,9 @@ apply_log_finish()
return(true);
}
extern void
os_io_init_simple(void);
bool
copy_back()
{
......@@ -1661,7 +1683,7 @@ copy_back()
}
srv_max_n_threads = 1000;
os_sync_mutex = NULL;
//os_sync_mutex = NULL;
ut_mem_init();
/* temporally dummy value to avoid crash */
srv_page_size_shift = 14;
......@@ -1764,10 +1786,9 @@ copy_back()
if (mkdirp(path, 0777, MYF(0)) < 0) {
char errbuf[MYSYS_STRERROR_SIZE];
my_strerror(errbuf, sizeof(errbuf), my_errno);
msg("Can not create directory %s: %s\n",
path, my_strerror(errbuf,
sizeof(errbuf), my_errno));
path, errbuf);
ret = false;
goto cleanup;
......@@ -1855,13 +1876,12 @@ copy_back()
ds_data = NULL;
sync_close();
sync_initialized = FALSE;
os_sync_free();
//os_sync_free();
mem_close();
os_sync_mutex = NULL;
//os_sync_mutex = NULL;
ut_free_all_mem();
sync_close();
sync_initialized = FALSE;
return(ret);
}
......@@ -1872,7 +1892,7 @@ decrypt_decompress_file(const char *filepath, uint thread_n)
char *dest_filepath = strdup(filepath);
bool needs_action = false;
cmd << "cat " << filepath;
cmd << IF_WIN("type ","cat ") << filepath;
if (ends_with(filepath, ".xbcrypt") && opt_decrypt) {
cmd << " | xbcrypt --decrypt --encrypt-algo="
......@@ -1926,7 +1946,7 @@ decrypt_decompress_file(const char *filepath, uint thread_n)
}
static
os_thread_ret_t
os_thread_ret_t STDCALL
decrypt_decompress_thread_func(void *arg)
{
bool ret = true;
......@@ -1974,7 +1994,7 @@ decrypt_decompress()
datadir_iter_t *it = NULL;
srv_max_n_threads = 1000;
os_sync_mutex = NULL;
//os_sync_mutex = NULL;
ut_mem_init();
os_sync_init();
sync_init();
......@@ -2008,40 +2028,43 @@ decrypt_decompress()
sync_close();
sync_initialized = FALSE;
os_sync_free();
os_sync_mutex = NULL;
//os_sync_free();
//os_sync_mutex = NULL;
ut_free_all_mem();
return(ret);
}
void
version_check()
/*
Copy some files from top level datadir.
Do not copy the Innodb files (ibdata1, redo log files),
as this is done in a separate step.
*/
static bool backup_files_from_datadir(const char *dir_path)
{
if (opt_password != NULL) {
setenv("option_mysql_password", opt_password, 1);
}
if (opt_user != NULL) {
setenv("option_mysql_user", opt_user, 1);
}
if (opt_host != NULL) {
setenv("option_mysql_host", opt_host, 1);
}
if (opt_socket != NULL) {
setenv("option_mysql_socket", opt_socket, 1);
}
if (opt_port != 0) {
char port[20];
snprintf(port, sizeof(port), "%u", opt_port);
setenv("option_mysql_port", port, 1);
}
FILE *pipe = popen("perl", "w");
if (pipe == NULL) {
return;
}
os_file_dir_t dir = os_file_opendir(dir_path, TRUE);
os_file_stat_t info;
bool ret = true;
while (os_file_readdir_next_file(dir_path, dir, &info) == 0) {
fputs((const char *)version_check_pl, pipe);
if (info.type != OS_FILE_TYPE_FILE)
continue;
pclose(pipe);
const char *pname = strrchr(info.name, IF_WIN('\\', '/'));
if (!pname)
pname = info.name;
/* Copy aria log files, and aws keys for encryption plugins.*/
const char *prefixes[] = { "aria_log", "aws-kms-key" };
for (size_t i = 0; i < array_elements(prefixes); i++) {
if (starts_with(pname, prefixes[i])) {
ret = copy_file(ds_data, info.name, info.name, 1);
if (!ret) {
break;
}
}
}
}
os_file_closedir(dir);
return ret;
}
extra/mariabackup/backup_copy.h
View file @ ce4c56db
......@@ -41,8 +41,6 @@ bool
copy_back();
bool
decrypt_decompress();
void
version_check();
bool
is_path_separator(char);
bool
......
extra/mariabackup/backup_mysql.cc
View file @ ce4c56db
......@@ -38,6 +38,7 @@ this program; if not, write to the Free Software Foundation, Inc., 59 Temple
Place, Suite 330, Boston, MA 02111-1307 USA
*******************************************************/
#define MYSQL_CLIENT
#include <my_global.h>
#include <mysql.h>
......@@ -47,10 +48,12 @@ Place, Suite 330, Boston, MA 02111-1307 USA
#include <limits>
#include "common.h"
#include "xtrabackup.h"
#include "xtrabackup_version.h"
#include "mysql_version.h"
#include "backup_copy.h"
#include "backup_mysql.h"
#include "mysqld.h"
#include "encryption_plugin.h"
#include <sstream>
char *tool_name;
......@@ -88,15 +91,7 @@ time_t history_lock_time;
MYSQL *mysql_connection;
extern "C" {
MYSQL * STDCALL
cli_mysql_real_connect(MYSQL *mysql,const char *host, const char *user,
const char *passwd, const char *db,
uint port, const char *unix_socket,ulong client_flag);
}
#define mysql_real_connect cli_mysql_real_connect
my_bool opt_ssl_verify_server_cert;
MYSQL *
xb_mysql_connect()
......@@ -136,11 +131,6 @@ xb_mysql_connect()
}
mysql_options(connection,MYSQL_OPT_SSL_VERIFY_SERVER_CERT,
(char*)&opt_ssl_verify_server_cert);
#if !defined(HAVE_YASSL)
if (opt_server_public_key && *opt_server_public_key)
mysql_options(connection, MYSQL_SERVER_PUBLIC_KEY,
opt_server_public_key);
#endif
#endif
if (!mysql_real_connect(connection,
......@@ -310,7 +300,7 @@ check_server_version(unsigned long version_number,
version_supported = version_supported
|| (version_number > 50500 && version_number < 50700);
version_supported = version_supported
|| ((version_number > 100000 && version_number < 100300)
|| ((version_number > 100000)
&& server_flavor == FLAVOR_MARIADB);
if (mysql51 && innodb_version == NULL) {
......@@ -596,7 +586,7 @@ select_incremental_lsn_from_history(lsn_t *incremental_lsn)
if (opt_incremental_history_name) {
mysql_real_escape_string(mysql_connection, buf,
opt_incremental_history_name,
strlen(opt_incremental_history_name));
(unsigned long)strlen(opt_incremental_history_name));
ut_snprintf(query, sizeof(query),
"SELECT innodb_to_lsn "
"FROM PERCONA_SCHEMA.xtrabackup_history "
......@@ -609,7 +599,7 @@ select_incremental_lsn_from_history(lsn_t *incremental_lsn)
if (opt_incremental_history_uuid) {
mysql_real_escape_string(mysql_connection, buf,
opt_incremental_history_uuid,
strlen(opt_incremental_history_uuid));
(unsigned long)strlen(opt_incremental_history_uuid));
ut_snprintf(query, sizeof(query),
"SELECT innodb_to_lsn "
"FROM PERCONA_SCHEMA.xtrabackup_history "
......@@ -756,7 +746,7 @@ have_queries_to_wait_for(MYSQL *connection, uint threshold)
static
void
kill_long_queries(MYSQL *connection, uint timeout)
kill_long_queries(MYSQL *connection, time_t timeout)
{
MYSQL_RES *result;
MYSQL_ROW row;
......@@ -768,15 +758,15 @@ kill_long_queries(MYSQL *connection, uint timeout)
all_queries = (opt_kill_long_query_type == QUERY_TYPE_ALL);
while ((row = mysql_fetch_row(result)) != NULL) {
const char *info = row[7];
int duration = atoi(row[5]);
long long duration = atoll(row[5]);
char *id = row[0];
if (info != NULL &&
duration >= (int)timeout &&
(time_t)duration >= timeout &&
((all_queries && is_query(info)) ||
is_select_query(info))) {
msg_ts("Killing query %s (duration %d sec): %s\n",
id, duration, info);
id, (int)duration, info);
ut_snprintf(kill_stmt, sizeof(kill_stmt),
"KILL %s", id);
xb_mysql_query(connection, kill_stmt, false, false);
......@@ -1378,7 +1368,18 @@ write_binlog_info(MYSQL *connection)
return(result);
}
static string escape_and_quote(MYSQL *mysql,const char *str)
{
if (!str)
return "NULL";
size_t len = strlen(str);
char* escaped = (char *)alloca(2 * len + 3);
escaped[0] = '\'';
size_t new_len = mysql_real_escape_string(mysql, escaped+1, str, len);
escaped[new_len + 1] = '\'';
escaped[new_len + 2] = 0;
return string(escaped);
}
/*********************************************************************//**
Writes xtrabackup_info file and if backup_history is enable creates
......@@ -1388,25 +1389,16 @@ backup. */
bool
write_xtrabackup_info(MYSQL *connection)
{
MYSQL_STMT *stmt;
MYSQL_BIND bind[19];
char *uuid = NULL;
char *server_version = NULL;
char buf_start_time[100];
char buf_end_time[100];
int idx;
tm tm;
my_bool null = TRUE;
ostringstream oss;
const char *xb_stream_name[] = {"file", "tar", "xbstream"};
const char *ins_query = "insert into PERCONA_SCHEMA.xtrabackup_history("
"uuid, name, tool_name, tool_command, tool_version, "
"ibbackup_version, server_version, start_time, end_time, "
"lock_time, binlog_pos, innodb_from_lsn, innodb_to_lsn, "
"partial, incremental, format, compact, compressed, "
"encrypted) "
"values(?,?,?,?,?,?,?,from_unixtime(?),from_unixtime(?),"
"?,?,?,?,?,?,?,?,?,?)";
ut_ad(xtrabackup_stream_fmt < 3);
......@@ -1419,6 +1411,11 @@ write_xtrabackup_info(MYSQL *connection)
localtime_r(&history_end_time, &tm);
strftime(buf_end_time, sizeof(buf_end_time),
"%Y-%m-%d %H:%M:%S", &tm);
bool is_partial = (xtrabackup_tables
|| xtrabackup_tables_file
|| xtrabackup_databases
|| xtrabackup_databases_file);
backup_file_printf(XTRABACKUP_INFO,
"uuid = %s\n"
"name = %s\n"
......@@ -1443,23 +1440,20 @@ write_xtrabackup_info(MYSQL *connection)
opt_history ? opt_history : "", /* name */
tool_name, /* tool_name */
tool_args, /* tool_command */
XTRABACKUP_VERSION, /* tool_version */
XTRABACKUP_VERSION, /* ibbackup_version */
MYSQL_SERVER_VERSION, /* tool_version */
MYSQL_SERVER_VERSION, /* ibbackup_version */
server_version, /* server_version */
buf_start_time, /* start_time */
buf_end_time, /* end_time */
history_lock_time, /* lock_time */
(int)history_lock_time, /* lock_time */
mysql_binlog_position ?
mysql_binlog_position : "", /* binlog_pos */
incremental_lsn, /* innodb_from_lsn */
metadata_to_lsn, /* innodb_to_lsn */
(xtrabackup_tables /* partial */
|| xtrabackup_tables_file
|| xtrabackup_databases
|| xtrabackup_databases_file) ? "Y" : "N",
is_partial? "Y" : "N",
xtrabackup_incremental ? "Y" : "N", /* incremental */
xb_stream_name[xtrabackup_stream_fmt], /* format */
xtrabackup_compact ? "Y" : "N", /* compact */
"N", /* compact */
xtrabackup_compress ? "compressed" : "N", /* compressed */
xtrabackup_encrypt ? "Y" : "N"); /* encrypted */
......@@ -1492,142 +1486,36 @@ write_xtrabackup_info(MYSQL *connection)
"encrypted ENUM('Y', 'N') DEFAULT NULL"
") CHARACTER SET utf8 ENGINE=innodb", false);
stmt = mysql_stmt_init(connection);
mysql_stmt_prepare(stmt, ins_query, strlen(ins_query));
memset(bind, 0, sizeof(bind));
idx = 0;
/* uuid */
bind[idx].buffer_type = MYSQL_TYPE_STRING;
bind[idx].buffer = uuid;
bind[idx].buffer_length = strlen(uuid);
++idx;
/* name */
bind[idx].buffer_type = MYSQL_TYPE_STRING;
bind[idx].buffer = (char*)(opt_history);
bind[idx].buffer_length = strlen(opt_history);
if (!(opt_history && *opt_history)) {
bind[idx].is_null = &null;
}
++idx;
/* tool_name */
bind[idx].buffer_type = MYSQL_TYPE_STRING;
bind[idx].buffer = tool_name;
bind[idx].buffer_length = strlen(tool_name);
++idx;
/* tool_command */
bind[idx].buffer_type = MYSQL_TYPE_STRING;
bind[idx].buffer = tool_args;
bind[idx].buffer_length = strlen(tool_args);
++idx;
/* tool_version */
bind[idx].buffer_type = MYSQL_TYPE_STRING;
bind[idx].buffer = (char*)(XTRABACKUP_VERSION);
bind[idx].buffer_length = strlen(XTRABACKUP_VERSION);
++idx;
/* ibbackup_version */
bind[idx].buffer_type = MYSQL_TYPE_STRING;
bind[idx].buffer = (char*)(XTRABACKUP_VERSION);
bind[idx].buffer_length = strlen(XTRABACKUP_VERSION);
++idx;
/* server_version */
bind[idx].buffer_type = MYSQL_TYPE_STRING;
bind[idx].buffer = server_version;
bind[idx].buffer_length = strlen(server_version);
++idx;
/* start_time */
bind[idx].buffer_type = MYSQL_TYPE_LONG;
bind[idx].buffer = &history_start_time;
++idx;
/* end_time */
bind[idx].buffer_type = MYSQL_TYPE_LONG;
bind[idx].buffer = &history_end_time;
++idx;
/* lock_time */
bind[idx].buffer_type = MYSQL_TYPE_LONG;
bind[idx].buffer = &history_lock_time;
++idx;
/* binlog_pos */
bind[idx].buffer_type = MYSQL_TYPE_STRING;
bind[idx].buffer = mysql_binlog_position;
if (mysql_binlog_position != NULL) {
bind[idx].buffer_length = strlen(mysql_binlog_position);
} else {
bind[idx].is_null = &null;
}
++idx;
/* innodb_from_lsn */
bind[idx].buffer_type = MYSQL_TYPE_LONGLONG;
bind[idx].buffer = (char*)(&incremental_lsn);
++idx;
/* innodb_to_lsn */
bind[idx].buffer_type = MYSQL_TYPE_LONGLONG;
bind[idx].buffer = (char*)(&metadata_to_lsn);
++idx;
/* partial (Y | N) */
bind[idx].buffer_type = MYSQL_TYPE_STRING;
bind[idx].buffer = (char*)((xtrabackup_tables
|| xtrabackup_tables_file
|| xtrabackup_databases
|| xtrabackup_databases_file) ? "Y" : "N");
bind[idx].buffer_length = 1;
++idx;
/* incremental (Y | N) */
bind[idx].buffer_type = MYSQL_TYPE_STRING;
bind[idx].buffer = (char*)(
(xtrabackup_incremental
|| xtrabackup_incremental_basedir
|| opt_incremental_history_name
|| opt_incremental_history_uuid) ? "Y" : "N");
bind[idx].buffer_length = 1;
++idx;
/* format (file | tar | xbstream) */
bind[idx].buffer_type = MYSQL_TYPE_STRING;
bind[idx].buffer = (char*)(xb_stream_name[xtrabackup_stream_fmt]);
bind[idx].buffer_length = strlen(xb_stream_name[xtrabackup_stream_fmt]);
++idx;
/* compact (Y | N) */
bind[idx].buffer_type = MYSQL_TYPE_STRING;
bind[idx].buffer = (char*)(xtrabackup_compact ? "Y" : "N");
bind[idx].buffer_length = 1;
++idx;
/* compressed (Y | N) */
bind[idx].buffer_type = MYSQL_TYPE_STRING;
bind[idx].buffer = (char*)(xtrabackup_compress ? "Y" : "N");
bind[idx].buffer_length = 1;
++idx;
/* encrypted (Y | N) */
bind[idx].buffer_type = MYSQL_TYPE_STRING;
bind[idx].buffer = (char*)(xtrabackup_encrypt ? "Y" : "N");
bind[idx].buffer_length = 1;
++idx;
ut_ad(idx == 19);
mysql_stmt_bind_param(stmt, bind);
mysql_stmt_execute(stmt);
mysql_stmt_close(stmt);
#define ESCAPE_BOOL(expr) ((expr)?"'Y'":"'N'")
oss << "insert into PERCONA_SCHEMA.xtrabackup_history("
<< "uuid, name, tool_name, tool_command, tool_version,"
<< "ibbackup_version, server_version, start_time, end_time,"
<< "lock_time, binlog_pos, innodb_from_lsn, innodb_to_lsn,"
<< "partial, incremental, format, compact, compressed, "
<< "encrypted) values("
<< escape_and_quote(connection, uuid) << ","
<< escape_and_quote(connection, opt_history) << ","
<< escape_and_quote(connection, tool_name) << ","
<< escape_and_quote(connection, tool_args) << ","
<< escape_and_quote(connection, MYSQL_SERVER_VERSION) << ","
<< escape_and_quote(connection, MYSQL_SERVER_VERSION) << ","
<< escape_and_quote(connection, server_version) << ","
<< "from_unixtime(" << history_start_time << "),"
<< "from_unixtime(" << history_end_time << "),"
<< history_lock_time << ","
<< escape_and_quote(connection, mysql_binlog_position) << ","
<< incremental_lsn << ","
<< metadata_to_lsn << ","
<< ESCAPE_BOOL(is_partial) << ","
<< ESCAPE_BOOL(xtrabackup_incremental)<< ","
<< escape_and_quote(connection,xb_stream_name[xtrabackup_stream_fmt]) <<","
<< ESCAPE_BOOL(false) << ","
<< ESCAPE_BOOL(xtrabackup_compress) << ","
<< ESCAPE_BOOL(xtrabackup_encrypt) <<")";
xb_mysql_query(mysql_connection, oss.str().c_str(), false);
cleanup:
......@@ -1637,10 +1525,11 @@ write_xtrabackup_info(MYSQL *connection)
return(true);
}
bool
write_backup_config_file()
extern const char *innodb_checksum_algorithm_names[];
bool write_backup_config_file()
{
return backup_file_printf("backup-my.cnf",
int rc= backup_file_printf("backup-my.cnf",
"# This MySQL options file was generated by innobackupex.\n\n"
"# The MySQL server\n"
"[mysqld]\n"
......@@ -1649,19 +1538,18 @@ write_backup_config_file()
"innodb_data_file_path=%s\n"
"innodb_log_files_in_group=%lu\n"
"innodb_log_file_size=%lld\n"
"innodb_fast_checksum=%s\n"
"innodb_page_size=%lu\n"
"innodb_log_block_size=%lu\n"
"innodb_undo_directory=%s\n"
"innodb_undo_tablespaces=%lu\n"
"%s%s\n"
"%s%s\n",
"%s%s\n"
"%s\n",
innodb_checksum_algorithm_names[srv_checksum_algorithm],
innodb_checksum_algorithm_names[srv_log_checksum_algorithm],
innobase_data_file_path,
srv_n_log_files,
innobase_log_file_size,
srv_fast_checksum ? "true" : "false",
srv_page_size,
srv_log_block_size,
srv_undo_dir,
......@@ -1671,7 +1559,9 @@ write_backup_config_file()
innobase_buffer_pool_filename ?
"innodb_buffer_pool_filename=" : "",
innobase_buffer_pool_filename ?
innobase_buffer_pool_filename : "");
innobase_buffer_pool_filename : "",
encryption_plugin_get_config());
return rc;
}
......
extra/mariabackup/changed_page_bitmap.cc
View file @ ce4c56db
......@@ -447,7 +447,7 @@ log_online_open_bitmap_file_read_only(
= os_file_create_simple_no_error_handling(0, bitmap_file->name,
OS_FILE_OPEN,
OS_FILE_READ_ONLY,
&success);
&success,0);
if (UNIV_UNLIKELY(!success)) {
/* Here and below assume that bitmap file names do not
......
extra/mariabackup/common.h
View file @ ce4c56db
......@@ -26,6 +26,46 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA
#include <fcntl.h>
#include <stdarg.h>
# define fil_is_user_tablespace_id(i) ((i) > srv_undo_tablespaces_open)
#ifdef _MSC_VER
#define stat _stati64
#define PATH_MAX MAX_PATH
#endif
#ifndef HAVE_VASPRINTF
static inline int vasprintf(char **strp, const char *fmt, va_list args)
{
int len;
#ifdef _MSC_VER
len = _vscprintf(fmt, args);
#else
len = vsnprintf(NULL, 0, fmt, args);
#endif
if (len < 0)
{
return -1;
}
*strp = (char *)malloc(len + 1);
if (!*strp)
{
return -1;
}
vsprintf(*strp, fmt, args);
return len;
}
static inline int asprintf(char **strp, const char *fmt,...)
{
va_list args;
va_start(args, fmt);
int len = vasprintf(strp, fmt, args);
va_end(args);
return len;
}
#endif
#define xb_a(expr) \
do { \
if (!(expr)) { \
......@@ -93,15 +133,15 @@ static inline int msg_ts(const char *fmt, ...)
/***********************************************************************
Computes bit shift for a given value. If the argument is not a power
of 2, returns 0.*/
static inline ulong
get_bit_shift(ulong value)
static inline size_t
get_bit_shift(size_t value)
{
ulong shift;
size_t shift;
if (value == 0)
return 0;
for (shift = 0; !(value & 1UL); shift++) {
for (shift = 0; !(value & 1); shift++) {
value >>= 1;
}
return (value >> 1) ? 0 : shift;
......
extra/mariabackup/compact.cc deleted 100644 → 0
View file @ d7714308
/******************************************************
XtraBackup: hot backup tool for InnoDB
(c) 2009-2014 Percona LLC and/or its affiliates.
Originally Created 3/3/2009 Yasufumi Kinoshita
Written by Alexey Kopytov, Aleksandr Kuzminsky, Stewart Smith, Vadim Tkachenko,
Yasufumi Kinoshita, Ignacio Nin and Baron Schwartz.
This program is free software; you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation; version 2 of the License.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program; if not, write to the Free Software
Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA
*******************************************************/
/* Compact backups implementation */
#include <my_base.h>
#include <table.h>
#include <univ.i>
#include <dict0mem.h>
#include <dict0priv.h>
#include <fsp0fsp.h>
#include <handler0alter.h>
#include <ibuf0ibuf.h>
#include <page0page.h>
#include <row0merge.h>
#include "common.h"
#include "write_filt.h"
#include "fil_cur.h"
#include "xtrabackup.h"
#include "ds_buffer.h"
#include "xb0xb.h"
/* Number of the first primary key page in an .ibd file */
#define XB_FIRST_CLUSTERED_INDEX_PAGE_NO 3
/* Suffix for page map files */
#define XB_PAGE_MAP_SUFFIX ".pmap"
#define XB_TMPFILE_SUFFIX ".tmp"
/* Page range */
struct page_range_t {
ulint from; /*!< range start */
ulint to; /*!< range end */
};
/* Cursor in a page map file */
struct page_map_cursor_t {
File fd; /*!< file descriptor */
IO_CACHE cache; /*!< IO_CACHE associated with fd */
};
/* Table descriptor for the index rebuild operation */
struct index_rebuild_table_t {
char* name; /* table name */
ulint space_id; /* space ID */
UT_LIST_NODE_T(index_rebuild_table_t) list; /* list node */
};
/* Thread descriptor for the index rebuild operation */
struct index_rebuild_thread_t {
ulint num; /* thread number */
pthread_t id; /* thread ID */
};
/* Empty page use to replace skipped pages in the data files */
static byte empty_page[UNIV_PAGE_SIZE_MAX];
static const char compacted_page_magic[] = "COMPACTP";
static const size_t compacted_page_magic_size =
sizeof(compacted_page_magic) - 1;
static const ulint compacted_page_magic_offset = FIL_PAGE_DATA;
/* Mutex protecting table_list */
static pthread_mutex_t table_list_mutex;
/* List of tablespaces to process by the index rebuild operation */
static UT_LIST_BASE_NODE_T(index_rebuild_table_t) table_list;
/************************************************************************
Compact page filter. */
static my_bool wf_compact_init(xb_write_filt_ctxt_t *ctxt, char *dst_name,
xb_fil_cur_t *cursor);
static my_bool wf_compact_process(xb_write_filt_ctxt_t *ctxt,
ds_file_t *dstfile);
static my_bool wf_compact_finalize(xb_write_filt_ctxt_t *ctxt,
ds_file_t *dstfile);
xb_write_filt_t wf_compact = {
&wf_compact_init,
&wf_compact_process,
&wf_compact_finalize,
NULL
};
/************************************************************************
Initialize the compact page filter.
@return TRUE on success, FALSE on error. */
static my_bool
wf_compact_init(xb_write_filt_ctxt_t *ctxt,
char *dst_name __attribute__((unused)), xb_fil_cur_t *cursor)
{
xb_wf_compact_ctxt_t *cp = &(ctxt->u.wf_compact_ctxt);
char page_map_name[FN_REFLEN];
MY_STAT mystat;
ctxt->cursor = cursor;
cp->clustered_index_found = FALSE;
cp->inside_skipped_range = FALSE;
cp->free_limit = 0;
/* Don't compact the system table space */
cp->skip = cursor->is_system;
if (cp->skip) {
return(TRUE);
}
snprintf(page_map_name, sizeof(page_map_name), "%s%s", dst_name,
XB_PAGE_MAP_SUFFIX);
cp->ds_buffer = ds_create(xtrabackup_target_dir, DS_TYPE_BUFFER);
if (cp->ds_buffer == NULL) {
return(FALSE);
}
ds_set_pipe(cp->ds_buffer, ds_meta);
memset(&mystat, 0, sizeof(mystat));
mystat.st_mtime = my_time(0);
cp->buffer = ds_open(cp->ds_buffer, page_map_name, &mystat);
if (cp->buffer == NULL) {
msg("xtrabackup: Error: cannot open output stream for %s\n",
page_map_name);
return(FALSE);
}
return(TRUE);
}
/************************************************************************
Check if the specified page should be skipped. We currently skip all
non-clustered index pages for compact backups.
@return TRUE if the page should be skipped. */
static my_bool
check_if_skip_page(xb_wf_compact_ctxt_t *cp, xb_fil_cur_t *cursor, ulint offset)
{
byte *page;
ulint page_no;
ulint page_type;
index_id_t index_id;
xb_ad(cursor->is_system == FALSE);
page = cursor->buf + cursor->page_size * offset;
page_no = cursor->buf_page_no + offset;
page_type = fil_page_get_type(page);
if (UNIV_UNLIKELY(page_no == 0)) {
cp->free_limit = mach_read_from_4(page + FSP_HEADER_OFFSET +
FSP_FREE_LIMIT);
} else if (UNIV_UNLIKELY(page_no == XB_FIRST_CLUSTERED_INDEX_PAGE_NO)) {
xb_ad(cp->clustered_index_found == FALSE);
if (page_type != FIL_PAGE_INDEX) {
/* Uninitialized clustered index root page, there's
nothing we can do to compact the space.*/
msg("[%02u] Uninitialized page type value (%lu) in the "
"clustered index root page of tablespace %s. "
"Will not be compacted.\n",
cursor->thread_n,
page_type, cursor->rel_path);
cp->skip = TRUE;
return(FALSE);
}
cp->clustered_index =
mach_read_from_8(page + PAGE_HEADER + PAGE_INDEX_ID);
cp->clustered_index_found = TRUE;
} else if (UNIV_UNLIKELY(page_no >= cp->free_limit)) {
/* Skip unused pages above free limit, if that value is set in
the FSP header.*/
return(cp->free_limit > 0);
} else if (cp->clustered_index_found && page_type == FIL_PAGE_INDEX) {
index_id = mach_read_from_8(page + PAGE_HEADER + PAGE_INDEX_ID);
if (index_id != cp->clustered_index) {
ulint fseg_hdr_space =
mach_read_from_4(page + PAGE_HEADER +
PAGE_BTR_SEG_TOP);
ulint fseg_hdr_page_no =
mach_read_from_4(page + PAGE_HEADER +
PAGE_BTR_SEG_TOP + 4);
ulint fseg_hdr_offset =
mach_read_from_2(page + PAGE_HEADER +
PAGE_BTR_SEG_TOP + 8);
/* Don't skip root index pages, i.e. the ones where the
above fields are defined. We need root index pages to be
able to correctly drop the indexes later, as they
contain fseg inode pointers. */
return(fseg_hdr_space == 0 &&
fseg_hdr_page_no == 0 &&
fseg_hdr_offset == 0);
}
}
return(FALSE);
}
/************************************************************************
Run the next batch of pages through the compact page filter.
@return TRUE on success, FALSE on error. */
static my_bool
wf_compact_process(xb_write_filt_ctxt_t *ctxt, ds_file_t *dstfile)
{
xb_fil_cur_t *cursor = ctxt->cursor;
ulint page_size = cursor->page_size;
byte *page;
byte *buf_end;
byte *write_from;
xb_wf_compact_ctxt_t *cp = &(ctxt->u.wf_compact_ctxt);
ulint i;
ulint page_no;
byte tmp[4];
if (cp->skip) {
return(!ds_write(dstfile, cursor->buf, cursor->buf_read));
}
write_from = NULL;
buf_end = cursor->buf + cursor->buf_read;
for (i = 0, page = cursor->buf; page < buf_end;
i++, page += page_size) {
page_no = cursor->buf_page_no + i;
if (!check_if_skip_page(cp, cursor, i)) {
if (write_from == NULL) {
write_from = page;
}
if (cp->inside_skipped_range) {
cp->inside_skipped_range = FALSE;
/* Write the last range endpoint to the
skipped pages map */
xb_ad(page_no > 0);
mach_write_to_4(tmp, page_no - 1);
if (ds_write(cp->buffer, tmp, sizeof(tmp))) {
return(FALSE);
}
}
continue;
}
if (write_from != NULL) {
/* The first skipped page in this block, write the
non-skipped ones to the data file */
if (ds_write(dstfile, write_from, page - write_from)) {
return(FALSE);
}
write_from = NULL;
}
if (!cp->inside_skipped_range) {
/* The first skipped page in range, write the first
range endpoint to the skipped pages map */
cp->inside_skipped_range = TRUE;
mach_write_to_4(tmp, page_no);
if (ds_write(cp->buffer, tmp, sizeof(tmp))) {
return(FALSE);
}
}
}
/* Write the remaining pages in the buffer, if any */
if (write_from != NULL &&
ds_write(dstfile, write_from, buf_end - write_from)) {
return(FALSE);
}
return(TRUE);
}
/************************************************************************
Close the compact filter's page map stream.
@return TRUE on success, FALSE on error. */
static my_bool
wf_compact_finalize(xb_write_filt_ctxt_t *ctxt,
ds_file_t *dstfile __attribute__((unused)))
{
xb_fil_cur_t *cursor = ctxt->cursor;
xb_wf_compact_ctxt_t *cp = &(ctxt->u.wf_compact_ctxt);
my_bool rc = TRUE;
/* Write the last endpoint of the current range, if the last pages of
the space have been skipped. */
if (cp->inside_skipped_range) {
byte tmp[4];
mach_write_to_4(tmp, cursor->space_size - 1);
if (ds_write(cp->buffer, tmp, sizeof(tmp))) {
return(FALSE);
}
cp->inside_skipped_range = FALSE;
}
if (cp->buffer) {
if (ds_close(cp->buffer)) {
rc = FALSE;
}
}
if (cp->ds_buffer) {
ds_destroy(cp->ds_buffer);
}
return(rc);
}
/************************************************************************
Open a page map file and return a cursor.
@return page map cursor, or NULL if the file doesn't exist. */
static page_map_cursor_t *
page_map_file_open(const char *path)
{
MY_STAT statinfo;
page_map_cursor_t *pmap_cur;
int rc;
if (my_stat(path, &statinfo, MYF(0)) == NULL) {
return(NULL);
}
/* The maximum possible page map file corresponds to a 64 TB tablespace
and the worst case when every other page was skipped. That is, 2^32/2
page ranges = 16 GB. */
xb_a(statinfo.st_size < (off_t) 16 * 1024 * 1024 * 1024);
/* Must be a series of 8-byte tuples */
xb_a(statinfo.st_size % 8 == 0);
pmap_cur = (page_map_cursor_t *) my_malloc(sizeof(page_map_cursor_t),
MYF(MY_FAE));
pmap_cur->fd = my_open(path, O_RDONLY, MYF(MY_WME));
xb_a(pmap_cur->fd != 0);
rc = init_io_cache(&pmap_cur->cache, pmap_cur->fd, 0, READ_CACHE,
0, 0, MYF(MY_WME));
xb_a(rc == 0);
return(pmap_cur);
}
/************************************************************************
Read the next range from a page map file and update the cursor.
@return TRUE on success, FALSE on end-of-file. */
static ibool
page_map_file_next(page_map_cursor_t *pmap_cur, page_range_t *range)
{
byte buf[8];
xb_ad(pmap_cur != NULL);
if (my_b_read(&pmap_cur->cache, buf, sizeof(buf))) {
return(FALSE);
}
range->from = mach_read_from_4(buf);
range->to = mach_read_from_4(buf + 4);
return(TRUE);
}
/************************************************************************
Close the page map cursor.*/
static void
page_map_file_close(page_map_cursor_t *pmap_cur)
{
int rc;
xb_ad(pmap_cur != NULL);
rc = end_io_cache(&pmap_cur->cache);
xb_a(rc == 0);
posix_fadvise(pmap_cur->fd, 0, 0, POSIX_FADV_DONTNEED);
rc = my_close(pmap_cur->fd, MY_WME);
xb_a(rc == 0);
my_free(pmap_cur);
}
/****************************************************************************
Expand a single data file according to the skipped pages maps created by
--compact.
@return TRUE on success, FALSE on failure. */
static my_bool
xb_expand_file(fil_node_t *node)
{
char pmapfile_path[FN_REFLEN];
char tmpfile_path[FN_REFLEN];
xb_fil_cur_t cursor;
xb_fil_cur_result_t res;
ds_ctxt_t *ds_local;
ds_ctxt_t *ds_buffer;
ds_file_t *tmpfile;
my_bool success = FALSE;
ulint i;
byte *page;
ulint page_expected_no;
page_map_cursor_t *pmap_cur;
ibool have_next_range;
page_range_t pmap_range;
xb_ad(trx_sys_sys_space(node->space->id) == FALSE);
snprintf(pmapfile_path, sizeof(pmapfile_path), "%s%s",
node->name, XB_PAGE_MAP_SUFFIX);
/* Skip files that don't have a corresponding page map file */
if (!(pmap_cur = page_map_file_open(pmapfile_path))) {
msg("Not expanding %s\n", node->name);
return(FALSE);
}
msg("Expanding %s\n", node->name);
ds_local = ds_create(".", DS_TYPE_LOCAL);
ds_buffer = ds_create(".", DS_TYPE_BUFFER);
xb_a(ds_local != NULL && ds_buffer != NULL);
ds_buffer_set_size(ds_buffer, FSP_EXTENT_SIZE * UNIV_PAGE_SIZE_MAX);
ds_set_pipe(ds_buffer, ds_local);
res = xb_fil_cur_open(&cursor, &rf_pass_through, node, 1);
xb_a(res == XB_FIL_CUR_SUCCESS);
snprintf(tmpfile_path, sizeof(tmpfile_path), "%s%s",
node->name, XB_TMPFILE_SUFFIX);
tmpfile = ds_open(ds_buffer, tmpfile_path, &cursor.statinfo);
if (tmpfile == NULL) {
msg("Could not open temporary file '%s'\n", tmpfile_path);
goto error;
}
have_next_range = page_map_file_next(pmap_cur, &pmap_range);
page_expected_no = 0;
/* Initialize and mark the empty page which is used to replace
skipped pages. */
memset(empty_page, 0, cursor.page_size);
memcpy(empty_page + compacted_page_magic_offset,
compacted_page_magic, compacted_page_magic_size);
mach_write_to_4(empty_page + FIL_PAGE_SPACE_OR_CHKSUM,
BUF_NO_CHECKSUM_MAGIC);
mach_write_to_4(empty_page + cursor.page_size -
FIL_PAGE_END_LSN_OLD_CHKSUM,
BUF_NO_CHECKSUM_MAGIC);
/* Main copy loop */
while ((res = xb_fil_cur_read(&cursor)) == XB_FIL_CUR_SUCCESS) {
for (i = 0, page = cursor.buf; i < cursor.buf_npages;
i++, page += cursor.page_size) {
ulint page_read_no;
page_read_no = mach_read_from_4(page + FIL_PAGE_OFFSET);
xb_a(!page_read_no || page_expected_no <= page_read_no);
if (have_next_range &&
page_expected_no == pmap_range.from) {
xb_a(pmap_range.from <= pmap_range.to);
/* Write empty pages instead of skipped ones, if
necessary. */
while (page_expected_no <= pmap_range.to) {
if (ds_write(tmpfile, empty_page,
cursor.page_size)) {
goto write_error;
}
page_expected_no++;
}
have_next_range =
page_map_file_next(pmap_cur,
&pmap_range);
}
/* Write the current page */
if (ds_write(tmpfile, page, cursor.page_size)) {
goto write_error;
}
page_expected_no++;
}
}
if (res != XB_FIL_CUR_EOF) {
goto error;
}
/* Write empty pages instead of trailing skipped ones, if any */
if (have_next_range) {
xb_a(page_expected_no == pmap_range.from);
xb_a(pmap_range.from <= pmap_range.to);
while (page_expected_no <= pmap_range.to) {
if (ds_write(tmpfile, empty_page,
cursor.page_size)) {
goto write_error;
}
page_expected_no++;
}
xb_a(!page_map_file_next(pmap_cur, &pmap_range));
}
/* Replace the original .ibd file with the expanded file */
if (my_rename(tmpfile_path, node->name, MYF(MY_WME))) {
msg("Failed to rename '%s' to '%s'\n",
tmpfile_path, node->name);
goto error;
}
my_delete(pmapfile_path, MYF(MY_WME));
if (!ds_close(tmpfile)) {
success = TRUE;
}
tmpfile = NULL;
goto end;
write_error:
msg("Write to '%s' failed\n", tmpfile_path);
error:
if (tmpfile != NULL) {
ds_close(tmpfile);
my_delete(tmpfile_path, MYF(MY_WME));
}
end:
ds_destroy(ds_buffer);
ds_destroy(ds_local);
xb_fil_cur_close(&cursor);
page_map_file_close(pmap_cur);
return(success);
}
/******************************************************************************
Expand the data files according to the skipped pages maps created by --compact.
@return TRUE on success, FALSE on failure. */
my_bool
xb_expand_datafiles(void)
/*=====================*/
{
ulint nfiles;
datafiles_iter_t *it = NULL;
fil_node_t *node;
fil_space_t *space;
msg("Starting to expand compacted .ibd files.\n");
/* Initialize the tablespace cache */
if (xb_data_files_init() != DB_SUCCESS) {
return(FALSE);
}
nfiles = UT_LIST_GET_LEN(fil_system->space_list);
xb_a(nfiles > 0);
it = datafiles_iter_new(fil_system);
if (it == NULL) {
msg("xtrabackup: error: datafiles_iter_new() failed.\n");
goto error;
}
while ((node = datafiles_iter_next(it)) != NULL) {
space = node->space;
/* System tablespace cannot be compacted */
if (!fil_is_user_tablespace_id(space->id)) {
continue;
}
if (!xb_expand_file(node)) {
goto error;
}
}
datafiles_iter_free(it);
xb_data_files_close();
return(TRUE);
error:
if (it != NULL) {
datafiles_iter_free(it);
}
xb_data_files_close();
return(FALSE);
}
/******************************************************************************
Callback used in buf_page_io_complete() to detect compacted pages.
@return TRUE if the page is marked as compacted, FALSE otherwise. */
ibool
buf_page_is_compacted(
/*==================*/
const byte* page) /*!< in: a database page */
{
return !memcmp(page + compacted_page_magic_offset,
compacted_page_magic, compacted_page_magic_size);
}
/*****************************************************************************
Builds an index definition corresponding to an index object. It is roughly
similar to innobase_create_index_def() / innobase_create_index_field_def() and
the opposite to dict_mem_index_create() / dict_mem_index_add_field(). */
static
void
xb_build_index_def(
/*=======================*/
mem_heap_t* heap, /*!< in: heap */
const dict_index_t* index, /*!< in: index */
index_def_t* index_def) /*!< out: index definition */
{
index_field_t* fields;
ulint n_fields;
ulint i;
ut_a(index->n_fields);
ut_ad(index->magic_n == DICT_INDEX_MAGIC_N);
/* Use n_user_defined_cols instead of n_fields, as the index will
contain a part of the primary key after n_user_defined_cols, and those
columns will be created automatically in
dict_index_build_internal_clust(). */
n_fields = index->n_user_defined_cols;
memset(index_def, 0, sizeof(*index_def));
index_def->name = mem_heap_strdup(heap, index->name);
index_def->ind_type = index->type;
fields = static_cast<index_field_t *>
(mem_heap_alloc(heap, n_fields * sizeof(*fields)));
for (i = 0; i < n_fields; i++) {
dict_field_t* field;
field = dict_index_get_nth_field(index, i);
fields[i].col_no = dict_col_get_no(field->col);
fields[i].prefix_len = field->prefix_len;
}
index_def->fields = fields;
index_def->n_fields = n_fields;
}
/* A dummy autoc_inc sequence for row_merge_build_indexes(). */
static ib_sequence_t null_seq(NULL, 0, 0);
/* A dummy table share and table for row_merge_build_indexes() error reporting.
Assumes that no errors are going to be reported. */
static struct TABLE_SHARE dummy_table_share;
static struct TABLE dummy_table;
/********************************************************************//**
Rebuild secondary indexes for a given table. */
static
void
xb_rebuild_indexes_for_table(
/*=========================*/
dict_table_t* table, /*!< in: table */
trx_t* trx, /*!< in: transaction handle */
ulint thread_n) /*!< in: thread number */
{
dict_index_t* index;
dict_index_t** indexes;
ulint n_indexes;
index_def_t* index_defs;
ulint i;
mem_heap_t* heap;
ulint error;
ulint* add_key_nums;
ut_ad(!mutex_own(&(dict_sys->mutex)));
ut_ad(table);
ut_a(UT_LIST_GET_LEN(table->indexes) > 0);
n_indexes = UT_LIST_GET_LEN(table->indexes) - 1;
if (!n_indexes) {
/* Only the primary key, nothing to do. */
return;
}
heap = mem_heap_create(1024);
indexes = (dict_index_t**) mem_heap_alloc(heap,
n_indexes * sizeof(*indexes));
index_defs = (index_def_t*) mem_heap_alloc(heap, n_indexes *
sizeof(*index_defs));
add_key_nums = static_cast<ulint *>
(mem_heap_alloc(heap, n_indexes * sizeof(*add_key_nums)));
/* Skip the primary key. */
index = dict_table_get_first_index(table);
ut_a(dict_index_is_clust(index));
row_mysql_lock_data_dictionary(trx);
for (i = 0; (index = dict_table_get_next_index(index)); i++) {
msg("[%02lu] Found index %s\n", thread_n, index->name);
/* Pretend that it's the current trx that created this index.
Required to avoid 5.6+ debug assertions. */
index->trx_id = trx->id;
xb_build_index_def(heap, index, &index_defs[i]);
/* In 5.6+, row_merge_drop_indexes() drops all the indexes on
the table that have the temp index prefix. It does not accept
an array of indexes to drop as in 5.5-. */
row_merge_rename_index_to_drop(trx, table->id, index->id);
}
ut_ad(i == n_indexes);
row_merge_drop_indexes(trx, table, TRUE);
index = dict_table_get_first_index(table);
ut_a(dict_index_is_clust(index));
index = dict_table_get_next_index(index);
while (index) {
/* In 5.6+, row_merge_drop_indexes() does not remove the
indexes from the dictionary cache nor from any foreign key
list. This may cause invalid dereferences as we try to access
the dropped indexes from other tables as FKs. */
dict_index_t* next_index = dict_table_get_next_index(index);
index->to_be_dropped = 1;
/* Patch up any FK referencing this index with NULL */
dict_foreign_replace_index(table, NULL, index);
dict_index_remove_from_cache(table, index);
index = next_index;
}
msg("[%02lu] Rebuilding %lu index(es).\n", thread_n, n_indexes);
error = row_merge_lock_table(trx, table, LOCK_X);
xb_a(error == DB_SUCCESS);
for (i = 0; i < n_indexes; i++) {
indexes[i] = row_merge_create_index(trx, table,
&index_defs[i]);
add_key_nums[i] = index_defs[i].key_number;
}
/* Commit trx to release latches on system tables */
trx_commit_for_mysql(trx);
trx_start_for_ddl(trx, TRX_DICT_OP_INDEX);
row_mysql_unlock_data_dictionary(trx);
/* Reacquire table lock for row_merge_build_indexes() */
error = row_merge_lock_table(trx, table, LOCK_X);
xb_a(error == DB_SUCCESS);
error = row_merge_build_indexes(trx, table, table, FALSE, indexes,
add_key_nums, n_indexes, &dummy_table,
NULL, NULL, ULINT_UNDEFINED, null_seq);
ut_a(error == DB_SUCCESS);
mem_heap_free(heap);
trx_commit_for_mysql(trx);
trx_start_for_ddl(trx, TRX_DICT_OP_INDEX);
}
/**************************************************************************
Worker thread function for index rebuild. */
static
void *
xb_rebuild_indexes_thread_func(
/*===========================*/
void* arg) /* thread context */
{
dict_table_t* table;
index_rebuild_table_t* rebuild_table;
index_rebuild_thread_t* thread;
trx_t* trx;
thread = (index_rebuild_thread_t *) arg;
trx = trx_allocate_for_mysql();
/* Suppress foreign key checks, as we are going to drop and recreate all
secondary keys. */
trx->check_foreigns = FALSE;
trx_start_for_ddl(trx, TRX_DICT_OP_INDEX);
/* Loop until there are no more tables in tables list */
for (;;) {
pthread_mutex_lock(&table_list_mutex);
rebuild_table = UT_LIST_GET_FIRST(table_list);
if (rebuild_table == NULL) {
pthread_mutex_unlock(&table_list_mutex);
break;
}
UT_LIST_REMOVE(list, table_list, rebuild_table);
pthread_mutex_unlock(&table_list_mutex);
ut_ad(rebuild_table->name);
ut_ad(fil_is_user_tablespace_id(rebuild_table->space_id));
row_mysql_lock_data_dictionary(trx);
table = dict_table_get_low(rebuild_table->name);
ut_d(table->n_ref_count++);
row_mysql_unlock_data_dictionary(trx);
ut_a(table != NULL);
ut_a(table->space == rebuild_table->space_id);
/* Discard change buffer entries for this space */
ibuf_delete_for_discarded_space(rebuild_table->space_id);
msg("[%02lu] Checking if there are indexes to rebuild in table "
"%s (space id: %lu)\n",
thread->num,
rebuild_table->name, rebuild_table->space_id);
xb_rebuild_indexes_for_table(table, trx, thread->num);
ut_d(table->n_ref_count--);
mem_free(rebuild_table->name);
mem_free(rebuild_table);
}
trx_commit_for_mysql(trx);
trx_free_for_mysql(trx);
return(NULL);
}
/******************************************************************************
Rebuild all secondary indexes in all tables in separate spaces. Called from
innobase_start_or_create_for_mysql(). */
void
xb_compact_rebuild_indexes(void)
/*=============================*/
{
dict_table_t* sys_tables;
dict_index_t* sys_index;
btr_pcur_t pcur;
const rec_t* rec;
mtr_t mtr;
const byte* field;
ulint len;
ulint space_id;
trx_t* trx;
index_rebuild_table_t* rebuild_table;
index_rebuild_thread_t* threads;
ulint i;
/* Set up the dummy table for the index rebuild error reporting */
dummy_table_share.fields = 0;
dummy_table.s = &dummy_table_share;
/* Iterate all tables that are not in the system tablespace and add them
to the list of tables to be rebuilt later. */
trx = trx_allocate_for_mysql();
trx_start_for_ddl(trx, TRX_DICT_OP_INDEX);
row_mysql_lock_data_dictionary(trx);
/* Enlarge the fatal lock wait timeout during index rebuild
operation. */
os_increment_counter_by_amount(server_mutex,
srv_fatal_semaphore_wait_threshold,
7200);
mtr_start(&mtr);
sys_tables = dict_table_get_low("SYS_TABLES");
sys_index = UT_LIST_GET_FIRST(sys_tables->indexes);
ut_a(!dict_table_is_comp(sys_tables));
pthread_mutex_init(&table_list_mutex, NULL);
UT_LIST_INIT(table_list);
btr_pcur_open_at_index_side(TRUE, sys_index, BTR_SEARCH_LEAF, &pcur,
TRUE, 0, &mtr);
for (;;) {
btr_pcur_move_to_next_user_rec(&pcur, &mtr);
rec = btr_pcur_get_rec(&pcur);
if (!btr_pcur_is_on_user_rec(&pcur)) {
/* end of index */
break;
}
if (rec_get_deleted_flag(rec, 0)) {
continue;
}
field = rec_get_nth_field_old(rec, 9, &len);
ut_a(len == 4);
space_id = mach_read_from_4(field);
/* Don't touch tables in the system tablespace */
if (!fil_is_user_tablespace_id(space_id)) {
continue;
}
field = rec_get_nth_field_old(rec, 0, &len);
rebuild_table = static_cast<index_rebuild_table_t *>
(mem_alloc(sizeof(*rebuild_table)));
rebuild_table->name = mem_strdupl((char*) field, len);
rebuild_table->space_id = space_id;
UT_LIST_ADD_LAST(list, table_list, rebuild_table);
}
btr_pcur_close(&pcur);
mtr_commit(&mtr);
row_mysql_unlock_data_dictionary(trx);
trx_commit_for_mysql(trx);
trx_free_for_mysql(trx);
/* Start worker threads for the index rebuild operation */
ut_ad(xtrabackup_rebuild_threads > 0);
if (xtrabackup_rebuild_threads > 1) {
msg("Starting %lu threads to rebuild indexes.\n",
xtrabackup_rebuild_threads);
}
threads = (index_rebuild_thread_t *)
mem_alloc(sizeof(*threads) *
xtrabackup_rebuild_threads);
for (i = 0; i < xtrabackup_rebuild_threads; i++) {
threads[i].num = i+1;
if (pthread_create(&threads[i].id, NULL,
xb_rebuild_indexes_thread_func,
&threads[i])) {
msg("error: pthread_create() failed: errno = %d\n",
errno);
ut_a(0);
}
}
/* Wait for worker threads to finish */
for (i = 0; i < xtrabackup_rebuild_threads; i++) {
pthread_join(threads[i].id, NULL);
}
mem_free(threads);
}
extra/mariabackup/compact.h deleted 100644 → 0
View file @ d7714308
/******************************************************
XtraBackup: hot backup tool for InnoDB
(c) 2009-2013 Percona LLC and/or its affiliates.
Originally Created 3/3/2009 Yasufumi Kinoshita
Written by Alexey Kopytov, Aleksandr Kuzminsky, Stewart Smith, Vadim Tkachenko,
Yasufumi Kinoshita, Ignacio Nin and Baron Schwartz.
This program is free software; you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation; version 2 of the License.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program; if not, write to the Free Software
Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA
*******************************************************/
#ifndef XB_COMPACT_H
#define XB_COMPACT_H
#include "write_filt.h"
/* Compact page filter context */
typedef struct {
my_bool skip;
ds_ctxt_t *ds_buffer;
ds_file_t *buffer;
index_id_t clustered_index;
my_bool clustered_index_found;
my_bool inside_skipped_range;
ulint free_limit;
} xb_wf_compact_ctxt_t;
/******************************************************************************
Expand the data files according to the skipped pages maps created by --compact.
@return TRUE on success, FALSE on failure. */
my_bool xb_expand_datafiles(void);
#endif
extra/mariabackup/datasink.c
View file @ ce4c56db
......@@ -46,7 +46,12 @@ ds_create(const char *root, ds_type_t type)
ds = &datasink_local;
break;
case DS_TYPE_ARCHIVE:
#ifdef HAVE_LIBARCHIVE
ds = &datasink_archive;
#else
msg("Error : mariabackup was built without libarchive support");
exit(EXIT_FAILURE);
#endif
break;
case DS_TYPE_XBSTREAM:
ds = &datasink_xbstream;
......@@ -55,8 +60,10 @@ ds_create(const char *root, ds_type_t type)
ds = &datasink_compress;
break;
case DS_TYPE_ENCRYPT:
ds = &datasink_encrypt;
msg("Error : mariabackup does not support encrypted backups.");
exit(EXIT_FAILURE);
break;
case DS_TYPE_TMPFILE:
ds = &datasink_tmpfile;
break;
......
extra/mariabackup/datasink.h
View file @ ce4c56db
......@@ -28,6 +28,7 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA
extern "C" {
#endif
extern char *xtrabackup_tmpdir;
struct datasink_struct;
typedef struct datasink_struct datasink_t;
......
extra/mariabackup/ds_compress.c
View file @ ce4c56db
......@@ -152,7 +152,7 @@ compress_open(ds_ctxt_t *ctxt, const char *path, MY_STAT *mystat)
/* Write the qpress file header */
name_len = strlen(new_name);
if (ds_write(dest_file, "F", 1) ||
write_uint32_le(dest_file, name_len) ||
write_uint32_le(dest_file, (uint)name_len) ||
/* we want to write the terminating \0 as well */
ds_write(dest_file, new_name, name_len + 1)) {
goto err;
......@@ -453,7 +453,7 @@ compress_worker_thread_func(void *arg)
with qpress implementation. */
thd->adler = adler32(0x00000001, (uchar *) thd->to,
thd->to_len);
(uInt)thd->to_len);
}
pthread_mutex_unlock(&thd->data_mutex);
......
extra/mariabackup/ds_encrypt.c
View file @ ce4c56db
......@@ -22,7 +22,7 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA
#include <my_base.h>
#include "common.h"
#include "datasink.h"
#ifdef HAVE_GRYPT
#if GCC_VERSION >= 4002
/* Workaround to avoid "gcry_ac_* is deprecated" warnings in gcrypt.h */
# pragma GCC diagnostic ignored "-Wdeprecated-declarations"
......@@ -615,3 +615,4 @@ encrypt_worker_thread_func(void *arg)
return NULL;
}
#endif /* HAVE_GCRYPT*/
extra/mariabackup/ds_encrypt.h
View file @ ce4c56db
......@@ -22,7 +22,7 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA
#define DS_ENCRYPT_H
#include "datasink.h"
#ifdef HAVE_GCRYPT
extern datasink_t datasink_encrypt;
#endif
#endif
extra/mariabackup/ds_local.c
View file @ ce4c56db
......@@ -53,9 +53,9 @@ local_init(const char *root)
&& my_errno != EEXIST && my_errno != EISDIR)
{
char errbuf[MYSYS_STRERROR_SIZE];
my_strerror(errbuf, sizeof(errbuf),my_errno);
my_error(EE_CANT_MKDIR, MYF(ME_BELL | ME_WAITTANG),
root, my_errno, my_strerror(errbuf, sizeof(errbuf),
my_errno));
root, my_errno,errbuf, my_errno);
return NULL;
}
......@@ -85,9 +85,9 @@ local_open(ds_ctxt_t *ctxt, const char *path,
dirname_part(dirpath, fullpath, &dirpath_len);
if (my_mkdir(dirpath, 0777, MYF(0)) < 0 && my_errno != EEXIST) {
char errbuf[MYSYS_STRERROR_SIZE];
my_strerror(errbuf, sizeof(errbuf), my_errno);
my_error(EE_CANT_MKDIR, MYF(ME_BELL | ME_WAITTANG),
dirpath, my_errno, my_strerror(errbuf, sizeof(errbuf),
my_errno));
dirpath, my_errno, errbuf);
return NULL;
}
......
extra/mariabackup/ds_stdout.c
View file @ ce4c56db
......@@ -79,7 +79,7 @@ stdout_open(ds_ctxt_t *ctxt __attribute__((unused)),
setmode(fileno(stdout), _O_BINARY);
#endif
stdout_file->fd = fileno(stdout);
stdout_file->fd = my_fileno(stdout);
file->path = (char *) stdout_file + sizeof(ds_stdout_file_t);
memcpy(file->path, fullpath, pathlen);
......
extra/mariabackup/ds_tmpfile.c
View file @ ce4c56db
......@@ -53,7 +53,6 @@ datasink_t datasink_tmpfile = {
&tmpfile_deinit
};
MY_TMPDIR mysql_tmpdir_list;
static ds_ctxt_t *
tmpfile_init(const char *root)
......@@ -90,7 +89,7 @@ tmpfile_open(ds_ctxt_t *ctxt, const char *path,
/* Create a temporary file in tmpdir. The file will be automatically
removed on close. Code copied from mysql_tmpfile(). */
fd = create_temp_file(tmp_path, my_tmpdir(&mysql_tmpdir_list),
fd = create_temp_file(tmp_path,xtrabackup_tmpdir,
"xbtemp",
#ifdef __WIN__
O_BINARY | O_TRUNC | O_SEQUENTIAL |
......
extra/mariabackup/encryption_plugin.cc 0 → 100644
View file @ ce4c56db
#include <mysqld.h>
#include <mysql.h>
#include <xtrabackup.h>
#include <encryption_plugin.h>
#include <backup_copy.h>
#include <sql_plugin.h>
#include <sstream>
#include <vector>
#include <common.h>
#include <backup_mysql.h>
extern struct st_maria_plugin *mysql_optional_plugins[];
extern struct st_maria_plugin *mysql_mandatory_plugins[];
static void encryption_plugin_init(int argc, char **argv);
extern char *xb_plugin_load;
extern char *xb_plugin_dir;
const int PLUGIN_MAX_ARGS = 1024;
vector<string> backup_plugins_args;
const char *QUERY_PLUGIN =
"SELECT plugin_name, plugin_library, @@plugin_dir"
" FROM information_schema.plugins WHERE plugin_type='ENCRYPTION'"
" AND plugin_status='ACTIVE'";
string encryption_plugin_config;
static void add_to_plugin_load_list(const char *plugin_def)
{
opt_plugin_load_list_ptr->push_back(new i_string(plugin_def));
}
static char XTRABACKUP_EXE[] = "xtrabackup";
void encryption_plugin_backup_init(MYSQL *mysql)
{
MYSQL_RES *result;
MYSQL_ROW row;
ostringstream oss;
char *argv[PLUGIN_MAX_ARGS];
int argc;
result = xb_mysql_query(mysql, QUERY_PLUGIN, true, true);
row = mysql_fetch_row(result);
if (!row)
{
mysql_free_result(result);
return;
}
char *name= row[0];
char *library= row[1];
char *dir= row[2];
#ifdef _WIN32
for (char *p = dir; *p; p++)
if (*p == '\\') *p = '/';
#endif
string plugin_load(name);
if (library)
plugin_load += string("=") + library;
oss << "plugin_load=" << plugin_load << endl;
/* Required to load the plugin later.*/
add_to_plugin_load_list(plugin_load.c_str());
strncpy(opt_plugin_dir, dir, FN_REFLEN);
oss << "plugin_dir=" << '"' << dir << '"' << endl;
/* Read plugin variables. */
char query[1024];
snprintf(query, 1024, "SHOW variables like '%s_%%'", name);
mysql_free_result(result);
result = xb_mysql_query(mysql, query, true, true);
while ((row = mysql_fetch_row(result)))
{
string arg("--");
arg += row[0];
arg += "=";
arg += row[1];
backup_plugins_args.push_back(arg);
oss << row[0] << "=" << row[1] << endl;
}
mysql_free_result(result);
/* Check whether to encrypt logs. */
result = xb_mysql_query(mysql, "select @@innodb_encrypt_log", true, true);
row = mysql_fetch_row(result);
srv_encrypt_log = (row != 0 && row[0][0] == '1');
oss << "innodb_encrypt_log=" << row[0] << endl;
mysql_free_result(result);
encryption_plugin_config = oss.str();
argc = 0;
argv[argc++] = XTRABACKUP_EXE;
for(size_t i = 0; i < backup_plugins_args.size(); i++)
{
argv[argc++] = (char *)backup_plugins_args[i].c_str();
if (argc == PLUGIN_MAX_ARGS - 2)
break;
}
argv[argc] = 0;
encryption_plugin_init(argc, argv);
}
const char *encryption_plugin_get_config()
{
return encryption_plugin_config.c_str();
}
extern int finalize_encryption_plugin(st_plugin_int *plugin);
void encryption_plugin_prepare_init(int argc, char **argv)
{
if (!xb_plugin_load)
{
/* This prevents crashes e.g in --stats with wrong my.cnf*/
finalize_encryption_plugin(0);
return;
}
add_to_plugin_load_list(xb_plugin_load);
if (xb_plugin_dir)
strncpy(opt_plugin_dir, xb_plugin_dir, FN_REFLEN);
char **new_argv = new char *[argc + 1];
new_argv[0] = XTRABACKUP_EXE;
memcpy(&new_argv[1], argv, argc*sizeof(char *));
encryption_plugin_init(argc+1, new_argv);
delete[] new_argv;
}
static void encryption_plugin_init(int argc, char **argv)
{
/* Patch optional and mandatory plugins, we only need to load the one in xb_plugin_load. */
mysql_optional_plugins[0] = mysql_mandatory_plugins[0] = 0;
msg("Loading encryption plugin\n");
for (int i= 1; i < argc; i++)
msg("\t Encryption plugin parameter : '%s'\n", argv[i]);
plugin_init(&argc, argv, PLUGIN_INIT_SKIP_PLUGIN_TABLE);
}
extra/mariabackup/encryption_plugin.h 0 → 100644
View file @ ce4c56db
#include <mysql.h>
#include <string>
extern void encryption_plugin_backup_init(MYSQL *mysql);
extern const char* encryption_plugin_get_config();
extern void encryption_plugin_prepare_init(int argc, char **argv);
//extern void encryption_plugin_init(int argc, char **argv);
extra/mariabackup/fil_cur.cc
View file @ ce4c56db
......@@ -33,6 +33,7 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA
#include "common.h"
#include "read_filt.h"
#include "xtrabackup.h"
#include "xb0xb.h"
/* Size of read buffer in pages (640 pages = 10M for 16K sized pages) */
#define XB_FIL_CUR_PAGES 640
......@@ -167,7 +168,7 @@ xb_fil_cur_open(
os_file_create_simple_no_error_handling(0, node->name,
OS_FILE_OPEN,
OS_FILE_READ_ONLY,
&success);
&success,0);
if (!success) {
/* The following call prints an error message */
os_file_get_last_error(TRUE);
......@@ -200,7 +201,7 @@ xb_fil_cur_open(
cursor->node = node;
cursor->file = node->handle;
if (my_fstat(cursor->file, &cursor->statinfo, MYF(MY_WME))) {
if (stat(cursor->abs_path, &cursor->statinfo)) {
msg("[%02u] xtrabackup: error: cannot stat %s\n",
thread_n, cursor->abs_path);
......@@ -253,7 +254,7 @@ xb_fil_cur_open(
cursor->buf_page_no = 0;
cursor->thread_n = thread_n;
cursor->space_size = cursor->statinfo.st_size / page_size;
cursor->space_size = (ulint)(cursor->statinfo.st_size / page_size);
cursor->read_filter = read_filter;
cursor->read_filter->init(&cursor->read_filter_ctxt, cursor,
......@@ -327,26 +328,32 @@ xb_fil_cur_read(
cursor->buf_read = 0;
cursor->buf_npages = 0;
cursor->buf_offset = offset;
cursor->buf_page_no = (ulint) (offset >> cursor->page_size_shift);
cursor->buf_page_no = (ulint)(offset >> cursor->page_size_shift);
success = os_file_read(cursor->file, cursor->buf, offset,
to_read);
(ulint)to_read);
if (!success) {
return(XB_FIL_CUR_ERROR);
}
fil_system_enter();
fil_space_t *space = fil_space_get_by_id(cursor->space_id);
fil_system_exit();
/* check pages for corruption and re-read if necessary. i.e. in case of
partially written pages */
for (page = cursor->buf, i = 0; i < npages;
page += cursor->page_size, i++) {
ib_int64_t page_no = cursor->buf_page_no + i;
if (buf_page_is_corrupted(TRUE, page, cursor->zip_size)) {
bool checksum_ok = fil_space_verify_crypt_checksum(page, cursor->zip_size,space, (ulint)page_no);
ulint page_no = cursor->buf_page_no + i;
if (!checksum_ok &&
buf_page_is_corrupted(true, page, cursor->zip_size,space)) {
if (cursor->is_system &&
page_no >= FSP_EXTENT_SIZE &&
page_no < FSP_EXTENT_SIZE * 3) {
page_no >= (ib_int64_t)FSP_EXTENT_SIZE &&
page_no < (ib_int64_t) FSP_EXTENT_SIZE * 3) {
/* skip doublewrite buffer pages */
xb_a(cursor->page_size == UNIV_PAGE_SIZE);
msg("[%02u] xtrabackup: "
......
extra/mariabackup/fil_cur.h
View file @ ce4c56db
......@@ -49,10 +49,10 @@ struct xb_fil_cur_t {
/*!< read filter context */
byte* orig_buf; /*!< read buffer */
byte* buf; /*!< aligned pointer for orig_buf */
ulint buf_size; /*!< buffer size in bytes */
ulint buf_read; /*!< number of read bytes in buffer
size_t buf_size; /*!< buffer size in bytes */
size_t buf_read; /*!< number of read bytes in buffer
after the last cursor read */
ulint buf_npages; /*!< number of pages in buffer after the
size_t buf_npages; /*!< number of pages in buffer after the
last cursor read */
ib_int64_t buf_offset; /*!< file offset of the first page in
buffer */
......
extra/mariabackup/innobackupex.cc
View file @ ce4c56db
......@@ -52,14 +52,12 @@ Place, Suite 330, Boston, MA 02111-1307 USA
#include <mysqld.h>
#include <my_default.h>
#include <my_getopt.h>
#include <strings.h>
#include <string>
#include <sstream>
#include <set>
#include "common.h"
#include "innobackupex.h"
#include "xtrabackup.h"
#include "xtrabackup_version.h"
#include "xbstream.h"
#include "fil_cur.h"
#include "write_filt.h"
......@@ -667,22 +665,6 @@ static struct my_option ibx_long_options[] =
(uchar*) &ibx_xtrabackup_parallel, (uchar*) &ibx_xtrabackup_parallel,
0, GET_INT, REQUIRED_ARG, 1, 1, INT_MAX, 0, 0, 0},
{"rebuild-indexes", OPT_REBUILD_INDEXES,
"This option only has effect when used together with the --apply-log "
"option and is passed directly to xtrabackup. When used, makes "
"xtrabackup rebuild all secondary indexes after applying the log. "
"This option is normally used to prepare compact backups. See the "
"XtraBackup manual for more information.",
(uchar*) &ibx_xtrabackup_rebuild_indexes,
(uchar*) &ibx_xtrabackup_rebuild_indexes,
0, GET_BOOL, NO_ARG, 0, 0, 0, 0, 0, 0},
{"rebuild-threads", OPT_REBUILD_THREADS,
"Use this number of threads to rebuild indexes in a compact backup. "
"Only has effect with --prepare and --rebuild-indexes.",
(uchar*) &ibx_xtrabackup_rebuild_threads,
(uchar*) &ibx_xtrabackup_rebuild_threads,
0, GET_UINT, REQUIRED_ARG, 1, 1, UINT_MAX, 0, 0, 0},
{"stream", OPT_STREAM, "This option specifies the format in which to "
"do the streamed backup. The option accepts a string argument. The "
......@@ -845,9 +827,9 @@ ibx_get_one_option(int optid,
exit(0);
break;
case 'v':
msg("innobackupex version %s %s (%s) (revision id: %s)\n",
XTRABACKUP_VERSION,
SYSTEM_TYPE, MACHINE_TYPE, XTRABACKUP_REVISION);
msg("innobackupex version %s %s (%s)\n",
MYSQL_SERVER_VERSION,
SYSTEM_TYPE, MACHINE_TYPE);
exit(0);
break;
case OPT_HISTORY:
......@@ -1039,7 +1021,6 @@ ibx_init()
/* setup xtrabackup options */
xb_close_files = ibx_xb_close_files;
xtrabackup_compact = ibx_xtrabackup_compact;
xtrabackup_compress_alg = ibx_xtrabackup_compress_alg;
xtrabackup_compress_threads = ibx_xtrabackup_compress_threads;
xtrabackup_compress_chunk_size = ibx_xtrabackup_compress_chunk_size;
......@@ -1057,8 +1038,6 @@ ibx_init()
xtrabackup_log_copy_interval = ibx_xtrabackup_log_copy_interval;
xtrabackup_incremental = ibx_xtrabackup_incremental;
xtrabackup_parallel = ibx_xtrabackup_parallel;
xtrabackup_rebuild_indexes = ibx_xtrabackup_rebuild_indexes;
xtrabackup_rebuild_threads = ibx_xtrabackup_rebuild_threads;
xtrabackup_stream_str = ibx_xtrabackup_stream_str;
xtrabackup_tables_file = ibx_xtrabackup_tables_file;
xtrabackup_throttle = ibx_xtrabackup_throttle;
......
extra/mariabackup/read_filt.cc
View file @ ce4c56db
......@@ -75,7 +75,7 @@ rf_pass_through_get_next_batch(
*read_batch_start = ctxt->offset;
*read_batch_len = ctxt->data_file_size - ctxt->offset;
if (*read_batch_len > ctxt->buffer_capacity) {
if (*read_batch_len > (ib_int64_t)ctxt->buffer_capacity) {
*read_batch_len = ctxt->buffer_capacity;
}
......@@ -128,7 +128,7 @@ rf_bitmap_get_next_batch(
{
ulint start_page_id;
start_page_id = ctxt->offset / ctxt->page_size;
start_page_id = (ulint)(ctxt->offset / ctxt->page_size);
xb_a (ctxt->offset % ctxt->page_size == 0);
......@@ -170,7 +170,7 @@ rf_bitmap_get_next_batch(
buffer capacity. The subsequent invocations will continue returning
the current block in buffer-sized pieces until ctxt->filter_batch_end
is reached, trigerring the next bitmap query. */
if (*read_batch_len > ctxt->buffer_capacity) {
if (*read_batch_len > (ib_int64_t)ctxt->buffer_capacity) {
*read_batch_len = ctxt->buffer_capacity;
}
......
extra/mariabackup/read_filt.h
View file @ ce4c56db
......@@ -33,14 +33,14 @@ struct xb_fil_cur_t;
struct xb_read_filt_ctxt_t {
ib_int64_t offset; /*!< current file offset */
ib_int64_t data_file_size; /*!< data file size */
ib_int64_t buffer_capacity;/*!< read buffer capacity */
ulint space_id; /*!< space id */
size_t buffer_capacity;/*!< read buffer capacity */
ib_int64_t space_id; /*!< space id */
/* The following fields used only in bitmap filter */
/* Move these to union if any other filters are added in future */
xb_page_bitmap_range *bitmap_range; /*!< changed page bitmap range
iterator for space_id */
ulint page_size; /*!< page size */
ulint filter_batch_end;/*!< the ending page id of the
size_t page_size; /*!< page size */
ulint filter_batch_end;/*!< the ending page id of the
current changed page block in
the bitmap */
};
......
extra/mariabackup/version_check.pl deleted 100644 → 0
View file @ d7714308
use warnings FATAL => 'all';
use strict;
use English qw(-no_match_vars);
use POSIX "strftime";
my @required_perl_version = (5, 0, 5);
my $required_perl_version_old_style = 5.005;
# check existence of DBD::mysql module
eval {
require DBD::mysql;
};
my $dbd_mysql_installed = $EVAL_ERROR ? 0 : 1;
my $now;
my %mysql;
my $prefix = "version_check";
# ###########################################################################
# HTTPMicro package
# This package is a copy without comments from the original. The original
# with comments and its test file can be found in the Bazaar repository at,
# lib/HTTPMicro.pm
# t/lib/HTTPMicro.t
# See https://launchpad.net/percona-toolkit for more information.
# ###########################################################################
{
package HTTPMicro;
BEGIN {
$HTTPMicro::VERSION = '0.001';
}
use strict;
use warnings;
use Carp ();
my @attributes;
BEGIN {
@attributes = qw(agent timeout);
no strict 'refs';
for my $accessor ( @attributes ) {
*{$accessor} = sub {
@_ > 1 ? $_[0]->{$accessor} = $_[1] : $_[0]->{$accessor};
};
}
}
sub new {
my($class, %args) = @_;
(my $agent = $class) =~ s{::}{-}g;
my $self = {
agent => $agent . "/" . ($class->VERSION || 0),
timeout => 60,
};
for my $key ( @attributes ) {
$self->{$key} = $args{$key} if exists $args{$key}
}
return bless $self, $class;
}
my %DefaultPort = (
http => 80,
https => 443,
);
sub request {
my ($self, $method, $url, $args) = @_;
@_ == 3 || (@_ == 4 && ref $args eq 'HASH')
or Carp::croak(q/Usage: $http->request(METHOD, URL, [HASHREF])/);
$args ||= {}; # we keep some state in this during _request
my $response;
for ( 0 .. 1 ) {
$response = eval { $self->_request($method, $url, $args) };
last unless $@ && $method eq 'GET'
&& $@ =~ m{^(?:Socket closed|Unexpected end)};
}
if (my $e = "$@") {
$response = {
success => q{},
status => 599,
reason => 'Internal Exception',
content => $e,
headers => {
'content-type' => 'text/plain',
'content-length' => length $e,
}
};
}
return $response;
}
sub _request {
my ($self, $method, $url, $args) = @_;
my ($scheme, $host, $port, $path_query) = $self->_split_url($url);
my $request = {
method => $method,
scheme => $scheme,
host_port => ($port == $DefaultPort{$scheme} ? $host : "$host:$port"),
uri => $path_query,
headers => {},
};
my $handle = HTTPMicro::Handle->new(timeout => $self->{timeout});
$handle->connect($scheme, $host, $port);
$self->_prepare_headers_and_cb($request, $args);
$handle->write_request_header(@{$request}{qw/method uri headers/});
$handle->write_content_body($request) if $request->{content};
my $response;
do { $response = $handle->read_response_header }
until (substr($response->{status},0,1) ne '1');
if (!($method eq 'HEAD' || $response->{status} =~ /^[23]04/)) {
$response->{content} = '';
$handle->read_content_body(sub { $_[1]->{content} .= $_[0] }, $response);
}
$handle->close;
$response->{success} = substr($response->{status},0,1) eq '2';
return $response;
}
sub _prepare_headers_and_cb {
my ($self, $request, $args) = @_;
for ($args->{headers}) {
next unless defined;
while (my ($k, $v) = each %$_) {
$request->{headers}{lc $k} = $v;
}
}
$request->{headers}{'host'} = $request->{host_port};
$request->{headers}{'connection'} = "close";
$request->{headers}{'user-agent'} ||= $self->{agent};
if (defined $args->{content}) {
$request->{headers}{'content-type'} ||= "application/octet-stream";
utf8::downgrade($args->{content}, 1)
or Carp::croak(q/Wide character in request message body/);
$request->{headers}{'content-length'} = length $args->{content};
$request->{content} = $args->{content};
}
return;
}
sub _split_url {
my $url = pop;
my ($scheme, $authority, $path_query) = $url =~ m<\A([^:/?#]+)://([^/?#]*)([^#]*)>
or Carp::croak(qq/Cannot parse URL: '$url'/);
$scheme = lc $scheme;
$path_query = "/$path_query" unless $path_query =~ m<\A/>;
my $host = (length($authority)) ? lc $authority : 'localhost';
$host =~ s/\A[^@]*@//; # userinfo
my $port = do {
$host =~ s/:([0-9]*)\z// && length $1
? $1
: $DefaultPort{$scheme}
};
return ($scheme, $host, $port, $path_query);
}
package
HTTPMicro::Handle; # hide from PAUSE/indexers
use strict;
use warnings;
use Carp qw[croak];
use Errno qw[EINTR EPIPE];
use IO::Socket qw[SOCK_STREAM];
sub BUFSIZE () { 32768 }
my $Printable = sub {
local $_ = shift;
s/\r/\\r/g;
s/\n/\\n/g;
s/\t/\\t/g;
s/([^\x20-\x7E])/sprintf('\\x%.2X', ord($1))/ge;
$_;
};
sub new {
my ($class, %args) = @_;
return bless {
rbuf => '',
timeout => 60,
max_line_size => 16384,
%args
}, $class;
}
my $ssl_verify_args = {
check_cn => "when_only",
wildcards_in_alt => "anywhere",
wildcards_in_cn => "anywhere"
};
sub connect {
@_ == 4 || croak(q/Usage: $handle->connect(scheme, host, port)/);
my ($self, $scheme, $host, $port) = @_;
if ( $scheme eq 'https' ) {
eval "require IO::Socket::SSL"
unless exists $INC{'IO/Socket/SSL.pm'};
croak(qq/IO::Socket::SSL must be installed for https support\n/)
unless $INC{'IO/Socket/SSL.pm'};
}
elsif ( $scheme ne 'http' ) {
croak(qq/Unsupported URL scheme '$scheme'\n/);
}
$self->{fh} = 'IO::Socket::INET'->new(
PeerHost => $host,
PeerPort => $port,
Proto => 'tcp',
Type => SOCK_STREAM,
Timeout => $self->{timeout}
) or croak(qq/Could not connect to '$host:$port': $@/);
binmode($self->{fh})
or croak(qq/Could not binmode() socket: '$!'/);
if ( $scheme eq 'https') {
IO::Socket::SSL->start_SSL($self->{fh});
ref($self->{fh}) eq 'IO::Socket::SSL'
or die(qq/SSL connection failed for $host\n/);
if ( $self->{fh}->can("verify_hostname") ) {
$self->{fh}->verify_hostname( $host, $ssl_verify_args )
or die(qq/SSL certificate not valid for $host\n/);
}
else {
my $fh = $self->{fh};
_verify_hostname_of_cert($host, _peer_certificate($fh), $ssl_verify_args)
or die(qq/SSL certificate not valid for $host\n/);
}
}
$self->{host} = $host;
$self->{port} = $port;
return $self;
}
sub close {
@_ == 1 || croak(q/Usage: $handle->close()/);
my ($self) = @_;
CORE::close($self->{fh})
or croak(qq/Could not close socket: '$!'/);
}
sub write {
@_ == 2 || croak(q/Usage: $handle->write(buf)/);
my ($self, $buf) = @_;
my $len = length $buf;
my $off = 0;
local $SIG{PIPE} = 'IGNORE';
while () {
$self->can_write
or croak(q/Timed out while waiting for socket to become ready for writing/);
my $r = syswrite($self->{fh}, $buf, $len, $off);
if (defined $r) {
$len -= $r;
$off += $r;
last unless $len > 0;
}
elsif ($! == EPIPE) {
croak(qq/Socket closed by remote server: $!/);
}
elsif ($! != EINTR) {
croak(qq/Could not write to socket: '$!'/);
}
}
return $off;
}
sub read {
@_ == 2 || @_ == 3 || croak(q/Usage: $handle->read(len)/);
my ($self, $len) = @_;
my $buf = '';
my $got = length $self->{rbuf};
if ($got) {
my $take = ($got < $len) ? $got : $len;
$buf = substr($self->{rbuf}, 0, $take, '');
$len -= $take;
}
while ($len > 0) {
$self->can_read
or croak(q/Timed out while waiting for socket to become ready for reading/);
my $r = sysread($self->{fh}, $buf, $len, length $buf);
if (defined $r) {
last unless $r;
$len -= $r;
}
elsif ($! != EINTR) {
croak(qq/Could not read from socket: '$!'/);
}
}
if ($len) {
croak(q/Unexpected end of stream/);
}
return $buf;
}
sub readline {
@_ == 1 || croak(q/Usage: $handle->readline()/);
my ($self) = @_;
while () {
if ($self->{rbuf} =~ s/\A ([^\x0D\x0A]* \x0D?\x0A)//x) {
return $1;
}
$self->can_read
or croak(q/Timed out while waiting for socket to become ready for reading/);
my $r = sysread($self->{fh}, $self->{rbuf}, BUFSIZE, length $self->{rbuf});
if (defined $r) {
last unless $r;
}
elsif ($! != EINTR) {
croak(qq/Could not read from socket: '$!'/);
}
}
croak(q/Unexpected end of stream while looking for line/);
}
sub read_header_lines {
@_ == 1 || @_ == 2 || croak(q/Usage: $handle->read_header_lines([headers])/);
my ($self, $headers) = @_;
$headers ||= {};
my $lines = 0;
my $val;
while () {
my $line = $self->readline;
if ($line =~ /\A ([^\x00-\x1F\x7F:]+) : [\x09\x20]* ([^\x0D\x0A]*)/x) {
my ($field_name) = lc $1;
$val = \($headers->{$field_name} = $2);
}
elsif ($line =~ /\A [\x09\x20]+ ([^\x0D\x0A]*)/x) {
$val
or croak(q/Unexpected header continuation line/);
next unless length $1;
$$val .= ' ' if length $$val;
$$val .= $1;
}
elsif ($line =~ /\A \x0D?\x0A \z/x) {
last;
}
else {
croak(q/Malformed header line: / . $Printable->($line));
}
}
return $headers;
}
sub write_header_lines {
(@_ == 2 && ref $_[1] eq 'HASH') || croak(q/Usage: $handle->write_header_lines(headers)/);
my($self, $headers) = @_;
my $buf = '';
while (my ($k, $v) = each %$headers) {
my $field_name = lc $k;
$field_name =~ /\A [\x21\x23-\x27\x2A\x2B\x2D\x2E\x30-\x39\x41-\x5A\x5E-\x7A\x7C\x7E]+ \z/x
or croak(q/Invalid HTTP header field name: / . $Printable->($field_name));
$field_name =~ s/\b(\w)/\u$1/g;
$buf .= "$field_name: $v\x0D\x0A";
}
$buf .= "\x0D\x0A";
return $self->write($buf);
}
sub read_content_body {
@_ == 3 || @_ == 4 || croak(q/Usage: $handle->read_content_body(callback, response, [read_length])/);
my ($self, $cb, $response, $len) = @_;
$len ||= $response->{headers}{'content-length'};
croak("No content-length in the returned response, and this "
. "UA doesn't implement chunking") unless defined $len;
while ($len > 0) {
my $read = ($len > BUFSIZE) ? BUFSIZE : $len;
$cb->($self->read($read), $response);
$len -= $read;
}
return;
}
sub write_content_body {
@_ == 2 || croak(q/Usage: $handle->write_content_body(request)/);
my ($self, $request) = @_;
my ($len, $content_length) = (0, $request->{headers}{'content-length'});
$len += $self->write($request->{content});
$len == $content_length
or croak(qq/Content-Length missmatch (got: $len expected: $content_length)/);
return $len;
}
sub read_response_header {
@_ == 1 || croak(q/Usage: $handle->read_response_header()/);
my ($self) = @_;
my $line = $self->readline;
$line =~ /\A (HTTP\/(0*\d+\.0*\d+)) [\x09\x20]+ ([0-9]{3}) [\x09\x20]+ ([^\x0D\x0A]*) \x0D?\x0A/x
or croak(q/Malformed Status-Line: / . $Printable->($line));
my ($protocol, $version, $status, $reason) = ($1, $2, $3, $4);
return {
status => $status,
reason => $reason,
headers => $self->read_header_lines,
protocol => $protocol,
};
}
sub write_request_header {
@_ == 4 || croak(q/Usage: $handle->write_request_header(method, request_uri, headers)/);
my ($self, $method, $request_uri, $headers) = @_;
return $self->write("$method $request_uri HTTP/1.1\x0D\x0A")
+ $self->write_header_lines($headers);
}
sub _do_timeout {
my ($self, $type, $timeout) = @_;
$timeout = $self->{timeout}
unless defined $timeout && $timeout >= 0;
my $fd = fileno $self->{fh};
defined $fd && $fd >= 0
or croak(q/select(2): 'Bad file descriptor'/);
my $initial = time;
my $pending = $timeout;
my $nfound;
vec(my $fdset = '', $fd, 1) = 1;
while () {
$nfound = ($type eq 'read')
? select($fdset, undef, undef, $pending)
: select(undef, $fdset, undef, $pending) ;
if ($nfound == -1) {
$! == EINTR
or croak(qq/select(2): '$!'/);
redo if !$timeout || ($pending = $timeout - (time - $initial)) > 0;
$nfound = 0;
}
last;
}
$! = 0;
return $nfound;
}
sub can_read {
@_ == 1 || @_ == 2 || croak(q/Usage: $handle->can_read([timeout])/);
my $self = shift;
return $self->_do_timeout('read', @_)
}
sub can_write {
@_ == 1 || @_ == 2 || croak(q/Usage: $handle->can_write([timeout])/);
my $self = shift;
return $self->_do_timeout('write', @_)
}
my $prog = <<'EOP';
BEGIN {
if ( defined &IO::Socket::SSL::CAN_IPV6 ) {
*CAN_IPV6 = \*IO::Socket::SSL::CAN_IPV6;
}
else {
constant->import( CAN_IPV6 => '' );
}
my %const = (
NID_CommonName => 13,
GEN_DNS => 2,
GEN_IPADD => 7,
);
while ( my ($name,$value) = each %const ) {
no strict 'refs';
*{$name} = UNIVERSAL::can( 'Net::SSLeay', $name ) || sub { $value };
}
}
{
my %dispatcher = (
issuer => sub { Net::SSLeay::X509_NAME_oneline( Net::SSLeay::X509_get_issuer_name( shift )) },
subject => sub { Net::SSLeay::X509_NAME_oneline( Net::SSLeay::X509_get_subject_name( shift )) },
);
if ( $Net::SSLeay::VERSION >= 1.30 ) {
$dispatcher{commonName} = sub {
my $cn = Net::SSLeay::X509_NAME_get_text_by_NID(
Net::SSLeay::X509_get_subject_name( shift ), NID_CommonName);
$cn =~s{\0$}{}; # work around Bug in Net::SSLeay <1.33
$cn;
}
} else {
$dispatcher{commonName} = sub {
croak "you need at least Net::SSLeay version 1.30 for getting commonName"
}
}
if ( $Net::SSLeay::VERSION >= 1.33 ) {
$dispatcher{subjectAltNames} = sub { Net::SSLeay::X509_get_subjectAltNames( shift ) };
} else {
$dispatcher{subjectAltNames} = sub {
return;
};
}
$dispatcher{authority} = $dispatcher{issuer};
$dispatcher{owner} = $dispatcher{subject};
$dispatcher{cn} = $dispatcher{commonName};
sub _peer_certificate {
my ($self, $field) = @_;
my $ssl = $self->_get_ssl_object or return;
my $cert = ${*$self}{_SSL_certificate}
||= Net::SSLeay::get_peer_certificate($ssl)
or return $self->error("Could not retrieve peer certificate");
if ($field) {
my $sub = $dispatcher{$field} or croak
"invalid argument for peer_certificate, valid are: ".join( " ",keys %dispatcher ).
"\nMaybe you need to upgrade your Net::SSLeay";
return $sub->($cert);
} else {
return $cert
}
}
my %scheme = (
ldap => {
wildcards_in_cn => 0,
wildcards_in_alt => 'leftmost',
check_cn => 'always',
},
http => {
wildcards_in_cn => 'anywhere',
wildcards_in_alt => 'anywhere',
check_cn => 'when_only',
},
smtp => {
wildcards_in_cn => 0,
wildcards_in_alt => 0,
check_cn => 'always'
},
none => {}, # do not check
);
$scheme{www} = $scheme{http}; # alias
$scheme{xmpp} = $scheme{http}; # rfc 3920
$scheme{pop3} = $scheme{ldap}; # rfc 2595
$scheme{imap} = $scheme{ldap}; # rfc 2595
$scheme{acap} = $scheme{ldap}; # rfc 2595
$scheme{nntp} = $scheme{ldap}; # rfc 4642
$scheme{ftp} = $scheme{http}; # rfc 4217
sub _verify_hostname_of_cert {
my $identity = shift;
my $cert = shift;
my $scheme = shift || 'none';
if ( ! ref($scheme) ) {
$scheme = $scheme{$scheme} or croak "scheme $scheme not defined";
}
return 1 if ! %$scheme; # 'none'
my $commonName = $dispatcher{cn}->($cert);
my @altNames = $dispatcher{subjectAltNames}->($cert);
if ( my $sub = $scheme->{callback} ) {
return $sub->($identity,$commonName,@altNames);
}
my $ipn;
if ( CAN_IPV6 and $identity =~m{:} ) {
$ipn = IO::Socket::SSL::inet_pton(IO::Socket::SSL::AF_INET6,$identity)
or croak "'$identity' is not IPv6, but neither IPv4 nor hostname";
} elsif ( $identity =~m{^\d+\.\d+\.\d+\.\d+$} ) {
$ipn = IO::Socket::SSL::inet_aton( $identity ) or croak "'$identity' is not IPv4, but neither IPv6 nor hostname";
} else {
if ( $identity =~m{[^a-zA-Z0-9_.\-]} ) {
$identity =~m{\0} and croak("name '$identity' has \\0 byte");
$identity = IO::Socket::SSL::idn_to_ascii($identity) or
croak "Warning: Given name '$identity' could not be converted to IDNA!";
}
}
my $check_name = sub {
my ($name,$identity,$wtyp) = @_;
$wtyp ||= '';
my $pattern;
if ( $wtyp eq 'anywhere' and $name =~m{^([a-zA-Z0-9_\-]*)\*(.+)} ) {
$pattern = qr{^\Q$1\E[a-zA-Z0-9_\-]*\Q$2\E$}i;
} elsif ( $wtyp eq 'leftmost' and $name =~m{^\*(\..+)$} ) {
$pattern = qr{^[a-zA-Z0-9_\-]*\Q$1\E$}i;
} else {
$pattern = qr{^\Q$name\E$}i;
}
return $identity =~ $pattern;
};
my $alt_dnsNames = 0;
while (@altNames) {
my ($type, $name) = splice (@altNames, 0, 2);
if ( $ipn and $type == GEN_IPADD ) {
return 1 if $ipn eq $name;
} elsif ( ! $ipn and $type == GEN_DNS ) {
$name =~s/\s+$//; $name =~s/^\s+//;
$alt_dnsNames++;
$check_name->($name,$identity,$scheme->{wildcards_in_alt})
and return 1;
}
}
if ( ! $ipn and (
$scheme->{check_cn} eq 'always' or
$scheme->{check_cn} eq 'when_only' and !$alt_dnsNames)) {
$check_name->($commonName,$identity,$scheme->{wildcards_in_cn})
and return 1;
}
return 0; # no match
}
}
EOP
eval { require IO::Socket::SSL };
if ( $INC{"IO/Socket/SSL.pm"} ) {
eval $prog;
die $@ if $@;
}
1;
}
# ###########################################################################
# End HTTPMicro package
# ###########################################################################
# ###########################################################################
# VersionCheck package
# This package is a copy without comments from the original. The original
# with comments and its test file can be found in the Bazaar repository at,
# lib/VersionCheck.pm
# t/lib/VersionCheck.t
# See https://launchpad.net/percona-toolkit for more information.
# ###########################################################################
{
package VersionCheck;
use strict;
use warnings FATAL => 'all';
use English qw(-no_match_vars);
use constant PTDEBUG => $ENV{PTDEBUG} || 0;
use Data::Dumper;
local $Data::Dumper::Indent = 1;
local $Data::Dumper::Sortkeys = 1;
local $Data::Dumper::Quotekeys = 0;
use Digest::MD5 qw(md5_hex);
use Sys::Hostname qw(hostname);
use File::Basename qw();
use File::Spec;
use FindBin qw();
eval {
require Percona::Toolkit;
require HTTPMicro;
};
{
my $file = 'percona-version-check';
my $home = $ENV{HOME} || $ENV{HOMEPATH} || $ENV{USERPROFILE} || '.';
my @vc_dirs = (
'/etc/percona',
'/etc/percona-toolkit',
'/tmp',
"$home",
);
if ($ENV{PTDEBUG_VERSION_CHECK_HOME}) {
@vc_dirs = ( $ENV{PTDEBUG_VERSION_CHECK_HOME} );
}
sub version_check_file {
foreach my $dir ( @vc_dirs ) {
if ( -d $dir && -w $dir ) {
PTDEBUG && _d('Version check file', $file, 'in', $dir);
return $dir . '/' . $file;
}
}
PTDEBUG && _d('Version check file', $file, 'in', $ENV{PWD});
return $file; # in the CWD
}
}
sub version_check_time_limit {
return 60 * 60 * 24; # one day
}
sub version_check {
my (%args) = @_;
my $instances = $args{instances} || [];
my $instances_to_check;
PTDEBUG && _d('FindBin::Bin:', $FindBin::Bin);
if ( !$args{force} ) {
if ( $FindBin::Bin
&& (-d "$FindBin::Bin/../.bzr" || -d "$FindBin::Bin/../../.bzr") ) {
PTDEBUG && _d("$FindBin::Bin/../.bzr disables --version-check");
return;
}
}
eval {
foreach my $instance ( @$instances ) {
my ($name, $id) = get_instance_id($instance);
$instance->{name} = $name;
$instance->{id} = $id;
}
push @$instances, { name => 'system', id => 0 };
$instances_to_check = get_instances_to_check(
instances => $instances,
vc_file => $args{vc_file}, # testing
now => $args{now}, # testing
);
PTDEBUG && _d(scalar @$instances_to_check, 'instances to check');
return unless @$instances_to_check;
my $protocol = 'https';
eval { require IO::Socket::SSL; };
if ( $EVAL_ERROR ) {
PTDEBUG && _d($EVAL_ERROR);
PTDEBUG && _d("SSL not available, won't run version_check");
return;
}
PTDEBUG && _d('Using', $protocol);
my $advice = pingback(
instances => $instances_to_check,
protocol => $protocol,
url => $args{url} # testing
|| $ENV{PERCONA_VERSION_CHECK_URL} # testing
|| "$protocol://v.percona.com",
);
if ( $advice ) {
PTDEBUG && _d('Advice:', Dumper($advice));
if ( scalar @$advice > 1) {
print "\n# " . scalar @$advice . " software updates are "
. "available:\n";
}
else {
print "\n# A software update is available:\n";
}
print join("\n", map { "# * $_" } @$advice), "\n\n";
}
};
if ( $EVAL_ERROR ) {
PTDEBUG && _d('Version check failed:', $EVAL_ERROR);
}
if ( @$instances_to_check ) {
eval {
update_check_times(
instances => $instances_to_check,
vc_file => $args{vc_file}, # testing
now => $args{now}, # testing
);
};
if ( $EVAL_ERROR ) {
PTDEBUG && _d('Error updating version check file:', $EVAL_ERROR);
}
}
if ( $ENV{PTDEBUG_VERSION_CHECK} ) {
warn "Exiting because the PTDEBUG_VERSION_CHECK "
. "environment variable is defined.\n";
exit 255;
}
return;
}
sub get_instances_to_check {
my (%args) = @_;
my $instances = $args{instances};
my $now = $args{now} || int(time);
my $vc_file = $args{vc_file} || version_check_file();
if ( !-f $vc_file ) {
PTDEBUG && _d('Version check file', $vc_file, 'does not exist;',
'version checking all instances');
return $instances;
}
open my $fh, '<', $vc_file or die "Cannot open $vc_file: $OS_ERROR";
chomp(my $file_contents = do { local $/ = undef; <$fh> });
PTDEBUG && _d('Version check file', $vc_file, 'contents:', $file_contents);
close $fh;
my %last_check_time_for = $file_contents =~ /^([^,]+),(.+)$/mg;
my $check_time_limit = version_check_time_limit();
my @instances_to_check;
foreach my $instance ( @$instances ) {
my $last_check_time = $last_check_time_for{ $instance->{id} };
PTDEBUG && _d('Intsance', $instance->{id}, 'last checked',
$last_check_time, 'now', $now, 'diff', $now - ($last_check_time || 0),
'hours until next check',
sprintf '%.2f',
($check_time_limit - ($now - ($last_check_time || 0))) / 3600);
if ( !defined $last_check_time
|| ($now - $last_check_time) >= $check_time_limit ) {
PTDEBUG && _d('Time to check', Dumper($instance));
push @instances_to_check, $instance;
}
}
return \@instances_to_check;
}
sub update_check_times {
my (%args) = @_;
my $instances = $args{instances};
my $now = $args{now} || int(time);
my $vc_file = $args{vc_file} || version_check_file();
PTDEBUG && _d('Updating last check time:', $now);
my %all_instances = map {
$_->{id} => { name => $_->{name}, ts => $now }
} @$instances;
if ( -f $vc_file ) {
open my $fh, '<', $vc_file or die "Cannot read $vc_file: $OS_ERROR";
my $contents = do { local $/ = undef; <$fh> };
close $fh;
foreach my $line ( split("\n", ($contents || '')) ) {
my ($id, $ts) = split(',', $line);
if ( !exists $all_instances{$id} ) {
$all_instances{$id} = { ts => $ts }; # original ts, not updated
}
}
}
open my $fh, '>', $vc_file or die "Cannot write to $vc_file: $OS_ERROR";
foreach my $id ( sort keys %all_instances ) {
PTDEBUG && _d('Updated:', $id, Dumper($all_instances{$id}));
print { $fh } $id . ',' . $all_instances{$id}->{ts} . "\n";
}
close $fh;
return;
}
sub get_instance_id {
my ($instance) = @_;
my $dbh = $instance->{dbh};
my $dsn = $instance->{dsn};
my $sql = q{SELECT CONCAT(@@hostname, @@port)};
PTDEBUG && _d($sql);
my ($name) = eval { $dbh->selectrow_array($sql) };
if ( $EVAL_ERROR ) {
PTDEBUG && _d($EVAL_ERROR);
$sql = q{SELECT @@hostname};
PTDEBUG && _d($sql);
($name) = eval { $dbh->selectrow_array($sql) };
if ( $EVAL_ERROR ) {
PTDEBUG && _d($EVAL_ERROR);
$name = ($dsn->{h} || 'localhost') . ($dsn->{P} || 3306);
}
else {
$sql = q{SHOW VARIABLES LIKE 'port'};
PTDEBUG && _d($sql);
my (undef, $port) = eval { $dbh->selectrow_array($sql) };
PTDEBUG && _d('port:', $port);
$name .= $port || '';
}
}
my $id = md5_hex($name);
PTDEBUG && _d('MySQL instance:', $id, $name, Dumper($dsn));
return $name, $id;
}
sub pingback {
my (%args) = @_;
my @required_args = qw(url instances);
foreach my $arg ( @required_args ) {
die "I need a $arg arugment" unless $args{$arg};
}
my $url = $args{url};
my $instances = $args{instances};
my $ua = $args{ua} || HTTPMicro->new( timeout => 3 );
my $response = $ua->request('GET', $url);
PTDEBUG && _d('Server response:', Dumper($response));
die "No response from GET $url"
if !$response;
die("GET on $url returned HTTP status $response->{status}; expected 200\n",
($response->{content} || '')) if $response->{status} != 200;
die("GET on $url did not return any programs to check")
if !$response->{content};
my $items = parse_server_response(
response => $response->{content}
);
die "Failed to parse server requested programs: $response->{content}"
if !scalar keys %$items;
my $versions = get_versions(
items => $items,
instances => $instances,
);
die "Failed to get any program versions; should have at least gotten Perl"
if !scalar keys %$versions;
my $client_content = encode_client_response(
items => $items,
versions => $versions,
general_id => md5_hex( hostname() ),
);
my $client_response = {
headers => { "X-Percona-Toolkit-Tool" => File::Basename::basename($0) },
content => $client_content,
};
PTDEBUG && _d('Client response:', Dumper($client_response));
$response = $ua->request('POST', $url, $client_response);
PTDEBUG && _d('Server suggestions:', Dumper($response));
die "No response from POST $url $client_response"
if !$response;
die "POST $url returned HTTP status $response->{status}; expected 200"
if $response->{status} != 200;
return unless $response->{content};
$items = parse_server_response(
response => $response->{content},
split_vars => 0,
);
die "Failed to parse server suggestions: $response->{content}"
if !scalar keys %$items;
my @suggestions = map { $_->{vars} }
sort { $a->{item} cmp $b->{item} }
values %$items;
return \@suggestions;
}
sub encode_client_response {
my (%args) = @_;
my @required_args = qw(items versions general_id);
foreach my $arg ( @required_args ) {
die "I need a $arg arugment" unless $args{$arg};
}
my ($items, $versions, $general_id) = @args{@required_args};
my @lines;
foreach my $item ( sort keys %$items ) {
next unless exists $versions->{$item};
if ( ref($versions->{$item}) eq 'HASH' ) {
my $mysql_versions = $versions->{$item};
for my $id ( sort keys %$mysql_versions ) {
push @lines, join(';', $id, $item, $mysql_versions->{$id});
}
}
else {
push @lines, join(';', $general_id, $item, $versions->{$item});
}
}
my $client_response = join("\n", @lines) . "\n";
return $client_response;
}
sub parse_server_response {
my (%args) = @_;
my @required_args = qw(response);
foreach my $arg ( @required_args ) {
die "I need a $arg arugment" unless $args{$arg};
}
my ($response) = @args{@required_args};
my %items = map {
my ($item, $type, $vars) = split(";", $_);
if ( !defined $args{split_vars} || $args{split_vars} ) {
$vars = [ split(",", ($vars || '')) ];
}
$item => {
item => $item,
type => $type,
vars => $vars,
};
} split("\n", $response);
PTDEBUG && _d('Items:', Dumper(\%items));
return \%items;
}
my %sub_for_type = (
os_version => \&get_os_version,
perl_version => \&get_perl_version,
perl_module_version => \&get_perl_module_version,
mysql_variable => \&get_mysql_variable,
);
sub valid_item {
my ($item) = @_;
return unless $item;
if ( !exists $sub_for_type{ $item->{type} } ) {
PTDEBUG && _d('Invalid type:', $item->{type});
return 0;
}
return 1;
}
sub get_versions {
my (%args) = @_;
my @required_args = qw(items);
foreach my $arg ( @required_args ) {
die "I need a $arg arugment" unless $args{$arg};
}
my ($items) = @args{@required_args};
my %versions;
foreach my $item ( values %$items ) {
next unless valid_item($item);
eval {
my $version = $sub_for_type{ $item->{type} }->(
item => $item,
instances => $args{instances},
);
if ( $version ) {
chomp $version unless ref($version);
$versions{$item->{item}} = $version;
}
};
if ( $EVAL_ERROR ) {
PTDEBUG && _d('Error getting version for', Dumper($item), $EVAL_ERROR);
}
}
return \%versions;
}
sub get_os_version {
if ( $OSNAME eq 'MSWin32' ) {
require Win32;
return Win32::GetOSDisplayName();
}
chomp(my $platform = `uname -s`);
PTDEBUG && _d('platform:', $platform);
return $OSNAME unless $platform;
chomp(my $lsb_release
= `which lsb_release 2>/dev/null | awk '{print \$1}'` || '');
PTDEBUG && _d('lsb_release:', $lsb_release);
my $release = "";
if ( $platform eq 'Linux' ) {
if ( -f "/etc/fedora-release" ) {
$release = `cat /etc/fedora-release`;
}
elsif ( -f "/etc/redhat-release" ) {
$release = `cat /etc/redhat-release`;
}
elsif ( -f "/etc/system-release" ) {
$release = `cat /etc/system-release`;
}
elsif ( $lsb_release ) {
$release = `$lsb_release -ds`;
}
elsif ( -f "/etc/lsb-release" ) {
$release = `grep DISTRIB_DESCRIPTION /etc/lsb-release`;
$release =~ s/^\w+="([^"]+)".+/$1/;
}
elsif ( -f "/etc/debian_version" ) {
chomp(my $rel = `cat /etc/debian_version`);
$release = "Debian $rel";
if ( -f "/etc/apt/sources.list" ) {
chomp(my $code_name = `awk '/^deb/ {print \$3}' /etc/apt/sources.list | awk -F/ '{print \$1}'| awk 'BEGIN {FS="|"} {print \$1}' | sort | uniq -c | sort -rn | head -n1 | awk '{print \$2}'`);
$release .= " ($code_name)" if $code_name;
}
}
elsif ( -f "/etc/os-release" ) { # openSUSE
chomp($release = `grep PRETTY_NAME /etc/os-release`);
$release =~ s/^PRETTY_NAME="(.+)"$/$1/;
}
elsif ( `ls /etc/*release 2>/dev/null` ) {
if ( `grep DISTRIB_DESCRIPTION /etc/*release 2>/dev/null` ) {
$release = `grep DISTRIB_DESCRIPTION /etc/*release | head -n1`;
}
else {
$release = `cat /etc/*release | head -n1`;
}
}
}
elsif ( $platform =~ m/(?:BSD|^Darwin)$/ ) {
my $rel = `uname -r`;
$release = "$platform $rel";
}
elsif ( $platform eq "SunOS" ) {
my $rel = `head -n1 /etc/release` || `uname -r`;
$release = "$platform $rel";
}
if ( !$release ) {
PTDEBUG && _d('Failed to get the release, using platform');
$release = $platform;
}
chomp($release);
$release =~ s/^"|"$//g;
PTDEBUG && _d('OS version =', $release);
return $release;
}
sub get_perl_version {
my (%args) = @_;
my $item = $args{item};
return unless $item;
my $version = sprintf '%vd', $PERL_VERSION;
PTDEBUG && _d('Perl version', $version);
return $version;
}
sub get_perl_module_version {
my (%args) = @_;
my $item = $args{item};
return unless $item;
my $var = '$' . $item->{item} . '::VERSION';
my $version = eval "use $item->{item}; $var;";
PTDEBUG && _d('Perl version for', $var, '=', $version);
return $version;
}
sub get_mysql_variable {
return get_from_mysql(
show => 'VARIABLES',
@_,
);
}
sub get_from_mysql {
my (%args) = @_;
my $show = $args{show};
my $item = $args{item};
my $instances = $args{instances};
return unless $show && $item;
if ( !$instances || !@$instances ) {
PTDEBUG && _d('Cannot check', $item,
'because there are no MySQL instances');
return;
}
if ($item->{item} eq 'MySQL' && $item->{type} eq 'mysql_variable') {
$item->{vars} = ['version_comment', 'version'];
}
my @versions;
my %version_for;
foreach my $instance ( @$instances ) {
next unless $instance->{id}; # special system instance has id=0
my $dbh = $instance->{dbh};
local $dbh->{FetchHashKeyName} = 'NAME_lc';
my $sql = qq/SHOW $show/;
PTDEBUG && _d($sql);
my $rows = $dbh->selectall_hashref($sql, 'variable_name');
my @versions;
foreach my $var ( @{$item->{vars}} ) {
$var = lc($var);
my $version = $rows->{$var}->{value};
PTDEBUG && _d('MySQL version for', $item->{item}, '=', $version,
'on', $instance->{name});
push @versions, $version;
}
$version_for{ $instance->{id} } = join(' ', @versions);
}
return \%version_for;
}
sub _d {
my ($package, undef, $line) = caller 0;
@_ = map { (my $temp = $_) =~ s/\n/\n# /g; $temp; }
map { defined $_ ? $_ : 'undef' }
@_;
print STDERR "# $package:$line $PID ", join(' ', @_), "\n";
}
1;
}
# ###########################################################################
# End VersionCheck package
# ###########################################################################
#
# parse_connection_options() subroutine parses connection-related command line
# options
#
sub parse_connection_options {
my $con = shift;
$con->{dsn} = 'dbi:mysql:';
# this option has to be first
if ($ENV{option_defaults_file}) {
$con->{dsn} .= ";mysql_read_default_file=$ENV{option_defaults_file}";
}
if ($ENV{option_defaults_extra_file}) {
$con->{dsn} .= ";mysql_read_default_file=$ENV{option_defaults_extra_file}";
}
$con->{dsn} .= ";mysql_read_default_group=xtrabackup";
if ($ENV{option_mysql_password}) {
$con->{dsn_password} = "$ENV{option_mysql_password}";
}
if ($ENV{option_mysql_user}) {
$con->{dsn_user} = "$ENV{option_mysql_user}";
}
if ($ENV{option_mysql_host}) {
$con->{dsn} .= ";host=$ENV{option_mysql_host}";
}
if ($ENV{option_mysql_port}) {
$con->{dsn} .= ";port=$ENV{option_mysql_port}";
}
if ($ENV{option_mysql_socket}) {
$con->{dsn} .= ";mysql_socket=$ENV{option_mysql_socket}";
}
}
#
# mysql_connect subroutine connects to MySQL server
#
sub mysql_connect {
my %con;
my %args = (
# Defaults
abort_on_error => 1,
@_
);
$con{abort_on_error} = $args{abort_on_error};
parse_connection_options(\%con);
$now = current_time();
print STDERR "$now $prefix Connecting to MySQL server with DSN '$con{dsn}'" .
(defined($con{dsn_user}) ? " as '$con{dsn_user}' " : "") .
" (using password: ";
if (defined($con{dsn_password})) {
print STDERR "YES).\n";
} else {
print STDERR "NO).\n";
}
eval {
$con{dbh}=DBI->connect($con{dsn}, $con{dsn_user},
$con{dsn_password}, { RaiseError => 1 });
};
if ($EVAL_ERROR) {
$con{connect_error}=$EVAL_ERROR;
} else {
$now = current_time();
print STDERR "$now $prefix Connected to MySQL server\n";
}
if ($args{abort_on_error}) {
if (!$dbd_mysql_installed) {
die "Failed to connect to MySQL server as " .
"DBD::mysql module is not installed";
} else {
if (!$con{dbh}) {
die "Failed to connect to MySQL server: " .
$con{connect_error};
}
}
}
if ($con{dbh}) {
$con{dbh}->do("SET SESSION wait_timeout=2147483");
}
return %con;
}
#
# return current local time as string in form "070816 12:23:15"
#
sub current_time {
return strftime("%y%m%d %H:%M:%S", localtime());
}
%mysql = mysql_connect(abort_on_error => 1);
$now = current_time();
print STDERR
"$now $prefix Executing a version check against the server...\n";
# Redirect STDOUT to STDERR, as VersionCheck prints alerts to STDOUT
select STDERR;
VersionCheck::version_check(
force => 1,
instances => [ {
dbh => $mysql{dbh},
dsn => $mysql{dsn}
}
]
);
# Restore STDOUT as the default filehandle
select STDOUT;
$now = current_time();
print STDERR "$now $prefix Done.\n";
extra/mariabackup/write_filt.cc
View file @ ce4c56db
......@@ -75,7 +75,7 @@ wf_incremental_init(xb_write_filt_ctxt_t *ctxt, char *dst_name,
ctxt->cursor = cursor;
/* allocate buffer for incremental backup (4096 pages) */
buf_size = (UNIV_PAGE_SIZE_MAX / 4 + 1) * UNIV_PAGE_SIZE_MAX;
buf_size = (cursor->page_size / 4 + 1) * cursor->page_size;
cp->delta_buf_base = static_cast<byte *>(ut_malloc(buf_size));
memset(cp->delta_buf_base, 0, buf_size);
cp->delta_buf = static_cast<byte *>
......
extra/mariabackup/write_filt.h
View file @ ce4c56db
......@@ -27,7 +27,6 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA
#include "fil_cur.h"
#include "datasink.h"
#include "compact.h"
/* Incremental page filter context */
typedef struct {
......@@ -41,7 +40,6 @@ typedef struct {
xb_fil_cur_t *cursor;
union {
xb_wf_incremental_ctxt_t wf_incremental_ctxt;
xb_wf_compact_ctxt_t wf_compact_ctxt;
} u;
} xb_write_filt_ctxt_t;
......@@ -56,6 +54,5 @@ typedef struct {
extern xb_write_filt_t wf_write_through;
extern xb_write_filt_t wf_incremental;
extern xb_write_filt_t wf_compact;
#endif /* XB_WRITE_FILT_H */
extra/mariabackup/wsrep.cc
View file @ ce4c56db
......@@ -46,7 +46,7 @@ permission notice:
#include <trx0sys.h>
#include "common.h"
#ifdef WITH_WSREP
#define WSREP_XID_PREFIX "WSREPXid"
#define WSREP_XID_PREFIX_LEN MYSQL_XID_PREFIX_LEN
#define WSREP_XID_UUID_OFFSET 8
......@@ -61,11 +61,11 @@ permission notice:
/* Galera UUID type - for all unique IDs */
typedef struct wsrep_uuid {
uint8_t data[16];
unsigned char data[16];
} wsrep_uuid_t;
/* sequence number of a writeset, etc. */
typedef int64_t wsrep_seqno_t;
typedef long long wsrep_seqno_t;
/* Undefined UUID */
static const wsrep_uuid_t WSREP_UUID_UNDEFINED = {{0,}};
......@@ -217,3 +217,4 @@ xb_write_galera_info(bool incremental_prepare)
fclose(fp);
}
#endif
extra/mariabackup/xb0xb.h 0 → 100644
View file @ ce4c56db
/******************************************************
Copyright (c) 2012 Percona LLC and/or its affiliates.
Declarations of XtraBackup functions called by InnoDB code.
This program is free software; you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation; version 2 of the License.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program; if not, write to the Free Software
Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA
*******************************************************/
#ifndef xb0xb_h
#define xb0xb_h
extern void os_io_init_simple(void);
extern os_file_t files[1000];
extern const char *innodb_checksum_algorithm_names[];
extern TYPELIB innodb_checksum_algorithm_typelib;
extern dberr_t open_or_create_data_files(
ibool* create_new_db,
#ifdef UNIV_LOG_ARCHIVE
lsn_t* min_arch_log_no,
lsn_t* max_arch_log_no,
#endif
lsn_t* min_flushed_lsn,
lsn_t* max_flushed_lsn,
ulint* sum_of_new_sizes)
;
int
fil_file_readdir_next_file(
/*=======================*/
dberr_t* err, /*!< out: this is set to DB_ERROR if an error
was encountered, otherwise not changed */
const char* dirname,/*!< in: directory name or path */
os_file_dir_t dir, /*!< in: directory stream */
os_file_stat_t* info) /*!< in/out: buffer where the
info is returned */;
buf_block_t* btr_node_ptr_get_child(
const rec_t* node_ptr,/*!< in: node pointer */
dict_index_t* index, /*!< in: index */
const ulint* offsets,/*!< in: array returned by rec_get_offsets() */
mtr_t* mtr) /*!< in: mtr */;
buf_block_t*
btr_root_block_get(
/*===============*/
const dict_index_t* index, /*!< in: index tree */
ulint mode, /*!< in: either RW_S_LATCH
or RW_X_LATCH */
mtr_t* mtr) /*!< in: mtr */;
fil_space_t*
fil_space_get_by_name(const char *);
ibool
recv_check_cp_is_consistent(const byte* buf);
void
innodb_log_checksum_func_update(
/*============================*/
ulint algorithm) /*!< in: algorithm */;
dberr_t recv_find_max_checkpoint(log_group_t** max_group, ulint* max_field);
dberr_t
srv_undo_tablespaces_init(
/*======================*/
ibool create_new_db,
ibool backup_mode,
const ulint n_conf_tablespaces,
ulint* n_opened);
#endif
extra/mariabackup/xb_regex.h
View file @ ce4c56db
......@@ -22,50 +22,27 @@ my_regex is used on Windows and native calls are used on POSIX platforms. */
#ifndef XB_REGEX_H
#define XB_REGEX_H
#ifdef _WIN32
#include <my_regex.h>
typedef my_regex_t xb_regex_t;
typedef my_regmatch_t xb_regmatch_t;
#define xb_regex_init() my_regex_init(&my_charset_latin1)
#define xb_regexec(preg,string,nmatch,pmatch,eflags) \
my_regexec(preg, string, nmatch, pmatch, eflags)
#define xb_regerror(errcode,preg,errbuf,errbuf_size) \
my_regerror(errcode, preg, errbuf, errbuf_size)
#define xb_regcomp(preg,regex,cflags) \
my_regcomp(preg, regex, cflags, &my_charset_latin1)
#define xb_regfree(preg) my_regfree(preg)
#define xb_regex_end() my_regex_end()
#else /* ! _WIN32 */
#ifdef HAVE_SYSTEM_REGEX
#include <regex.h>
#else
#include <pcreposix.h>
#endif
typedef regex_t xb_regex_t;
typedef regmatch_t xb_regmatch_t;
typedef regex_t* xb_regex_t;
#define xb_regex_init() do { } while(0)
#define xb_regex_init()
#define xb_regexec(preg,string,nmatch,pmatch,eflags) \
#define xb_regexec(preg,string,nmatch,pmatch,eflags) \
regexec(preg, string, nmatch, pmatch, eflags)
#define xb_regerror(errcode,preg,errbuf,errbuf_size) \
#define xb_regerror(errcode,preg,errbuf,errbuf_size) \
regerror(errcode, preg, errbuf, errbuf_size)
#define xb_regcomp(preg,regex,cflags) \
#define xb_regcomp(preg,regex,cflags) \
regcomp(preg, regex, cflags)
#define xb_regfree(preg) regfree(preg)
#define xb_regex_end() do { } while (0)
#endif /* _WIN32 */
#define xb_regex_end()
#endif /* XB_REGEX_H */
extra/mariabackup/xbcrypt_common.c
View file @ ce4c56db
......@@ -27,6 +27,7 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA
# pragma GCC diagnostic ignored "-Wdeprecated-declarations"
#endif
#ifdef HAVE_GRYPT
#include <gcrypt.h>
#if GCC_VERSION >= 4002
......@@ -58,3 +59,4 @@ xb_crypt_create_iv(void* ivbuf, size_t ivlen)
{
gcry_create_nonce(ivbuf, ivlen);
}
#endif
\ No newline at end of file
extra/mariabackup/xbcrypt_write.c
View file @ ce4c56db
......@@ -73,7 +73,7 @@ int xb_crypt_write_chunk(xb_wcrypt_t *crypt, const void *buf, size_t olen,
int8store(ptr, (ulonglong)elen); /* encrypted (actual) size */
ptr += 8;
checksum = crc32(0, buf, elen);
checksum = crc32(0, buf, (uint)elen);
int4store(ptr, checksum); /* checksum */
ptr += 4;
......
extra/mariabackup/xbstream.c
View file @ ce4c56db
......@@ -208,15 +208,15 @@ int
stream_one_file(File file, xb_wstream_file_t *xbfile)
{
uchar *buf;
size_t bytes;
size_t offset;
ssize_t bytes;
my_off_t offset;
posix_fadvise(file, 0, 0, POSIX_FADV_SEQUENTIAL);
offset = my_tell(file, MYF(MY_WME));
buf = (uchar*)(my_malloc(XBSTREAM_BUFFER_SIZE, MYF(MY_FAE)));
while ((bytes = my_read(file, buf, XBSTREAM_BUFFER_SIZE,
while ((bytes = (ssize_t)my_read(file, buf, XBSTREAM_BUFFER_SIZE,
MYF(MY_WME))) > 0) {
if (xb_stream_write_data(xbfile, buf, bytes)) {
msg("%s: xb_stream_write_data() failed.\n",
......@@ -232,7 +232,7 @@ stream_one_file(File file, xb_wstream_file_t *xbfile)
my_free(buf);
if (bytes == (size_t) -1) {
if (bytes < 0) {
return 1;
}
......
extra/mariabackup/xbstream_read.c
View file @ ce4c56db
......@@ -48,13 +48,13 @@ xb_stream_read_new(void)
stream->buffer = my_malloc(INIT_BUFFER_LEN, MYF(MY_FAE));
stream->buflen = INIT_BUFFER_LEN;
stream->fd = fileno(stdin);
stream->offset = 0;
#ifdef __WIN__
setmode(stream->fd, _O_BINARY);
setmode(fileno(stdin), _O_BINARY);
#endif
stream->fd = my_fileno(stdin);
stream->offset = 0;
return stream;
}
......
extra/mariabackup/xbstream_write.c
View file @ ce4c56db
......@@ -34,7 +34,7 @@ struct xb_wstream_struct {
struct xb_wstream_file_struct {
xb_wstream_t *stream;
char *path;
ulong path_len;
size_t path_len;
char chunk[XB_STREAM_MIN_CHUNK_SIZE];
char *chunk_ptr;
size_t chunk_free;
......@@ -54,7 +54,7 @@ xb_stream_default_write_callback(xb_wstream_file_t *file __attribute__((unused))
void *userdata __attribute__((unused)),
const void *buf, size_t len)
{
if (my_write(fileno(stdout), buf, len, MYF(MY_WME | MY_NABP)))
if (my_write(my_fileno(stdout), buf, len, MYF(MY_WME | MY_NABP)))
return -1;
return len;
}
......@@ -77,7 +77,7 @@ xb_stream_write_open(xb_wstream_t *stream, const char *path,
xb_stream_write_callback *onwrite)
{
xb_wstream_file_t *file;
ulong path_len;
size_t path_len;
path_len = strlen(path);
......@@ -90,7 +90,19 @@ xb_stream_write_open(xb_wstream_t *stream, const char *path,
path_len + 1, MYF(MY_FAE));
file->path = (char *) (file + 1);
#ifdef _WIN32
/* Normalize path on Windows, so we can restore elsewhere.*/
{
int i;
for (i = 0; ; i++) {
file->path[i] = (path[i] == '\\') ? '/' : path[i];
if (!path[i])
break;
}
}
#else
memcpy(file->path, path, path_len + 1);
#endif
file->path_len = path_len;
file->stream = stream;
......@@ -208,7 +220,7 @@ xb_stream_write_chunk(xb_wstream_file_t *file, const void *buf, size_t len)
int8store(ptr, file->offset); /* Payload offset */
ptr += 8;
checksum = crc32(0, buf, len); /* checksum */
checksum = crc32(0, buf, (uint)len); /* checksum */
int4store(ptr, checksum);
ptr += 4;
......
extra/mariabackup/xtrabackup.cc
View file @ ce4c56db
......@@ -53,9 +53,9 @@ Place, Suite 330, Boston, MA 02111-1307 USA
#ifdef __linux__
# include <sys/prctl.h>
#include <sys/resource.h>
#endif
#include <sys/resource.h>
#include <btr0sea.h>
#include <dict0priv.h>
......@@ -74,7 +74,6 @@ Place, Suite 330, Boston, MA 02111-1307 USA
#define G_PTR uchar*
#include "common.h"
#include "xtrabackup_version.h"
#include "datasink.h"
#include "xb_regex.h"
......@@ -91,6 +90,10 @@ Place, Suite 330, Boston, MA 02111-1307 USA
#include "backup_mysql.h"
#include "backup_copy.h"
#include "backup_mysql.h"
#include "xb0xb.h"
#include "encryption_plugin.h"
#include <sql_plugin.h>
#include <srv0srv.h>
/* TODO: replace with appropriate macros used in InnoDB 5.6 */
#define PAGE_ZIP_MIN_SIZE_SHIFT 10
......@@ -143,15 +146,16 @@ lsn_t xtrabackup_archived_to_lsn = 0; /* for --archived-to-lsn */
char *xtrabackup_tables = NULL;
char *xtrabackup_tmpdir;
/* List of regular expressions for filtering */
typedef struct xb_regex_list_node_struct xb_regex_list_node_t;
struct xb_regex_list_node_struct {
UT_LIST_NODE_T(xb_regex_list_node_t) regex_list;
xb_regex_t regex;
regex_t regex;
};
static UT_LIST_BASE_NODE_T(xb_regex_list_node_t) regex_list;
static xb_regmatch_t tables_regmatch[1];
static regmatch_t tables_regmatch[1];
char *xtrabackup_tables_file = NULL;
static hash_table_t* tables_hash = NULL;
......@@ -210,6 +214,7 @@ ulint xtrabackup_rebuild_threads = 1;
/* sleep interval beetween log copy iterations in log copying thread
in milliseconds (default is 1 second) */
ulint xtrabackup_log_copy_interval = 1000;
static ulong max_buf_pool_modified_pct;
/* Ignored option (--log) for MySQL option compatibility */
char* log_ignored_opt = NULL;
......@@ -294,8 +299,6 @@ it every INNOBASE_WAKE_INTERVAL'th step. */
#define INNOBASE_WAKE_INTERVAL 32
ulong innobase_active_counter = 0;
ibool srv_compact_backup = FALSE;
ibool srv_rebuild_indexes = FALSE;
static char *xtrabackup_debug_sync = NULL;
......@@ -309,13 +312,15 @@ lsn_t min_flushed_lsn= 0;
lsn_t max_flushed_lsn= 0;
/* The size of archived log file */
size_t xtrabackup_arch_file_size = 0ULL;
ib_int64_t xtrabackup_arch_file_size = 0ULL;
/* The minimal LSN of found archived log files */
lsn_t xtrabackup_arch_first_file_lsn = 0ULL;
/* The maximum LSN of found archived log files */
lsn_t xtrabackup_arch_last_file_lsn = 0ULL;
ulong xb_open_files_limit= 0;
char *xb_plugin_dir;
char *xb_plugin_load;
my_bool xb_close_files= FALSE;
/* Datasinks */
......@@ -574,6 +579,9 @@ enum options_xtrabackup
OPT_XTRA_INCREMENTAL_FORCE_SCAN,
OPT_DEFAULTS_GROUP,
OPT_OPEN_FILES_LIMIT,
OPT_PLUGIN_DIR,
OPT_PLUGIN_LOAD,
OPT_INNODB_ENCRYPT_LOG,
OPT_CLOSE_FILES,
OPT_CORE_FILE,
......@@ -682,7 +690,13 @@ struct my_option xb_client_options[] =
0, GET_BOOL, NO_ARG, 0, 0, 0, 0, 0, 0},
{"stream", OPT_XTRA_STREAM, "Stream all backup files to the standard output "
"in the specified format. Currently the only supported format is 'tar'.",
"in the specified format."
#ifdef HAVE_LIBARCHIVE
"Supported formats are 'tar' and 'xbstream'."
#else
"Supported format is 'xbstream'."
#endif
,
(G_PTR*) &xtrabackup_stream_str, (G_PTR*) &xtrabackup_stream_str, 0, GET_STR,
REQUIRED_ARG, 0, 0, 0, 0, 0, 0},
......@@ -1001,12 +1015,6 @@ struct my_option xb_client_options[] =
#include "sslopt-longopts.h"
#if !defined(HAVE_YASSL)
{"server-public-key-path", OPT_SERVER_PUBLIC_KEY,
"File path to the server public RSA key in PEM format.",
&opt_server_public_key, &opt_server_public_key, 0,
GET_STR, REQUIRED_ARG, 0, 0, 0, 0, 0, 0},
#endif
{ 0, 0, 0, 0, 0, 0, GET_NO_ARG, NO_ARG, 0, 0, 0, 0, 0, 0}
};
......@@ -1098,11 +1106,7 @@ Disable with --skip-innodb-doublewrite.", (G_PTR*) &innobase_use_doublewrite,
(G_PTR*) &innobase_file_per_table,
(G_PTR*) &innobase_file_per_table, 0, GET_BOOL, NO_ARG,
FALSE, 0, 0, 0, 0, 0},
{"innodb_flush_log_at_trx_commit", OPT_INNODB_FLUSH_LOG_AT_TRX_COMMIT,
"Set to 0 (write and flush once per second), 1 (write and flush at each commit) or 2 (write at commit, flush once per second).",
(G_PTR*) &srv_flush_log_at_trx_commit,
(G_PTR*) &srv_flush_log_at_trx_commit,
0, GET_ULONG, OPT_ARG, 1, 0, 2, 0, 0, 0},
{"innodb_flush_method", OPT_INNODB_FLUSH_METHOD,
"With which method to flush data.", (G_PTR*) &innobase_unix_file_flush_method,
(G_PTR*) &innobase_unix_file_flush_method, 0, GET_STR, REQUIRED_ARG, 0, 0, 0,
......@@ -1203,6 +1207,18 @@ Disable with --skip-innodb-doublewrite.", (G_PTR*) &innobase_use_doublewrite,
(G_PTR*) &defaults_group, (G_PTR*) &defaults_group,
0, GET_STR, REQUIRED_ARG, 0, 0, 0, 0, 0, 0},
{"plugin-dir", OPT_PLUGIN_DIR, "Server plugin directory",
&xb_plugin_dir, &xb_plugin_dir,
0, GET_STR, REQUIRED_ARG, 0, 0, 0, 0, 0, 0 },
{ "plugin-load", OPT_PLUGIN_LOAD, "encrypton plugin to load",
&xb_plugin_load, &xb_plugin_load,
0, GET_STR, REQUIRED_ARG, 0, 0, 0, 0, 0, 0 },
{ "innodb-encrypt-log", OPT_INNODB_ENCRYPT_LOG, "encrypton plugin to load",
&srv_encrypt_log, &srv_encrypt_log,
0, GET_BOOL, REQUIRED_ARG, 0, 0, 0, 0, 0, 0 },
{"open_files_limit", OPT_OPEN_FILES_LIMIT, "the maximum number of file "
"descriptors to reserve with setrlimit().",
(G_PTR*) &xb_open_files_limit, (G_PTR*) &xb_open_files_limit, 0, GET_ULONG,
......@@ -1276,9 +1292,8 @@ static const char *xb_server_default_groups[]=
static void print_version(void)
{
msg("%s version %s based on MySQL server %s %s (%s) (revision id: %s)\n",
my_progname, XTRABACKUP_VERSION, MYSQL_SERVER_VERSION, SYSTEM_TYPE,
MACHINE_TYPE, XTRABACKUP_REVISION);
msg("%s based on MariaDB server %s %s (%s) \n",
my_progname, MYSQL_SERVER_VERSION, SYSTEM_TYPE, MACHINE_TYPE);
}
static void usage(void)
......@@ -1515,10 +1530,10 @@ xb_init_log_block_size(void)
{
srv_log_block_size = 0;
if (innobase_log_block_size != 512) {
uint n_shift = get_bit_shift(innobase_log_block_size);;
uint n_shift = (uint)get_bit_shift(innobase_log_block_size);;
if (n_shift > 0) {
srv_log_block_size = (1 << n_shift);
srv_log_block_size = (ulint)(1LL << n_shift);
msg("InnoDB: The log block size is set to %lu.\n",
srv_log_block_size);
}
......@@ -1541,13 +1556,13 @@ innodb_init_param(void)
static char current_dir[3]; /* Set if using current lib */
my_bool ret;
char *default_path;
srv_is_being_started = TRUE;
/* === some variables from mysqld === */
memset((G_PTR) &mysql_tmpdir_list, 0, sizeof(mysql_tmpdir_list));
if (init_tmpdir(&mysql_tmpdir_list, opt_mysql_tmpdir))
exit(EXIT_FAILURE);
xtrabackup_tmpdir = my_tmpdir(&mysql_tmpdir_list);
/* dummy for initialize all_charsets[] */
get_charset_name(0);
......@@ -1555,7 +1570,7 @@ innodb_init_param(void)
srv_page_size_shift = 0;
if (innobase_page_size != (1LL << 14)) {
int n_shift = get_bit_shift((ulint) innobase_page_size);
int n_shift = (int)get_bit_shift((ulint) innobase_page_size);
if (n_shift >= 12 && n_shift <= UNIV_PAGE_SIZE_SHIFT_MAX) {
srv_page_size_shift = n_shift;
......@@ -1576,8 +1591,6 @@ innodb_init_param(void)
goto error;
}
srv_fast_checksum = (ibool) innobase_fast_checksum;
/* Check that values don't overflow on 32-bit systems. */
if (sizeof(ulint) == 4) {
if (xtrabackup_use_memory > UINT_MAX32) {
......@@ -1738,10 +1751,15 @@ innodb_init_param(void)
}
btr_search_enabled = (char) innobase_adaptive_hash_index;
btr_search_index_num = 1;
os_use_large_pages = (ibool) innobase_use_large_pages;
os_large_page_size = (ulint) innobase_large_page_size;
if (!innobase_log_arch_dir) {
static char default_dir[3] = "./";
srv_arch_dir = default_dir;
}
row_rollback_on_timeout = (ibool) innobase_rollback_on_timeout;
srv_file_per_table = (my_bool) innobase_file_per_table;
......@@ -1848,7 +1866,7 @@ static my_bool
innodb_init(void)
{
int err;
srv_is_being_started = TRUE;
err = innobase_start_or_create_for_mysql();
if (err != DB_SUCCESS) {
......@@ -1947,12 +1965,6 @@ xtrabackup_read_metadata(char *filename)
}
/* Optional fields */
if (fscanf(fp, "compact = %d\n", &t) == 1) {
xtrabackup_compact = (t == 1);
} else {
xtrabackup_compact = 0;
}
if (fscanf(fp, "recover_binlog_info = %d\n", &t) == 1) {
recover_binlog_info = (t == 1);
}
......@@ -1981,7 +1993,7 @@ xtrabackup_print_metadata(char *buf, size_t buf_len)
metadata_from_lsn,
metadata_to_lsn,
metadata_last_lsn,
MY_TEST(xtrabackup_compact == TRUE),
MY_TEST(false),
MY_TEST(opt_binlog_info == BINLOG_INFO_LOCKLESS));
}
......@@ -2166,7 +2178,7 @@ check_if_table_matches_filters(const char *name)
/* Check against regular expressions list */
for (node = UT_LIST_GET_FIRST(regex_list); node;
node = UT_LIST_GET_NEXT(regex_list, node)) {
regres = xb_regexec(&node->regex, name, 1,
regres = regexec(&node->regex, name, 1,
tables_regmatch, 0);
if (regres != REG_NOMATCH) {
......@@ -2213,7 +2225,7 @@ check_if_skip_table(
dbname = NULL;
tbname = name;
while ((ptr = strchr(tbname, SRV_PATH_SEPARATOR)) != NULL) {
while ((ptr = strchr(tbname, '/')) != NULL) {
dbname = tbname;
tbname = ptr + 1;
}
......@@ -2287,10 +2299,10 @@ xb_get_zip_size(os_file_t file)
ibool success;
ulint space;
buf = static_cast<byte *>(ut_malloc(2 * UNIV_PAGE_SIZE_MAX));
page = static_cast<byte *>(ut_align(buf, UNIV_PAGE_SIZE_MAX));
buf = static_cast<byte *>(ut_malloc(2 * UNIV_PAGE_SIZE));
page = static_cast<byte *>(ut_align(buf, UNIV_PAGE_SIZE));
success = os_file_read(file, page, 0, UNIV_PAGE_SIZE_MAX);
success = os_file_read(file, page, 0, UNIV_PAGE_SIZE);
if (!success) {
goto end;
}
......@@ -2392,8 +2404,6 @@ xtrabackup_copy_datafile(fil_node_t* node, uint thread_n)
/* Setup the page write filter */
if (xtrabackup_incremental) {
write_filter = &wf_incremental;
} else if (xtrabackup_compact) {
write_filter = &wf_compact;
} else {
write_filter = &wf_write_through;
}
......@@ -2484,6 +2494,7 @@ static
void
xtrabackup_choose_lsn_offset(lsn_t start_lsn)
{
#if SUPPORT_PERCONA_5_5
ulint no, alt_no, expected_no;
ulint blocks_in_group;
lsn_t tmp_offset, end_lsn;
......@@ -2566,8 +2577,11 @@ xtrabackup_choose_lsn_offset(lsn_t start_lsn)
decision to choose one over the other. Die just
like a Buridan's ass */
ut_a(lsn_chosen == 1);
#endif
}
extern ibool log_block_checksum_is_ok_or_old_format(const byte* block);
/*******************************************************//**
Scans log from a buffer and writes new log data to the outpud datasinc.
@return true if success */
......@@ -2704,13 +2718,22 @@ xtrabackup_scan_log_recs(
if (!*finished) {
write_size = RECV_SCAN_SIZE;
} else {
write_size = ut_uint64_align_up(scanned_lsn,
OS_FILE_LOG_BLOCK_SIZE) - start_lsn;
write_size = (ulint)(ut_uint64_align_up(scanned_lsn,
OS_FILE_LOG_BLOCK_SIZE) - start_lsn);
if (!is_last && scanned_lsn % OS_FILE_LOG_BLOCK_SIZE) {
write_size -= OS_FILE_LOG_BLOCK_SIZE;
}
}
if (write_size == 0) {
return(true);
}
if (srv_encrypt_log) {
log_encrypt_before_write(scanned_checkpoint_no,
log_sys->buf, write_size);
}
if (ds_write(dst_log_file, log_sys->buf, write_size)) {
msg("xtrabackup: Error: "
"write to logfile failed\n");
......@@ -2756,7 +2779,7 @@ xtrabackup_copy_logfile(lsn_t from_lsn, my_bool is_last)
mutex_enter(&log_sys->mutex);
log_group_read_log_seg(LOG_RECOVER, log_sys->buf,
group, start_lsn, end_lsn);
group, start_lsn, end_lsn, false);
if (!xtrabackup_scan_log_recs(group, is_last,
start_lsn, &contiguous_lsn, &group_scanned_lsn,
......@@ -3133,6 +3156,7 @@ xb_load_tablespaces(void)
ibool create_new_db;
ulint err;
ulint sum_of_new_sizes;
lsn_t min_arch_logno, max_arch_logno;
for (i = 0; i < srv_n_file_io_threads; i++) {
thread_nr[i] = i;
......@@ -3144,6 +3168,7 @@ xb_load_tablespaces(void)
os_thread_sleep(200000); /*0.2 sec*/
err = open_or_create_data_files(&create_new_db,
&min_arch_logno, &max_arch_logno,
&min_flushed_lsn, &max_flushed_lsn,
&sum_of_new_sizes);
if (err != DB_SUCCESS) {
......@@ -3225,19 +3250,9 @@ xb_data_files_close(void)
for (i = 0; i < 1000; i++) {
os_aio_wake_all_threads_at_shutdown();
os_mutex_enter(os_sync_mutex);
if (os_thread_count == 0) {
os_mutex_exit(os_sync_mutex);
os_thread_sleep(10000);
break;
}
os_mutex_exit(os_sync_mutex);
os_thread_sleep(10000);
}
......@@ -3411,7 +3426,7 @@ xb_register_regex(
node = static_cast<xb_regex_list_node_t *>
(ut_malloc(sizeof(xb_regex_list_node_t)));
ret = xb_regcomp(&node->regex, regex, REG_EXTENDED);
ret = regcomp(&node->regex, regex, REG_EXTENDED);
if (ret != 0) {
xb_regerror(ret, &node->regex, errbuf, sizeof(errbuf));
msg("xtrabackup: error: tables regcomp(%s): %s\n",
......@@ -3545,7 +3560,7 @@ xb_filters_free()
while (UT_LIST_GET_LEN(regex_list) > 0) {
xb_regex_list_node_t* node = UT_LIST_GET_FIRST(regex_list);
UT_LIST_REMOVE(regex_list, regex_list, node);
xb_regfree(&node->regex);
regfree(&node->regex);
ut_free(node);
}
......@@ -3602,7 +3617,7 @@ open_or_create_log_file(
files[i] = os_file_create(innodb_file_log_key, name,
OS_FILE_OPEN, OS_FILE_NORMAL,
OS_LOG_FILE, &ret);
OS_LOG_FILE, &ret,0);
if (ret == FALSE) {
fprintf(stderr, "InnoDB: Error in opening %s\n", name);
......@@ -3631,12 +3646,12 @@ open_or_create_log_file(
which is for this log group */
fil_space_create(name,
2 * k + SRV_LOG_SPACE_FIRST_ID, 0, FIL_LOG);
2 * k + SRV_LOG_SPACE_FIRST_ID, 0, FIL_LOG, 0, 0);
}
ut_a(fil_validate());
ut_a(fil_node_create(name, srv_log_file_size,
ut_a(fil_node_create(name, (ulint)srv_log_file_size,
2 * k + SRV_LOG_SPACE_FIRST_ID, FALSE));
if (i == 0) {
log_group_init(k, srv_n_log_files,
......@@ -3765,7 +3780,7 @@ xtrabackup_backup_func(void)
srv_read_only_mode = TRUE;
srv_backup_mode = TRUE;
srv_close_files = xb_close_files;
srv_close_files = (bool)xb_close_files;
if (srv_close_files)
msg("xtrabackup: warning: close-files specified. Use it "
......@@ -3779,7 +3794,7 @@ xtrabackup_backup_func(void)
xb_normalize_init_values();
#ifndef __WIN__
if (srv_file_flush_method_str == NULL) {
/* These are the default options */
srv_unix_file_flush_method = SRV_UNIX_FSYNC;
......@@ -3808,13 +3823,14 @@ xtrabackup_backup_func(void)
"innodb_flush_method\n", srv_file_flush_method_str);
exit(EXIT_FAILURE);
}
#else /* __WIN__ */
/* We can only use synchronous unbuffered IO on Windows for now */
if (srv_file_flush_method_str != NULL) {
msg("xtrabackupp: Warning: "
"ignoring innodb_flush_method = %s on Windows.\n");
"ignoring innodb_flush_method = %s on Windows.\n", srv_file_flush_method_str);
}
#ifdef _WIN32
srv_win_file_flush_method = SRV_WIN_IO_UNBUFFERED;
srv_use_native_aio = FALSE;
#endif
......@@ -3836,10 +3852,14 @@ xtrabackup_backup_func(void)
computers */
}
os_sync_mutex = NULL;
srv_general_init();
ut_crc32_init();
#ifdef WITH_INNODB_DISALLOW_WRITES
srv_allow_writes_event = os_event_create();
os_event_set(srv_allow_writes_event);
#endif
xb_filters_init();
{
......@@ -3901,7 +3921,7 @@ xtrabackup_backup_func(void)
}
/* create target dir if not exist */
if (!my_stat(xtrabackup_target_dir,&stat_info,MYF(0))
if (!xtrabackup_stream_str && !my_stat(xtrabackup_target_dir,&stat_info,MYF(0))
&& (my_mkdir(xtrabackup_target_dir,0777,MYF(0)) < 0)){
msg("xtrabackup: Error: cannot mkdir %d: %s\n",
my_errno, xtrabackup_target_dir);
......@@ -3950,10 +3970,10 @@ xtrabackup_backup_func(void)
mutex_exit(&log_sys->mutex);
reread_log_header:
fil_io(OS_FILE_READ | OS_FILE_LOG, TRUE, max_cp_group->space_id,
fil_io(OS_FILE_READ | OS_FILE_LOG, true, max_cp_group->space_id,
0,
0, 0, LOG_FILE_HDR_SIZE,
log_hdr_buf, max_cp_group);
log_hdr_buf, max_cp_group, NULL);
/* check consistency of log file header to copy */
mutex_enter(&log_sys->mutex);
......@@ -4427,7 +4447,7 @@ xtrabackup_stats_level(
}
static void
xtrabackup_stats_func(void)
xtrabackup_stats_func(int argc, char **argv)
{
ulint n;
......@@ -4439,7 +4459,7 @@ xtrabackup_stats_func(void)
exit(EXIT_FAILURE);
}
msg("xtrabackup: cd to %s\n", mysql_real_data_home);
encryption_plugin_prepare_init(argc, argv);
mysql_data_home= mysql_data_home_buff;
mysql_data_home[0]=FN_CURLIB; // all paths are relative from here
mysql_data_home[1]=0;
......@@ -4654,7 +4674,7 @@ xtrabackup_init_temp_log(void)
ibool success;
ulint field;
byte log_buf[UNIV_PAGE_SIZE_MAX * 128]; /* 2 MB */
byte* log_buf= (byte *)malloc(UNIV_PAGE_SIZE_MAX * 128); /* 2 MB */
ib_int64_t file_size;
......@@ -4668,6 +4688,10 @@ xtrabackup_init_temp_log(void)
max_no = 0;
if (!log_buf) {
goto error;
}
if (!xb_init_log_block_size()) {
goto error;
}
......@@ -4688,7 +4712,7 @@ xtrabackup_init_temp_log(void)
src_file = os_file_create_simple_no_error_handling(0, src_path,
OS_FILE_OPEN,
OS_FILE_READ_WRITE,
&success);
&success,0);
if (!success) {
/* The following call prints an error message */
os_file_get_last_error(TRUE);
......@@ -4700,7 +4724,7 @@ xtrabackup_init_temp_log(void)
src_file = os_file_create_simple_no_error_handling(0, dst_path,
OS_FILE_OPEN,
OS_FILE_READ_WRITE,
&success);
&success,0);
if (!success) {
os_file_get_last_error(TRUE);
msg(" xtrabackup: Fatal error: cannot find %s.\n",
......@@ -4798,7 +4822,7 @@ xtrabackup_init_temp_log(void)
mach_write_to_4(log_buf + LOG_CHECKPOINT_1
+ LOG_CHECKPOINT_OFFSET_LOW32,
LOG_FILE_HDR_SIZE +
(max_lsn -
(ulint)(max_lsn -
ut_uint64_align_down(max_lsn,
OS_FILE_LOG_BLOCK_SIZE)));
mach_write_to_4(log_buf + LOG_CHECKPOINT_1
......@@ -4815,7 +4839,7 @@ xtrabackup_init_temp_log(void)
mach_write_to_4(log_buf + LOG_CHECKPOINT_2
+ LOG_CHECKPOINT_OFFSET_LOW32,
LOG_FILE_HDR_SIZE +
(max_lsn -
(ulint)(max_lsn -
ut_uint64_align_down(max_lsn,
OS_FILE_LOG_BLOCK_SIZE)));
mach_write_to_4(log_buf + LOG_CHECKPOINT_2
......@@ -4916,15 +4940,17 @@ xtrabackup_init_temp_log(void)
goto error;
}
xtrabackup_logfile_is_renamed = TRUE;
free(log_buf);
return(FALSE);
skip_modify:
free(log_buf);
os_file_close(src_file);
src_file = XB_FILE_UNDEFINED;
return(FALSE);
error:
free(log_buf);
if (src_file != XB_FILE_UNDEFINED)
os_file_close(src_file);
msg("xtrabackup: Error: xtrabackup_init_temp_log() failed.\n");
......@@ -4977,7 +5003,7 @@ xb_space_create_file(
*file = os_file_create_simple_no_error_handling(0, path, OS_FILE_CREATE,
OS_FILE_READ_WRITE,
&ret);
&ret,0);
if (!ret) {
msg("xtrabackup: cannot create file %s\n", path);
return ret;
......@@ -5171,7 +5197,7 @@ xb_delta_open_matching_space(
/* No matching space found. create the new one. */
if (!fil_space_create(dest_space_name, space_id, 0,
FIL_TABLESPACE)) {
FIL_TABLESPACE, 0, false)) {
msg("xtrabackup: Cannot create tablespace %s\n",
dest_space_name);
goto exit;
......@@ -5201,7 +5227,7 @@ xb_delta_open_matching_space(
file = os_file_create_simple_no_error_handling(0, real_name,
OS_FILE_OPEN,
OS_FILE_READ_WRITE,
&ok);
&ok,0);
if (ok) {
*success = TRUE;
......@@ -5290,7 +5316,7 @@ xtrabackup_apply_delta(
src_file = os_file_create_simple_no_error_handling(0, src_path,
OS_FILE_OPEN,
OS_FILE_READ_WRITE,
&success);
&success,0);
if (!success) {
os_file_get_last_error(TRUE);
msg("xtrabackup: error: cannot open %s\n", src_path);
......@@ -5315,11 +5341,11 @@ xtrabackup_apply_delta(
/* allocate buffer for incremental backup (4096 pages) */
incremental_buffer_base = static_cast<byte *>
(ut_malloc((UNIV_PAGE_SIZE_MAX / 4 + 1) *
UNIV_PAGE_SIZE_MAX));
(ut_malloc((page_size / 4 + 1) *
page_size));
incremental_buffer = static_cast<byte *>
(ut_align(incremental_buffer_base,
UNIV_PAGE_SIZE_MAX));
page_size));
msg("Applying %s to %s...\n", src_path, dst_path);
......@@ -5637,7 +5663,7 @@ xtrabackup_close_temp_log(my_bool clear_flag)
src_file = os_file_create_simple_no_error_handling(0, src_path,
OS_FILE_OPEN,
OS_FILE_READ_WRITE,
&success);
&success,0);
if (!success) {
goto error;
}
......@@ -5701,7 +5727,7 @@ xb_export_cfg_write_index_fields(
}
/* Include the NUL byte in the length. */
ib_uint32_t len = strlen(field->name) + 1;
ib_uint32_t len = (ib_uint32_t)strlen(field->name) + 1;
ut_a(len > 1);
mach_write_to_4(row, len);
......@@ -5791,7 +5817,7 @@ xb_export_cfg_write_indexes(
/* Write the length of the index name.
NUL byte is included in the length. */
ib_uint32_t len = strlen(index->name) + 1;
ib_uint32_t len = (ib_uint32_t)strlen(index->name) + 1;
ut_a(len > 1);
mach_write_to_4(row, len);
......@@ -5865,7 +5891,7 @@ xb_export_cfg_write_table(
col_name = dict_table_get_col_name(table, dict_col_get_no(col));
/* Include the NUL byte in the length. */
len = strlen(col_name) + 1;
len = (ib_uint32_t)strlen(col_name) + 1;
ut_a(len > 1);
mach_write_to_4(row, len);
......@@ -5909,7 +5935,7 @@ xb_export_cfg_write_header(
const char* hostname = "Hostname unknown";
/* The server hostname includes the NUL byte. */
len = strlen(hostname) + 1;
len = (ib_uint32_t)strlen(hostname) + 1;
mach_write_to_4(value, len);
if (fwrite(&value, 1, sizeof(value), file) != sizeof(value)
......@@ -5922,7 +5948,7 @@ xb_export_cfg_write_header(
/* The table name includes the NUL byte. */
ut_a(table->name != 0);
len = strlen(table->name) + 1;
len = (ib_uint32_t)strlen(table->name) + 1;
/* Write the table name. */
mach_write_to_4(value, len);
......@@ -6122,7 +6148,7 @@ store_binlog_info(
}
static void
xtrabackup_prepare_func(void)
xtrabackup_prepare_func(int argc, char ** argv)
{
ulint err;
datafiles_iter_t *it;
......@@ -6140,6 +6166,8 @@ xtrabackup_prepare_func(void)
}
msg("xtrabackup: cd to %s\n", xtrabackup_real_target_dir);
encryption_plugin_prepare_init(argc, argv);
xtrabackup_target_dir= mysql_data_home_buff;
xtrabackup_target_dir[0]=FN_CURLIB; // all paths are relative from here
xtrabackup_target_dir[1]=0;
......@@ -6193,7 +6221,6 @@ xtrabackup_prepare_func(void)
/* Create logfiles for recovery from 'xtrabackup_logfile', before start InnoDB */
srv_max_n_threads = 1000;
os_sync_mutex = NULL;
ut_mem_init();
/* temporally dummy value to avoid crash */
srv_page_size_shift = 14;
......@@ -6204,6 +6231,11 @@ xtrabackup_prepare_func(void)
mem_init(srv_mem_pool_size);
ut_crc32_init();
#ifdef WITH_INNODB_DISALLOW_WRITES
srv_allow_writes_event = os_event_create();
os_event_set(srv_allow_writes_event);
#endif
xb_filters_init();
if(!innobase_log_arch_dir && xtrabackup_init_temp_log())
......@@ -6213,25 +6245,6 @@ xtrabackup_prepare_func(void)
goto error_cleanup;
}
/* Expand compacted datafiles */
if (xtrabackup_compact) {
srv_compact_backup = TRUE;
if (!xb_expand_datafiles()) {
goto error_cleanup;
}
/* Reset the 'compact' flag in xtrabackup_checkpoints so we
don't expand on subsequent invocations. */
xtrabackup_compact = FALSE;
if (!xtrabackup_write_metadata(metadata_path)) {
msg("xtrabackup: error: xtrabackup_write_metadata() "
"failed\n");
goto error_cleanup;
}
}
xb_normalize_init_values();
if (xtrabackup_incremental || innobase_log_arch_dir) {
......@@ -6262,17 +6275,16 @@ xtrabackup_prepare_func(void)
xb_filter_hash_free(inc_dir_tables_hash);
}
sync_close();
sync_initialized = FALSE;
if (fil_system) {
fil_close();
}
os_sync_free();
mem_close();
os_sync_mutex = NULL;
ut_free_all_mem();
innodb_free_param();
sync_close();
sync_initialized = FALSE;
/* Reset the configuration as it might have been changed by
xb_data_files_init(). */
......@@ -6280,8 +6292,7 @@ xtrabackup_prepare_func(void)
goto error_cleanup;
}
srv_apply_log_only = (ibool) xtrabackup_apply_log_only;
srv_rebuild_indexes = (ibool) xtrabackup_rebuild_indexes;
srv_apply_log_only = (bool) xtrabackup_apply_log_only;
/* increase IO threads */
if(srv_n_file_io_threads < 10) {
......@@ -6291,7 +6302,7 @@ xtrabackup_prepare_func(void)
if (innobase_log_arch_dir) {
srv_arch_dir = innobase_log_arch_dir;
srv_archive_recovery = TRUE;
srv_archive_recovery = true;
if (xtrabackup_archived_to_lsn) {
if (xtrabackup_archived_to_lsn < metadata_last_lsn) {
msg("xtrabackup: warning: logs applying lsn "
......@@ -6317,7 +6328,7 @@ xtrabackup_prepare_func(void)
Unfinished transactions are not rolled back during log applying
as they can be finished at the firther files applyings.
*/
srv_apply_log_only = xtrabackup_apply_log_only = TRUE;
srv_apply_log_only = xtrabackup_apply_log_only = true;
if (!xtrabackup_arch_search_files(min_flushed_lsn)) {
goto error_cleanup;
......@@ -6340,6 +6351,12 @@ xtrabackup_prepare_func(void)
"xtrabackup: Using %lld bytes for buffer pool "
"(set by --use-memory parameter)\n", xtrabackup_use_memory);
srv_max_buf_pool_modified_pct = (double)max_buf_pool_modified_pct;
if (srv_max_dirty_pages_pct_lwm > srv_max_buf_pool_modified_pct) {
srv_max_dirty_pages_pct_lwm = srv_max_buf_pool_modified_pct;
}
if(innodb_init())
goto error_cleanup;
......@@ -6423,15 +6440,20 @@ xtrabackup_prepare_func(void)
/* node exist == file exist, here */
strcpy(info_file_path, node->name);
#ifdef _WIN32
for (int i = 0; info_file_path[i]; i++)
if (info_file_path[i] == '\\')
info_file_path[i]= '/';
#endif
strcpy(info_file_path +
strlen(info_file_path) -
4, ".exp");
len = strlen(info_file_path);
len =(ib_uint32_t)strlen(info_file_path);
p = info_file_path;
prev = NULL;
while ((next = strchr(p, SRV_PATH_SEPARATOR)) != NULL)
while ((next = strchr(p, '/')) != NULL)
{
prev = p;
p = next + 1;
......@@ -6503,7 +6525,7 @@ xtrabackup_prepare_func(void)
info_file_path,
OS_FILE_OVERWRITE,
OS_FILE_NORMAL, OS_DATA_FILE,
&success);
&success,0);
if (!success) {
os_file_get_last_error(TRUE);
goto next_node;
......@@ -6569,8 +6591,9 @@ xtrabackup_prepare_func(void)
}
exit(EXIT_FAILURE);
}
#ifdef WITH_WSREP
xb_write_galera_info(xtrabackup_incremental);
#endif
if(innodb_end())
goto error_cleanup;
......@@ -6578,7 +6601,6 @@ xtrabackup_prepare_func(void)
innodb_free_param();
sync_initialized = FALSE;
os_sync_mutex = NULL;
/* re-init necessary components */
ut_mem_init();
......@@ -6630,9 +6652,7 @@ xtrabackup_prepare_func(void)
if (fil_system) {
fil_close();
}
os_sync_free();
// mem_close();
os_sync_mutex = NULL;
ut_free_all_mem();
/* start InnoDB once again to create log files */
......@@ -6643,8 +6663,7 @@ xtrabackup_prepare_func(void)
goto error;
}
srv_apply_log_only = FALSE;
srv_rebuild_indexes = FALSE;
srv_apply_log_only = false;
/* increase IO threads */
if(srv_n_file_io_threads < 10) {
......@@ -6676,40 +6695,6 @@ xtrabackup_prepare_func(void)
exit(EXIT_FAILURE);
}
/**************************************************************************
Signals-related setup. */
static
void
setup_signals()
/*===========*/
{
struct sigaction sa;
/* Print a stacktrace on some signals */
sa.sa_flags = SA_RESETHAND | SA_NODEFER;
sigemptyset(&sa.sa_mask);
sigprocmask(SIG_SETMASK,&sa.sa_mask,NULL);
#ifdef HAVE_STACKTRACE
my_init_stacktrace();
#endif
sa.sa_handler = handle_fatal_signal;
sigaction(SIGSEGV, &sa, NULL);
sigaction(SIGABRT, &sa, NULL);
sigaction(SIGBUS, &sa, NULL);
sigaction(SIGILL, &sa, NULL);
sigaction(SIGFPE, &sa, NULL);
#ifdef __linux__
/* Ensure xtrabackup process is killed when the parent one
(innobackupex) is terminated with an unhandled signal */
if (prctl(PR_SET_PDEATHSIG, SIGKILL)) {
msg("prctl() failed with errno = %d\n", errno);
exit(EXIT_FAILURE);
}
#endif
}
/**************************************************************************
Append group name to xb_load_default_groups list. */
static
......@@ -6778,11 +6763,6 @@ xb_init()
}
if (xtrabackup_backup) {
if (!opt_noversioncheck) {
version_check();
}
if ((mysql_connection = xb_mysql_connect()) == NULL) {
return(false);
}
......@@ -6791,6 +6771,7 @@ xb_init()
return(false);
}
encryption_plugin_backup_init(mysql_connection);
history_start_time = time(NULL);
}
......@@ -6798,9 +6779,74 @@ xb_init()
return(true);
}
extern void init_signals(void);
#include <sql_locale.h>
/* Messages . Avoid loading errmsg.sys file */
void setup_error_messages()
{
static const char *all_msgs[ER_ERROR_LAST - ER_ERROR_FIRST +1];
my_default_lc_messages = &my_locale_en_US;
my_default_lc_messages->errmsgs->errmsgs = all_msgs;
/* Populate the necessary error messages */
struct {
int id;
const char *fmt;
}
xb_msgs[] =
{
{ ER_DATABASE_NAME,"Database" },
{ ER_TABLE_NAME,"Table"},
{ ER_PARTITION_NAME, "Partition" },
{ ER_SUBPARTITION_NAME, "Subpartition" },
{ ER_TEMPORARY_NAME, "Temporary"},
{ ER_RENAMED_NAME, "Renamed"},
{ ER_CANT_FIND_DL_ENTRY, "Can't find symbol '%-.128s' in library"},
{ ER_CANT_OPEN_LIBRARY, "Can't open shared library '%-.192s' (errno: %d, %-.128s)" },
{ ER_OUTOFMEMORY, "Out of memory; restart server and try again (needed %d bytes)" },
{ ER_CANT_OPEN_LIBRARY, "Can't open shared library '%-.192s' (errno: %d, %-.128s)" },
{ ER_UDF_NO_PATHS, "No paths allowed for shared library" },
{ ER_CANT_INITIALIZE_UDF,"Can't initialize function '%-.192s'; %-.80s"},
{ ER_PLUGIN_IS_NOT_LOADED,"Plugin '%-.192s' is not loaded" }
};
for (int i = 0; i < (int)array_elements(all_msgs); i++)
all_msgs[i] = "Unknown error";
for (int i = 0; i < (int)array_elements(xb_msgs); i++)
all_msgs[xb_msgs[i].id - ER_ERROR_FIRST] = xb_msgs[i].fmt;
}
extern my_bool(*dict_check_if_skip_table)(const char* name) ;
void
handle_options(int argc, char **argv, char ***argv_client, char ***argv_server)
{
/* Setup some variables for Innodb.*/
srv_xtrabackup = true;
files_charset_info = &my_charset_utf8_general_ci;
dict_check_if_skip_table = check_if_skip_table;
setup_error_messages();
sys_var_init();
plugin_mutex_init();
mysql_rwlock_init(key_rwlock_LOCK_system_variables_hash, &LOCK_system_variables_hash);
opt_stack_trace = 1;
test_flags |= TEST_SIGINT;
init_signals();
#ifndef _WIN32
/* Exit process on SIGINT. */
my_sigset(SIGINT, SIG_DFL);
#endif
sf_leaking_memory = 0; /* don't report memory leaks on early exist */
int i;
int ho_error;
......@@ -6811,9 +6857,6 @@ handle_options(int argc, char **argv, char ***argv_client, char ***argv_server)
int argc_client = argc;
int argc_server = argc;
*argv_client = argv;
*argv_server = argv;
/* scan options for group and config file to load defaults from */
for (i = 1; i < argc; i++) {
......@@ -6859,12 +6902,27 @@ handle_options(int argc, char **argv, char ***argv_client, char ***argv_server)
if (prepare && target_dir) {
snprintf(conf_file, sizeof(conf_file),
"%s/backup-my.cnf", target_dir);
if (!strncmp(argv[1], "--defaults-file=", 16)) {
/* Remove defaults-file*/
for (int i = 2; ; i++) {
if ((argv[i-1]= argv[i]) == 0)
break;
}
argc--;
}
}
*argv_client = argv;
*argv_server = argv;
if (load_defaults(conf_file, xb_server_default_groups,
&argc_server, argv_server)) {
exit(EXIT_FAILURE);
}
int n;
for (n = 0; (*argv_server)[n]; n++) {};
argc_server = n;
print_param_str <<
"# This MySQL options file was generated by XtraBackup.\n"
"[" << defaults_group << "]\n";
......@@ -6879,6 +6937,7 @@ handle_options(int argc, char **argv, char ***argv_client, char ***argv_server)
optp->u_max_value = (G_PTR *) &global_max_value;
}
/* Throw a descriptive error if --defaults-file or --defaults-extra-file
is not the first command line argument */
for (int i = 2 ; i < argc ; i++) {
......@@ -6913,6 +6972,9 @@ handle_options(int argc, char **argv, char ***argv_client, char ***argv_server)
exit(EXIT_FAILURE);
}
for (n = 0; (*argv_client)[n]; n++) {};
argc_client = n;
if (strcmp(base_name(my_progname), INNOBACKUPEX_BIN_NAME) == 0 &&
argc_client > 0) {
/* emulate innobackupex script */
......@@ -6959,10 +7021,16 @@ int main(int argc, char **argv)
{
char **client_defaults, **server_defaults;
char cwd[FN_REFLEN];
my_bool is_symdir;
setup_signals();
if (argc > 1 && (strcmp(argv[1], "--innobackupex") == 0))
{
argv++;
argc--;
argv[0] = "innobackupex";
innobackupex_mode = true;
}
init_signals();
MY_INIT(argv[0]);
pthread_key_create(&THR_THD, NULL);
......@@ -6977,11 +7045,18 @@ int main(int argc, char **argv)
exit(EXIT_FAILURE);
}
system_charset_info= &my_charset_utf8_general_ci;
system_charset_info = &my_charset_utf8_general_ci;
key_map_full.set_all();
handle_options(argc, argv, &client_defaults, &server_defaults);
int argc_server;
for (argc_server = 0; server_defaults[argc_server]; argc_server++) {}
int argc_client;
for (argc_client = 0; client_defaults[argc_client]; argc_client++) {}
if (innobackupex_mode) {
if (!ibx_init()) {
exit(EXIT_FAILURE);
......@@ -7002,14 +7077,14 @@ int main(int argc, char **argv)
my_load_path(xtrabackup_real_target_dir,
xtrabackup_target_dir, cwd);
unpack_dirname(xtrabackup_real_target_dir,
xtrabackup_real_target_dir, &is_symdir);
xtrabackup_real_target_dir);
xtrabackup_target_dir= xtrabackup_real_target_dir;
if (xtrabackup_incremental_basedir) {
my_load_path(xtrabackup_real_incremental_basedir,
xtrabackup_incremental_basedir, cwd);
unpack_dirname(xtrabackup_real_incremental_basedir,
xtrabackup_real_incremental_basedir, &is_symdir);
xtrabackup_real_incremental_basedir);
xtrabackup_incremental_basedir =
xtrabackup_real_incremental_basedir;
}
......@@ -7018,7 +7093,7 @@ int main(int argc, char **argv)
my_load_path(xtrabackup_real_incremental_dir,
xtrabackup_incremental_dir, cwd);
unpack_dirname(xtrabackup_real_incremental_dir,
xtrabackup_real_incremental_dir, &is_symdir);
xtrabackup_real_incremental_dir);
xtrabackup_incremental_dir = xtrabackup_real_incremental_dir;
}
......@@ -7026,7 +7101,7 @@ int main(int argc, char **argv)
my_load_path(xtrabackup_real_extra_lsndir,
xtrabackup_extra_lsndir, cwd);
unpack_dirname(xtrabackup_real_extra_lsndir,
xtrabackup_real_extra_lsndir, &is_symdir);
xtrabackup_real_extra_lsndir);
xtrabackup_extra_lsndir = xtrabackup_real_extra_lsndir;
}
......@@ -7181,11 +7256,12 @@ int main(int argc, char **argv)
/* --stats */
if (xtrabackup_stats)
xtrabackup_stats_func();
xtrabackup_stats_func(argc_server,server_defaults);
/* --prepare */
if (xtrabackup_prepare)
xtrabackup_prepare_func();
if (xtrabackup_prepare) {
xtrabackup_prepare_func(argc_server, server_defaults);
}
if (xtrabackup_copy_back || xtrabackup_move_back) {
if (!check_if_param_set("datadir")) {
......@@ -7206,7 +7282,6 @@ int main(int argc, char **argv)
ibx_cleanup();
}
xb_regex_end();
free_defaults(client_defaults);
free_defaults(server_defaults);
......
extra/mariabackup/xtrabackup.h
View file @ ce4c56db
......@@ -63,8 +63,6 @@ extern lsn_t checkpoint_lsn_start;
extern xb_page_bitmap *changed_page_bitmap;
extern ulint xtrabackup_rebuild_threads;
extern char *xtrabackup_incremental;
extern my_bool xtrabackup_incremental_force_scan;
......@@ -80,7 +78,6 @@ extern char *xtrabackup_tables_file;
extern char *xtrabackup_databases;
extern char *xtrabackup_databases_file;
extern my_bool xtrabackup_compact;
extern ibool xtrabackup_compress;
extern ibool xtrabackup_encrypt;
......@@ -106,8 +103,14 @@ extern int xtrabackup_parallel;
extern my_bool xb_close_files;
extern const char *xtrabackup_compress_alg;
extern uint xtrabackup_compress_threads;
extern ulonglong xtrabackup_compress_chunk_size;
#ifdef __cplusplus
extern "C"{
#endif
extern uint xtrabackup_compress_threads;
extern ulonglong xtrabackup_compress_chunk_size;
#ifdef __cplusplus
}
#endif
extern ulong xtrabackup_encrypt_algo;
extern uint xtrabackup_encrypt_threads;
extern ulonglong xtrabackup_encrypt_chunk_size;
......@@ -116,7 +119,6 @@ extern char *xtrabackup_incremental_basedir;
extern char *xtrabackup_extra_lsndir;
extern char *xtrabackup_incremental_dir;
extern ulint xtrabackup_log_copy_interval;
extern my_bool xtrabackup_rebuild_indexes;
extern char *xtrabackup_stream_str;
extern long xtrabackup_throttle;
extern longlong xtrabackup_use_memory;
......@@ -165,14 +167,6 @@ extern uint opt_safe_slave_backup_timeout;
extern const char *opt_history;
extern my_bool opt_decrypt;
#if defined(HAVE_OPENSSL)
extern my_bool opt_use_ssl;
extern my_bool opt_ssl_verify_server_cert;
#if !defined(HAVE_YASSL)
extern char *opt_server_public_key;
#endif
#endif
enum binlog_info_enum { BINLOG_INFO_OFF, BINLOG_INFO_LOCKLESS, BINLOG_INFO_ON,
BINLOG_INFO_AUTO};
......@@ -217,6 +211,14 @@ Check if parameter is set in defaults file or via command line argument
bool
check_if_param_set(const char *param);
#if defined(HAVE_OPENSSL)
extern my_bool opt_use_ssl;
extern my_bool opt_ssl_verify_server_cert;
#if !defined(HAVE_YASSL)
extern char *opt_server_public_key;
#endif
#endif
void
xtrabackup_backup_func(void);
......
extra/mariabackup/xtrabackup_version.h.in deleted 100644 → 0
View file @ d7714308
/******************************************************
Copyright (c) 2013 Percona LLC and/or its affiliates.
Version numbers definitions.
This program is free software; you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation; version 2 of the License.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program; if not, write to the Free Software
Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA
*******************************************************/
#ifndef XB_VERSION_H
#define XB_VERSION_H
#define XTRABACKUP_VERSION "@XB_VERSION@"
#define XTRABACKUP_REVISION "@XB_REVISION@"
#endif /* XB_VERSION_H */
storage/xtradb/CMakeLists.txt
View file @ ce4c56db
......@@ -506,3 +506,6 @@ MYSQL_ADD_PLUGIN(xtradb ${INNOBASE_SOURCES} STORAGE_ENGINE
IF(TARGET xtradb AND NOT XTRADB_OK)
MESSAGE(FATAL_ERROR "Percona XtraDB is not supported on this platform")
ENDIF()
ADD_SUBDIRECTORY(${CMAKE_SOURCE_DIR}/extra/mariabackup ${CMAKE_BINARY_DIR}/extra/mariabackup)
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment
GitLab Nexedi Edition | About GitLab | About Nexedi | 沪ICP备2021021310号-2 | 沪ICP备2021021310号-7