Skip to content

M
MariaDB
Commit ecb25df2 authored by Vladislav Vaintroub's avatar Vladislav Vaintroub Committed by Sergei Golubchik
Browse files
Options

Xtrabackup 2.3.8

parent c8ac0244
Show whitespace changes
Inline Side-by-side
Showing with 2414 additions and 422 deletions
extra/mariabackup/CMakeLists.txt
View file @ ecb25df2
# Copyright (c) 2013 Percona LLC and/or its affiliates. # Copyright (c) 2013, 2017 Percona LLC and/or its affiliates.
# #
# This program is free software; you can redistribute it and/or modify # This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by # it under the terms of the GNU General Public License as published by
# the Free Software Foundation; version 2 of the License. # the Free Software Foundation; version 2 of the License.
#
# This program is distributed in the hope that it will be useful, # This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of # but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details. # GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License # You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software # along with this program; if not, write to the Free Software
# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA
...@@ -129,6 +129,7 @@ INCLUDE_DIRECTORIES( ...@@ -129,6 +129,7 @@ INCLUDE_DIRECTORIES(
${CMAKE_SOURCE_DIR}/sql ${CMAKE_SOURCE_DIR}/sql
${CMAKE_CURRENT_SOURCE_DIR}/quicklz ${CMAKE_CURRENT_SOURCE_DIR}/quicklz
${CMAKE_CURRENT_SOURCE_DIR} ${CMAKE_CURRENT_SOURCE_DIR}
${CMAKE_CURRENT_SOURCE_DIR}/crc
) )
IF(NOT HAVE_SYSTEM_REGEX) IF(NOT HAVE_SYSTEM_REGEX)
...@@ -156,7 +157,6 @@ MYSQL_ADD_EXECUTABLE(mariabackup ...@@ -156,7 +157,6 @@ MYSQL_ADD_EXECUTABLE(mariabackup
${DS_ARCHIVE_SOURCE} ${DS_ARCHIVE_SOURCE}
ds_buffer.c ds_buffer.c
ds_compress.c ds_compress.c
ds_encrypt.c
ds_local.c ds_local.c
ds_stdout.c ds_stdout.c
ds_tmpfile.c ds_tmpfile.c
...@@ -166,8 +166,6 @@ MYSQL_ADD_EXECUTABLE(mariabackup ...@@ -166,8 +166,6 @@ MYSQL_ADD_EXECUTABLE(mariabackup
read_filt.cc read_filt.cc
write_filt.cc write_filt.cc
wsrep.cc wsrep.cc
xbcrypt_common.c
xbcrypt_write.c
xbstream_write.c xbstream_write.c
backup_mysql.cc backup_mysql.cc
backup_copy.cc backup_copy.cc
...@@ -181,9 +179,10 @@ MYSQL_ADD_EXECUTABLE(mariabackup ...@@ -181,9 +179,10 @@ MYSQL_ADD_EXECUTABLE(mariabackup
# Export all symbols on Unix, for better crash callstacks # Export all symbols on Unix, for better crash callstacks
SET_TARGET_PROPERTIES(mariabackup PROPERTIES ENABLE_EXPORTS TRUE) SET_TARGET_PROPERTIES(mariabackup PROPERTIES ENABLE_EXPORTS TRUE)
ADD_SUBDIRECTORY(crc)
TARGET_LINK_LIBRARIES(mariabackup sql) TARGET_LINK_LIBRARIES(mariabackup sql crc)
IF(NOT HAVE_SYSTEM_REGEX) IF(NOT HAVE_SYSTEM_REGEX)
TARGET_LINK_LIBRARIES(mariabackup pcreposix) TARGET_LINK_LIBRARIES(mariabackup pcreposix)
...@@ -201,13 +200,13 @@ MYSQL_ADD_EXECUTABLE(mbstream ...@@ -201,13 +200,13 @@ MYSQL_ADD_EXECUTABLE(mbstream
xbstream.c xbstream.c
xbstream_read.c xbstream_read.c
xbstream_write.c xbstream_write.c
COMPONENT backup COMPONENT backup
) )
TARGET_LINK_LIBRARIES(mbstream TARGET_LINK_LIBRARIES(mbstream
mysys mysys
crc
) )
IF(MSVC) IF(MSVC)
......
extra/mariabackup/backup_copy.cc
View file @ ecb25df2
...@@ -265,6 +265,11 @@ datadir_iter_next_database(datadir_iter_t *it) ...@@ -265,6 +265,11 @@ datadir_iter_next_database(datadir_iter_t *it)
return(true); return(true);
} }
if (check_if_skip_database_by_path(it->dbpath)) {
msg("Skipping db: %s\n", it->dbpath);
continue;
}
/* We want wrong directory permissions to be a fatal error for /* We want wrong directory permissions to be a fatal error for
XtraBackup. */ XtraBackup. */
it->dbdir = os_file_opendir(it->dbpath, TRUE); it->dbdir = os_file_opendir(it->dbpath, TRUE);
...@@ -1704,7 +1709,7 @@ copy_back() ...@@ -1704,7 +1709,7 @@ copy_back()
for (i = 1; i <= srv_undo_tablespaces; i++) { for (i = 1; i <= srv_undo_tablespaces; i++) {
char filename[20]; char filename[20];
sprintf(filename, "undo%03lu", i); sprintf(filename, "undo%03u", (uint)i);
if (!(ret = copy_or_move_file(filename, filename, if (!(ret = copy_or_move_file(filename, filename,
dst_dir, 1))) { dst_dir, 1))) {
goto cleanup; goto cleanup;
......
extra/mariabackup/backup_mysql.cc
View file @ ecb25df2
...@@ -1414,7 +1414,10 @@ write_xtrabackup_info(MYSQL *connection) ...@@ -1414,7 +1414,10 @@ write_xtrabackup_info(MYSQL *connection)
bool is_partial = (xtrabackup_tables bool is_partial = (xtrabackup_tables
|| xtrabackup_tables_file || xtrabackup_tables_file
|| xtrabackup_databases || xtrabackup_databases
|| xtrabackup_databases_file); || xtrabackup_databases_file
|| xtrabackup_tables_exclude
|| xtrabackup_databases_exclude
);
backup_file_printf(XTRABACKUP_INFO, backup_file_printf(XTRABACKUP_INFO,
"uuid = %s\n" "uuid = %s\n"
......
extra/mariabackup/crc/CMakeLists.txt 0 → 100644
View file @ ecb25df2
# Copyright (c) 2017 Percona LLC and/or its affiliates.
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation; version 2 of the License.
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA
PROJECT(crc C)
IF(NOT CMAKE_CROSSCOMPILING AND NOT MSVC)
STRING(TOLOWER ${CMAKE_SYSTEM_PROCESSOR} processor)
IF(processor MATCHES "86" OR processor MATCHES "amd64" OR processor MATCHES "x64")
# Check for PCLMUL instruction
CHECK_C_SOURCE_RUNS("
int main()
{
asm volatile (\"pclmulqdq \\$0x00, %%xmm1, %%xmm0\":::\"cc\");
return 0;
}" HAVE_CLMUL_INSTRUCTION)
ENDIF()
ENDIF()
IF(HAVE_CLMUL_INSTRUCTION)
ADD_DEFINITIONS(-DHAVE_CLMUL_INSTRUCTION)
ENDIF()
ADD_LIBRARY(crc crc_glue.c crc-intel-pclmul.c)
extra/mariabackup/crc/config.h.cmake 0 → 100644
View file @ ecb25df2
/******************************************************
Copyright (c) 2017 Percona LLC and/or its affiliates.
Zlib compatible CRC-32 implementation.
This program is free software; you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation; version 2 of the License.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program; if not, write to the Free Software
Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA
*******************************************************/
#cmakedefine HAVE_CLMUL_INSTRUCTION 1
extra/mariabackup/crc/crc-intel-pclmul.c 0 → 100644
View file @ ecb25df2
/******************************************************
Copyright (c) 2017 Percona LLC and/or its affiliates.
CRC32 using Intel's PCLMUL instruction.
This program is free software; you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation; version 2 of the License.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program; if not, write to the Free Software
Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA
*******************************************************/
/* crc-intel-pclmul.c - Intel PCLMUL accelerated CRC implementation
* Copyright (C) 2016 Jussi Kivilinna <jussi.kivilinna@iki.fi>
*
* This file is part of Libgcrypt.
*
* Libgcrypt is free software; you can redistribute it and/or modify
* it under the terms of the GNU Lesser General Public License as
* published by the Free Software Foundation; either version 2.1 of
* the License, or (at your option) any later version.
*
* Libgcrypt is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Lesser General Public License for more details.
*
* You should have received a copy of the GNU Lesser General Public
* License along with this program; if not, write to the Free Software
* Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA
*
*/
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <stdint.h>
# define U64_C(c) (c ## UL)
typedef uint32_t u32;
typedef uint16_t u16;
typedef uint64_t u64;
#ifndef byte
typedef uint8_t byte;
#endif
# define _gcry_bswap32 __builtin_bswap32
#if __GNUC__ >= 4 && defined(__x86_64__) && defined(HAVE_CLMUL_INSTRUCTION)
#if _GCRY_GCC_VERSION >= 40400 /* 4.4 */
/* Prevent compiler from issuing SSE instructions between asm blocks. */
# pragma GCC target("no-sse")
#endif
#define ALIGNED_16 __attribute__ ((aligned (16)))
struct u16_unaligned_s
{
u16 a;
} __attribute__((packed, aligned (1), may_alias));
/* Constants structure for generic reflected/non-reflected CRC32 CLMUL
* functions. */
struct crc32_consts_s
{
/* k: { x^(32*17), x^(32*15), x^(32*5), x^(32*3), x^(32*2), 0 } mod P(x) */
u64 k[6];
/* my_p: { floor(x^64 / P(x)), P(x) } */
u64 my_p[2];
};
/* CLMUL constants for CRC32 and CRC32RFC1510. */
static const struct crc32_consts_s crc32_consts ALIGNED_16 =
{
{ /* k[6] = reverse_33bits( x^(32*y) mod P(x) ) */
U64_C(0x154442bd4), U64_C(0x1c6e41596), /* y = { 17, 15 } */
U64_C(0x1751997d0), U64_C(0x0ccaa009e), /* y = { 5, 3 } */
U64_C(0x163cd6124), 0 /* y = 2 */
},
{ /* my_p[2] = reverse_33bits ( { floor(x^64 / P(x)), P(x) } ) */
U64_C(0x1f7011641), U64_C(0x1db710641)
}
};
/* Common constants for CRC32 algorithms. */
static const byte crc32_refl_shuf_shift[3 * 16] ALIGNED_16 =
{
0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07,
0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f,
0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
};
static const byte crc32_partial_fold_input_mask[16 + 16] ALIGNED_16 =
{
0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
};
static const u64 crc32_merge9to15_shuf[15 - 9 + 1][2] ALIGNED_16 =
{
{ U64_C(0x0706050403020100), U64_C(0xffffffffffffff0f) }, /* 9 */
{ U64_C(0x0706050403020100), U64_C(0xffffffffffff0f0e) },
{ U64_C(0x0706050403020100), U64_C(0xffffffffff0f0e0d) },
{ U64_C(0x0706050403020100), U64_C(0xffffffff0f0e0d0c) },
{ U64_C(0x0706050403020100), U64_C(0xffffff0f0e0d0c0b) },
{ U64_C(0x0706050403020100), U64_C(0xffff0f0e0d0c0b0a) },
{ U64_C(0x0706050403020100), U64_C(0xff0f0e0d0c0b0a09) }, /* 15 */
};
static const u64 crc32_merge5to7_shuf[7 - 5 + 1][2] ALIGNED_16 =
{
{ U64_C(0xffffff0703020100), U64_C(0xffffffffffffffff) }, /* 5 */
{ U64_C(0xffff070603020100), U64_C(0xffffffffffffffff) },
{ U64_C(0xff07060503020100), U64_C(0xffffffffffffffff) }, /* 7 */
};
/* PCLMUL functions for reflected CRC32. */
static inline void
crc32_reflected_bulk (u32 *pcrc, const byte *inbuf, size_t inlen,
const struct crc32_consts_s *consts)
{
if (inlen >= 8 * 16)
{
asm volatile ("movd %[crc], %%xmm4\n\t"
"movdqu %[inbuf_0], %%xmm0\n\t"
"movdqu %[inbuf_1], %%xmm1\n\t"
"movdqu %[inbuf_2], %%xmm2\n\t"
"movdqu %[inbuf_3], %%xmm3\n\t"
"pxor %%xmm4, %%xmm0\n\t"
:
: [inbuf_0] "m" (inbuf[0 * 16]),
[inbuf_1] "m" (inbuf[1 * 16]),
[inbuf_2] "m" (inbuf[2 * 16]),
[inbuf_3] "m" (inbuf[3 * 16]),
[crc] "m" (*pcrc)
);
inbuf += 4 * 16;
inlen -= 4 * 16;
asm volatile ("movdqa %[k1k2], %%xmm4\n\t"
:
: [k1k2] "m" (consts->k[1 - 1])
);
/* Fold by 4. */
while (inlen >= 4 * 16)
{
asm volatile ("movdqu %[inbuf_0], %%xmm5\n\t"
"movdqa %%xmm0, %%xmm6\n\t"
"pclmulqdq $0x00, %%xmm4, %%xmm0\n\t"
"pclmulqdq $0x11, %%xmm4, %%xmm6\n\t"
"pxor %%xmm5, %%xmm0\n\t"
"pxor %%xmm6, %%xmm0\n\t"
"movdqu %[inbuf_1], %%xmm5\n\t"
"movdqa %%xmm1, %%xmm6\n\t"
"pclmulqdq $0x00, %%xmm4, %%xmm1\n\t"
"pclmulqdq $0x11, %%xmm4, %%xmm6\n\t"
"pxor %%xmm5, %%xmm1\n\t"
"pxor %%xmm6, %%xmm1\n\t"
"movdqu %[inbuf_2], %%xmm5\n\t"
"movdqa %%xmm2, %%xmm6\n\t"
"pclmulqdq $0x00, %%xmm4, %%xmm2\n\t"
"pclmulqdq $0x11, %%xmm4, %%xmm6\n\t"
"pxor %%xmm5, %%xmm2\n\t"
"pxor %%xmm6, %%xmm2\n\t"
"movdqu %[inbuf_3], %%xmm5\n\t"
"movdqa %%xmm3, %%xmm6\n\t"
"pclmulqdq $0x00, %%xmm4, %%xmm3\n\t"
"pclmulqdq $0x11, %%xmm4, %%xmm6\n\t"
"pxor %%xmm5, %%xmm3\n\t"
"pxor %%xmm6, %%xmm3\n\t"
:
: [inbuf_0] "m" (inbuf[0 * 16]),
[inbuf_1] "m" (inbuf[1 * 16]),
[inbuf_2] "m" (inbuf[2 * 16]),
[inbuf_3] "m" (inbuf[3 * 16])
);
inbuf += 4 * 16;
inlen -= 4 * 16;
}
asm volatile ("movdqa %[k3k4], %%xmm6\n\t"
"movdqa %[my_p], %%xmm5\n\t"
:
: [k3k4] "m" (consts->k[3 - 1]),
[my_p] "m" (consts->my_p[0])
);
/* Fold 4 to 1. */
asm volatile ("movdqa %%xmm0, %%xmm4\n\t"
"pclmulqdq $0x00, %%xmm6, %%xmm0\n\t"
"pclmulqdq $0x11, %%xmm6, %%xmm4\n\t"
"pxor %%xmm1, %%xmm0\n\t"
"pxor %%xmm4, %%xmm0\n\t"
"movdqa %%xmm0, %%xmm4\n\t"
"pclmulqdq $0x00, %%xmm6, %%xmm0\n\t"
"pclmulqdq $0x11, %%xmm6, %%xmm4\n\t"
"pxor %%xmm2, %%xmm0\n\t"
"pxor %%xmm4, %%xmm0\n\t"
"movdqa %%xmm0, %%xmm4\n\t"
"pclmulqdq $0x00, %%xmm6, %%xmm0\n\t"
"pclmulqdq $0x11, %%xmm6, %%xmm4\n\t"
"pxor %%xmm3, %%xmm0\n\t"
"pxor %%xmm4, %%xmm0\n\t"
:
:
);
}
else
{
asm volatile ("movd %[crc], %%xmm1\n\t"
"movdqu %[inbuf], %%xmm0\n\t"
"movdqa %[k3k4], %%xmm6\n\t"
"pxor %%xmm1, %%xmm0\n\t"
"movdqa %[my_p], %%xmm5\n\t"
:
: [inbuf] "m" (*inbuf),
[crc] "m" (*pcrc),
[k3k4] "m" (consts->k[3 - 1]),
[my_p] "m" (consts->my_p[0])
);
inbuf += 16;
inlen -= 16;
}
/* Fold by 1. */
if (inlen >= 16)
{
while (inlen >= 16)
{
/* Load next block to XMM2. Fold XMM0 to XMM0:XMM1. */
asm volatile ("movdqu %[inbuf], %%xmm2\n\t"
"movdqa %%xmm0, %%xmm1\n\t"
"pclmulqdq $0x00, %%xmm6, %%xmm0\n\t"
"pclmulqdq $0x11, %%xmm6, %%xmm1\n\t"
"pxor %%xmm2, %%xmm0\n\t"
"pxor %%xmm1, %%xmm0\n\t"
:
: [inbuf] "m" (*inbuf)
);
inbuf += 16;
inlen -= 16;
}
}
/* Partial fold. */
if (inlen)
{
/* Load last input and add padding zeros. */
asm volatile ("movdqu %[shr_shuf], %%xmm3\n\t"
"movdqu %[shl_shuf], %%xmm4\n\t"
"movdqu %[mask], %%xmm2\n\t"
"movdqa %%xmm0, %%xmm1\n\t"
"pshufb %%xmm4, %%xmm0\n\t"
"movdqu %[inbuf], %%xmm4\n\t"
"pshufb %%xmm3, %%xmm1\n\t"
"pand %%xmm4, %%xmm2\n\t"
"por %%xmm1, %%xmm2\n\t"
"movdqa %%xmm0, %%xmm1\n\t"
"pclmulqdq $0x00, %%xmm6, %%xmm0\n\t"
"pclmulqdq $0x11, %%xmm6, %%xmm1\n\t"
"pxor %%xmm2, %%xmm0\n\t"
"pxor %%xmm1, %%xmm0\n\t"
:
: [inbuf] "m" (*(inbuf - 16 + inlen)),
[mask] "m" (crc32_partial_fold_input_mask[inlen]),
[shl_shuf] "m" (crc32_refl_shuf_shift[inlen]),
[shr_shuf] "m" (crc32_refl_shuf_shift[inlen + 16])
);
inbuf += inlen;
inlen -= inlen;
}
/* Final fold. */
asm volatile (/* reduce 128-bits to 96-bits */
"movdqa %%xmm0, %%xmm1\n\t"
"pclmulqdq $0x10, %%xmm6, %%xmm0\n\t"
"psrldq $8, %%xmm1\n\t"
"pxor %%xmm1, %%xmm0\n\t"
/* reduce 96-bits to 64-bits */
"pshufd $0xfc, %%xmm0, %%xmm1\n\t" /* [00][00][00][x] */
"pshufd $0xf9, %%xmm0, %%xmm0\n\t" /* [00][00][x>>64][x>>32] */
"pclmulqdq $0x00, %[k5], %%xmm1\n\t" /* [00][00][xx][xx] */
"pxor %%xmm1, %%xmm0\n\t" /* top 64-bit are zero */
/* barrett reduction */
"pshufd $0xf3, %%xmm0, %%xmm1\n\t" /* [00][00][x>>32][00] */
"pslldq $4, %%xmm0\n\t" /* [??][x>>32][??][??] */
"pclmulqdq $0x00, %%xmm5, %%xmm1\n\t" /* [00][xx][xx][00] */
"pclmulqdq $0x10, %%xmm5, %%xmm1\n\t" /* [00][xx][xx][00] */
"pxor %%xmm1, %%xmm0\n\t"
/* store CRC */
"pextrd $2, %%xmm0, %[out]\n\t"
: [out] "=m" (*pcrc)
: [k5] "m" (consts->k[5 - 1])
);
}
static inline void
crc32_reflected_less_than_16 (u32 *pcrc, const byte *inbuf, size_t inlen,
const struct crc32_consts_s *consts)
{
if (inlen < 4)
{
u32 crc = *pcrc;
u32 data;
asm volatile ("movdqa %[my_p], %%xmm5\n\t"
:
: [my_p] "m" (consts->my_p[0])
);
if (inlen == 1)
{
data = inbuf[0];
data ^= crc;
data <<= 24;
crc >>= 8;
}
else if (inlen == 2)
{
data = ((const struct u16_unaligned_s *)inbuf)->a;
data ^= crc;
data <<= 16;
crc >>= 16;
}
else
{
data = ((const struct u16_unaligned_s *)inbuf)->a;
data |= inbuf[2] << 16;
data ^= crc;
data <<= 8;
crc >>= 24;
}
/* Barrett reduction */
asm volatile ("movd %[in], %%xmm0\n\t"
"movd %[crc], %%xmm1\n\t"
"pclmulqdq $0x00, %%xmm5, %%xmm0\n\t" /* [00][00][xx][xx] */
"psllq $32, %%xmm1\n\t"
"pshufd $0xfc, %%xmm0, %%xmm0\n\t" /* [00][00][00][x] */
"pclmulqdq $0x10, %%xmm5, %%xmm0\n\t" /* [00][00][xx][xx] */
"pxor %%xmm1, %%xmm0\n\t"
"pextrd $1, %%xmm0, %[out]\n\t"
: [out] "=m" (*pcrc)
: [in] "rm" (data),
[crc] "rm" (crc)
);
}
else if (inlen == 4)
{
/* Barrett reduction */
asm volatile ("movd %[crc], %%xmm1\n\t"
"movd %[in], %%xmm0\n\t"
"movdqa %[my_p], %%xmm5\n\t"
"pxor %%xmm1, %%xmm0\n\t"
"pclmulqdq $0x00, %%xmm5, %%xmm0\n\t" /* [00][00][xx][xx] */
"pshufd $0xfc, %%xmm0, %%xmm0\n\t" /* [00][00][00][x] */
"pclmulqdq $0x10, %%xmm5, %%xmm0\n\t" /* [00][00][xx][xx] */
"pextrd $1, %%xmm0, %[out]\n\t"
: [out] "=m" (*pcrc)
: [in] "m" (*inbuf),
[crc] "m" (*pcrc),
[my_p] "m" (consts->my_p[0])
);
}
else
{
asm volatile ("movdqu %[shuf], %%xmm4\n\t"
"movd %[crc], %%xmm1\n\t"
"movdqa %[my_p], %%xmm5\n\t"
"movdqa %[k3k4], %%xmm6\n\t"
:
: [shuf] "m" (crc32_refl_shuf_shift[inlen]),
[crc] "m" (*pcrc),
[my_p] "m" (consts->my_p[0]),
[k3k4] "m" (consts->k[3 - 1])
);
if (inlen >= 8)
{
asm volatile ("movq %[inbuf], %%xmm0\n\t"
:
: [inbuf] "m" (*inbuf)
);
if (inlen > 8)
{
asm volatile (/*"pinsrq $1, %[inbuf_tail], %%xmm0\n\t"*/
"movq %[inbuf_tail], %%xmm2\n\t"
"punpcklqdq %%xmm2, %%xmm0\n\t"
"pshufb %[merge_shuf], %%xmm0\n\t"
:
: [inbuf_tail] "m" (inbuf[inlen - 8]),
[merge_shuf] "m"
(*crc32_merge9to15_shuf[inlen - 9])
);
}
}
else
{
asm volatile ("movd %[inbuf], %%xmm0\n\t"
"pinsrd $1, %[inbuf_tail], %%xmm0\n\t"
"pshufb %[merge_shuf], %%xmm0\n\t"
:
: [inbuf] "m" (*inbuf),
[inbuf_tail] "m" (inbuf[inlen - 4]),
[merge_shuf] "m"
(*crc32_merge5to7_shuf[inlen - 5])
);
}
/* Final fold. */
asm volatile ("pxor %%xmm1, %%xmm0\n\t"
"pshufb %%xmm4, %%xmm0\n\t"
/* reduce 128-bits to 96-bits */
"movdqa %%xmm0, %%xmm1\n\t"
"pclmulqdq $0x10, %%xmm6, %%xmm0\n\t"
"psrldq $8, %%xmm1\n\t"
"pxor %%xmm1, %%xmm0\n\t" /* top 32-bit are zero */
/* reduce 96-bits to 64-bits */
"pshufd $0xfc, %%xmm0, %%xmm1\n\t" /* [00][00][00][x] */
"pshufd $0xf9, %%xmm0, %%xmm0\n\t" /* [00][00][x>>64][x>>32] */
"pclmulqdq $0x00, %[k5], %%xmm1\n\t" /* [00][00][xx][xx] */
"pxor %%xmm1, %%xmm0\n\t" /* top 64-bit are zero */
/* barrett reduction */
"pshufd $0xf3, %%xmm0, %%xmm1\n\t" /* [00][00][x>>32][00] */
"pslldq $4, %%xmm0\n\t" /* [??][x>>32][??][??] */
"pclmulqdq $0x00, %%xmm5, %%xmm1\n\t" /* [00][xx][xx][00] */
"pclmulqdq $0x10, %%xmm5, %%xmm1\n\t" /* [00][xx][xx][00] */
"pxor %%xmm1, %%xmm0\n\t"
/* store CRC */
"pextrd $2, %%xmm0, %[out]\n\t"
: [out] "=m" (*pcrc)
: [k5] "m" (consts->k[5 - 1])
);
}
}
void
crc32_intel_pclmul (u32 *pcrc, const byte *inbuf, size_t inlen)
{
const struct crc32_consts_s *consts = &crc32_consts;
#if defined(__x86_64__) && defined(__WIN64__)
char win64tmp[2 * 16];
/* XMM6-XMM7 need to be restored after use. */
asm volatile ("movdqu %%xmm6, 0*16(%0)\n\t"
"movdqu %%xmm7, 1*16(%0)\n\t"
:
: "r" (win64tmp)
: "memory");
#endif
if (!inlen)
return;
if (inlen >= 16)
crc32_reflected_bulk(pcrc, inbuf, inlen, consts);
else
crc32_reflected_less_than_16(pcrc, inbuf, inlen, consts);
#if defined(__x86_64__) && defined(__WIN64__)
/* Restore used registers. */
asm volatile("movdqu 0*16(%0), %%xmm6\n\t"
"movdqu 1*16(%0), %%xmm7\n\t"
:
: "r" (win64tmp)
: "memory");
#endif
}
#endif
extra/mariabackup/crc/crc-intel-pclmul.h 0 → 100644
View file @ ecb25df2
/******************************************************
Copyright (c) 2017 Percona LLC and/or its affiliates.
CRC32 using Intel's PCLMUL instruction.
This program is free software; you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation; version 2 of the License.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program; if not, write to the Free Software
Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA
*******************************************************/
#include <stdint.h>
#include <stddef.h>
void
crc32_intel_pclmul(uint32_t *pcrc, const uint8_t *inbuf, size_t inlen);
extra/mariabackup/crc/crc_glue.c 0 → 100644
View file @ ecb25df2
/******************************************************
Copyright (c) 2017 Percona LLC and/or its affiliates.
Zlib compatible CRC-32 implementation.
This program is free software; you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation; version 2 of the License.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program; if not, write to the Free Software
Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA
*******************************************************/
#include <zlib.h>
#include <stdint.h>
#include <string.h>
#include "crc_glue.h"
#include "crc-intel-pclmul.h"
#if __GNUC__ >= 4 && defined(__x86_64__)
static int pclmul_enabled = 0;
#endif
#if defined(__GNUC__) && defined(__x86_64__)
static
uint32_t
cpuid(uint32_t* ecx, uint32_t* edx)
{
uint32_t level;
asm("cpuid" : "=a" (level) : "a" (0) : "ebx", "ecx", "edx");
if (level < 1) {
return level;
}
asm("cpuid" : "=c" (*ecx), "=d" (*edx)
: "a" (1)
: "ebx");
return level;
}
#endif
void crc_init() {
#if defined(__GNUC__) && defined(__x86_64__)
uint32_t ecx, edx;
if (cpuid(&ecx, &edx) > 0) {
pclmul_enabled = ((ecx >> 19) & 1) && ((ecx >> 1) & 1);
}
#endif
}
unsigned long crc32_iso3309(unsigned long crc, const unsigned char *buf, unsigned int len)
{
#if __GNUC__ >= 4 && defined(__x86_64__) && defined(HAVE_CLMUL_INSTRUCTION)
if (pclmul_enabled) {
uint32_t crc_accum = crc ^ 0xffffffffL;
crc32_intel_pclmul(&crc_accum, buf, len);
return crc_accum ^ 0xffffffffL;
}
#endif
return crc32(crc, buf, len);
}
extra/mariabackup/crc/crc_glue.h 0 → 100644
View file @ ecb25df2
/******************************************************
Copyright (c) 2017 Percona LLC and/or its affiliates.
Zlib compatible CRC-32 implementation.
This program is free software; you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation; version 2 of the License.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program; if not, write to the Free Software
Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA
*******************************************************/
#ifdef __cplusplus
extern "C" {
#endif
void crc_init();
unsigned long crc32_iso3309(unsigned long crc, const unsigned char *buf, unsigned int len);
#ifdef __cplusplus
}
#endif
extra/mariabackup/datasink.c
View file @ ecb25df2
...@@ -27,7 +27,6 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA ...@@ -27,7 +27,6 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA
#include "ds_local.h" #include "ds_local.h"
#include "ds_stdout.h" #include "ds_stdout.h"
#include "ds_tmpfile.h" #include "ds_tmpfile.h"
#include "ds_encrypt.h"
#include "ds_buffer.h" #include "ds_buffer.h"
/************************************************************************ /************************************************************************
...@@ -60,6 +59,7 @@ ds_create(const char *root, ds_type_t type) ...@@ -60,6 +59,7 @@ ds_create(const char *root, ds_type_t type)
ds = &datasink_compress; ds = &datasink_compress;
break; break;
case DS_TYPE_ENCRYPT: case DS_TYPE_ENCRYPT:
case DS_TYPE_DECRYPT:
msg("Error : mariabackup does not support encrypted backups."); msg("Error : mariabackup does not support encrypted backups.");
exit(EXIT_FAILURE); exit(EXIT_FAILURE);
break; break;
......
extra/mariabackup/datasink.h
View file @ ecb25df2
...@@ -61,6 +61,7 @@ typedef enum { ...@@ -61,6 +61,7 @@ typedef enum {
DS_TYPE_XBSTREAM, DS_TYPE_XBSTREAM,
DS_TYPE_COMPRESS, DS_TYPE_COMPRESS,
DS_TYPE_ENCRYPT, DS_TYPE_ENCRYPT,
DS_TYPE_DECRYPT,
DS_TYPE_TMPFILE, DS_TYPE_TMPFILE,
DS_TYPE_BUFFER DS_TYPE_BUFFER
} ds_type_t; } ds_type_t;
......
extra/mariabackup/ds_decrypt.c 0 → 100644
View file @ ecb25df2
/******************************************************
Copyright (c) 2017 Percona LLC and/or its affiliates.
Encryption datasink implementation for XtraBackup.
This program is free software; you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation; version 2 of the License.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program; if not, write to the Free Software
Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA
*******************************************************/
#include <my_base.h>
#include "common.h"
#include "datasink.h"
#include "xbcrypt.h"
#include "xbcrypt_common.h"
#include "crc_glue.h"
typedef struct {
pthread_t id;
uint num;
pthread_mutex_t ctrl_mutex;
pthread_cond_t ctrl_cond;
pthread_mutex_t data_mutex;
pthread_cond_t data_cond;
my_bool started;
my_bool data_avail;
my_bool cancelled;
my_bool failed;
const uchar *from;
size_t from_len;
uchar *to;
size_t to_len;
size_t to_size;
const uchar *iv;
size_t iv_len;
unsigned long long offset;
my_bool hash_appended;
gcry_cipher_hd_t cipher_handle;
xb_rcrypt_result_t parse_result;
} crypt_thread_ctxt_t;
typedef struct {
crypt_thread_ctxt_t *threads;
uint nthreads;
int encrypt_algo;
size_t chunk_size;
char *encrypt_key;
char *encrypt_key_file;
} ds_decrypt_ctxt_t;
typedef struct {
ds_decrypt_ctxt_t *crypt_ctxt;
size_t bytes_processed;
ds_file_t *dest_file;
uchar *buf;
size_t buf_len;
size_t buf_size;
} ds_decrypt_file_t;
int ds_decrypt_encrypt_threads = 1;
static ds_ctxt_t *decrypt_init(const char *root);
static ds_file_t *decrypt_open(ds_ctxt_t *ctxt, const char *path,
MY_STAT *mystat);
static int decrypt_write(ds_file_t *file, const void *buf, size_t len);
static int decrypt_close(ds_file_t *file);
static void decrypt_deinit(ds_ctxt_t *ctxt);
datasink_t datasink_decrypt = {
&decrypt_init,
&decrypt_open,
&decrypt_write,
&decrypt_close,
&decrypt_deinit
};
static crypt_thread_ctxt_t *create_worker_threads(uint n);
static void destroy_worker_threads(crypt_thread_ctxt_t *threads, uint n);
static void *decrypt_worker_thread_func(void *arg);
static
ds_ctxt_t *
decrypt_init(const char *root)
{
ds_ctxt_t *ctxt;
ds_decrypt_ctxt_t *decrypt_ctxt;
crypt_thread_ctxt_t *threads;
if (xb_crypt_init(NULL)) {
return NULL;
}
/* Create and initialize the worker threads */
threads = create_worker_threads(ds_decrypt_encrypt_threads);
if (threads == NULL) {
msg("decrypt: failed to create worker threads.\n");
return NULL;
}
ctxt = (ds_ctxt_t *) my_malloc(sizeof(ds_ctxt_t) +
sizeof(ds_decrypt_ctxt_t),
MYF(MY_FAE));
decrypt_ctxt = (ds_decrypt_ctxt_t *) (ctxt + 1);
decrypt_ctxt->threads = threads;
decrypt_ctxt->nthreads = ds_decrypt_encrypt_threads;
ctxt->ptr = decrypt_ctxt;
ctxt->root = my_strdup(root, MYF(MY_FAE));
return ctxt;
}
static
ds_file_t *
decrypt_open(ds_ctxt_t *ctxt, const char *path, MY_STAT *mystat)
{
ds_ctxt_t *dest_ctxt;
ds_decrypt_ctxt_t *crypt_ctxt;
ds_decrypt_file_t *crypt_file;
char new_name[FN_REFLEN];
ds_file_t *file;
xb_ad(ctxt->pipe_ctxt != NULL);
dest_ctxt = ctxt->pipe_ctxt;
crypt_ctxt = (ds_decrypt_ctxt_t *) ctxt->ptr;
file = (ds_file_t *) my_malloc(sizeof(ds_file_t) +
sizeof(ds_decrypt_file_t),
MYF(MY_FAE|MY_ZEROFILL));
crypt_file = (ds_decrypt_file_t *) (file + 1);
/* Remove the .xbcrypt extension from the filename */
strncpy(new_name, path, FN_REFLEN);
new_name[strlen(new_name) - 8] = 0;
crypt_file->dest_file = ds_open(dest_ctxt, new_name, mystat);
if (crypt_file->dest_file == NULL) {
msg("decrypt: ds_open(\"%s\") failed.\n", new_name);
goto err;
}
crypt_file->crypt_ctxt = crypt_ctxt;
crypt_file->buf = NULL;
crypt_file->buf_size = 0;
crypt_file->buf_len = 0;
file->ptr = crypt_file;
file->path = crypt_file->dest_file->path;
return file;
err:
if (crypt_file->dest_file) {
ds_close(crypt_file->dest_file);
}
my_free(file);
return NULL;
}
#define CHECK_BUF_SIZE(ptr, size, buf, len) \
if (ptr + size - buf > (ssize_t) len) { \
result = XB_CRYPT_READ_INCOMPLETE; \
goto exit; \
}
static
xb_rcrypt_result_t
parse_xbcrypt_chunk(crypt_thread_ctxt_t *thd, const uchar *buf, size_t len,
size_t *bytes_processed)
{
const uchar *ptr;
uint version;
ulong checksum, checksum_exp;
ulonglong tmp;
xb_rcrypt_result_t result = XB_CRYPT_READ_CHUNK;
*bytes_processed = 0;
ptr = buf;
CHECK_BUF_SIZE(ptr, XB_CRYPT_CHUNK_MAGIC_SIZE, buf, len);
if (memcmp(ptr, XB_CRYPT_CHUNK_MAGIC3,
XB_CRYPT_CHUNK_MAGIC_SIZE) == 0) {
version = 3;
} else if (memcmp(ptr, XB_CRYPT_CHUNK_MAGIC2,
XB_CRYPT_CHUNK_MAGIC_SIZE) == 0) {
version = 2;
} else if (memcmp(ptr, XB_CRYPT_CHUNK_MAGIC1,
XB_CRYPT_CHUNK_MAGIC_SIZE) == 0) {
version = 1;
} else {
msg("%s:%s: wrong chunk magic at offset 0x%llx.\n",
my_progname, __FUNCTION__, thd->offset);
result = XB_CRYPT_READ_ERROR;
goto exit;
}
ptr += XB_CRYPT_CHUNK_MAGIC_SIZE;
thd->offset += XB_CRYPT_CHUNK_MAGIC_SIZE;
CHECK_BUF_SIZE(ptr, 8, buf, len);
tmp = uint8korr(ptr); /* reserved */
ptr += 8;
thd->offset += 8;
CHECK_BUF_SIZE(ptr, 8, buf, len);
tmp = uint8korr(ptr); /* original size */
ptr += 8;
if (tmp > INT_MAX) {
msg("%s:%s: invalid original size at offset 0x%llx.\n",
my_progname, __FUNCTION__, thd->offset);
result = XB_CRYPT_READ_ERROR;
goto exit;
}
thd->offset += 8;
thd->to_len = (size_t)tmp;
if (thd->to_size < thd->to_len + XB_CRYPT_HASH_LEN) {
thd->to = (uchar *) my_realloc(
thd->to,
thd->to_len + XB_CRYPT_HASH_LEN,
MYF(MY_FAE | MY_ALLOW_ZERO_PTR));
thd->to_size = thd->to_len;
}
CHECK_BUF_SIZE(ptr, 8, buf, len);
tmp = uint8korr(ptr); /* encrypted size */
ptr += 8;
if (tmp > INT_MAX) {
msg("%s:%s: invalid encrypted size at offset 0x%llx.\n",
my_progname, __FUNCTION__, thd->offset);
result = XB_CRYPT_READ_ERROR;
goto exit;
}
thd->offset += 8;
thd->from_len = (size_t)tmp;
xb_a(thd->from_len <= thd->to_len + XB_CRYPT_HASH_LEN);
CHECK_BUF_SIZE(ptr, 4, buf, len);
checksum_exp = uint4korr(ptr); /* checksum */
ptr += 4;
thd->offset += 4;
/* iv size */
if (version == 1) {
thd->iv_len = 0;
thd->iv = NULL;
} else {
CHECK_BUF_SIZE(ptr, 8, buf, len);
tmp = uint8korr(ptr);
if (tmp > INT_MAX) {
msg("%s:%s: invalid iv size at offset 0x%llx.\n",
my_progname, __FUNCTION__, thd->offset);
result = XB_CRYPT_READ_ERROR;
goto exit;
}
ptr += 8;
thd->offset += 8;
thd->iv_len = (size_t)tmp;
}
if (thd->iv_len > 0) {
CHECK_BUF_SIZE(ptr, thd->iv_len, buf, len);
thd->iv = ptr;
ptr += thd->iv_len;
}
/* for version euqals 2 we need to read in the iv data but do not init
CTR with it */
if (version == 2) {
thd->iv_len = 0;
thd->iv = 0;
}
if (thd->from_len > 0) {
CHECK_BUF_SIZE(ptr, thd->from_len, buf, len);
thd->from = ptr;
ptr += thd->from_len;
}
xb_ad(thd->from_len <= thd->to_len);
checksum = crc32_iso3309(0, thd->from, thd->from_len);
if (checksum != checksum_exp) {
msg("%s:%s invalid checksum at offset 0x%llx, "
"expected 0x%lx, actual 0x%lx.\n", my_progname,
__FUNCTION__, thd->offset, checksum_exp, checksum);
result = XB_CRYPT_READ_ERROR;
goto exit;
}
thd->offset += thd->from_len;
thd->hash_appended = version > 2;
exit:
*bytes_processed = (size_t) (ptr - buf);
return result;
}
static
int
decrypt_write(ds_file_t *file, const void *buf, size_t len)
{
ds_decrypt_file_t *crypt_file;
ds_decrypt_ctxt_t *crypt_ctxt;
crypt_thread_ctxt_t *threads;
crypt_thread_ctxt_t *thd;
uint nthreads;
uint i;
size_t bytes_processed;
xb_rcrypt_result_t parse_result = XB_CRYPT_READ_CHUNK;
my_bool err = FALSE;
crypt_file = (ds_decrypt_file_t *) file->ptr;
crypt_ctxt = crypt_file->crypt_ctxt;
threads = crypt_ctxt->threads;
nthreads = crypt_ctxt->nthreads;
if (crypt_file->buf_len > 0) {
thd = threads;
pthread_mutex_lock(&thd->ctrl_mutex);
do {
if (parse_result == XB_CRYPT_READ_INCOMPLETE) {
crypt_file->buf_size = crypt_file->buf_size * 2;
crypt_file->buf = (uchar *) my_realloc(
crypt_file->buf,
crypt_file->buf_size,
MYF(MY_FAE|MY_ALLOW_ZERO_PTR));
}
memcpy(crypt_file->buf + crypt_file->buf_len,
buf, MY_MIN(crypt_file->buf_size -
crypt_file->buf_len, len));
parse_result = parse_xbcrypt_chunk(
thd, crypt_file->buf,
crypt_file->buf_size, &bytes_processed);
if (parse_result == XB_CRYPT_READ_ERROR) {
pthread_mutex_unlock(&thd->ctrl_mutex);
return 1;
}
} while (parse_result == XB_CRYPT_READ_INCOMPLETE &&
crypt_file->buf_size < len);
if (parse_result != XB_CRYPT_READ_CHUNK) {
msg("decrypt: incomplete data.\n");
pthread_mutex_unlock(&thd->ctrl_mutex);
return 1;
}
pthread_mutex_lock(&thd->data_mutex);
thd->data_avail = TRUE;
pthread_cond_signal(&thd->data_cond);
pthread_mutex_unlock(&thd->data_mutex);
len -= bytes_processed - crypt_file->buf_len;
buf += bytes_processed - crypt_file->buf_len;
/* reap */
pthread_mutex_lock(&thd->data_mutex);
while (thd->data_avail == TRUE) {
pthread_cond_wait(&thd->data_cond,
&thd->data_mutex);
}
if (thd->failed) {
msg("decrypt: failed to decrypt chunk.\n");
err = TRUE;
}
xb_a(thd->to_len > 0);
if (!err &&
ds_write(crypt_file->dest_file, thd->to, thd->to_len)) {
msg("decrypt: write to destination failed.\n");
err = TRUE;
}
crypt_file->bytes_processed += thd->from_len;
pthread_mutex_unlock(&thd->data_mutex);
pthread_mutex_unlock(&thd->ctrl_mutex);
crypt_file->buf_len = 0;
if (err) {
return 1;
}
}
while (parse_result == XB_CRYPT_READ_CHUNK && len > 0) {
uint max_thread;
for (i = 0; i < nthreads; i++) {
thd = threads + i;
pthread_mutex_lock(&thd->ctrl_mutex);
parse_result = parse_xbcrypt_chunk(
thd, buf, len, &bytes_processed);
if (parse_result == XB_CRYPT_READ_ERROR) {
pthread_mutex_unlock(&thd->ctrl_mutex);
err = TRUE;
break;
}
thd->parse_result = parse_result;
if (parse_result != XB_CRYPT_READ_CHUNK) {
pthread_mutex_unlock(&thd->ctrl_mutex);
break;
}
pthread_mutex_lock(&thd->data_mutex);
thd->data_avail = TRUE;
pthread_cond_signal(&thd->data_cond);
pthread_mutex_unlock(&thd->data_mutex);
len -= bytes_processed;
buf += bytes_processed;
}
max_thread = (i < nthreads) ? i : nthreads - 1;
/* Reap and write decrypted data */
for (i = 0; i <= max_thread; i++) {
thd = threads + i;
if (thd->parse_result != XB_CRYPT_READ_CHUNK) {
break;
}
pthread_mutex_lock(&thd->data_mutex);
while (thd->data_avail == TRUE) {
pthread_cond_wait(&thd->data_cond,
&thd->data_mutex);
}
if (thd->failed) {
msg("decrypt: failed to decrypt chunk.\n");
err = TRUE;
}
xb_a(thd->to_len > 0);
if (!err && ds_write(crypt_file->dest_file, thd->to,
thd->to_len)) {
msg("decrypt: write to destination failed.\n");
err = TRUE;
}
crypt_file->bytes_processed += thd->from_len;
pthread_mutex_unlock(&thd->data_mutex);
pthread_mutex_unlock(&thd->ctrl_mutex);
}
if (err) {
return 1;
}
}
if (parse_result == XB_CRYPT_READ_INCOMPLETE && len > 0) {
crypt_file->buf_len = len;
if (crypt_file->buf_size < len) {
crypt_file->buf = (uchar *) my_realloc(
crypt_file->buf,
crypt_file->buf_len,
MYF(MY_FAE | MY_ALLOW_ZERO_PTR));
crypt_file->buf_size = len;
}
memcpy(crypt_file->buf, buf, len);
}
return 0;
}
static
int
decrypt_close(ds_file_t *file)
{
ds_decrypt_file_t *crypt_file;
ds_file_t *dest_file;
int rc = 0;
crypt_file = (ds_decrypt_file_t *) file->ptr;
dest_file = crypt_file->dest_file;
if (ds_close(dest_file)) {
rc = 1;
}
my_free(crypt_file->buf);
my_free(file);
return rc;
}
static
void
decrypt_deinit(ds_ctxt_t *ctxt)
{
ds_decrypt_ctxt_t *crypt_ctxt;
xb_ad(ctxt->pipe_ctxt != NULL);
crypt_ctxt = (ds_decrypt_ctxt_t *) ctxt->ptr;
destroy_worker_threads(crypt_ctxt->threads, crypt_ctxt->nthreads);
my_free(ctxt->root);
my_free(ctxt);
}
static
crypt_thread_ctxt_t *
create_worker_threads(uint n)
{
crypt_thread_ctxt_t *threads;
uint i;
threads = (crypt_thread_ctxt_t *)
my_malloc(sizeof(crypt_thread_ctxt_t) * n,
MYF(MY_FAE | MY_ZEROFILL));
for (i = 0; i < n; i++) {
crypt_thread_ctxt_t *thd = threads + i;
thd->num = i + 1;
/* Initialize the control mutex and condition var */
if (pthread_mutex_init(&thd->ctrl_mutex, NULL) ||
pthread_cond_init(&thd->ctrl_cond, NULL)) {
goto err;
}
/* Initialize and data mutex and condition var */
if (pthread_mutex_init(&thd->data_mutex, NULL) ||
pthread_cond_init(&thd->data_cond, NULL)) {
goto err;
}
xb_crypt_cipher_open(&thd->cipher_handle);
pthread_mutex_lock(&thd->ctrl_mutex);
if (pthread_create(&thd->id, NULL, decrypt_worker_thread_func,
thd)) {
msg("decrypt: pthread_create() failed: "
"errno = %d\n", errno);
goto err;
}
}
/* Wait for the threads to start */
for (i = 0; i < n; i++) {
crypt_thread_ctxt_t *thd = threads + i;
while (thd->started == FALSE)
pthread_cond_wait(&thd->ctrl_cond, &thd->ctrl_mutex);
pthread_mutex_unlock(&thd->ctrl_mutex);
}
return threads;
err:
return NULL;
}
static
void
destroy_worker_threads(crypt_thread_ctxt_t *threads, uint n)
{
uint i;
for (i = 0; i < n; i++) {
crypt_thread_ctxt_t *thd = threads + i;
pthread_mutex_lock(&thd->data_mutex);
threads[i].cancelled = TRUE;
pthread_cond_signal(&thd->data_cond);
pthread_mutex_unlock(&thd->data_mutex);
pthread_join(thd->id, NULL);
pthread_cond_destroy(&thd->data_cond);
pthread_mutex_destroy(&thd->data_mutex);
pthread_cond_destroy(&thd->ctrl_cond);
pthread_mutex_destroy(&thd->ctrl_mutex);
xb_crypt_cipher_close(thd->cipher_handle);
my_free(thd->to);
}
my_free(threads);
}
static
void *
decrypt_worker_thread_func(void *arg)
{
crypt_thread_ctxt_t *thd = (crypt_thread_ctxt_t *) arg;
pthread_mutex_lock(&thd->ctrl_mutex);
pthread_mutex_lock(&thd->data_mutex);
thd->started = TRUE;
pthread_cond_signal(&thd->ctrl_cond);
pthread_mutex_unlock(&thd->ctrl_mutex);
while (1) {
thd->data_avail = FALSE;
pthread_cond_signal(&thd->data_cond);
while (!thd->data_avail && !thd->cancelled) {
pthread_cond_wait(&thd->data_cond, &thd->data_mutex);
}
if (thd->cancelled)
break;
if (xb_crypt_decrypt(thd->cipher_handle, thd->from,
thd->from_len, thd->to, &thd->to_len,
thd->iv, thd->iv_len,
thd->hash_appended)) {
thd->failed = TRUE;
continue;
}
}
pthread_mutex_unlock(&thd->data_mutex);
return NULL;
}
extra/mariabackup/ds_decrypt.h 0 → 100644
View file @ ecb25df2
/******************************************************
Copyright (c) 2017 Percona LLC and/or its affiliates.
Encryption interface for XtraBackup.
This program is free software; you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation; version 2 of the License.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program; if not, write to the Free Software
Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA
*******************************************************/
#ifndef DS_DECRYPT_H
#define DS_DECRYPT_H
#include "datasink.h"
extern datasink_t datasink_decrypt;
extern int ds_decrypt_encrypt_threads;
#endif
extra/mariabackup/ds_encrypt.c
View file @ ecb25df2
...@@ -22,25 +22,11 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA ...@@ -22,25 +22,11 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA
#include <my_base.h> #include <my_base.h>
#include "common.h" #include "common.h"
#include "datasink.h" #include "datasink.h"
#include "xbcrypt_common.h"
#ifdef HAVE_GRYPT #ifdef HAVE_GRYPT
#if GCC_VERSION >= 4002
/* Workaround to avoid "gcry_ac_* is deprecated" warnings in gcrypt.h */
# pragma GCC diagnostic ignored "-Wdeprecated-declarations"
#endif
#include <gcrypt.h>
#if GCC_VERSION >= 4002
# pragma GCC diagnostic warning "-Wdeprecated-declarations"
#endif
#include "xbcrypt.h" #include "xbcrypt.h"
#if !defined(GCRYPT_VERSION_NUMBER) || (GCRYPT_VERSION_NUMBER < 0x010600) #define XB_CRYPT_CHUNK_SIZE ((size_t) (ds_encrypt_encrypt_chunk_size))
GCRY_THREAD_OPTION_PTHREAD_IMPL;
#endif
#define XB_CRYPT_CHUNK_SIZE ((size_t) (xtrabackup_encrypt_chunk_size))
typedef struct { typedef struct {
pthread_t id; pthread_t id;
...@@ -52,10 +38,10 @@ typedef struct { ...@@ -52,10 +38,10 @@ typedef struct {
my_bool started; my_bool started;
my_bool data_avail; my_bool data_avail;
my_bool cancelled; my_bool cancelled;
const char *from; const uchar *from;
size_t from_len; size_t from_len;
char *to; uchar *to;
char *iv; uchar *iv;
size_t to_len; size_t to_len;
gcry_cipher_hd_t cipher_handle; gcry_cipher_hd_t cipher_handle;
} crypt_thread_ctxt_t; } crypt_thread_ctxt_t;
...@@ -73,11 +59,8 @@ typedef struct { ...@@ -73,11 +59,8 @@ typedef struct {
} ds_encrypt_file_t; } ds_encrypt_file_t;
/* Encryption options */ /* Encryption options */
extern ulong xtrabackup_encrypt_algo; uint ds_encrypt_encrypt_threads;
extern char *xtrabackup_encrypt_key; ulonglong ds_encrypt_encrypt_chunk_size;
extern char *xtrabackup_encrypt_key_file;
extern uint xtrabackup_encrypt_threads;
extern ulonglong xtrabackup_encrypt_chunk_size;
static ds_ctxt_t *encrypt_init(const char *root); static ds_ctxt_t *encrypt_init(const char *root);
static ds_file_t *encrypt_open(ds_ctxt_t *ctxt, const char *path, static ds_file_t *encrypt_open(ds_ctxt_t *ctxt, const char *path,
...@@ -98,12 +81,7 @@ static crypt_thread_ctxt_t *create_worker_threads(uint n); ...@@ -98,12 +81,7 @@ static crypt_thread_ctxt_t *create_worker_threads(uint n);
static void destroy_worker_threads(crypt_thread_ctxt_t *threads, uint n); static void destroy_worker_threads(crypt_thread_ctxt_t *threads, uint n);
static void *encrypt_worker_thread_func(void *arg); static void *encrypt_worker_thread_func(void *arg);
static uint encrypt_algos[] = { GCRY_CIPHER_NONE, GCRY_CIPHER_AES128, static uint encrypt_iv_len = 0;
GCRY_CIPHER_AES192, GCRY_CIPHER_AES256 };
static uint encrypt_algo;
static const uint encrypt_mode = GCRY_CIPHER_MODE_CTR;
static uint encrypt_key_len = 0;
static size_t encrypt_iv_len = 0;
static static
ssize_t ssize_t
...@@ -129,87 +107,13 @@ encrypt_init(const char *root) ...@@ -129,87 +107,13 @@ encrypt_init(const char *root)
ds_ctxt_t *ctxt; ds_ctxt_t *ctxt;
ds_encrypt_ctxt_t *encrypt_ctxt; ds_encrypt_ctxt_t *encrypt_ctxt;
crypt_thread_ctxt_t *threads; crypt_thread_ctxt_t *threads;
gcry_error_t gcry_error;
/* Acording to gcrypt docs (and my testing), setting up the threading
callbacks must be done first, so, lets give it a shot */
#if !defined(GCRYPT_VERSION_NUMBER) || (GCRYPT_VERSION_NUMBER < 0x010600)
gcry_error = gcry_control(GCRYCTL_SET_THREAD_CBS, &gcry_threads_pthread);
if (gcry_error) {
msg("encrypt: unable to set libgcrypt thread cbs - "
"%s : %s\n",
gcry_strsource(gcry_error),
gcry_strerror(gcry_error));
return NULL;
}
#endif
/* Version check should be the very next call because it
makes sure that important subsystems are intialized. */
if (!gcry_control(GCRYCTL_ANY_INITIALIZATION_P)) {
const char *gcrypt_version;
gcrypt_version = gcry_check_version(NULL);
/* No other library has already initialized libgcrypt. */
if (!gcrypt_version) {
msg("encrypt: failed to initialize libgcrypt\n");
return NULL;
} else {
msg("encrypt: using gcrypt %s\n", gcrypt_version);
}
}
/* Disable the gcry secure memory, not dealing with this for now */ if (xb_crypt_init(&encrypt_iv_len)) {
gcry_error = gcry_control(GCRYCTL_DISABLE_SECMEM, 0);
if (gcry_error) {
msg("encrypt: unable to disable libgcrypt secmem - "
"%s : %s\n",
gcry_strsource(gcry_error),
gcry_strerror(gcry_error));
return NULL;
}
/* Finalize gcry initialization. */
gcry_error = gcry_control(GCRYCTL_INITIALIZATION_FINISHED, 0);
if (gcry_error) {
msg("encrypt: unable to finish libgcrypt initialization - "
"%s : %s\n",
gcry_strsource(gcry_error),
gcry_strerror(gcry_error));
return NULL;
}
/* Determine the algorithm */
encrypt_algo = encrypt_algos[xtrabackup_encrypt_algo];
/* Set up the iv length */
encrypt_iv_len = gcry_cipher_get_algo_blklen(encrypt_algo);
xb_a(encrypt_iv_len > 0);
/* Now set up the key */
if (xtrabackup_encrypt_key == NULL &&
xtrabackup_encrypt_key_file == NULL) {
msg("encrypt: no encryption key or key file specified.\n");
return NULL;
} else if (xtrabackup_encrypt_key && xtrabackup_encrypt_key_file) {
msg("encrypt: both encryption key and key file specified.\n");
return NULL;
} else if (xtrabackup_encrypt_key_file) {
if (!xb_crypt_read_key_file(xtrabackup_encrypt_key_file,
(void**)&xtrabackup_encrypt_key,
&encrypt_key_len)) {
msg("encrypt: unable to read encryption key file"
" \"%s\".\n", xtrabackup_encrypt_key_file);
return NULL;
}
} else if (xtrabackup_encrypt_key) {
encrypt_key_len = strlen(xtrabackup_encrypt_key);
} else {
msg("encrypt: no encryption key or key file specified.\n");
return NULL; return NULL;
} }
/* Create and initialize the worker threads */ /* Create and initialize the worker threads */
threads = create_worker_threads(xtrabackup_encrypt_threads); threads = create_worker_threads(ds_encrypt_encrypt_threads);
if (threads == NULL) { if (threads == NULL) {
msg("encrypt: failed to create worker threads.\n"); msg("encrypt: failed to create worker threads.\n");
return NULL; return NULL;
...@@ -221,7 +125,7 @@ encrypt_init(const char *root) ...@@ -221,7 +125,7 @@ encrypt_init(const char *root)
encrypt_ctxt = (ds_encrypt_ctxt_t *) (ctxt + 1); encrypt_ctxt = (ds_encrypt_ctxt_t *) (ctxt + 1);
encrypt_ctxt->threads = threads; encrypt_ctxt->threads = threads;
encrypt_ctxt->nthreads = xtrabackup_encrypt_threads; encrypt_ctxt->nthreads = ds_encrypt_encrypt_threads;
ctxt->ptr = encrypt_ctxt; ctxt->ptr = encrypt_ctxt;
ctxt->root = my_strdup(root, MYF(MY_FAE)); ctxt->root = my_strdup(root, MYF(MY_FAE));
...@@ -294,7 +198,7 @@ encrypt_write(ds_file_t *file, const void *buf, size_t len) ...@@ -294,7 +198,7 @@ encrypt_write(ds_file_t *file, const void *buf, size_t len)
crypt_thread_ctxt_t *thd; crypt_thread_ctxt_t *thd;
uint nthreads; uint nthreads;
uint i; uint i;
const char *ptr; const uchar *ptr;
crypt_file = (ds_encrypt_file_t *) file->ptr; crypt_file = (ds_encrypt_file_t *) file->ptr;
crypt_ctxt = crypt_file->crypt_ctxt; crypt_ctxt = crypt_file->crypt_ctxt;
...@@ -302,7 +206,7 @@ encrypt_write(ds_file_t *file, const void *buf, size_t len) ...@@ -302,7 +206,7 @@ encrypt_write(ds_file_t *file, const void *buf, size_t len)
threads = crypt_ctxt->threads; threads = crypt_ctxt->threads;
nthreads = crypt_ctxt->nthreads; nthreads = crypt_ctxt->nthreads;
ptr = (const char *) buf; ptr = (const uchar *) buf;
while (len > 0) { while (len > 0) {
uint max_thread; uint max_thread;
...@@ -403,10 +307,6 @@ encrypt_deinit(ds_ctxt_t *ctxt) ...@@ -403,10 +307,6 @@ encrypt_deinit(ds_ctxt_t *ctxt)
my_free(ctxt->root); my_free(ctxt->root);
my_free(ctxt); my_free(ctxt);
if (xtrabackup_encrypt_key)
my_free(xtrabackup_encrypt_key);
if (xtrabackup_encrypt_key_file)
my_free(xtrabackup_encrypt_key_file);
} }
static static
...@@ -427,11 +327,10 @@ create_worker_threads(uint n) ...@@ -427,11 +327,10 @@ create_worker_threads(uint n)
thd->cancelled = FALSE; thd->cancelled = FALSE;
thd->data_avail = FALSE; thd->data_avail = FALSE;
thd->to = (char *) my_malloc(XB_CRYPT_CHUNK_SIZE + thd->to = (uchar *) my_malloc(XB_CRYPT_CHUNK_SIZE +
XB_CRYPT_HASH_LEN, MYF(MY_FAE)); XB_CRYPT_HASH_LEN, MYF(MY_FAE));
thd->iv = (char *) my_malloc(encrypt_iv_len, thd->iv = (uchar *) my_malloc(encrypt_iv_len, MYF(MY_FAE));
MYF(MY_FAE));
/* Initialize the control mutex and condition var */ /* Initialize the control mutex and condition var */
if (pthread_mutex_init(&thd->ctrl_mutex, NULL) || if (pthread_mutex_init(&thd->ctrl_mutex, NULL) ||
...@@ -445,34 +344,10 @@ create_worker_threads(uint n) ...@@ -445,34 +344,10 @@ create_worker_threads(uint n)
goto err; goto err;
} }
if (encrypt_algo != GCRY_CIPHER_NONE) { if (xb_crypt_cipher_open(&thd->cipher_handle)) {
gcry_error_t gcry_error;
gcry_error = gcry_cipher_open(&thd->cipher_handle,
encrypt_algo,
encrypt_mode, 0);
if (gcry_error) {
msg("encrypt: unable to open libgcrypt"
" cipher - %s : %s\n",
gcry_strsource(gcry_error),
gcry_strerror(gcry_error));
gcry_cipher_close(thd->cipher_handle);
goto err; goto err;
} }
gcry_error = gcry_cipher_setkey(thd->cipher_handle,
xtrabackup_encrypt_key,
encrypt_key_len);
if (gcry_error) {
msg("encrypt: unable to set libgcrypt"
" cipher key - %s : %s\n",
gcry_strsource(gcry_error),
gcry_strerror(gcry_error));
gcry_cipher_close(thd->cipher_handle);
goto err;
}
}
pthread_mutex_lock(&thd->ctrl_mutex); pthread_mutex_lock(&thd->ctrl_mutex);
if (pthread_create(&thd->id, NULL, encrypt_worker_thread_func, if (pthread_create(&thd->id, NULL, encrypt_worker_thread_func,
...@@ -519,8 +394,7 @@ destroy_worker_threads(crypt_thread_ctxt_t *threads, uint n) ...@@ -519,8 +394,7 @@ destroy_worker_threads(crypt_thread_ctxt_t *threads, uint n)
pthread_cond_destroy(&thd->ctrl_cond); pthread_cond_destroy(&thd->ctrl_cond);
pthread_mutex_destroy(&thd->ctrl_mutex); pthread_mutex_destroy(&thd->ctrl_mutex);
if (encrypt_algo != GCRY_CIPHER_NONE) xb_crypt_cipher_close(thd->cipher_handle);
gcry_cipher_close(thd->cipher_handle);
my_free(thd->to); my_free(thd->to);
my_free(thd->iv); my_free(thd->iv);
...@@ -555,60 +429,14 @@ encrypt_worker_thread_func(void *arg) ...@@ -555,60 +429,14 @@ encrypt_worker_thread_func(void *arg)
if (thd->cancelled) if (thd->cancelled)
break; break;
/* ensure that XB_CRYPT_HASH_LEN is the correct length
of XB_CRYPT_HASH hashing algorithm output */
assert(gcry_md_get_algo_dlen(XB_CRYPT_HASH) ==
XB_CRYPT_HASH_LEN);
memcpy(thd->to, thd->from, thd->from_len);
gcry_md_hash_buffer(XB_CRYPT_HASH, thd->to + thd->from_len,
thd->from, thd->from_len);
thd->to_len = thd->from_len; thd->to_len = thd->from_len;
if (encrypt_algo != GCRY_CIPHER_NONE) { if (xb_crypt_encrypt(thd->cipher_handle, thd->from,
gcry_error_t gcry_error; thd->from_len, thd->to, &thd->to_len,
thd->iv)) {
gcry_error = gcry_cipher_reset(thd->cipher_handle);
if (gcry_error) {
msg("encrypt: unable to reset cipher - "
"%s : %s\n",
gcry_strsource(gcry_error),
gcry_strerror(gcry_error));
thd->to_len = 0; thd->to_len = 0;
continue; continue;
} }
xb_crypt_create_iv(thd->iv, encrypt_iv_len);
gcry_error = gcry_cipher_setctr(thd->cipher_handle,
thd->iv,
encrypt_iv_len);
if (gcry_error) {
msg("encrypt: unable to set cipher ctr - "
"%s : %s\n",
gcry_strsource(gcry_error),
gcry_strerror(gcry_error));
thd->to_len = 0;
continue;
}
gcry_error = gcry_cipher_encrypt(thd->cipher_handle,
thd->to,
thd->to_len +
XB_CRYPT_HASH_LEN,
thd->to,
thd->from_len +
XB_CRYPT_HASH_LEN);
if (gcry_error) {
msg("encrypt: unable to encrypt buffer - "
"%s : %s\n", gcry_strsource(gcry_error),
gcry_strerror(gcry_error));
thd->to_len = 0;
}
} else {
memcpy(thd->to, thd->from,
thd->from_len + XB_CRYPT_HASH_LEN);
}
thd->to_len += XB_CRYPT_HASH_LEN;
} }
pthread_mutex_unlock(&thd->data_mutex); pthread_mutex_unlock(&thd->data_mutex);
......
extra/mariabackup/ds_encrypt.h
View file @ ecb25df2
...@@ -25,4 +25,9 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA ...@@ -25,4 +25,9 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA
#ifdef HAVE_GCRYPT #ifdef HAVE_GCRYPT
extern datasink_t datasink_encrypt; extern datasink_t datasink_encrypt;
#endif #endif
/* Encryption options */
extern uint ds_encrypt_encrypt_threads;
extern ulonglong ds_encrypt_encrypt_chunk_size;
#endif #endif
extra/mariabackup/xbcrypt.c
View file @ ecb25df2
...@@ -22,7 +22,8 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA ...@@ -22,7 +22,8 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA
#include <my_getopt.h> #include <my_getopt.h>
#include "common.h" #include "common.h"
#include "xbcrypt.h" #include "xbcrypt.h"
#include <gcrypt.h> #include "xbcrypt_common.h"
#include "crc_glue.h"
#if !defined(GCRYPT_VERSION_NUMBER) || (GCRYPT_VERSION_NUMBER < 0x010600) #if !defined(GCRYPT_VERSION_NUMBER) || (GCRYPT_VERSION_NUMBER < 0x010600)
GCRY_THREAD_OPTION_PTHREAD_IMPL; GCRY_THREAD_OPTION_PTHREAD_IMPL;
...@@ -138,6 +139,8 @@ main(int argc, char **argv) ...@@ -138,6 +139,8 @@ main(int argc, char **argv)
MY_INIT(argv[0]); MY_INIT(argv[0]);
crc_init();
if (get_options(&argc, &argv)) { if (get_options(&argc, &argv)) {
goto err; goto err;
} }
...@@ -402,7 +405,7 @@ mode_decrypt(File filein, File fileout) ...@@ -402,7 +405,7 @@ mode_decrypt(File filein, File fileout)
/* ensure that XB_CRYPT_HASH_LEN is the correct length /* ensure that XB_CRYPT_HASH_LEN is the correct length
of XB_CRYPT_HASH hashing algorithm output */ of XB_CRYPT_HASH hashing algorithm output */
assert(gcry_md_get_algo_dlen(XB_CRYPT_HASH) == xb_a(gcry_md_get_algo_dlen(XB_CRYPT_HASH) ==
XB_CRYPT_HASH_LEN); XB_CRYPT_HASH_LEN);
gcry_md_hash_buffer(XB_CRYPT_HASH, hash, decryptbuf, gcry_md_hash_buffer(XB_CRYPT_HASH, hash, decryptbuf,
originalsize); originalsize);
...@@ -529,8 +532,7 @@ mode_encrypt(File filein, File fileout) ...@@ -529,8 +532,7 @@ mode_encrypt(File filein, File fileout)
/* ensure that XB_CRYPT_HASH_LEN is the correct length /* ensure that XB_CRYPT_HASH_LEN is the correct length
of XB_CRYPT_HASH hashing algorithm output */ of XB_CRYPT_HASH hashing algorithm output */
assert(XB_CRYPT_HASH_LEN == xb_a(XB_CRYPT_HASH_LEN == gcry_md_get_algo_dlen(XB_CRYPT_HASH));
gcry_md_get_algo_dlen(XB_CRYPT_HASH));
gcry_md_hash_buffer(XB_CRYPT_HASH, chunkbuf + bytesread, gcry_md_hash_buffer(XB_CRYPT_HASH, chunkbuf + bytesread,
chunkbuf, bytesread); chunkbuf, bytesread);
......
extra/mariabackup/xbcrypt.h
View file @ ecb25df2
...@@ -65,6 +65,7 @@ xb_rcrypt_t *xb_crypt_read_open(void *userdata, ...@@ -65,6 +65,7 @@ xb_rcrypt_t *xb_crypt_read_open(void *userdata,
typedef enum { typedef enum {
XB_CRYPT_READ_CHUNK, XB_CRYPT_READ_CHUNK,
XB_CRYPT_READ_INCOMPLETE,
XB_CRYPT_READ_EOF, XB_CRYPT_READ_EOF,
XB_CRYPT_READ_ERROR XB_CRYPT_READ_ERROR
} xb_rcrypt_result_t; } xb_rcrypt_result_t;
...@@ -75,10 +76,4 @@ xb_rcrypt_result_t xb_crypt_read_chunk(xb_rcrypt_t *crypt, void **buf, ...@@ -75,10 +76,4 @@ xb_rcrypt_result_t xb_crypt_read_chunk(xb_rcrypt_t *crypt, void **buf,
int xb_crypt_read_close(xb_rcrypt_t *crypt); int xb_crypt_read_close(xb_rcrypt_t *crypt);
/******************************************************************************
Utility interface */
my_bool xb_crypt_read_key_file(const char *filename,
void** key, uint *keylength);
void xb_crypt_create_iv(void* ivbuf, size_t ivlen);
#endif #endif
extra/mariabackup/xbcrypt_common.c
View file @ ecb25df2
/****************************************************** /******************************************************
Copyright (c) 2013 Percona LLC and/or its affiliates. Copyright (c) 2013, 2017 Percona LLC and/or its affiliates.
Encryption configuration file interface for XtraBackup. Encryption configuration file interface for XtraBackup.
...@@ -21,19 +21,27 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA ...@@ -21,19 +21,27 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA
#include <my_base.h> #include <my_base.h>
#include "common.h" #include "common.h"
#include "xbcrypt.h" #include "xbcrypt.h"
#include "xbcrypt_common.h"
#if GCC_VERSION >= 4002 /* Encryption options */
/* Workaround to avoid "gcry_ac_* is deprecated" warnings in gcrypt.h */ char *ds_encrypt_key = NULL;
# pragma GCC diagnostic ignored "-Wdeprecated-declarations" char *ds_encrypt_key_file = NULL;
#endif ulong ds_encrypt_algo;
static uint encrypt_key_len;
static uint encrypt_iv_len;
static const uint encrypt_mode = GCRY_CIPHER_MODE_CTR;
#ifdef HAVE_GRYPT static uint encrypt_algos[] = { GCRY_CIPHER_NONE, GCRY_CIPHER_AES128,
#include <gcrypt.h> GCRY_CIPHER_AES192, GCRY_CIPHER_AES256 };
static uint encrypt_algo;
#if GCC_VERSION >= 4002 #if !defined(GCRYPT_VERSION_NUMBER) || (GCRYPT_VERSION_NUMBER < 0x010600)
# pragma GCC diagnostic warning "-Wdeprecated-declarations" GCRY_THREAD_OPTION_PTHREAD_IMPL;
#endif #endif
my_bool my_bool
xb_crypt_read_key_file(const char *filename, void** key, uint *keylength) xb_crypt_read_key_file(const char *filename, void** key, uint *keylength)
{ {
...@@ -59,4 +67,262 @@ xb_crypt_create_iv(void* ivbuf, size_t ivlen) ...@@ -59,4 +67,262 @@ xb_crypt_create_iv(void* ivbuf, size_t ivlen)
{ {
gcry_create_nonce(ivbuf, ivlen); gcry_create_nonce(ivbuf, ivlen);
} }
gcry_error_t
xb_crypt_init(uint *iv_len)
{
gcry_error_t gcry_error;
/* Acording to gcrypt docs (and my testing), setting up the threading
callbacks must be done first, so, lets give it a shot */
#if !defined(GCRYPT_VERSION_NUMBER) || (GCRYPT_VERSION_NUMBER < 0x010600)
gcry_error = gcry_control(GCRYCTL_SET_THREAD_CBS, &gcry_threads_pthread);
if (gcry_error) {
msg("encryption: unable to set libgcrypt thread cbs - "
"%s : %s\n",
gcry_strsource(gcry_error),
gcry_strerror(gcry_error));
return gcry_error;
}
#endif
/* Version check should be the very next call because it
makes sure that important subsystems are intialized. */
if (!gcry_control(GCRYCTL_ANY_INITIALIZATION_P)) {
const char *gcrypt_version;
gcrypt_version = gcry_check_version(NULL);
/* No other library has already initialized libgcrypt. */
if (!gcrypt_version) {
msg("encryption: failed to initialize libgcrypt\n");
return 1;
} else {
msg("encryption: using gcrypt %s\n", gcrypt_version);
}
}
/* Disable the gcry secure memory, not dealing with this for now */
gcry_error = gcry_control(GCRYCTL_DISABLE_SECMEM, 0);
if (gcry_error) {
msg("encryption: unable to disable libgcrypt secmem - "
"%s : %s\n",
gcry_strsource(gcry_error),
gcry_strerror(gcry_error));
return gcry_error;
}
/* Finalize gcry initialization. */
gcry_error = gcry_control(GCRYCTL_INITIALIZATION_FINISHED, 0);
if (gcry_error) {
msg("encryption: unable to finish libgcrypt initialization - "
"%s : %s\n",
gcry_strsource(gcry_error),
gcry_strerror(gcry_error));
return gcry_error;
}
/* Determine the algorithm */
encrypt_algo = encrypt_algos[ds_encrypt_algo];
/* Set up the iv length */
encrypt_iv_len = gcry_cipher_get_algo_blklen(encrypt_algo);
xb_a(encrypt_iv_len > 0);
if (iv_len != NULL) {
*iv_len = encrypt_iv_len;
}
/* Now set up the key */
if (ds_encrypt_key == NULL &&
ds_encrypt_key_file == NULL) {
msg("encryption: no encryption key or key file specified.\n");
return gcry_error;
} else if (ds_encrypt_key && ds_encrypt_key_file) {
msg("encryption: both encryption key and key file specified.\n");
return gcry_error;
} else if (ds_encrypt_key_file) {
if (!xb_crypt_read_key_file(ds_encrypt_key_file,
(void**)&ds_encrypt_key,
&encrypt_key_len)) {
msg("encryption: unable to read encryption key file"
" \"%s\".\n", ds_encrypt_key_file);
return gcry_error;
}
} else if (ds_encrypt_key) {
encrypt_key_len = strlen(ds_encrypt_key);
} else {
msg("encryption: no encryption key or key file specified.\n");
return gcry_error;
}
return 0;
}
gcry_error_t
xb_crypt_cipher_open(gcry_cipher_hd_t *cipher_handle)
{
if (encrypt_algo != GCRY_CIPHER_NONE) {
gcry_error_t gcry_error;
gcry_error = gcry_cipher_open(cipher_handle,
encrypt_algo,
encrypt_mode, 0);
if (gcry_error) {
msg("encryption: unable to open libgcrypt"
" cipher - %s : %s\n",
gcry_strsource(gcry_error),
gcry_strerror(gcry_error));
gcry_cipher_close(*cipher_handle);
return gcry_error;
}
gcry_error = gcry_cipher_setkey(*cipher_handle,
ds_encrypt_key,
encrypt_key_len);
if (gcry_error) {
msg("encryption: unable to set libgcrypt"
" cipher key - %s : %s\n",
gcry_strsource(gcry_error),
gcry_strerror(gcry_error));
gcry_cipher_close(*cipher_handle);
return gcry_error;
}
return gcry_error;
}
return 0;
}
void
xb_crypt_cipher_close(gcry_cipher_hd_t cipher_handle)
{
if (encrypt_algo != GCRY_CIPHER_NONE)
gcry_cipher_close(cipher_handle);
}
gcry_error_t
xb_crypt_decrypt(gcry_cipher_hd_t cipher_handle, const uchar *from,
size_t from_len, uchar *to, size_t *to_len,
const uchar *iv, size_t iv_len, my_bool hash_appended)
{
*to_len = from_len;
if (encrypt_algo != GCRY_CIPHER_NONE) {
gcry_error_t gcry_error;
gcry_error = gcry_cipher_reset(cipher_handle);
if (gcry_error) {
msg("%s:encryption: unable to reset libgcrypt"
" cipher - %s : %s\n", my_progname,
gcry_strsource(gcry_error),
gcry_strerror(gcry_error));
return gcry_error;
}
if (iv_len > 0) {
gcry_error = gcry_cipher_setctr(cipher_handle,
iv, iv_len);
}
if (gcry_error) {
msg("%s:encryption: unable to set cipher iv - "
"%s : %s\n", my_progname,
gcry_strsource(gcry_error),
gcry_strerror(gcry_error));
return gcry_error;
}
/* Try to decrypt it */
gcry_error = gcry_cipher_decrypt(cipher_handle, to, *to_len,
from, from_len);
if (gcry_error) {
msg("%s:encryption: unable to decrypt chunk - "
"%s : %s\n", my_progname,
gcry_strsource(gcry_error),
gcry_strerror(gcry_error));
gcry_cipher_close(cipher_handle);
return gcry_error;
}
if (hash_appended) {
uchar hash[XB_CRYPT_HASH_LEN];
*to_len -= XB_CRYPT_HASH_LEN;
/* ensure that XB_CRYPT_HASH_LEN is the correct length
of XB_CRYPT_HASH hashing algorithm output */
xb_ad(gcry_md_get_algo_dlen(XB_CRYPT_HASH) ==
XB_CRYPT_HASH_LEN);
gcry_md_hash_buffer(XB_CRYPT_HASH, hash, to,
*to_len);
if (memcmp(hash, (char *) to + *to_len,
XB_CRYPT_HASH_LEN) != 0) {
msg("%s:%s invalid plaintext hash. "
"Wrong encrytion key specified?\n",
my_progname, __FUNCTION__);
return 1;
}
}
} else {
memcpy(to, from, *to_len);
}
return 0;
}
gcry_error_t
xb_crypt_encrypt(gcry_cipher_hd_t cipher_handle, const uchar *from,
size_t from_len, uchar *to, size_t *to_len, uchar *iv)
{
gcry_error_t gcry_error;
/* ensure that XB_CRYPT_HASH_LEN is the correct length
of XB_CRYPT_HASH hashing algorithm output */
xb_ad(gcry_md_get_algo_dlen(XB_CRYPT_HASH) ==
XB_CRYPT_HASH_LEN);
memcpy(to, from, from_len);
gcry_md_hash_buffer(XB_CRYPT_HASH, to + from_len,
from, from_len);
*to_len = from_len;
if (encrypt_algo != GCRY_CIPHER_NONE) {
gcry_error = gcry_cipher_reset(cipher_handle);
if (gcry_error) {
msg("encrypt: unable to reset cipher - "
"%s : %s\n",
gcry_strsource(gcry_error),
gcry_strerror(gcry_error));
return gcry_error;
}
xb_crypt_create_iv(iv, encrypt_iv_len);
gcry_error = gcry_cipher_setctr(cipher_handle, iv,
encrypt_iv_len);
if (gcry_error) {
msg("encrypt: unable to set cipher ctr - "
"%s : %s\n",
gcry_strsource(gcry_error),
gcry_strerror(gcry_error));
return gcry_error;
}
gcry_error = gcry_cipher_encrypt(cipher_handle, to,
*to_len + XB_CRYPT_HASH_LEN,
to,
from_len + XB_CRYPT_HASH_LEN);
if (gcry_error) {
msg("encrypt: unable to encrypt buffer - "
"%s : %s\n", gcry_strsource(gcry_error),
gcry_strerror(gcry_error));
return gcry_error;
}
} else {
memcpy(to, from, from_len + XB_CRYPT_HASH_LEN);
}
*to_len += XB_CRYPT_HASH_LEN;
return 0;
}
#endif #endif
\ No newline at end of file
extra/mariabackup/xbcrypt_common.h 0 → 100644
View file @ ecb25df2
/******************************************************
Copyright (c) 2017 Percona LLC and/or its affiliates.
Encryption datasink implementation for XtraBackup.
This program is free software; you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation; version 2 of the License.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program; if not, write to the Free Software
Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA
*******************************************************/
#include <my_base.h>
#if HAVE_GCRYPT
#if GCC_VERSION >= 4002
/* Workaround to avoid "gcry_ac_* is deprecated" warnings in gcrypt.h */
# pragma GCC diagnostic ignored "-Wdeprecated-declarations"
#endif
#include <gcrypt.h>
extern char *ds_encrypt_key;
extern char *ds_encrypt_key_file;
extern int ds_encrypt_threads;
extern ulong ds_encrypt_algo;
/******************************************************************************
Utility interface */
my_bool xb_crypt_read_key_file(const char *filename,
void** key, uint *keylength);
void xb_crypt_create_iv(void* ivbuf, size_t ivlen);
/* Initialize gcrypt and setup encryption key and IV lengths */
gcry_error_t
xb_crypt_init(uint *iv_len);
/* Setup gcrypt cipher */
gcry_error_t
xb_crypt_cipher_open(gcry_cipher_hd_t *cipher_handle);
/* Close gcrypt cipher */
void
xb_crypt_cipher_close(gcry_cipher_hd_t cipher_handle);
/* Decrypt buffer */
gcry_error_t
xb_crypt_decrypt(gcry_cipher_hd_t cipher_handle, const uchar *from,
size_t from_len, uchar *to, size_t *to_len, const uchar *iv,
size_t iv_len, my_bool hash_appended);
/* Encrypt buffer */
gcry_error_t
xb_crypt_encrypt(gcry_cipher_hd_t cipher_handle, const uchar *from,
size_t from_len, uchar *to, size_t *to_len, uchar *iv);
#endif
extra/mariabackup/xbcrypt_read.c
View file @ ecb25df2
...@@ -19,6 +19,7 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA ...@@ -19,6 +19,7 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA
*******************************************************/ *******************************************************/
#include "xbcrypt.h" #include "xbcrypt.h"
#include "crc_glue.h"
struct xb_rcrypt_struct { struct xb_rcrypt_struct {
void *userdata; void *userdata;
...@@ -212,7 +213,7 @@ xb_crypt_read_chunk(xb_rcrypt_t *crypt, void **buf, size_t *olen, size_t *elen, ...@@ -212,7 +213,7 @@ xb_crypt_read_chunk(xb_rcrypt_t *crypt, void **buf, size_t *olen, size_t *elen,
} }
} }
checksum = crc32(0, crypt->buffer, *elen); checksum = crc32_iso3309(0, crypt->buffer, *elen);
if (checksum != checksum_exp) { if (checksum != checksum_exp) {
msg("%s:%s invalid checksum at offset 0x%llx, " msg("%s:%s invalid checksum at offset 0x%llx, "
"expected 0x%lx, actual 0x%lx.\n", my_progname, __FUNCTION__, "expected 0x%lx, actual 0x%lx.\n", my_progname, __FUNCTION__,
......
extra/mariabackup/xbcrypt_write.c
View file @ ecb25df2
...@@ -19,6 +19,7 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA ...@@ -19,6 +19,7 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA
*******************************************************/ *******************************************************/
#include "xbcrypt.h" #include "xbcrypt.h"
#include "crc_glue.h"
struct xb_wcrypt_struct { struct xb_wcrypt_struct {
void *userdata; void *userdata;
...@@ -73,7 +74,7 @@ int xb_crypt_write_chunk(xb_wcrypt_t *crypt, const void *buf, size_t olen, ...@@ -73,7 +74,7 @@ int xb_crypt_write_chunk(xb_wcrypt_t *crypt, const void *buf, size_t olen,
int8store(ptr, (ulonglong)elen); /* encrypted (actual) size */ int8store(ptr, (ulonglong)elen); /* encrypted (actual) size */
ptr += 8; ptr += 8;
checksum = crc32(0, buf, (uint)elen); checksum = crc32_iso3309(0, buf, elen);
int4store(ptr, checksum); /* checksum */ int4store(ptr, checksum); /* checksum */
ptr += 4; ptr += 4;
......
extra/mariabackup/xbstream.c
View file @ ecb25df2
...@@ -22,10 +22,13 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA ...@@ -22,10 +22,13 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA
#include <my_base.h> #include <my_base.h>
#include <my_getopt.h> #include <my_getopt.h>
#include <hash.h> #include <hash.h>
#include <my_pthread.h>
#include "common.h" #include "common.h"
#include "xbstream.h" #include "xbstream.h"
#include "ds_local.h" #include "xbcrypt_common.h"
#include "ds_stdout.h" #include "datasink.h"
#include "ds_decrypt.h"
#include "crc_glue.h"
#define XBSTREAM_VERSION "1.0" #define XBSTREAM_VERSION "1.0"
#define XBSTREAM_BUFFER_SIZE (10 * 1024 * 1024UL) #define XBSTREAM_BUFFER_SIZE (10 * 1024 * 1024UL)
...@@ -38,6 +41,12 @@ typedef enum { ...@@ -38,6 +41,12 @@ typedef enum {
RUN_MODE_EXTRACT RUN_MODE_EXTRACT
} run_mode_t; } run_mode_t;
const char *xbstream_encrypt_algo_names[] =
{ "NONE", "AES128", "AES192", "AES256", NullS};
TYPELIB xbstream_encrypt_algo_typelib=
{array_elements(xbstream_encrypt_algo_names)-1,"",
xbstream_encrypt_algo_names, NULL};
/* Need the following definitions to avoid linking with ds_*.o and their link /* Need the following definitions to avoid linking with ds_*.o and their link
dependencies */ dependencies */
datasink_t datasink_archive; datasink_t datasink_archive;
...@@ -50,6 +59,15 @@ datasink_t datasink_buffer; ...@@ -50,6 +59,15 @@ datasink_t datasink_buffer;
static run_mode_t opt_mode; static run_mode_t opt_mode;
static char * opt_directory = NULL; static char * opt_directory = NULL;
static my_bool opt_verbose = 0; static my_bool opt_verbose = 0;
static int opt_parallel = 1;
static ulong opt_encrypt_algo;
static char *opt_encrypt_key_file = NULL;
static void *opt_encrypt_key = NULL;
static int opt_encrypt_threads = 1;
enum {
OPT_ENCRYPT_THREADS = 256
};
static struct my_option my_long_options[] = static struct my_option my_long_options[] =
{ {
...@@ -65,21 +83,46 @@ static struct my_option my_long_options[] = ...@@ -65,21 +83,46 @@ static struct my_option my_long_options[] =
GET_STR_ALLOC, REQUIRED_ARG, 0, 0, 0, 0, 0, 0}, GET_STR_ALLOC, REQUIRED_ARG, 0, 0, 0, 0, 0, 0},
{"verbose", 'v', "Print verbose output.", &opt_verbose, &opt_verbose, {"verbose", 'v', "Print verbose output.", &opt_verbose, &opt_verbose,
0, GET_BOOL, NO_ARG, 0, 0, 0, 0, 0, 0}, 0, GET_BOOL, NO_ARG, 0, 0, 0, 0, 0, 0},
{"parallel", 'p', "Number of worker threads for reading / writing.",
&opt_parallel, &opt_parallel, 0, GET_INT, REQUIRED_ARG,
1, 1, INT_MAX, 0, 0, 0},
{"decrypt", 'd', "Decrypt files ending with .xbcrypt.",
&opt_encrypt_algo, &opt_encrypt_algo, &xbstream_encrypt_algo_typelib,
GET_ENUM, REQUIRED_ARG, 0, 0, 0, 0, 0, 0},
{"encrypt-key", 'k', "Encryption key.",
&opt_encrypt_key, &opt_encrypt_key, 0,
GET_STR_ALLOC, REQUIRED_ARG, 0, 0, 0, 0, 0, 0},
{"encrypt-key-file", 'f', "File which contains encryption key.",
&opt_encrypt_key_file, &opt_encrypt_key_file, 0,
GET_STR_ALLOC, REQUIRED_ARG, 0, 0, 0, 0, 0, 0},
{"encrypt-threads", OPT_ENCRYPT_THREADS,
"Number of threads for parallel data encryption. "
"The default value is 1.",
&opt_encrypt_threads, &opt_encrypt_threads,
0, GET_INT, REQUIRED_ARG, 1, 1, INT_MAX, 0, 0, 0},
{0, 0, 0, 0, 0, 0, GET_NO_ARG, NO_ARG, 0, 0, 0, 0, 0, 0} {0, 0, 0, 0, 0, 0, GET_NO_ARG, NO_ARG, 0, 0, 0, 0, 0, 0}
}; };
typedef struct {
HASH *filehash;
xb_rstream_t *stream;
ds_ctxt_t *ds_ctxt;
ds_ctxt_t *ds_decrypt_ctxt;
pthread_mutex_t *mutex;
} extract_ctxt_t;
typedef struct { typedef struct {
char *path; char *path;
uint pathlen; uint pathlen;
my_off_t offset; my_off_t offset;
ds_ctxt_t *ds_ctxt;
ds_file_t *file; ds_file_t *file;
pthread_mutex_t mutex;
} file_entry_t; } file_entry_t;
static int get_options(int *argc, char ***argv); static int get_options(int *argc, char ***argv);
static int mode_create(int argc, char **argv); static int mode_create(int argc, char **argv);
static int mode_extract(int argc, char **argv); static int mode_extract(int n_threads, int argc, char **argv);
static my_bool get_one_option(int optid, const struct my_option *opt, static my_bool get_one_option(int optid, const struct my_option *opt,
char *argument); char *argument);
...@@ -88,6 +131,8 @@ main(int argc, char **argv) ...@@ -88,6 +131,8 @@ main(int argc, char **argv)
{ {
MY_INIT(argv[0]); MY_INIT(argv[0]);
crc_init();
if (get_options(&argc, &argv)) { if (get_options(&argc, &argv)) {
goto err; goto err;
} }
...@@ -104,7 +149,8 @@ main(int argc, char **argv) ...@@ -104,7 +149,8 @@ main(int argc, char **argv)
if (opt_mode == RUN_MODE_CREATE && mode_create(argc, argv)) { if (opt_mode == RUN_MODE_CREATE && mode_create(argc, argv)) {
goto err; goto err;
} else if (opt_mode == RUN_MODE_EXTRACT && mode_extract(argc, argv)) { } else if (opt_mode == RUN_MODE_EXTRACT &&
mode_extract(opt_parallel, argc, argv)) {
goto err; goto err;
} }
...@@ -302,9 +348,22 @@ mode_create(int argc, char **argv) ...@@ -302,9 +348,22 @@ mode_create(int argc, char **argv)
return 1; return 1;
} }
/************************************************************************
Check if string ends with given suffix.
@return true if string ends with given suffix. */
static
my_bool
ends_with(const char *str, const char *suffix)
{
size_t suffix_len = strlen(suffix);
size_t str_len = strlen(str);
return(str_len >= suffix_len
&& strcmp(str + str_len - suffix_len, suffix) == 0);
}
static static
file_entry_t * file_entry_t *
file_entry_new(ds_ctxt_t *ds_ctxt, const char *path, uint pathlen) file_entry_new(extract_ctxt_t *ctxt, const char *path, uint pathlen)
{ {
file_entry_t *entry; file_entry_t *entry;
ds_file_t *file; ds_file_t *file;
...@@ -321,7 +380,11 @@ file_entry_new(ds_ctxt_t *ds_ctxt, const char *path, uint pathlen) ...@@ -321,7 +380,11 @@ file_entry_new(ds_ctxt_t *ds_ctxt, const char *path, uint pathlen)
} }
entry->pathlen = pathlen; entry->pathlen = pathlen;
file = ds_open(ds_ctxt, path, NULL); if (ctxt->ds_decrypt_ctxt && ends_with(path, ".xbcrypt")) {
file = ds_open(ctxt->ds_decrypt_ctxt, path, NULL);
} else {
file = ds_open(ctxt->ds_ctxt, path, NULL);
}
if (file == NULL) { if (file == NULL) {
msg("%s: failed to create file.\n", my_progname); msg("%s: failed to create file.\n", my_progname);
goto err; goto err;
...@@ -332,7 +395,8 @@ file_entry_new(ds_ctxt_t *ds_ctxt, const char *path, uint pathlen) ...@@ -332,7 +395,8 @@ file_entry_new(ds_ctxt_t *ds_ctxt, const char *path, uint pathlen)
} }
entry->file = file; entry->file = file;
entry->ds_ctxt = ds_ctxt;
pthread_mutex_init(&entry->mutex, NULL);
return entry; return entry;
...@@ -358,68 +422,77 @@ static ...@@ -358,68 +422,77 @@ static
void void
file_entry_free(file_entry_t *entry) file_entry_free(file_entry_t *entry)
{ {
pthread_mutex_destroy(&entry->mutex);
ds_close(entry->file); ds_close(entry->file);
my_free(entry->path); my_free(entry->path);
my_free(entry); my_free(entry);
} }
static static
int void *
mode_extract(int argc __attribute__((unused)), extract_worker_thread_func(void *arg)
char **argv __attribute__((unused)))
{ {
xb_rstream_t *stream;
xb_rstream_result_t res;
xb_rstream_chunk_t chunk; xb_rstream_chunk_t chunk;
HASH filehash;
file_entry_t *entry; file_entry_t *entry;
ds_ctxt_t *ds_ctxt; xb_rstream_result_t res;
stream = xb_stream_read_new(); extract_ctxt_t *ctxt = (extract_ctxt_t *) arg;
if (stream == NULL) {
msg("%s: xb_stream_read_new() failed.\n", my_progname);
return 1;
}
/* If --directory is specified, it is already set as CWD by now. */ my_thread_init();
ds_ctxt = ds_create(".", DS_TYPE_LOCAL);
if (my_hash_init(&filehash, &my_charset_bin, START_FILE_HASH_SIZE, memset(&chunk, 0, sizeof(chunk));
0, 0, (my_hash_get_key) get_file_entry_key,
(my_hash_free_key) file_entry_free, MYF(0))) { while (1) {
msg("%s: failed to initialize file hash.\n", my_progname);
goto err; pthread_mutex_lock(ctxt->mutex);
res = xb_stream_read_chunk(ctxt->stream, &chunk);
if (res != XB_STREAM_READ_CHUNK) {
pthread_mutex_unlock(ctxt->mutex);
break;
} }
while ((res = xb_stream_read_chunk(stream, &chunk)) ==
XB_STREAM_READ_CHUNK) {
/* If unknown type and ignorable flag is set, skip this chunk */ /* If unknown type and ignorable flag is set, skip this chunk */
if (chunk.type == XB_CHUNK_TYPE_UNKNOWN && \ if (chunk.type == XB_CHUNK_TYPE_UNKNOWN && \
!(chunk.flags & XB_STREAM_FLAG_IGNORABLE)) { !(chunk.flags & XB_STREAM_FLAG_IGNORABLE)) {
pthread_mutex_unlock(ctxt->mutex);
continue; continue;
} }
/* See if we already have this file open */ /* See if we already have this file open */
entry = (file_entry_t *) my_hash_search(&filehash, entry = (file_entry_t *) my_hash_search(ctxt->filehash,
(uchar *) chunk.path, (uchar *) chunk.path,
chunk.pathlen); chunk.pathlen);
if (entry == NULL) { if (entry == NULL) {
entry = file_entry_new(ds_ctxt, chunk.path, entry = file_entry_new(ctxt,
chunk.path,
chunk.pathlen); chunk.pathlen);
if (entry == NULL) { if (entry == NULL) {
goto err; pthread_mutex_unlock(ctxt->mutex);
break;
} }
if (my_hash_insert(&filehash, (uchar *) entry)) { if (my_hash_insert(ctxt->filehash, (uchar *) entry)) {
msg("%s: my_hash_insert() failed.\n", msg("%s: my_hash_insert() failed.\n",
my_progname); my_progname);
goto err; pthread_mutex_unlock(ctxt->mutex);
break;
} }
} }
if (chunk.type == XB_CHUNK_TYPE_EOF) { pthread_mutex_lock(&entry->mutex);
my_hash_delete(&filehash, (uchar *) entry);
pthread_mutex_unlock(ctxt->mutex);
res = xb_stream_validate_checksum(&chunk);
if (res != XB_STREAM_READ_CHUNK) {
pthread_mutex_unlock(&entry->mutex);
break;
}
if (chunk.type == XB_CHUNK_TYPE_EOF) {
pthread_mutex_unlock(&entry->mutex);
continue; continue;
} }
...@@ -427,30 +500,114 @@ mode_extract(int argc __attribute__((unused)), ...@@ -427,30 +500,114 @@ mode_extract(int argc __attribute__((unused)),
msg("%s: out-of-order chunk: real offset = 0x%llx, " msg("%s: out-of-order chunk: real offset = 0x%llx, "
"expected offset = 0x%llx\n", my_progname, "expected offset = 0x%llx\n", my_progname,
chunk.offset, entry->offset); chunk.offset, entry->offset);
goto err; pthread_mutex_unlock(&entry->mutex);
res = XB_STREAM_READ_ERROR;
break;
} }
if (ds_write(entry->file, chunk.data, chunk.length)) { if (ds_write(entry->file, chunk.data, chunk.length)) {
msg("%s: my_write() failed.\n", my_progname); msg("%s: my_write() failed.\n", my_progname);
goto err; pthread_mutex_unlock(&entry->mutex);
res = XB_STREAM_READ_ERROR;
break;
} }
entry->offset += chunk.length; entry->offset += chunk.length;
};
if (res == XB_STREAM_READ_ERROR) { pthread_mutex_unlock(&entry->mutex);
goto err; }
if (chunk.data)
my_free(chunk.data);
my_thread_end();
return (void *)(res);
}
static
int
mode_extract(int n_threads, int argc __attribute__((unused)),
char **argv __attribute__((unused)))
{
xb_rstream_t *stream = NULL;
HASH filehash;
ds_ctxt_t *ds_ctxt = NULL;
ds_ctxt_t *ds_decrypt_ctxt = NULL;
extract_ctxt_t ctxt;
int i;
pthread_t *tids = NULL;
void **retvals = NULL;
pthread_mutex_t mutex;
int ret = 0;
if (my_hash_init(&filehash, &my_charset_bin, START_FILE_HASH_SIZE,
0, 0, (my_hash_get_key) get_file_entry_key,
(my_hash_free_key) file_entry_free, MYF(0))) {
msg("%s: failed to initialize file hash.\n", my_progname);
return 1;
} }
if (pthread_mutex_init(&mutex, NULL)) {
msg("%s: failed to initialize mutex.\n", my_progname);
my_hash_free(&filehash); my_hash_free(&filehash);
ds_destroy(ds_ctxt); return 1;
xb_stream_read_done(stream); }
/* If --directory is specified, it is already set as CWD by now. */
ds_ctxt = ds_create(".", DS_TYPE_LOCAL);
if (ds_ctxt == NULL) {
ret = 1;
goto exit;
}
stream = xb_stream_read_new();
if (stream == NULL) {
msg("%s: xb_stream_read_new() failed.\n", my_progname);
pthread_mutex_destroy(&mutex);
ret = 1;
goto exit;
}
ctxt.stream = stream;
ctxt.filehash = &filehash;
ctxt.ds_ctxt = ds_ctxt;
ctxt.ds_decrypt_ctxt = ds_decrypt_ctxt;
ctxt.mutex = &mutex;
tids = malloc(sizeof(pthread_t) * n_threads);
retvals = malloc(sizeof(void*) * n_threads);
for (i = 0; i < n_threads; i++)
pthread_create(tids + i, NULL, extract_worker_thread_func,
&ctxt);
for (i = 0; i < n_threads; i++)
pthread_join(tids[i], retvals + i);
for (i = 0; i < n_threads; i++) {
if ((ulong)retvals[i] == XB_STREAM_READ_ERROR) {
ret = 1;
goto exit;
}
}
exit:
pthread_mutex_destroy(&mutex);
free(tids);
free(retvals);
return 0;
err:
my_hash_free(&filehash); my_hash_free(&filehash);
if (ds_ctxt != NULL) {
ds_destroy(ds_ctxt); ds_destroy(ds_ctxt);
}
if (ds_decrypt_ctxt) {
ds_destroy(ds_decrypt_ctxt);
}
xb_stream_read_done(stream); xb_stream_read_done(stream);
return 1; return ret;
} }
extra/mariabackup/xbstream.h
View file @ ecb25df2
/****************************************************** /******************************************************
Copyright (c) 2011-2013 Percona LLC and/or its affiliates. Copyright (c) 2011-2017 Percona LLC and/or its affiliates.
The xbstream format interface. The xbstream format interface.
...@@ -89,8 +89,10 @@ typedef struct { ...@@ -89,8 +89,10 @@ typedef struct {
char path[FN_REFLEN]; char path[FN_REFLEN];
size_t length; size_t length;
my_off_t offset; my_off_t offset;
my_off_t checksum_offset;
void *data; void *data;
ulong checksum; ulong checksum;
size_t buflen;
} xb_rstream_chunk_t; } xb_rstream_chunk_t;
xb_rstream_t *xb_stream_read_new(void); xb_rstream_t *xb_stream_read_new(void);
...@@ -100,4 +102,6 @@ xb_rstream_result_t xb_stream_read_chunk(xb_rstream_t *stream, ...@@ -100,4 +102,6 @@ xb_rstream_result_t xb_stream_read_chunk(xb_rstream_t *stream,
int xb_stream_read_done(xb_rstream_t *stream); int xb_stream_read_done(xb_rstream_t *stream);
int xb_stream_validate_checksum(xb_rstream_chunk_t *chunk);
#endif #endif
extra/mariabackup/xbstream_read.c
View file @ ecb25df2
/****************************************************** /******************************************************
Copyright (c) 2011-2013 Percona LLC and/or its affiliates. Copyright (c) 2011-2017 Percona LLC and/or its affiliates.
The xbstream format reader implementation. The xbstream format reader implementation.
...@@ -23,6 +23,7 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA ...@@ -23,6 +23,7 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA
#include <zlib.h> #include <zlib.h>
#include "common.h" #include "common.h"
#include "xbstream.h" #include "xbstream.h"
#include "crc_glue.h"
/* Allocate 1 MB for the payload buffer initially */ /* Allocate 1 MB for the payload buffer initially */
#define INIT_BUFFER_LEN (1024 * 1024) #define INIT_BUFFER_LEN (1024 * 1024)
...@@ -34,8 +35,6 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA ...@@ -34,8 +35,6 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA
struct xb_rstream_struct { struct xb_rstream_struct {
my_off_t offset; my_off_t offset;
File fd; File fd;
void *buffer;
size_t buflen;
}; };
xb_rstream_t * xb_rstream_t *
...@@ -45,9 +44,6 @@ xb_stream_read_new(void) ...@@ -45,9 +44,6 @@ xb_stream_read_new(void)
stream = (xb_rstream_t *) my_malloc(sizeof(xb_rstream_t), MYF(MY_FAE)); stream = (xb_rstream_t *) my_malloc(sizeof(xb_rstream_t), MYF(MY_FAE));
stream->buffer = my_malloc(INIT_BUFFER_LEN, MYF(MY_FAE));
stream->buflen = INIT_BUFFER_LEN;
#ifdef __WIN__ #ifdef __WIN__
setmode(fileno(stdin), _O_BINARY); setmode(fileno(stdin), _O_BINARY);
#endif #endif
...@@ -71,6 +67,23 @@ validate_chunk_type(uchar code) ...@@ -71,6 +67,23 @@ validate_chunk_type(uchar code)
} }
} }
int
xb_stream_validate_checksum(xb_rstream_chunk_t *chunk)
{
ulong checksum;
checksum = crc32_iso3309(0, chunk->data, (uint)chunk->length);
if (checksum != chunk->checksum) {
msg("xb_stream_read_chunk(): invalid checksum at offset "
"0x%llx: expected 0x%lx, read 0x%lx.\n",
(ulonglong) chunk->checksum_offset, chunk->checksum,
checksum);
return XB_STREAM_READ_ERROR;
}
return XB_STREAM_READ_CHUNK;
}
#define F_READ(buf,len) \ #define F_READ(buf,len) \
do { \ do { \
if (xb_read_full(fd, buf, len) < len) { \ if (xb_read_full(fd, buf, len) < len) { \
...@@ -87,8 +100,6 @@ xb_stream_read_chunk(xb_rstream_t *stream, xb_rstream_chunk_t *chunk) ...@@ -87,8 +100,6 @@ xb_stream_read_chunk(xb_rstream_t *stream, xb_rstream_chunk_t *chunk)
uint pathlen; uint pathlen;
size_t tbytes; size_t tbytes;
ulonglong ullval; ulonglong ullval;
ulong checksum_exp;
ulong checksum;
File fd = stream->fd; File fd = stream->fd;
xb_ad(sizeof(tmpbuf) >= CHUNK_HEADER_CONSTANT_LEN); xb_ad(sizeof(tmpbuf) >= CHUNK_HEADER_CONSTANT_LEN);
...@@ -178,39 +189,30 @@ xb_stream_read_chunk(xb_rstream_t *stream, xb_rstream_chunk_t *chunk) ...@@ -178,39 +189,30 @@ xb_stream_read_chunk(xb_rstream_t *stream, xb_rstream_chunk_t *chunk)
stream->offset += 8; stream->offset += 8;
/* Reallocate the buffer if needed */ /* Reallocate the buffer if needed */
if (chunk->length > stream->buflen) { if (chunk->length > chunk->buflen) {
stream->buffer = my_realloc(stream->buffer, chunk->length, chunk->data = my_realloc(chunk->data, chunk->length,
MYF(MY_WME)); MYF(MY_WME | MY_ALLOW_ZERO_PTR));
if (stream->buffer == NULL) { if (chunk->data == NULL) {
msg("xb_stream_read_chunk(): failed to increase buffer " msg("xb_stream_read_chunk(): failed to increase buffer "
"to %lu bytes.\n", (ulong) chunk->length); "to %lu bytes.\n", (ulong) chunk->length);
goto err; goto err;
} }
stream->buflen = chunk->length; chunk->buflen = chunk->length;
} }
/* Checksum */ /* Checksum */
F_READ(tmpbuf, 4); F_READ(tmpbuf, 4);
checksum_exp = uint4korr(tmpbuf); chunk->checksum = uint4korr(tmpbuf);
chunk->checksum_offset = stream->offset;
/* Payload */ /* Payload */
if (chunk->length > 0) { if (chunk->length > 0) {
F_READ(stream->buffer, chunk->length); F_READ(chunk->data, chunk->length);
stream->offset += chunk->length; stream->offset += chunk->length;
} }
checksum = crc32(0, stream->buffer, chunk->length);
if (checksum != checksum_exp) {
msg("xb_stream_read_chunk(): invalid checksum at offset "
"0x%llx: expected 0x%lx, read 0x%lx.\n",
(ulonglong) stream->offset, checksum_exp, checksum);
goto err;
}
stream->offset += 4; stream->offset += 4;
chunk->data = stream->buffer;
chunk->checksum = checksum;
return XB_STREAM_READ_CHUNK; return XB_STREAM_READ_CHUNK;
err: err:
...@@ -220,7 +222,6 @@ xb_stream_read_chunk(xb_rstream_t *stream, xb_rstream_chunk_t *chunk) ...@@ -220,7 +222,6 @@ xb_stream_read_chunk(xb_rstream_t *stream, xb_rstream_chunk_t *chunk)
int int
xb_stream_read_done(xb_rstream_t *stream) xb_stream_read_done(xb_rstream_t *stream)
{ {
my_free(stream->buffer);
my_free(stream); my_free(stream);
return 0; return 0;
......
extra/mariabackup/xbstream_write.c
View file @ ecb25df2
/****************************************************** /******************************************************
Copyright (c) 2011-2013 Percona LLC and/or its affiliates. Copyright (c) 2011-2017 Percona LLC and/or its affiliates.
The xbstream format writer implementation. The xbstream format writer implementation.
...@@ -23,6 +23,7 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA ...@@ -23,6 +23,7 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA
#include <zlib.h> #include <zlib.h>
#include "common.h" #include "common.h"
#include "xbstream.h" #include "xbstream.h"
#include "crc_glue.h"
/* Group writes smaller than this into a single chunk */ /* Group writes smaller than this into a single chunk */
#define XB_STREAM_MIN_CHUNK_SIZE (10 * 1024 * 1024) #define XB_STREAM_MIN_CHUNK_SIZE (10 * 1024 * 1024)
...@@ -215,12 +216,13 @@ xb_stream_write_chunk(xb_wstream_file_t *file, const void *buf, size_t len) ...@@ -215,12 +216,13 @@ xb_stream_write_chunk(xb_wstream_file_t *file, const void *buf, size_t len)
int8store(ptr, len); /* Payload length */ int8store(ptr, len); /* Payload length */
ptr += 8; ptr += 8;
checksum = crc32_iso3309(0, buf, (uint)len); /* checksum */
pthread_mutex_lock(&stream->mutex); pthread_mutex_lock(&stream->mutex);
int8store(ptr, file->offset); /* Payload offset */ int8store(ptr, file->offset); /* Payload offset */
ptr += 8; ptr += 8;
checksum = crc32(0, buf, (uint)len); /* checksum */
int4store(ptr, checksum); int4store(ptr, checksum);
ptr += 4; ptr += 4;
......
extra/mariabackup/xtrabackup.cc
View file @ ecb25df2
/****************************************************** /******************************************************
XtraBackup: hot backup tool for InnoDB XtraBackup: hot backup tool for InnoDB
(c) 2009-2015 Percona LLC and/or its affiliates (c) 2009-2017 Percona LLC and/or its affiliates
Originally Created 3/3/2009 Yasufumi Kinoshita Originally Created 3/3/2009 Yasufumi Kinoshita
Written by Alexey Kopytov, Aleksandr Kuzminsky, Stewart Smith, Vadim Tkachenko, Written by Alexey Kopytov, Aleksandr Kuzminsky, Stewart Smith, Vadim Tkachenko,
Yasufumi Kinoshita, Ignacio Nin and Baron Schwartz. Yasufumi Kinoshita, Ignacio Nin and Baron Schwartz.
...@@ -67,6 +67,7 @@ Place, Suite 330, Boston, MA 02111-1307 USA ...@@ -67,6 +67,7 @@ Place, Suite 330, Boston, MA 02111-1307 USA
#include <srv0start.h> #include <srv0start.h>
#include <buf0dblwr.h> #include <buf0dblwr.h>
#include <list>
#include <sstream> #include <sstream>
#include <set> #include <set>
#include <mysql.h> #include <mysql.h>
...@@ -94,6 +95,7 @@ Place, Suite 330, Boston, MA 02111-1307 USA ...@@ -94,6 +95,7 @@ Place, Suite 330, Boston, MA 02111-1307 USA
#include "encryption_plugin.h" #include "encryption_plugin.h"
#include <sql_plugin.h> #include <sql_plugin.h>
#include <srv0srv.h> #include <srv0srv.h>
#include <crc_glue.h>
/* TODO: replace with appropriate macros used in InnoDB 5.6 */ /* TODO: replace with appropriate macros used in InnoDB 5.6 */
#define PAGE_ZIP_MIN_SIZE_SHIFT 10 #define PAGE_ZIP_MIN_SIZE_SHIFT 10
...@@ -144,25 +146,24 @@ char xtrabackup_real_incremental_dir[FN_REFLEN]; ...@@ -144,25 +146,24 @@ char xtrabackup_real_incremental_dir[FN_REFLEN];
lsn_t xtrabackup_archived_to_lsn = 0; /* for --archived-to-lsn */ lsn_t xtrabackup_archived_to_lsn = 0; /* for --archived-to-lsn */
char *xtrabackup_tables = NULL;
char *xtrabackup_tmpdir; char *xtrabackup_tmpdir;
/* List of regular expressions for filtering */ char *xtrabackup_tables = NULL;
typedef struct xb_regex_list_node_struct xb_regex_list_node_t;
struct xb_regex_list_node_struct {
UT_LIST_NODE_T(xb_regex_list_node_t) regex_list;
regex_t regex;
};
static UT_LIST_BASE_NODE_T(xb_regex_list_node_t) regex_list;
static regmatch_t tables_regmatch[1];
char *xtrabackup_tables_file = NULL; char *xtrabackup_tables_file = NULL;
static hash_table_t* tables_hash = NULL; char *xtrabackup_tables_exclude = NULL;
typedef std::list<regex_t> regex_list_t;
static regex_list_t regex_include_list;
static regex_list_t regex_exclude_list;
static hash_table_t* tables_include_hash = NULL;
static hash_table_t* tables_exclude_hash = NULL;
char *xtrabackup_databases = NULL; char *xtrabackup_databases = NULL;
char *xtrabackup_databases_file = NULL; char *xtrabackup_databases_file = NULL;
static hash_table_t* databases_hash = NULL; char *xtrabackup_databases_exclude = NULL;
static hash_table_t* databases_include_hash = NULL;
static hash_table_t* databases_exclude_hash = NULL;
static hash_table_t* inc_dir_tables_hash; static hash_table_t* inc_dir_tables_hash;
...@@ -615,6 +616,8 @@ enum options_xtrabackup ...@@ -615,6 +616,8 @@ enum options_xtrabackup
OPT_SSL_VERIFY_SERVER_CERT, OPT_SSL_VERIFY_SERVER_CERT,
OPT_SERVER_PUBLIC_KEY, OPT_SERVER_PUBLIC_KEY,
OPT_XTRA_TABLES_EXCLUDE,
OPT_XTRA_DATABASES_EXCLUDE,
}; };
struct my_option xb_client_options[] = struct my_option xb_client_options[] =
...@@ -685,6 +688,16 @@ struct my_option xb_client_options[] = ...@@ -685,6 +688,16 @@ struct my_option xb_client_options[] =
"filtering by list of databases in the file.", "filtering by list of databases in the file.",
(G_PTR*) &xtrabackup_databases_file, (G_PTR*) &xtrabackup_databases_file, (G_PTR*) &xtrabackup_databases_file, (G_PTR*) &xtrabackup_databases_file,
0, GET_STR, REQUIRED_ARG, 0, 0, 0, 0, 0, 0}, 0, GET_STR, REQUIRED_ARG, 0, 0, 0, 0, 0, 0},
{"tables-exclude", OPT_XTRA_TABLES_EXCLUDE, "filtering by regexp for table names. "
"Operates the same way as --tables, but matched names are excluded from backup. "
"Note that this option has a higher priority than --tables.",
(G_PTR*) &xtrabackup_tables_exclude, (G_PTR*) &xtrabackup_tables_exclude,
0, GET_STR, REQUIRED_ARG, 0, 0, 0, 0, 0, 0},
{"databases-exclude", OPT_XTRA_DATABASES_EXCLUDE, "Excluding databases based on name, "
"Operates the same way as --databases, but matched names are excluded from backup. "
"Note that this option has a higher priority than --databases.",
(G_PTR*) &xtrabackup_databases_exclude, (G_PTR*) &xtrabackup_databases_exclude,
0, GET_STR, REQUIRED_ARG, 0, 0, 0, 0, 0, 0},
{"create-ib-logfile", OPT_XTRA_CREATE_IB_LOGFILE, "** not work for now** creates ib_logfile* also after '--prepare'. ### If you want create ib_logfile*, only re-execute this command in same options. ###", {"create-ib-logfile", OPT_XTRA_CREATE_IB_LOGFILE, "** not work for now** creates ib_logfile* also after '--prepare'. ### If you want create ib_logfile*, only re-execute this command in same options. ###",
(G_PTR*) &xtrabackup_create_ib_logfile, (G_PTR*) &xtrabackup_create_ib_logfile, (G_PTR*) &xtrabackup_create_ib_logfile, (G_PTR*) &xtrabackup_create_ib_logfile,
0, GET_BOOL, NO_ARG, 0, 0, 0, 0, 0, 0}, 0, GET_BOOL, NO_ARG, 0, 0, 0, 0, 0, 0},
...@@ -1036,8 +1049,8 @@ struct my_option xb_server_options[] = ...@@ -1036,8 +1049,8 @@ struct my_option xb_server_options[] =
(G_PTR*) &opt_mysql_tmpdir, (G_PTR*) &opt_mysql_tmpdir,
(G_PTR*) &opt_mysql_tmpdir, 0, GET_STR, REQUIRED_ARG, 0, 0, 0, 0, 0, 0}, (G_PTR*) &opt_mysql_tmpdir, 0, GET_STR, REQUIRED_ARG, 0, 0, 0, 0, 0, 0},
{"parallel", OPT_XTRA_PARALLEL, {"parallel", OPT_XTRA_PARALLEL,
"Number of threads to use for parallel datafiles transfer. Does not have " "Number of threads to use for parallel datafiles transfer. "
"any effect in the stream mode. The default value is 1.", "The default value is 1.",
(G_PTR*) &xtrabackup_parallel, (G_PTR*) &xtrabackup_parallel, 0, GET_INT, (G_PTR*) &xtrabackup_parallel, (G_PTR*) &xtrabackup_parallel, 0, GET_INT,
REQUIRED_ARG, 1, 1, INT_MAX, 0, 0, 0}, REQUIRED_ARG, 1, 1, INT_MAX, 0, 0, 0},
...@@ -2162,43 +2175,139 @@ xtrabackup_io_throttling(void) ...@@ -2162,43 +2175,139 @@ xtrabackup_io_throttling(void)
} }
} }
static
my_bool regex_list_check_match(
const regex_list_t& list,
const char* name)
{
regmatch_t tables_regmatch[1];
for (regex_list_t::const_iterator i = list.begin(), end = list.end();
i != end; ++i) {
const regex_t& regex = *i;
int regres = regexec(&regex, name, 1, tables_regmatch, 0);
if (regres != REG_NOMATCH) {
return(TRUE);
}
}
return(FALSE);
}
static
my_bool
find_filter_in_hashtable(
const char* name,
hash_table_t* table,
xb_filter_entry_t** result
)
{
xb_filter_entry_t* found = NULL;
HASH_SEARCH(name_hash, table, ut_fold_string(name),
xb_filter_entry_t*,
found, (void) 0,
!strcmp(found->name, name));
if (found && result) {
*result = found;
}
return (found != NULL);
}
/************************************************************************ /************************************************************************
Checks if a given table name matches any of specifications in the --tables or Checks if a given table name matches any of specifications given in
--tables-file options. regex_list or tables_hash.
@return TRUE on match. */ @return TRUE on match or both regex_list and tables_hash are empty.*/
static my_bool static my_bool
check_if_table_matches_filters(const char *name) check_if_table_matches_filters(const char *name,
const regex_list_t& regex_list,
hash_table_t* tables_hash)
{ {
int regres; if (regex_list.empty() && !tables_hash) {
xb_filter_entry_t* table; return(FALSE);
xb_regex_list_node_t* node; }
if (UT_LIST_GET_LEN(regex_list)) {
/* Check against regular expressions list */
for (node = UT_LIST_GET_FIRST(regex_list); node;
node = UT_LIST_GET_NEXT(regex_list, node)) {
regres = regexec(&node->regex, name, 1,
tables_regmatch, 0);
if (regres != REG_NOMATCH) {
if (regex_list_check_match(regex_list, name)) {
return(TRUE); return(TRUE);
} }
}
if (tables_hash && find_filter_in_hashtable(name, tables_hash, NULL)) {
return(TRUE);
} }
if (tables_hash) { return FALSE;
HASH_SEARCH(name_hash, tables_hash, ut_fold_string(name), }
xb_filter_entry_t*,
table, (void) 0,
!strcmp(table->name, name));
if (table) {
return(TRUE); enum skip_database_check_result {
DATABASE_SKIP,
DATABASE_SKIP_SOME_TABLES,
DATABASE_DONT_SKIP,
DATABASE_DONT_SKIP_UNLESS_EXPLICITLY_EXCLUDED,
};
/************************************************************************
Checks if a database specified by name should be skipped from backup based on
the --databases, --databases_file or --databases_exclude options.
@return TRUE if entire database should be skipped,
FALSE otherwise.
*/
static
skip_database_check_result
check_if_skip_database(
const char* name /*!< in: path to the database */
)
{
/* There are some filters for databases, check them */
xb_filter_entry_t* database = NULL;
if (databases_exclude_hash &&
find_filter_in_hashtable(name, databases_exclude_hash,
&database) &&
!database->has_tables) {
/* Database is found and there are no tables specified,
skip entire db. */
return DATABASE_SKIP;
}
if (databases_include_hash) {
if (!find_filter_in_hashtable(name, databases_include_hash,
&database)) {
/* Database isn't found, skip the database */
return DATABASE_SKIP;
} else if (database->has_tables) {
return DATABASE_SKIP_SOME_TABLES;
} else {
return DATABASE_DONT_SKIP_UNLESS_EXPLICITLY_EXCLUDED;
} }
} }
return DATABASE_DONT_SKIP;
}
/************************************************************************
Checks if a database specified by path should be skipped from backup based on
the --databases, --databases_file or --databases_exclude options.
@return TRUE if the table should be skipped. */
my_bool
check_if_skip_database_by_path(
const char* path /*!< in: path to the db directory. */
)
{
if (databases_include_hash == NULL &&
databases_exclude_hash == NULL) {
return(FALSE); return(FALSE);
}
const char* db_name = strrchr(path, SRV_PATH_SEPARATOR);
if (db_name == NULL) {
db_name = path;
} else {
++db_name;
}
return check_if_skip_database(db_name) == DATABASE_SKIP;
} }
/************************************************************************ /************************************************************************
...@@ -2217,9 +2326,12 @@ check_if_skip_table( ...@@ -2217,9 +2326,12 @@ check_if_skip_table(
const char *ptr; const char *ptr;
char *eptr; char *eptr;
if (UT_LIST_GET_LEN(regex_list) == 0 && if (regex_exclude_list.empty() &&
tables_hash == NULL && regex_include_list.empty() &&
databases_hash == NULL) { tables_include_hash == NULL &&
tables_exclude_hash == NULL &&
databases_include_hash == NULL &&
databases_exclude_hash == NULL) {
return(FALSE); return(FALSE);
} }
...@@ -2237,23 +2349,10 @@ check_if_skip_table( ...@@ -2237,23 +2349,10 @@ check_if_skip_table(
strncpy(buf, dbname, FN_REFLEN); strncpy(buf, dbname, FN_REFLEN);
buf[tbname - 1 - dbname] = 0; buf[tbname - 1 - dbname] = 0;
if (databases_hash) { const skip_database_check_result skip_database =
/* There are some filters for databases, check them */ check_if_skip_database(buf);
xb_filter_entry_t* database; if (skip_database == DATABASE_SKIP) {
return (TRUE);
HASH_SEARCH(name_hash, databases_hash, ut_fold_string(buf),
xb_filter_entry_t*,
database, (void) 0,
!strcmp(database->name, buf));
/* Table's database isn't found, skip the table */
if (!database) {
return(TRUE);
}
/* There aren't tables specified for the database,
it should be backed up entirely */
if (!database->has_tables) {
return(FALSE);
}
} }
buf[FN_REFLEN - 1] = '\0'; buf[FN_REFLEN - 1] = '\0';
...@@ -2270,21 +2369,43 @@ check_if_skip_table( ...@@ -2270,21 +2369,43 @@ check_if_skip_table(
/* For partitioned tables first try to match against the regexp /* For partitioned tables first try to match against the regexp
without truncating the #P#... suffix so we can backup individual without truncating the #P#... suffix so we can backup individual
partitions with regexps like '^test[.]t#P#p5' */ partitions with regexps like '^test[.]t#P#p5' */
if (check_if_table_matches_filters(buf)) { if (check_if_table_matches_filters(buf, regex_exclude_list,
tables_exclude_hash)) {
return(TRUE);
}
if (check_if_table_matches_filters(buf, regex_include_list,
tables_include_hash)) {
return(FALSE); return(FALSE);
} }
if ((eptr = strstr(buf, "#P#")) != NULL) { if ((eptr = strstr(buf, "#P#")) != NULL) {
*eptr = 0; *eptr = 0;
if (check_if_table_matches_filters(buf)) { if (check_if_table_matches_filters(buf, regex_exclude_list,
tables_exclude_hash)) {
return (TRUE);
}
if (check_if_table_matches_filters(buf, regex_include_list,
tables_include_hash)) {
return(FALSE); return(FALSE);
} }
} }
if (skip_database == DATABASE_DONT_SKIP_UNLESS_EXPLICITLY_EXCLUDED) {
/* Database is in include-list, and qualified name wasn't
found in any of exclusion filters.*/
return (FALSE);
}
if (skip_database == DATABASE_SKIP_SOME_TABLES ||
!regex_include_list.empty() ||
tables_include_hash) {
/* Include lists are present, but qualified name
failed to match any.*/
return(TRUE); return(TRUE);
}
return(FALSE);
} }
/*********************************************************************** /***********************************************************************
...@@ -3040,6 +3161,8 @@ xtrabackup_init_datasinks(void) ...@@ -3040,6 +3161,8 @@ xtrabackup_init_datasinks(void)
if (xtrabackup_encrypt) { if (xtrabackup_encrypt) {
ds_ctxt_t *ds; ds_ctxt_t *ds;
ds = ds_create(xtrabackup_target_dir, DS_TYPE_ENCRYPT); ds = ds_create(xtrabackup_target_dir, DS_TYPE_ENCRYPT);
xtrabackup_add_datasink(ds); xtrabackup_add_datasink(ds);
...@@ -3360,7 +3483,10 @@ static ...@@ -3360,7 +3483,10 @@ static
void void
xb_register_filter_entry( xb_register_filter_entry(
/*=====================*/ /*=====================*/
const char* name) /*!< in: name */ const char* name, /*!< in: name */
hash_table_t** databases_hash,
hash_table_t** tables_hash
)
{ {
const char* p; const char* p;
size_t namelen; size_t namelen;
...@@ -3376,25 +3502,45 @@ xb_register_filter_entry( ...@@ -3376,25 +3502,45 @@ xb_register_filter_entry(
strncpy(dbname, name, p - name); strncpy(dbname, name, p - name);
dbname[p - name] = 0; dbname[p - name] = 0;
if (databases_hash) { if (*databases_hash) {
HASH_SEARCH(name_hash, databases_hash, HASH_SEARCH(name_hash, (*databases_hash),
ut_fold_string(dbname), ut_fold_string(dbname),
xb_filter_entry_t*, xb_filter_entry_t*,
db_entry, (void) 0, db_entry, (void) 0,
!strcmp(db_entry->name, dbname)); !strcmp(db_entry->name, dbname));
} }
if (!db_entry) { if (!db_entry) {
db_entry = xb_add_filter(dbname, &databases_hash); db_entry = xb_add_filter(dbname, databases_hash);
} }
db_entry->has_tables = TRUE; db_entry->has_tables = TRUE;
xb_add_filter(name, &tables_hash); xb_add_filter(name, tables_hash);
} else { } else {
xb_validate_name(name, namelen); xb_validate_name(name, namelen);
xb_add_filter(name, &databases_hash); xb_add_filter(name, databases_hash);
} }
} }
static
void
xb_register_include_filter_entry(
const char* name
)
{
xb_register_filter_entry(name, &databases_include_hash,
&tables_include_hash);
}
static
void
xb_register_exclude_filter_entry(
const char* name
)
{
xb_register_filter_entry(name, &databases_exclude_hash,
&tables_exclude_hash);
}
/*********************************************************************** /***********************************************************************
Register new table for the filter. */ Register new table for the filter. */
static static
...@@ -3408,33 +3554,52 @@ xb_register_table( ...@@ -3408,33 +3554,52 @@ xb_register_table(
exit(EXIT_FAILURE); exit(EXIT_FAILURE);
} }
xb_register_filter_entry(name); xb_register_include_filter_entry(name);
} }
/***********************************************************************
Register new regex for the filter. */
static static
void void
xb_register_regex( xb_add_regex_to_list(
/*==============*/ const char* regex, /*!< in: regex */
const char* regex) /*!< in: regex */ const char* error_context, /*!< in: context to error message */
regex_list_t* list) /*! in: list to put new regex to */
{ {
xb_regex_list_node_t* node;
char errbuf[100]; char errbuf[100];
int ret; int ret;
node = static_cast<xb_regex_list_node_t *> regex_t compiled_regex;
(ut_malloc(sizeof(xb_regex_list_node_t))); ret = regcomp(&compiled_regex, regex, REG_EXTENDED);
ret = regcomp(&node->regex, regex, REG_EXTENDED);
if (ret != 0) { if (ret != 0) {
xb_regerror(ret, &node->regex, errbuf, sizeof(errbuf)); regerror(ret, &compiled_regex, errbuf, sizeof(errbuf));
msg("xtrabackup: error: tables regcomp(%s): %s\n", msg("xtrabackup: error: %s regcomp(%s): %s\n",
regex, errbuf); error_context, regex, errbuf);
exit(EXIT_FAILURE); exit(EXIT_FAILURE);
} }
UT_LIST_ADD_LAST(regex_list, regex_list, node); list->push_back(compiled_regex);
}
/***********************************************************************
Register new regex for the include filter. */
static
void
xb_register_include_regex(
/*==============*/
const char* regex) /*!< in: regex */
{
xb_add_regex_to_list(regex, "tables", &regex_include_list);
}
/***********************************************************************
Register new regex for the exclude filter. */
static
void
xb_register_exclude_regex(
/*==============*/
const char* regex) /*!< in: regex */
{
xb_add_regex_to_list(regex, "tables-exclude", &regex_exclude_list);
} }
typedef void (*insert_entry_func_t)(const char*); typedef void (*insert_entry_func_t)(const char*);
...@@ -3500,26 +3665,34 @@ static ...@@ -3500,26 +3665,34 @@ static
void void
xb_filters_init() xb_filters_init()
{ {
UT_LIST_INIT(regex_list);
if (xtrabackup_databases) { if (xtrabackup_databases) {
xb_load_list_string(xtrabackup_databases, " \t", xb_load_list_string(xtrabackup_databases, " \t",
xb_register_filter_entry); xb_register_include_filter_entry);
} }
if (xtrabackup_databases_file) { if (xtrabackup_databases_file) {
xb_load_list_file(xtrabackup_databases_file, xb_load_list_file(xtrabackup_databases_file,
xb_register_filter_entry); xb_register_include_filter_entry);
}
if (xtrabackup_databases_exclude) {
xb_load_list_string(xtrabackup_databases_exclude, " \t",
xb_register_exclude_filter_entry);
} }
if (xtrabackup_tables) { if (xtrabackup_tables) {
xb_load_list_string(xtrabackup_tables, ",", xb_load_list_string(xtrabackup_tables, ",",
xb_register_regex); xb_register_include_regex);
} }
if (xtrabackup_tables_file) { if (xtrabackup_tables_file) {
xb_load_list_file(xtrabackup_tables_file, xb_register_table); xb_load_list_file(xtrabackup_tables_file, xb_register_table);
} }
if (xtrabackup_tables_exclude) {
xb_load_list_string(xtrabackup_tables_exclude, ",",
xb_register_exclude_regex);
}
} }
static static
...@@ -3551,25 +3724,37 @@ xb_filter_hash_free(hash_table_t* hash) ...@@ -3551,25 +3724,37 @@ xb_filter_hash_free(hash_table_t* hash)
hash_table_free(hash); hash_table_free(hash);
} }
static void xb_regex_list_free(regex_list_t* list)
{
while (list->size() > 0) {
xb_regfree(&list->front());
list->pop_front();
}
}
/************************************************************************ /************************************************************************
Destroy table filters for partial backup. */ Destroy table filters for partial backup. */
static static
void void
xb_filters_free() xb_filters_free()
{ {
while (UT_LIST_GET_LEN(regex_list) > 0) { xb_regex_list_free(&regex_include_list);
xb_regex_list_node_t* node = UT_LIST_GET_FIRST(regex_list); xb_regex_list_free(&regex_exclude_list);
UT_LIST_REMOVE(regex_list, regex_list, node);
regfree(&node->regex); if (tables_include_hash) {
ut_free(node); xb_filter_hash_free(tables_include_hash);
} }
if (tables_hash) { if (tables_exclude_hash) {
xb_filter_hash_free(tables_hash); xb_filter_hash_free(tables_exclude_hash);
} }
if (databases_hash) { if (databases_include_hash) {
xb_filter_hash_free(databases_hash); xb_filter_hash_free(databases_include_hash);
}
if (databases_exclude_hash) {
xb_filter_hash_free(databases_exclude_hash);
} }
} }
...@@ -3854,6 +4039,7 @@ xtrabackup_backup_func(void) ...@@ -3854,6 +4039,7 @@ xtrabackup_backup_func(void)
srv_general_init(); srv_general_init();
ut_crc32_init(); ut_crc32_init();
crc_init();
#ifdef WITH_INNODB_DISALLOW_WRITES #ifdef WITH_INNODB_DISALLOW_WRITES
srv_allow_writes_event = os_event_create(); srv_allow_writes_event = os_event_create();
...@@ -6014,7 +6200,7 @@ xb_export_cfg_write( ...@@ -6014,7 +6200,7 @@ xb_export_cfg_write(
file = fopen(file_path, "w+b"); file = fopen(file_path, "w+b");
if (file == NULL) { if (file == NULL) {
msg("xtrabackup: Error: cannot close %s\n", node->name); msg("xtrabackup: Error: cannot open %s\n", node->name);
success = false; success = false;
} else { } else {
...@@ -6473,20 +6659,23 @@ xtrabackup_prepare_func(int argc, char ** argv) ...@@ -6473,20 +6659,23 @@ xtrabackup_prepare_func(int argc, char ** argv)
table_name); table_name);
goto next_node; goto next_node;
} }
index = dict_table_get_first_index(table);
n_index = UT_LIST_GET_LEN(table->indexes);
if (n_index > 31) {
msg("xtrabackup: error: "
"sorry, cannot export over "
"31 indexes for now.\n");
goto next_node;
}
/* Write MySQL 5.6 .cfg file */ /* Write MySQL 5.6 .cfg file */
if (!xb_export_cfg_write(node, table)) { if (!xb_export_cfg_write(node, table)) {
goto next_node; goto next_node;
} }
index = dict_table_get_first_index(table);
n_index = UT_LIST_GET_LEN(table->indexes);
if (n_index > 31) {
msg("xtrabackup: warning: table '%s' has more "
"than 31 indexes, .exp file was not "
"generated. Table will fail to import "
"on server version prior to 5.6.\n",
table->name);
goto next_node;
}
/* init exp file */ /* init exp file */
memset(page, 0, UNIV_PAGE_SIZE); memset(page, 0, UNIV_PAGE_SIZE);
mach_write_to_4(page , 0x78706f72UL); mach_write_to_4(page , 0x78706f72UL);
...@@ -6659,6 +6848,12 @@ xtrabackup_prepare_func(int argc, char ** argv) ...@@ -6659,6 +6848,12 @@ xtrabackup_prepare_func(int argc, char ** argv)
if (!xtrabackup_apply_log_only) { if (!xtrabackup_apply_log_only) {
/* xtrabackup_incremental_dir is used to indicate that
we are going to apply incremental backup. Here we already
applied incremental backup and are about to do final prepare
of the full backup */
xtrabackup_incremental_dir = NULL;
if(innodb_init_param()) { if(innodb_init_param()) {
goto error; goto error;
} }
...@@ -7016,20 +7211,24 @@ handle_options(int argc, char **argv, char ***argv_client, char ***argv_server) ...@@ -7016,20 +7211,24 @@ handle_options(int argc, char **argv, char ***argv_client, char ***argv_server)
} }
/* ================= main =================== */ /* ================= main =================== */
extern my_bool(*fil_check_if_skip_database_by_path)(const char* name);
int main(int argc, char **argv) int main(int argc, char **argv)
{ {
char **client_defaults, **server_defaults; char **client_defaults, **server_defaults;
char cwd[FN_REFLEN]; char cwd[FN_REFLEN];
static char INNOBACKUPEX_EXE[]= "innobackupex";
if (argc > 1 && (strcmp(argv[1], "--innobackupex") == 0)) if (argc > 1 && (strcmp(argv[1], "--innobackupex") == 0))
{ {
argv++; argv++;
argc--; argc--;
argv[0] = "innobackupex"; argv[0] = INNOBACKUPEX_EXE;
innobackupex_mode = true; innobackupex_mode = true;
} }
/* Setup skip fil_load_single_tablespaces callback.*/
fil_check_if_skip_database_by_path = check_if_skip_database_by_path;
init_signals(); init_signals();
MY_INIT(argv[0]); MY_INIT(argv[0]);
......
extra/mariabackup/xtrabackup.h
View file @ ecb25df2
...@@ -77,6 +77,8 @@ extern char *xtrabackup_tables; ...@@ -77,6 +77,8 @@ extern char *xtrabackup_tables;
extern char *xtrabackup_tables_file; extern char *xtrabackup_tables_file;
extern char *xtrabackup_databases; extern char *xtrabackup_databases;
extern char *xtrabackup_databases_file; extern char *xtrabackup_databases_file;
extern char *xtrabackup_tables_exclude;
extern char *xtrabackup_databases_exclude;
extern ibool xtrabackup_compress; extern ibool xtrabackup_compress;
extern ibool xtrabackup_encrypt; extern ibool xtrabackup_encrypt;
...@@ -205,6 +207,17 @@ check_if_skip_table( ...@@ -205,6 +207,17 @@ check_if_skip_table(
/******************/ /******************/
const char* name); /*!< in: path to the table */ const char* name); /*!< in: path to the table */
/************************************************************************
Checks if a database specified by path should be skipped from backup based on
the --databases, --databases_file or --databases_exclude options.
@return TRUE if the table should be skipped. */
my_bool
check_if_skip_database_by_path(
const char* path /*!< in: path to the db directory. */
);
/************************************************************************ /************************************************************************
Check if parameter is set in defaults file or via command line argument Check if parameter is set in defaults file or via command line argument
@return true if parameter is set. */ @return true if parameter is set. */
......
mysql-test/suite/mariabackup/partial_exclude.result 0 → 100644
View file @ ecb25df2
CREATE TABLE t1(i INT) ENGINE INNODB;
INSERT INTO t1 VALUES(1);
CREATE TABLE t2(i int) ENGINE INNODB;
CREATE DATABASE db2;
USE db2;
CREATE TABLE t1(i INT) ENGINE INNODB;
USE test;
# xtrabackup backup
t1.ibd
DROP TABLE t1;
DROP TABLE t2;
DROP DATABASE db2;
mysql-test/suite/mariabackup/partial_exclude.test 0 → 100644
View file @ ecb25df2
# Test --databases-exclude and --tables-exclude feature of xtrabackup 2.3.8
CREATE TABLE t1(i INT) ENGINE INNODB;
INSERT INTO t1 VALUES(1);
CREATE TABLE t2(i int) ENGINE INNODB;
CREATE DATABASE db2;
USE db2;
CREATE TABLE t1(i INT) ENGINE INNODB;
USE test;
echo # xtrabackup backup;
let $targetdir=$MYSQLTEST_VARDIR/tmp/backup;
--disable_result_log
exec $XTRABACKUP --defaults-file=$MYSQLTEST_VARDIR/my.cnf --backup "--tables-exclude=test.*2" "--databases-exclude=db2" --target-dir=$targetdir;
--enable_result_log
# check that only t1 table is in backup (t2 is excluded)
list_files $targetdir/test *.ibd;
# check that db2 database is not in the backup (excluded)
--error 1
list_files $targetdir/db2 *.ibd;
DROP TABLE t1;
DROP TABLE t2;
DROP DATABASE db2;
rmdir $targetdir;
mysql-test/suite/mariabackup/xbstream.test
View file @ ecb25df2
...@@ -9,7 +9,7 @@ echo # xtrabackup backup to stream; ...@@ -9,7 +9,7 @@ echo # xtrabackup backup to stream;
exec $XTRABACKUP --defaults-file=$MYSQLTEST_VARDIR/my.cnf --backup --stream=xbstream > $streamfile 2>$targetdir/backup_stream.log; exec $XTRABACKUP --defaults-file=$MYSQLTEST_VARDIR/my.cnf --backup --stream=xbstream > $streamfile 2>$targetdir/backup_stream.log;
echo # xbstream extract; echo # xbstream extract;
--disable_result_log --disable_result_log
exec $XBSTREAM -x -C $targetdir < $streamfile; exec $XBSTREAM -x -C $targetdir --parallel=16 < $streamfile;
echo # xtrabackup prepare; echo # xtrabackup prepare;
exec $XTRABACKUP --prepare --target-dir=$targetdir; exec $XTRABACKUP --prepare --target-dir=$targetdir;
......
storage/xtradb/fil/fil0fil.cc
View file @ ecb25df2
...@@ -71,6 +71,7 @@ static ulint srv_data_read, srv_data_written; ...@@ -71,6 +71,7 @@ static ulint srv_data_read, srv_data_written;
MYSQL_PLUGIN_IMPORT extern my_bool lower_case_file_system; MYSQL_PLUGIN_IMPORT extern my_bool lower_case_file_system;
/* /*
IMPLEMENTATION OF THE TABLESPACE MEMORY CACHE IMPLEMENTATION OF THE TABLESPACE MEMORY CACHE
============================================= =============================================
...@@ -5379,6 +5380,9 @@ fil_file_readdir_next_file( ...@@ -5379,6 +5380,9 @@ fil_file_readdir_next_file(
return(-1); return(-1);
} }
my_bool(*fil_check_if_skip_database_by_path)(const char* name);
#define CHECK_TIME_EVERY_N_FILES 10 #define CHECK_TIME_EVERY_N_FILES 10
/********************************************************************//** /********************************************************************//**
At the server startup, if we need crash recovery, scans the database At the server startup, if we need crash recovery, scans the database
...@@ -5449,7 +5453,19 @@ fil_load_single_table_tablespaces(ibool (*pred)(const char*, const char*)) ...@@ -5449,7 +5453,19 @@ fil_load_single_table_tablespaces(ibool (*pred)(const char*, const char*))
"%s/%s", fil_path_to_mysql_datadir, dbinfo.name); "%s/%s", fil_path_to_mysql_datadir, dbinfo.name);
srv_normalize_path_for_win(dbpath); srv_normalize_path_for_win(dbpath);
if (IS_XTRABACKUP()) {
ut_a(fil_check_if_skip_database_by_path);
if (fil_check_if_skip_database_by_path(dbpath)) {
fprintf(stderr, "Skipping db: %s\n", dbpath);
dbdir = NULL;
} else {
/* We want wrong directory permissions to be a fatal
error for XtraBackup. */
dbdir = os_file_opendir(dbpath, TRUE);
}
} else {
dbdir = os_file_opendir(dbpath, FALSE); dbdir = os_file_opendir(dbpath, FALSE);
}
if (dbdir != NULL) { if (dbdir != NULL) {
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment
GitLab Nexedi Edition | About GitLab | About Nexedi | 沪ICP备2021021310号-2 | 沪ICP备2021021310号-7