Skip to content

M
MariaDB
Commit ecb25df2 authored by Vladislav Vaintroub's avatar Vladislav Vaintroub Committed by Sergei Golubchik
Browse files
Options

Xtrabackup 2.3.8

parent c8ac0244
Show whitespace changes
Inline Side-by-side
Showing with 2414 additions and 422 deletions
extra/mariabackup/CMakeLists.txt
View file @ ecb25df2
# Copyright (c) 2013 Percona LLC and/or its affiliates.
# Copyright (c) 2013, 2017 Percona LLC and/or its affiliates.
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation; version 2 of the License.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA
......@@ -129,6 +129,7 @@ INCLUDE_DIRECTORIES(
${CMAKE_SOURCE_DIR}/sql
${CMAKE_CURRENT_SOURCE_DIR}/quicklz
${CMAKE_CURRENT_SOURCE_DIR}
${CMAKE_CURRENT_SOURCE_DIR}/crc
)
IF(NOT HAVE_SYSTEM_REGEX)
......@@ -156,7 +157,6 @@ MYSQL_ADD_EXECUTABLE(mariabackup
${DS_ARCHIVE_SOURCE}
ds_buffer.c
ds_compress.c
ds_encrypt.c
ds_local.c
ds_stdout.c
ds_tmpfile.c
......@@ -166,8 +166,6 @@ MYSQL_ADD_EXECUTABLE(mariabackup
read_filt.cc
write_filt.cc
wsrep.cc
xbcrypt_common.c
xbcrypt_write.c
xbstream_write.c
backup_mysql.cc
backup_copy.cc
......@@ -181,9 +179,10 @@ MYSQL_ADD_EXECUTABLE(mariabackup
# Export all symbols on Unix, for better crash callstacks
SET_TARGET_PROPERTIES(mariabackup PROPERTIES ENABLE_EXPORTS TRUE)
ADD_SUBDIRECTORY(crc)
TARGET_LINK_LIBRARIES(mariabackup sql)
TARGET_LINK_LIBRARIES(mariabackup sql crc)
IF(NOT HAVE_SYSTEM_REGEX)
TARGET_LINK_LIBRARIES(mariabackup pcreposix)
......@@ -201,13 +200,13 @@ MYSQL_ADD_EXECUTABLE(mbstream
xbstream.c
xbstream_read.c
xbstream_write.c
COMPONENT backup
)
TARGET_LINK_LIBRARIES(mbstream
mysys
crc
)
IF(MSVC)
......
extra/mariabackup/backup_copy.cc
View file @ ecb25df2
......@@ -265,6 +265,11 @@ datadir_iter_next_database(datadir_iter_t *it)
return(true);
}
if (check_if_skip_database_by_path(it->dbpath)) {
msg("Skipping db: %s\n", it->dbpath);
continue;
}
/* We want wrong directory permissions to be a fatal error for
XtraBackup. */
it->dbdir = os_file_opendir(it->dbpath, TRUE);
......@@ -1704,7 +1709,7 @@ copy_back()
for (i = 1; i <= srv_undo_tablespaces; i++) {
char filename[20];
sprintf(filename, "undo%03lu", i);
sprintf(filename, "undo%03u", (uint)i);
if (!(ret = copy_or_move_file(filename, filename,
dst_dir, 1))) {
goto cleanup;
......
extra/mariabackup/backup_mysql.cc
View file @ ecb25df2
......@@ -1414,7 +1414,10 @@ write_xtrabackup_info(MYSQL *connection)
bool is_partial = (xtrabackup_tables
|| xtrabackup_tables_file
|| xtrabackup_databases
|| xtrabackup_databases_file);
|| xtrabackup_databases_file
|| xtrabackup_tables_exclude
|| xtrabackup_databases_exclude
);
backup_file_printf(XTRABACKUP_INFO,
"uuid = %s\n"
......
extra/mariabackup/crc/CMakeLists.txt 0 → 100644
View file @ ecb25df2
# Copyright (c) 2017 Percona LLC and/or its affiliates.
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation; version 2 of the License.
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA
PROJECT(crc C)
IF(NOT CMAKE_CROSSCOMPILING AND NOT MSVC)
STRING(TOLOWER ${CMAKE_SYSTEM_PROCESSOR} processor)
IF(processor MATCHES "86" OR processor MATCHES "amd64" OR processor MATCHES "x64")
# Check for PCLMUL instruction
CHECK_C_SOURCE_RUNS("
int main()
{
asm volatile (\"pclmulqdq \\$0x00, %%xmm1, %%xmm0\":::\"cc\");
return 0;
}" HAVE_CLMUL_INSTRUCTION)
ENDIF()
ENDIF()
IF(HAVE_CLMUL_INSTRUCTION)
ADD_DEFINITIONS(-DHAVE_CLMUL_INSTRUCTION)
ENDIF()
ADD_LIBRARY(crc crc_glue.c crc-intel-pclmul.c)
extra/mariabackup/crc/config.h.cmake 0 → 100644
View file @ ecb25df2
/******************************************************
Copyright (c) 2017 Percona LLC and/or its affiliates.
Zlib compatible CRC-32 implementation.
This program is free software; you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation; version 2 of the License.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program; if not, write to the Free Software
Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA
*******************************************************/
#cmakedefine HAVE_CLMUL_INSTRUCTION 1
extra/mariabackup/crc/crc-intel-pclmul.c 0 → 100644
View file @ ecb25df2
/******************************************************
Copyright (c) 2017 Percona LLC and/or its affiliates.
CRC32 using Intel's PCLMUL instruction.
This program is free software; you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation; version 2 of the License.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program; if not, write to the Free Software
Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA
*******************************************************/
/* crc-intel-pclmul.c - Intel PCLMUL accelerated CRC implementation
* Copyright (C) 2016 Jussi Kivilinna <jussi.kivilinna@iki.fi>
*
* This file is part of Libgcrypt.
*
* Libgcrypt is free software; you can redistribute it and/or modify
* it under the terms of the GNU Lesser General Public License as
* published by the Free Software Foundation; either version 2.1 of
* the License, or (at your option) any later version.
*
* Libgcrypt is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Lesser General Public License for more details.
*
* You should have received a copy of the GNU Lesser General Public
* License along with this program; if not, write to the Free Software
* Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA
*
*/
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <stdint.h>
# define U64_C(c) (c ## UL)
typedef uint32_t u32;
typedef uint16_t u16;
typedef uint64_t u64;
#ifndef byte
typedef uint8_t byte;
#endif
# define _gcry_bswap32 __builtin_bswap32
#if __GNUC__ >= 4 && defined(__x86_64__) && defined(HAVE_CLMUL_INSTRUCTION)
#if _GCRY_GCC_VERSION >= 40400 /* 4.4 */
/* Prevent compiler from issuing SSE instructions between asm blocks. */
# pragma GCC target("no-sse")
#endif
#define ALIGNED_16 __attribute__ ((aligned (16)))
struct u16_unaligned_s
{
u16 a;
} __attribute__((packed, aligned (1), may_alias));
/* Constants structure for generic reflected/non-reflected CRC32 CLMUL
* functions. */
struct crc32_consts_s
{
/* k: { x^(32*17), x^(32*15), x^(32*5), x^(32*3), x^(32*2), 0 } mod P(x) */
u64 k[6];
/* my_p: { floor(x^64 / P(x)), P(x) } */
u64 my_p[2];
};
/* CLMUL constants for CRC32 and CRC32RFC1510. */
static const struct crc32_consts_s crc32_consts ALIGNED_16 =
{
{ /* k[6] = reverse_33bits( x^(32*y) mod P(x) ) */
U64_C(0x154442bd4), U64_C(0x1c6e41596), /* y = { 17, 15 } */
U64_C(0x1751997d0), U64_C(0x0ccaa009e), /* y = { 5, 3 } */
U64_C(0x163cd6124), 0 /* y = 2 */
},
{ /* my_p[2] = reverse_33bits ( { floor(x^64 / P(x)), P(x) } ) */
U64_C(0x1f7011641), U64_C(0x1db710641)
}
};
/* Common constants for CRC32 algorithms. */
static const byte crc32_refl_shuf_shift[3 * 16] ALIGNED_16 =
{
0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07,
0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f,
0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
};
static const byte crc32_partial_fold_input_mask[16 + 16] ALIGNED_16 =
{
0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
};
static const u64 crc32_merge9to15_shuf[15 - 9 + 1][2] ALIGNED_16 =
{
{ U64_C(0x0706050403020100), U64_C(0xffffffffffffff0f) }, /* 9 */
{ U64_C(0x0706050403020100), U64_C(0xffffffffffff0f0e) },
{ U64_C(0x0706050403020100), U64_C(0xffffffffff0f0e0d) },
{ U64_C(0x0706050403020100), U64_C(0xffffffff0f0e0d0c) },
{ U64_C(0x0706050403020100), U64_C(0xffffff0f0e0d0c0b) },
{ U64_C(0x0706050403020100), U64_C(0xffff0f0e0d0c0b0a) },
{ U64_C(0x0706050403020100), U64_C(0xff0f0e0d0c0b0a09) }, /* 15 */
};
static const u64 crc32_merge5to7_shuf[7 - 5 + 1][2] ALIGNED_16 =
{
{ U64_C(0xffffff0703020100), U64_C(0xffffffffffffffff) }, /* 5 */
{ U64_C(0xffff070603020100), U64_C(0xffffffffffffffff) },
{ U64_C(0xff07060503020100), U64_C(0xffffffffffffffff) }, /* 7 */
};
/* PCLMUL functions for reflected CRC32. */
static inline void
crc32_reflected_bulk (u32 *pcrc, const byte *inbuf, size_t inlen,
const struct crc32_consts_s *consts)
{
if (inlen >= 8 * 16)
{
asm volatile ("movd %[crc], %%xmm4\n\t"
"movdqu %[inbuf_0], %%xmm0\n\t"
"movdqu %[inbuf_1], %%xmm1\n\t"
"movdqu %[inbuf_2], %%xmm2\n\t"
"movdqu %[inbuf_3], %%xmm3\n\t"
"pxor %%xmm4, %%xmm0\n\t"
:
: [inbuf_0] "m" (inbuf[0 * 16]),
[inbuf_1] "m" (inbuf[1 * 16]),
[inbuf_2] "m" (inbuf[2 * 16]),
[inbuf_3] "m" (inbuf[3 * 16]),
[crc] "m" (*pcrc)
);
inbuf += 4 * 16;
inlen -= 4 * 16;
asm volatile ("movdqa %[k1k2], %%xmm4\n\t"
:
: [k1k2] "m" (consts->k[1 - 1])
);
/* Fold by 4. */
while (inlen >= 4 * 16)
{
asm volatile ("movdqu %[inbuf_0], %%xmm5\n\t"
"movdqa %%xmm0, %%xmm6\n\t"
"pclmulqdq $0x00, %%xmm4, %%xmm0\n\t"
"pclmulqdq $0x11, %%xmm4, %%xmm6\n\t"
"pxor %%xmm5, %%xmm0\n\t"
"pxor %%xmm6, %%xmm0\n\t"
"movdqu %[inbuf_1], %%xmm5\n\t"
"movdqa %%xmm1, %%xmm6\n\t"
"pclmulqdq $0x00, %%xmm4, %%xmm1\n\t"
"pclmulqdq $0x11, %%xmm4, %%xmm6\n\t"
"pxor %%xmm5, %%xmm1\n\t"
"pxor %%xmm6, %%xmm1\n\t"
"movdqu %[inbuf_2], %%xmm5\n\t"
"movdqa %%xmm2, %%xmm6\n\t"
"pclmulqdq $0x00, %%xmm4, %%xmm2\n\t"
"pclmulqdq $0x11, %%xmm4, %%xmm6\n\t"
"pxor %%xmm5, %%xmm2\n\t"
"pxor %%xmm6, %%xmm2\n\t"
"movdqu %[inbuf_3], %%xmm5\n\t"
"movdqa %%xmm3, %%xmm6\n\t"
"pclmulqdq $0x00, %%xmm4, %%xmm3\n\t"
"pclmulqdq $0x11, %%xmm4, %%xmm6\n\t"
"pxor %%xmm5, %%xmm3\n\t"
"pxor %%xmm6, %%xmm3\n\t"
:
: [inbuf_0] "m" (inbuf[0 * 16]),
[inbuf_1] "m" (inbuf[1 * 16]),
[inbuf_2] "m" (inbuf[2 * 16]),
[inbuf_3] "m" (inbuf[3 * 16])
);
inbuf += 4 * 16;
inlen -= 4 * 16;
}
asm volatile ("movdqa %[k3k4], %%xmm6\n\t"
"movdqa %[my_p], %%xmm5\n\t"
:
: [k3k4] "m" (consts->k[3 - 1]),
[my_p] "m" (consts->my_p[0])
);
/* Fold 4 to 1. */
asm volatile ("movdqa %%xmm0, %%xmm4\n\t"
"pclmulqdq $0x00, %%xmm6, %%xmm0\n\t"
"pclmulqdq $0x11, %%xmm6, %%xmm4\n\t"
"pxor %%xmm1, %%xmm0\n\t"
"pxor %%xmm4, %%xmm0\n\t"
"movdqa %%xmm0, %%xmm4\n\t"
"pclmulqdq $0x00, %%xmm6, %%xmm0\n\t"
"pclmulqdq $0x11, %%xmm6, %%xmm4\n\t"
"pxor %%xmm2, %%xmm0\n\t"
"pxor %%xmm4, %%xmm0\n\t"
"movdqa %%xmm0, %%xmm4\n\t"
"pclmulqdq $0x00, %%xmm6, %%xmm0\n\t"
"pclmulqdq $0x11, %%xmm6, %%xmm4\n\t"
"pxor %%xmm3, %%xmm0\n\t"
"pxor %%xmm4, %%xmm0\n\t"
:
:
);
}
else
{
asm volatile ("movd %[crc], %%xmm1\n\t"
"movdqu %[inbuf], %%xmm0\n\t"
"movdqa %[k3k4], %%xmm6\n\t"
"pxor %%xmm1, %%xmm0\n\t"
"movdqa %[my_p], %%xmm5\n\t"
:
: [inbuf] "m" (*inbuf),
[crc] "m" (*pcrc),
[k3k4] "m" (consts->k[3 - 1]),
[my_p] "m" (consts->my_p[0])
);
inbuf += 16;
inlen -= 16;
}
/* Fold by 1. */
if (inlen >= 16)
{
while (inlen >= 16)
{
/* Load next block to XMM2. Fold XMM0 to XMM0:XMM1. */
asm volatile ("movdqu %[inbuf], %%xmm2\n\t"
"movdqa %%xmm0, %%xmm1\n\t"
"pclmulqdq $0x00, %%xmm6, %%xmm0\n\t"
"pclmulqdq $0x11, %%xmm6, %%xmm1\n\t"
"pxor %%xmm2, %%xmm0\n\t"
"pxor %%xmm1, %%xmm0\n\t"
:
: [inbuf] "m" (*inbuf)
);
inbuf += 16;
inlen -= 16;
}
}
/* Partial fold. */
if (inlen)
{
/* Load last input and add padding zeros. */
asm volatile ("movdqu %[shr_shuf], %%xmm3\n\t"
"movdqu %[shl_shuf], %%xmm4\n\t"
"movdqu %[mask], %%xmm2\n\t"
"movdqa %%xmm0, %%xmm1\n\t"
"pshufb %%xmm4, %%xmm0\n\t"
"movdqu %[inbuf], %%xmm4\n\t"
"pshufb %%xmm3, %%xmm1\n\t"
"pand %%xmm4, %%xmm2\n\t"
"por %%xmm1, %%xmm2\n\t"
"movdqa %%xmm0, %%xmm1\n\t"
"pclmulqdq $0x00, %%xmm6, %%xmm0\n\t"
"pclmulqdq $0x11, %%xmm6, %%xmm1\n\t"
"pxor %%xmm2, %%xmm0\n\t"
"pxor %%xmm1, %%xmm0\n\t"
:
: [inbuf] "m" (*(inbuf - 16 + inlen)),
[mask] "m" (crc32_partial_fold_input_mask[inlen]),
[shl_shuf] "m" (crc32_refl_shuf_shift[inlen]),
[shr_shuf] "m" (crc32_refl_shuf_shift[inlen + 16])
);
inbuf += inlen;
inlen -= inlen;
}
/* Final fold. */
asm volatile (/* reduce 128-bits to 96-bits */
"movdqa %%xmm0, %%xmm1\n\t"
"pclmulqdq $0x10, %%xmm6, %%xmm0\n\t"
"psrldq $8, %%xmm1\n\t"
"pxor %%xmm1, %%xmm0\n\t"
/* reduce 96-bits to 64-bits */
"pshufd $0xfc, %%xmm0, %%xmm1\n\t" /* [00][00][00][x] */
"pshufd $0xf9, %%xmm0, %%xmm0\n\t" /* [00][00][x>>64][x>>32] */
"pclmulqdq $0x00, %[k5], %%xmm1\n\t" /* [00][00][xx][xx] */
"pxor %%xmm1, %%xmm0\n\t" /* top 64-bit are zero */
/* barrett reduction */
"pshufd $0xf3, %%xmm0, %%xmm1\n\t" /* [00][00][x>>32][00] */
"pslldq $4, %%xmm0\n\t" /* [??][x>>32][??][??] */
"pclmulqdq $0x00, %%xmm5, %%xmm1\n\t" /* [00][xx][xx][00] */
"pclmulqdq $0x10, %%xmm5, %%xmm1\n\t" /* [00][xx][xx][00] */
"pxor %%xmm1, %%xmm0\n\t"
/* store CRC */
"pextrd $2, %%xmm0, %[out]\n\t"
: [out] "=m" (*pcrc)
: [k5] "m" (consts->k[5 - 1])
);
}
static inline void
crc32_reflected_less_than_16 (u32 *pcrc, const byte *inbuf, size_t inlen,
const struct crc32_consts_s *consts)
{
if (inlen < 4)
{
u32 crc = *pcrc;
u32 data;
asm volatile ("movdqa %[my_p], %%xmm5\n\t"
:
: [my_p] "m" (consts->my_p[0])
);
if (inlen == 1)
{
data = inbuf[0];
data ^= crc;
data <<= 24;
crc >>= 8;
}
else if (inlen == 2)
{
data = ((const struct u16_unaligned_s *)inbuf)->a;
data ^= crc;
data <<= 16;
crc >>= 16;
}
else
{
data = ((const struct u16_unaligned_s *)inbuf)->a;
data |= inbuf[2] << 16;
data ^= crc;
data <<= 8;
crc >>= 24;
}
/* Barrett reduction */
asm volatile ("movd %[in], %%xmm0\n\t"
"movd %[crc], %%xmm1\n\t"
"pclmulqdq $0x00, %%xmm5, %%xmm0\n\t" /* [00][00][xx][xx] */
"psllq $32, %%xmm1\n\t"
"pshufd $0xfc, %%xmm0, %%xmm0\n\t" /* [00][00][00][x] */
"pclmulqdq $0x10, %%xmm5, %%xmm0\n\t" /* [00][00][xx][xx] */
"pxor %%xmm1, %%xmm0\n\t"
"pextrd $1, %%xmm0, %[out]\n\t"
: [out] "=m" (*pcrc)
: [in] "rm" (data),
[crc] "rm" (crc)
);
}
else if (inlen == 4)
{
/* Barrett reduction */
asm volatile ("movd %[crc], %%xmm1\n\t"
"movd %[in], %%xmm0\n\t"
"movdqa %[my_p], %%xmm5\n\t"
"pxor %%xmm1, %%xmm0\n\t"
"pclmulqdq $0x00, %%xmm5, %%xmm0\n\t" /* [00][00][xx][xx] */
"pshufd $0xfc, %%xmm0, %%xmm0\n\t" /* [00][00][00][x] */
"pclmulqdq $0x10, %%xmm5, %%xmm0\n\t" /* [00][00][xx][xx] */
"pextrd $1, %%xmm0, %[out]\n\t"
: [out] "=m" (*pcrc)
: [in] "m" (*inbuf),
[crc] "m" (*pcrc),
[my_p] "m" (consts->my_p[0])
);
}
else
{
asm volatile ("movdqu %[shuf], %%xmm4\n\t"
"movd %[crc], %%xmm1\n\t"
"movdqa %[my_p], %%xmm5\n\t"
"movdqa %[k3k4], %%xmm6\n\t"
:
: [shuf] "m" (crc32_refl_shuf_shift[inlen]),
[crc] "m" (*pcrc),
[my_p] "m" (consts->my_p[0]),
[k3k4] "m" (consts->k[3 - 1])
);
if (inlen >= 8)
{
asm volatile ("movq %[inbuf], %%xmm0\n\t"
:
: [inbuf] "m" (*inbuf)
);
if (inlen > 8)
{
asm volatile (/*"pinsrq $1, %[inbuf_tail], %%xmm0\n\t"*/
"movq %[inbuf_tail], %%xmm2\n\t"
"punpcklqdq %%xmm2, %%xmm0\n\t"
"pshufb %[merge_shuf], %%xmm0\n\t"
:
: [inbuf_tail] "m" (inbuf[inlen - 8]),
[merge_shuf] "m"
(*crc32_merge9to15_shuf[inlen - 9])
);
}
}
else
{
asm volatile ("movd %[inbuf], %%xmm0\n\t"
"pinsrd $1, %[inbuf_tail], %%xmm0\n\t"
"pshufb %[merge_shuf], %%xmm0\n\t"
:
: [inbuf] "m" (*inbuf),
[inbuf_tail] "m" (inbuf[inlen - 4]),
[merge_shuf] "m"
(*crc32_merge5to7_shuf[inlen - 5])
);
}
/* Final fold. */
asm volatile ("pxor %%xmm1, %%xmm0\n\t"
"pshufb %%xmm4, %%xmm0\n\t"
/* reduce 128-bits to 96-bits */
"movdqa %%xmm0, %%xmm1\n\t"
"pclmulqdq $0x10, %%xmm6, %%xmm0\n\t"
"psrldq $8, %%xmm1\n\t"
"pxor %%xmm1, %%xmm0\n\t" /* top 32-bit are zero */
/* reduce 96-bits to 64-bits */
"pshufd $0xfc, %%xmm0, %%xmm1\n\t" /* [00][00][00][x] */
"pshufd $0xf9, %%xmm0, %%xmm0\n\t" /* [00][00][x>>64][x>>32] */
"pclmulqdq $0x00, %[k5], %%xmm1\n\t" /* [00][00][xx][xx] */
"pxor %%xmm1, %%xmm0\n\t" /* top 64-bit are zero */
/* barrett reduction */
"pshufd $0xf3, %%xmm0, %%xmm1\n\t" /* [00][00][x>>32][00] */
"pslldq $4, %%xmm0\n\t" /* [??][x>>32][??][??] */
"pclmulqdq $0x00, %%xmm5, %%xmm1\n\t" /* [00][xx][xx][00] */
"pclmulqdq $0x10, %%xmm5, %%xmm1\n\t" /* [00][xx][xx][00] */
"pxor %%xmm1, %%xmm0\n\t"
/* store CRC */
"pextrd $2, %%xmm0, %[out]\n\t"
: [out] "=m" (*pcrc)
: [k5] "m" (consts->k[5 - 1])
);
}
}
void
crc32_intel_pclmul (u32 *pcrc, const byte *inbuf, size_t inlen)
{
const struct crc32_consts_s *consts = &crc32_consts;
#if defined(__x86_64__) && defined(__WIN64__)
char win64tmp[2 * 16];
/* XMM6-XMM7 need to be restored after use. */
asm volatile ("movdqu %%xmm6, 0*16(%0)\n\t"
"movdqu %%xmm7, 1*16(%0)\n\t"
:
: "r" (win64tmp)
: "memory");
#endif
if (!inlen)
return;
if (inlen >= 16)
crc32_reflected_bulk(pcrc, inbuf, inlen, consts);
else
crc32_reflected_less_than_16(pcrc, inbuf, inlen, consts);
#if defined(__x86_64__) && defined(__WIN64__)
/* Restore used registers. */
asm volatile("movdqu 0*16(%0), %%xmm6\n\t"
"movdqu 1*16(%0), %%xmm7\n\t"
:
: "r" (win64tmp)
: "memory");
#endif
}
#endif
extra/mariabackup/crc/crc-intel-pclmul.h 0 → 100644
View file @ ecb25df2
/******************************************************
Copyright (c) 2017 Percona LLC and/or its affiliates.
CRC32 using Intel's PCLMUL instruction.
This program is free software; you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation; version 2 of the License.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program; if not, write to the Free Software
Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA
*******************************************************/
#include <stdint.h>
#include <stddef.h>
void
crc32_intel_pclmul(uint32_t *pcrc, const uint8_t *inbuf, size_t inlen);
extra/mariabackup/crc/crc_glue.c 0 → 100644
View file @ ecb25df2
/******************************************************
Copyright (c) 2017 Percona LLC and/or its affiliates.
Zlib compatible CRC-32 implementation.
This program is free software; you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation; version 2 of the License.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program; if not, write to the Free Software
Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA
*******************************************************/
#include <zlib.h>
#include <stdint.h>
#include <string.h>
#include "crc_glue.h"
#include "crc-intel-pclmul.h"
#if __GNUC__ >= 4 && defined(__x86_64__)
static int pclmul_enabled = 0;
#endif
#if defined(__GNUC__) && defined(__x86_64__)
static
uint32_t
cpuid(uint32_t* ecx, uint32_t* edx)
{
uint32_t level;
asm("cpuid" : "=a" (level) : "a" (0) : "ebx", "ecx", "edx");
if (level < 1) {
return level;
}
asm("cpuid" : "=c" (*ecx), "=d" (*edx)
: "a" (1)
: "ebx");
return level;
}
#endif
void crc_init() {
#if defined(__GNUC__) && defined(__x86_64__)
uint32_t ecx, edx;
if (cpuid(&ecx, &edx) > 0) {
pclmul_enabled = ((ecx >> 19) & 1) && ((ecx >> 1) & 1);
}
#endif
}
unsigned long crc32_iso3309(unsigned long crc, const unsigned char *buf, unsigned int len)
{
#if __GNUC__ >= 4 && defined(__x86_64__) && defined(HAVE_CLMUL_INSTRUCTION)
if (pclmul_enabled) {
uint32_t crc_accum = crc ^ 0xffffffffL;
crc32_intel_pclmul(&crc_accum, buf, len);
return crc_accum ^ 0xffffffffL;
}
#endif
return crc32(crc, buf, len);
}
extra/mariabackup/crc/crc_glue.h 0 → 100644
View file @ ecb25df2
/******************************************************
Copyright (c) 2017 Percona LLC and/or its affiliates.
Zlib compatible CRC-32 implementation.
This program is free software; you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation; version 2 of the License.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program; if not, write to the Free Software
Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA
*******************************************************/
#ifdef __cplusplus
extern "C" {
#endif
void crc_init();
unsigned long crc32_iso3309(unsigned long crc, const unsigned char *buf, unsigned int len);
#ifdef __cplusplus
}
#endif
extra/mariabackup/datasink.c
View file @ ecb25df2
......@@ -27,7 +27,6 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA
#include "ds_local.h"
#include "ds_stdout.h"
#include "ds_tmpfile.h"
#include "ds_encrypt.h"
#include "ds_buffer.h"
/************************************************************************
......@@ -60,6 +59,7 @@ ds_create(const char *root, ds_type_t type)
ds = &datasink_compress;
break;
case DS_TYPE_ENCRYPT:
case DS_TYPE_DECRYPT:
msg("Error : mariabackup does not support encrypted backups.");
exit(EXIT_FAILURE);
break;
......
extra/mariabackup/datasink.h
View file @ ecb25df2
......@@ -61,6 +61,7 @@ typedef enum {
DS_TYPE_XBSTREAM,
DS_TYPE_COMPRESS,
DS_TYPE_ENCRYPT,
DS_TYPE_DECRYPT,
DS_TYPE_TMPFILE,
DS_TYPE_BUFFER
} ds_type_t;
......
extra/mariabackup/ds_decrypt.c 0 → 100644
View file @ ecb25df2
/******************************************************
Copyright (c) 2017 Percona LLC and/or its affiliates.
Encryption datasink implementation for XtraBackup.
This program is free software; you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation; version 2 of the License.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program; if not, write to the Free Software
Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA
*******************************************************/
#include <my_base.h>
#include "common.h"
#include "datasink.h"
#include "xbcrypt.h"
#include "xbcrypt_common.h"
#include "crc_glue.h"
typedef struct {
pthread_t id;
uint num;
pthread_mutex_t ctrl_mutex;
pthread_cond_t ctrl_cond;
pthread_mutex_t data_mutex;
pthread_cond_t data_cond;
my_bool started;
my_bool data_avail;
my_bool cancelled;
my_bool failed;
const uchar *from;
size_t from_len;
uchar *to;
size_t to_len;
size_t to_size;
const uchar *iv;
size_t iv_len;
unsigned long long offset;
my_bool hash_appended;
gcry_cipher_hd_t cipher_handle;
xb_rcrypt_result_t parse_result;
} crypt_thread_ctxt_t;
typedef struct {
crypt_thread_ctxt_t *threads;
uint nthreads;
int encrypt_algo;
size_t chunk_size;
char *encrypt_key;
char *encrypt_key_file;
} ds_decrypt_ctxt_t;
typedef struct {
ds_decrypt_ctxt_t *crypt_ctxt;
size_t bytes_processed;
ds_file_t *dest_file;
uchar *buf;
size_t buf_len;
size_t buf_size;
} ds_decrypt_file_t;
int ds_decrypt_encrypt_threads = 1;
static ds_ctxt_t *decrypt_init(const char *root);
static ds_file_t *decrypt_open(ds_ctxt_t *ctxt, const char *path,
MY_STAT *mystat);
static int decrypt_write(ds_file_t *file, const void *buf, size_t len);
static int decrypt_close(ds_file_t *file);
static void decrypt_deinit(ds_ctxt_t *ctxt);
datasink_t datasink_decrypt = {
&decrypt_init,
&decrypt_open,
&decrypt_write,
&decrypt_close,
&decrypt_deinit
};
static crypt_thread_ctxt_t *create_worker_threads(uint n);
static void destroy_worker_threads(crypt_thread_ctxt_t *threads, uint n);
static void *decrypt_worker_thread_func(void *arg);
static
ds_ctxt_t *
decrypt_init(const char *root)
{
ds_ctxt_t *ctxt;
ds_decrypt_ctxt_t *decrypt_ctxt;
crypt_thread_ctxt_t *threads;
if (xb_crypt_init(NULL)) {
return NULL;
}
/* Create and initialize the worker threads */
threads = create_worker_threads(ds_decrypt_encrypt_threads);
if (threads == NULL) {
msg("decrypt: failed to create worker threads.\n");
return NULL;
}
ctxt = (ds_ctxt_t *) my_malloc(sizeof(ds_ctxt_t) +
sizeof(ds_decrypt_ctxt_t),
MYF(MY_FAE));
decrypt_ctxt = (ds_decrypt_ctxt_t *) (ctxt + 1);
decrypt_ctxt->threads = threads;
decrypt_ctxt->nthreads = ds_decrypt_encrypt_threads;
ctxt->ptr = decrypt_ctxt;
ctxt->root = my_strdup(root, MYF(MY_FAE));
return ctxt;
}
static
ds_file_t *
decrypt_open(ds_ctxt_t *ctxt, const char *path, MY_STAT *mystat)
{
ds_ctxt_t *dest_ctxt;
ds_decrypt_ctxt_t *crypt_ctxt;
ds_decrypt_file_t *crypt_file;
char new_name[FN_REFLEN];
ds_file_t *file;
xb_ad(ctxt->pipe_ctxt != NULL);
dest_ctxt = ctxt->pipe_ctxt;
crypt_ctxt = (ds_decrypt_ctxt_t *) ctxt->ptr;
file = (ds_file_t *) my_malloc(sizeof(ds_file_t) +
sizeof(ds_decrypt_file_t),
MYF(MY_FAE|MY_ZEROFILL));
crypt_file = (ds_decrypt_file_t *) (file + 1);
/* Remove the .xbcrypt extension from the filename */
strncpy(new_name, path, FN_REFLEN);
new_name[strlen(new_name) - 8] = 0;
crypt_file->dest_file = ds_open(dest_ctxt, new_name, mystat);
if (crypt_file->dest_file == NULL) {
msg("decrypt: ds_open(\"%s\") failed.\n", new_name);
goto err;
}
crypt_file->crypt_ctxt = crypt_ctxt;
crypt_file->buf = NULL;
crypt_file->buf_size = 0;
crypt_file->buf_len = 0;
file->ptr = crypt_file;
file->path = crypt_file->dest_file->path;
return file;
err:
if (crypt_file->dest_file) {
ds_close(crypt_file->dest_file);
}
my_free(file);
return NULL;
}
#define CHECK_BUF_SIZE(ptr, size, buf, len) \
if (ptr + size - buf > (ssize_t) len) { \
result = XB_CRYPT_READ_INCOMPLETE; \
goto exit; \
}
static
xb_rcrypt_result_t
parse_xbcrypt_chunk(crypt_thread_ctxt_t *thd, const uchar *buf, size_t len,
size_t *bytes_processed)
{
const uchar *ptr;
uint version;
ulong checksum, checksum_exp;
ulonglong tmp;
xb_rcrypt_result_t result = XB_CRYPT_READ_CHUNK;
*bytes_processed = 0;
ptr = buf;
CHECK_BUF_SIZE(ptr, XB_CRYPT_CHUNK_MAGIC_SIZE, buf, len);
if (memcmp(ptr, XB_CRYPT_CHUNK_MAGIC3,
XB_CRYPT_CHUNK_MAGIC_SIZE) == 0) {
version = 3;
} else if (memcmp(ptr, XB_CRYPT_CHUNK_MAGIC2,
XB_CRYPT_CHUNK_MAGIC_SIZE) == 0) {
version = 2;
} else if (memcmp(ptr, XB_CRYPT_CHUNK_MAGIC1,
XB_CRYPT_CHUNK_MAGIC_SIZE) == 0) {
version = 1;
} else {
msg("%s:%s: wrong chunk magic at offset 0x%llx.\n",
my_progname, __FUNCTION__, thd->offset);
result = XB_CRYPT_READ_ERROR;
goto exit;
}
ptr += XB_CRYPT_CHUNK_MAGIC_SIZE;
thd->offset += XB_CRYPT_CHUNK_MAGIC_SIZE;
CHECK_BUF_SIZE(ptr, 8, buf, len);
tmp = uint8korr(ptr); /* reserved */
ptr += 8;
thd->offset += 8;
CHECK_BUF_SIZE(ptr, 8, buf, len);
tmp = uint8korr(ptr); /* original size */
ptr += 8;
if (tmp > INT_MAX) {
msg("%s:%s: invalid original size at offset 0x%llx.\n",
my_progname, __FUNCTION__, thd->offset);
result = XB_CRYPT_READ_ERROR;
goto exit;
}
thd->offset += 8;
thd->to_len = (size_t)tmp;
if (thd->to_size < thd->to_len + XB_CRYPT_HASH_LEN) {
thd->to = (uchar *) my_realloc(
thd->to,
thd->to_len + XB_CRYPT_HASH_LEN,
MYF(MY_FAE | MY_ALLOW_ZERO_PTR));
thd->to_size = thd->to_len;
}
CHECK_BUF_SIZE(ptr, 8, buf, len);
tmp = uint8korr(ptr); /* encrypted size */
ptr += 8;
if (tmp > INT_MAX) {
msg("%s:%s: invalid encrypted size at offset 0x%llx.\n",
my_progname, __FUNCTION__, thd->offset);
result = XB_CRYPT_READ_ERROR;
goto exit;
}
thd->offset += 8;
thd->from_len = (size_t)tmp;
xb_a(thd->from_len <= thd->to_len + XB_CRYPT_HASH_LEN);
CHECK_BUF_SIZE(ptr, 4, buf, len);
checksum_exp = uint4korr(ptr); /* checksum */
ptr += 4;
thd->offset += 4;
/* iv size */
if (version == 1) {
thd->iv_len = 0;
thd->iv = NULL;
} else {
CHECK_BUF_SIZE(ptr, 8, buf, len);
tmp = uint8korr(ptr);
if (tmp > INT_MAX) {
msg("%s:%s: invalid iv size at offset 0x%llx.\n",
my_progname, __FUNCTION__, thd->offset);
result = XB_CRYPT_READ_ERROR;
goto exit;
}
ptr += 8;
thd->offset += 8;
thd->iv_len = (size_t)tmp;
}
if (thd->iv_len > 0) {
CHECK_BUF_SIZE(ptr, thd->iv_len, buf, len);
thd->iv = ptr;
ptr += thd->iv_len;
}
/* for version euqals 2 we need to read in the iv data but do not init
CTR with it */
if (version == 2) {
thd->iv_len = 0;
thd->iv = 0;
}
if (thd->from_len > 0) {
CHECK_BUF_SIZE(ptr, thd->from_len, buf, len);
thd->from = ptr;
ptr += thd->from_len;
}
xb_ad(thd->from_len <= thd->to_len);
checksum = crc32_iso3309(0, thd->from, thd->from_len);
if (checksum != checksum_exp) {
msg("%s:%s invalid checksum at offset 0x%llx, "
"expected 0x%lx, actual 0x%lx.\n", my_progname,
__FUNCTION__, thd->offset, checksum_exp, checksum);
result = XB_CRYPT_READ_ERROR;
goto exit;
}
thd->offset += thd->from_len;
thd->hash_appended = version > 2;
exit:
*bytes_processed = (size_t) (ptr - buf);
return result;
}
static
int
decrypt_write(ds_file_t *file, const void *buf, size_t len)
{
ds_decrypt_file_t *crypt_file;
ds_decrypt_ctxt_t *crypt_ctxt;
crypt_thread_ctxt_t *threads;
crypt_thread_ctxt_t *thd;
uint nthreads;
uint i;
size_t bytes_processed;
xb_rcrypt_result_t parse_result = XB_CRYPT_READ_CHUNK;
my_bool err = FALSE;
crypt_file = (ds_decrypt_file_t *) file->ptr;
crypt_ctxt = crypt_file->crypt_ctxt;
threads = crypt_ctxt->threads;
nthreads = crypt_ctxt->nthreads;
if (crypt_file->buf_len > 0) {
thd = threads;
pthread_mutex_lock(&thd->ctrl_mutex);
do {
if (parse_result == XB_CRYPT_READ_INCOMPLETE) {
crypt_file->buf_size = crypt_file->buf_size * 2;
crypt_file->buf = (uchar *) my_realloc(
crypt_file->buf,
crypt_file->buf_size,
MYF(MY_FAE|MY_ALLOW_ZERO_PTR));
}
memcpy(crypt_file->buf + crypt_file->buf_len,
buf, MY_MIN(crypt_file->buf_size -
crypt_file->buf_len, len));
parse_result = parse_xbcrypt_chunk(
thd, crypt_file->buf,
crypt_file->buf_size, &bytes_processed);
if (parse_result == XB_CRYPT_READ_ERROR) {
pthread_mutex_unlock(&thd->ctrl_mutex);
return 1;
}
} while (parse_result == XB_CRYPT_READ_INCOMPLETE &&
crypt_file->buf_size < len);
if (parse_result != XB_CRYPT_READ_CHUNK) {
msg("decrypt: incomplete data.\n");
pthread_mutex_unlock(&thd->ctrl_mutex);
return 1;
}
pthread_mutex_lock(&thd->data_mutex);
thd->data_avail = TRUE;
pthread_cond_signal(&thd->data_cond);
pthread_mutex_unlock(&thd->data_mutex);
len -= bytes_processed - crypt_file->buf_len;
buf += bytes_processed - crypt_file->buf_len;
/* reap */
pthread_mutex_lock(&thd->data_mutex);
while (thd->data_avail == TRUE) {
pthread_cond_wait(&thd->data_cond,
&thd->data_mutex);
}
if (thd->failed) {
msg("decrypt: failed to decrypt chunk.\n");
err = TRUE;
}
xb_a(thd->to_len > 0);
if (!err &&
ds_write(crypt_file->dest_file, thd->to, thd->to_len)) {
msg("decrypt: write to destination failed.\n");
err = TRUE;
}
crypt_file->bytes_processed += thd->from_len;
pthread_mutex_unlock(&thd->data_mutex);
pthread_mutex_unlock(&thd->ctrl_mutex);
crypt_file->buf_len = 0;
if (err) {
return 1;
}
}
while (parse_result == XB_CRYPT_READ_CHUNK && len > 0) {
uint max_thread;
for (i = 0; i < nthreads; i++) {
thd = threads + i;
pthread_mutex_lock(&thd->ctrl_mutex);
parse_result = parse_xbcrypt_chunk(
thd, buf, len, &bytes_processed);
if (parse_result == XB_CRYPT_READ_ERROR) {
pthread_mutex_unlock(&thd->ctrl_mutex);
err = TRUE;
break;
}
thd->parse_result = parse_result;
if (parse_result != XB_CRYPT_READ_CHUNK) {
pthread_mutex_unlock(&thd->ctrl_mutex);
break;
}
pthread_mutex_lock(&thd->data_mutex);
thd->data_avail = TRUE;
pthread_cond_signal(&thd->data_cond);
pthread_mutex_unlock(&thd->data_mutex);
len -= bytes_processed;
buf += bytes_processed;
}
max_thread = (i < nthreads) ? i : nthreads - 1;
/* Reap and write decrypted data */
for (i = 0; i <= max_thread; i++) {
thd = threads + i;
if (thd->parse_result != XB_CRYPT_READ_CHUNK) {
break;
}
pthread_mutex_lock(&thd->data_mutex);
while (thd->data_avail == TRUE) {
pthread_cond_wait(&thd->data_cond,
&thd->data_mutex);
}
if (thd->failed) {
msg("decrypt: failed to decrypt chunk.\n");
err = TRUE;
}
xb_a(thd->to_len > 0);
if (!err && ds_write(crypt_file->dest_file, thd->to,
thd->to_len)) {
msg("decrypt: write to destination failed.\n");
err = TRUE;
}
crypt_file->bytes_processed += thd->from_len;
pthread_mutex_unlock(&thd->data_mutex);
pthread_mutex_unlock(&thd->ctrl_mutex);
}
if (err) {
return 1;
}
}
if (parse_result == XB_CRYPT_READ_INCOMPLETE && len > 0) {
crypt_file->buf_len = len;
if (crypt_file->buf_size < len) {
crypt_file->buf = (uchar *) my_realloc(
crypt_file->buf,
crypt_file->buf_len,
MYF(MY_FAE | MY_ALLOW_ZERO_PTR));
crypt_file->buf_size = len;
}
memcpy(crypt_file->buf, buf, len);
}
return 0;
}
static
int
decrypt_close(ds_file_t *file)
{
ds_decrypt_file_t *crypt_file;
ds_file_t *dest_file;
int rc = 0;
crypt_file = (ds_decrypt_file_t *) file->ptr;
dest_file = crypt_file->dest_file;
if (ds_close(dest_file)) {
rc = 1;
}
my_free(crypt_file->buf);
my_free(file);
return rc;
}
static
void
decrypt_deinit(ds_ctxt_t *ctxt)
{
ds_decrypt_ctxt_t *crypt_ctxt;
xb_ad(ctxt->pipe_ctxt != NULL);
crypt_ctxt = (ds_decrypt_ctxt_t *) ctxt->ptr;
destroy_worker_threads(crypt_ctxt->threads, crypt_ctxt->nthreads);
my_free(ctxt->root);
my_free(ctxt);
}
static
crypt_thread_ctxt_t *
create_worker_threads(uint n)
{
crypt_thread_ctxt_t *threads;
uint i;
threads = (crypt_thread_ctxt_t *)
my_malloc(sizeof(crypt_thread_ctxt_t) * n,
MYF(MY_FAE | MY_ZEROFILL));
for (i = 0; i < n; i++) {
crypt_thread_ctxt_t *thd = threads + i;
thd->num = i + 1;
/* Initialize the control mutex and condition var */
if (pthread_mutex_init(&thd->ctrl_mutex, NULL) ||
pthread_cond_init(&thd->ctrl_cond, NULL)) {
goto err;
}
/* Initialize and data mutex and condition var */
if (pthread_mutex_init(&thd->data_mutex, NULL) ||
pthread_cond_init(&thd->data_cond, NULL)) {
goto err;
}
xb_crypt_cipher_open(&thd->cipher_handle);
pthread_mutex_lock(&thd->ctrl_mutex);
if (pthread_create(&thd->id, NULL, decrypt_worker_thread_func,
thd)) {
msg("decrypt: pthread_create() failed: "
"errno = %d\n", errno);
goto err;
}
}
/* Wait for the threads to start */
for (i = 0; i < n; i++) {
crypt_thread_ctxt_t *thd = threads + i;
while (thd->started == FALSE)
pthread_cond_wait(&thd->ctrl_cond, &thd->ctrl_mutex);
pthread_mutex_unlock(&thd->ctrl_mutex);
}
return threads;
err:
return NULL;
}
static
void
destroy_worker_threads(crypt_thread_ctxt_t *threads, uint n)
{
uint i;
for (i = 0; i < n; i++) {
crypt_thread_ctxt_t *thd = threads + i;
pthread_mutex_lock(&thd->data_mutex);
threads[i].cancelled = TRUE;
pthread_cond_signal(&thd->data_cond);
pthread_mutex_unlock(&thd->data_mutex);
pthread_join(thd->id, NULL);
pthread_cond_destroy(&thd->data_cond);
pthread_mutex_destroy(&thd->data_mutex);
pthread_cond_destroy(&thd->ctrl_cond);
pthread_mutex_destroy(&thd->ctrl_mutex);
xb_crypt_cipher_close(thd->cipher_handle);
my_free(thd->to);
}
my_free(threads);
}
static
void *
decrypt_worker_thread_func(void *arg)
{
crypt_thread_ctxt_t *thd = (crypt_thread_ctxt_t *) arg;
pthread_mutex_lock(&thd->ctrl_mutex);
pthread_mutex_lock(&thd->data_mutex);
thd->started = TRUE;
pthread_cond_signal(&thd->ctrl_cond);
pthread_mutex_unlock(&thd->ctrl_mutex);
while (1) {
thd->data_avail = FALSE;
pthread_cond_signal(&thd->data_cond);
while (!thd->data_avail && !thd->cancelled) {
pthread_cond_wait(&thd->data_cond, &thd->data_mutex);
}
if (thd->cancelled)
break;
if (xb_crypt_decrypt(thd->cipher_handle, thd->from,
thd->from_len, thd->to, &thd->to_len,
thd->iv, thd->iv_len,
thd->hash_appended)) {
thd->failed = TRUE;
continue;
}
}
pthread_mutex_unlock(&thd->data_mutex);
return NULL;
}
extra/mariabackup/ds_decrypt.h 0 → 100644
View file @ ecb25df2
/******************************************************
Copyright (c) 2017 Percona LLC and/or its affiliates.
Encryption interface for XtraBackup.
This program is free software; you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation; version 2 of the License.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program; if not, write to the Free Software
Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA
*******************************************************/
#ifndef DS_DECRYPT_H
#define DS_DECRYPT_H
#include "datasink.h"
extern datasink_t datasink_decrypt;
extern int ds_decrypt_encrypt_threads;
#endif
extra/mariabackup/ds_encrypt.c
View file @ ecb25df2
......@@ -22,25 +22,11 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA
#include <my_base.h>
#include "common.h"
#include "datasink.h"
#include "xbcrypt_common.h"
#ifdef HAVE_GRYPT
#if GCC_VERSION >= 4002
/* Workaround to avoid "gcry_ac_* is deprecated" warnings in gcrypt.h */
# pragma GCC diagnostic ignored "-Wdeprecated-declarations"
#endif
#include <gcrypt.h>
#if GCC_VERSION >= 4002
# pragma GCC diagnostic warning "-Wdeprecated-declarations"
#endif
#include "xbcrypt.h"
#if !defined(GCRYPT_VERSION_NUMBER) || (GCRYPT_VERSION_NUMBER < 0x010600)
GCRY_THREAD_OPTION_PTHREAD_IMPL;
#endif
#define XB_CRYPT_CHUNK_SIZE ((size_t) (xtrabackup_encrypt_chunk_size))
#define XB_CRYPT_CHUNK_SIZE ((size_t) (ds_encrypt_encrypt_chunk_size))
typedef struct {
pthread_t id;
......@@ -52,10 +38,10 @@ typedef struct {
my_bool started;
my_bool data_avail;
my_bool cancelled;
const char *from;
const uchar *from;
size_t from_len;
char *to;
char *iv;
uchar *to;
uchar *iv;
size_t to_len;
gcry_cipher_hd_t cipher_handle;
} crypt_thread_ctxt_t;
......@@ -73,11 +59,8 @@ typedef struct {
} ds_encrypt_file_t;
/* Encryption options */
extern ulong xtrabackup_encrypt_algo;
extern char *xtrabackup_encrypt_key;
extern char *xtrabackup_encrypt_key_file;
extern uint xtrabackup_encrypt_threads;
extern ulonglong xtrabackup_encrypt_chunk_size;
uint ds_encrypt_encrypt_threads;
ulonglong ds_encrypt_encrypt_chunk_size;
static ds_ctxt_t *encrypt_init(const char *root);
static ds_file_t *encrypt_open(ds_ctxt_t *ctxt, const char *path,
......@@ -98,12 +81,7 @@ static crypt_thread_ctxt_t *create_worker_threads(uint n);
static void destroy_worker_threads(crypt_thread_ctxt_t *threads, uint n);
static void *encrypt_worker_thread_func(void *arg);
static uint encrypt_algos[] = { GCRY_CIPHER_NONE, GCRY_CIPHER_AES128,
GCRY_CIPHER_AES192, GCRY_CIPHER_AES256 };
static uint encrypt_algo;
static const uint encrypt_mode = GCRY_CIPHER_MODE_CTR;
static uint encrypt_key_len = 0;
static size_t encrypt_iv_len = 0;
static uint encrypt_iv_len = 0;
static
ssize_t
......@@ -129,87 +107,13 @@ encrypt_init(const char *root)
ds_ctxt_t *ctxt;
ds_encrypt_ctxt_t *encrypt_ctxt;
crypt_thread_ctxt_t *threads;
gcry_error_t gcry_error;
/* Acording to gcrypt docs (and my testing), setting up the threading
callbacks must be done first, so, lets give it a shot */
#if !defined(GCRYPT_VERSION_NUMBER) || (GCRYPT_VERSION_NUMBER < 0x010600)
gcry_error = gcry_control(GCRYCTL_SET_THREAD_CBS, &gcry_threads_pthread);
if (gcry_error) {
msg("encrypt: unable to set libgcrypt thread cbs - "
"%s : %s\n",
gcry_strsource(gcry_error),
gcry_strerror(gcry_error));
return NULL;
}
#endif
/* Version check should be the very next call because it
makes sure that important subsystems are intialized. */
if (!gcry_control(GCRYCTL_ANY_INITIALIZATION_P)) {
const char *gcrypt_version;
gcrypt_version = gcry_check_version(NULL);
/* No other library has already initialized libgcrypt. */
if (!gcrypt_version) {
msg("encrypt: failed to initialize libgcrypt\n");
return NULL;
} else {
msg("encrypt: using gcrypt %s\n", gcrypt_version);
}
}
/* Disable the gcry secure memory, not dealing with this for now */
gcry_error = gcry_control(GCRYCTL_DISABLE_SECMEM, 0);
if (gcry_error) {
msg("encrypt: unable to disable libgcrypt secmem - "
"%s : %s\n",
gcry_strsource(gcry_error),
gcry_strerror(gcry_error));
return NULL;
}
/* Finalize gcry initialization. */
gcry_error = gcry_control(GCRYCTL_INITIALIZATION_FINISHED, 0);
if (gcry_error) {
msg("encrypt: unable to finish libgcrypt initialization - "
"%s : %s\n",
gcry_strsource(gcry_error),
gcry_strerror(gcry_error));
return NULL;
}
/* Determine the algorithm */
encrypt_algo = encrypt_algos[xtrabackup_encrypt_algo];
/* Set up the iv length */
encrypt_iv_len = gcry_cipher_get_algo_blklen(encrypt_algo);
xb_a(encrypt_iv_len > 0);
/* Now set up the key */
if (xtrabackup_encrypt_key == NULL &&
xtrabackup_encrypt_key_file == NULL) {
msg("encrypt: no encryption key or key file specified.\n");
return NULL;
} else if (xtrabackup_encrypt_key && xtrabackup_encrypt_key_file) {
msg("encrypt: both encryption key and key file specified.\n");
return NULL;
} else if (xtrabackup_encrypt_key_file) {
if (!xb_crypt_read_key_file(xtrabackup_encrypt_key_file,
(void**)&xtrabackup_encrypt_key,
&encrypt_key_len)) {
msg("encrypt: unable to read encryption key file"
" \"%s\".\n", xtrabackup_encrypt_key_file);
return NULL;
}
} else if (xtrabackup_encrypt_key) {
encrypt_key_len = strlen(xtrabackup_encrypt_key);
} else {
msg("encrypt: no encryption key or key file specified.\n");
if (xb_crypt_init(&encrypt_iv_len)) {
return NULL;
}
/* Create and initialize the worker threads */
threads = create_worker_threads(xtrabackup_encrypt_threads);
threads = create_worker_threads(ds_encrypt_encrypt_threads);
if (threads == NULL) {
msg("encrypt: failed to create worker threads.\n");
return NULL;
......@@ -221,7 +125,7 @@ encrypt_init(const char *root)
encrypt_ctxt = (ds_encrypt_ctxt_t *) (ctxt + 1);
encrypt_ctxt->threads = threads;
encrypt_ctxt->nthreads = xtrabackup_encrypt_threads;
encrypt_ctxt->nthreads = ds_encrypt_encrypt_threads;
ctxt->ptr = encrypt_ctxt;
ctxt->root = my_strdup(root, MYF(MY_FAE));
......@@ -294,7 +198,7 @@ encrypt_write(ds_file_t *file, const void *buf, size_t len)
crypt_thread_ctxt_t *thd;
uint nthreads;
uint i;
const char *ptr;
const uchar *ptr;
crypt_file = (ds_encrypt_file_t *) file->ptr;
crypt_ctxt = crypt_file->crypt_ctxt;
......@@ -302,7 +206,7 @@ encrypt_write(ds_file_t *file, const void *buf, size_t len)
threads = crypt_ctxt->threads;
nthreads = crypt_ctxt->nthreads;
ptr = (const char *) buf;
ptr = (const uchar *) buf;
while (len > 0) {
uint max_thread;
......@@ -403,10 +307,6 @@ encrypt_deinit(ds_ctxt_t *ctxt)
my_free(ctxt->root);
my_free(ctxt);
if (xtrabackup_encrypt_key)
my_free(xtrabackup_encrypt_key);
if (xtrabackup_encrypt_key_file)
my_free(xtrabackup_encrypt_key_file);
}
static
......@@ -427,11 +327,10 @@ create_worker_threads(uint n)
thd->cancelled = FALSE;
thd->data_avail = FALSE;
thd->to = (char *) my_malloc(XB_CRYPT_CHUNK_SIZE +
thd->to = (uchar *) my_malloc(XB_CRYPT_CHUNK_SIZE +
XB_CRYPT_HASH_LEN, MYF(MY_FAE));
thd->iv = (char *) my_malloc(encrypt_iv_len,
MYF(MY_FAE));
thd->iv = (uchar *) my_malloc(encrypt_iv_len, MYF(MY_FAE));
/* Initialize the control mutex and condition var */
if (pthread_mutex_init(&thd->ctrl_mutex, NULL) ||
......@@ -445,34 +344,10 @@ create_worker_threads(uint n)
goto err;
}
if (encrypt_algo != GCRY_CIPHER_NONE) {
gcry_error_t gcry_error;
gcry_error = gcry_cipher_open(&thd->cipher_handle,
encrypt_algo,
encrypt_mode, 0);
if (gcry_error) {
msg("encrypt: unable to open libgcrypt"
" cipher - %s : %s\n",
gcry_strsource(gcry_error),
gcry_strerror(gcry_error));
gcry_cipher_close(thd->cipher_handle);
if (xb_crypt_cipher_open(&thd->cipher_handle)) {
goto err;
}
gcry_error = gcry_cipher_setkey(thd->cipher_handle,
xtrabackup_encrypt_key,
encrypt_key_len);
if (gcry_error) {
msg("encrypt: unable to set libgcrypt"
" cipher key - %s : %s\n",
gcry_strsource(gcry_error),
gcry_strerror(gcry_error));
gcry_cipher_close(thd->cipher_handle);
goto err;
}
}
pthread_mutex_lock(&thd->ctrl_mutex);
if (pthread_create(&thd->id, NULL, encrypt_worker_thread_func,
......@@ -519,8 +394,7 @@ destroy_worker_threads(crypt_thread_ctxt_t *threads, uint n)
pthread_cond_destroy(&thd->ctrl_cond);
pthread_mutex_destroy(&thd->ctrl_mutex);
if (encrypt_algo != GCRY_CIPHER_NONE)
gcry_cipher_close(thd->cipher_handle);
xb_crypt_cipher_close(thd->cipher_handle);
my_free(thd->to);
my_free(thd->iv);
......@@ -555,60 +429,14 @@ encrypt_worker_thread_func(void *arg)
if (thd->cancelled)
break;
/* ensure that XB_CRYPT_HASH_LEN is the correct length
of XB_CRYPT_HASH hashing algorithm output */
assert(gcry_md_get_algo_dlen(XB_CRYPT_HASH) ==
XB_CRYPT_HASH_LEN);
memcpy(thd->to, thd->from, thd->from_len);
gcry_md_hash_buffer(XB_CRYPT_HASH, thd->to + thd->from_len,
thd->from, thd->from_len);
thd->to_len = thd->from_len;
if (encrypt_algo != GCRY_CIPHER_NONE) {
gcry_error_t gcry_error;
gcry_error = gcry_cipher_reset(thd->cipher_handle);
if (gcry_error) {
msg("encrypt: unable to reset cipher - "
"%s : %s\n",
gcry_strsource(gcry_error),
gcry_strerror(gcry_error));
if (xb_crypt_encrypt(thd->cipher_handle, thd->from,
thd->from_len, thd->to, &thd->to_len,
thd->iv)) {
thd->to_len = 0;
continue;
}
xb_crypt_create_iv(thd->iv, encrypt_iv_len);
gcry_error = gcry_cipher_setctr(thd->cipher_handle,
thd->iv,
encrypt_iv_len);
if (gcry_error) {
msg("encrypt: unable to set cipher ctr - "
"%s : %s\n",
gcry_strsource(gcry_error),
gcry_strerror(gcry_error));
thd->to_len = 0;
continue;
}
gcry_error = gcry_cipher_encrypt(thd->cipher_handle,
thd->to,
thd->to_len +
XB_CRYPT_HASH_LEN,
thd->to,
thd->from_len +
XB_CRYPT_HASH_LEN);
if (gcry_error) {
msg("encrypt: unable to encrypt buffer - "
"%s : %s\n", gcry_strsource(gcry_error),
gcry_strerror(gcry_error));
thd->to_len = 0;
}
} else {
memcpy(thd->to, thd->from,
thd->from_len + XB_CRYPT_HASH_LEN);
}
thd->to_len += XB_CRYPT_HASH_LEN;
}
pthread_mutex_unlock(&thd->data_mutex);
......
extra/mariabackup/ds_encrypt.h
View file @ ecb25df2
......@@ -25,4 +25,9 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA
#ifdef HAVE_GCRYPT
extern datasink_t datasink_encrypt;
#endif
/* Encryption options */
extern uint ds_encrypt_encrypt_threads;
extern ulonglong ds_encrypt_encrypt_chunk_size;
#endif
extra/mariabackup/xbcrypt.c
View file @ ecb25df2
......@@ -22,7 +22,8 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA
#include <my_getopt.h>
#include "common.h"
#include "xbcrypt.h"
#include <gcrypt.h>
#include "xbcrypt_common.h"
#include "crc_glue.h"
#if !defined(GCRYPT_VERSION_NUMBER) || (GCRYPT_VERSION_NUMBER < 0x010600)
GCRY_THREAD_OPTION_PTHREAD_IMPL;
......@@ -138,6 +139,8 @@ main(int argc, char **argv)
MY_INIT(argv[0]);
crc_init();
if (get_options(&argc, &argv)) {
goto err;
}
......@@ -402,7 +405,7 @@ mode_decrypt(File filein, File fileout)
/* ensure that XB_CRYPT_HASH_LEN is the correct length
of XB_CRYPT_HASH hashing algorithm output */
assert(gcry_md_get_algo_dlen(XB_CRYPT_HASH) ==
xb_a(gcry_md_get_algo_dlen(XB_CRYPT_HASH) ==
XB_CRYPT_HASH_LEN);
gcry_md_hash_buffer(XB_CRYPT_HASH, hash, decryptbuf,
originalsize);
......@@ -529,8 +532,7 @@ mode_encrypt(File filein, File fileout)
/* ensure that XB_CRYPT_HASH_LEN is the correct length
of XB_CRYPT_HASH hashing algorithm output */
assert(XB_CRYPT_HASH_LEN ==
gcry_md_get_algo_dlen(XB_CRYPT_HASH));
xb_a(XB_CRYPT_HASH_LEN == gcry_md_get_algo_dlen(XB_CRYPT_HASH));
gcry_md_hash_buffer(XB_CRYPT_HASH, chunkbuf + bytesread,
chunkbuf, bytesread);
......
extra/mariabackup/xbcrypt.h
View file @ ecb25df2
......@@ -65,6 +65,7 @@ xb_rcrypt_t *xb_crypt_read_open(void *userdata,
typedef enum {
XB_CRYPT_READ_CHUNK,
XB_CRYPT_READ_INCOMPLETE,
XB_CRYPT_READ_EOF,
XB_CRYPT_READ_ERROR
} xb_rcrypt_result_t;
......@@ -75,10 +76,4 @@ xb_rcrypt_result_t xb_crypt_read_chunk(xb_rcrypt_t *crypt, void **buf,
int xb_crypt_read_close(xb_rcrypt_t *crypt);
/******************************************************************************
Utility interface */
my_bool xb_crypt_read_key_file(const char *filename,
void** key, uint *keylength);
void xb_crypt_create_iv(void* ivbuf, size_t ivlen);
#endif
extra/mariabackup/xbcrypt_common.c
View file @ ecb25df2
/******************************************************
Copyright (c) 2013 Percona LLC and/or its affiliates.
Copyright (c) 2013, 2017 Percona LLC and/or its affiliates.
Encryption configuration file interface for XtraBackup.
......@@ -21,19 +21,27 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA
#include <my_base.h>
#include "common.h"
#include "xbcrypt.h"
#include "xbcrypt_common.h"
#if GCC_VERSION >= 4002
/* Workaround to avoid "gcry_ac_* is deprecated" warnings in gcrypt.h */
# pragma GCC diagnostic ignored "-Wdeprecated-declarations"
#endif
/* Encryption options */
char *ds_encrypt_key = NULL;
char *ds_encrypt_key_file = NULL;
ulong ds_encrypt_algo;
static uint encrypt_key_len;
static uint encrypt_iv_len;
static const uint encrypt_mode = GCRY_CIPHER_MODE_CTR;
#ifdef HAVE_GRYPT
#include <gcrypt.h>
static uint encrypt_algos[] = { GCRY_CIPHER_NONE, GCRY_CIPHER_AES128,
GCRY_CIPHER_AES192, GCRY_CIPHER_AES256 };
static uint encrypt_algo;
#if GCC_VERSION >= 4002
# pragma GCC diagnostic warning "-Wdeprecated-declarations"
#if !defined(GCRYPT_VERSION_NUMBER) || (GCRYPT_VERSION_NUMBER < 0x010600)
GCRY_THREAD_OPTION_PTHREAD_IMPL;
#endif
my_bool
xb_crypt_read_key_file(const char *filename, void** key, uint *keylength)
{
......@@ -59,4 +67,262 @@ xb_crypt_create_iv(void* ivbuf, size_t ivlen)
{
gcry_create_nonce(ivbuf, ivlen);
}
gcry_error_t
xb_crypt_init(uint *iv_len)
{
gcry_error_t gcry_error;
/* Acording to gcrypt docs (and my testing), setting up the threading
callbacks must be done first, so, lets give it a shot */
#if !defined(GCRYPT_VERSION_NUMBER) || (GCRYPT_VERSION_NUMBER < 0x010600)
gcry_error = gcry_control(GCRYCTL_SET_THREAD_CBS, &gcry_threads_pthread);
if (gcry_error) {
msg("encryption: unable to set libgcrypt thread cbs - "
"%s : %s\n",
gcry_strsource(gcry_error),
gcry_strerror(gcry_error));
return gcry_error;
}
#endif
/* Version check should be the very next call because it
makes sure that important subsystems are intialized. */
if (!gcry_control(GCRYCTL_ANY_INITIALIZATION_P)) {
const char *gcrypt_version;
gcrypt_version = gcry_check_version(NULL);
/* No other library has already initialized libgcrypt. */
if (!gcrypt_version) {
msg("encryption: failed to initialize libgcrypt\n");
return 1;
} else {
msg("encryption: using gcrypt %s\n", gcrypt_version);
}
}
/* Disable the gcry secure memory, not dealing with this for now */
gcry_error = gcry_control(GCRYCTL_DISABLE_SECMEM, 0);
if (gcry_error) {
msg("encryption: unable to disable libgcrypt secmem - "
"%s : %s\n",
gcry_strsource(gcry_error),
gcry_strerror(gcry_error));
return gcry_error;
}
/* Finalize gcry initialization. */
gcry_error = gcry_control(GCRYCTL_INITIALIZATION_FINISHED, 0);
if (gcry_error) {
msg("encryption: unable to finish libgcrypt initialization - "
"%s : %s\n",
gcry_strsource(gcry_error),
gcry_strerror(gcry_error));
return gcry_error;
}
/* Determine the algorithm */
encrypt_algo = encrypt_algos[ds_encrypt_algo];
/* Set up the iv length */
encrypt_iv_len = gcry_cipher_get_algo_blklen(encrypt_algo);
xb_a(encrypt_iv_len > 0);
if (iv_len != NULL) {
*iv_len = encrypt_iv_len;
}
/* Now set up the key */
if (ds_encrypt_key == NULL &&
ds_encrypt_key_file == NULL) {
msg("encryption: no encryption key or key file specified.\n");
return gcry_error;
} else if (ds_encrypt_key && ds_encrypt_key_file) {
msg("encryption: both encryption key and key file specified.\n");
return gcry_error;
} else if (ds_encrypt_key_file) {
if (!xb_crypt_read_key_file(ds_encrypt_key_file,
(void**)&ds_encrypt_key,
&encrypt_key_len)) {
msg("encryption: unable to read encryption key file"
" \"%s\".\n", ds_encrypt_key_file);
return gcry_error;
}
} else if (ds_encrypt_key) {
encrypt_key_len = strlen(ds_encrypt_key);
} else {
msg("encryption: no encryption key or key file specified.\n");
return gcry_error;
}
return 0;
}
gcry_error_t
xb_crypt_cipher_open(gcry_cipher_hd_t *cipher_handle)
{
if (encrypt_algo != GCRY_CIPHER_NONE) {
gcry_error_t gcry_error;
gcry_error = gcry_cipher_open(cipher_handle,
encrypt_algo,
encrypt_mode, 0);
if (gcry_error) {
msg("encryption: unable to open libgcrypt"
" cipher - %s : %s\n",
gcry_strsource(gcry_error),
gcry_strerror(gcry_error));
gcry_cipher_close(*cipher_handle);
return gcry_error;
}
gcry_error = gcry_cipher_setkey(*cipher_handle,
ds_encrypt_key,
encrypt_key_len);
if (gcry_error) {
msg("encryption: unable to set libgcrypt"
" cipher key - %s : %s\n",
gcry_strsource(gcry_error),
gcry_strerror(gcry_error));
gcry_cipher_close(*cipher_handle);
return gcry_error;
}
return gcry_error;
}
return 0;
}
void
xb_crypt_cipher_close(gcry_cipher_hd_t cipher_handle)
{
if (encrypt_algo != GCRY_CIPHER_NONE)
gcry_cipher_close(cipher_handle);
}
gcry_error_t
xb_crypt_decrypt(gcry_cipher_hd_t cipher_handle, const uchar *from,
size_t from_len, uchar *to, size_t *to_len,
const uchar *iv, size_t iv_len, my_bool hash_appended)
{
*to_len = from_len;
if (encrypt_algo != GCRY_CIPHER_NONE) {
gcry_error_t gcry_error;
gcry_error = gcry_cipher_reset(cipher_handle);
if (gcry_error) {
msg("%s:encryption: unable to reset libgcrypt"
" cipher - %s : %s\n", my_progname,
gcry_strsource(gcry_error),
gcry_strerror(gcry_error));
return gcry_error;
}
if (iv_len > 0) {
gcry_error = gcry_cipher_setctr(cipher_handle,
iv, iv_len);
}
if (gcry_error) {
msg("%s:encryption: unable to set cipher iv - "
"%s : %s\n", my_progname,
gcry_strsource(gcry_error),
gcry_strerror(gcry_error));
return gcry_error;
}
/* Try to decrypt it */
gcry_error = gcry_cipher_decrypt(cipher_handle, to, *to_len,
from, from_len);
if (gcry_error) {
msg("%s:encryption: unable to decrypt chunk - "
"%s : %s\n", my_progname,
gcry_strsource(gcry_error),
gcry_strerror(gcry_error));
gcry_cipher_close(cipher_handle);
return gcry_error;
}
if (hash_appended) {
uchar hash[XB_CRYPT_HASH_LEN];
*to_len -= XB_CRYPT_HASH_LEN;
/* ensure that XB_CRYPT_HASH_LEN is the correct length
of XB_CRYPT_HASH hashing algorithm output */
xb_ad(gcry_md_get_algo_dlen(XB_CRYPT_HASH) ==
XB_CRYPT_HASH_LEN);
gcry_md_hash_buffer(XB_CRYPT_HASH, hash, to,
*to_len);
if (memcmp(hash, (char *) to + *to_len,
XB_CRYPT_HASH_LEN) != 0) {
msg("%s:%s invalid plaintext hash. "
"Wrong encrytion key specified?\n",
my_progname, __FUNCTION__);
return 1;
}
}
} else {
memcpy(to, from, *to_len);
}
return 0;
}
gcry_error_t
xb_crypt_encrypt(gcry_cipher_hd_t cipher_handle, const uchar *from,
size_t from_len, uchar *to, size_t *to_len, uchar *iv)
{
gcry_error_t gcry_error;
/* ensure that XB_CRYPT_HASH_LEN is the correct length
of XB_CRYPT_HASH hashing algorithm output */
xb_ad(gcry_md_get_algo_dlen(XB_CRYPT_HASH) ==
XB_CRYPT_HASH_LEN);
memcpy(to, from, from_len);
gcry_md_hash_buffer(XB_CRYPT_HASH, to + from_len,
from, from_len);
*to_len = from_len;
if (encrypt_algo != GCRY_CIPHER_NONE) {
gcry_error = gcry_cipher_reset(cipher_handle);
if (gcry_error) {
msg("encrypt: unable to reset cipher - "
"%s : %s\n",
gcry_strsource(gcry_error),
gcry_strerror(gcry_error));
return gcry_error;
}
xb_crypt_create_iv(iv, encrypt_iv_len);
gcry_error = gcry_cipher_setctr(cipher_handle, iv,
encrypt_iv_len);
if (gcry_error) {
msg("encrypt: unable to set cipher ctr - "
"%s : %s\n",
gcry_strsource(gcry_error),
gcry_strerror(gcry_error));
return gcry_error;
}
gcry_error = gcry_cipher_encrypt(cipher_handle, to,
*to_len + XB_CRYPT_HASH_LEN,
to,
from_len + XB_CRYPT_HASH_LEN);
if (gcry_error) {
msg("encrypt: unable to encrypt buffer - "
"%s : %s\n", gcry_strsource(gcry_error),
gcry_strerror(gcry_error));
return gcry_error;
}
} else {
memcpy(to, from, from_len + XB_CRYPT_HASH_LEN);
}
*to_len += XB_CRYPT_HASH_LEN;
return 0;
}
#endif
\ No newline at end of file
extra/mariabackup/xbcrypt_common.h 0 → 100644
View file @ ecb25df2
/******************************************************
Copyright (c) 2017 Percona LLC and/or its affiliates.
Encryption datasink implementation for XtraBackup.
This program is free software; you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation; version 2 of the License.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program; if not, write to the Free Software
Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA
*******************************************************/
#include <my_base.h>
#if HAVE_GCRYPT
#if GCC_VERSION >= 4002
/* Workaround to avoid "gcry_ac_* is deprecated" warnings in gcrypt.h */
# pragma GCC diagnostic ignored "-Wdeprecated-declarations"
#endif
#include <gcrypt.h>
extern char *ds_encrypt_key;
extern char *ds_encrypt_key_file;
extern int ds_encrypt_threads;
extern ulong ds_encrypt_algo;
/******************************************************************************
Utility interface */
my_bool xb_crypt_read_key_file(const char *filename,
void** key, uint *keylength);
void xb_crypt_create_iv(void* ivbuf, size_t ivlen);
/* Initialize gcrypt and setup encryption key and IV lengths */
gcry_error_t
xb_crypt_init(uint *iv_len);
/* Setup gcrypt cipher */
gcry_error_t
xb_crypt_cipher_open(gcry_cipher_hd_t *cipher_handle);
/* Close gcrypt cipher */
void
xb_crypt_cipher_close(gcry_cipher_hd_t cipher_handle);
/* Decrypt buffer */
gcry_error_t
xb_crypt_decrypt(gcry_cipher_hd_t cipher_handle, const uchar *from,
size_t from_len, uchar *to, size_t *to_len, const uchar *iv,
size_t iv_len, my_bool hash_appended);
/* Encrypt buffer */
gcry_error_t
xb_crypt_encrypt(gcry_cipher_hd_t cipher_handle, const uchar *from,
size_t from_len, uchar *to, size_t *to_len, uchar *iv);
#endif
extra/mariabackup/xbcrypt_read.c
View file @ ecb25df2
......@@ -19,6 +19,7 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA
*******************************************************/
#include "xbcrypt.h"
#include "crc_glue.h"
struct xb_rcrypt_struct {
void *userdata;
......@@ -212,7 +213,7 @@ xb_crypt_read_chunk(xb_rcrypt_t *crypt, void **buf, size_t *olen, size_t *elen,
}
}
checksum = crc32(0, crypt->buffer, *elen);
checksum = crc32_iso3309(0, crypt->buffer, *elen);
if (checksum != checksum_exp) {
msg("%s:%s invalid checksum at offset 0x%llx, "
"expected 0x%lx, actual 0x%lx.\n", my_progname, __FUNCTION__,
......
extra/mariabackup/xbcrypt_write.c
View file @ ecb25df2
......@@ -19,6 +19,7 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA
*******************************************************/
#include "xbcrypt.h"
#include "crc_glue.h"
struct xb_wcrypt_struct {
void *userdata;
......@@ -73,7 +74,7 @@ int xb_crypt_write_chunk(xb_wcrypt_t *crypt, const void *buf, size_t olen,
int8store(ptr, (ulonglong)elen); /* encrypted (actual) size */
ptr += 8;
checksum = crc32(0, buf, (uint)elen);
checksum = crc32_iso3309(0, buf, elen);
int4store(ptr, checksum); /* checksum */
ptr += 4;
......
extra/mariabackup/xbstream.c
View file @ ecb25df2
......@@ -22,10 +22,13 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA
#include <my_base.h>
#include <my_getopt.h>
#include <hash.h>
#include <my_pthread.h>
#include "common.h"
#include "xbstream.h"
#include "ds_local.h"
#include "ds_stdout.h"
#include "xbcrypt_common.h"
#include "datasink.h"
#include "ds_decrypt.h"
#include "crc_glue.h"
#define XBSTREAM_VERSION "1.0"
#define XBSTREAM_BUFFER_SIZE (10 * 1024 * 1024UL)
......@@ -38,6 +41,12 @@ typedef enum {
RUN_MODE_EXTRACT
} run_mode_t;
const char *xbstream_encrypt_algo_names[] =
{ "NONE", "AES128", "AES192", "AES256", NullS};
TYPELIB xbstream_encrypt_algo_typelib=
{array_elements(xbstream_encrypt_algo_names)-1,"",
xbstream_encrypt_algo_names, NULL};
/* Need the following definitions to avoid linking with ds_*.o and their link
dependencies */
datasink_t datasink_archive;
......@@ -50,6 +59,15 @@ datasink_t datasink_buffer;
static run_mode_t opt_mode;
static char * opt_directory = NULL;
static my_bool opt_verbose = 0;
static int opt_parallel = 1;
static ulong opt_encrypt_algo;
static char *opt_encrypt_key_file = NULL;
static void *opt_encrypt_key = NULL;
static int opt_encrypt_threads = 1;
enum {
OPT_ENCRYPT_THREADS = 256
};
static struct my_option my_long_options[] =
{
......@@ -65,21 +83,46 @@ static struct my_option my_long_options[] =
GET_STR_ALLOC, REQUIRED_ARG, 0, 0, 0, 0, 0, 0},
{"verbose", 'v', "Print verbose output.", &opt_verbose, &opt_verbose,
0, GET_BOOL, NO_ARG, 0, 0, 0, 0, 0, 0},
{"parallel", 'p', "Number of worker threads for reading / writing.",
&opt_parallel, &opt_parallel, 0, GET_INT, REQUIRED_ARG,
1, 1, INT_MAX, 0, 0, 0},
{"decrypt", 'd', "Decrypt files ending with .xbcrypt.",
&opt_encrypt_algo, &opt_encrypt_algo, &xbstream_encrypt_algo_typelib,
GET_ENUM, REQUIRED_ARG, 0, 0, 0, 0, 0, 0},
{"encrypt-key", 'k', "Encryption key.",
&opt_encrypt_key, &opt_encrypt_key, 0,
GET_STR_ALLOC, REQUIRED_ARG, 0, 0, 0, 0, 0, 0},
{"encrypt-key-file", 'f', "File which contains encryption key.",
&opt_encrypt_key_file, &opt_encrypt_key_file, 0,
GET_STR_ALLOC, REQUIRED_ARG, 0, 0, 0, 0, 0, 0},
{"encrypt-threads", OPT_ENCRYPT_THREADS,
"Number of threads for parallel data encryption. "
"The default value is 1.",
&opt_encrypt_threads, &opt_encrypt_threads,
0, GET_INT, REQUIRED_ARG, 1, 1, INT_MAX, 0, 0, 0},
{0, 0, 0, 0, 0, 0, GET_NO_ARG, NO_ARG, 0, 0, 0, 0, 0, 0}
};
typedef struct {
HASH *filehash;
xb_rstream_t *stream;
ds_ctxt_t *ds_ctxt;
ds_ctxt_t *ds_decrypt_ctxt;
pthread_mutex_t *mutex;
} extract_ctxt_t;
typedef struct {
char *path;
uint pathlen;
my_off_t offset;
ds_ctxt_t *ds_ctxt;
ds_file_t *file;
pthread_mutex_t mutex;
} file_entry_t;
static int get_options(int *argc, char ***argv);
static int mode_create(int argc, char **argv);
static int mode_extract(int argc, char **argv);
static int mode_extract(int n_threads, int argc, char **argv);
static my_bool get_one_option(int optid, const struct my_option *opt,
char *argument);
......@@ -88,6 +131,8 @@ main(int argc, char **argv)
{
MY_INIT(argv[0]);
crc_init();
if (get_options(&argc, &argv)) {
goto err;
}
......@@ -104,7 +149,8 @@ main(int argc, char **argv)
if (opt_mode == RUN_MODE_CREATE && mode_create(argc, argv)) {
goto err;
} else if (opt_mode == RUN_MODE_EXTRACT && mode_extract(argc, argv)) {
} else if (opt_mode == RUN_MODE_EXTRACT &&
mode_extract(opt_parallel, argc, argv)) {
goto err;
}
......@@ -302,9 +348,22 @@ mode_create(int argc, char **argv)
return 1;
}
/************************************************************************
Check if string ends with given suffix.
@return true if string ends with given suffix. */
static
my_bool
ends_with(const char *str, const char *suffix)
{
size_t suffix_len = strlen(suffix);
size_t str_len = strlen(str);
return(str_len >= suffix_len
&& strcmp(str + str_len - suffix_len, suffix) == 0);
}
static
file_entry_t *
file_entry_new(ds_ctxt_t *ds_ctxt, const char *path, uint pathlen)
file_entry_new(extract_ctxt_t *ctxt, const char *path, uint pathlen)
{
file_entry_t *entry;
ds_file_t *file;
......@@ -321,7 +380,11 @@ file_entry_new(ds_ctxt_t *ds_ctxt, const char *path, uint pathlen)
}
entry->pathlen = pathlen;
file = ds_open(ds_ctxt, path, NULL);
if (ctxt->ds_decrypt_ctxt && ends_with(path, ".xbcrypt")) {
file = ds_open(ctxt->ds_decrypt_ctxt, path, NULL);
} else {
file = ds_open(ctxt->ds_ctxt, path, NULL);
}
if (file == NULL) {
msg("%s: failed to create file.\n", my_progname);
goto err;
......@@ -332,7 +395,8 @@ file_entry_new(ds_ctxt_t *ds_ctxt, const char *path, uint pathlen)
}
entry->file = file;
entry->ds_ctxt = ds_ctxt;
pthread_mutex_init(&entry->mutex, NULL);
return entry;
......@@ -358,68 +422,77 @@ static
void
file_entry_free(file_entry_t *entry)
{
pthread_mutex_destroy(&entry->mutex);
ds_close(entry->file);
my_free(entry->path);
my_free(entry);
}
static
int
mode_extract(int argc __attribute__((unused)),
char **argv __attribute__((unused)))
void *
extract_worker_thread_func(void *arg)
{
xb_rstream_t *stream;
xb_rstream_result_t res;
xb_rstream_chunk_t chunk;
HASH filehash;
file_entry_t *entry;
ds_ctxt_t *ds_ctxt;
xb_rstream_result_t res;
stream = xb_stream_read_new();
if (stream == NULL) {
msg("%s: xb_stream_read_new() failed.\n", my_progname);
return 1;
}
extract_ctxt_t *ctxt = (extract_ctxt_t *) arg;
/* If --directory is specified, it is already set as CWD by now. */
ds_ctxt = ds_create(".", DS_TYPE_LOCAL);
my_thread_init();
if (my_hash_init(&filehash, &my_charset_bin, START_FILE_HASH_SIZE,
0, 0, (my_hash_get_key) get_file_entry_key,
(my_hash_free_key) file_entry_free, MYF(0))) {
msg("%s: failed to initialize file hash.\n", my_progname);
goto err;
memset(&chunk, 0, sizeof(chunk));
while (1) {
pthread_mutex_lock(ctxt->mutex);
res = xb_stream_read_chunk(ctxt->stream, &chunk);
if (res != XB_STREAM_READ_CHUNK) {
pthread_mutex_unlock(ctxt->mutex);
break;
}
while ((res = xb_stream_read_chunk(stream, &chunk)) ==
XB_STREAM_READ_CHUNK) {
/* If unknown type and ignorable flag is set, skip this chunk */
if (chunk.type == XB_CHUNK_TYPE_UNKNOWN && \
!(chunk.flags & XB_STREAM_FLAG_IGNORABLE)) {
pthread_mutex_unlock(ctxt->mutex);
continue;
}
/* See if we already have this file open */
entry = (file_entry_t *) my_hash_search(&filehash,
entry = (file_entry_t *) my_hash_search(ctxt->filehash,
(uchar *) chunk.path,
chunk.pathlen);
if (entry == NULL) {
entry = file_entry_new(ds_ctxt, chunk.path,
entry = file_entry_new(ctxt,
chunk.path,
chunk.pathlen);
if (entry == NULL) {
goto err;
pthread_mutex_unlock(ctxt->mutex);
break;
}
if (my_hash_insert(&filehash, (uchar *) entry)) {
if (my_hash_insert(ctxt->filehash, (uchar *) entry)) {
msg("%s: my_hash_insert() failed.\n",
my_progname);
goto err;
pthread_mutex_unlock(ctxt->mutex);
break;
}
}
if (chunk.type == XB_CHUNK_TYPE_EOF) {
my_hash_delete(&filehash, (uchar *) entry);
pthread_mutex_lock(&entry->mutex);
pthread_mutex_unlock(ctxt->mutex);
res = xb_stream_validate_checksum(&chunk);
if (res != XB_STREAM_READ_CHUNK) {
pthread_mutex_unlock(&entry->mutex);
break;
}
if (chunk.type == XB_CHUNK_TYPE_EOF) {
pthread_mutex_unlock(&entry->mutex);
continue;
}
......@@ -427,30 +500,114 @@ mode_extract(int argc __attribute__((unused)),
msg("%s: out-of-order chunk: real offset = 0x%llx, "
"expected offset = 0x%llx\n", my_progname,
chunk.offset, entry->offset);
goto err;
pthread_mutex_unlock(&entry->mutex);
res = XB_STREAM_READ_ERROR;
break;
}
if (ds_write(entry->file, chunk.data, chunk.length)) {
msg("%s: my_write() failed.\n", my_progname);
goto err;
pthread_mutex_unlock(&entry->mutex);
res = XB_STREAM_READ_ERROR;
break;
}
entry->offset += chunk.length;
};
if (res == XB_STREAM_READ_ERROR) {
goto err;
pthread_mutex_unlock(&entry->mutex);
}
if (chunk.data)
my_free(chunk.data);
my_thread_end();
return (void *)(res);
}
static
int
mode_extract(int n_threads, int argc __attribute__((unused)),
char **argv __attribute__((unused)))
{
xb_rstream_t *stream = NULL;
HASH filehash;
ds_ctxt_t *ds_ctxt = NULL;
ds_ctxt_t *ds_decrypt_ctxt = NULL;
extract_ctxt_t ctxt;
int i;
pthread_t *tids = NULL;
void **retvals = NULL;
pthread_mutex_t mutex;
int ret = 0;
if (my_hash_init(&filehash, &my_charset_bin, START_FILE_HASH_SIZE,
0, 0, (my_hash_get_key) get_file_entry_key,
(my_hash_free_key) file_entry_free, MYF(0))) {
msg("%s: failed to initialize file hash.\n", my_progname);
return 1;
}
if (pthread_mutex_init(&mutex, NULL)) {
msg("%s: failed to initialize mutex.\n", my_progname);
my_hash_free(&filehash);
ds_destroy(ds_ctxt);
xb_stream_read_done(stream);
return 1;
}
/* If --directory is specified, it is already set as CWD by now. */
ds_ctxt = ds_create(".", DS_TYPE_LOCAL);
if (ds_ctxt == NULL) {
ret = 1;
goto exit;
}
stream = xb_stream_read_new();
if (stream == NULL) {
msg("%s: xb_stream_read_new() failed.\n", my_progname);
pthread_mutex_destroy(&mutex);
ret = 1;
goto exit;
}
ctxt.stream = stream;
ctxt.filehash = &filehash;
ctxt.ds_ctxt = ds_ctxt;
ctxt.ds_decrypt_ctxt = ds_decrypt_ctxt;
ctxt.mutex = &mutex;
tids = malloc(sizeof(pthread_t) * n_threads);
retvals = malloc(sizeof(void*) * n_threads);
for (i = 0; i < n_threads; i++)
pthread_create(tids + i, NULL, extract_worker_thread_func,
&ctxt);
for (i = 0; i < n_threads; i++)
pthread_join(tids[i], retvals + i);
for (i = 0; i < n_threads; i++) {
if ((ulong)retvals[i] == XB_STREAM_READ_ERROR) {
ret = 1;
goto exit;
}
}
exit:
pthread_mutex_destroy(&mutex);
free(tids);
free(retvals);
return 0;
err:
my_hash_free(&filehash);
if (ds_ctxt != NULL) {
ds_destroy(ds_ctxt);
}
if (ds_decrypt_ctxt) {
ds_destroy(ds_decrypt_ctxt);
}
xb_stream_read_done(stream);
return 1;
return ret;
}
extra/mariabackup/xbstream.h
View file @ ecb25df2
/******************************************************
Copyright (c) 2011-2013 Percona LLC and/or its affiliates.
Copyright (c) 2011-2017 Percona LLC and/or its affiliates.
The xbstream format interface.
......@@ -89,8 +89,10 @@ typedef struct {
char path[FN_REFLEN];
size_t length;
my_off_t offset;
my_off_t checksum_offset;
void *data;
ulong checksum;
size_t buflen;
} xb_rstream_chunk_t;
xb_rstream_t *xb_stream_read_new(void);
......@@ -100,4 +102,6 @@ xb_rstream_result_t xb_stream_read_chunk(xb_rstream_t *stream,
int xb_stream_read_done(xb_rstream_t *stream);
int xb_stream_validate_checksum(xb_rstream_chunk_t *chunk);
#endif
extra/mariabackup/xbstream_read.c
View file @ ecb25df2
/******************************************************
Copyright (c) 2011-2013 Percona LLC and/or its affiliates.
Copyright (c) 2011-2017 Percona LLC and/or its affiliates.
The xbstream format reader implementation.
......@@ -23,6 +23,7 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA
#include <zlib.h>
#include "common.h"
#include "xbstream.h"
#include "crc_glue.h"
/* Allocate 1 MB for the payload buffer initially */
#define INIT_BUFFER_LEN (1024 * 1024)
......@@ -34,8 +35,6 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA
struct xb_rstream_struct {
my_off_t offset;
File fd;
void *buffer;
size_t buflen;
};
xb_rstream_t *
......@@ -45,9 +44,6 @@ xb_stream_read_new(void)
stream = (xb_rstream_t *) my_malloc(sizeof(xb_rstream_t), MYF(MY_FAE));
stream->buffer = my_malloc(INIT_BUFFER_LEN, MYF(MY_FAE));
stream->buflen = INIT_BUFFER_LEN;
#ifdef __WIN__
setmode(fileno(stdin), _O_BINARY);
#endif
......@@ -71,6 +67,23 @@ validate_chunk_type(uchar code)
}
}
int
xb_stream_validate_checksum(xb_rstream_chunk_t *chunk)
{
ulong checksum;
checksum = crc32_iso3309(0, chunk->data, (uint)chunk->length);
if (checksum != chunk->checksum) {
msg("xb_stream_read_chunk(): invalid checksum at offset "
"0x%llx: expected 0x%lx, read 0x%lx.\n",
(ulonglong) chunk->checksum_offset, chunk->checksum,
checksum);
return XB_STREAM_READ_ERROR;
}
return XB_STREAM_READ_CHUNK;
}
#define F_READ(buf,len) \
do { \
if (xb_read_full(fd, buf, len) < len) { \
......@@ -87,8 +100,6 @@ xb_stream_read_chunk(xb_rstream_t *stream, xb_rstream_chunk_t *chunk)
uint pathlen;
size_t tbytes;
ulonglong ullval;
ulong checksum_exp;
ulong checksum;
File fd = stream->fd;
xb_ad(sizeof(tmpbuf) >= CHUNK_HEADER_CONSTANT_LEN);
......@@ -178,39 +189,30 @@ xb_stream_read_chunk(xb_rstream_t *stream, xb_rstream_chunk_t *chunk)
stream->offset += 8;
/* Reallocate the buffer if needed */
if (chunk->length > stream->buflen) {
stream->buffer = my_realloc(stream->buffer, chunk->length,
MYF(MY_WME));
if (stream->buffer == NULL) {
if (chunk->length > chunk->buflen) {
chunk->data = my_realloc(chunk->data, chunk->length,
MYF(MY_WME | MY_ALLOW_ZERO_PTR));
if (chunk->data == NULL) {
msg("xb_stream_read_chunk(): failed to increase buffer "
"to %lu bytes.\n", (ulong) chunk->length);
goto err;
}
stream->buflen = chunk->length;
chunk->buflen = chunk->length;
}
/* Checksum */
F_READ(tmpbuf, 4);
checksum_exp = uint4korr(tmpbuf);
chunk->checksum = uint4korr(tmpbuf);
chunk->checksum_offset = stream->offset;
/* Payload */
if (chunk->length > 0) {
F_READ(stream->buffer, chunk->length);
F_READ(chunk->data, chunk->length);
stream->offset += chunk->length;
}
checksum = crc32(0, stream->buffer, chunk->length);
if (checksum != checksum_exp) {
msg("xb_stream_read_chunk(): invalid checksum at offset "
"0x%llx: expected 0x%lx, read 0x%lx.\n",
(ulonglong) stream->offset, checksum_exp, checksum);
goto err;
}
stream->offset += 4;
chunk->data = stream->buffer;
chunk->checksum = checksum;
return XB_STREAM_READ_CHUNK;
err:
......@@ -220,7 +222,6 @@ xb_stream_read_chunk(xb_rstream_t *stream, xb_rstream_chunk_t *chunk)
int
xb_stream_read_done(xb_rstream_t *stream)
{
my_free(stream->buffer);
my_free(stream);
return 0;
......
extra/mariabackup/xbstream_write.c
View file @ ecb25df2
/******************************************************
Copyright (c) 2011-2013 Percona LLC and/or its affiliates.
Copyright (c) 2011-2017 Percona LLC and/or its affiliates.
The xbstream format writer implementation.
......@@ -23,6 +23,7 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA
#include <zlib.h>
#include "common.h"
#include "xbstream.h"
#include "crc_glue.h"
/* Group writes smaller than this into a single chunk */
#define XB_STREAM_MIN_CHUNK_SIZE (10 * 1024 * 1024)
......@@ -215,12 +216,13 @@ xb_stream_write_chunk(xb_wstream_file_t *file, const void *buf, size_t len)
int8store(ptr, len); /* Payload length */
ptr += 8;
checksum = crc32_iso3309(0, buf, (uint)len); /* checksum */
pthread_mutex_lock(&stream->mutex);
int8store(ptr, file->offset); /* Payload offset */
ptr += 8;
checksum = crc32(0, buf, (uint)len); /* checksum */
int4store(ptr, checksum);
ptr += 4;
......
extra/mariabackup/xtrabackup.cc
View file @ ecb25df2
/******************************************************
XtraBackup: hot backup tool for InnoDB
(c) 2009-2015 Percona LLC and/or its affiliates
(c) 2009-2017 Percona LLC and/or its affiliates
Originally Created 3/3/2009 Yasufumi Kinoshita
Written by Alexey Kopytov, Aleksandr Kuzminsky, Stewart Smith, Vadim Tkachenko,
Yasufumi Kinoshita, Ignacio Nin and Baron Schwartz.
......@@ -67,6 +67,7 @@ Place, Suite 330, Boston, MA 02111-1307 USA
#include <srv0start.h>
#include <buf0dblwr.h>
#include <list>
#include <sstream>
#include <set>
#include <mysql.h>
......@@ -94,6 +95,7 @@ Place, Suite 330, Boston, MA 02111-1307 USA
#include "encryption_plugin.h"
#include <sql_plugin.h>
#include <srv0srv.h>
#include <crc_glue.h>
/* TODO: replace with appropriate macros used in InnoDB 5.6 */
#define PAGE_ZIP_MIN_SIZE_SHIFT 10
......@@ -144,25 +146,24 @@ char xtrabackup_real_incremental_dir[FN_REFLEN];
lsn_t xtrabackup_archived_to_lsn = 0; /* for --archived-to-lsn */
char *xtrabackup_tables = NULL;
char *xtrabackup_tmpdir;
/* List of regular expressions for filtering */
typedef struct xb_regex_list_node_struct xb_regex_list_node_t;
struct xb_regex_list_node_struct {
UT_LIST_NODE_T(xb_regex_list_node_t) regex_list;
regex_t regex;
};
static UT_LIST_BASE_NODE_T(xb_regex_list_node_t) regex_list;
static regmatch_t tables_regmatch[1];
char *xtrabackup_tables = NULL;
char *xtrabackup_tables_file = NULL;
static hash_table_t* tables_hash = NULL;
char *xtrabackup_tables_exclude = NULL;
typedef std::list<regex_t> regex_list_t;
static regex_list_t regex_include_list;
static regex_list_t regex_exclude_list;
static hash_table_t* tables_include_hash = NULL;
static hash_table_t* tables_exclude_hash = NULL;
char *xtrabackup_databases = NULL;
char *xtrabackup_databases_file = NULL;
static hash_table_t* databases_hash = NULL;
char *xtrabackup_databases_exclude = NULL;
static hash_table_t* databases_include_hash = NULL;
static hash_table_t* databases_exclude_hash = NULL;
static hash_table_t* inc_dir_tables_hash;
......@@ -615,6 +616,8 @@ enum options_xtrabackup
OPT_SSL_VERIFY_SERVER_CERT,
OPT_SERVER_PUBLIC_KEY,
OPT_XTRA_TABLES_EXCLUDE,
OPT_XTRA_DATABASES_EXCLUDE,
};
struct my_option xb_client_options[] =
......@@ -685,6 +688,16 @@ struct my_option xb_client_options[] =
"filtering by list of databases in the file.",
(G_PTR*) &xtrabackup_databases_file, (G_PTR*) &xtrabackup_databases_file,
0, GET_STR, REQUIRED_ARG, 0, 0, 0, 0, 0, 0},
{"tables-exclude", OPT_XTRA_TABLES_EXCLUDE, "filtering by regexp for table names. "
"Operates the same way as --tables, but matched names are excluded from backup. "
"Note that this option has a higher priority than --tables.",
(G_PTR*) &xtrabackup_tables_exclude, (G_PTR*) &xtrabackup_tables_exclude,
0, GET_STR, REQUIRED_ARG, 0, 0, 0, 0, 0, 0},
{"databases-exclude", OPT_XTRA_DATABASES_EXCLUDE, "Excluding databases based on name, "
"Operates the same way as --databases, but matched names are excluded from backup. "
"Note that this option has a higher priority than --databases.",
(G_PTR*) &xtrabackup_databases_exclude, (G_PTR*) &xtrabackup_databases_exclude,
0, GET_STR, REQUIRED_ARG, 0, 0, 0, 0, 0, 0},
{"create-ib-logfile", OPT_XTRA_CREATE_IB_LOGFILE, "** not work for now** creates ib_logfile* also after '--prepare'. ### If you want create ib_logfile*, only re-execute this command in same options. ###",
(G_PTR*) &xtrabackup_create_ib_logfile, (G_PTR*) &xtrabackup_create_ib_logfile,
0, GET_BOOL, NO_ARG, 0, 0, 0, 0, 0, 0},
......@@ -1036,8 +1049,8 @@ struct my_option xb_server_options[] =
(G_PTR*) &opt_mysql_tmpdir,
(G_PTR*) &opt_mysql_tmpdir, 0, GET_STR, REQUIRED_ARG, 0, 0, 0, 0, 0, 0},
{"parallel", OPT_XTRA_PARALLEL,
"Number of threads to use for parallel datafiles transfer. Does not have "
"any effect in the stream mode. The default value is 1.",
"Number of threads to use for parallel datafiles transfer. "
"The default value is 1.",
(G_PTR*) &xtrabackup_parallel, (G_PTR*) &xtrabackup_parallel, 0, GET_INT,
REQUIRED_ARG, 1, 1, INT_MAX, 0, 0, 0},
......@@ -2162,43 +2175,139 @@ xtrabackup_io_throttling(void)
}
}
static
my_bool regex_list_check_match(
const regex_list_t& list,
const char* name)
{
regmatch_t tables_regmatch[1];
for (regex_list_t::const_iterator i = list.begin(), end = list.end();
i != end; ++i) {
const regex_t& regex = *i;
int regres = regexec(&regex, name, 1, tables_regmatch, 0);
if (regres != REG_NOMATCH) {
return(TRUE);
}
}
return(FALSE);
}
static
my_bool
find_filter_in_hashtable(
const char* name,
hash_table_t* table,
xb_filter_entry_t** result
)
{
xb_filter_entry_t* found = NULL;
HASH_SEARCH(name_hash, table, ut_fold_string(name),
xb_filter_entry_t*,
found, (void) 0,
!strcmp(found->name, name));
if (found && result) {
*result = found;
}
return (found != NULL);
}
/************************************************************************
Checks if a given table name matches any of specifications in the --tables or
--tables-file options.
Checks if a given table name matches any of specifications given in
regex_list or tables_hash.
@return TRUE on match. */
@return TRUE on match or both regex_list and tables_hash are empty.*/
static my_bool
check_if_table_matches_filters(const char *name)
check_if_table_matches_filters(const char *name,
const regex_list_t& regex_list,
hash_table_t* tables_hash)
{
int regres;
xb_filter_entry_t* table;
xb_regex_list_node_t* node;
if (UT_LIST_GET_LEN(regex_list)) {
/* Check against regular expressions list */
for (node = UT_LIST_GET_FIRST(regex_list); node;
node = UT_LIST_GET_NEXT(regex_list, node)) {
regres = regexec(&node->regex, name, 1,
tables_regmatch, 0);
if (regres != REG_NOMATCH) {
if (regex_list.empty() && !tables_hash) {
return(FALSE);
}
if (regex_list_check_match(regex_list, name)) {
return(TRUE);
}
}
if (tables_hash && find_filter_in_hashtable(name, tables_hash, NULL)) {
return(TRUE);
}
if (tables_hash) {
HASH_SEARCH(name_hash, tables_hash, ut_fold_string(name),
xb_filter_entry_t*,
table, (void) 0,
!strcmp(table->name, name));
if (table) {
return FALSE;
}
return(TRUE);
enum skip_database_check_result {
DATABASE_SKIP,
DATABASE_SKIP_SOME_TABLES,
DATABASE_DONT_SKIP,
DATABASE_DONT_SKIP_UNLESS_EXPLICITLY_EXCLUDED,
};
/************************************************************************
Checks if a database specified by name should be skipped from backup based on
the --databases, --databases_file or --databases_exclude options.
@return TRUE if entire database should be skipped,
FALSE otherwise.
*/
static
skip_database_check_result
check_if_skip_database(
const char* name /*!< in: path to the database */
)
{
/* There are some filters for databases, check them */
xb_filter_entry_t* database = NULL;
if (databases_exclude_hash &&
find_filter_in_hashtable(name, databases_exclude_hash,
&database) &&
!database->has_tables) {
/* Database is found and there are no tables specified,
skip entire db. */
return DATABASE_SKIP;
}
if (databases_include_hash) {
if (!find_filter_in_hashtable(name, databases_include_hash,
&database)) {
/* Database isn't found, skip the database */
return DATABASE_SKIP;
} else if (database->has_tables) {
return DATABASE_SKIP_SOME_TABLES;
} else {
return DATABASE_DONT_SKIP_UNLESS_EXPLICITLY_EXCLUDED;
}
}
return DATABASE_DONT_SKIP;
}
/************************************************************************
Checks if a database specified by path should be skipped from backup based on
the --databases, --databases_file or --databases_exclude options.
@return TRUE if the table should be skipped. */
my_bool
check_if_skip_database_by_path(
const char* path /*!< in: path to the db directory. */
)
{
if (databases_include_hash == NULL &&
databases_exclude_hash == NULL) {
return(FALSE);
}
const char* db_name = strrchr(path, SRV_PATH_SEPARATOR);
if (db_name == NULL) {
db_name = path;
} else {
++db_name;
}
return check_if_skip_database(db_name) == DATABASE_SKIP;
}
/************************************************************************
......@@ -2217,9 +2326,12 @@ check_if_skip_table(
const char *ptr;
char *eptr;
if (UT_LIST_GET_LEN(regex_list) == 0 &&
tables_hash == NULL &&
databases_hash == NULL) {
if (regex_exclude_list.empty() &&
regex_include_list.empty() &&
tables_include_hash == NULL &&
tables_exclude_hash == NULL &&
databases_include_hash == NULL &&
databases_exclude_hash == NULL) {
return(FALSE);
}
......@@ -2237,23 +2349,10 @@ check_if_skip_table(
strncpy(buf, dbname, FN_REFLEN);
buf[tbname - 1 - dbname] = 0;
if (databases_hash) {
/* There are some filters for databases, check them */
xb_filter_entry_t* database;
HASH_SEARCH(name_hash, databases_hash, ut_fold_string(buf),
xb_filter_entry_t*,
database, (void) 0,
!strcmp(database->name, buf));
/* Table's database isn't found, skip the table */
if (!database) {
return(TRUE);
}
/* There aren't tables specified for the database,
it should be backed up entirely */
if (!database->has_tables) {
return(FALSE);
}
const skip_database_check_result skip_database =
check_if_skip_database(buf);
if (skip_database == DATABASE_SKIP) {
return (TRUE);
}
buf[FN_REFLEN - 1] = '\0';
......@@ -2270,21 +2369,43 @@ check_if_skip_table(
/* For partitioned tables first try to match against the regexp
without truncating the #P#... suffix so we can backup individual
partitions with regexps like '^test[.]t#P#p5' */
if (check_if_table_matches_filters(buf)) {
if (check_if_table_matches_filters(buf, regex_exclude_list,
tables_exclude_hash)) {
return(TRUE);
}
if (check_if_table_matches_filters(buf, regex_include_list,
tables_include_hash)) {
return(FALSE);
}
if ((eptr = strstr(buf, "#P#")) != NULL) {
*eptr = 0;
if (check_if_table_matches_filters(buf)) {
if (check_if_table_matches_filters(buf, regex_exclude_list,
tables_exclude_hash)) {
return (TRUE);
}
if (check_if_table_matches_filters(buf, regex_include_list,
tables_include_hash)) {
return(FALSE);
}
}
if (skip_database == DATABASE_DONT_SKIP_UNLESS_EXPLICITLY_EXCLUDED) {
/* Database is in include-list, and qualified name wasn't
found in any of exclusion filters.*/
return (FALSE);
}
if (skip_database == DATABASE_SKIP_SOME_TABLES ||
!regex_include_list.empty() ||
tables_include_hash) {
/* Include lists are present, but qualified name
failed to match any.*/
return(TRUE);
}
return(FALSE);
}
/***********************************************************************
......@@ -3040,6 +3161,8 @@ xtrabackup_init_datasinks(void)
if (xtrabackup_encrypt) {
ds_ctxt_t *ds;
ds = ds_create(xtrabackup_target_dir, DS_TYPE_ENCRYPT);
xtrabackup_add_datasink(ds);
......@@ -3360,7 +3483,10 @@ static
void
xb_register_filter_entry(
/*=====================*/
const char* name) /*!< in: name */
const char* name, /*!< in: name */
hash_table_t** databases_hash,
hash_table_t** tables_hash
)
{
const char* p;
size_t namelen;
......@@ -3376,25 +3502,45 @@ xb_register_filter_entry(
strncpy(dbname, name, p - name);
dbname[p - name] = 0;
if (databases_hash) {
HASH_SEARCH(name_hash, databases_hash,
if (*databases_hash) {
HASH_SEARCH(name_hash, (*databases_hash),
ut_fold_string(dbname),
xb_filter_entry_t*,
db_entry, (void) 0,
!strcmp(db_entry->name, dbname));
}
if (!db_entry) {
db_entry = xb_add_filter(dbname, &databases_hash);
db_entry = xb_add_filter(dbname, databases_hash);
}
db_entry->has_tables = TRUE;
xb_add_filter(name, &tables_hash);
xb_add_filter(name, tables_hash);
} else {
xb_validate_name(name, namelen);
xb_add_filter(name, &databases_hash);
xb_add_filter(name, databases_hash);
}
}
static
void
xb_register_include_filter_entry(
const char* name
)
{
xb_register_filter_entry(name, &databases_include_hash,
&tables_include_hash);
}
static
void
xb_register_exclude_filter_entry(
const char* name
)
{
xb_register_filter_entry(name, &databases_exclude_hash,
&tables_exclude_hash);
}
/***********************************************************************
Register new table for the filter. */
static
......@@ -3408,33 +3554,52 @@ xb_register_table(
exit(EXIT_FAILURE);
}
xb_register_filter_entry(name);
xb_register_include_filter_entry(name);
}
/***********************************************************************
Register new regex for the filter. */
static
void
xb_register_regex(
/*==============*/
const char* regex) /*!< in: regex */
xb_add_regex_to_list(
const char* regex, /*!< in: regex */
const char* error_context, /*!< in: context to error message */
regex_list_t* list) /*! in: list to put new regex to */
{
xb_regex_list_node_t* node;
char errbuf[100];
int ret;
node = static_cast<xb_regex_list_node_t *>
(ut_malloc(sizeof(xb_regex_list_node_t)));
regex_t compiled_regex;
ret = regcomp(&compiled_regex, regex, REG_EXTENDED);
ret = regcomp(&node->regex, regex, REG_EXTENDED);
if (ret != 0) {
xb_regerror(ret, &node->regex, errbuf, sizeof(errbuf));
msg("xtrabackup: error: tables regcomp(%s): %s\n",
regex, errbuf);
regerror(ret, &compiled_regex, errbuf, sizeof(errbuf));
msg("xtrabackup: error: %s regcomp(%s): %s\n",
error_context, regex, errbuf);
exit(EXIT_FAILURE);
}
UT_LIST_ADD_LAST(regex_list, regex_list, node);
list->push_back(compiled_regex);
}
/***********************************************************************
Register new regex for the include filter. */
static
void
xb_register_include_regex(
/*==============*/
const char* regex) /*!< in: regex */
{
xb_add_regex_to_list(regex, "tables", &regex_include_list);
}
/***********************************************************************
Register new regex for the exclude filter. */
static
void
xb_register_exclude_regex(
/*==============*/
const char* regex) /*!< in: regex */
{
xb_add_regex_to_list(regex, "tables-exclude", &regex_exclude_list);
}
typedef void (*insert_entry_func_t)(const char*);
......@@ -3500,26 +3665,34 @@ static
void
xb_filters_init()
{
UT_LIST_INIT(regex_list);
if (xtrabackup_databases) {
xb_load_list_string(xtrabackup_databases, " \t",
xb_register_filter_entry);
xb_register_include_filter_entry);
}
if (xtrabackup_databases_file) {
xb_load_list_file(xtrabackup_databases_file,
xb_register_filter_entry);
xb_register_include_filter_entry);
}
if (xtrabackup_databases_exclude) {
xb_load_list_string(xtrabackup_databases_exclude, " \t",
xb_register_exclude_filter_entry);
}
if (xtrabackup_tables) {
xb_load_list_string(xtrabackup_tables, ",",
xb_register_regex);
xb_register_include_regex);
}
if (xtrabackup_tables_file) {
xb_load_list_file(xtrabackup_tables_file, xb_register_table);
}
if (xtrabackup_tables_exclude) {
xb_load_list_string(xtrabackup_tables_exclude, ",",
xb_register_exclude_regex);
}
}
static
......@@ -3551,25 +3724,37 @@ xb_filter_hash_free(hash_table_t* hash)
hash_table_free(hash);
}
static void xb_regex_list_free(regex_list_t* list)
{
while (list->size() > 0) {
xb_regfree(&list->front());
list->pop_front();
}
}
/************************************************************************
Destroy table filters for partial backup. */
static
void
xb_filters_free()
{
while (UT_LIST_GET_LEN(regex_list) > 0) {
xb_regex_list_node_t* node = UT_LIST_GET_FIRST(regex_list);
UT_LIST_REMOVE(regex_list, regex_list, node);
regfree(&node->regex);
ut_free(node);
xb_regex_list_free(&regex_include_list);
xb_regex_list_free(&regex_exclude_list);
if (tables_include_hash) {
xb_filter_hash_free(tables_include_hash);
}
if (tables_hash) {
xb_filter_hash_free(tables_hash);
if (tables_exclude_hash) {
xb_filter_hash_free(tables_exclude_hash);
}
if (databases_hash) {
xb_filter_hash_free(databases_hash);
if (databases_include_hash) {
xb_filter_hash_free(databases_include_hash);
}
if (databases_exclude_hash) {
xb_filter_hash_free(databases_exclude_hash);
}
}
......@@ -3854,6 +4039,7 @@ xtrabackup_backup_func(void)
srv_general_init();
ut_crc32_init();
crc_init();
#ifdef WITH_INNODB_DISALLOW_WRITES
srv_allow_writes_event = os_event_create();
......@@ -6014,7 +6200,7 @@ xb_export_cfg_write(
file = fopen(file_path, "w+b");
if (file == NULL) {
msg("xtrabackup: Error: cannot close %s\n", node->name);
msg("xtrabackup: Error: cannot open %s\n", node->name);
success = false;
} else {
......@@ -6473,20 +6659,23 @@ xtrabackup_prepare_func(int argc, char ** argv)
table_name);
goto next_node;
}
index = dict_table_get_first_index(table);
n_index = UT_LIST_GET_LEN(table->indexes);
if (n_index > 31) {
msg("xtrabackup: error: "
"sorry, cannot export over "
"31 indexes for now.\n");
goto next_node;
}
/* Write MySQL 5.6 .cfg file */
if (!xb_export_cfg_write(node, table)) {
goto next_node;
}
index = dict_table_get_first_index(table);
n_index = UT_LIST_GET_LEN(table->indexes);
if (n_index > 31) {
msg("xtrabackup: warning: table '%s' has more "
"than 31 indexes, .exp file was not "
"generated. Table will fail to import "
"on server version prior to 5.6.\n",
table->name);
goto next_node;
}
/* init exp file */
memset(page, 0, UNIV_PAGE_SIZE);
mach_write_to_4(page , 0x78706f72UL);
......@@ -6659,6 +6848,12 @@ xtrabackup_prepare_func(int argc, char ** argv)
if (!xtrabackup_apply_log_only) {
/* xtrabackup_incremental_dir is used to indicate that
we are going to apply incremental backup. Here we already
applied incremental backup and are about to do final prepare
of the full backup */
xtrabackup_incremental_dir = NULL;
if(innodb_init_param()) {
goto error;
}
......@@ -7016,20 +7211,24 @@ handle_options(int argc, char **argv, char ***argv_client, char ***argv_server)
}
/* ================= main =================== */
extern my_bool(*fil_check_if_skip_database_by_path)(const char* name);
int main(int argc, char **argv)
{
char **client_defaults, **server_defaults;
char cwd[FN_REFLEN];
static char INNOBACKUPEX_EXE[]= "innobackupex";
if (argc > 1 && (strcmp(argv[1], "--innobackupex") == 0))
{
argv++;
argc--;
argv[0] = "innobackupex";
argv[0] = INNOBACKUPEX_EXE;
innobackupex_mode = true;
}
/* Setup skip fil_load_single_tablespaces callback.*/
fil_check_if_skip_database_by_path = check_if_skip_database_by_path;
init_signals();
MY_INIT(argv[0]);
......
extra/mariabackup/xtrabackup.h
View file @ ecb25df2
......@@ -77,6 +77,8 @@ extern char *xtrabackup_tables;
extern char *xtrabackup_tables_file;
extern char *xtrabackup_databases;
extern char *xtrabackup_databases_file;
extern char *xtrabackup_tables_exclude;
extern char *xtrabackup_databases_exclude;
extern ibool xtrabackup_compress;
extern ibool xtrabackup_encrypt;
......@@ -205,6 +207,17 @@ check_if_skip_table(
/******************/
const char* name); /*!< in: path to the table */
/************************************************************************
Checks if a database specified by path should be skipped from backup based on
the --databases, --databases_file or --databases_exclude options.
@return TRUE if the table should be skipped. */
my_bool
check_if_skip_database_by_path(
const char* path /*!< in: path to the db directory. */
);
/************************************************************************
Check if parameter is set in defaults file or via command line argument
@return true if parameter is set. */
......
mysql-test/suite/mariabackup/partial_exclude.result 0 → 100644
View file @ ecb25df2
CREATE TABLE t1(i INT) ENGINE INNODB;
INSERT INTO t1 VALUES(1);
CREATE TABLE t2(i int) ENGINE INNODB;
CREATE DATABASE db2;
USE db2;
CREATE TABLE t1(i INT) ENGINE INNODB;
USE test;
# xtrabackup backup
t1.ibd
DROP TABLE t1;
DROP TABLE t2;
DROP DATABASE db2;
mysql-test/suite/mariabackup/partial_exclude.test 0 → 100644
View file @ ecb25df2
# Test --databases-exclude and --tables-exclude feature of xtrabackup 2.3.8
CREATE TABLE t1(i INT) ENGINE INNODB;
INSERT INTO t1 VALUES(1);
CREATE TABLE t2(i int) ENGINE INNODB;
CREATE DATABASE db2;
USE db2;
CREATE TABLE t1(i INT) ENGINE INNODB;
USE test;
echo # xtrabackup backup;
let $targetdir=$MYSQLTEST_VARDIR/tmp/backup;
--disable_result_log
exec $XTRABACKUP --defaults-file=$MYSQLTEST_VARDIR/my.cnf --backup "--tables-exclude=test.*2" "--databases-exclude=db2" --target-dir=$targetdir;
--enable_result_log
# check that only t1 table is in backup (t2 is excluded)
list_files $targetdir/test *.ibd;
# check that db2 database is not in the backup (excluded)
--error 1
list_files $targetdir/db2 *.ibd;
DROP TABLE t1;
DROP TABLE t2;
DROP DATABASE db2;
rmdir $targetdir;
mysql-test/suite/mariabackup/xbstream.test
View file @ ecb25df2
......@@ -9,7 +9,7 @@ echo # xtrabackup backup to stream;
exec $XTRABACKUP --defaults-file=$MYSQLTEST_VARDIR/my.cnf --backup --stream=xbstream > $streamfile 2>$targetdir/backup_stream.log;
echo # xbstream extract;
--disable_result_log
exec $XBSTREAM -x -C $targetdir < $streamfile;
exec $XBSTREAM -x -C $targetdir --parallel=16 < $streamfile;
echo # xtrabackup prepare;
exec $XTRABACKUP --prepare --target-dir=$targetdir;
......
storage/xtradb/fil/fil0fil.cc
View file @ ecb25df2
......@@ -71,6 +71,7 @@ static ulint srv_data_read, srv_data_written;
MYSQL_PLUGIN_IMPORT extern my_bool lower_case_file_system;
/*
IMPLEMENTATION OF THE TABLESPACE MEMORY CACHE
=============================================
......@@ -5379,6 +5380,9 @@ fil_file_readdir_next_file(
return(-1);
}
my_bool(*fil_check_if_skip_database_by_path)(const char* name);
#define CHECK_TIME_EVERY_N_FILES 10
/********************************************************************//**
At the server startup, if we need crash recovery, scans the database
......@@ -5449,7 +5453,19 @@ fil_load_single_table_tablespaces(ibool (*pred)(const char*, const char*))
"%s/%s", fil_path_to_mysql_datadir, dbinfo.name);
srv_normalize_path_for_win(dbpath);
if (IS_XTRABACKUP()) {
ut_a(fil_check_if_skip_database_by_path);
if (fil_check_if_skip_database_by_path(dbpath)) {
fprintf(stderr, "Skipping db: %s\n", dbpath);
dbdir = NULL;
} else {
/* We want wrong directory permissions to be a fatal
error for XtraBackup. */
dbdir = os_file_opendir(dbpath, TRUE);
}
} else {
dbdir = os_file_opendir(dbpath, FALSE);
}
if (dbdir != NULL) {
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment
GitLab Nexedi Edition | About GitLab | About Nexedi | 沪ICP备2021021310号-2 | 沪ICP备2021021310号-7