Commit c73fa2d7 authored by Marko Mäkelä's avatar Marko Mäkelä

Merge 10.1 into 10.2

This will also change the minimum and maximum value of
innodb_log_file_size to 1MiB and 512GiB, respectively.
parents cf4a6abe d1e182d6
...@@ -1774,7 +1774,7 @@ copy_back() ...@@ -1774,7 +1774,7 @@ copy_back()
const char *ext_list[] = {"backup-my.cnf", "xtrabackup_logfile", const char *ext_list[] = {"backup-my.cnf", "xtrabackup_logfile",
"xtrabackup_binary", "xtrabackup_binlog_info", "xtrabackup_binary", "xtrabackup_binlog_info",
"xtrabackup_checkpoints", ".qp", ".pmap", ".tmp", "xtrabackup_checkpoints", ".qp", ".pmap", ".tmp",
".xbcrypt", NULL}; NULL};
const char *filename; const char *filename;
char c_tmp; char c_tmp;
int i_tmp; int i_tmp;
...@@ -1807,7 +1807,7 @@ copy_back() ...@@ -1807,7 +1807,7 @@ copy_back()
filename = base_name(node.filepath); filename = base_name(node.filepath);
/* skip .qp and .xbcrypt files */ /* skip .qp files */
if (filename_matches(filename, ext_list)) { if (filename_matches(filename, ext_list)) {
continue; continue;
} }
...@@ -1899,24 +1899,8 @@ decrypt_decompress_file(const char *filepath, uint thread_n) ...@@ -1899,24 +1899,8 @@ decrypt_decompress_file(const char *filepath, uint thread_n)
cmd << IF_WIN("type ","cat ") << filepath; cmd << IF_WIN("type ","cat ") << filepath;
if (ends_with(filepath, ".xbcrypt") && opt_decrypt) {
cmd << " | xbcrypt --decrypt --encrypt-algo="
<< xtrabackup_encrypt_algo_names[opt_decrypt_algo];
if (xtrabackup_encrypt_key) {
cmd << " --encrypt-key=" << xtrabackup_encrypt_key;
} else {
cmd << " --encrypt-key-file="
<< xtrabackup_encrypt_key_file;
}
dest_filepath[strlen(dest_filepath) - 8] = 0;
message << "decrypting";
needs_action = true;
}
if (opt_decompress if (opt_decompress
&& (ends_with(filepath, ".qp") && ends_with(filepath, ".qp")) {
|| (ends_with(filepath, ".qp.xbcrypt")
&& opt_decrypt))) {
cmd << " | qpress -dio "; cmd << " | qpress -dio ";
dest_filepath[strlen(dest_filepath) - 3] = 0; dest_filepath[strlen(dest_filepath) - 3] = 0;
if (needs_action) { if (needs_action) {
...@@ -1967,8 +1951,7 @@ decrypt_decompress_thread_func(void *arg) ...@@ -1967,8 +1951,7 @@ decrypt_decompress_thread_func(void *arg)
continue; continue;
} }
if (!ends_with(node.filepath, ".qp") if (!ends_with(node.filepath, ".qp")) {
&& !ends_with(node.filepath, ".xbcrypt")) {
continue; continue;
} }
......
...@@ -1435,9 +1435,7 @@ write_xtrabackup_info(MYSQL *connection) ...@@ -1435,9 +1435,7 @@ write_xtrabackup_info(MYSQL *connection)
"partial = %s\n" "partial = %s\n"
"incremental = %s\n" "incremental = %s\n"
"format = %s\n" "format = %s\n"
"compact = %s\n" "compressed = %s\n",
"compressed = %s\n"
"encrypted = %s\n",
uuid, /* uuid */ uuid, /* uuid */
opt_history ? opt_history : "", /* name */ opt_history ? opt_history : "", /* name */
tool_name, /* tool_name */ tool_name, /* tool_name */
...@@ -1455,9 +1453,7 @@ write_xtrabackup_info(MYSQL *connection) ...@@ -1455,9 +1453,7 @@ write_xtrabackup_info(MYSQL *connection)
is_partial? "Y" : "N", is_partial? "Y" : "N",
xtrabackup_incremental ? "Y" : "N", /* incremental */ xtrabackup_incremental ? "Y" : "N", /* incremental */
xb_stream_name[xtrabackup_stream_fmt], /* format */ xb_stream_name[xtrabackup_stream_fmt], /* format */
"N", /* compact */ xtrabackup_compress ? "compressed" : "N"); /* compressed */
xtrabackup_compress ? "compressed" : "N", /* compressed */
xtrabackup_encrypt ? "Y" : "N"); /* encrypted */
if (!opt_history) { if (!opt_history) {
goto cleanup; goto cleanup;
...@@ -1483,9 +1479,7 @@ write_xtrabackup_info(MYSQL *connection) ...@@ -1483,9 +1479,7 @@ write_xtrabackup_info(MYSQL *connection)
"partial ENUM('Y', 'N') DEFAULT NULL," "partial ENUM('Y', 'N') DEFAULT NULL,"
"incremental ENUM('Y', 'N') DEFAULT NULL," "incremental ENUM('Y', 'N') DEFAULT NULL,"
"format ENUM('file', 'tar', 'xbstream') DEFAULT NULL," "format ENUM('file', 'tar', 'xbstream') DEFAULT NULL,"
"compact ENUM('Y', 'N') DEFAULT NULL," "compressed ENUM('Y', 'N') DEFAULT NULL"
"compressed ENUM('Y', 'N') DEFAULT NULL,"
"encrypted ENUM('Y', 'N') DEFAULT NULL"
") CHARACTER SET utf8 ENGINE=innodb", false); ") CHARACTER SET utf8 ENGINE=innodb", false);
...@@ -1495,8 +1489,8 @@ write_xtrabackup_info(MYSQL *connection) ...@@ -1495,8 +1489,8 @@ write_xtrabackup_info(MYSQL *connection)
<< "uuid, name, tool_name, tool_command, tool_version," << "uuid, name, tool_name, tool_command, tool_version,"
<< "ibbackup_version, server_version, start_time, end_time," << "ibbackup_version, server_version, start_time, end_time,"
<< "lock_time, binlog_pos, innodb_from_lsn, innodb_to_lsn," << "lock_time, binlog_pos, innodb_from_lsn, innodb_to_lsn,"
<< "partial, incremental, format, compact, compressed, " << "partial, incremental, format, compressed) "
<< "encrypted) values(" << "values("
<< escape_and_quote(connection, uuid) << "," << escape_and_quote(connection, uuid) << ","
<< escape_and_quote(connection, opt_history) << "," << escape_and_quote(connection, opt_history) << ","
<< escape_and_quote(connection, tool_name) << "," << escape_and_quote(connection, tool_name) << ","
...@@ -1513,9 +1507,7 @@ write_xtrabackup_info(MYSQL *connection) ...@@ -1513,9 +1507,7 @@ write_xtrabackup_info(MYSQL *connection)
<< ESCAPE_BOOL(is_partial) << "," << ESCAPE_BOOL(is_partial) << ","
<< ESCAPE_BOOL(xtrabackup_incremental)<< "," << ESCAPE_BOOL(xtrabackup_incremental)<< ","
<< escape_and_quote(connection,xb_stream_name[xtrabackup_stream_fmt]) <<"," << escape_and_quote(connection,xb_stream_name[xtrabackup_stream_fmt]) <<","
<< ESCAPE_BOOL(false) << "," << ESCAPE_BOOL(xtrabackup_compress) << ")";
<< ESCAPE_BOOL(xtrabackup_compress) << ","
<< ESCAPE_BOOL(xtrabackup_encrypt) <<")";
xb_mysql_query(mysql_connection, oss.str().c_str(), false); xb_mysql_query(mysql_connection, oss.str().c_str(), false);
...@@ -1581,14 +1573,6 @@ char *make_argv(char *buf, size_t len, int argc, char **argv) ...@@ -1581,14 +1573,6 @@ char *make_argv(char *buf, size_t len, int argc, char **argv)
if (strncmp(*argv, "--password", strlen("--password")) == 0) { if (strncmp(*argv, "--password", strlen("--password")) == 0) {
arg = "--password=..."; arg = "--password=...";
} }
if (strncmp(*argv, "--encrypt-key",
strlen("--encrypt-key")) == 0) {
arg = "--encrypt-key=...";
}
if (strncmp(*argv, "--encrypt_key",
strlen("--encrypt_key")) == 0) {
arg = "--encrypt_key=...";
}
left-= ut_snprintf(buf + len - left, left, left-= ut_snprintf(buf + len - left, left,
"%s%c", arg, argc > 1 ? ' ' : 0); "%s%c", arg, argc > 1 ? ' ' : 0);
++argv; --argc; ++argv; --argc;
......
This diff is collapsed.
/******************************************************
Copyright (c) 2017 Percona LLC and/or its affiliates.
Encryption interface for XtraBackup.
This program is free software; you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation; version 2 of the License.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program; if not, write to the Free Software
Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA
*******************************************************/
#ifndef DS_DECRYPT_H
#define DS_DECRYPT_H
#include "datasink.h"
extern datasink_t datasink_decrypt;
extern int ds_decrypt_encrypt_threads;
#endif
This diff is collapsed.
/******************************************************
Copyright (c) 2013 Percona LLC and/or its affiliates.
Encryption interface for XtraBackup.
This program is free software; you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation; version 2 of the License.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program; if not, write to the Free Software
Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA
*******************************************************/
#ifndef DS_ENCRYPT_H
#define DS_ENCRYPT_H
#include "datasink.h"
#ifdef HAVE_GCRYPT
extern datasink_t datasink_encrypt;
#endif
/* Encryption options */
extern uint ds_encrypt_encrypt_threads;
extern ulonglong ds_encrypt_encrypt_chunk_size;
#endif
...@@ -126,7 +126,6 @@ void encryption_plugin_prepare_init(int argc, char **argv) ...@@ -126,7 +126,6 @@ void encryption_plugin_prepare_init(int argc, char **argv)
if (!xb_plugin_load) if (!xb_plugin_load)
{ {
/* This prevents crashes e.g in --stats with wrong my.cnf*/
finalize_encryption_plugin(0); finalize_encryption_plugin(0);
return; return;
} }
......
This diff is collapsed.
...@@ -44,19 +44,6 @@ dberr_t* err, /*!< out: this is set to DB_ERROR if an error ...@@ -44,19 +44,6 @@ dberr_t* err, /*!< out: this is set to DB_ERROR if an error
os_file_dir_t dir, /*!< in: directory stream */ os_file_dir_t dir, /*!< in: directory stream */
os_file_stat_t* info) /*!< in/out: buffer where the os_file_stat_t* info) /*!< in/out: buffer where the
info is returned */; info is returned */;
buf_block_t* btr_node_ptr_get_child(
const rec_t* node_ptr,/*!< in: node pointer */
dict_index_t* index, /*!< in: index */
const ulint* offsets,/*!< in: array returned by rec_get_offsets() */
mtr_t* mtr) /*!< in: mtr */;
buf_block_t*
btr_root_block_get(
/*===============*/
const dict_index_t* index, /*!< in: index tree */
ulint mode, /*!< in: either RW_S_LATCH
or RW_X_LATCH */
mtr_t* mtr) /*!< in: mtr */;
fil_space_t* fil_space_t*
fil_space_get_by_name(const char *); fil_space_get_by_name(const char *);
ibool ibool
......
This diff is collapsed.
/******************************************************
Copyright (c) 2011 Percona LLC and/or its affiliates.
Encryption interface for XtraBackup.
This program is free software; you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation; version 2 of the License.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program; if not, write to the Free Software
Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA
*******************************************************/
#ifndef XBCRYPT_H
#define XBCRYPT_H
#include <my_base.h>
#include "common.h"
#define XB_CRYPT_CHUNK_MAGIC1 "XBCRYP01"
#define XB_CRYPT_CHUNK_MAGIC2 "XBCRYP02"
#define XB_CRYPT_CHUNK_MAGIC3 "XBCRYP03" /* must be same size as ^^ */
#define XB_CRYPT_CHUNK_MAGIC_CURRENT XB_CRYPT_CHUNK_MAGIC3
#define XB_CRYPT_CHUNK_MAGIC_SIZE (sizeof(XB_CRYPT_CHUNK_MAGIC1)-1)
#define XB_CRYPT_HASH GCRY_MD_SHA256
#define XB_CRYPT_HASH_LEN 32
/******************************************************************************
Write interface */
typedef struct xb_wcrypt_struct xb_wcrypt_t;
/* Callback on write for i/o, must return # of bytes written or -1 on error */
typedef ssize_t xb_crypt_write_callback(void *userdata,
const void *buf, size_t len);
xb_wcrypt_t *xb_crypt_write_open(void *userdata,
xb_crypt_write_callback *onwrite);
/* Takes buffer, original length, encrypted length iv and iv length, formats
output buffer and calls write callback.
Returns 0 on success, 1 on error */
int xb_crypt_write_chunk(xb_wcrypt_t *crypt, const void *buf, size_t olen,
size_t elen, const void *iv, size_t ivlen);
/* Returns 0 on success, 1 on error */
int xb_crypt_write_close(xb_wcrypt_t *crypt);
/******************************************************************************
Read interface */
typedef struct xb_rcrypt_struct xb_rcrypt_t;
/* Callback on read for i/o, must return # of bytes read or -1 on error */
typedef size_t xb_crypt_read_callback(void *userdata, void *buf, size_t len);
xb_rcrypt_t *xb_crypt_read_open(void *userdata,
xb_crypt_read_callback *onread);
typedef enum {
XB_CRYPT_READ_CHUNK,
XB_CRYPT_READ_INCOMPLETE,
XB_CRYPT_READ_EOF,
XB_CRYPT_READ_ERROR
} xb_rcrypt_result_t;
xb_rcrypt_result_t xb_crypt_read_chunk(xb_rcrypt_t *crypt, void **buf,
size_t *olen, size_t *elen, void **iv,
size_t *ivlen, my_bool *hash_appended);
int xb_crypt_read_close(xb_rcrypt_t *crypt);
#endif
/******************************************************
Copyright (c) 2013, 2017 Percona LLC and/or its affiliates.
Encryption configuration file interface for XtraBackup.
This program is free software; you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation; version 2 of the License.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program; if not, write to the Free Software
Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA
*******************************************************/
#include <my_base.h>
#include "common.h"
#include "xbcrypt.h"
#include "xbcrypt_common.h"
/* Encryption options */
char *ds_encrypt_key = NULL;
char *ds_encrypt_key_file = NULL;
ulong ds_encrypt_algo;
static uint encrypt_key_len;
static uint encrypt_iv_len;
static const uint encrypt_mode = GCRY_CIPHER_MODE_CTR;
static uint encrypt_algos[] = { GCRY_CIPHER_NONE, GCRY_CIPHER_AES128,
GCRY_CIPHER_AES192, GCRY_CIPHER_AES256 };
static uint encrypt_algo;
#if !defined(GCRYPT_VERSION_NUMBER) || (GCRYPT_VERSION_NUMBER < 0x010600)
GCRY_THREAD_OPTION_PTHREAD_IMPL;
#endif
my_bool
xb_crypt_read_key_file(const char *filename, void** key, uint *keylength)
{
FILE *fp;
if (!(fp = my_fopen(filename, O_RDONLY, MYF(0)))) {
msg("%s:%s: unable to open config file \"%s\", errno(%d)\n",
my_progname, __FUNCTION__, filename, my_errno);
return FALSE;
}
fseek(fp, 0 , SEEK_END);
*keylength = ftell(fp);
rewind(fp);
*key = my_malloc(*keylength, MYF(MY_FAE));
*keylength = fread(*key, 1, *keylength, fp);
my_fclose(fp, MYF(0));
return TRUE;
}
void
xb_crypt_create_iv(void* ivbuf, size_t ivlen)
{
gcry_create_nonce(ivbuf, ivlen);
}
gcry_error_t
xb_crypt_init(uint *iv_len)
{
gcry_error_t gcry_error;
/* Acording to gcrypt docs (and my testing), setting up the threading
callbacks must be done first, so, lets give it a shot */
#if !defined(GCRYPT_VERSION_NUMBER) || (GCRYPT_VERSION_NUMBER < 0x010600)
gcry_error = gcry_control(GCRYCTL_SET_THREAD_CBS, &gcry_threads_pthread);
if (gcry_error) {
msg("encryption: unable to set libgcrypt thread cbs - "
"%s : %s\n",
gcry_strsource(gcry_error),
gcry_strerror(gcry_error));
return gcry_error;
}
#endif
/* Version check should be the very next call because it
makes sure that important subsystems are intialized. */
if (!gcry_control(GCRYCTL_ANY_INITIALIZATION_P)) {
const char *gcrypt_version;
gcrypt_version = gcry_check_version(NULL);
/* No other library has already initialized libgcrypt. */
if (!gcrypt_version) {
msg("encryption: failed to initialize libgcrypt\n");
return 1;
} else {
msg("encryption: using gcrypt %s\n", gcrypt_version);
}
}
/* Disable the gcry secure memory, not dealing with this for now */
gcry_error = gcry_control(GCRYCTL_DISABLE_SECMEM, 0);
if (gcry_error) {
msg("encryption: unable to disable libgcrypt secmem - "
"%s : %s\n",
gcry_strsource(gcry_error),
gcry_strerror(gcry_error));
return gcry_error;
}
/* Finalize gcry initialization. */
gcry_error = gcry_control(GCRYCTL_INITIALIZATION_FINISHED, 0);
if (gcry_error) {
msg("encryption: unable to finish libgcrypt initialization - "
"%s : %s\n",
gcry_strsource(gcry_error),
gcry_strerror(gcry_error));
return gcry_error;
}
/* Determine the algorithm */
encrypt_algo = encrypt_algos[ds_encrypt_algo];
/* Set up the iv length */
encrypt_iv_len = gcry_cipher_get_algo_blklen(encrypt_algo);
xb_a(encrypt_iv_len > 0);
if (iv_len != NULL) {
*iv_len = encrypt_iv_len;
}
/* Now set up the key */
if (ds_encrypt_key == NULL &&
ds_encrypt_key_file == NULL) {
msg("encryption: no encryption key or key file specified.\n");
return gcry_error;
} else if (ds_encrypt_key && ds_encrypt_key_file) {
msg("encryption: both encryption key and key file specified.\n");
return gcry_error;
} else if (ds_encrypt_key_file) {
if (!xb_crypt_read_key_file(ds_encrypt_key_file,
(void**)&ds_encrypt_key,
&encrypt_key_len)) {
msg("encryption: unable to read encryption key file"
" \"%s\".\n", ds_encrypt_key_file);
return gcry_error;
}
} else if (ds_encrypt_key) {
encrypt_key_len = strlen(ds_encrypt_key);
} else {
msg("encryption: no encryption key or key file specified.\n");
return gcry_error;
}
return 0;
}
gcry_error_t
xb_crypt_cipher_open(gcry_cipher_hd_t *cipher_handle)
{
if (encrypt_algo != GCRY_CIPHER_NONE) {
gcry_error_t gcry_error;
gcry_error = gcry_cipher_open(cipher_handle,
encrypt_algo,
encrypt_mode, 0);
if (gcry_error) {
msg("encryption: unable to open libgcrypt"
" cipher - %s : %s\n",
gcry_strsource(gcry_error),
gcry_strerror(gcry_error));
gcry_cipher_close(*cipher_handle);
return gcry_error;
}
gcry_error = gcry_cipher_setkey(*cipher_handle,
ds_encrypt_key,
encrypt_key_len);
if (gcry_error) {
msg("encryption: unable to set libgcrypt"
" cipher key - %s : %s\n",
gcry_strsource(gcry_error),
gcry_strerror(gcry_error));
gcry_cipher_close(*cipher_handle);
return gcry_error;
}
return gcry_error;
}
return 0;
}
void
xb_crypt_cipher_close(gcry_cipher_hd_t cipher_handle)
{
if (encrypt_algo != GCRY_CIPHER_NONE)
gcry_cipher_close(cipher_handle);
}
gcry_error_t
xb_crypt_decrypt(gcry_cipher_hd_t cipher_handle, const uchar *from,
size_t from_len, uchar *to, size_t *to_len,
const uchar *iv, size_t iv_len, my_bool hash_appended)
{
*to_len = from_len;
if (encrypt_algo != GCRY_CIPHER_NONE) {
gcry_error_t gcry_error;
gcry_error = gcry_cipher_reset(cipher_handle);
if (gcry_error) {
msg("%s:encryption: unable to reset libgcrypt"
" cipher - %s : %s\n", my_progname,
gcry_strsource(gcry_error),
gcry_strerror(gcry_error));
return gcry_error;
}
if (iv_len > 0) {
gcry_error = gcry_cipher_setctr(cipher_handle,
iv, iv_len);
}
if (gcry_error) {
msg("%s:encryption: unable to set cipher iv - "
"%s : %s\n", my_progname,
gcry_strsource(gcry_error),
gcry_strerror(gcry_error));
return gcry_error;
}
/* Try to decrypt it */
gcry_error = gcry_cipher_decrypt(cipher_handle, to, *to_len,
from, from_len);
if (gcry_error) {
msg("%s:encryption: unable to decrypt chunk - "
"%s : %s\n", my_progname,
gcry_strsource(gcry_error),
gcry_strerror(gcry_error));
gcry_cipher_close(cipher_handle);
return gcry_error;
}
if (hash_appended) {
uchar hash[XB_CRYPT_HASH_LEN];
*to_len -= XB_CRYPT_HASH_LEN;
/* ensure that XB_CRYPT_HASH_LEN is the correct length
of XB_CRYPT_HASH hashing algorithm output */
xb_ad(gcry_md_get_algo_dlen(XB_CRYPT_HASH) ==
XB_CRYPT_HASH_LEN);
gcry_md_hash_buffer(XB_CRYPT_HASH, hash, to,
*to_len);
if (memcmp(hash, (char *) to + *to_len,
XB_CRYPT_HASH_LEN) != 0) {
msg("%s:%s invalid plaintext hash. "
"Wrong encrytion key specified?\n",
my_progname, __FUNCTION__);
return 1;
}
}
} else {
memcpy(to, from, *to_len);
}
return 0;
}
gcry_error_t
xb_crypt_encrypt(gcry_cipher_hd_t cipher_handle, const uchar *from,
size_t from_len, uchar *to, size_t *to_len, uchar *iv)
{
gcry_error_t gcry_error;
/* ensure that XB_CRYPT_HASH_LEN is the correct length
of XB_CRYPT_HASH hashing algorithm output */
xb_ad(gcry_md_get_algo_dlen(XB_CRYPT_HASH) ==
XB_CRYPT_HASH_LEN);
memcpy(to, from, from_len);
gcry_md_hash_buffer(XB_CRYPT_HASH, to + from_len,
from, from_len);
*to_len = from_len;
if (encrypt_algo != GCRY_CIPHER_NONE) {
gcry_error = gcry_cipher_reset(cipher_handle);
if (gcry_error) {
msg("encrypt: unable to reset cipher - "
"%s : %s\n",
gcry_strsource(gcry_error),
gcry_strerror(gcry_error));
return gcry_error;
}
xb_crypt_create_iv(iv, encrypt_iv_len);
gcry_error = gcry_cipher_setctr(cipher_handle, iv,
encrypt_iv_len);
if (gcry_error) {
msg("encrypt: unable to set cipher ctr - "
"%s : %s\n",
gcry_strsource(gcry_error),
gcry_strerror(gcry_error));
return gcry_error;
}
gcry_error = gcry_cipher_encrypt(cipher_handle, to,
*to_len + XB_CRYPT_HASH_LEN,
to,
from_len + XB_CRYPT_HASH_LEN);
if (gcry_error) {
msg("encrypt: unable to encrypt buffer - "
"%s : %s\n", gcry_strsource(gcry_error),
gcry_strerror(gcry_error));
return gcry_error;
}
} else {
memcpy(to, from, from_len + XB_CRYPT_HASH_LEN);
}
*to_len += XB_CRYPT_HASH_LEN;
return 0;
}
#endif
\ No newline at end of file
/******************************************************
Copyright (c) 2017 Percona LLC and/or its affiliates.
Encryption datasink implementation for XtraBackup.
This program is free software; you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation; version 2 of the License.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program; if not, write to the Free Software
Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA
*******************************************************/
#include <my_base.h>
#if HAVE_GCRYPT
#if GCC_VERSION >= 4002
/* Workaround to avoid "gcry_ac_* is deprecated" warnings in gcrypt.h */
# pragma GCC diagnostic ignored "-Wdeprecated-declarations"
#endif
#include <gcrypt.h>
extern char *ds_encrypt_key;
extern char *ds_encrypt_key_file;
extern int ds_encrypt_threads;
extern ulong ds_encrypt_algo;
/******************************************************************************
Utility interface */
my_bool xb_crypt_read_key_file(const char *filename,
void** key, uint *keylength);
void xb_crypt_create_iv(void* ivbuf, size_t ivlen);
/* Initialize gcrypt and setup encryption key and IV lengths */
gcry_error_t
xb_crypt_init(uint *iv_len);
/* Setup gcrypt cipher */
gcry_error_t
xb_crypt_cipher_open(gcry_cipher_hd_t *cipher_handle);
/* Close gcrypt cipher */
void
xb_crypt_cipher_close(gcry_cipher_hd_t cipher_handle);
/* Decrypt buffer */
gcry_error_t
xb_crypt_decrypt(gcry_cipher_hd_t cipher_handle, const uchar *from,
size_t from_len, uchar *to, size_t *to_len, const uchar *iv,
size_t iv_len, my_bool hash_appended);
/* Encrypt buffer */
gcry_error_t
xb_crypt_encrypt(gcry_cipher_hd_t cipher_handle, const uchar *from,
size_t from_len, uchar *to, size_t *to_len, uchar *iv);
#endif
/******************************************************
Copyright (c) 2013 Percona LLC and/or its affiliates.
The xbcrypt format reader implementation.
This program is free software; you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation; version 2 of the License.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program; if not, write to the Free Software
Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA
*******************************************************/
#include "xbcrypt.h"
#include "crc_glue.h"
struct xb_rcrypt_struct {
void *userdata;
xb_crypt_read_callback *read;
void *buffer;
size_t bufsize;
void *ivbuffer;
size_t ivbufsize;
ulonglong offset;
};
xb_rcrypt_t *
xb_crypt_read_open(void *userdata, xb_crypt_read_callback *onread)
{
xb_rcrypt_t *crypt;
xb_ad(onread);
crypt = (xb_rcrypt_t *) my_malloc(sizeof(xb_rcrypt_t), MYF(MY_FAE));
crypt->userdata = userdata;
crypt->read = onread;
crypt->buffer = NULL;
crypt->bufsize = 0;
crypt->offset = 0;
crypt->ivbuffer = NULL;
crypt->ivbufsize = 0;
return crypt;
}
xb_rcrypt_result_t
xb_crypt_read_chunk(xb_rcrypt_t *crypt, void **buf, size_t *olen, size_t *elen,
void **iv, size_t *ivlen, my_bool *hash_appended)
{
uchar tmpbuf[XB_CRYPT_CHUNK_MAGIC_SIZE + 8 + 8 + 8 + 4];
uchar *ptr;
ulonglong tmp;
ulong checksum, checksum_exp, version;
size_t bytesread;
xb_rcrypt_result_t result = XB_CRYPT_READ_CHUNK;
if ((bytesread = crypt->read(crypt->userdata, tmpbuf, sizeof(tmpbuf)))
!= sizeof(tmpbuf)) {
if (bytesread == 0) {
result = XB_CRYPT_READ_EOF;
goto err;
} else {
msg("%s:%s: unable to read chunk header data at "
"offset 0x%llx.\n",
my_progname, __FUNCTION__, crypt->offset);
result = XB_CRYPT_READ_ERROR;
goto err;
}
}
ptr = tmpbuf;
if (memcmp(ptr, XB_CRYPT_CHUNK_MAGIC3,
XB_CRYPT_CHUNK_MAGIC_SIZE) == 0) {
version = 3;
} else if (memcmp(ptr, XB_CRYPT_CHUNK_MAGIC2,
XB_CRYPT_CHUNK_MAGIC_SIZE) == 0) {
version = 2;
} else if (memcmp(ptr, XB_CRYPT_CHUNK_MAGIC1,
XB_CRYPT_CHUNK_MAGIC_SIZE) == 0) {
version = 1;
} else {
msg("%s:%s: wrong chunk magic at offset 0x%llx.\n",
my_progname, __FUNCTION__, crypt->offset);
result = XB_CRYPT_READ_ERROR;
goto err;
}
ptr += XB_CRYPT_CHUNK_MAGIC_SIZE;
crypt->offset += XB_CRYPT_CHUNK_MAGIC_SIZE;
tmp = uint8korr(ptr); /* reserved */
ptr += 8;
crypt->offset += 8;
tmp = uint8korr(ptr); /* original size */
ptr += 8;
if (tmp > INT_MAX) {
msg("%s:%s: invalid original size at offset 0x%llx.\n",
my_progname, __FUNCTION__, crypt->offset);
result = XB_CRYPT_READ_ERROR;
goto err;
}
crypt->offset += 8;
*olen = (size_t)tmp;
tmp = uint8korr(ptr); /* encrypted size */
ptr += 8;
if (tmp > INT_MAX) {
msg("%s:%s: invalid encrypted size at offset 0x%llx.\n",
my_progname, __FUNCTION__, crypt->offset);
result = XB_CRYPT_READ_ERROR;
goto err;
}
crypt->offset += 8;
*elen = (size_t)tmp;
checksum_exp = uint4korr(ptr); /* checksum */
ptr += 4;
crypt->offset += 4;
/* iv size */
if (version == 1) {
*ivlen = 0;
*iv = 0;
} else {
if ((bytesread = crypt->read(crypt->userdata, tmpbuf, 8))
!= 8) {
if (bytesread == 0) {
result = XB_CRYPT_READ_EOF;
goto err;
} else {
msg("%s:%s: unable to read chunk iv size at "
"offset 0x%llx.\n",
my_progname, __FUNCTION__, crypt->offset);
result = XB_CRYPT_READ_ERROR;
goto err;
}
}
tmp = uint8korr(tmpbuf);
if (tmp > INT_MAX) {
msg("%s:%s: invalid iv size at offset 0x%llx.\n",
my_progname, __FUNCTION__, crypt->offset);
result = XB_CRYPT_READ_ERROR;
goto err;
}
crypt->offset += 8;
*ivlen = (size_t)tmp;
}
if (*ivlen > crypt->ivbufsize) {
crypt->ivbuffer = my_realloc(crypt->ivbuffer, *ivlen,
MYF(MY_WME | MY_ALLOW_ZERO_PTR));
if (crypt->ivbuffer == NULL) {
msg("%s:%s: failed to increase iv buffer to "
"%llu bytes.\n", my_progname, __FUNCTION__,
(ulonglong)*ivlen);
result = XB_CRYPT_READ_ERROR;
goto err;
}
crypt->ivbufsize = *ivlen;
}
if (*ivlen > 0) {
if (crypt->read(crypt->userdata, crypt->ivbuffer, *ivlen)
!= *ivlen) {
msg("%s:%s: failed to read %lld bytes for chunk iv "
"at offset 0x%llx.\n", my_progname, __FUNCTION__,
(ulonglong)*ivlen, crypt->offset);
result = XB_CRYPT_READ_ERROR;
goto err;
}
*iv = crypt->ivbuffer;
}
/* for version euqals 2 we need to read in the iv data but do not init
CTR with it */
if (version == 2) {
*ivlen = 0;
*iv = 0;
}
if (*olen > crypt->bufsize) {
crypt->buffer = my_realloc(crypt->buffer, *olen,
MYF(MY_WME | MY_ALLOW_ZERO_PTR));
if (crypt->buffer == NULL) {
msg("%s:%s: failed to increase buffer to "
"%llu bytes.\n", my_progname, __FUNCTION__,
(ulonglong)*olen);
result = XB_CRYPT_READ_ERROR;
goto err;
}
crypt->bufsize = *olen;
}
if (*elen > 0) {
if (crypt->read(crypt->userdata, crypt->buffer, *elen)
!= *elen) {
msg("%s:%s: failed to read %lld bytes for chunk payload "
"at offset 0x%llx.\n", my_progname, __FUNCTION__,
(ulonglong)*elen, crypt->offset);
result = XB_CRYPT_READ_ERROR;
goto err;
}
}
checksum = crc32_iso3309(0, crypt->buffer, *elen);
if (checksum != checksum_exp) {
msg("%s:%s invalid checksum at offset 0x%llx, "
"expected 0x%lx, actual 0x%lx.\n", my_progname, __FUNCTION__,
crypt->offset, checksum_exp, checksum);
result = XB_CRYPT_READ_ERROR;
goto err;
}
crypt->offset += *elen;
*buf = crypt->buffer;
*hash_appended = version > 2;
goto exit;
err:
*buf = NULL;
*olen = 0;
*elen = 0;
*ivlen = 0;
*iv = 0;
exit:
return result;
}
int xb_crypt_read_close(xb_rcrypt_t *crypt)
{
if (crypt->buffer)
my_free(crypt->buffer);
if (crypt->ivbuffer)
my_free(crypt->ivbuffer);
my_free(crypt);
return 0;
}
/******************************************************
Copyright (c) 2013 Percona LLC and/or its affiliates.
The xbcrypt format writer implementation.
This program is free software; you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation; version 2 of the License.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program; if not, write to the Free Software
Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA
*******************************************************/
#include "xbcrypt.h"
#include "crc_glue.h"
struct xb_wcrypt_struct {
void *userdata;
xb_crypt_write_callback *write;
};
xb_wcrypt_t *
xb_crypt_write_open(void *userdata, xb_crypt_write_callback *onwrite)
{
xb_wcrypt_t *crypt;
xb_ad(onwrite);
crypt = (xb_wcrypt_t *) my_malloc(sizeof(xb_wcrypt_t), MYF(MY_FAE));
crypt->userdata = userdata;
crypt->write = onwrite;
return crypt;
}
int xb_crypt_write_chunk(xb_wcrypt_t *crypt, const void *buf, size_t olen,
size_t elen, const void *iv, size_t ivlen)
{
uchar tmpbuf[XB_CRYPT_CHUNK_MAGIC_SIZE + 8 + 8 + 8 + 4 + 8];
uchar *ptr;
ulong checksum;
xb_ad(olen <= INT_MAX);
if (olen > INT_MAX)
return 0;
xb_ad(elen <= INT_MAX);
if (elen > INT_MAX)
return 0;
xb_ad(ivlen <= INT_MAX);
if (ivlen > INT_MAX)
return 0;
ptr = tmpbuf;
memcpy(ptr, XB_CRYPT_CHUNK_MAGIC_CURRENT, XB_CRYPT_CHUNK_MAGIC_SIZE);
ptr += XB_CRYPT_CHUNK_MAGIC_SIZE;
int8store(ptr, (ulonglong)0); /* reserved */
ptr += 8;
int8store(ptr, (ulonglong)olen); /* original size */
ptr += 8;
int8store(ptr, (ulonglong)elen); /* encrypted (actual) size */
ptr += 8;
checksum = crc32_iso3309(0, buf, elen);
int4store(ptr, checksum); /* checksum */
ptr += 4;
int8store(ptr, (ulonglong)ivlen); /* iv size */
ptr += 8;
xb_ad(ptr <= tmpbuf + sizeof(tmpbuf));
if (crypt->write(crypt->userdata, tmpbuf, ptr-tmpbuf) == -1)
return 1;
if (crypt->write(crypt->userdata, iv, ivlen) == -1)
return 1;
if (crypt->write(crypt->userdata, buf, elen) == -1)
return 1;
return 0;
}
int xb_crypt_write_close(xb_wcrypt_t *crypt)
{
my_free(crypt);
return 0;
}
...@@ -25,9 +25,7 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA ...@@ -25,9 +25,7 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA
#include <my_pthread.h> #include <my_pthread.h>
#include "common.h" #include "common.h"
#include "xbstream.h" #include "xbstream.h"
#include "xbcrypt_common.h"
#include "datasink.h" #include "datasink.h"
#include "ds_decrypt.h"
#include "crc_glue.h" #include "crc_glue.h"
#define XBSTREAM_VERSION "1.0" #define XBSTREAM_VERSION "1.0"
...@@ -41,33 +39,18 @@ typedef enum { ...@@ -41,33 +39,18 @@ typedef enum {
RUN_MODE_EXTRACT RUN_MODE_EXTRACT
} run_mode_t; } run_mode_t;
const char *xbstream_encrypt_algo_names[] =
{ "NONE", "AES128", "AES192", "AES256", NullS};
TYPELIB xbstream_encrypt_algo_typelib=
{array_elements(xbstream_encrypt_algo_names)-1,"",
xbstream_encrypt_algo_names, NULL};
/* Need the following definitions to avoid linking with ds_*.o and their link /* Need the following definitions to avoid linking with ds_*.o and their link
dependencies */ dependencies */
datasink_t datasink_archive; datasink_t datasink_archive;
datasink_t datasink_xbstream; datasink_t datasink_xbstream;
datasink_t datasink_compress; datasink_t datasink_compress;
datasink_t datasink_tmpfile; datasink_t datasink_tmpfile;
datasink_t datasink_encrypt;
datasink_t datasink_buffer; datasink_t datasink_buffer;
static run_mode_t opt_mode; static run_mode_t opt_mode;
static char * opt_directory = NULL; static char * opt_directory = NULL;
static my_bool opt_verbose = 0; static my_bool opt_verbose = 0;
static int opt_parallel = 1; static int opt_parallel = 1;
static ulong opt_encrypt_algo;
static char *opt_encrypt_key_file = NULL;
static void *opt_encrypt_key = NULL;
static int opt_encrypt_threads = 1;
enum {
OPT_ENCRYPT_THREADS = 256
};
static struct my_option my_long_options[] = static struct my_option my_long_options[] =
{ {
...@@ -86,20 +69,6 @@ static struct my_option my_long_options[] = ...@@ -86,20 +69,6 @@ static struct my_option my_long_options[] =
{"parallel", 'p', "Number of worker threads for reading / writing.", {"parallel", 'p', "Number of worker threads for reading / writing.",
&opt_parallel, &opt_parallel, 0, GET_INT, REQUIRED_ARG, &opt_parallel, &opt_parallel, 0, GET_INT, REQUIRED_ARG,
1, 1, INT_MAX, 0, 0, 0}, 1, 1, INT_MAX, 0, 0, 0},
{"decrypt", 'd', "Decrypt files ending with .xbcrypt.",
&opt_encrypt_algo, &opt_encrypt_algo, &xbstream_encrypt_algo_typelib,
GET_ENUM, REQUIRED_ARG, 0, 0, 0, 0, 0, 0},
{"encrypt-key", 'k', "Encryption key.",
&opt_encrypt_key, &opt_encrypt_key, 0,
GET_STR_ALLOC, REQUIRED_ARG, 0, 0, 0, 0, 0, 0},
{"encrypt-key-file", 'f', "File which contains encryption key.",
&opt_encrypt_key_file, &opt_encrypt_key_file, 0,
GET_STR_ALLOC, REQUIRED_ARG, 0, 0, 0, 0, 0, 0},
{"encrypt-threads", OPT_ENCRYPT_THREADS,
"Number of threads for parallel data encryption. "
"The default value is 1.",
&opt_encrypt_threads, &opt_encrypt_threads,
0, GET_INT, REQUIRED_ARG, 1, 1, INT_MAX, 0, 0, 0},
{0, 0, 0, 0, 0, 0, GET_NO_ARG, NO_ARG, 0, 0, 0, 0, 0, 0} {0, 0, 0, 0, 0, 0, GET_NO_ARG, NO_ARG, 0, 0, 0, 0, 0, 0}
}; };
...@@ -108,7 +77,6 @@ typedef struct { ...@@ -108,7 +77,6 @@ typedef struct {
HASH *filehash; HASH *filehash;
xb_rstream_t *stream; xb_rstream_t *stream;
ds_ctxt_t *ds_ctxt; ds_ctxt_t *ds_ctxt;
ds_ctxt_t *ds_decrypt_ctxt;
pthread_mutex_t *mutex; pthread_mutex_t *mutex;
} extract_ctxt_t; } extract_ctxt_t;
...@@ -348,19 +316,6 @@ mode_create(int argc, char **argv) ...@@ -348,19 +316,6 @@ mode_create(int argc, char **argv)
return 1; return 1;
} }
/************************************************************************
Check if string ends with given suffix.
@return true if string ends with given suffix. */
static
my_bool
ends_with(const char *str, const char *suffix)
{
size_t suffix_len = strlen(suffix);
size_t str_len = strlen(str);
return(str_len >= suffix_len
&& strcmp(str + str_len - suffix_len, suffix) == 0);
}
static static
file_entry_t * file_entry_t *
file_entry_new(extract_ctxt_t *ctxt, const char *path, uint pathlen) file_entry_new(extract_ctxt_t *ctxt, const char *path, uint pathlen)
...@@ -380,11 +335,8 @@ file_entry_new(extract_ctxt_t *ctxt, const char *path, uint pathlen) ...@@ -380,11 +335,8 @@ file_entry_new(extract_ctxt_t *ctxt, const char *path, uint pathlen)
} }
entry->pathlen = pathlen; entry->pathlen = pathlen;
if (ctxt->ds_decrypt_ctxt && ends_with(path, ".xbcrypt")) { file = ds_open(ctxt->ds_ctxt, path, NULL);
file = ds_open(ctxt->ds_decrypt_ctxt, path, NULL);
} else {
file = ds_open(ctxt->ds_ctxt, path, NULL);
}
if (file == NULL) { if (file == NULL) {
msg("%s: failed to create file.\n", my_progname); msg("%s: failed to create file.\n", my_progname);
goto err; goto err;
...@@ -534,7 +486,6 @@ mode_extract(int n_threads, int argc __attribute__((unused)), ...@@ -534,7 +486,6 @@ mode_extract(int n_threads, int argc __attribute__((unused)),
xb_rstream_t *stream = NULL; xb_rstream_t *stream = NULL;
HASH filehash; HASH filehash;
ds_ctxt_t *ds_ctxt = NULL; ds_ctxt_t *ds_ctxt = NULL;
ds_ctxt_t *ds_decrypt_ctxt = NULL;
extract_ctxt_t ctxt; extract_ctxt_t ctxt;
int i; int i;
pthread_t *tids = NULL; pthread_t *tids = NULL;
...@@ -574,7 +525,6 @@ mode_extract(int n_threads, int argc __attribute__((unused)), ...@@ -574,7 +525,6 @@ mode_extract(int n_threads, int argc __attribute__((unused)),
ctxt.stream = stream; ctxt.stream = stream;
ctxt.filehash = &filehash; ctxt.filehash = &filehash;
ctxt.ds_ctxt = ds_ctxt; ctxt.ds_ctxt = ds_ctxt;
ctxt.ds_decrypt_ctxt = ds_decrypt_ctxt;
ctxt.mutex = &mutex; ctxt.mutex = &mutex;
tids = malloc(sizeof(pthread_t) * n_threads); tids = malloc(sizeof(pthread_t) * n_threads);
...@@ -604,9 +554,6 @@ mode_extract(int n_threads, int argc __attribute__((unused)), ...@@ -604,9 +554,6 @@ mode_extract(int n_threads, int argc __attribute__((unused)),
if (ds_ctxt != NULL) { if (ds_ctxt != NULL) {
ds_destroy(ds_ctxt); ds_destroy(ds_ctxt);
} }
if (ds_decrypt_ctxt) {
ds_destroy(ds_decrypt_ctxt);
}
xb_stream_read_done(stream); xb_stream_read_done(stream);
return ret; return ret;
......
This diff is collapsed.
...@@ -81,7 +81,6 @@ extern char *xtrabackup_tables_exclude; ...@@ -81,7 +81,6 @@ extern char *xtrabackup_tables_exclude;
extern char *xtrabackup_databases_exclude; extern char *xtrabackup_databases_exclude;
extern ibool xtrabackup_compress; extern ibool xtrabackup_compress;
extern ibool xtrabackup_encrypt;
extern my_bool xtrabackup_backup; extern my_bool xtrabackup_backup;
extern my_bool xtrabackup_prepare; extern my_bool xtrabackup_prepare;
...@@ -92,15 +91,10 @@ extern my_bool xtrabackup_decrypt_decompress; ...@@ -92,15 +91,10 @@ extern my_bool xtrabackup_decrypt_decompress;
extern char *innobase_data_file_path; extern char *innobase_data_file_path;
extern char *innobase_doublewrite_file; extern char *innobase_doublewrite_file;
extern char *xtrabackup_encrypt_key;
extern char *xtrabackup_encrypt_key_file;
extern longlong innobase_log_file_size; extern longlong innobase_log_file_size;
extern long innobase_log_files_in_group; extern long innobase_log_files_in_group;
extern longlong innobase_page_size; extern longlong innobase_page_size;
extern const char *xtrabackup_encrypt_algo_names[];
extern TYPELIB xtrabackup_encrypt_algo_typelib;
extern int xtrabackup_parallel; extern int xtrabackup_parallel;
extern my_bool xb_close_files; extern my_bool xb_close_files;
...@@ -113,9 +107,6 @@ extern "C"{ ...@@ -113,9 +107,6 @@ extern "C"{
#ifdef __cplusplus #ifdef __cplusplus
} }
#endif #endif
extern ulong xtrabackup_encrypt_algo;
extern uint xtrabackup_encrypt_threads;
extern ulonglong xtrabackup_encrypt_chunk_size;
extern my_bool xtrabackup_export; extern my_bool xtrabackup_export;
extern char *xtrabackup_incremental_basedir; extern char *xtrabackup_incremental_basedir;
extern char *xtrabackup_extra_lsndir; extern char *xtrabackup_extra_lsndir;
...@@ -158,8 +149,6 @@ extern TYPELIB query_type_typelib; ...@@ -158,8 +149,6 @@ extern TYPELIB query_type_typelib;
extern ulong opt_lock_wait_query_type; extern ulong opt_lock_wait_query_type;
extern ulong opt_kill_long_query_type; extern ulong opt_kill_long_query_type;
extern ulong opt_decrypt_algo;
extern uint opt_kill_long_queries_timeout; extern uint opt_kill_long_queries_timeout;
extern uint opt_lock_wait_timeout; extern uint opt_lock_wait_timeout;
extern uint opt_lock_wait_threshold; extern uint opt_lock_wait_threshold;
...@@ -167,7 +156,6 @@ extern uint opt_debug_sleep_before_unlock; ...@@ -167,7 +156,6 @@ extern uint opt_debug_sleep_before_unlock;
extern uint opt_safe_slave_backup_timeout; extern uint opt_safe_slave_backup_timeout;
extern const char *opt_history; extern const char *opt_history;
extern my_bool opt_decrypt;
enum binlog_info_enum { BINLOG_INFO_OFF, BINLOG_INFO_LOCKLESS, BINLOG_INFO_ON, enum binlog_info_enum { BINLOG_INFO_OFF, BINLOG_INFO_LOCKLESS, BINLOG_INFO_ON,
BINLOG_INFO_AUTO}; BINLOG_INFO_AUTO};
......
...@@ -3049,3 +3049,21 @@ SET DEBUG_SYNC= 'RESET'; ...@@ -3049,3 +3049,21 @@ SET DEBUG_SYNC= 'RESET';
disconnect con1; disconnect con1;
disconnect con2; disconnect con2;
disconnect con3; disconnect con3;
#
# MDEV-12620 - set lock_wait_timeout = 1;flush tables with read lock;
# lock not released after timeout
#
CREATE TABLE t1(a INT) ENGINE=InnoDB;
SET debug_sync='open_tables_after_open_and_process_table SIGNAL ready WAIT_FOR go';
SELECT * FROM t1;
connect con1,localhost,root,,;
SET debug_sync='now WAIT_FOR ready';
SET lock_wait_timeout=1;
FLUSH TABLES WITH READ LOCK;
ERROR HY000: Lock wait timeout exceeded; try restarting transaction
SET debug_sync='now SIGNAL go';
connection default;
a
SET debug_sync='RESET';
DROP TABLE t1;
disconnect con1;
# Test resizing the InnoDB redo log. # Test resizing the InnoDB redo log.
--source include/have_innodb.inc --source include/innodb_page_size_small.inc
# Embedded server tests do not support restarting # Embedded server tests do not support restarting
--source include/not_embedded.inc --source include/not_embedded.inc
# DBUG_EXECUTE_IF is needed # DBUG_EXECUTE_IF is needed
...@@ -24,6 +24,12 @@ call mtr.add_suppression("InnoDB: Unable to open .*ib_logfile0. to check native ...@@ -24,6 +24,12 @@ call mtr.add_suppression("InnoDB: Unable to open .*ib_logfile0. to check native
FLUSH TABLES; FLUSH TABLES;
--enable_query_log --enable_query_log
--let $restart_parameters= --innodb-thread-concurrency=1 --innodb-log-file-size=1m --innodb-log-files-in-group=2
--source include/restart_mysqld.inc
--let $restart_parameters= --innodb-thread-concurrency=100 --innodb-log-file-size=10M --innodb-log-files-in-group=2
--source include/restart_mysqld.inc
CREATE TABLE t1(a INT PRIMARY KEY) ENGINE=InnoDB; CREATE TABLE t1(a INT PRIMARY KEY) ENGINE=InnoDB;
BEGIN; BEGIN;
INSERT INTO t1 VALUES (42); INSERT INTO t1 VALUES (42);
......
...@@ -13,7 +13,6 @@ INSERT INTO t VALUES(2); ...@@ -13,7 +13,6 @@ INSERT INTO t VALUES(2);
echo # xtrabackup prepare; echo # xtrabackup prepare;
--disable_result_log --disable_result_log
exec $XTRABACKUP --prepare --target-dir=$targetdir; exec $XTRABACKUP --prepare --target-dir=$targetdir;
exec $XTRABACKUP --defaults-file=$targetdir/backup-my.cnf --stats --datadir=$targetdir;
-- source include/restart_and_restore.inc -- source include/restart_and_restore.inc
--enable_result_log --enable_result_log
......
...@@ -27,11 +27,6 @@ echo # Prepare full backup, apply incremental one; ...@@ -27,11 +27,6 @@ echo # Prepare full backup, apply incremental one;
exec $XTRABACKUP --prepare --apply-log-only --target-dir=$basedir; exec $XTRABACKUP --prepare --apply-log-only --target-dir=$basedir;
exec $XTRABACKUP --prepare --target-dir=$basedir --incremental-dir=$incremental_dir; exec $XTRABACKUP --prepare --target-dir=$basedir --incremental-dir=$incremental_dir;
# stats also can support encryption, but needs plugin-load and plugin variables, they are stored in backup-my.cnf
# We need to prepare again to create log files though.
exec $XTRABACKUP --prepare --target-dir=$basedir;
exec $XTRABACKUP --defaults-file=$basedir/backup-my.cnf --stats --datadir=$basedir;
echo # Restore and check results; echo # Restore and check results;
let $targetdir=$basedir; let $targetdir=$basedir;
-- source include/restart_and_restore.inc -- source include/restart_and_restore.inc
......
...@@ -19,7 +19,6 @@ echo # xtrabackup prepare; ...@@ -19,7 +19,6 @@ echo # xtrabackup prepare;
--disable_result_log --disable_result_log
exec $XTRABACKUP --prepare --target-dir=$targetdir; exec $XTRABACKUP --prepare --target-dir=$targetdir;
exec $XTRABACKUP --defaults-file=$targetdir/backup-my.cnf --stats --datadir=$targetdir ;
-- source include/restart_and_restore.inc -- source include/restart_and_restore.inc
--enable_result_log --enable_result_log
......
...@@ -1778,8 +1778,8 @@ DEFAULT_VALUE 50331648 ...@@ -1778,8 +1778,8 @@ DEFAULT_VALUE 50331648
VARIABLE_SCOPE GLOBAL VARIABLE_SCOPE GLOBAL
VARIABLE_TYPE BIGINT VARIABLE_TYPE BIGINT
VARIABLE_COMMENT Size of each log file in a log group. VARIABLE_COMMENT Size of each log file in a log group.
NUMERIC_MIN_VALUE 4194304 NUMERIC_MIN_VALUE 1048576
NUMERIC_MAX_VALUE 9223372036854775807 NUMERIC_MAX_VALUE 549755813888
NUMERIC_BLOCK_SIZE 1048576 NUMERIC_BLOCK_SIZE 1048576
ENUM_VALUE_LIST NULL ENUM_VALUE_LIST NULL
READ_ONLY YES READ_ONLY YES
......
...@@ -4075,6 +4075,30 @@ disconnect con2; ...@@ -4075,6 +4075,30 @@ disconnect con2;
disconnect con3; disconnect con3;
--echo #
--echo # MDEV-12620 - set lock_wait_timeout = 1;flush tables with read lock;
--echo # lock not released after timeout
--echo #
CREATE TABLE t1(a INT) ENGINE=InnoDB;
SET debug_sync='open_tables_after_open_and_process_table SIGNAL ready WAIT_FOR go';
send SELECT * FROM t1;
connect (con1,localhost,root,,);
SET debug_sync='now WAIT_FOR ready';
# lock_wait_timeout should be 0 in 10.3, so that we don't have to wait at all
SET lock_wait_timeout=1;
--error ER_LOCK_WAIT_TIMEOUT
FLUSH TABLES WITH READ LOCK;
SET debug_sync='now SIGNAL go';
connection default;
reap;
SET debug_sync='RESET';
DROP TABLE t1;
disconnect con1;
# Check that all connections opened by test cases in this file are really # Check that all connections opened by test cases in this file are really
# gone so execution of other tests won't be affected by their presence. # gone so execution of other tests won't be affected by their presence.
--source include/wait_until_count_sessions.inc --source include/wait_until_count_sessions.inc
...@@ -253,7 +253,8 @@ bool reload_acl_and_cache(THD *thd, unsigned long long options, ...@@ -253,7 +253,8 @@ bool reload_acl_and_cache(THD *thd, unsigned long long options,
NOTE: my_error() has been already called by reopen_tables() within NOTE: my_error() has been already called by reopen_tables() within
close_cached_tables(). close_cached_tables().
*/ */
result= 1; thd->global_read_lock.unlock_global_read_lock(thd);
return 1;
} }
if (thd->global_read_lock.make_global_read_lock_block_commit(thd)) // Killed if (thd->global_read_lock.make_global_read_lock_block_commit(thd)) // Killed
......
...@@ -21202,7 +21202,7 @@ static MYSQL_SYSVAR_LONG(log_buffer_size, innobase_log_buffer_size, ...@@ -21202,7 +21202,7 @@ static MYSQL_SYSVAR_LONG(log_buffer_size, innobase_log_buffer_size,
static MYSQL_SYSVAR_LONGLONG(log_file_size, innobase_log_file_size, static MYSQL_SYSVAR_LONGLONG(log_file_size, innobase_log_file_size,
PLUGIN_VAR_RQCMDARG | PLUGIN_VAR_READONLY, PLUGIN_VAR_RQCMDARG | PLUGIN_VAR_READONLY,
"Size of each log file in a log group.", "Size of each log file in a log group.",
NULL, NULL, 48*1024*1024L, 4*1024*1024L, LLONG_MAX, 1024*1024L); NULL, NULL, 48 << 20, 1 << 20, 512ULL << 30, 1 << 20);
static MYSQL_SYSVAR_ULONG(log_files_in_group, srv_n_log_files, static MYSQL_SYSVAR_ULONG(log_files_in_group, srv_n_log_files,
PLUGIN_VAR_RQCMDARG | PLUGIN_VAR_READONLY, PLUGIN_VAR_RQCMDARG | PLUGIN_VAR_READONLY,
......
...@@ -677,14 +677,13 @@ log_set_capacity() ...@@ -677,14 +677,13 @@ log_set_capacity()
{ {
lsn_t margin; lsn_t margin;
ulint free; ulint free;
bool success = true;
lsn_t smallest_capacity;
log_mutex_enter(); lsn_t smallest_capacity = ((srv_log_file_size_requested
<< srv_page_size_shift)
smallest_capacity = log_group_get_capacity(&log_sys->log); - LOG_FILE_HDR_SIZE)
* srv_n_log_files;
/* Add extra safety */ /* Add extra safety */
smallest_capacity = smallest_capacity - smallest_capacity / 10; smallest_capacity -= smallest_capacity / 10;
/* For each OS thread we must reserve so much free space in the /* For each OS thread we must reserve so much free space in the
smallest log group that it can accommodate the log entries produced smallest log group that it can accommodate the log entries produced
...@@ -694,15 +693,20 @@ log_set_capacity() ...@@ -694,15 +693,20 @@ log_set_capacity()
free = LOG_CHECKPOINT_FREE_PER_THREAD * (10 + srv_thread_concurrency) free = LOG_CHECKPOINT_FREE_PER_THREAD * (10 + srv_thread_concurrency)
+ LOG_CHECKPOINT_EXTRA_FREE; + LOG_CHECKPOINT_EXTRA_FREE;
if (free >= smallest_capacity / 2) { if (free >= smallest_capacity / 2) {
success = false; ib::error() << "Cannot continue operation. ib_logfiles are too"
" small for innodb_thread_concurrency="
goto failure; << srv_thread_concurrency << ". The combined size of"
} else { " ib_logfiles should be bigger than"
margin = smallest_capacity - free; " 200 kB * innodb_thread_concurrency. "
<< INNODB_PARAMETERS_MSG;
return(false);
} }
margin = smallest_capacity - free;
margin = margin - margin / 10; /* Add still some extra safety */ margin = margin - margin / 10; /* Add still some extra safety */
log_mutex_enter();
log_sys->log_group_capacity = smallest_capacity; log_sys->log_group_capacity = smallest_capacity;
log_sys->max_modified_age_async = margin log_sys->max_modified_age_async = margin
...@@ -714,19 +718,9 @@ log_set_capacity() ...@@ -714,19 +718,9 @@ log_set_capacity()
/ LOG_POOL_CHECKPOINT_RATIO_ASYNC; / LOG_POOL_CHECKPOINT_RATIO_ASYNC;
log_sys->max_checkpoint_age = margin; log_sys->max_checkpoint_age = margin;
failure:
log_mutex_exit(); log_mutex_exit();
if (!success) { return(true);
ib::error() << "Cannot continue operation. ib_logfiles are too"
" small for innodb_thread_concurrency="
<< srv_thread_concurrency << ". The combined size of"
" ib_logfiles should be bigger than"
" 200 kB * innodb_thread_concurrency. "
<< INNODB_PARAMETERS_MSG;
}
return(success);
} }
/** Initializes the redo logging subsystem. */ /** Initializes the redo logging subsystem. */
......
...@@ -860,43 +860,16 @@ ibool ...@@ -860,43 +860,16 @@ ibool
log_calc_max_ages(void) log_calc_max_ages(void)
/*===================*/ /*===================*/
{ {
log_group_t* group;
lsn_t margin; lsn_t margin;
ulint free; ulint free;
ibool success = TRUE;
lsn_t smallest_capacity;
lsn_t archive_margin;
lsn_t smallest_archive_margin;
mutex_enter(&(log_sys->mutex));
group = UT_LIST_GET_FIRST(log_sys->log_groups);
ut_ad(group);
smallest_capacity = LSN_MAX;
smallest_archive_margin = LSN_MAX;
while (group) {
if (log_group_get_capacity(group) < smallest_capacity) {
smallest_capacity = log_group_get_capacity(group); lsn_t smallest_capacity = ((srv_log_file_size_requested
} << srv_page_size_shift)
- LOG_FILE_HDR_SIZE)
archive_margin = log_group_get_capacity(group) * srv_n_log_files;
- (group->file_size - LOG_FILE_HDR_SIZE)
- LOG_ARCHIVE_EXTRA_MARGIN;
if (archive_margin < smallest_archive_margin) {
smallest_archive_margin = archive_margin;
}
group = UT_LIST_GET_NEXT(log_groups, group);
}
/* Add extra safety */ /* Add extra safety */
smallest_capacity = smallest_capacity - smallest_capacity / 10; smallest_capacity -= smallest_capacity / 10;
/* For each OS thread we must reserve so much free space in the /* For each OS thread we must reserve so much free space in the
smallest log group that it can accommodate the log entries produced smallest log group that it can accommodate the log entries produced
...@@ -906,15 +879,16 @@ log_calc_max_ages(void) ...@@ -906,15 +879,16 @@ log_calc_max_ages(void)
free = LOG_CHECKPOINT_FREE_PER_THREAD * (10 + srv_thread_concurrency) free = LOG_CHECKPOINT_FREE_PER_THREAD * (10 + srv_thread_concurrency)
+ LOG_CHECKPOINT_EXTRA_FREE; + LOG_CHECKPOINT_EXTRA_FREE;
if (free >= smallest_capacity / 2) { if (free >= smallest_capacity / 2) {
success = FALSE; ib_logf(IB_LOG_LEVEL_FATAL,
"The combined size of ib_logfiles"
goto failure; " should be bigger than\n"
} else { "InnoDB: 200 kB * innodb_thread_concurrency.");
margin = smallest_capacity - free;
} }
margin = smallest_capacity - free;
margin = margin - margin / 10; /* Add still some extra safety */ margin = margin - margin / 10; /* Add still some extra safety */
mutex_enter(&log_sys->mutex);
log_sys->log_group_capacity = smallest_capacity; log_sys->log_group_capacity = smallest_capacity;
log_sys->max_modified_age_async = margin log_sys->max_modified_age_async = margin
...@@ -927,22 +901,17 @@ log_calc_max_ages(void) ...@@ -927,22 +901,17 @@ log_calc_max_ages(void)
log_sys->max_checkpoint_age = margin; log_sys->max_checkpoint_age = margin;
#ifdef UNIV_LOG_ARCHIVE #ifdef UNIV_LOG_ARCHIVE
log_sys->max_archived_lsn_age = smallest_archive_margin; lsn_t archive_margin = smallest_capacity
- (srv_log_file_size_requested - LOG_FILE_HDR_SIZE)
- LOG_ARCHIVE_EXTRA_MARGIN;
log_sys->max_archived_lsn_age = archive_margin;
log_sys->max_archived_lsn_age_async = smallest_archive_margin log_sys->max_archived_lsn_age_async = archive_margin
- smallest_archive_margin / LOG_ARCHIVE_RATIO_ASYNC; - archive_margin / LOG_ARCHIVE_RATIO_ASYNC;
#endif /* UNIV_LOG_ARCHIVE */ #endif /* UNIV_LOG_ARCHIVE */
failure: mutex_exit(&log_sys->mutex);
mutex_exit(&(log_sys->mutex));
if (!success) {
ib_logf(IB_LOG_LEVEL_FATAL,
"The combined size of ib_logfiles"
" should be bigger than\n"
"InnoDB: 200 kB * innodb_thread_concurrency.");
}
return(success); return(true);
} }
/******************************************************//** /******************************************************//**
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment