Xtrabackup 2.3.8

extra/mariabackup/CMakeLists.txt

-# Copyright (c) 2013 Percona LLC and/or its affiliates.
+# Copyright (c) 2013, 2017 Percona LLC and/or its affiliates.
 #
 # This program is free software; you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by
 # the Free Software Foundation; version 2 of the License.
-
+#
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
-
+#
 # You should have received a copy of the GNU General Public License
 # along with this program; if not, write to the Free Software
 # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA
@@ -129,6 +129,7 @@ INCLUDE_DIRECTORIES(
   ${CMAKE_SOURCE_DIR}/sql
   ${CMAKE_CURRENT_SOURCE_DIR}/quicklz
   ${CMAKE_CURRENT_SOURCE_DIR}
+  ${CMAKE_CURRENT_SOURCE_DIR}/crc
   )
 
 IF(NOT HAVE_SYSTEM_REGEX)
@@ -156,7 +157,6 @@ MYSQL_ADD_EXECUTABLE(mariabackup
   ${DS_ARCHIVE_SOURCE}
   ds_buffer.c
   ds_compress.c
-  ds_encrypt.c
   ds_local.c
   ds_stdout.c
   ds_tmpfile.c
@@ -166,8 +166,6 @@ MYSQL_ADD_EXECUTABLE(mariabackup
   read_filt.cc
   write_filt.cc
   wsrep.cc
-  xbcrypt_common.c
-  xbcrypt_write.c
   xbstream_write.c
   backup_mysql.cc
   backup_copy.cc
@@ -181,9 +179,10 @@ MYSQL_ADD_EXECUTABLE(mariabackup
 
 # Export all symbols on Unix, for better crash callstacks
 SET_TARGET_PROPERTIES(mariabackup PROPERTIES ENABLE_EXPORTS TRUE)
+ADD_SUBDIRECTORY(crc)
 
 
-TARGET_LINK_LIBRARIES(mariabackup sql)
+TARGET_LINK_LIBRARIES(mariabackup sql crc)
 
 IF(NOT HAVE_SYSTEM_REGEX)
   TARGET_LINK_LIBRARIES(mariabackup pcreposix)
@@ -201,13 +200,13 @@ MYSQL_ADD_EXECUTABLE(mbstream
   xbstream.c
   xbstream_read.c
   xbstream_write.c
-
   COMPONENT backup
   )
 
 
 TARGET_LINK_LIBRARIES(mbstream
   mysys
+  crc
 )
 
 IF(MSVC)

extra/mariabackup/backup_copy.cc

@@ -265,6 +265,11 @@ datadir_iter_next_database(datadir_iter_t *it)
 			return(true);
 		}
 
+		if (check_if_skip_database_by_path(it->dbpath)) {
+			msg("Skipping db: %s\n", it->dbpath);
+			continue;
+		}
+
 		/* We want wrong directory permissions to be a fatal error for
 		XtraBackup. */
 		it->dbdir = os_file_opendir(it->dbpath, TRUE);
@@ -1704,7 +1709,7 @@ copy_back()
 
 		for (i = 1; i <= srv_undo_tablespaces; i++) {
 			char filename[20];
-			sprintf(filename, "undo%03lu", i);
+			sprintf(filename, "undo%03u", (uint)i);
 			if (!(ret = copy_or_move_file(filename, filename,
 				                      dst_dir, 1))) {
 				goto cleanup;

extra/mariabackup/backup_mysql.cc

@@ -1414,7 +1414,10 @@ write_xtrabackup_info(MYSQL *connection)
 	bool is_partial = (xtrabackup_tables
 		|| xtrabackup_tables_file
 		|| xtrabackup_databases
-		|| xtrabackup_databases_file);
+		|| xtrabackup_databases_file
+		|| xtrabackup_tables_exclude
+		|| xtrabackup_databases_exclude
+		);
 
 	backup_file_printf(XTRABACKUP_INFO,
 		"uuid = %s\n"

extra/mariabackup/crc/CMakeLists.txt

+# Copyright (c) 2017 Percona LLC and/or its affiliates.
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; version 2 of the License.
+
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA
+
+PROJECT(crc C)
+
+IF(NOT CMAKE_CROSSCOMPILING AND NOT MSVC)
+  STRING(TOLOWER ${CMAKE_SYSTEM_PROCESSOR}  processor)
+  IF(processor MATCHES "86" OR processor MATCHES "amd64" OR processor MATCHES "x64")
+  # Check for PCLMUL instruction
+  CHECK_C_SOURCE_RUNS("
+  int main()
+  {
+    asm volatile (\"pclmulqdq \\$0x00, %%xmm1, %%xmm0\":::\"cc\");
+    return 0;
+  }"  HAVE_CLMUL_INSTRUCTION)
+  ENDIF()
+ENDIF()
+IF(HAVE_CLMUL_INSTRUCTION)
+  ADD_DEFINITIONS(-DHAVE_CLMUL_INSTRUCTION)
+ENDIF()
+ADD_LIBRARY(crc crc_glue.c crc-intel-pclmul.c)

extra/mariabackup/crc/config.h.cmake

+/******************************************************
+Copyright (c) 2017 Percona LLC and/or its affiliates.
+
+Zlib compatible CRC-32 implementation.
+
+This program is free software; you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation; version 2 of the License.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program; if not, write to the Free Software
+Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA
+
+*******************************************************/
+
+#cmakedefine HAVE_CLMUL_INSTRUCTION 1

extra/mariabackup/crc/crc-intel-pclmul.h

+/******************************************************
+Copyright (c) 2017 Percona LLC and/or its affiliates.
+
+CRC32 using Intel's PCLMUL instruction.
+
+This program is free software; you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation; version 2 of the License.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program; if not, write to the Free Software
+Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA
+
+*******************************************************/
+
+#include <stdint.h>
+#include <stddef.h>
+
+void
+crc32_intel_pclmul(uint32_t *pcrc, const uint8_t *inbuf, size_t inlen);

extra/mariabackup/crc/crc_glue.c

+/******************************************************
+Copyright (c) 2017 Percona LLC and/or its affiliates.
+
+Zlib compatible CRC-32 implementation.
+
+This program is free software; you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation; version 2 of the License.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program; if not, write to the Free Software
+Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA
+
+*******************************************************/
+
+#include <zlib.h>
+#include <stdint.h>
+#include <string.h>
+#include "crc_glue.h"
+#include "crc-intel-pclmul.h"
+
+#if __GNUC__ >= 4 && defined(__x86_64__)
+static int pclmul_enabled = 0;
+#endif
+
+#if defined(__GNUC__) && defined(__x86_64__)
+static
+uint32_t
+cpuid(uint32_t*	ecx, uint32_t*	edx)
+{
+	uint32_t level;
+
+	asm("cpuid" : "=a"  (level) : "a" (0) : "ebx", "ecx", "edx");
+
+	if (level < 1) {
+		return level;
+	}
+
+	asm("cpuid" : "=c" (*ecx), "=d" (*edx)
+	    : "a" (1)
+	    : "ebx");
+
+	return level;
+}
+#endif
+
+void crc_init() {
+#if defined(__GNUC__) && defined(__x86_64__)
+	uint32_t ecx, edx;
+
+	if (cpuid(&ecx, &edx) > 0) {
+		pclmul_enabled = ((ecx >> 19) & 1) && ((ecx >> 1) & 1);
+	}
+#endif
+}
+
+unsigned long crc32_iso3309(unsigned long crc, const unsigned char *buf, unsigned int len)
+{
+#if __GNUC__ >= 4 && defined(__x86_64__) && defined(HAVE_CLMUL_INSTRUCTION)
+	if (pclmul_enabled) {
+		uint32_t crc_accum = crc ^ 0xffffffffL;
+		crc32_intel_pclmul(&crc_accum, buf, len);
+		return crc_accum ^ 0xffffffffL;
+	}
+#endif
+	return crc32(crc, buf, len);
+}

extra/mariabackup/crc/crc_glue.h

+/******************************************************
+Copyright (c) 2017 Percona LLC and/or its affiliates.
+
+Zlib compatible CRC-32 implementation.
+
+This program is free software; you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation; version 2 of the License.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program; if not, write to the Free Software
+Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA
+
+*******************************************************/
+
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+void crc_init();
+unsigned long crc32_iso3309(unsigned long crc, const unsigned char *buf, unsigned int len);
+
+#ifdef __cplusplus
+}
+#endif

extra/mariabackup/datasink.c

@@ -27,7 +27,6 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA
 #include "ds_local.h"
 #include "ds_stdout.h"
 #include "ds_tmpfile.h"
-#include "ds_encrypt.h"
 #include "ds_buffer.h"
 
 /************************************************************************
@@ -60,6 +59,7 @@ ds_create(const char *root, ds_type_t type)
 		ds = &datasink_compress;
 		break;
 	case DS_TYPE_ENCRYPT:
+  case DS_TYPE_DECRYPT:
 		msg("Error : mariabackup does not support encrypted backups.");
 		exit(EXIT_FAILURE);
 		break;

extra/mariabackup/datasink.h

@@ -61,6 +61,7 @@ typedef enum {
 	DS_TYPE_XBSTREAM,
 	DS_TYPE_COMPRESS,
 	DS_TYPE_ENCRYPT,
+	DS_TYPE_DECRYPT,
 	DS_TYPE_TMPFILE,
 	DS_TYPE_BUFFER
 } ds_type_t;

extra/mariabackup/ds_decrypt.h

+/******************************************************
+Copyright (c) 2017 Percona LLC and/or its affiliates.
+
+Encryption interface for XtraBackup.
+
+This program is free software; you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation; version 2 of the License.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program; if not, write to the Free Software
+Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA
+
+*******************************************************/
+
+#ifndef DS_DECRYPT_H
+#define DS_DECRYPT_H
+
+#include "datasink.h"
+
+extern datasink_t	datasink_decrypt;
+
+extern int		ds_decrypt_encrypt_threads;
+
+#endif

extra/mariabackup/ds_encrypt.c

@@ -22,25 +22,11 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA
 #include <my_base.h>
 #include "common.h"
 #include "datasink.h"
+#include "xbcrypt_common.h"
 #ifdef HAVE_GRYPT
-#if GCC_VERSION >= 4002
-/* Workaround to avoid "gcry_ac_* is deprecated" warnings in gcrypt.h */
-#  pragma GCC diagnostic ignored "-Wdeprecated-declarations"
-#endif
-
-#include <gcrypt.h>
-
-#if GCC_VERSION >= 4002
-#  pragma GCC diagnostic warning "-Wdeprecated-declarations"
-#endif
-
 #include "xbcrypt.h"
 
-#if !defined(GCRYPT_VERSION_NUMBER) || (GCRYPT_VERSION_NUMBER < 0x010600)
-GCRY_THREAD_OPTION_PTHREAD_IMPL;
-#endif
-
-#define XB_CRYPT_CHUNK_SIZE ((size_t) (xtrabackup_encrypt_chunk_size))
+#define XB_CRYPT_CHUNK_SIZE ((size_t) (ds_encrypt_encrypt_chunk_size))
 
 typedef struct {
 	pthread_t		id;
@@ -52,10 +38,10 @@ typedef struct {
 	my_bool			started;
 	my_bool			data_avail;
 	my_bool			cancelled;
-	const char 		*from;
+	const uchar 		*from;
 	size_t			from_len;
-	char			*to;
-	char			*iv;
+	uchar			*to;
+	uchar			*iv;
 	size_t			to_len;
 	gcry_cipher_hd_t	cipher_handle;
 } crypt_thread_ctxt_t;
@@ -73,11 +59,8 @@ typedef struct {
 } ds_encrypt_file_t;
 
 /* Encryption options */
-extern ulong		xtrabackup_encrypt_algo;
-extern char		*xtrabackup_encrypt_key;
-extern char		*xtrabackup_encrypt_key_file;
-extern uint		xtrabackup_encrypt_threads;
-extern ulonglong	xtrabackup_encrypt_chunk_size;
+uint		ds_encrypt_encrypt_threads;
+ulonglong	ds_encrypt_encrypt_chunk_size;
 
 static ds_ctxt_t *encrypt_init(const char *root);
 static ds_file_t *encrypt_open(ds_ctxt_t *ctxt, const char *path,
@@ -98,12 +81,7 @@ static crypt_thread_ctxt_t *create_worker_threads(uint n);
 static void destroy_worker_threads(crypt_thread_ctxt_t *threads, uint n);
 static void *encrypt_worker_thread_func(void *arg);
 
-static uint encrypt_algos[] = { GCRY_CIPHER_NONE, GCRY_CIPHER_AES128,
-				GCRY_CIPHER_AES192, GCRY_CIPHER_AES256 };
-static uint encrypt_algo;
-static const uint encrypt_mode = GCRY_CIPHER_MODE_CTR;
-static uint encrypt_key_len = 0;
-static size_t encrypt_iv_len = 0;
+static uint encrypt_iv_len = 0;
 
 static
 ssize_t
@@ -129,87 +107,13 @@ encrypt_init(const char *root)
 	ds_ctxt_t		*ctxt;
 	ds_encrypt_ctxt_t	*encrypt_ctxt;
 	crypt_thread_ctxt_t	*threads;
-	gcry_error_t 		gcry_error;
-
-	/* Acording to gcrypt docs (and my testing), setting up the threading
-	   callbacks must be done first, so, lets give it a shot */
-#if !defined(GCRYPT_VERSION_NUMBER) || (GCRYPT_VERSION_NUMBER < 0x010600)
-	gcry_error = gcry_control(GCRYCTL_SET_THREAD_CBS, &gcry_threads_pthread);
-	if (gcry_error) {
-		msg("encrypt: unable to set libgcrypt thread cbs - "
-		    "%s : %s\n",
-		    gcry_strsource(gcry_error),
-		    gcry_strerror(gcry_error));
-		return NULL;
-	}
-#endif
-
-	/* Version check should be the very next call because it
-	makes sure that important subsystems are intialized. */
-	if (!gcry_control(GCRYCTL_ANY_INITIALIZATION_P)) {
-		const char	*gcrypt_version;
-		gcrypt_version = gcry_check_version(NULL);
-		/* No other library has already initialized libgcrypt. */
-		if (!gcrypt_version) {
-			msg("encrypt: failed to initialize libgcrypt\n");
-			return NULL;
-		} else {
-			msg("encrypt: using gcrypt %s\n", gcrypt_version);
-		}
-	}
 
-	/* Disable the gcry secure memory, not dealing with this for now */
-	gcry_error = gcry_control(GCRYCTL_DISABLE_SECMEM, 0);
-	if (gcry_error) {
-		msg("encrypt: unable to disable libgcrypt secmem - "
-		    "%s : %s\n",
-		    gcry_strsource(gcry_error),
-		    gcry_strerror(gcry_error));
-		return NULL;
-	}
-
-	/* Finalize gcry initialization. */
-	gcry_error = gcry_control(GCRYCTL_INITIALIZATION_FINISHED, 0);
-	if (gcry_error) {
-		msg("encrypt: unable to finish libgcrypt initialization - "
-		    "%s : %s\n",
-		    gcry_strsource(gcry_error),
-		    gcry_strerror(gcry_error));
-		return NULL;
-	}
-
-	/* Determine the algorithm */
-	encrypt_algo = encrypt_algos[xtrabackup_encrypt_algo];
-
-	/* Set up the iv length */
-	encrypt_iv_len = gcry_cipher_get_algo_blklen(encrypt_algo);
-	xb_a(encrypt_iv_len > 0);
-
-	/* Now set up the key */
-	if (xtrabackup_encrypt_key == NULL &&
-	    xtrabackup_encrypt_key_file == NULL) {
-		msg("encrypt: no encryption key or key file specified.\n");
-		return NULL;
-	} else if (xtrabackup_encrypt_key && xtrabackup_encrypt_key_file) {
-		msg("encrypt: both encryption key and key file specified.\n");
-		return NULL;
-	} else if (xtrabackup_encrypt_key_file) {
-		if (!xb_crypt_read_key_file(xtrabackup_encrypt_key_file,
-					    (void**)&xtrabackup_encrypt_key,
-					    &encrypt_key_len)) {
-			msg("encrypt: unable to read encryption key file"
-			    " \"%s\".\n", xtrabackup_encrypt_key_file);
-			return NULL;
-		}
-	} else if (xtrabackup_encrypt_key) {
-		encrypt_key_len = strlen(xtrabackup_encrypt_key);
-	} else {
-		msg("encrypt: no encryption key or key file specified.\n");
+	if (xb_crypt_init(&encrypt_iv_len)) {
 		return NULL;
 	}
 
 	/* Create and initialize the worker threads */
-	threads = create_worker_threads(xtrabackup_encrypt_threads);
+	threads = create_worker_threads(ds_encrypt_encrypt_threads);
 	if (threads == NULL) {
 		msg("encrypt: failed to create worker threads.\n");
 		return NULL;
@@ -221,7 +125,7 @@ encrypt_init(const char *root)
 
 	encrypt_ctxt = (ds_encrypt_ctxt_t *) (ctxt + 1);
 	encrypt_ctxt->threads = threads;
-	encrypt_ctxt->nthreads = xtrabackup_encrypt_threads;
+	encrypt_ctxt->nthreads = ds_encrypt_encrypt_threads;
 
 	ctxt->ptr = encrypt_ctxt;
 	ctxt->root = my_strdup(root, MYF(MY_FAE));
@@ -294,7 +198,7 @@ encrypt_write(ds_file_t *file, const void *buf, size_t len)
 	crypt_thread_ctxt_t	*thd;
 	uint			nthreads;
 	uint			i;
-	const char		*ptr;
+	const uchar		*ptr;
 
 	crypt_file = (ds_encrypt_file_t *) file->ptr;
 	crypt_ctxt = crypt_file->crypt_ctxt;
@@ -302,7 +206,7 @@ encrypt_write(ds_file_t *file, const void *buf, size_t len)
 	threads = crypt_ctxt->threads;
 	nthreads = crypt_ctxt->nthreads;
 
-	ptr = (const char *) buf;
+	ptr = (const uchar *) buf;
 	while (len > 0) {
 		uint max_thread;
 
@@ -403,10 +307,6 @@ encrypt_deinit(ds_ctxt_t *ctxt)
 
 	my_free(ctxt->root);
 	my_free(ctxt);
-	if (xtrabackup_encrypt_key)
-		my_free(xtrabackup_encrypt_key);
-	if (xtrabackup_encrypt_key_file)
-		my_free(xtrabackup_encrypt_key_file);
 }
 
 static
@@ -427,11 +327,10 @@ create_worker_threads(uint n)
 		thd->cancelled = FALSE;
 		thd->data_avail = FALSE;
 
-		thd->to = (char *) my_malloc(XB_CRYPT_CHUNK_SIZE +
-					     XB_CRYPT_HASH_LEN, MYF(MY_FAE));
+		thd->to = (uchar *) my_malloc(XB_CRYPT_CHUNK_SIZE +
+					      XB_CRYPT_HASH_LEN, MYF(MY_FAE));
 
-		thd->iv = (char *) my_malloc(encrypt_iv_len,
-						   MYF(MY_FAE));
+		thd->iv = (uchar *) my_malloc(encrypt_iv_len, MYF(MY_FAE));
 
 		/* Initialize the control mutex and condition var */
 		if (pthread_mutex_init(&thd->ctrl_mutex, NULL) ||
@@ -445,32 +344,8 @@ create_worker_threads(uint n)
 			goto err;
 		}
 
-		if (encrypt_algo != GCRY_CIPHER_NONE) {
-			gcry_error_t 		gcry_error;
-
-			gcry_error = gcry_cipher_open(&thd->cipher_handle,
-						      encrypt_algo,
-						      encrypt_mode, 0);
-			if (gcry_error) {
-				msg("encrypt: unable to open libgcrypt"
-				    " cipher - %s : %s\n",
-				    gcry_strsource(gcry_error),
-				    gcry_strerror(gcry_error));
-				gcry_cipher_close(thd->cipher_handle);
-				goto err;
-			}
-
-			gcry_error = gcry_cipher_setkey(thd->cipher_handle,
-							xtrabackup_encrypt_key,
-							encrypt_key_len);
-			if (gcry_error) {
-				msg("encrypt: unable to set libgcrypt"
-				    " cipher key - %s : %s\n",
-				    gcry_strsource(gcry_error),
-				    gcry_strerror(gcry_error));
-				gcry_cipher_close(thd->cipher_handle);
-				goto err;
-			}
+		if (xb_crypt_cipher_open(&thd->cipher_handle)) {
+			goto err;
 		}
 
 		pthread_mutex_lock(&thd->ctrl_mutex);
@@ -519,8 +394,7 @@ destroy_worker_threads(crypt_thread_ctxt_t *threads, uint n)
 		pthread_cond_destroy(&thd->ctrl_cond);
 		pthread_mutex_destroy(&thd->ctrl_mutex);
 
-		if (encrypt_algo != GCRY_CIPHER_NONE)
-			gcry_cipher_close(thd->cipher_handle);
+		xb_crypt_cipher_close(thd->cipher_handle);
 
 		my_free(thd->to);
 		my_free(thd->iv);
@@ -555,60 +429,14 @@ encrypt_worker_thread_func(void *arg)
 		if (thd->cancelled)
 			break;
 
-		/* ensure that XB_CRYPT_HASH_LEN is the correct length
-		of XB_CRYPT_HASH hashing algorithm output */
-		assert(gcry_md_get_algo_dlen(XB_CRYPT_HASH) ==
-		       XB_CRYPT_HASH_LEN);
-
-		memcpy(thd->to, thd->from, thd->from_len);
-		gcry_md_hash_buffer(XB_CRYPT_HASH, thd->to + thd->from_len,
-				    thd->from, thd->from_len);
 		thd->to_len = thd->from_len;
 
-		if (encrypt_algo != GCRY_CIPHER_NONE) {
-			gcry_error_t 		gcry_error;
-
-			gcry_error = gcry_cipher_reset(thd->cipher_handle);
-			if (gcry_error) {
-				msg("encrypt: unable to reset cipher - "
-				    "%s : %s\n",
-				    gcry_strsource(gcry_error),
-				    gcry_strerror(gcry_error));
-				thd->to_len = 0;
-				continue;
-			}
-
-			xb_crypt_create_iv(thd->iv, encrypt_iv_len);
-			gcry_error = gcry_cipher_setctr(thd->cipher_handle,
-							thd->iv,
-							encrypt_iv_len);
-			if (gcry_error) {
-				msg("encrypt: unable to set cipher ctr - "
-				    "%s : %s\n",
-				    gcry_strsource(gcry_error),
-				    gcry_strerror(gcry_error));
-				thd->to_len = 0;
-				continue;
-			}
-
-			gcry_error = gcry_cipher_encrypt(thd->cipher_handle,
-							 thd->to,
-							 thd->to_len +
-							 XB_CRYPT_HASH_LEN,
-							 thd->to,
-							 thd->from_len +
-							 XB_CRYPT_HASH_LEN);
-			if (gcry_error) {
-				msg("encrypt: unable to encrypt buffer - "
-				    "%s : %s\n", gcry_strsource(gcry_error),
-				    gcry_strerror(gcry_error));
-				thd->to_len = 0;
-			}
-		} else {
-			memcpy(thd->to, thd->from,
-			       thd->from_len + XB_CRYPT_HASH_LEN);
+		if (xb_crypt_encrypt(thd->cipher_handle, thd->from,
+				     thd->from_len, thd->to, &thd->to_len,
+				     thd->iv)) {
+			thd->to_len = 0;
+			continue;
 		}
-		thd->to_len += XB_CRYPT_HASH_LEN;
 	}
 
 	pthread_mutex_unlock(&thd->data_mutex);

extra/mariabackup/ds_encrypt.h

@@ -25,4 +25,9 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA
 #ifdef HAVE_GCRYPT
 extern datasink_t datasink_encrypt;
 #endif
+/* Encryption options */
+extern uint		ds_encrypt_encrypt_threads;
+extern ulonglong	ds_encrypt_encrypt_chunk_size;
+
+
 #endif

extra/mariabackup/xbcrypt.c

@@ -22,7 +22,8 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA
 #include <my_getopt.h>
 #include "common.h"
 #include "xbcrypt.h"
-#include <gcrypt.h>
+#include "xbcrypt_common.h"
+#include "crc_glue.h"
 
 #if !defined(GCRYPT_VERSION_NUMBER) || (GCRYPT_VERSION_NUMBER < 0x010600)
 GCRY_THREAD_OPTION_PTHREAD_IMPL;
@@ -138,6 +139,8 @@ main(int argc, char **argv)
 
 	MY_INIT(argv[0]);
 
+	crc_init();
+
 	if (get_options(&argc, &argv)) {
 		goto err;
 	}
@@ -402,8 +405,8 @@ mode_decrypt(File filein, File fileout)
 
 			/* ensure that XB_CRYPT_HASH_LEN is the correct length
 			of XB_CRYPT_HASH hashing algorithm output */
-			assert(gcry_md_get_algo_dlen(XB_CRYPT_HASH) ==
-			       XB_CRYPT_HASH_LEN);
+			xb_a(gcry_md_get_algo_dlen(XB_CRYPT_HASH) ==
+			     XB_CRYPT_HASH_LEN);
 			gcry_md_hash_buffer(XB_CRYPT_HASH, hash, decryptbuf,
 					    originalsize);
 			if (memcmp(hash, (char *) decryptbuf + originalsize,
@@ -529,8 +532,7 @@ mode_encrypt(File filein, File fileout)
 
 		/* ensure that XB_CRYPT_HASH_LEN is the correct length
 		of XB_CRYPT_HASH hashing algorithm output */
-		assert(XB_CRYPT_HASH_LEN ==
-		       gcry_md_get_algo_dlen(XB_CRYPT_HASH));
+		xb_a(XB_CRYPT_HASH_LEN == gcry_md_get_algo_dlen(XB_CRYPT_HASH));
 		gcry_md_hash_buffer(XB_CRYPT_HASH, chunkbuf + bytesread,
 				    chunkbuf, bytesread);
 

extra/mariabackup/xbcrypt.h

@@ -65,6 +65,7 @@ xb_rcrypt_t *xb_crypt_read_open(void *userdata,
 
 typedef enum {
 	XB_CRYPT_READ_CHUNK,
+	XB_CRYPT_READ_INCOMPLETE,
 	XB_CRYPT_READ_EOF,
 	XB_CRYPT_READ_ERROR
 } xb_rcrypt_result_t;
@@ -75,10 +76,4 @@ xb_rcrypt_result_t xb_crypt_read_chunk(xb_rcrypt_t *crypt, void **buf,
 
 int xb_crypt_read_close(xb_rcrypt_t *crypt);
 
-/******************************************************************************
-Utility interface */
-my_bool xb_crypt_read_key_file(const char *filename,
-			       void** key, uint *keylength);
-
-void xb_crypt_create_iv(void* ivbuf, size_t ivlen);
 #endif

extra/mariabackup/xbcrypt_common.c

 /******************************************************
-Copyright (c) 2013 Percona LLC and/or its affiliates.
+Copyright (c) 2013, 2017 Percona LLC and/or its affiliates.
 
 Encryption configuration file interface for XtraBackup.
 
@@ -21,19 +21,27 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA
 #include <my_base.h>
 #include "common.h"
 #include "xbcrypt.h"
+#include "xbcrypt_common.h"
 
-#if GCC_VERSION >= 4002
-/* Workaround to avoid "gcry_ac_* is deprecated" warnings in gcrypt.h */
-#  pragma GCC diagnostic ignored "-Wdeprecated-declarations"
-#endif
+/* Encryption options */
+char		*ds_encrypt_key = NULL;
+char		*ds_encrypt_key_file = NULL;
+ulong		ds_encrypt_algo;
+
+static uint	encrypt_key_len;
+static uint	encrypt_iv_len;
 
-#ifdef HAVE_GRYPT
-#include <gcrypt.h>
+static const uint encrypt_mode = GCRY_CIPHER_MODE_CTR;
 
-#if GCC_VERSION >= 4002
-#  pragma GCC diagnostic warning "-Wdeprecated-declarations"
+static uint encrypt_algos[] = { GCRY_CIPHER_NONE, GCRY_CIPHER_AES128,
+				GCRY_CIPHER_AES192, GCRY_CIPHER_AES256 };
+static uint encrypt_algo;
+
+#if !defined(GCRYPT_VERSION_NUMBER) || (GCRYPT_VERSION_NUMBER < 0x010600)
+GCRY_THREAD_OPTION_PTHREAD_IMPL;
 #endif
 
+
 my_bool
 xb_crypt_read_key_file(const char *filename, void** key, uint *keylength)
 {
@@ -59,4 +67,262 @@ xb_crypt_create_iv(void* ivbuf, size_t ivlen)
 {
 	gcry_create_nonce(ivbuf, ivlen);
 }
+
+gcry_error_t
+xb_crypt_init(uint *iv_len)
+{
+	gcry_error_t 		gcry_error;
+
+	/* Acording to gcrypt docs (and my testing), setting up the threading
+	   callbacks must be done first, so, lets give it a shot */
+#if !defined(GCRYPT_VERSION_NUMBER) || (GCRYPT_VERSION_NUMBER < 0x010600)
+	gcry_error = gcry_control(GCRYCTL_SET_THREAD_CBS, &gcry_threads_pthread);
+	if (gcry_error) {
+		msg("encryption: unable to set libgcrypt thread cbs - "
+		    "%s : %s\n",
+		    gcry_strsource(gcry_error),
+		    gcry_strerror(gcry_error));
+		return gcry_error;
+	}
+#endif
+
+	/* Version check should be the very next call because it
+	makes sure that important subsystems are intialized. */
+	if (!gcry_control(GCRYCTL_ANY_INITIALIZATION_P)) {
+		const char	*gcrypt_version;
+		gcrypt_version = gcry_check_version(NULL);
+		/* No other library has already initialized libgcrypt. */
+		if (!gcrypt_version) {
+			msg("encryption: failed to initialize libgcrypt\n");
+			return 1;
+		} else {
+			msg("encryption: using gcrypt %s\n", gcrypt_version);
+		}
+	}
+
+	/* Disable the gcry secure memory, not dealing with this for now */
+	gcry_error = gcry_control(GCRYCTL_DISABLE_SECMEM, 0);
+	if (gcry_error) {
+		msg("encryption: unable to disable libgcrypt secmem - "
+		    "%s : %s\n",
+		    gcry_strsource(gcry_error),
+		    gcry_strerror(gcry_error));
+		return gcry_error;
+	}
+
+	/* Finalize gcry initialization. */
+	gcry_error = gcry_control(GCRYCTL_INITIALIZATION_FINISHED, 0);
+	if (gcry_error) {
+		msg("encryption: unable to finish libgcrypt initialization - "
+		    "%s : %s\n",
+		    gcry_strsource(gcry_error),
+		    gcry_strerror(gcry_error));
+		return gcry_error;
+	}
+
+	/* Determine the algorithm */
+	encrypt_algo = encrypt_algos[ds_encrypt_algo];
+
+	/* Set up the iv length */
+	encrypt_iv_len = gcry_cipher_get_algo_blklen(encrypt_algo);
+	xb_a(encrypt_iv_len > 0);
+	if (iv_len != NULL) {
+		*iv_len = encrypt_iv_len;
+	}
+
+	/* Now set up the key */
+	if (ds_encrypt_key == NULL &&
+	    ds_encrypt_key_file == NULL) {
+		msg("encryption: no encryption key or key file specified.\n");
+		return gcry_error;
+	} else if (ds_encrypt_key && ds_encrypt_key_file) {
+		msg("encryption: both encryption key and key file specified.\n");
+		return gcry_error;
+	} else if (ds_encrypt_key_file) {
+		if (!xb_crypt_read_key_file(ds_encrypt_key_file,
+					    (void**)&ds_encrypt_key,
+					    &encrypt_key_len)) {
+			msg("encryption: unable to read encryption key file"
+			    " \"%s\".\n", ds_encrypt_key_file);
+			return gcry_error;
+		}
+	} else if (ds_encrypt_key) {
+		encrypt_key_len = strlen(ds_encrypt_key);
+	} else {
+		msg("encryption: no encryption key or key file specified.\n");
+		return gcry_error;
+	}
+
+	return 0;
+}
+
+gcry_error_t
+xb_crypt_cipher_open(gcry_cipher_hd_t *cipher_handle)
+{
+	if (encrypt_algo != GCRY_CIPHER_NONE) {
+		gcry_error_t 		gcry_error;
+
+		gcry_error = gcry_cipher_open(cipher_handle,
+					      encrypt_algo,
+					      encrypt_mode, 0);
+		if (gcry_error) {
+			msg("encryption: unable to open libgcrypt"
+			    " cipher - %s : %s\n",
+			    gcry_strsource(gcry_error),
+			    gcry_strerror(gcry_error));
+			gcry_cipher_close(*cipher_handle);
+			return gcry_error;
+		}
+
+		gcry_error = gcry_cipher_setkey(*cipher_handle,
+						ds_encrypt_key,
+						encrypt_key_len);
+		if (gcry_error) {
+			msg("encryption: unable to set libgcrypt"
+			    " cipher key - %s : %s\n",
+			    gcry_strsource(gcry_error),
+			    gcry_strerror(gcry_error));
+			gcry_cipher_close(*cipher_handle);
+			return gcry_error;
+		}
+		return gcry_error;
+	}
+	return 0;
+}
+
+void
+xb_crypt_cipher_close(gcry_cipher_hd_t cipher_handle)
+{
+	if (encrypt_algo != GCRY_CIPHER_NONE)
+		gcry_cipher_close(cipher_handle);
+}
+
+gcry_error_t
+xb_crypt_decrypt(gcry_cipher_hd_t cipher_handle, const uchar *from,
+		 size_t from_len, uchar *to, size_t *to_len,
+		 const uchar *iv, size_t iv_len, my_bool hash_appended)
+{
+	*to_len = from_len;
+
+	if (encrypt_algo != GCRY_CIPHER_NONE) {
+
+		gcry_error_t	gcry_error;
+
+		gcry_error = gcry_cipher_reset(cipher_handle);
+		if (gcry_error) {
+			msg("%s:encryption: unable to reset libgcrypt"
+			    " cipher - %s : %s\n", my_progname,
+			    gcry_strsource(gcry_error),
+			    gcry_strerror(gcry_error));
+			return gcry_error;
+		}
+
+		if (iv_len > 0) {
+			gcry_error = gcry_cipher_setctr(cipher_handle,
+							iv, iv_len);
+		}
+		if (gcry_error) {
+			msg("%s:encryption: unable to set cipher iv - "
+			    "%s : %s\n", my_progname,
+			    gcry_strsource(gcry_error),
+			    gcry_strerror(gcry_error));
+			return gcry_error;
+		}
+
+		/* Try to decrypt it */
+		gcry_error = gcry_cipher_decrypt(cipher_handle, to, *to_len,
+						 from, from_len);
+		if (gcry_error) {
+			msg("%s:encryption: unable to decrypt chunk - "
+			    "%s : %s\n", my_progname,
+			    gcry_strsource(gcry_error),
+			    gcry_strerror(gcry_error));
+			gcry_cipher_close(cipher_handle);
+			return gcry_error;
+		}
+
+		if (hash_appended) {
+			uchar hash[XB_CRYPT_HASH_LEN];
+
+			*to_len -= XB_CRYPT_HASH_LEN;
+
+			/* ensure that XB_CRYPT_HASH_LEN is the correct length
+			of XB_CRYPT_HASH hashing algorithm output */
+			xb_ad(gcry_md_get_algo_dlen(XB_CRYPT_HASH) ==
+			      XB_CRYPT_HASH_LEN);
+			gcry_md_hash_buffer(XB_CRYPT_HASH, hash, to,
+					    *to_len);
+			if (memcmp(hash, (char *) to + *to_len,
+				   XB_CRYPT_HASH_LEN) != 0) {
+				msg("%s:%s invalid plaintext hash. "
+				    "Wrong encrytion key specified?\n",
+				    my_progname, __FUNCTION__);
+				return 1;
+			}
+		}
+
+	} else {
+		memcpy(to, from, *to_len);
+	}
+
+	return 0;
+}
+
+gcry_error_t
+xb_crypt_encrypt(gcry_cipher_hd_t cipher_handle, const uchar *from,
+		 size_t from_len, uchar *to, size_t *to_len, uchar *iv)
+{
+	gcry_error_t 		gcry_error;
+
+	/* ensure that XB_CRYPT_HASH_LEN is the correct length
+	of XB_CRYPT_HASH hashing algorithm output */
+	xb_ad(gcry_md_get_algo_dlen(XB_CRYPT_HASH) ==
+	      XB_CRYPT_HASH_LEN);
+
+	memcpy(to, from, from_len);
+	gcry_md_hash_buffer(XB_CRYPT_HASH, to + from_len,
+			    from, from_len);
+
+	*to_len = from_len;
+
+	if (encrypt_algo != GCRY_CIPHER_NONE) {
+
+		gcry_error = gcry_cipher_reset(cipher_handle);
+		if (gcry_error) {
+			msg("encrypt: unable to reset cipher - "
+			    "%s : %s\n",
+			    gcry_strsource(gcry_error),
+			    gcry_strerror(gcry_error));
+			return gcry_error;
+		}
+
+		xb_crypt_create_iv(iv, encrypt_iv_len);
+		gcry_error = gcry_cipher_setctr(cipher_handle, iv,
+						encrypt_iv_len);
+		if (gcry_error) {
+			msg("encrypt: unable to set cipher ctr - "
+			    "%s : %s\n",
+			    gcry_strsource(gcry_error),
+			    gcry_strerror(gcry_error));
+			return gcry_error;
+		}
+
+		gcry_error = gcry_cipher_encrypt(cipher_handle, to,
+						 *to_len + XB_CRYPT_HASH_LEN,
+						 to,
+						 from_len + XB_CRYPT_HASH_LEN);
+		if (gcry_error) {
+			msg("encrypt: unable to encrypt buffer - "
+			    "%s : %s\n", gcry_strsource(gcry_error),
+			    gcry_strerror(gcry_error));
+			return gcry_error;
+		}
+	} else {
+		memcpy(to, from, from_len + XB_CRYPT_HASH_LEN);
+	}
+
+	*to_len += XB_CRYPT_HASH_LEN;
+
+	return 0;
+}
 #endif
\ No newline at end of file

extra/mariabackup/xbcrypt_common.h

+/******************************************************
+Copyright (c) 2017 Percona LLC and/or its affiliates.
+
+Encryption datasink implementation for XtraBackup.
+
+This program is free software; you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation; version 2 of the License.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program; if not, write to the Free Software
+Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA
+
+*******************************************************/
+
+#include <my_base.h>
+#if HAVE_GCRYPT
+#if GCC_VERSION >= 4002
+/* Workaround to avoid "gcry_ac_* is deprecated" warnings in gcrypt.h */
+#  pragma GCC diagnostic ignored "-Wdeprecated-declarations"
+#endif
+
+#include <gcrypt.h>
+
+extern char		*ds_encrypt_key;
+extern char		*ds_encrypt_key_file;
+extern int		ds_encrypt_threads;
+extern ulong		ds_encrypt_algo;
+
+/******************************************************************************
+Utility interface */
+my_bool xb_crypt_read_key_file(const char *filename,
+			       void** key, uint *keylength);
+
+void xb_crypt_create_iv(void* ivbuf, size_t ivlen);
+
+/* Initialize gcrypt and setup encryption key and IV lengths */
+gcry_error_t
+xb_crypt_init(uint *iv_len);
+
+/* Setup gcrypt cipher */
+gcry_error_t
+xb_crypt_cipher_open(gcry_cipher_hd_t *cipher_handle);
+
+/* Close gcrypt cipher */
+void
+xb_crypt_cipher_close(gcry_cipher_hd_t cipher_handle);
+
+/* Decrypt buffer */
+gcry_error_t
+xb_crypt_decrypt(gcry_cipher_hd_t cipher_handle, const uchar *from,
+		 size_t from_len, uchar *to, size_t *to_len, const uchar *iv,
+		 size_t iv_len, my_bool hash_appended);
+
+/* Encrypt buffer */
+gcry_error_t
+xb_crypt_encrypt(gcry_cipher_hd_t cipher_handle, const uchar *from,
+  size_t from_len, uchar *to, size_t *to_len, uchar *iv);
+#endif

extra/mariabackup/xbcrypt_read.c

@@ -19,6 +19,7 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA
 *******************************************************/
 
 #include "xbcrypt.h"
+#include "crc_glue.h"
 
 struct xb_rcrypt_struct {
 	void				*userdata;
@@ -212,7 +213,7 @@ xb_crypt_read_chunk(xb_rcrypt_t *crypt, void **buf, size_t *olen, size_t *elen,
 		}
 	}
 
-	checksum = crc32(0, crypt->buffer, *elen);
+	checksum = crc32_iso3309(0, crypt->buffer, *elen);
 	if (checksum != checksum_exp) {
 		msg("%s:%s invalid checksum at offset 0x%llx, "
 		    "expected 0x%lx, actual 0x%lx.\n", my_progname, __FUNCTION__,

extra/mariabackup/xbcrypt_write.c

@@ -19,6 +19,7 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA
 *******************************************************/
 
 #include "xbcrypt.h"
+#include "crc_glue.h"
 
 struct xb_wcrypt_struct {
 	void				*userdata;
@@ -73,7 +74,7 @@ int xb_crypt_write_chunk(xb_wcrypt_t *crypt, const void *buf, size_t olen,
 	int8store(ptr, (ulonglong)elen); /* encrypted (actual) size */
 	ptr += 8;
 
-	checksum = crc32(0, buf, (uint)elen);
+	checksum = crc32_iso3309(0, buf, elen);
 	int4store(ptr, checksum);	/* checksum */
 	ptr += 4;
 

extra/mariabackup/xbstream.h

 /******************************************************
-Copyright (c) 2011-2013 Percona LLC and/or its affiliates.
+Copyright (c) 2011-2017 Percona LLC and/or its affiliates.
 
 The xbstream format interface.
 
@@ -89,8 +89,10 @@ typedef struct {
 	char		path[FN_REFLEN];
 	size_t		length;
 	my_off_t	offset;
+	my_off_t	checksum_offset;
 	void		*data;
 	ulong		checksum;
+	size_t		buflen;
 } xb_rstream_chunk_t;
 
 xb_rstream_t *xb_stream_read_new(void);
@@ -100,4 +102,6 @@ xb_rstream_result_t xb_stream_read_chunk(xb_rstream_t *stream,
 
 int xb_stream_read_done(xb_rstream_t *stream);
 
+int xb_stream_validate_checksum(xb_rstream_chunk_t *chunk);
+
 #endif

extra/mariabackup/xbstream_read.c

 /******************************************************
-Copyright (c) 2011-2013 Percona LLC and/or its affiliates.
+Copyright (c) 2011-2017 Percona LLC and/or its affiliates.
 
 The xbstream format reader implementation.
 
@@ -23,6 +23,7 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA
 #include <zlib.h>
 #include "common.h"
 #include "xbstream.h"
+#include "crc_glue.h"
 
 /* Allocate 1 MB for the payload buffer initially */
 #define INIT_BUFFER_LEN (1024 * 1024)
@@ -34,8 +35,6 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA
 struct xb_rstream_struct {
 	my_off_t	offset;
 	File 		fd;
-	void 		*buffer;
-	size_t		buflen;
 };
 
 xb_rstream_t *
@@ -45,9 +44,6 @@ xb_stream_read_new(void)
 
 	stream = (xb_rstream_t *) my_malloc(sizeof(xb_rstream_t), MYF(MY_FAE));
 
-	stream->buffer = my_malloc(INIT_BUFFER_LEN, MYF(MY_FAE));
-	stream->buflen = INIT_BUFFER_LEN;
-
 #ifdef __WIN__
 	setmode(fileno(stdin), _O_BINARY);
 #endif
@@ -71,6 +67,23 @@ validate_chunk_type(uchar code)
 	}
 }
 
+int
+xb_stream_validate_checksum(xb_rstream_chunk_t *chunk)
+{
+	ulong	checksum;
+
+	checksum = crc32_iso3309(0, chunk->data, (uint)chunk->length);
+	if (checksum != chunk->checksum) {
+		msg("xb_stream_read_chunk(): invalid checksum at offset "
+		    "0x%llx: expected 0x%lx, read 0x%lx.\n",
+		    (ulonglong) chunk->checksum_offset, chunk->checksum,
+		    checksum);
+		return XB_STREAM_READ_ERROR;
+	}
+
+	return XB_STREAM_READ_CHUNK;
+}
+
 #define F_READ(buf,len)                                       	\
 	do {                                                      	\
 		if (xb_read_full(fd, buf, len) < len) {	\
@@ -87,8 +100,6 @@ xb_stream_read_chunk(xb_rstream_t *stream, xb_rstream_chunk_t *chunk)
 	uint		pathlen;
 	size_t		tbytes;
 	ulonglong	ullval;
-	ulong		checksum_exp;
-	ulong		checksum;
 	File		fd = stream->fd;
 
 	xb_ad(sizeof(tmpbuf) >= CHUNK_HEADER_CONSTANT_LEN);
@@ -178,39 +189,30 @@ xb_stream_read_chunk(xb_rstream_t *stream, xb_rstream_chunk_t *chunk)
 	stream->offset += 8;
 
 	/* Reallocate the buffer if needed */
-	if (chunk->length > stream->buflen) {
-		stream->buffer = my_realloc(stream->buffer, chunk->length,
-					    MYF(MY_WME));
-		if (stream->buffer == NULL) {
+	if (chunk->length > chunk->buflen) {
+		chunk->data = my_realloc(chunk->data, chunk->length,
+					 MYF(MY_WME | MY_ALLOW_ZERO_PTR));
+		if (chunk->data == NULL) {
 			msg("xb_stream_read_chunk(): failed to increase buffer "
 			    "to %lu bytes.\n", (ulong) chunk->length);
 			goto err;
 		}
-		stream->buflen = chunk->length;
+		chunk->buflen = chunk->length;
 	}
 
 	/* Checksum */
 	F_READ(tmpbuf, 4);
-	checksum_exp = uint4korr(tmpbuf);
+	chunk->checksum = uint4korr(tmpbuf);
+	chunk->checksum_offset = stream->offset;
 
 	/* Payload */
 	if (chunk->length > 0) {
-		F_READ(stream->buffer, chunk->length);
+		F_READ(chunk->data, chunk->length);
 		stream->offset += chunk->length;
 	}
 
-	checksum = crc32(0, stream->buffer, chunk->length);
-	if (checksum != checksum_exp) {
-		msg("xb_stream_read_chunk(): invalid checksum at offset "
-		    "0x%llx: expected 0x%lx, read 0x%lx.\n",
-		    (ulonglong) stream->offset, checksum_exp, checksum);
-		goto err;
-	}
 	stream->offset += 4;
 
-	chunk->data = stream->buffer;
-	chunk->checksum = checksum;
-
 	return XB_STREAM_READ_CHUNK;
 
 err:
@@ -220,7 +222,6 @@ xb_stream_read_chunk(xb_rstream_t *stream, xb_rstream_chunk_t *chunk)
 int
 xb_stream_read_done(xb_rstream_t *stream)
 {
-	my_free(stream->buffer);
 	my_free(stream);
 
 	return 0;

extra/mariabackup/xbstream_write.c

 /******************************************************
-Copyright (c) 2011-2013 Percona LLC and/or its affiliates.
+Copyright (c) 2011-2017 Percona LLC and/or its affiliates.
 
 The xbstream format writer implementation.
 
@@ -23,6 +23,7 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA
 #include <zlib.h>
 #include "common.h"
 #include "xbstream.h"
+#include "crc_glue.h"
 
 /* Group writes smaller than this into a single chunk */
 #define XB_STREAM_MIN_CHUNK_SIZE (10 * 1024 * 1024)
@@ -215,12 +216,13 @@ xb_stream_write_chunk(xb_wstream_file_t *file, const void *buf, size_t len)
 	int8store(ptr, len);                     /* Payload length */
 	ptr += 8;
 
+	checksum = crc32_iso3309(0, buf, (uint)len);   /* checksum */
+
 	pthread_mutex_lock(&stream->mutex);
 
 	int8store(ptr, file->offset);            /* Payload offset */
 	ptr += 8;
 
-	checksum = crc32(0, buf, (uint)len);           /* checksum */
 	int4store(ptr, checksum);
 	ptr += 4;
 

extra/mariabackup/xtrabackup.h

@@ -77,6 +77,8 @@ extern char		*xtrabackup_tables;
 extern char		*xtrabackup_tables_file;
 extern char		*xtrabackup_databases;
 extern char		*xtrabackup_databases_file;
+extern char		*xtrabackup_tables_exclude;
+extern char		*xtrabackup_databases_exclude;
 
 extern ibool		xtrabackup_compress;
 extern ibool		xtrabackup_encrypt;
@@ -205,6 +207,17 @@ check_if_skip_table(
 /******************/
 	const char*	name);	/*!< in: path to the table */
 
+
+/************************************************************************
+Checks if a database specified by path should be skipped from backup based on
+the --databases, --databases_file or --databases_exclude options.
+
+@return TRUE if the table should be skipped. */
+my_bool
+check_if_skip_database_by_path(
+	const char* path /*!< in: path to the db directory. */
+);
+
 /************************************************************************
 Check if parameter is set in defaults file or via command line argument
 @return true if parameter is set. */

mysql-test/suite/mariabackup/partial_exclude.result

+CREATE TABLE t1(i INT) ENGINE INNODB;
+INSERT INTO t1 VALUES(1);
+CREATE TABLE t2(i int) ENGINE INNODB;
+CREATE DATABASE db2;
+USE db2;
+CREATE TABLE t1(i INT) ENGINE INNODB;
+USE test;
+# xtrabackup backup
+t1.ibd
+DROP TABLE t1;
+DROP TABLE t2;
+DROP DATABASE db2;

mysql-test/suite/mariabackup/partial_exclude.test

+# Test --databases-exclude and --tables-exclude feature of xtrabackup 2.3.8
+
+CREATE TABLE t1(i INT) ENGINE INNODB;
+INSERT INTO t1 VALUES(1);
+CREATE TABLE t2(i int) ENGINE INNODB;
+
+CREATE DATABASE db2;
+USE db2;
+CREATE TABLE t1(i INT) ENGINE INNODB;
+
+USE test;
+
+echo # xtrabackup backup;
+
+let $targetdir=$MYSQLTEST_VARDIR/tmp/backup;
+--disable_result_log
+exec $XTRABACKUP --defaults-file=$MYSQLTEST_VARDIR/my.cnf  --backup "--tables-exclude=test.*2" "--databases-exclude=db2" --target-dir=$targetdir;
+--enable_result_log
+
+# check that only t1 table is in backup (t2 is excluded)
+list_files $targetdir/test *.ibd;
+# check that db2 database is not in the backup (excluded)
+--error 1
+list_files $targetdir/db2 *.ibd;
+
+DROP TABLE t1;
+DROP TABLE t2;
+DROP DATABASE db2;
+rmdir $targetdir;
+

mysql-test/suite/mariabackup/xbstream.test

@@ -9,7 +9,7 @@ echo # xtrabackup backup to stream;
 exec $XTRABACKUP --defaults-file=$MYSQLTEST_VARDIR/my.cnf  --backup  --stream=xbstream > $streamfile 2>$targetdir/backup_stream.log;
 echo # xbstream extract;
 --disable_result_log
-exec $XBSTREAM -x -C $targetdir  < $streamfile;
+exec $XBSTREAM -x -C $targetdir --parallel=16 < $streamfile;
 
 echo # xtrabackup prepare;
 exec $XTRABACKUP --prepare --target-dir=$targetdir;

storage/xtradb/fil/fil0fil.cc

@@ -71,6 +71,7 @@ static ulint srv_data_read, srv_data_written;
 
 MYSQL_PLUGIN_IMPORT extern my_bool lower_case_file_system;
 
+
 /*
 		IMPLEMENTATION OF THE TABLESPACE MEMORY CACHE
 		=============================================
@@ -5379,6 +5380,9 @@ fil_file_readdir_next_file(
 	return(-1);
 }
 
+
+my_bool(*fil_check_if_skip_database_by_path)(const char* name);
+
 #define CHECK_TIME_EVERY_N_FILES   10
 /********************************************************************//**
 At the server startup, if we need crash recovery, scans the database
@@ -5449,7 +5453,19 @@ fil_load_single_table_tablespaces(ibool (*pred)(const char*, const char*))
 			    "%s/%s", fil_path_to_mysql_datadir, dbinfo.name);
 		srv_normalize_path_for_win(dbpath);
 
-		dbdir = os_file_opendir(dbpath, FALSE);
+		if (IS_XTRABACKUP()) {
+			ut_a(fil_check_if_skip_database_by_path);
+			if (fil_check_if_skip_database_by_path(dbpath)) {
+				fprintf(stderr, "Skipping db: %s\n", dbpath);
+				dbdir = NULL;
+			} else {
+				/* We want wrong directory permissions to be a fatal
+				error for XtraBackup. */
+				dbdir = os_file_opendir(dbpath, TRUE);
+			}
+		} else {
+			dbdir = os_file_opendir(dbpath, FALSE);
+		}
 
 		if (dbdir != NULL) {