Commit 35034d81 authored by Sergei Golubchik's avatar Sergei Golubchik Committed by Oleksandr Byelkin

S3 is pluggable now

parent 4acafaae
...@@ -623,6 +623,15 @@ Description: Connect storage engine for MariaDB ...@@ -623,6 +623,15 @@ Description: Connect storage engine for MariaDB
other interesting features. other interesting features.
This package contains the Connect plugin for MariaDB. This package contains the Connect plugin for MariaDB.
Package: mariadb-plugin-s3
Architecture: any
Depends: libcurl4,
mariadb-server-10.5 (= ${binary:Version}),
${misc:Depends},
${shlibs:Depends}
Description: S3 storage engine for MariaDB
Read only table stored in S3.
Package: mariadb-plugin-rocksdb Package: mariadb-plugin-rocksdb
Architecture: amd64 arm64 mips64el ppc64el Architecture: amd64 arm64 mips64el ppc64el
Depends: mariadb-server-10.5 (= ${binary:Version}), Depends: mariadb-server-10.5 (= ${binary:Version}),
......
etc/mysql/mariadb.conf.d/s3.cnf
usr/bin/aria_s3_copy
usr/lib/mysql/plugin/ha_s3.so
/* Copyright (C) 2018 MariaDB corporation /* Copyright (C) 2018,2020 MariaDB Corporation Ab
This program is free software; you can redistribute it and/or modify This program is free software; you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by it under the terms of the GNU General Public License as published by
......
...@@ -2,6 +2,7 @@ ...@@ -2,6 +2,7 @@
!include include/default_client.cnf !include include/default_client.cnf
[mysqld.1] [mysqld.1]
plugin-load-add=ha_s3
s3=ON s3=ON
#s3-host-name=s3.amazonaws.com #s3-host-name=s3.amazonaws.com
#s3-protocol-version=Amazon #s3-protocol-version=Amazon
......
...@@ -2,7 +2,7 @@ package My::Suite::S3; ...@@ -2,7 +2,7 @@ package My::Suite::S3;
@ISA = qw(My::Suite); @ISA = qw(My::Suite);
return "Need S3 engine" unless $::mysqld_variables{'s3'} eq "ON"; return "Need S3 engine" unless $::mysqld_variables{'s3'} eq "ON" or $ENV{HA_S3_SO};
bless { }; bless { };
# Copyright (C) 2007 MySQL AB # Copyright (C) 2007 MySQL AB
# Copyright (C) 2009,2020 MariaDB Corporation Ab
# #
# This program is free software; you can redistribute it and/or modify # This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by # it under the terms of the GNU General Public License as published by
...@@ -105,7 +106,7 @@ OPTION(USE_ARIA_FOR_TMP_TABLES "Use Aria for temporary tables" ON) ...@@ -105,7 +106,7 @@ OPTION(USE_ARIA_FOR_TMP_TABLES "Use Aria for temporary tables" ON)
# #
INCLUDE (CheckIncludeFiles) INCLUDE (CheckIncludeFiles)
SET(S3_SOURCES ha_s3.cc s3_func.c SET(S3_SOURCES s3_func.c
libmarias3/src/debug.c libmarias3/src/error.c libmarias3/src/marias3.c libmarias3/src/debug.c libmarias3/src/error.c libmarias3/src/marias3.c
libmarias3/src/request.c libmarias3/src/response.c libmarias3/src/sha256.c libmarias3/src/request.c libmarias3/src/response.c libmarias3/src/sha256.c
libmarias3/src/sha256-internal.c libmarias3/src/xml.c) libmarias3/src/sha256-internal.c libmarias3/src/xml.c)
...@@ -115,18 +116,13 @@ IF(NOT PLUGIN_S3 STREQUAL NO) ...@@ -115,18 +116,13 @@ IF(NOT PLUGIN_S3 STREQUAL NO)
ENDIF() ENDIF()
IF (CURL_FOUND) IF (CURL_FOUND)
MYSQL_ADD_PLUGIN(s3 ${S3_SOURCES} STORAGE_ENGINE STATIC_ONLY MYSQL_ADD_PLUGIN(s3 ha_s3.cc ${S3_SOURCES} COMPONENT s3-engine
LINK_LIBRARIES aria myisam mysys mysys_ssl curl LINK_LIBRARIES curl STORAGE_ENGINE NOT_EMBEDDED CONFIG s3.cnf)
NOT_EMBEDDED)
ENDIF() ENDIF()
IF(TARGET s3) IF(TARGET s3)
MYSQL_ADD_EXECUTABLE(aria_s3_copy aria_s3_copy.cc COMPONENT Server) MYSQL_ADD_EXECUTABLE(aria_s3_copy aria_s3_copy.cc ${S3_SOURCES} COMPONENT s3-engine)
TARGET_LINK_LIBRARIES(aria_s3_copy s3) TARGET_LINK_LIBRARIES(aria_s3_copy aria myisam mysys mysys_ssl curl)
INCLUDE_DIRECTORIES(${CMAKE_CURRENT_SOURCE_DIR}/libmarias3) INCLUDE_DIRECTORIES(${CMAKE_CURRENT_SOURCE_DIR}/libmarias3)
ADD_DEFINITIONS(-DWITH_S3_STORAGE_ENGINE) ADD_DEFINITIONS(-DWITH_S3_STORAGE_ENGINE)
TARGET_LINK_LIBRARIES(aria s3)
ENDIF() ENDIF()
/* Copyright (C) 2004-2008 MySQL AB & MySQL Finland AB & TCX DataKonsult AB /* Copyright (C) 2004-2008 MySQL AB & MySQL Finland AB & TCX DataKonsult AB
Copyright (C) 2008-2009 Sun Microsystems, Inc. Copyright (C) 2008-2009 Sun Microsystems, Inc.
Copyright (c) 2009, 2017, MariaDB Corporation. Copyright (c) 2009, 2020, MariaDB Corporation Ab
This program is free software; you can redistribute it and/or modify This program is free software; you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by it under the terms of the GNU General Public License as published by
...@@ -72,7 +72,7 @@ const char *zerofill_error_msg= ...@@ -72,7 +72,7 @@ const char *zerofill_error_msg=
corrupted. corrupted.
*/ */
ulonglong maria_recover_options= HA_RECOVER_NONE; ulonglong maria_recover_options= HA_RECOVER_NONE;
handlerton *maria_hton; handlerton __attribute__((visibility("default"))) *maria_hton;
/* bits in maria_recover_options */ /* bits in maria_recover_options */
const char *maria_recover_names[]= const char *maria_recover_names[]=
...@@ -2688,6 +2688,16 @@ int ha_maria::extra_opt(enum ha_extra_function operation, ulong cache_size) ...@@ -2688,6 +2688,16 @@ int ha_maria::extra_opt(enum ha_extra_function operation, ulong cache_size)
} }
bool ha_maria::auto_repair(int error) const
{
/* Always auto-repair moved tables (error == HA_ERR_OLD_FILE) */
return ((MY_TEST(maria_recover_options & HA_RECOVER_ANY) &&
error == HA_ERR_CRASHED_ON_USAGE) ||
error == HA_ERR_OLD_FILE);
}
int ha_maria::delete_all_rows() int ha_maria::delete_all_rows()
{ {
THD *thd= table->in_use; THD *thd= table->in_use;
...@@ -3348,6 +3358,14 @@ ha_rows ha_maria::records_in_range(uint inx, const key_range *min_key, ...@@ -3348,6 +3358,14 @@ ha_rows ha_maria::records_in_range(uint inx, const key_range *min_key,
} }
FT_INFO *ha_maria::ft_init_ext(uint flags, uint inx, String * key)
{
return maria_ft_init_search(flags, file, inx,
(uchar *) key->ptr(), key->length(),
key->charset(), table->record[0]);
}
int ha_maria::ft_read(uchar * buf) int ha_maria::ft_read(uchar * buf)
{ {
int error; int error;
......
#ifndef HA_MARIA_INCLUDED #ifndef HA_MARIA_INCLUDED
#define HA_MARIA_INCLUDED #define HA_MARIA_INCLUDED
/* Copyright (C) 2006,2004 MySQL AB & MySQL Finland AB & TCX DataKonsult AB /* Copyright (C) 2006, 2004 MySQL AB & MySQL Finland AB & TCX DataKonsult AB
Copyright (c) 2009, 2020, MariaDB Corporation Ab
This program is free software; you can redistribute it and/or modify This program is free software; you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by it under the terms of the GNU General Public License as published by
...@@ -38,7 +39,7 @@ C_MODE_END ...@@ -38,7 +39,7 @@ C_MODE_END
extern TYPELIB maria_recover_typelib; extern TYPELIB maria_recover_typelib;
extern ulonglong maria_recover_options; extern ulonglong maria_recover_options;
class ha_maria :public handler class __attribute__((visibility("default"))) ha_maria :public handler
{ {
public: public:
MARIA_HA *file; MARIA_HA *file;
...@@ -97,12 +98,7 @@ class ha_maria :public handler ...@@ -97,12 +98,7 @@ class ha_maria :public handler
ft_handler->please->reinit_search(ft_handler); ft_handler->please->reinit_search(ft_handler);
return 0; return 0;
} }
FT_INFO *ft_init_ext(uint flags, uint inx, String * key) FT_INFO *ft_init_ext(uint flags, uint inx, String * key);
{
return maria_ft_init_search(flags, file, inx,
(uchar *) key->ptr(), key->length(),
key->charset(), table->record[0]);
}
int ft_read(uchar * buf); int ft_read(uchar * buf);
int index_init(uint idx, bool sorted); int index_init(uint idx, bool sorted);
int index_end(); int index_end();
...@@ -146,14 +142,7 @@ class ha_maria :public handler ...@@ -146,14 +142,7 @@ class ha_maria :public handler
bool check_and_repair(THD * thd); bool check_and_repair(THD * thd);
bool is_crashed() const; bool is_crashed() const;
bool is_changed() const; bool is_changed() const;
bool auto_repair(int error) const bool auto_repair(int error) const;
{
/* Always auto-repair moved tables (error == HA_ERR_OLD_FILE) */
return ((MY_TEST(maria_recover_options & HA_RECOVER_ANY) &&
error == HA_ERR_CRASHED_ON_USAGE) ||
error == HA_ERR_OLD_FILE);
}
int optimize(THD * thd, HA_CHECK_OPT * check_opt); int optimize(THD * thd, HA_CHECK_OPT * check_opt);
int assign_to_keycache(THD * thd, HA_CHECK_OPT * check_opt); int assign_to_keycache(THD * thd, HA_CHECK_OPT * check_opt);
int preload_keys(THD * thd, HA_CHECK_OPT * check_opt); int preload_keys(THD * thd, HA_CHECK_OPT * check_opt);
......
/* Copyright (C) 2019 MariaDB Corppration AB /* Copyright (C) 2019, 2020 MariaDB Corporation Ab
This program is free software; you can redistribute it and/or modify This program is free software; you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by it under the terms of the GNU General Public License as published by
...@@ -1008,9 +1008,24 @@ static int ha_s3_init(void *p) ...@@ -1008,9 +1008,24 @@ static int ha_s3_init(void *p)
s3_init_library(); s3_init_library();
if (s3_debug) if (s3_debug)
ms3_debug(); ms3_debug();
struct s3_func s3f_real =
{
ms3_set_option, s3_free, ms3_deinit, s3_unique_file_number,
read_index_header, s3_check_frm_version, s3_info_copy,
set_database_and_table_from_path, s3_open_connection
};
s3f= s3f_real;
return res ? HA_ERR_INITIALIZATION : 0; return res ? HA_ERR_INITIALIZATION : 0;
} }
static int ha_s3_deinit(void*)
{
bzero(&s3f, sizeof(s3f));
return 0;
}
static SHOW_VAR status_variables[]= { static SHOW_VAR status_variables[]= {
{"pagecache_blocks_not_flushed", {"pagecache_blocks_not_flushed",
(char*) &s3_pagecache.global_blocks_changed, SHOW_LONG}, (char*) &s3_pagecache.global_blocks_changed, SHOW_LONG},
...@@ -1057,7 +1072,7 @@ maria_declare_plugin(s3) ...@@ -1057,7 +1072,7 @@ maria_declare_plugin(s3)
"ALTER TABLE table_name ENGINE=s3", "ALTER TABLE table_name ENGINE=s3",
PLUGIN_LICENSE_GPL, PLUGIN_LICENSE_GPL,
ha_s3_init, /* Plugin Init */ ha_s3_init, /* Plugin Init */
NULL, /* Plugin Deinit */ ha_s3_deinit, /* Plugin Deinit */
0x0100, /* 1.0 */ 0x0100, /* 1.0 */
status_variables, /* status variables */ status_variables, /* status variables */
system_variables, /* system variables */ system_variables, /* system variables */
......
/* Copyright (C) 2018 MariaDB corporation /* Copyright (C) 2018, 2020 MariaDB Corporation Ab
This program is free software; you can redistribute it and/or modify This program is free software; you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by it under the terms of the GNU General Public License as published by
...@@ -20,8 +20,6 @@ ...@@ -20,8 +20,6 @@
#include "ma_checkpoint.h" #include "ma_checkpoint.h"
#include <aria_backup.h> #include <aria_backup.h>
static uchar *_ma_base_info_read(uchar *ptr, MARIA_BASE_INFO *base);
/** /**
@brief Get capabilites for an Aria table @brief Get capabilites for an Aria table
...@@ -32,6 +30,7 @@ static uchar *_ma_base_info_read(uchar *ptr, MARIA_BASE_INFO *base); ...@@ -32,6 +30,7 @@ static uchar *_ma_base_info_read(uchar *ptr, MARIA_BASE_INFO *base);
@return X errno @return X errno
*/ */
int aria_get_capabilities(File kfile, ARIA_TABLE_CAPABILITIES *cap)__attribute__((visibility("default"))) ;
int aria_get_capabilities(File kfile, ARIA_TABLE_CAPABILITIES *cap) int aria_get_capabilities(File kfile, ARIA_TABLE_CAPABILITIES *cap)
{ {
MARIA_SHARE share; MARIA_SHARE share;
...@@ -102,20 +101,13 @@ int aria_get_capabilities(File kfile, ARIA_TABLE_CAPABILITIES *cap) ...@@ -102,20 +101,13 @@ int aria_get_capabilities(File kfile, ARIA_TABLE_CAPABILITIES *cap)
err: err:
my_free(disc_cache); my_free(disc_cache);
DBUG_RETURN(error); DBUG_RETURN(error);
} /* maria_get_capabilities */ } /* aria_get_capabilities */
/* /****************************************************************************
This is a copy of my_base_info_read from ma_open(). ** store MARIA_BASE_INFO
The base information will never change (something may be added ****************************************************************************/
last, but not relevant for maria_get_capabilities), so it's safe to
copy it here.
The copy is done to avoid linking in the fill Aria library just
because maria_backup uses maria_get_capabilities()
*/
static uchar *_ma_base_info_read(uchar *ptr, MARIA_BASE_INFO *base) uchar *_ma_base_info_read(uchar *ptr, MARIA_BASE_INFO *base)
{ {
bmove(base->uuid, ptr, MY_UUID_SIZE); ptr+= MY_UUID_SIZE; bmove(base->uuid, ptr, MY_UUID_SIZE); ptr+= MY_UUID_SIZE;
base->keystart= mi_sizekorr(ptr); ptr+= 8; base->keystart= mi_sizekorr(ptr); ptr+= 8;
......
/* Copyright (C) 2006 MySQL AB & MySQL Finland AB & TCX DataKonsult AB /* Copyright (C) 2006 MySQL AB & MySQL Finland AB & TCX DataKonsult AB
Copyright (c) 2010, 2020, MariaDB Corporation Ab
This program is free software; you can redistribute it and/or modify This program is free software; you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by it under the terms of the GNU General Public License as published by
...@@ -253,7 +254,7 @@ int maria_close(register MARIA_HA *info) ...@@ -253,7 +254,7 @@ int maria_close(register MARIA_HA *info)
delete_dynamic(&info->pinned_pages); delete_dynamic(&info->pinned_pages);
#ifdef WITH_S3_STORAGE_ENGINE #ifdef WITH_S3_STORAGE_ENGINE
if (info->s3) if (info->s3)
ms3_deinit(info->s3); s3f.deinit(info->s3);
#endif /* WITH_S3_STORAGE_ENGINE */ #endif /* WITH_S3_STORAGE_ENGINE */
my_free(info); my_free(info);
......
/* Copyright (C) 2006 MySQL AB & MySQL Finland AB & TCX DataKonsult AB /* Copyright (C) 2006 MySQL AB & MySQL Finland AB & TCX DataKonsult AB
Copyright (c) 2009, 2019, MariaDB Corporation. Copyright (c) 2009, 2020, MariaDB Corporation Ab
This program is free software; you can redistribute it and/or modify This program is free software; you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by it under the terms of the GNU General Public License as published by
...@@ -39,7 +39,6 @@ static my_bool maria_scan_init_dummy(MARIA_HA *info); ...@@ -39,7 +39,6 @@ static my_bool maria_scan_init_dummy(MARIA_HA *info);
static void maria_scan_end_dummy(MARIA_HA *info); static void maria_scan_end_dummy(MARIA_HA *info);
static my_bool maria_once_init_dummy(MARIA_SHARE *, File); static my_bool maria_once_init_dummy(MARIA_SHARE *, File);
static my_bool maria_once_end_dummy(MARIA_SHARE *); static my_bool maria_once_end_dummy(MARIA_SHARE *);
static uchar *_ma_base_info_read(uchar *ptr, MARIA_BASE_INFO *base);
static uchar *_ma_state_info_read(uchar *ptr, MARIA_STATE_INFO *state); static uchar *_ma_state_info_read(uchar *ptr, MARIA_STATE_INFO *state);
#define get_next_element(to,pos,size) { memcpy((char*) to,pos,(size_t) size); \ #define get_next_element(to,pos,size) { memcpy((char*) to,pos,(size_t) size); \
...@@ -318,7 +317,7 @@ MARIA_HA *maria_open(const char *name, int mode, uint open_flags, ...@@ -318,7 +317,7 @@ MARIA_HA *maria_open(const char *name, int mode, uint open_flags,
else else
{ {
strmake(name_buff, name, sizeof(name_buff)-1); /* test_if_reopen() */ strmake(name_buff, name, sizeof(name_buff)-1); /* test_if_reopen() */
if (!(s3_client= s3_open_connection(s3))) if (!(s3_client= s3f.open_connection(s3)))
{ {
internal_table= 1; /* Avoid unlock on error */ internal_table= 1; /* Avoid unlock on error */
goto err; goto err;
...@@ -371,7 +370,7 @@ MARIA_HA *maria_open(const char *name, int mode, uint open_flags, ...@@ -371,7 +370,7 @@ MARIA_HA *maria_open(const char *name, int mode, uint open_flags,
else else
{ {
errpos= 1; errpos= 1;
if (set_database_and_table_from_path(s3, name_buff)) if (s3f.set_database_and_table_from_path(s3, name_buff))
{ {
my_printf_error(HA_ERR_NO_SUCH_TABLE, my_printf_error(HA_ERR_NO_SUCH_TABLE,
"Can't find database and path from %s", MYF(0), "Can't find database and path from %s", MYF(0),
...@@ -379,17 +378,17 @@ MARIA_HA *maria_open(const char *name, int mode, uint open_flags, ...@@ -379,17 +378,17 @@ MARIA_HA *maria_open(const char *name, int mode, uint open_flags,
my_errno= HA_ERR_NO_SUCH_TABLE; my_errno= HA_ERR_NO_SUCH_TABLE;
goto err; goto err;
} }
if (!(share_s3= share->s3_path= s3_info_copy(s3))) if (!(share_s3= share->s3_path= s3f.info_copy(s3)))
goto err; /* EiOM */ goto err; /* EiOM */
/* Check if table has changed in S3 */ /* Check if table has changed in S3 */
if (s3_check_frm_version(s3_client, share_s3) == 1) if (s3f.check_frm_version(s3_client, share_s3) == 1)
{ {
my_errno= HA_ERR_TABLE_DEF_CHANGED; my_errno= HA_ERR_TABLE_DEF_CHANGED;
goto err; goto err;
} }
if (read_index_header(s3_client, share_s3, &index_header)) if (s3f.read_index_header(s3_client, share_s3, &index_header))
goto err; goto err;
if (index_header.length < head_length) if (index_header.length < head_length)
{ {
...@@ -398,7 +397,7 @@ MARIA_HA *maria_open(const char *name, int mode, uint open_flags, ...@@ -398,7 +397,7 @@ MARIA_HA *maria_open(const char *name, int mode, uint open_flags,
} }
memcpy(share->state.header.file_version, index_header.str, memcpy(share->state.header.file_version, index_header.str,
head_length); head_length);
kfile= s3_unique_file_number(); kfile= s3f.unique_file_number();
} }
#endif /* WITH_S3_STORAGE_ENGINE */ #endif /* WITH_S3_STORAGE_ENGINE */
...@@ -972,7 +971,7 @@ MARIA_HA *maria_open(const char *name, int mode, uint open_flags, ...@@ -972,7 +971,7 @@ MARIA_HA *maria_open(const char *name, int mode, uint open_flags,
} }
#ifdef WITH_S3_STORAGE_ENGINE #ifdef WITH_S3_STORAGE_ENGINE
else else
data_file= info.dfile.file= s3_unique_file_number(); data_file= info.dfile.file= s3f.unique_file_number();
#endif /* WITH_S3_STORAGE_ENGINE */ #endif /* WITH_S3_STORAGE_ENGINE */
} }
errpos= 5; errpos= 5;
...@@ -1149,7 +1148,7 @@ MARIA_HA *maria_open(const char *name, int mode, uint open_flags, ...@@ -1149,7 +1148,7 @@ MARIA_HA *maria_open(const char *name, int mode, uint open_flags,
if (s3_client) if (s3_client)
{ {
size_t block_size= share->base.s3_block_size; size_t block_size= share->base.s3_block_size;
ms3_set_option(s3_client, MS3_OPT_BUFFER_CHUNK_SIZE, &block_size); s3f.set_option(s3_client, MS3_OPT_BUFFER_CHUNK_SIZE, &block_size);
} }
#endif /* WITH_S3_STORAGE_ENGINE */ #endif /* WITH_S3_STORAGE_ENGINE */
} }
...@@ -1162,7 +1161,7 @@ MARIA_HA *maria_open(const char *name, int mode, uint open_flags, ...@@ -1162,7 +1161,7 @@ MARIA_HA *maria_open(const char *name, int mode, uint open_flags,
#ifdef WITH_S3_STORAGE_ENGINE #ifdef WITH_S3_STORAGE_ENGINE
if (index_header.alloc_ptr) if (index_header.alloc_ptr)
s3_free(&index_header); s3f.free(&index_header);
#endif /* WITH_S3_STORAGE_ENGINE */ #endif /* WITH_S3_STORAGE_ENGINE */
if (!(m_info= maria_clone_internal(share, mode, data_file, if (!(m_info= maria_clone_internal(share, mode, data_file,
...@@ -1224,9 +1223,9 @@ MARIA_HA *maria_open(const char *name, int mode, uint open_flags, ...@@ -1224,9 +1223,9 @@ MARIA_HA *maria_open(const char *name, int mode, uint open_flags,
} }
#ifdef WITH_S3_STORAGE_ENGINE #ifdef WITH_S3_STORAGE_ENGINE
if (s3_client) if (s3_client)
ms3_deinit(s3_client); s3f.deinit(s3_client);
if (index_header.alloc_ptr) if (index_header.alloc_ptr)
s3_free(&index_header); s3f.free(&index_header);
#endif /* WITH_S3_STORAGE_ENGINE */ #endif /* WITH_S3_STORAGE_ENGINE */
if (!internal_table) if (!internal_table)
mysql_mutex_unlock(&THR_LOCK_maria); mysql_mutex_unlock(&THR_LOCK_maria);
...@@ -1725,7 +1724,7 @@ uint _ma_state_info_read_dsk(File file __attribute__((unused)), ...@@ -1725,7 +1724,7 @@ uint _ma_state_info_read_dsk(File file __attribute__((unused)),
/**************************************************************************** /****************************************************************************
** store and read of MARIA_BASE_INFO ** store MARIA_BASE_INFO
****************************************************************************/ ****************************************************************************/
uint _ma_base_info_write(File file, MARIA_BASE_INFO *base) uint _ma_base_info_write(File file, MARIA_BASE_INFO *base)
...@@ -1775,49 +1774,6 @@ uint _ma_base_info_write(File file, MARIA_BASE_INFO *base) ...@@ -1775,49 +1774,6 @@ uint _ma_base_info_write(File file, MARIA_BASE_INFO *base)
} }
static uchar *_ma_base_info_read(uchar *ptr, MARIA_BASE_INFO *base)
{
bmove(base->uuid, ptr, MY_UUID_SIZE); ptr+= MY_UUID_SIZE;
base->keystart= mi_sizekorr(ptr); ptr+= 8;
base->max_data_file_length= mi_sizekorr(ptr); ptr+= 8;
base->max_key_file_length= mi_sizekorr(ptr); ptr+= 8;
base->records= (ha_rows) mi_sizekorr(ptr); ptr+= 8;
base->reloc= (ha_rows) mi_sizekorr(ptr); ptr+= 8;
base->mean_row_length= mi_uint4korr(ptr); ptr+= 4;
base->reclength= mi_uint4korr(ptr); ptr+= 4;
base->pack_reclength= mi_uint4korr(ptr); ptr+= 4;
base->min_pack_length= mi_uint4korr(ptr); ptr+= 4;
base->max_pack_length= mi_uint4korr(ptr); ptr+= 4;
base->min_block_length= mi_uint4korr(ptr); ptr+= 4;
base->fields= mi_uint2korr(ptr); ptr+= 2;
base->fixed_not_null_fields= mi_uint2korr(ptr); ptr+= 2;
base->fixed_not_null_fields_length= mi_uint2korr(ptr);ptr+= 2;
base->max_field_lengths= mi_uint2korr(ptr); ptr+= 2;
base->pack_fields= mi_uint2korr(ptr); ptr+= 2;
base->extra_options= mi_uint2korr(ptr); ptr+= 2;
base->null_bytes= mi_uint2korr(ptr); ptr+= 2;
base->original_null_bytes= mi_uint2korr(ptr); ptr+= 2;
base->field_offsets= mi_uint2korr(ptr); ptr+= 2;
base->language= mi_uint2korr(ptr); ptr+= 2;
base->block_size= mi_uint2korr(ptr); ptr+= 2;
base->rec_reflength= *ptr++;
base->key_reflength= *ptr++;
base->keys= *ptr++;
base->auto_key= *ptr++;
base->born_transactional= *ptr++;
base->compression_algorithm= *ptr++;
base->pack_bytes= mi_uint2korr(ptr); ptr+= 2;
base->blobs= mi_uint2korr(ptr); ptr+= 2;
base->max_key_block_length= mi_uint2korr(ptr); ptr+= 2;
base->max_key_length= mi_uint2korr(ptr); ptr+= 2;
base->extra_alloc_bytes= mi_uint2korr(ptr); ptr+= 2;
base->extra_alloc_procent= *ptr++;
base->s3_block_size= mi_uint3korr(ptr); ptr+= 3;
ptr+= 13;
return ptr;
}
/*-------------------------------------------------------------------------- /*--------------------------------------------------------------------------
maria_keydef maria_keydef
---------------------------------------------------------------------------*/ ---------------------------------------------------------------------------*/
......
/* Copyright (C) 2006 MySQL AB /* Copyright (C) 2006 MySQL AB
Copyright (c) 2011, 2020, MariaDB Corporation Ab
This program is free software; you can redistribute it and/or modify This program is free software; you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by it under the terms of the GNU General Public License as published by
...@@ -238,7 +239,7 @@ extern PAGECACHE dflt_pagecache_var, *dflt_pagecache; ...@@ -238,7 +239,7 @@ extern PAGECACHE dflt_pagecache_var, *dflt_pagecache;
extern size_t init_pagecache(PAGECACHE *pagecache, size_t use_mem, extern size_t init_pagecache(PAGECACHE *pagecache, size_t use_mem,
uint division_limit, uint age_threshold, uint division_limit, uint age_threshold,
uint block_size, uint changed_blocks_hash_size, uint block_size, uint changed_blocks_hash_size,
myf my_read_flags); myf my_read_flags)__attribute__((visibility("default"))) ;
extern size_t resize_pagecache(PAGECACHE *pagecache, extern size_t resize_pagecache(PAGECACHE *pagecache,
size_t use_mem, uint division_limit, size_t use_mem, uint division_limit,
uint age_threshold, uint changed_blocks_hash_size); uint age_threshold, uint changed_blocks_hash_size);
...@@ -318,7 +319,7 @@ extern int flush_pagecache_blocks_with_filter(PAGECACHE *keycache, ...@@ -318,7 +319,7 @@ extern int flush_pagecache_blocks_with_filter(PAGECACHE *keycache,
PAGECACHE_FILE *file, PAGECACHE_FILE *file,
enum flush_type type, enum flush_type type,
PAGECACHE_FLUSH_FILTER filter, PAGECACHE_FLUSH_FILTER filter,
void *filter_arg); void *filter_arg)__attribute__((visibility("default"))) ;
extern my_bool pagecache_delete(PAGECACHE *pagecache, extern my_bool pagecache_delete(PAGECACHE *pagecache,
PAGECACHE_FILE *file, PAGECACHE_FILE *file,
pgcache_page_no_t pageno, pgcache_page_no_t pageno,
...@@ -334,7 +335,7 @@ extern my_bool pagecache_delete_pages(PAGECACHE *pagecache, ...@@ -334,7 +335,7 @@ extern my_bool pagecache_delete_pages(PAGECACHE *pagecache,
uint page_count, uint page_count,
enum pagecache_page_lock lock, enum pagecache_page_lock lock,
my_bool flush); my_bool flush);
extern void end_pagecache(PAGECACHE *keycache, my_bool cleanup); extern void end_pagecache(PAGECACHE *keycache, my_bool cleanup)__attribute__((visibility("default"))) ;
extern my_bool pagecache_collect_changed_blocks_with_lsn(PAGECACHE *pagecache, extern my_bool pagecache_collect_changed_blocks_with_lsn(PAGECACHE *pagecache,
LEX_STRING *str, LEX_STRING *str,
LSN *min_lsn); LSN *min_lsn);
......
/* Copyright (C) 2006 MySQL AB & MySQL Finland AB & TCX DataKonsult AB /* Copyright (C) 2006 MySQL AB & MySQL Finland AB & TCX DataKonsult AB
Copyright (c) 2010, 2020, MariaDB Corporation Ab
This program is free software; you can redistribute it and/or modify This program is free software; you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by it under the terms of the GNU General Public License as published by
...@@ -32,7 +33,7 @@ uchar maria_pack_file_magic[]= ...@@ -32,7 +33,7 @@ uchar maria_pack_file_magic[]=
/* Unique number for this maria instance */ /* Unique number for this maria instance */
uchar maria_uuid[MY_UUID_SIZE]; uchar maria_uuid[MY_UUID_SIZE];
uint maria_quick_table_bits=9; uint maria_quick_table_bits=9;
ulong maria_block_size= MARIA_KEY_BLOCK_LENGTH; ulong __attribute__((visibility("default"))) maria_block_size= MARIA_KEY_BLOCK_LENGTH;
my_bool maria_flush= 0, maria_single_user= 0; my_bool maria_flush= 0, maria_single_user= 0;
my_bool maria_delay_key_write= 0, maria_page_checksums= 1; my_bool maria_delay_key_write= 0, maria_page_checksums= 1;
my_bool maria_inited= FALSE; my_bool maria_inited= FALSE;
...@@ -146,3 +147,8 @@ PSI_file_key key_file_translog, key_file_kfile, key_file_dfile, ...@@ -146,3 +147,8 @@ PSI_file_key key_file_translog, key_file_kfile, key_file_dfile,
/* Note that PSI_stage_info globals must always be declared. */ /* Note that PSI_stage_info globals must always be declared. */
PSI_stage_info stage_waiting_for_a_resource= { 0, "Waiting for a resource", 0}; PSI_stage_info stage_waiting_for_a_resource= { 0, "Waiting for a resource", 0};
#ifdef WITH_S3_STORAGE_ENGINE
#include "s3_func.h"
struct s3_func __attribute__((visibility("default"))) s3f;
#endif
/* Copyright (C) 2006 MySQL AB & MySQL Finland AB & TCX DataKonsult AB /* Copyright (C) 2006 MySQL AB & MySQL Finland AB & TCX DataKonsult AB
Copyright (c) 2009, 2020, MariaDB Corporation Ab
This program is free software; you can redistribute it and/or modify This program is free software; you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by it under the terms of the GNU General Public License as published by
...@@ -606,6 +607,7 @@ typedef struct st_ma_base_info ...@@ -606,6 +607,7 @@ typedef struct st_ma_base_info
my_bool born_transactional; my_bool born_transactional;
} MARIA_BASE_INFO; } MARIA_BASE_INFO;
uchar *_ma_base_info_read(uchar *ptr, MARIA_BASE_INFO *base);
/* Structs used intern in database */ /* Structs used intern in database */
...@@ -1616,7 +1618,7 @@ extern size_t _ma_nommap_pwrite(MARIA_HA *info, const uchar *Buffer, ...@@ -1616,7 +1618,7 @@ extern size_t _ma_nommap_pwrite(MARIA_HA *info, const uchar *Buffer,
#define MA_STATE_INFO_WRITE_FULL_INFO 2 #define MA_STATE_INFO_WRITE_FULL_INFO 2
/* intern_lock taking is needed */ /* intern_lock taking is needed */
#define MA_STATE_INFO_WRITE_LOCK 4 #define MA_STATE_INFO_WRITE_LOCK 4
uint _ma_state_info_write(MARIA_SHARE *share, uint pWrite); uint _ma_state_info_write(MARIA_SHARE *share, uint pWrite)__attribute__((visibility("default"))) ;
uint _ma_state_info_write_sub(File file, MARIA_STATE_INFO *state, uint pWrite); uint _ma_state_info_write_sub(File file, MARIA_STATE_INFO *state, uint pWrite);
uint _ma_state_info_read_dsk(File file, MARIA_STATE_INFO *state); uint _ma_state_info_read_dsk(File file, MARIA_STATE_INFO *state);
uint _ma_base_info_write(File file, MARIA_BASE_INFO *base); uint _ma_base_info_write(File file, MARIA_BASE_INFO *base);
...@@ -1660,7 +1662,8 @@ void _ma_remap_file(MARIA_HA *info, my_off_t size); ...@@ -1660,7 +1662,8 @@ void _ma_remap_file(MARIA_HA *info, my_off_t size);
MARIA_RECORD_POS _ma_write_init_default(MARIA_HA *info, const uchar *record); MARIA_RECORD_POS _ma_write_init_default(MARIA_HA *info, const uchar *record);
my_bool _ma_write_abort_default(MARIA_HA *info); my_bool _ma_write_abort_default(MARIA_HA *info);
int maria_delete_table_files(const char *name, my_bool temporary, int maria_delete_table_files(const char *name, my_bool temporary,
myf flags); myf flags)__attribute__((visibility("default"))) ;
/* /*
This cannot be in my_base.h as it clashes with HA_SPATIAL. This cannot be in my_base.h as it clashes with HA_SPATIAL.
......
[mariadb]
plugin-load-add=ha_s3
#
# Uncomment and configure the S3 engine
#
#s3-host-name=s3.amazonaws.com
#s3-protocol-version=Amazon
#s3-bucket=...
#s3-access-key=...
#s3-secret-key=...
#s3-region=eu-north-1
#ifndef S3_FUNC_INCLUDED #ifndef S3_FUNC_INCLUDED
#define S3_FUNC_INCLUDED #define S3_FUNC_INCLUDED
/* Copyright (C) 2019 MariaDB Corporation Ab /* Copyright (C) 2019, 2020 MariaDB Corporation Ab
This program is free software; you can redistribute it and/or modify This program is free software; you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by it under the terms of the GNU General Public License as published by
...@@ -20,11 +20,23 @@ ...@@ -20,11 +20,23 @@
*/ */
#ifdef WITH_S3_STORAGE_ENGINE #ifdef WITH_S3_STORAGE_ENGINE
C_MODE_START
#include <libmarias3/marias3.h> #include <libmarias3/marias3.h>
C_MODE_START
#define DEFAULT_AWS_HOST_NAME "s3.amazonaws.com" #define DEFAULT_AWS_HOST_NAME "s3.amazonaws.com"
extern struct s3_func {
uint8_t (*set_option)(ms3_st *, ms3_set_option_t, void *);
void (*free)(S3_BLOCK *);
void (*deinit)(ms3_st *);
int32 (*unique_file_number)(void);
my_bool (*read_index_header)(ms3_st *, S3_INFO *, S3_BLOCK *);
int (*check_frm_version)(ms3_st *, S3_INFO *);
S3_INFO *(*info_copy)(S3_INFO *);
my_bool (*set_database_and_table_from_path)(S3_INFO *, const char *);
ms3_st *(*open_connection)(S3_INFO *);
} s3f;
extern TYPELIB s3_protocol_typelib; extern TYPELIB s3_protocol_typelib;
/* Store information about a s3 connection */ /* Store information about a s3 connection */
...@@ -34,7 +46,6 @@ typedef struct s3_info ...@@ -34,7 +46,6 @@ typedef struct s3_info
/* Connection strings */ /* Connection strings */
LEX_CSTRING access_key, secret_key, region, bucket, host_name; LEX_CSTRING access_key, secret_key, region, bucket, host_name;
/* Will be set by caller or by ma_open() */ /* Will be set by caller or by ma_open() */
LEX_CSTRING database, table; LEX_CSTRING database, table;
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment