Skip to content
Projects
Groups
Snippets
Help
Loading...
Help
Support
Keyboard shortcuts
?
Submit feedback
Contribute to GitLab
Sign in / Register
Toggle navigation
M
MariaDB
Project overview
Project overview
Details
Activity
Releases
Repository
Repository
Files
Commits
Branches
Tags
Contributors
Graph
Compare
Issues
0
Issues
0
List
Boards
Labels
Milestones
Merge Requests
0
Merge Requests
0
CI / CD
CI / CD
Pipelines
Jobs
Schedules
Analytics
Analytics
CI / CD
Repository
Value Stream
Wiki
Wiki
Snippets
Snippets
Members
Members
Collapse sidebar
Close sidebar
Activity
Graph
Create a new issue
Jobs
Commits
Issue Boards
Open sidebar
nexedi
MariaDB
Commits
35034d81
Commit
35034d81
authored
Jun 18, 2020
by
Sergei Golubchik
Committed by
Oleksandr Byelkin
Jun 19, 2020
Browse files
Options
Browse Files
Download
Email Patches
Plain Diff
S3 is pluggable now
parent
4acafaae
Changes
17
Hide whitespace changes
Inline
Side-by-side
Showing
17 changed files
with
127 additions
and
114 deletions
+127
-114
debian/control
debian/control
+9
-0
debian/mariadb-plugin-s3.install
debian/mariadb-plugin-s3.install
+3
-0
include/aria_backup.h
include/aria_backup.h
+1
-1
mysql-test/suite/s3/my.cnf
mysql-test/suite/s3/my.cnf
+1
-0
mysql-test/suite/s3/suite.pm
mysql-test/suite/s3/suite.pm
+1
-1
storage/maria/CMakeLists.txt
storage/maria/CMakeLists.txt
+6
-10
storage/maria/ha_maria.cc
storage/maria/ha_maria.cc
+20
-2
storage/maria/ha_maria.h
storage/maria/ha_maria.h
+5
-16
storage/maria/ha_s3.cc
storage/maria/ha_s3.cc
+17
-2
storage/maria/ma_backup.c
storage/maria/ma_backup.c
+7
-15
storage/maria/ma_close.c
storage/maria/ma_close.c
+2
-1
storage/maria/ma_open.c
storage/maria/ma_open.c
+13
-57
storage/maria/ma_pagecache.h
storage/maria/ma_pagecache.h
+4
-3
storage/maria/ma_static.c
storage/maria/ma_static.c
+7
-1
storage/maria/maria_def.h
storage/maria/maria_def.h
+5
-2
storage/maria/s3.cnf
storage/maria/s3.cnf
+12
-0
storage/maria/s3_func.h
storage/maria/s3_func.h
+14
-3
No files found.
debian/control
View file @
35034d81
...
@@ -623,6 +623,15 @@ Description: Connect storage engine for MariaDB
...
@@ -623,6 +623,15 @@ Description: Connect storage engine for MariaDB
other
interesting
features
.
other
interesting
features
.
This
package
contains
the
Connect
plugin
for
MariaDB
.
This
package
contains
the
Connect
plugin
for
MariaDB
.
Package
:
mariadb
-
plugin
-
s3
Architecture
:
any
Depends
:
libcurl4
,
mariadb
-
server
-
10.5
(=
${
binary
:
Version
}),
${
misc
:
Depends
},
${
shlibs
:
Depends
}
Description
:
S3
storage
engine
for
MariaDB
Read
only
table
stored
in
S3
.
Package
:
mariadb
-
plugin
-
rocksdb
Package
:
mariadb
-
plugin
-
rocksdb
Architecture
:
amd64
arm64
mips64el
ppc64el
Architecture
:
amd64
arm64
mips64el
ppc64el
Depends
:
mariadb
-
server
-
10.5
(=
${
binary
:
Version
}),
Depends
:
mariadb
-
server
-
10.5
(=
${
binary
:
Version
}),
...
...
debian/mariadb-plugin-s3.install
0 → 100644
View file @
35034d81
etc
/
mysql
/
mariadb
.
conf
.
d
/
s3
.
cnf
usr
/
bin
/
aria_s3_copy
usr
/
lib
/
mysql
/
plugin
/
ha_s3
.
so
include/aria_backup.h
View file @
35034d81
/* Copyright (C) 2018
MariaDB corporation
/* Copyright (C) 2018
,2020 MariaDB Corporation Ab
This program is free software; you can redistribute it and/or modify
This program is free software; you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
it under the terms of the GNU General Public License as published by
...
...
mysql-test/suite/s3/my.cnf
View file @
35034d81
...
@@ -2,6 +2,7 @@
...
@@ -2,6 +2,7 @@
!include include/default_client.cnf
!include include/default_client.cnf
[mysqld.1]
[mysqld.1]
plugin-load-add=ha_s3
s3=ON
s3=ON
#s3-host-name=s3.amazonaws.com
#s3-host-name=s3.amazonaws.com
#s3-protocol-version=Amazon
#s3-protocol-version=Amazon
...
...
mysql-test/suite/s3/suite.pm
View file @
35034d81
...
@@ -2,7 +2,7 @@ package My::Suite::S3;
...
@@ -2,7 +2,7 @@ package My::Suite::S3;
@ISA
=
qw(My::Suite)
;
@ISA
=
qw(My::Suite)
;
return
"
Need S3 engine
"
unless
$::mysqld_variables
{'
s3
'}
eq
"
ON
";
return
"
Need S3 engine
"
unless
$::mysqld_variables
{'
s3
'}
eq
"
ON
"
or
$ENV
{
HA_S3_SO
}
;
bless
{
};
bless
{
};
storage/maria/CMakeLists.txt
View file @
35034d81
# Copyright (C) 2007 MySQL AB
# Copyright (C) 2007 MySQL AB
# Copyright (C) 2009,2020 MariaDB Corporation Ab
#
#
# This program is free software; you can redistribute it and/or modify
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# it under the terms of the GNU General Public License as published by
...
@@ -105,7 +106,7 @@ OPTION(USE_ARIA_FOR_TMP_TABLES "Use Aria for temporary tables" ON)
...
@@ -105,7 +106,7 @@ OPTION(USE_ARIA_FOR_TMP_TABLES "Use Aria for temporary tables" ON)
#
#
INCLUDE
(
CheckIncludeFiles
)
INCLUDE
(
CheckIncludeFiles
)
SET
(
S3_SOURCES
ha_s3.cc
s3_func.c
SET
(
S3_SOURCES s3_func.c
libmarias3/src/debug.c libmarias3/src/error.c libmarias3/src/marias3.c
libmarias3/src/debug.c libmarias3/src/error.c libmarias3/src/marias3.c
libmarias3/src/request.c libmarias3/src/response.c libmarias3/src/sha256.c
libmarias3/src/request.c libmarias3/src/response.c libmarias3/src/sha256.c
libmarias3/src/sha256-internal.c libmarias3/src/xml.c
)
libmarias3/src/sha256-internal.c libmarias3/src/xml.c
)
...
@@ -115,18 +116,13 @@ IF(NOT PLUGIN_S3 STREQUAL NO)
...
@@ -115,18 +116,13 @@ IF(NOT PLUGIN_S3 STREQUAL NO)
ENDIF
()
ENDIF
()
IF
(
CURL_FOUND
)
IF
(
CURL_FOUND
)
MYSQL_ADD_PLUGIN
(
s3
${
S3_SOURCES
}
STORAGE_ENGINE STATIC_ONLY
MYSQL_ADD_PLUGIN
(
s3 ha_s3.cc
${
S3_SOURCES
}
COMPONENT s3-engine
LINK_LIBRARIES aria myisam mysys mysys_ssl curl
LINK_LIBRARIES curl STORAGE_ENGINE NOT_EMBEDDED CONFIG s3.cnf
)
NOT_EMBEDDED
)
ENDIF
()
ENDIF
()
IF
(
TARGET s3
)
IF
(
TARGET s3
)
MYSQL_ADD_EXECUTABLE
(
aria_s3_copy aria_s3_copy.cc COMPONENT Server
)
MYSQL_ADD_EXECUTABLE
(
aria_s3_copy aria_s3_copy.cc
${
S3_SOURCES
}
COMPONENT s3-engine
)
TARGET_LINK_LIBRARIES
(
aria_s3_copy s3
)
TARGET_LINK_LIBRARIES
(
aria_s3_copy aria myisam mysys mysys_ssl curl
)
INCLUDE_DIRECTORIES
(
${
CMAKE_CURRENT_SOURCE_DIR
}
/libmarias3
)
INCLUDE_DIRECTORIES
(
${
CMAKE_CURRENT_SOURCE_DIR
}
/libmarias3
)
ADD_DEFINITIONS
(
-DWITH_S3_STORAGE_ENGINE
)
ADD_DEFINITIONS
(
-DWITH_S3_STORAGE_ENGINE
)
TARGET_LINK_LIBRARIES
(
aria s3
)
ENDIF
()
ENDIF
()
storage/maria/ha_maria.cc
View file @
35034d81
/* Copyright (C) 2004-2008 MySQL AB & MySQL Finland AB & TCX DataKonsult AB
/* Copyright (C) 2004-2008 MySQL AB & MySQL Finland AB & TCX DataKonsult AB
Copyright (C) 2008-2009 Sun Microsystems, Inc.
Copyright (C) 2008-2009 Sun Microsystems, Inc.
Copyright (c) 2009, 20
17, MariaDB Corporation.
Copyright (c) 2009, 20
20, MariaDB Corporation Ab
This program is free software; you can redistribute it and/or modify
This program is free software; you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
it under the terms of the GNU General Public License as published by
...
@@ -72,7 +72,7 @@ const char *zerofill_error_msg=
...
@@ -72,7 +72,7 @@ const char *zerofill_error_msg=
corrupted.
corrupted.
*/
*/
ulonglong
maria_recover_options
=
HA_RECOVER_NONE
;
ulonglong
maria_recover_options
=
HA_RECOVER_NONE
;
handlerton
*
maria_hton
;
handlerton
__attribute__
((
visibility
(
"default"
)))
*
maria_hton
;
/* bits in maria_recover_options */
/* bits in maria_recover_options */
const
char
*
maria_recover_names
[]
=
const
char
*
maria_recover_names
[]
=
...
@@ -2688,6 +2688,16 @@ int ha_maria::extra_opt(enum ha_extra_function operation, ulong cache_size)
...
@@ -2688,6 +2688,16 @@ int ha_maria::extra_opt(enum ha_extra_function operation, ulong cache_size)
}
}
bool
ha_maria
::
auto_repair
(
int
error
)
const
{
/* Always auto-repair moved tables (error == HA_ERR_OLD_FILE) */
return
((
MY_TEST
(
maria_recover_options
&
HA_RECOVER_ANY
)
&&
error
==
HA_ERR_CRASHED_ON_USAGE
)
||
error
==
HA_ERR_OLD_FILE
);
}
int
ha_maria
::
delete_all_rows
()
int
ha_maria
::
delete_all_rows
()
{
{
THD
*
thd
=
table
->
in_use
;
THD
*
thd
=
table
->
in_use
;
...
@@ -3348,6 +3358,14 @@ ha_rows ha_maria::records_in_range(uint inx, const key_range *min_key,
...
@@ -3348,6 +3358,14 @@ ha_rows ha_maria::records_in_range(uint inx, const key_range *min_key,
}
}
FT_INFO
*
ha_maria
::
ft_init_ext
(
uint
flags
,
uint
inx
,
String
*
key
)
{
return
maria_ft_init_search
(
flags
,
file
,
inx
,
(
uchar
*
)
key
->
ptr
(),
key
->
length
(),
key
->
charset
(),
table
->
record
[
0
]);
}
int
ha_maria
::
ft_read
(
uchar
*
buf
)
int
ha_maria
::
ft_read
(
uchar
*
buf
)
{
{
int
error
;
int
error
;
...
...
storage/maria/ha_maria.h
View file @
35034d81
#ifndef HA_MARIA_INCLUDED
#ifndef HA_MARIA_INCLUDED
#define HA_MARIA_INCLUDED
#define HA_MARIA_INCLUDED
/* Copyright (C) 2006,2004 MySQL AB & MySQL Finland AB & TCX DataKonsult AB
/* Copyright (C) 2006, 2004 MySQL AB & MySQL Finland AB & TCX DataKonsult AB
Copyright (c) 2009, 2020, MariaDB Corporation Ab
This program is free software; you can redistribute it and/or modify
This program is free software; you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
it under the terms of the GNU General Public License as published by
...
@@ -38,7 +39,7 @@ C_MODE_END
...
@@ -38,7 +39,7 @@ C_MODE_END
extern
TYPELIB
maria_recover_typelib
;
extern
TYPELIB
maria_recover_typelib
;
extern
ulonglong
maria_recover_options
;
extern
ulonglong
maria_recover_options
;
class
ha_maria
:
public
handler
class
__attribute__
((
visibility
(
"default"
)))
ha_maria
:
public
handler
{
{
public:
public:
MARIA_HA
*
file
;
MARIA_HA
*
file
;
...
@@ -97,12 +98,7 @@ class ha_maria :public handler
...
@@ -97,12 +98,7 @@ class ha_maria :public handler
ft_handler
->
please
->
reinit_search
(
ft_handler
);
ft_handler
->
please
->
reinit_search
(
ft_handler
);
return
0
;
return
0
;
}
}
FT_INFO
*
ft_init_ext
(
uint
flags
,
uint
inx
,
String
*
key
)
FT_INFO
*
ft_init_ext
(
uint
flags
,
uint
inx
,
String
*
key
);
{
return
maria_ft_init_search
(
flags
,
file
,
inx
,
(
uchar
*
)
key
->
ptr
(),
key
->
length
(),
key
->
charset
(),
table
->
record
[
0
]);
}
int
ft_read
(
uchar
*
buf
);
int
ft_read
(
uchar
*
buf
);
int
index_init
(
uint
idx
,
bool
sorted
);
int
index_init
(
uint
idx
,
bool
sorted
);
int
index_end
();
int
index_end
();
...
@@ -146,14 +142,7 @@ class ha_maria :public handler
...
@@ -146,14 +142,7 @@ class ha_maria :public handler
bool
check_and_repair
(
THD
*
thd
);
bool
check_and_repair
(
THD
*
thd
);
bool
is_crashed
()
const
;
bool
is_crashed
()
const
;
bool
is_changed
()
const
;
bool
is_changed
()
const
;
bool
auto_repair
(
int
error
)
const
bool
auto_repair
(
int
error
)
const
;
{
/* Always auto-repair moved tables (error == HA_ERR_OLD_FILE) */
return
((
MY_TEST
(
maria_recover_options
&
HA_RECOVER_ANY
)
&&
error
==
HA_ERR_CRASHED_ON_USAGE
)
||
error
==
HA_ERR_OLD_FILE
);
}
int
optimize
(
THD
*
thd
,
HA_CHECK_OPT
*
check_opt
);
int
optimize
(
THD
*
thd
,
HA_CHECK_OPT
*
check_opt
);
int
assign_to_keycache
(
THD
*
thd
,
HA_CHECK_OPT
*
check_opt
);
int
assign_to_keycache
(
THD
*
thd
,
HA_CHECK_OPT
*
check_opt
);
int
preload_keys
(
THD
*
thd
,
HA_CHECK_OPT
*
check_opt
);
int
preload_keys
(
THD
*
thd
,
HA_CHECK_OPT
*
check_opt
);
...
...
storage/maria/ha_s3.cc
View file @
35034d81
/* Copyright (C) 2019
MariaDB Corppration AB
/* Copyright (C) 2019
, 2020 MariaDB Corporation Ab
This program is free software; you can redistribute it and/or modify
This program is free software; you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
it under the terms of the GNU General Public License as published by
...
@@ -1008,9 +1008,24 @@ static int ha_s3_init(void *p)
...
@@ -1008,9 +1008,24 @@ static int ha_s3_init(void *p)
s3_init_library
();
s3_init_library
();
if
(
s3_debug
)
if
(
s3_debug
)
ms3_debug
();
ms3_debug
();
struct
s3_func
s3f_real
=
{
ms3_set_option
,
s3_free
,
ms3_deinit
,
s3_unique_file_number
,
read_index_header
,
s3_check_frm_version
,
s3_info_copy
,
set_database_and_table_from_path
,
s3_open_connection
};
s3f
=
s3f_real
;
return
res
?
HA_ERR_INITIALIZATION
:
0
;
return
res
?
HA_ERR_INITIALIZATION
:
0
;
}
}
static
int
ha_s3_deinit
(
void
*
)
{
bzero
(
&
s3f
,
sizeof
(
s3f
));
return
0
;
}
static
SHOW_VAR
status_variables
[]
=
{
static
SHOW_VAR
status_variables
[]
=
{
{
"pagecache_blocks_not_flushed"
,
{
"pagecache_blocks_not_flushed"
,
(
char
*
)
&
s3_pagecache
.
global_blocks_changed
,
SHOW_LONG
},
(
char
*
)
&
s3_pagecache
.
global_blocks_changed
,
SHOW_LONG
},
...
@@ -1057,7 +1072,7 @@ maria_declare_plugin(s3)
...
@@ -1057,7 +1072,7 @@ maria_declare_plugin(s3)
"ALTER TABLE table_name ENGINE=s3"
,
"ALTER TABLE table_name ENGINE=s3"
,
PLUGIN_LICENSE_GPL
,
PLUGIN_LICENSE_GPL
,
ha_s3_init
,
/* Plugin Init */
ha_s3_init
,
/* Plugin Init */
NULL
,
/* Plugin Deinit */
ha_s3_deinit
,
/* Plugin Deinit */
0x0100
,
/* 1.0 */
0x0100
,
/* 1.0 */
status_variables
,
/* status variables */
status_variables
,
/* status variables */
system_variables
,
/* system variables */
system_variables
,
/* system variables */
...
...
storage/maria/ma_backup.c
View file @
35034d81
/* Copyright (C) 2018
MariaDB corporation
/* Copyright (C) 2018
, 2020 MariaDB Corporation Ab
This program is free software; you can redistribute it and/or modify
This program is free software; you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
it under the terms of the GNU General Public License as published by
...
@@ -20,8 +20,6 @@
...
@@ -20,8 +20,6 @@
#include "ma_checkpoint.h"
#include "ma_checkpoint.h"
#include <aria_backup.h>
#include <aria_backup.h>
static
uchar
*
_ma_base_info_read
(
uchar
*
ptr
,
MARIA_BASE_INFO
*
base
);
/**
/**
@brief Get capabilites for an Aria table
@brief Get capabilites for an Aria table
...
@@ -32,6 +30,7 @@ static uchar *_ma_base_info_read(uchar *ptr, MARIA_BASE_INFO *base);
...
@@ -32,6 +30,7 @@ static uchar *_ma_base_info_read(uchar *ptr, MARIA_BASE_INFO *base);
@return X errno
@return X errno
*/
*/
int
aria_get_capabilities
(
File
kfile
,
ARIA_TABLE_CAPABILITIES
*
cap
)
__attribute__
((
visibility
(
"default"
)))
;
int
aria_get_capabilities
(
File
kfile
,
ARIA_TABLE_CAPABILITIES
*
cap
)
int
aria_get_capabilities
(
File
kfile
,
ARIA_TABLE_CAPABILITIES
*
cap
)
{
{
MARIA_SHARE
share
;
MARIA_SHARE
share
;
...
@@ -102,20 +101,13 @@ int aria_get_capabilities(File kfile, ARIA_TABLE_CAPABILITIES *cap)
...
@@ -102,20 +101,13 @@ int aria_get_capabilities(File kfile, ARIA_TABLE_CAPABILITIES *cap)
err:
err:
my_free
(
disc_cache
);
my_free
(
disc_cache
);
DBUG_RETURN
(
error
);
DBUG_RETURN
(
error
);
}
/* maria_get_capabilities */
}
/* aria_get_capabilities */
/*
/****************************************************************************
This is a copy of my_base_info_read from ma_open().
** store MARIA_BASE_INFO
The base information will never change (something may be added
****************************************************************************/
last, but not relevant for maria_get_capabilities), so it's safe to
copy it here.
The copy is done to avoid linking in the fill Aria library just
because maria_backup uses maria_get_capabilities()
*/
static
uchar
*
_ma_base_info_read
(
uchar
*
ptr
,
MARIA_BASE_INFO
*
base
)
uchar
*
_ma_base_info_read
(
uchar
*
ptr
,
MARIA_BASE_INFO
*
base
)
{
{
bmove
(
base
->
uuid
,
ptr
,
MY_UUID_SIZE
);
ptr
+=
MY_UUID_SIZE
;
bmove
(
base
->
uuid
,
ptr
,
MY_UUID_SIZE
);
ptr
+=
MY_UUID_SIZE
;
base
->
keystart
=
mi_sizekorr
(
ptr
);
ptr
+=
8
;
base
->
keystart
=
mi_sizekorr
(
ptr
);
ptr
+=
8
;
...
...
storage/maria/ma_close.c
View file @
35034d81
/* Copyright (C) 2006 MySQL AB & MySQL Finland AB & TCX DataKonsult AB
/* Copyright (C) 2006 MySQL AB & MySQL Finland AB & TCX DataKonsult AB
Copyright (c) 2010, 2020, MariaDB Corporation Ab
This program is free software; you can redistribute it and/or modify
This program is free software; you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
it under the terms of the GNU General Public License as published by
...
@@ -253,7 +254,7 @@ int maria_close(register MARIA_HA *info)
...
@@ -253,7 +254,7 @@ int maria_close(register MARIA_HA *info)
delete_dynamic
(
&
info
->
pinned_pages
);
delete_dynamic
(
&
info
->
pinned_pages
);
#ifdef WITH_S3_STORAGE_ENGINE
#ifdef WITH_S3_STORAGE_ENGINE
if
(
info
->
s3
)
if
(
info
->
s3
)
ms3_
deinit
(
info
->
s3
);
s3f
.
deinit
(
info
->
s3
);
#endif
/* WITH_S3_STORAGE_ENGINE */
#endif
/* WITH_S3_STORAGE_ENGINE */
my_free
(
info
);
my_free
(
info
);
...
...
storage/maria/ma_open.c
View file @
35034d81
/* Copyright (C) 2006 MySQL AB & MySQL Finland AB & TCX DataKonsult AB
/* Copyright (C) 2006 MySQL AB & MySQL Finland AB & TCX DataKonsult AB
Copyright (c) 2009, 20
19, MariaDB Corporation.
Copyright (c) 2009, 20
20, MariaDB Corporation Ab
This program is free software; you can redistribute it and/or modify
This program is free software; you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
it under the terms of the GNU General Public License as published by
...
@@ -39,7 +39,6 @@ static my_bool maria_scan_init_dummy(MARIA_HA *info);
...
@@ -39,7 +39,6 @@ static my_bool maria_scan_init_dummy(MARIA_HA *info);
static
void
maria_scan_end_dummy
(
MARIA_HA
*
info
);
static
void
maria_scan_end_dummy
(
MARIA_HA
*
info
);
static
my_bool
maria_once_init_dummy
(
MARIA_SHARE
*
,
File
);
static
my_bool
maria_once_init_dummy
(
MARIA_SHARE
*
,
File
);
static
my_bool
maria_once_end_dummy
(
MARIA_SHARE
*
);
static
my_bool
maria_once_end_dummy
(
MARIA_SHARE
*
);
static
uchar
*
_ma_base_info_read
(
uchar
*
ptr
,
MARIA_BASE_INFO
*
base
);
static
uchar
*
_ma_state_info_read
(
uchar
*
ptr
,
MARIA_STATE_INFO
*
state
);
static
uchar
*
_ma_state_info_read
(
uchar
*
ptr
,
MARIA_STATE_INFO
*
state
);
#define get_next_element(to,pos,size) { memcpy((char*) to,pos,(size_t) size); \
#define get_next_element(to,pos,size) { memcpy((char*) to,pos,(size_t) size); \
...
@@ -318,7 +317,7 @@ MARIA_HA *maria_open(const char *name, int mode, uint open_flags,
...
@@ -318,7 +317,7 @@ MARIA_HA *maria_open(const char *name, int mode, uint open_flags,
else
else
{
{
strmake
(
name_buff
,
name
,
sizeof
(
name_buff
)
-
1
);
/* test_if_reopen() */
strmake
(
name_buff
,
name
,
sizeof
(
name_buff
)
-
1
);
/* test_if_reopen() */
if
(
!
(
s3_client
=
s3
_
open_connection
(
s3
)))
if
(
!
(
s3_client
=
s3
f
.
open_connection
(
s3
)))
{
{
internal_table
=
1
;
/* Avoid unlock on error */
internal_table
=
1
;
/* Avoid unlock on error */
goto
err
;
goto
err
;
...
@@ -371,7 +370,7 @@ MARIA_HA *maria_open(const char *name, int mode, uint open_flags,
...
@@ -371,7 +370,7 @@ MARIA_HA *maria_open(const char *name, int mode, uint open_flags,
else
else
{
{
errpos
=
1
;
errpos
=
1
;
if
(
set_database_and_table_from_path
(
s3
,
name_buff
))
if
(
s
3f
.
s
et_database_and_table_from_path
(
s3
,
name_buff
))
{
{
my_printf_error
(
HA_ERR_NO_SUCH_TABLE
,
my_printf_error
(
HA_ERR_NO_SUCH_TABLE
,
"Can't find database and path from %s"
,
MYF
(
0
),
"Can't find database and path from %s"
,
MYF
(
0
),
...
@@ -379,17 +378,17 @@ MARIA_HA *maria_open(const char *name, int mode, uint open_flags,
...
@@ -379,17 +378,17 @@ MARIA_HA *maria_open(const char *name, int mode, uint open_flags,
my_errno
=
HA_ERR_NO_SUCH_TABLE
;
my_errno
=
HA_ERR_NO_SUCH_TABLE
;
goto
err
;
goto
err
;
}
}
if
(
!
(
share_s3
=
share
->
s3_path
=
s3
_
info_copy
(
s3
)))
if
(
!
(
share_s3
=
share
->
s3_path
=
s3
f
.
info_copy
(
s3
)))
goto
err
;
/* EiOM */
goto
err
;
/* EiOM */
/* Check if table has changed in S3 */
/* Check if table has changed in S3 */
if
(
s3
_
check_frm_version
(
s3_client
,
share_s3
)
==
1
)
if
(
s3
f
.
check_frm_version
(
s3_client
,
share_s3
)
==
1
)
{
{
my_errno
=
HA_ERR_TABLE_DEF_CHANGED
;
my_errno
=
HA_ERR_TABLE_DEF_CHANGED
;
goto
err
;
goto
err
;
}
}
if
(
read_index_header
(
s3_client
,
share_s3
,
&
index_header
))
if
(
s3f
.
read_index_header
(
s3_client
,
share_s3
,
&
index_header
))
goto
err
;
goto
err
;
if
(
index_header
.
length
<
head_length
)
if
(
index_header
.
length
<
head_length
)
{
{
...
@@ -398,7 +397,7 @@ MARIA_HA *maria_open(const char *name, int mode, uint open_flags,
...
@@ -398,7 +397,7 @@ MARIA_HA *maria_open(const char *name, int mode, uint open_flags,
}
}
memcpy
(
share
->
state
.
header
.
file_version
,
index_header
.
str
,
memcpy
(
share
->
state
.
header
.
file_version
,
index_header
.
str
,
head_length
);
head_length
);
kfile
=
s3
_
unique_file_number
();
kfile
=
s3
f
.
unique_file_number
();
}
}
#endif
/* WITH_S3_STORAGE_ENGINE */
#endif
/* WITH_S3_STORAGE_ENGINE */
...
@@ -972,7 +971,7 @@ MARIA_HA *maria_open(const char *name, int mode, uint open_flags,
...
@@ -972,7 +971,7 @@ MARIA_HA *maria_open(const char *name, int mode, uint open_flags,
}
}
#ifdef WITH_S3_STORAGE_ENGINE
#ifdef WITH_S3_STORAGE_ENGINE
else
else
data_file
=
info
.
dfile
.
file
=
s3
_
unique_file_number
();
data_file
=
info
.
dfile
.
file
=
s3
f
.
unique_file_number
();
#endif
/* WITH_S3_STORAGE_ENGINE */
#endif
/* WITH_S3_STORAGE_ENGINE */
}
}
errpos
=
5
;
errpos
=
5
;
...
@@ -1149,7 +1148,7 @@ MARIA_HA *maria_open(const char *name, int mode, uint open_flags,
...
@@ -1149,7 +1148,7 @@ MARIA_HA *maria_open(const char *name, int mode, uint open_flags,
if
(
s3_client
)
if
(
s3_client
)
{
{
size_t
block_size
=
share
->
base
.
s3_block_size
;
size_t
block_size
=
share
->
base
.
s3_block_size
;
ms3_
set_option
(
s3_client
,
MS3_OPT_BUFFER_CHUNK_SIZE
,
&
block_size
);
s3f
.
set_option
(
s3_client
,
MS3_OPT_BUFFER_CHUNK_SIZE
,
&
block_size
);
}
}
#endif
/* WITH_S3_STORAGE_ENGINE */
#endif
/* WITH_S3_STORAGE_ENGINE */
}
}
...
@@ -1162,7 +1161,7 @@ MARIA_HA *maria_open(const char *name, int mode, uint open_flags,
...
@@ -1162,7 +1161,7 @@ MARIA_HA *maria_open(const char *name, int mode, uint open_flags,
#ifdef WITH_S3_STORAGE_ENGINE
#ifdef WITH_S3_STORAGE_ENGINE
if
(
index_header
.
alloc_ptr
)
if
(
index_header
.
alloc_ptr
)
s3
_
free
(
&
index_header
);
s3
f
.
free
(
&
index_header
);
#endif
/* WITH_S3_STORAGE_ENGINE */
#endif
/* WITH_S3_STORAGE_ENGINE */
if
(
!
(
m_info
=
maria_clone_internal
(
share
,
mode
,
data_file
,
if
(
!
(
m_info
=
maria_clone_internal
(
share
,
mode
,
data_file
,
...
@@ -1224,9 +1223,9 @@ MARIA_HA *maria_open(const char *name, int mode, uint open_flags,
...
@@ -1224,9 +1223,9 @@ MARIA_HA *maria_open(const char *name, int mode, uint open_flags,
}
}
#ifdef WITH_S3_STORAGE_ENGINE
#ifdef WITH_S3_STORAGE_ENGINE
if
(
s3_client
)
if
(
s3_client
)
ms3_
deinit
(
s3_client
);
s3f
.
deinit
(
s3_client
);
if
(
index_header
.
alloc_ptr
)
if
(
index_header
.
alloc_ptr
)
s3
_
free
(
&
index_header
);
s3
f
.
free
(
&
index_header
);
#endif
/* WITH_S3_STORAGE_ENGINE */
#endif
/* WITH_S3_STORAGE_ENGINE */
if
(
!
internal_table
)
if
(
!
internal_table
)
mysql_mutex_unlock
(
&
THR_LOCK_maria
);
mysql_mutex_unlock
(
&
THR_LOCK_maria
);
...
@@ -1725,7 +1724,7 @@ uint _ma_state_info_read_dsk(File file __attribute__((unused)),
...
@@ -1725,7 +1724,7 @@ uint _ma_state_info_read_dsk(File file __attribute__((unused)),
/****************************************************************************
/****************************************************************************
** store
and read of
MARIA_BASE_INFO
** store MARIA_BASE_INFO
****************************************************************************/
****************************************************************************/
uint
_ma_base_info_write
(
File
file
,
MARIA_BASE_INFO
*
base
)
uint
_ma_base_info_write
(
File
file
,
MARIA_BASE_INFO
*
base
)
...
@@ -1775,49 +1774,6 @@ uint _ma_base_info_write(File file, MARIA_BASE_INFO *base)
...
@@ -1775,49 +1774,6 @@ uint _ma_base_info_write(File file, MARIA_BASE_INFO *base)
}
}
static
uchar
*
_ma_base_info_read
(
uchar
*
ptr
,
MARIA_BASE_INFO
*
base
)
{
bmove
(
base
->
uuid
,
ptr
,
MY_UUID_SIZE
);
ptr
+=
MY_UUID_SIZE
;
base
->
keystart
=
mi_sizekorr
(
ptr
);
ptr
+=
8
;
base
->
max_data_file_length
=
mi_sizekorr
(
ptr
);
ptr
+=
8
;
base
->
max_key_file_length
=
mi_sizekorr
(
ptr
);
ptr
+=
8
;
base
->
records
=
(
ha_rows
)
mi_sizekorr
(
ptr
);
ptr
+=
8
;
base
->
reloc
=
(
ha_rows
)
mi_sizekorr
(
ptr
);
ptr
+=
8
;
base
->
mean_row_length
=
mi_uint4korr
(
ptr
);
ptr
+=
4
;
base
->
reclength
=
mi_uint4korr
(
ptr
);
ptr
+=
4
;
base
->
pack_reclength
=
mi_uint4korr
(
ptr
);
ptr
+=
4
;
base
->
min_pack_length
=
mi_uint4korr
(
ptr
);
ptr
+=
4
;
base
->
max_pack_length
=
mi_uint4korr
(
ptr
);
ptr
+=
4
;
base
->
min_block_length
=
mi_uint4korr
(
ptr
);
ptr
+=
4
;
base
->
fields
=
mi_uint2korr
(
ptr
);
ptr
+=
2
;
base
->
fixed_not_null_fields
=
mi_uint2korr
(
ptr
);
ptr
+=
2
;
base
->
fixed_not_null_fields_length
=
mi_uint2korr
(
ptr
);
ptr
+=
2
;
base
->
max_field_lengths
=
mi_uint2korr
(
ptr
);
ptr
+=
2
;
base
->
pack_fields
=
mi_uint2korr
(
ptr
);
ptr
+=
2
;
base
->
extra_options
=
mi_uint2korr
(
ptr
);
ptr
+=
2
;
base
->
null_bytes
=
mi_uint2korr
(
ptr
);
ptr
+=
2
;
base
->
original_null_bytes
=
mi_uint2korr
(
ptr
);
ptr
+=
2
;
base
->
field_offsets
=
mi_uint2korr
(
ptr
);
ptr
+=
2
;
base
->
language
=
mi_uint2korr
(
ptr
);
ptr
+=
2
;
base
->
block_size
=
mi_uint2korr
(
ptr
);
ptr
+=
2
;
base
->
rec_reflength
=
*
ptr
++
;
base
->
key_reflength
=
*
ptr
++
;
base
->
keys
=
*
ptr
++
;
base
->
auto_key
=
*
ptr
++
;
base
->
born_transactional
=
*
ptr
++
;
base
->
compression_algorithm
=
*
ptr
++
;
base
->
pack_bytes
=
mi_uint2korr
(
ptr
);
ptr
+=
2
;
base
->
blobs
=
mi_uint2korr
(
ptr
);
ptr
+=
2
;
base
->
max_key_block_length
=
mi_uint2korr
(
ptr
);
ptr
+=
2
;
base
->
max_key_length
=
mi_uint2korr
(
ptr
);
ptr
+=
2
;
base
->
extra_alloc_bytes
=
mi_uint2korr
(
ptr
);
ptr
+=
2
;
base
->
extra_alloc_procent
=
*
ptr
++
;
base
->
s3_block_size
=
mi_uint3korr
(
ptr
);
ptr
+=
3
;
ptr
+=
13
;
return
ptr
;
}
/*--------------------------------------------------------------------------
/*--------------------------------------------------------------------------
maria_keydef
maria_keydef
---------------------------------------------------------------------------*/
---------------------------------------------------------------------------*/
...
...
storage/maria/ma_pagecache.h
View file @
35034d81
/* Copyright (C) 2006 MySQL AB
/* Copyright (C) 2006 MySQL AB
Copyright (c) 2011, 2020, MariaDB Corporation Ab
This program is free software; you can redistribute it and/or modify
This program is free software; you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
it under the terms of the GNU General Public License as published by
...
@@ -238,7 +239,7 @@ extern PAGECACHE dflt_pagecache_var, *dflt_pagecache;
...
@@ -238,7 +239,7 @@ extern PAGECACHE dflt_pagecache_var, *dflt_pagecache;
extern
size_t
init_pagecache
(
PAGECACHE
*
pagecache
,
size_t
use_mem
,
extern
size_t
init_pagecache
(
PAGECACHE
*
pagecache
,
size_t
use_mem
,
uint
division_limit
,
uint
age_threshold
,
uint
division_limit
,
uint
age_threshold
,
uint
block_size
,
uint
changed_blocks_hash_size
,
uint
block_size
,
uint
changed_blocks_hash_size
,
myf
my_read_flags
);
myf
my_read_flags
)
__attribute__
((
visibility
(
"default"
)))
;
extern
size_t
resize_pagecache
(
PAGECACHE
*
pagecache
,
extern
size_t
resize_pagecache
(
PAGECACHE
*
pagecache
,
size_t
use_mem
,
uint
division_limit
,
size_t
use_mem
,
uint
division_limit
,
uint
age_threshold
,
uint
changed_blocks_hash_size
);
uint
age_threshold
,
uint
changed_blocks_hash_size
);
...
@@ -318,7 +319,7 @@ extern int flush_pagecache_blocks_with_filter(PAGECACHE *keycache,
...
@@ -318,7 +319,7 @@ extern int flush_pagecache_blocks_with_filter(PAGECACHE *keycache,
PAGECACHE_FILE
*
file
,
PAGECACHE_FILE
*
file
,
enum
flush_type
type
,
enum
flush_type
type
,
PAGECACHE_FLUSH_FILTER
filter
,
PAGECACHE_FLUSH_FILTER
filter
,
void
*
filter_arg
);
void
*
filter_arg
)
__attribute__
((
visibility
(
"default"
)))
;
extern
my_bool
pagecache_delete
(
PAGECACHE
*
pagecache
,
extern
my_bool
pagecache_delete
(
PAGECACHE
*
pagecache
,
PAGECACHE_FILE
*
file
,
PAGECACHE_FILE
*
file
,
pgcache_page_no_t
pageno
,
pgcache_page_no_t
pageno
,
...
@@ -334,7 +335,7 @@ extern my_bool pagecache_delete_pages(PAGECACHE *pagecache,
...
@@ -334,7 +335,7 @@ extern my_bool pagecache_delete_pages(PAGECACHE *pagecache,
uint
page_count
,
uint
page_count
,
enum
pagecache_page_lock
lock
,
enum
pagecache_page_lock
lock
,
my_bool
flush
);
my_bool
flush
);
extern
void
end_pagecache
(
PAGECACHE
*
keycache
,
my_bool
cleanup
);
extern
void
end_pagecache
(
PAGECACHE
*
keycache
,
my_bool
cleanup
)
__attribute__
((
visibility
(
"default"
)))
;
extern
my_bool
pagecache_collect_changed_blocks_with_lsn
(
PAGECACHE
*
pagecache
,
extern
my_bool
pagecache_collect_changed_blocks_with_lsn
(
PAGECACHE
*
pagecache
,
LEX_STRING
*
str
,
LEX_STRING
*
str
,
LSN
*
min_lsn
);
LSN
*
min_lsn
);
...
...
storage/maria/ma_static.c
View file @
35034d81
/* Copyright (C) 2006 MySQL AB & MySQL Finland AB & TCX DataKonsult AB
/* Copyright (C) 2006 MySQL AB & MySQL Finland AB & TCX DataKonsult AB
Copyright (c) 2010, 2020, MariaDB Corporation Ab
This program is free software; you can redistribute it and/or modify
This program is free software; you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
it under the terms of the GNU General Public License as published by
...
@@ -32,7 +33,7 @@ uchar maria_pack_file_magic[]=
...
@@ -32,7 +33,7 @@ uchar maria_pack_file_magic[]=
/* Unique number for this maria instance */
/* Unique number for this maria instance */
uchar
maria_uuid
[
MY_UUID_SIZE
];
uchar
maria_uuid
[
MY_UUID_SIZE
];
uint
maria_quick_table_bits
=
9
;
uint
maria_quick_table_bits
=
9
;
ulong
maria_block_size
=
MARIA_KEY_BLOCK_LENGTH
;
ulong
__attribute__
((
visibility
(
"default"
)))
maria_block_size
=
MARIA_KEY_BLOCK_LENGTH
;
my_bool
maria_flush
=
0
,
maria_single_user
=
0
;
my_bool
maria_flush
=
0
,
maria_single_user
=
0
;
my_bool
maria_delay_key_write
=
0
,
maria_page_checksums
=
1
;
my_bool
maria_delay_key_write
=
0
,
maria_page_checksums
=
1
;
my_bool
maria_inited
=
FALSE
;
my_bool
maria_inited
=
FALSE
;
...
@@ -146,3 +147,8 @@ PSI_file_key key_file_translog, key_file_kfile, key_file_dfile,
...
@@ -146,3 +147,8 @@ PSI_file_key key_file_translog, key_file_kfile, key_file_dfile,
/* Note that PSI_stage_info globals must always be declared. */
/* Note that PSI_stage_info globals must always be declared. */
PSI_stage_info
stage_waiting_for_a_resource
=
{
0
,
"Waiting for a resource"
,
0
};
PSI_stage_info
stage_waiting_for_a_resource
=
{
0
,
"Waiting for a resource"
,
0
};
#ifdef WITH_S3_STORAGE_ENGINE
#include "s3_func.h"
struct
s3_func
__attribute__
((
visibility
(
"default"
)))
s3f
;
#endif
storage/maria/maria_def.h
View file @
35034d81
/* Copyright (C) 2006 MySQL AB & MySQL Finland AB & TCX DataKonsult AB
/* Copyright (C) 2006 MySQL AB & MySQL Finland AB & TCX DataKonsult AB
Copyright (c) 2009, 2020, MariaDB Corporation Ab
This program is free software; you can redistribute it and/or modify
This program is free software; you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
it under the terms of the GNU General Public License as published by
...
@@ -606,6 +607,7 @@ typedef struct st_ma_base_info
...
@@ -606,6 +607,7 @@ typedef struct st_ma_base_info
my_bool
born_transactional
;
my_bool
born_transactional
;
}
MARIA_BASE_INFO
;
}
MARIA_BASE_INFO
;
uchar
*
_ma_base_info_read
(
uchar
*
ptr
,
MARIA_BASE_INFO
*
base
);
/* Structs used intern in database */
/* Structs used intern in database */
...
@@ -1616,7 +1618,7 @@ extern size_t _ma_nommap_pwrite(MARIA_HA *info, const uchar *Buffer,
...
@@ -1616,7 +1618,7 @@ extern size_t _ma_nommap_pwrite(MARIA_HA *info, const uchar *Buffer,
#define MA_STATE_INFO_WRITE_FULL_INFO 2
#define MA_STATE_INFO_WRITE_FULL_INFO 2
/* intern_lock taking is needed */
/* intern_lock taking is needed */
#define MA_STATE_INFO_WRITE_LOCK 4
#define MA_STATE_INFO_WRITE_LOCK 4
uint
_ma_state_info_write
(
MARIA_SHARE
*
share
,
uint
pWrite
);
uint
_ma_state_info_write
(
MARIA_SHARE
*
share
,
uint
pWrite
)
__attribute__
((
visibility
(
"default"
)))
;
uint
_ma_state_info_write_sub
(
File
file
,
MARIA_STATE_INFO
*
state
,
uint
pWrite
);
uint
_ma_state_info_write_sub
(
File
file
,
MARIA_STATE_INFO
*
state
,
uint
pWrite
);
uint
_ma_state_info_read_dsk
(
File
file
,
MARIA_STATE_INFO
*
state
);
uint
_ma_state_info_read_dsk
(
File
file
,
MARIA_STATE_INFO
*
state
);
uint
_ma_base_info_write
(
File
file
,
MARIA_BASE_INFO
*
base
);
uint
_ma_base_info_write
(
File
file
,
MARIA_BASE_INFO
*
base
);
...
@@ -1660,7 +1662,8 @@ void _ma_remap_file(MARIA_HA *info, my_off_t size);
...
@@ -1660,7 +1662,8 @@ void _ma_remap_file(MARIA_HA *info, my_off_t size);
MARIA_RECORD_POS
_ma_write_init_default
(
MARIA_HA
*
info
,
const
uchar
*
record
);
MARIA_RECORD_POS
_ma_write_init_default
(
MARIA_HA
*
info
,
const
uchar
*
record
);
my_bool
_ma_write_abort_default
(
MARIA_HA
*
info
);
my_bool
_ma_write_abort_default
(
MARIA_HA
*
info
);
int
maria_delete_table_files
(
const
char
*
name
,
my_bool
temporary
,
int
maria_delete_table_files
(
const
char
*
name
,
my_bool
temporary
,
myf
flags
);
myf
flags
)
__attribute__
((
visibility
(
"default"
)))
;
/*
/*
This cannot be in my_base.h as it clashes with HA_SPATIAL.
This cannot be in my_base.h as it clashes with HA_SPATIAL.
...
...
storage/maria/s3.cnf
0 → 100644
View file @
35034d81
[mariadb]
plugin-load-add=ha_s3
#
# Uncomment and configure the S3 engine
#
#s3-host-name=s3.amazonaws.com
#s3-protocol-version=Amazon
#s3-bucket=...
#s3-access-key=...
#s3-secret-key=...
#s3-region=eu-north-1
storage/maria/s3_func.h
View file @
35034d81
#ifndef S3_FUNC_INCLUDED
#ifndef S3_FUNC_INCLUDED
#define S3_FUNC_INCLUDED
#define S3_FUNC_INCLUDED
/* Copyright (C) 2019 MariaDB Corporation Ab
/* Copyright (C) 2019
, 2020
MariaDB Corporation Ab
This program is free software; you can redistribute it and/or modify
This program is free software; you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
it under the terms of the GNU General Public License as published by
...
@@ -20,11 +20,23 @@
...
@@ -20,11 +20,23 @@
*/
*/
#ifdef WITH_S3_STORAGE_ENGINE
#ifdef WITH_S3_STORAGE_ENGINE
C_MODE_START
#include <libmarias3/marias3.h>
#include <libmarias3/marias3.h>
C_MODE_START
#define DEFAULT_AWS_HOST_NAME "s3.amazonaws.com"
#define DEFAULT_AWS_HOST_NAME "s3.amazonaws.com"
extern
struct
s3_func
{
uint8_t
(
*
set_option
)(
ms3_st
*
,
ms3_set_option_t
,
void
*
);
void
(
*
free
)(
S3_BLOCK
*
);
void
(
*
deinit
)(
ms3_st
*
);
int32
(
*
unique_file_number
)(
void
);
my_bool
(
*
read_index_header
)(
ms3_st
*
,
S3_INFO
*
,
S3_BLOCK
*
);
int
(
*
check_frm_version
)(
ms3_st
*
,
S3_INFO
*
);
S3_INFO
*
(
*
info_copy
)(
S3_INFO
*
);
my_bool
(
*
set_database_and_table_from_path
)(
S3_INFO
*
,
const
char
*
);
ms3_st
*
(
*
open_connection
)(
S3_INFO
*
);
}
s3f
;
extern
TYPELIB
s3_protocol_typelib
;
extern
TYPELIB
s3_protocol_typelib
;
/* Store information about a s3 connection */
/* Store information about a s3 connection */
...
@@ -34,7 +46,6 @@ typedef struct s3_info
...
@@ -34,7 +46,6 @@ typedef struct s3_info
/* Connection strings */
/* Connection strings */
LEX_CSTRING
access_key
,
secret_key
,
region
,
bucket
,
host_name
;
LEX_CSTRING
access_key
,
secret_key
,
region
,
bucket
,
host_name
;
/* Will be set by caller or by ma_open() */
/* Will be set by caller or by ma_open() */
LEX_CSTRING
database
,
table
;
LEX_CSTRING
database
,
table
;
...
...
Write
Preview
Markdown
is supported
0%
Try again
or
attach a new file
Attach a file
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment