Skip to content

S
slapos
Commit 17bf5b6a authored by Thomas Gambier's avatar Thomas Gambier 🚴🏼
Browse files
Options

Replication Manager Upgrade 

See merge request nexedi/slapos!1092
parents ad04d19a bee8ef72
Show whitespace changes
Inline Side-by-side
Showing with 659 additions and 164 deletions
component/replication-manager/buildout.cfg
View file @ 17bf5b6a
...@@ -28,5 +28,6 @@ command = set -e ...@@ -28,5 +28,6 @@ command = set -e
<= go-git-package <= go-git-package
go.importpath = github.com/signal18/replication-manager go.importpath = github.com/signal18/replication-manager
repository = https://github.com/signal18/replication-manager repository = https://github.com/signal18/replication-manager
branch = 2.1 #branch = 2.1
revision = 9167a82c81af8f7be41cf51bc9be8a37dc3d8c03 branch = develop
revision = 838ffeb205ea4477f1c1bda607940fa07d8bcf51
\ No newline at end of file
slapos/recipe/generic_mysql/mysql.py
View file @ 17bf5b6a
from __future__ import print_function
import os import os
import subprocess import subprocess
import time import time
...@@ -16,35 +18,39 @@ def updateMysql(mysql_upgrade_binary, mysql_binary, mysql_script_file): ...@@ -16,35 +18,39 @@ def updateMysql(mysql_upgrade_binary, mysql_binary, mysql_script_file):
while True: while True:
while True: while True:
mysql_upgrade = subprocess.Popen(mysql_upgrade_binary, mysql_upgrade = subprocess.Popen(mysql_upgrade_binary,
stdout=subprocess.PIPE, stderr=subprocess.STDOUT) stdout=subprocess.PIPE, stderr=subprocess.STDOUT,
universal_newlines=True)
result = mysql_upgrade.communicate()[0] result = mysql_upgrade.communicate()[0]
if mysql_upgrade.returncode: if mysql_upgrade.returncode:
print "Command %r failed with result:\n%s" % (mysql_upgrade_binary, result) print("Command %r failed with result:\n%s" % (mysql_upgrade_binary, result))
break break
print "MySQL database upgraded with result:\n%s" % result print("MySQL database upgraded with result:\n%s" % result)
mysql = subprocess.Popen(mysql_list, stdin=subprocess.PIPE, mysql = subprocess.Popen(mysql_list, stdin=subprocess.PIPE,
stdout=subprocess.PIPE, stderr=subprocess.STDOUT) stdout=subprocess.PIPE, stderr=subprocess.STDOUT,
universal_newlines=True)
result = mysql.communicate(mysql_script)[0] result = mysql.communicate(mysql_script)[0]
if mysql.returncode: if mysql.returncode:
print 'Command %r failed with:\n%s' % (mysql_list, result) print('Command %r failed with:\n%s' % (mysql_list, result))
break break
# import timezone database # import timezone database
mysql_tzinfo_to_sql = subprocess.Popen(mysql_tzinfo_to_sql_list, stdin=subprocess.PIPE, mysql_tzinfo_to_sql = subprocess.Popen(mysql_tzinfo_to_sql_list, stdin=subprocess.PIPE,
stdout=subprocess.PIPE, stderr=subprocess.PIPE) stdout=subprocess.PIPE, stderr=subprocess.PIPE,
universal_newlines=True)
timezone_sql = mysql_tzinfo_to_sql.communicate()[0] timezone_sql = mysql_tzinfo_to_sql.communicate()[0]
if mysql_tzinfo_to_sql.returncode != 0: if mysql_tzinfo_to_sql.returncode != 0:
print 'Command %r failed with:\n%s' % (mysql_tzinfo_to_sql_list, result) print('Command %r failed with:\n%s' % (mysql_tzinfo_to_sql_list, result))
break break
mysql = subprocess.Popen(mysql_list + ('mysql',), stdin=subprocess.PIPE, mysql = subprocess.Popen(mysql_list + ('mysql',), stdin=subprocess.PIPE,
stdout=subprocess.PIPE, stderr=subprocess.STDOUT) stdout=subprocess.PIPE, stderr=subprocess.STDOUT,
universal_newlines=True)
result = mysql.communicate(timezone_sql)[0] result = mysql.communicate(timezone_sql)[0]
if mysql.returncode: if mysql.returncode:
print 'Command %r failed with:\n%s' % (mysql_list, result) print('Command %r failed with:\n%s' % (mysql_list, result))
break break
print 'SlapOS initialisation script succesfully applied on database.' print('SlapOS initialisation script succesfully applied on database.')
return return
sleep = min(sleep+1, 30) sleep = min(sleep+1, 30)
print 'Sleeping for %ss and retrying' % sleep print('Sleeping for %ss and retrying' % sleep)
sys.stdout.flush() sys.stdout.flush()
sys.stderr.flush() sys.stderr.flush()
time.sleep(sleep) time.sleep(sleep)
software/repman/buildout.hash.cfg
View file @ 17bf5b6a
...@@ -14,19 +14,19 @@ ...@@ -14,19 +14,19 @@
# not need these here). # not need these here).
[instance.cfg] [instance.cfg]
filename = instance.cfg.in filename = instance.cfg.in
md5sum = 8a08be95a04f1a47098c4fdef80bdfed md5sum = af2fc4a7a0f782fed2cb1112ef3cb397
[instance-repman.cfg] [instance-repman.cfg]
_update_hash_filename_ = instance-repman.cfg.jinja2.in _update_hash_filename_ = instance-repman.cfg.jinja2.in
md5sum = 697a1b546c883da45c14dbcd2d73b2b9 md5sum = c6e41783b8e3e844b6319953a8d665ec
[config-toml.in] [config-toml.in]
_update_hash_filename_ = templates/config.toml.in _update_hash_filename_ = templates/config.toml.in
md5sum = 2ee2c4bc5f985c11c2167b819d261256 md5sum = dc9fdcf6c3c775da6b0b72f83c4a4105
[config-cluster-toml.in] [config-cluster-toml.in]
_update_hash_filename_ = templates/cluster-config.toml.in _update_hash_filename_ = templates/cluster-config.toml.in
md5sum = c78c6be9537d3dffbb526b0e819bacb5 md5sum = ff44b81dc1177e32019ddbf4f1c6b616
[nginx.conf.in] [nginx.conf.in]
_update_hash_filename_ = templates/nginx.conf.in _update_hash_filename_ = templates/nginx.conf.in
...@@ -34,7 +34,7 @@ md5sum = 0eeb24c6aa0760f0d33c4cc2828ddf30 ...@@ -34,7 +34,7 @@ md5sum = 0eeb24c6aa0760f0d33c4cc2828ddf30
[template-mariadb.cfg] [template-mariadb.cfg]
_update_hash_filename_ = instance-mariadb.cfg.jinja2.in _update_hash_filename_ = instance-mariadb.cfg.jinja2.in
md5sum = a5c204cac552754520aee0570d379723 md5sum = 938f1e8087395757f5ae861d20500658
[template-my-cnf] [template-my-cnf]
_update_hash_filename_ = templates/my.cnf.in _update_hash_filename_ = templates/my.cnf.in
...@@ -42,15 +42,19 @@ md5sum = f3661b788099bb31d71ba6e7d36836d9 ...@@ -42,15 +42,19 @@ md5sum = f3661b788099bb31d71ba6e7d36836d9
[template-mariadb-initial-setup] [template-mariadb-initial-setup]
_update_hash_filename_ = templates/mariadb_initial_setup.sql.in _update_hash_filename_ = templates/mariadb_initial_setup.sql.in
md5sum = df44c2d6fb1971df582345daedad280c md5sum = 47b76144e1b116580c8acf08274af976
[template-publish-slave-information]
_update_hash_filename_ = publish-database-slave-parameters.cfg.in
md5sum = 9616477ff993c55c2b43bf8797db6994
[mariadb-init-root-sql] [mariadb-init-root-sql]
_update_hash_filename_ = templates/mariadb_init_root.sql.in _update_hash_filename_ = templates/mariadb_init_root.sql.in
md5sum = d8a0a7d72b02f911dafda655941b805d md5sum = 6e5c02542568f250aacd84694249efc0
[init-root-wrapper-in] [init-root-wrapper-in]
_update_hash_filename_ = templates/init_root_wrapper.in _update_hash_filename_ = templates/init_root_wrapper.in
md5sum = 7661e3b2f66982b9b17bd2ab73e2e8ef md5sum = c203f40a58386310a433b58fd345a341
[repman-manager-sh.in] [repman-manager-sh.in]
_update_hash_filename_ = templates/repman-manager.sh.in _update_hash_filename_ = templates/repman-manager.sh.in
...@@ -58,12 +62,16 @@ md5sum = 852dfab6d798aa1382eec4de2fd624f9 ...@@ -58,12 +62,16 @@ md5sum = 852dfab6d798aa1382eec4de2fd624f9
[dbjobs-in] [dbjobs-in]
_update_hash_filename_ = templates/dbjobs.in _update_hash_filename_ = templates/dbjobs.in
md5sum = d2ebd2ec55bf8489789a52c808729925 md5sum = d5c19d866c5a76d3bc2f434a91b1553a
[mysqld-need-start.sh.in] [mysqld-need-start.sh.in]
_update_hash_filename_ = templates/mysqld-need-start.sh.in _update_hash_filename_ = templates/mysqld-need-start.sh.in
md5sum = e9bcee5dc1318fe3acda2663472214f5 md5sum = 8f14054b3761b167cf8a86f4f084a574
[proxy-need-start-stop.sh.in] [proxy-need-start-stop.sh.in]
_update_hash_filename_ = templates/proxy-need-start-stop.sh.in _update_hash_filename_ = templates/proxy-need-start-stop.sh.in
md5sum = 455aaf369bf5141758dc57f2c0e67b08 md5sum = 455aaf369bf5141758dc57f2c0e67b08
[slave-db-manage.in]
_update_hash_filename_ = templates/slave-db-manage.in
md5sum = b45313ae5fb06972cc7fc945e34e434a
software/repman/instance-mariadb.cfg.jinja2.in
View file @ 17bf5b6a
...@@ -17,17 +17,11 @@ ...@@ -17,17 +17,11 @@
[{{ section('publish') }}] [{{ section('publish') }}]
recipe = slapos.cookbook:publish.serialised recipe = slapos.cookbook:publish.serialised
-extends = publish-early
database-host = {{ host }}:{{ port }} database-host = {{ host }}:{{ port }}
monitor-base-url = ${monitor-publish-parameters:monitor-base-url} monitor-base-url = ${monitor-publish-parameters:monitor-base-url}
partition-path = ${buildout:directory} partition-path = ${buildout:directory}
receiver-port = ${dbjob-parameter:socat-port} receiver-port = ${dbjob-parameter:socat-port}
csr-id = ${caucase-csr-id:csr-id}
[publish-early]
recipe = slapos.cookbook:publish-early
-init =
database-list = {{ dumps(database_list) }}
[jinja2-template-base] [jinja2-template-base]
recipe = slapos.recipe.template:jinja2 recipe = slapos.recipe.template:jinja2
...@@ -37,6 +31,22 @@ mode = 644 ...@@ -37,6 +31,22 @@ mode = 644
< = jinja2-template-base < = jinja2-template-base
mode = 755 mode = 755
[jinja2-template-script-base]
< = jinja2-template-executable
context =
key tmp_dir directory:tmp
key partition_dir buildout:directory
raw username {{ slapparameter_dict['repman-user'] }}
raw password {{ slapparameter_dict['repman-passwd'] }}
raw cluster_name {{ slapparameter_dict['cluster'] }}
raw secure_url {{ slapparameter_dict['repman-secure-url'] }}
raw mysql_conf ${directory:etc}/mysql/my.cnf
raw dash_bin {{ dash }}
raw jq_bin {{ jq_bin }}
raw curl_bin {{ curl_bin }}
raw mysql_bin {{ parameter_dict['mariadb-location'] }}/bin/mysql
${:extra-context}
[simplefile] [simplefile]
< = jinja2-template-base < = jinja2-template-base
template = inline:{{ '{{ content }}' }} template = inline:{{ '{{ content }}' }}
...@@ -53,10 +63,54 @@ context = key content {{content_section_name}}:content ...@@ -53,10 +63,54 @@ context = key content {{content_section_name}}:content
mode = {{ mode }} mode = {{ mode }}
{%- endmacro %} {%- endmacro %}
{% import "caucase" as caucase with context %}
{{ caucase.updater(
prefix='caucase-updater',
buildout_bin_directory=buildout_bin_directory,
updater_path='${directory:services}/caucase-updater',
url=slapparameter_dict['caucase-url'],
data_dir='${mysql-directory:ssl}',
crt_path='${mysql-directory:ssl}/server-cert.pem',
ca_path='${mysql-directory:ssl}/ca-cert.pem',
crl_path='${mysql-directory:ssl}/crl.pem',
key_path='${mysql-directory:ssl}/server-key.pem',
on_renew=None,
max_sleep=None,
template_csr_pem=None,
openssl=openssl_bin,
)}}
{% do part_list.append('caucase-updater') -%}
{% do part_list.append('caucase-updater-promise') -%}
[get-csr-id]
recipe = plone.recipe.command
output = ${directory:tmp}/csr_id
command =
if [ -s "${mysql-directory:ssl}/server-cert.pem" ]; then
RESULT="None";
else
if [ -f "${caucase-updater-csr:csr}" ]; then
RESULT=$({{ caucase_bin_client }} --ca-url {{ slapparameter_dict['caucase-url'] }} --send-csr ${caucase-updater-csr:csr} | cut -d ' ' -f1)
if [ ! $? -eq 0 ]; then
RESULT="None";
fi
fi
fi
cat <<EOF > ${:output}
[caucase]
csr-id = $(echo $RESULT)
EOF
update-command = ${:command}
[caucase-csr-id]
recipe = slapos.cookbook:zero-knowledge.read
file-path = ${get-csr-id:output}
csr-id =
[my-cnf-parameters] [my-cnf-parameters]
socket = ${directory:run}/mariadb.sock socket = ${directory:run}/mariadb.sock
ip = {{ ip }} ip = {{ ip }}
data-directory = ${directory:srv}/mariadb data-directory = ${mysql-directory:data}
pid-file = ${directory:run}/mariadb.pid pid-file = ${directory:run}/mariadb.pid
plugin-directory = {{ dumps(parameter_dict['mroonga-mariadb-plugin-dir']) }} plugin-directory = {{ dumps(parameter_dict['mroonga-mariadb-plugin-dir']) }}
groonga-plugins-path = {{ parameter_dict['groonga-plugins-path'] }} groonga-plugins-path = {{ parameter_dict['groonga-plugins-path'] }}
...@@ -66,9 +120,6 @@ innodb-log-file-size = {{ dumps(slapparameter_dict.get('innodb-log-file-size', 0 ...@@ -66,9 +120,6 @@ innodb-log-file-size = {{ dumps(slapparameter_dict.get('innodb-log-file-size', 0
innodb-file-per-table = {{ dumps(slapparameter_dict.get('innodb-file-per-table', 0)) }} innodb-file-per-table = {{ dumps(slapparameter_dict.get('innodb-file-per-table', 0)) }}
innodb-log-buffer-size = {{ dumps(slapparameter_dict.get('innodb-log-buffer-size', 0)) }} innodb-log-buffer-size = {{ dumps(slapparameter_dict.get('innodb-log-buffer-size', 0)) }}
relaxed-writes = {{ dumps(slapparameter_dict.get('relaxed-writes', False)) }} relaxed-writes = {{ dumps(slapparameter_dict.get('relaxed-writes', False)) }}
ssl-crt = ${directory:mariadb-ssl}/crt.pem
ssl-key = ${directory:mariadb-ssl}/key.pem
ssl-ca-crt = ${certificate-authority:ca-dir}/cacert.pem
[my-cnf] [my-cnf]
< = jinja2-template-base < = jinja2-template-base
...@@ -89,9 +140,11 @@ database-list = {{ dumps(database_list) }} ...@@ -89,9 +140,11 @@ database-list = {{ dumps(database_list) }}
mroonga-mariadb-install-sql = {{ dumps(parameter_dict['mroonga-mariadb-install-sql']) }} mroonga-mariadb-install-sql = {{ dumps(parameter_dict['mroonga-mariadb-install-sql']) }}
root-user = repman root-user = repman
heartbeat-user = {{ slapparameter_dict['heartbeat-user'] }} heartbeat-user = {{ slapparameter_dict['heartbeat-user'] }}
require-ssl = {{ dumps(slapparameter_dict['require-ssl']) }}
[init-script] [init-script]
< = jinja2-template-executable < = jinja2-template-executable
extensions = jinja2.ext.do
# XXX: is there a better location ? # XXX: is there a better location ?
rendered = ${directory:etc}/mariadb_initial_setup.sql rendered = ${directory:etc}/mariadb_initial_setup.sql
template = {{ parameter_dict['template-mariadb-initial-setup'] }} template = {{ parameter_dict['template-mariadb-initial-setup'] }}
...@@ -106,20 +159,26 @@ init-script = ${init-script:rendered} ...@@ -106,20 +159,26 @@ init-script = ${init-script:rendered}
mysql_tzinfo_to_sql = ${binary-wrap-mysql_tzinfo_to_sql:wrapper-path} mysql_tzinfo_to_sql = ${binary-wrap-mysql_tzinfo_to_sql:wrapper-path}
[{{ section('update-mysql-script') }}] [{{ section('update-mysql-script') }}]
< = jinja2-template-executable < = jinja2-template-script-base
rendered = ${directory:scripts}/mariadb_update rendered = ${directory:scripts}/mariadb_update
init-password = ${directory:etc}/.init-passwd.done init-password = ${directory:etc}/.init-passwd.done
upgrade-done = ${directory:lib}/mariadb-update-done upgrade-done = ${directory:lib}/mariadb-update-done
context = extra-context =
key init_password_done :init-password key init_password_done :init-password
key upgrade_done :upgrade-done key upgrade_done :upgrade-done
key init_root_sql init-root-sql:rendered key init_root_sql init-root-sql:rendered
key mysql_update update-mysql:output key mysql_update update-mysql:output
raw mysql_conf ${directory:etc}/mysql/my.cnf key init_database_sql init-script:rendered
raw dash_bin {{ dash }}
raw mysql_bin {{ parameter_dict['mariadb-location'] }}/bin/mysql
template = {{ parameter_dict['template-init-root-wrapper'] }} template = {{ parameter_dict['template-init-root-wrapper'] }}
[{{ section('mysql-slave-db-cleanup') }}]
< = jinja2-template-script-base
rendered = ${directory:scripts}/manage-slave-db
db-name = {{ slapparameter_dict['database-name'] }}
extra-context =
key database_name :db-name
template = {{ parameter_dict['template-manage-db'] }}
[mysqld] [mysqld]
< = jinja2-template-executable < = jinja2-template-executable
rendered = ${directory:bin}/mysqld rendered = ${directory:bin}/mysqld
...@@ -136,19 +195,19 @@ environ = ...@@ -136,19 +195,19 @@ environ =
{{ variable }} {{ variable }}
{%- endfor %} {%- endfor %}
[ca-mysqld] [mysqld-launcher]
<= certificate-authority recipe = slapos.cookbook:wrapper
recipe = slapos.cookbook:certificate_authority.request command-line = ${mysqld:rendered}
key-file = ${my-cnf-parameters:ssl-key} wrapper-path = ${directory:controller}/mariadb
cert-file = ${my-cnf-parameters:ssl-crt} wait-for-files =
executable = ${mysqld:rendered} ${mysql-directory:mysql}/my.cnf
wrapper = ${directory:controller}/mariadb ${mysql-directory:ssl}/server-cert.pem
{% import "supervisord_lib" as supervisord_lib with context %} {% import "supervisord_lib" as supervisord_lib with context %}
{{ supervisord_lib.supervisord("mariadb-ctl", buildout_bin_directory, supervisord_conf, use_service_hash=False) }} {{ supervisord_lib.supervisord("mariadb-ctl", buildout_bin_directory, supervisord_conf, use_service_hash=False) }}
{% do part_list.append("supervisord-mariadb-ctl") -%} {% do part_list.append("supervisord-mariadb-ctl") -%}
{% set maradb_program_dict = {"name": "mariadb", "command": "${ca-mysqld:wrapper}", {% set maradb_program_dict = {"name": "mariadb", "command": "${mysqld-launcher:wrapper-path}",
"stopwaitsecs": 86400, "environment": [], "stopwaitsecs": 86400, "startretries": 10, "autorestart": True, "environment": [],
"stdout_logfile": "${directory:log}/mariadb_stdout.log", "stdout_logfile": "${directory:log}/mariadb_stdout.log",
"stderr_logfile": "${directory:log}/mariadb_stdout.log" } %} "stderr_logfile": "${directory:log}/mariadb_stdout.log" } %}
...@@ -178,13 +237,18 @@ target-directory = ${directory:bin} ...@@ -178,13 +237,18 @@ target-directory = ${directory:bin}
link-binary = {{ dumps(parameter_dict['link-binary']) }} link-binary = {{ dumps(parameter_dict['link-binary']) }}
[binary-wrap-base] [binary-wrap-base]
recipe = slapos.cookbook:wrapper
# Note: --defaults-file must be the first argument, otherwise wrapped binary # Note: --defaults-file must be the first argument, otherwise wrapped binary
# will reject it. # will reject it.
command-line = recipe = slapos.recipe.template:jinja2
"{{ parameter_dict['mariadb-location'] }}/bin/${:command}" rendered = ${directory:bin}/${:command}
--defaults-file="${directory:etc}/mysql/my.cnf" --protocol=socket ${:extra-args} mode = 755
wrapper-path = ${directory:bin}/${:command} template =
inline:#!/bin/sh
exec {{ parameter_dict['mariadb-location'] }}/bin/${:command} \
--defaults-file="${directory:etc}/mysql/my.cnf" \
--protocol=socket \
${:extra-args} "$@"
wrapper-path = ${:rendered}
extra-args = extra-args =
[binary-wrap-mysql] [binary-wrap-mysql]
...@@ -208,6 +272,11 @@ command = mysqladmin ...@@ -208,6 +272,11 @@ command = mysqladmin
<= binary-wrap-base <= binary-wrap-base
command-line = "{{ parameter_dict['mariadb-location'] }}/bin/${:command}" --skip-write-binlog command-line = "{{ parameter_dict['mariadb-location'] }}/bin/${:command}" --skip-write-binlog
command = mysql_tzinfo_to_sql command = mysql_tzinfo_to_sql
template =
inline:#!/bin/sh
exec {{ parameter_dict['mariadb-location'] }}/bin/${:command} \
--defaults-file="${directory:etc}/mysql/my.cnf" \
--skip-write-binlog "$@"
[binary-wrap-pt-digest] [binary-wrap-pt-digest]
<= binary-wrap-base <= binary-wrap-base
...@@ -226,26 +295,29 @@ plugin = ${:etc}/plugin ...@@ -226,26 +295,29 @@ plugin = ${:etc}/plugin
srv = ${buildout:directory}/srv srv = ${buildout:directory}/srv
tmp = ${buildout:directory}/tmp tmp = ${buildout:directory}/tmp
backup = ${:srv}/backup backup = ${:srv}/backup
mariadb-backup-full = ${:backup}/mariadb-full
mariadb-backup-incremental = ${:backup}/mariadb-incremental
mariadb-ssl = ${:etc}/mariadb-ssl
var = ${buildout:directory}/var var = ${buildout:directory}/var
lib = ${:var}/lib lib = ${:var}/lib
mysql = ${:lib}/mysql
log = ${:var}/log log = ${:var}/log
run = ${:var}/run run = ${:var}/run
config-tmp = ${:tmp}/config config-tmp = ${:tmp}/config
custom = ${directory:etc}/mysql/custom
[mysql-directory]
recipe = slapos.cookbook:mkdirectory
mysql = ${directory:etc}/mysql
ssl = ${:mysql}/ssl
data = ${directory:lib}/mysql
custom = ${:mysql}/custom.d
[dbjob-parameter] [dbjob-parameter]
bash-bin = {{ bash_bin }} bash-bin = {{ bash_bin }}
db-user = ${init-script-parameters:root-user} db-user = ${init-script-parameters:root-user}
db-password = ${init-script-parameters:password} db-password = ${init-script-parameters:password}
mysql-dir = ${directory:mysql} mysql-dir = ${mysql-directory:data}
dbjob-cnf = ${directory:etc}/mysql/my.cnf dbjob-cnf = ${mysql-directory:mysql}/dbjob.cnf
log-dir = ${directory:mysql}/.system/logs log-dir = ${mysql-directory:data}/.system/logs
tmp-dir = ${directory:tmp} tmp-dir = ${directory:tmp}
mysqld-socket = ${my-cnf-parameters:socket} mysqld-socket = ${my-cnf-parameters:socket}
mysql-client = ${binary-wrap-mysql:wrapper-path}
socat-port = {{ int(port) + 9 }} socat-port = {{ int(port) + 9 }}
restart-script = ${mysqld-restart-script:rendered} restart-script = ${mysqld-restart-script:rendered}
socat-location = {{ parameter_dict['socat-location'] }} socat-location = {{ parameter_dict['socat-location'] }}
...@@ -309,11 +381,11 @@ template = inline:#!{{ bash_bin }} ...@@ -309,11 +381,11 @@ template = inline:#!{{ bash_bin }}
cd ${directory:config-tmp} && cd ${directory:config-tmp} &&
{{ curl_bin }} -o config.tar.gz {{ slapparameter_dict['repman-url'] }}/api/clusters/${:cluster}/servers/{{ host }}/{{ port }}/config {{ curl_bin }} -o config.tar.gz {{ slapparameter_dict['repman-url'] }}/api/clusters/${:cluster}/servers/{{ host }}/{{ port }}/config
tar -xzf config.tar.gz tar -xzf config.tar.gz
cp -r data/.system ${directory:mysql} rm -r etc/mysql/ssl # we don't need generated ssl files
rm -rf ${directory:etc}/mysql cp -r data/.system ${mysql-directory:data}
cp -r etc/mysql ${directory:etc} cp -r etc/mysql/* ${mysql-directory:mysql}
ln -sf ${directory:mysql}/.system ${directory:var}/system ln -sf ${mysql-directory:data}/.system ${directory:var}/system
ln -sf ${my-cnf:rendered} ${directory:etc}/mysql/custom/01_mariadb.cnf ln -sf ${my-cnf:rendered} ${mysql-directory:custom}/01_mariadb.cnf
[{{ section('install-mysql-config') }}] [{{ section('install-mysql-config') }}]
recipe = plone.recipe.command recipe = plone.recipe.command
......
software/repman/instance-repman-input-schema.json
View file @ 17bf5b6a
...@@ -113,7 +113,8 @@ ...@@ -113,7 +113,8 @@
"mroonga", "mroonga",
"utctime", "utctime",
"readcommitted", "readcommitted",
"nohandshake" "nohandshake",
"ssl"
] ]
}, },
"http-session-lifetime": { "http-session-lifetime": {
...@@ -140,6 +141,12 @@ ...@@ -140,6 +141,12 @@
"type": "boolean", "type": "boolean",
"default": true "default": true
}, },
"require-ssl": {
"title": "Enable REQUIRE SSL for repman database users",
"description": "Enable secure connection between repman and mariadb databases. This option is an initial value, used only when creating database users.",
"type": "boolean",
"default": false
},
"repman-cluster-dict": { "repman-cluster-dict": {
"title": "Replication Manager clusters definition", "title": "Replication Manager clusters definition",
"description": "Replication Manager clusters definition", "description": "Replication Manager clusters definition",
...@@ -197,7 +204,7 @@ ...@@ -197,7 +204,7 @@
}, },
"proxy-tags": { "proxy-tags": {
"title": "Proxy tag list", "title": "Proxy tag list",
"description": "playbook configuration tags. Default: [\"pkg\", \"masterslave\", \"linux\", \"noreadwritesplit\"]", "description": "playbook configuration tags. Default: [\"pkg\", \"masterslave\", \"linux\", \"noreadwritesplit\", \"ssl\"]",
"type": "array", "type": "array",
"items": { "items": {
"type": "string" "type": "string"
...@@ -206,7 +213,8 @@ ...@@ -206,7 +213,8 @@
"pkg", "pkg",
"masterslave", "masterslave",
"linux", "linux",
"noreadwritesplit" "noreadwritesplit",
"ssl"
] ]
}, },
"logical-backup-cron": { "logical-backup-cron": {
......
software/repman/instance-repman-slave-input-schema.json 0 → 100644
View file @ 17bf5b6a
{
"$schema": "http://json-schema.org/draft-04/schema",
"properties": {
"db_user": {
"description": "Database User, default is 'user'. This parameter is set only if database is not created yet.",
"title": "Database User",
"type": "string",
"default": "user"
},
"db_password": {
"description": "Database password. If no password set, a password will be generated. This parameter is set only if database is not created yet.",
"title": "Initial database password",
"type": "string",
"default": ""
},
"db_charset": {
"description": "Database Character set information. This parameter is set only if database is not created yet.",
"title": "Database Character set information",
"type": "string",
"enum": [
"latin1",
"latin2",
"latin5",
"latin7",
"ascii",
"latin5",
"utf8",
"utf8mb4",
"utf16",
"utf16le",
"utf32",
"binary",
"utf8_general_ci",
"utf8_unicode_ci",
"utf8_unicode_ci",
"utf8mb4_bin",
"utf8mb4_general_ci",
"utf8mb4_unicode_ci",
"latin1_general_ci",
"latin1_bin",
"ascii_general_ci",
"ascii_bin"
],
"default": ""
}
},
"title": "Input Parameters",
"type": "object"
}
software/repman/instance-repman.cfg.jinja2.in
View file @ 17bf5b6a
...@@ -8,8 +8,23 @@ ...@@ -8,8 +8,23 @@
{% set ip = (ipv6_set | list)[0] -%} {% set ip = (ipv6_set | list)[0] -%}
{% set ipv4 = (ipv4_set | list)[0] -%} {% set ipv4 = (ipv4_set | list)[0] -%}
{% set cluster_list = [] -%} {% set cluster_list = [] -%}
{% set tag_list = ["gtidstrict", "bind", "pkg", "innodb", "noquerycache", "slow", "pfs", "linux", "readonly", "diskmonitor", "sqlerror", "compressbinlog", "bm4ci", "mroonga", "utctime", "readcommitted", "nohandshake"] -%} {% set tag_list = ["gtidstrict", "bind", "pkg", "innodb", "noquerycache", "slow",
"pfs", "linux", "readonly", "diskmonitor", "sqlerror", "compressbinlog", "bm4ci",
"mroonga", "utctime", "readcommitted", "nohandshake", "ssl"] -%}
{% set frontend_parameter_dict = slapparameter_dict.get('slave-frontend', {}) -%} {% set frontend_parameter_dict = slapparameter_dict.get('slave-frontend', {}) -%}
{% set database_slave_list = [] -%}
{% set db_name_list = [] -%}
{% set count = namespace(value=2) %}
{% set caucase_bind = '[' ~ ip ~ ']:8890' -%}
{% set caucase_url = 'http://' ~ caucase_bind -%}
{% set csrid_list = [] -%}
{% macro password(name) -%}
[{{ name }}-password]
recipe = slapos.cookbook:generate.password
bytes = 12
{% endmacro -%}
[directory] [directory]
recipe = slapos.cookbook:mkdirectory recipe = slapos.cookbook:mkdirectory
...@@ -25,11 +40,25 @@ log = ${:var}/log ...@@ -25,11 +40,25 @@ log = ${:var}/log
data = ${:var}/lib data = ${:var}/lib
nginx-prefix = ${:var}/nginx nginx-prefix = ${:var}/nginx
tmp = ${:home}/tmp tmp = ${:home}/tmp
backup-caucased = ${:srv}/backup-caucased
caucased = ${:srv}/caucased
ssl = ${:etc}/ssl
[proxysql-directory]
recipe = slapos.cookbook:mkdirectory
config = ${directory:etc}/proxysql
ssl = ${:config}/ssl
{% import "supervisord_lib" as supervisord_lib with context %} {% import "supervisord_lib" as supervisord_lib with context %}
{% set proxysql_controller = "proxysql-ctl" -%} {% set proxysql_controller = "proxysql-ctl" -%}
{{ supervisord_lib.supervisord(proxysql_controller, buildout_bin_directory, supervisord_conf, use_service_hash=False) }} {{ supervisord_lib.supervisord(proxysql_controller, buildout_bin_directory, supervisord_conf, use_service_hash=False) }}
[proxysql-controller-update]
recipe = slapos.cookbook:wrapper
command-line = ${proxysql-ctl-bin:wrapper-path} update
wrapper-path = ${directory:scripts}/proxysql-ctl-update
{% do part_list.append("supervisord-proxysql-ctl") -%} {% do part_list.append("supervisord-proxysql-ctl") -%}
{% do part_list.append("proxysql-controller-update") -%}
[request-common] [request-common]
recipe = slapos.cookbook:request.serialised recipe = slapos.cookbook:request.serialised
...@@ -39,6 +68,7 @@ key-file = ${slap-connection:key-file} ...@@ -39,6 +68,7 @@ key-file = ${slap-connection:key-file}
cert-file = ${slap-connection:cert-file} cert-file = ${slap-connection:cert-file}
computer-id = ${slap-connection:computer-id} computer-id = ${slap-connection:computer-id}
partition-id = ${slap-connection:partition-id} partition-id = ${slap-connection:partition-id}
config-caucase-url = {{ caucase_url }}
[download-proxy-config] [download-proxy-config]
recipe = slapos.recipe.template:jinja2 recipe = slapos.recipe.template:jinja2
...@@ -48,16 +78,35 @@ template = inline:#!{{ bash_bin }} ...@@ -48,16 +78,35 @@ template = inline:#!{{ bash_bin }}
PORT=$3 PORT=$3
CONFIG=$4 CONFIG=$4
if [ -z "$CONFIG" ]; then if [ -z "$CONFIG" ]; then
CONFIG="${repman:proxies}/proxysql-$NAME.cnf" CONFIG="${proxysql-directory:config}/proxysql.cnf"
fi fi
mkdir -p ${repman:config-tmp}/proxies mkdir -p ${repman:config-tmp}/proxies
cd ${repman:config-tmp}/proxies cd ${repman:config-tmp}/proxies
{{ curl_bin }} -o proxies-$NAME.tar.gz ${nginx-parameter:repman-url}/api/clusters/$NAME/servers/$HOST/$PORT/config {{ curl_bin }} -o proxies-$NAME.tar.gz ${nginx-parameter:repman-url}/api/clusters/$NAME/servers/$HOST/$PORT/config
tar -xzf proxies-$NAME.tar.gz tar -xzf proxies-$NAME.tar.gz
cp conf/proxysql.cnf $CONFIG cp etc/proxysql/proxysql.cnf $CONFIG
rendered = ${directory:bin}/update-proxysql-config rendered = ${directory:bin}/update-proxysql-config
mode = 755 mode = 755
{% for instance_dict in slave_instance_list -%}
{% set slave_dict = {
'name': 'db_%s' % instance_dict['slave_reference'].replace('-', '_').lower(),
'user': instance_dict.get('db_user', 'user'),
'password': instance_dict.get('db_password', '${' ~ instance_dict['slave_reference'] ~ '-password:passwd}'),
'slave_reference': instance_dict['slave_reference'],
'charset': instance_dict.get('db_charset', ''),
'require_ssl': True
} -%}
{% do database_slave_list.append(slave_dict) -%}
{% do db_name_list.append(slave_dict['name']) -%}
{{ password(instance_dict['slave_reference']) }}
{% endfor %}
[database-slave-information]
{% for slave_dict in database_slave_list -%}
{{ slave_dict['name'] }} = !py!{{ slave_dict }}
{% endfor %}
{% set db_list = db_name_list | join(' ') -%}
{% do mariadb_dict.__setitem__('computer-memory-percent-threshold', 80) -%} {% do mariadb_dict.__setitem__('computer-memory-percent-threshold', 80) -%}
{% set default_parameter_dict = {"cluster1": {"name": "cluster1", "db-prefered-master": "", {% set default_parameter_dict = {"cluster1": {"name": "cluster1", "db-prefered-master": "",
"database-amount": 2, "proxysql-user": "external", "database-amount": 2, "proxysql-user": "external",
...@@ -67,22 +116,23 @@ mode = 755 ...@@ -67,22 +116,23 @@ mode = 755
{% do mariadb_dict.__setitem__('innodb-file-per-table', parameter_dict.get('innodb-file-per-table', 1)) -%} {% do mariadb_dict.__setitem__('innodb-file-per-table', parameter_dict.get('innodb-file-per-table', 1)) -%}
{% do mariadb_dict.__setitem__('use-ipv6', parameter_dict.get('use-ipv6', True)) -%} {% do mariadb_dict.__setitem__('use-ipv6', parameter_dict.get('use-ipv6', True)) -%}
{% set database_list = parameter_dict.get('database-list', [{'name': 'repdb', 'user': 'user', 'password': 'insecure'}]) -%}
# Request mariadb instances # Request mariadb instances
{% set db_amount = parameter_dict.get('database-amount', 2) -%} {% set db_amount = parameter_dict.get('database-amount', 2) -%}
{% if db_amount < 2 -%} {% if db_amount < 2 -%}
{% set db_amount = 2 -%} {% set db_amount = 2 -%}
{% endif -%} {% endif -%}
{% set count.value = count.value + db_amount %}
{% for i in range(0, db_amount) -%} {% for i in range(0, db_amount) -%}
{% do mariadb_dict.__setitem__('tcp-port', 2099 + (i * 100)) -%} {% do mariadb_dict.__setitem__('tcp-port', 2099 + (i * 100)) -%}
{% set section = 'request-mariadb-' ~ i -%} {% set section = 'request-mariadb-' ~ i -%}
{% set dbname = 'Mariadb-' ~ i -%} {% set dbname = 'Mariadb-' ~ i %}
[{{ section }}] [{{ section }}]
<= request-common <= request-common
software-type = mariadb software-type = mariadb
name = {{ dbname }} name = {{ dbname ~ '-' ~ name}}
sla-computer_guid = {{ dumps(parameter_dict.get('-sla-' ~ i ~'-computer_guid', '')) }} sla-computer_guid = {{ dumps(parameter_dict.get('-sla-' ~ i ~'-computer_guid', '')) }}
{% for key, value in mariadb_dict.items() -%} {% for key, value in mariadb_dict.items() -%}
config-{{ key }} = {{ dumps(value) }} config-{{ key }} = {{ dumps(value) }}
...@@ -91,23 +141,27 @@ config-monitor-passwd = ${publish-early:monitor-password} ...@@ -91,23 +141,27 @@ config-monitor-passwd = ${publish-early:monitor-password}
config-root-password = ${publish-early:db-root-password} config-root-password = ${publish-early:db-root-password}
config-repman-user = ${repman-parameter:username} config-repman-user = ${repman-parameter:username}
config-heartbeat-user = ${repman-parameter:heartbeat-user} config-heartbeat-user = ${repman-parameter:heartbeat-user}
#config-repman-passwd = ${repman-parameter:password} config-repman-passwd = ${repman-parameter:password}
config-repman-url = ${nginx-parameter:backend-url} config-repman-url = ${nginx-parameter:backend-url}
config-repman-secure-url = ${nginx-parameter:backend-ssl-url} config-repman-secure-url = ${nginx-parameter:backend-ssl-url}
config-cluster = {{ name }} config-cluster = {{ name }}
config-name = {{ dbname }} config-name = {{ dbname }}
config-database-list = {{ dumps(database_list) }} config-database-list = !py!{{ database_slave_list }}
config-database-name = {{ dumps(db_list) }}
config-require-ssl = {{ dumps(slapparameter_dict.get('require-ssl', False)) }}
return = return =
database-host database-host
receiver-port receiver-port
monitor-base-url monitor-base-url
partition-path partition-path
csr-id
{% do part_list.append(section) -%} {% do part_list.append(section) -%}
{% do mariadb_server_list.append('${' ~ section ~ ':connection-database-host}') -%} {% do mariadb_server_list.append('${' ~ section ~ ':connection-database-host}') -%}
{% do receiver_port_list.append('${' ~ section ~ ':connection-receiver-port}') -%} {% do receiver_port_list.append('${' ~ section ~ ':connection-receiver-port}') -%}
{% do mariadb_path_list.append('${' ~ section ~ ':connection-partition-path}') -%} {% do mariadb_path_list.append('${' ~ section ~ ':connection-partition-path}') -%}
{% do monitor_base_url_dict.__setitem__('mariadb' ~ i, '${' ~ section ~ ':connection-monitor-base-url}') -%} {% do monitor_base_url_dict.__setitem__('mariadb' ~ i, '${' ~ section ~ ':connection-monitor-base-url}') -%}
{% do csrid_list.append('${' ~ section ~ ':connection-csr-id}') -%}
{% endfor -%} {% endfor -%}
...@@ -129,6 +183,7 @@ maximum = 7132 ...@@ -129,6 +183,7 @@ maximum = 7132
{% for key, value in parameter_dict.items() -%} {% for key, value in parameter_dict.items() -%}
{{ key }} = {{ value }} {{ key }} = {{ value }}
{% endfor -%} {% endfor -%}
name = {{ name }}
proxysql-user = {{ parameter_dict.get("proxysql-user", "external") }} proxysql-user = {{ parameter_dict.get("proxysql-user", "external") }}
proxy-port = {{ '${' ~ name ~ '-port:port}' }} proxy-port = {{ '${' ~ name ~ '-port:port}' }}
proxy-admin-port = {{ '${' ~ name ~ '-admin-port:port}' }} proxy-admin-port = {{ '${' ~ name ~ '-admin-port:port}' }}
...@@ -151,7 +206,7 @@ password = ${repman-parameter:password} ...@@ -151,7 +206,7 @@ password = ${repman-parameter:password}
proxysql-partition = ${buildout:directory} proxysql-partition = ${buildout:directory}
receiver-port-list = {{ receiver_port_list | join(',') }} receiver-port-list = {{ receiver_port_list | join(',') }}
enabled-tags = {{ slapparameter_dict.get("tag-list", tag_list) | join(',') }} enabled-tags = {{ slapparameter_dict.get("tag-list", tag_list) | join(',') }}
proxy-tags = {{ parameter_dict.get("proxy-tags", ["pkg", "masterslave", "linux", "noreadwritesplit"]) | join(',') }} proxy-tags = {{ parameter_dict.get("proxy-tags", ["pkg", "masterslave", "linux", "noreadwritesplit", "ssl"]) | join(',') }}
logical-backup-cron = {{ parameter_dict.get("logical-backup-cron", "0 22 * * *") }} logical-backup-cron = {{ parameter_dict.get("logical-backup-cron", "0 22 * * *") }}
physical-backup-cron = {{ parameter_dict.get("physical-backup-cron", "0 0 * * *") }} physical-backup-cron = {{ parameter_dict.get("physical-backup-cron", "0 0 * * *") }}
proxy-cpu-cores = {{ parameter_dict.get("proxy-cpu-cores", 2) }} proxy-cpu-cores = {{ parameter_dict.get("proxy-cpu-cores", 2) }}
...@@ -191,7 +246,7 @@ context = ...@@ -191,7 +246,7 @@ context =
recipe = plone.recipe.command recipe = plone.recipe.command
# if Repman is not started, cannot download config from server # if Repman is not started, cannot download config from server
stop-on-error = false stop-on-error = false
config = ${repman:proxies}/proxysql-{{ name }}.cnf config = ${proxysql-directory:config}/proxysql.cnf
data = ${repman:proxy-data}/{{ name }} data = ${repman:proxy-data}/{{ name }}
command = command =
mkdir -p ${:data} && mkdir -p ${:data} &&
...@@ -209,6 +264,7 @@ command-line = ...@@ -209,6 +264,7 @@ command-line =
wrapper-path = ${directory:controller}/proxysql-{{ name }} wrapper-path = ${directory:controller}/proxysql-{{ name }}
wait-for-files = wait-for-files =
${repman:bootstrap}/{{ name }}_bootstrapped ${repman:bootstrap}/{{ name }}_bootstrapped
${config-proxysql-{{ name }}:config}
depends = depends =
{{ '${proxysql-' ~ name ~ '-admin-promise:recipe}' }} {{ '${proxysql-' ~ name ~ '-admin-promise:recipe}' }}
{{ '${proxysql-' ~ name ~ '-promise:recipe}' }} {{ '${proxysql-' ~ name ~ '-promise:recipe}' }}
...@@ -237,7 +293,7 @@ config-port = {{ '${' ~ name ~ '-cluster-parameter:proxy-port}' }} ...@@ -237,7 +293,7 @@ config-port = {{ '${' ~ name ~ '-cluster-parameter:proxy-port}' }}
{% set service_name = "proxysql-" ~ name -%} {% set service_name = "proxysql-" ~ name -%}
{% set proxysql_dict = {"name": service_name, "command": "${" ~ service_name ~ "-wrapper:wrapper-path}", {% set proxysql_dict = {"name": service_name, "command": "${" ~ service_name ~ "-wrapper:wrapper-path}",
"stopwaitsecs": 60, "environment": [], "stopwaitsecs": 60, "startretries": 10, "autorestart": True, "environment": [],
"stdout_logfile": "${repman:proxies-log}/" ~ service_name ~ ".log", "stdout_logfile": "${repman:proxies-log}/" ~ service_name ~ ".log",
"stderr_logfile": "${repman:proxies-log}/" ~ service_name ~ ".log" } %} "stderr_logfile": "${repman:proxies-log}/" ~ service_name ~ ".log" } %}
...@@ -247,22 +303,126 @@ config-port = {{ '${' ~ name ~ '-cluster-parameter:proxy-port}' }} ...@@ -247,22 +303,126 @@ config-port = {{ '${' ~ name ~ '-cluster-parameter:proxy-port}' }}
{% do part_list.append('config-' ~ name) -%} {% do part_list.append('config-' ~ name) -%}
{% do cluster_list.append("{'name': '" ~ name ~ "', 'host': '" ~ ipv4 ~ "', 'port': '${" ~ name ~ "-cluster-parameter:proxy-admin-port}'}") -%} {% do cluster_list.append("{'name': '" ~ name ~ "', 'host': '" ~ ipv4 ~ "', 'port': '${" ~ name ~ "-cluster-parameter:proxy-admin-port}'}") -%}
{% set publish_database_list = [] -%}
{% set publish_database_v6_list = [] -%}
{% for database in database_list -%}
{% if database.get('user') -%}
{% do publish_database_list.append("mysql://" ~ database['user'] ~ ":" ~ database['password'] ~ "@" ~ ipv4 ~ ":${" ~ name ~ "-cluster-parameter:proxy-port}/" ~ database['name']) -%}
{% do publish_database_v6_list.append("mysql://" ~ database['user'] ~ ":" ~ database['password'] ~ "@[" ~ ip ~ "]:${" ~ name ~ "-cluster-parameter:proxy-port}/" ~ database['name']) -%}
{% else -%}
{% do publish_database_list.append("mysql://" ~ ipv4 ~ ":${" ~ name ~ "-cluster-parameter:proxy-port}/" ~ database['name']) -%}
{% do publish_database_v6_list.append("mysql://[" ~ ip ~ "]:${" ~ name ~ "-cluster-parameter:proxy-port}/" ~ database['name']) -%}
{% endif -%}
{% endfor -%}
{% do publish_dict.__setitem__(name ~ '-database-list', "!py!['" ~ publish_database_list | join("', '") ~ "']") -%} [{{ name}}-publish-slave-information]
{% do publish_dict.__setitem__(name ~ '-database-list-v6', "!py!['" ~ publish_database_v6_list | join("', '") ~ "']") -%} recipe = slapos.cookbook:switch-softwaretype
default = {{ 'dynamic-' ~name ~ '-publish-slave-information:rendered' }}
RootSoftwareInstance = ${:default}
[dynamic-{{ name}}-publish-slave-information]
recipe = slapos.recipe.template:jinja2
rendered = ${buildout:directory}/database-publish-slave-information.cfg
extensions = jinja2.ext.do
template = {{ template_publish_slave_information }}
context =
import json_module json
key proxy_port {{ name }}-cluster-parameter:proxy-port
raw eggs_directory {{ eggs_directory }}
raw develop_eggs_directory {{ develop_eggs_directory }}
raw ipv6 {{ ip }}
raw ipv4 {{ ipv4 }}
section slave_dict database-slave-information
{% do part_list.append(name ~ '-publish-slave-information') -%}
{% endfor -%} {% endfor -%}
# deploy caucase
{% import "caucase" as caucase with context %}
{{ caucase.caucased(
prefix='caucased',
buildout_bin_directory=buildout_bin_directory,
caucased_path='${directory:service}/caucased',
backup_dir='${directory:backup-caucased}',
data_dir='${directory:caucased}',
netloc=caucase_bind,
tmp='${directory:tmp}',
service_auto_approve_count=count.value,
user_auto_approve_count=1,
key_len=2048,
)}}
{% do part_list.append('caucased') -%}
{% do part_list.append('caucased-promise') -%}
{% do publish_dict.__setitem__('caucase-http-url', caucase_url) -%}
{{ caucase.updater(
prefix='caucase-updater',
buildout_bin_directory=buildout_bin_directory,
updater_path='${directory:services}/caucase-updater',
url=caucase_url,
data_dir='${directory:srv}/caucase-updater',
crt_path='${directory:ssl}/repman-cert.crt',
ca_path='${directory:srv}/caucase-updater/ca.crt',
crl_path='${directory:srv}/caucase-updater/crl.pem',
key_path='${directory:ssl}/repman-cert.key',
on_renew=None,
max_sleep=None,
template_csr_pem=None,
openssl=openssl_bin,
)}}
{% do part_list.append('caucase-updater') -%}
{% do part_list.append('caucase-updater-promise') -%}
#caucase user certificate
{{ caucase.updater(
prefix='caucase-user-updater',
buildout_bin_directory=buildout_bin_directory,
updater_path='${directory:services}/caucase-user-updater',
url=caucase_url,
data_dir='${directory:srv}/caucase-user-updater',
crt_path='${directory:ssl}/caucase.user.crt',
ca_path='${directory:srv}/caucase-user-updater/ca.crt',
crl_path='${directory:srv}/caucase-user-updater/crl.pem',
key_path='${directory:ssl}/caucase.user.key',
on_renew=None,
max_sleep=None,
template_csr_pem=None,
openssl=openssl_bin,
mode='user',
)}}
{% do part_list.append('caucase-user-updater') -%}
{% do part_list.append('caucase-user-updater-promise') -%}
#caucase proxysql certificate
{{ caucase.updater(
prefix='caucase-proxysql-updater',
buildout_bin_directory=buildout_bin_directory,
updater_path='${directory:services}/caucase-proxysql-updater',
url=caucase_url,
data_dir='${proxysql-directory:ssl}',
crt_path='${proxysql-directory:ssl}/client-cert.pem',
ca_path='${proxysql-directory:ssl}/ca-cert.pem',
crl_path='${proxysql-directory:ssl}/crl.pem',
key_path='${proxysql-directory:ssl}/client-key.pem',
on_renew=None,
max_sleep=None,
template_csr_pem=None,
openssl=openssl_bin,
)}}
{% do part_list.append('caucase-proxysql-updater') -%}
{% do part_list.append('caucase-proxysql-updater-promise') -%}
[caucase-sign-csr]
recipe = slapos.recipe.template:jinja2
mode = 755
rendered = ${directory:bin}/caucase-sign
template =
inline:#!/bin/sh
cp ${directory:ssl}/caucase.user.key ${directory:ssl}/caucase-full.key
cat ${directory:ssl}/caucase.user.crt >> ${directory:ssl}/caucase-full.key
for csr_id in {{ csrid_list | join(' ') }}; do
if [ "$csr_id" = "None" ] || [ -z "$csr_id"]; then
continue
fi
{{ buildout_bin_directory }}/caucase --ca-url {{ caucase_url }} --user-key ${directory:ssl}/caucase-full.key --sign-csr $csr_id
done
[cron-caucase-sign-csr]
recipe = slapos.cookbook:cron.d
cron-entries = ${cron:cron-entries}
name = caucase-sign-csr
frequency = * * * * *
command = ${caucase-sign-csr:rendered}
{% do part_list.append('cron-caucase-sign-csr') -%}
[slap-configuration] [slap-configuration]
recipe = slapos.cookbook:slapconfiguration recipe = slapos.cookbook:slapconfiguration
computer = ${slap-connection:computer-id} computer = ${slap-connection:computer-id}
...@@ -286,7 +446,6 @@ etc = ${directory:etc}/repman ...@@ -286,7 +446,6 @@ etc = ${directory:etc}/repman
data-dir = ${directory:var}/lib data-dir = ${directory:var}/lib
root-dir = ${directory:srv}/repman root-dir = ${directory:srv}/repman
clusters = ${:etc}/cluster.d clusters = ${:etc}/cluster.d
proxies = ${:etc}/proxy
proxy-data = ${:data-dir}/proxy proxy-data = ${:data-dir}/proxy
config-tmp = ${directory:tmp}/config config-tmp = ${directory:tmp}/config
bootstrap = ${:etc}/bootstrap bootstrap = ${:etc}/bootstrap
...@@ -296,8 +455,9 @@ proxies-log = ${directory:log}/proxy ...@@ -296,8 +455,9 @@ proxies-log = ${directory:log}/proxy
ipv6 = ${instance-parameter:ipv6-random} ipv6 = ${instance-parameter:ipv6-random}
port = ${instance-parameter:nginx-port} port = ${instance-parameter:nginx-port}
ssl-port = ${instance-parameter:nginx-ssl-port} ssl-port = ${instance-parameter:nginx-ssl-port}
ssl-certificate = ${ca-nginx:cert-file} ssl-certificate = ${directory:ssl}/repman-cert.crt
ssl-key = ${ca-nginx:key-file} ca-certificate = ${directory:srv}/caucase-updater/ca.crt
ssl-key = ${directory:ssl}/repman-cert.key
pid-file = ${directory:run}/nginx.pid pid-file = ${directory:run}/nginx.pid
access-log = ${directory:log}/nginx_access.log access-log = ${directory:log}/nginx_access.log
error-log = ${directory:log}/nginx_error.log error-log = ${directory:log}/nginx_error.log
...@@ -345,6 +505,8 @@ mail-to = {{ slapparameter_dict.get("mail-to", "") }} ...@@ -345,6 +505,8 @@ mail-to = {{ slapparameter_dict.get("mail-to", "") }}
http-session-lifetime = {{ slapparameter_dict.get("http-session-lifetime", 86400) }} http-session-lifetime = {{ slapparameter_dict.get("http-session-lifetime", 86400) }}
http-refresh-interval = {{ slapparameter_dict.get("http-refresh-interval", 4) }} http-refresh-interval = {{ slapparameter_dict.get("http-refresh-interval", 4) }}
ssl-cert = ${directory:ssl}/repman-cert.crt
ssl-key = ${directory:ssl}/repman-cert.key
[repman-config-folder] [repman-config-folder]
recipe = plone.recipe.command recipe = plone.recipe.command
...@@ -392,6 +554,9 @@ depends = ...@@ -392,6 +554,9 @@ depends =
${replication-manager-reload:recipe} ${replication-manager-reload:recipe}
${repman-listen-promise:recipe} ${repman-listen-promise:recipe}
${repman-listen-ssl-promise:recipe} ${repman-listen-ssl-promise:recipe}
wait-for-files =
${directory:ssl}/repman-cert.crt
${directory:ssl}/repman-cert.key
[repman-config.toml] [repman-config.toml]
recipe = slapos.recipe.template:jinja2 recipe = slapos.recipe.template:jinja2
...@@ -428,11 +593,12 @@ command-line = ...@@ -428,11 +593,12 @@ command-line =
{{ nginx_bin }} {{ nginx_bin }}
-p ${directory:nginx-prefix} -p ${directory:nginx-prefix}
-c ${nginx-conf:rendered} -c ${nginx-conf:rendered}
wrapper-path = ${directory:bin}/nginx-start wrapper-path = ${directory:services}/nginx
wait-for-files = wait-for-files =
${ca-directory:certs}/nginx.key ${directory:ssl}/repman-cert.crt
${ca-directory:certs}/nginx.crt ${directory:srv}/caucase-updater/ca.crt
${nginx-graceful-wrapper:rendered} ${directory:ssl}/repman-cert.key
[nginx-graceful-wrapper] [nginx-graceful-wrapper]
recipe = slapos.recipe.template:jinja2 recipe = slapos.recipe.template:jinja2
...@@ -442,20 +608,6 @@ rendered = ${directory:scripts}/nginx-graceful ...@@ -442,20 +608,6 @@ rendered = ${directory:scripts}/nginx-graceful
context = context =
mode = 755 mode = 755
[ca-nginx]
<= certificate-authority
recipe = slapos.cookbook:certificate_authority.request
cert-file = ${ca-directory:certs}/nginx.crt
key-file = ${ca-directory:certs}/nginx.key
executable = ${nginx-launcher:wrapper-path}
wrapper = ${directory:bin}/ca-nginx
[ca-nginx-service]
recipe = slapos.cookbook:wrapper
command-line = ${ca-nginx:wrapper}
wrapper-path = ${directory:services}/nginx
hash-existing-files = ${buildout:directory}/software_release/buildout.cfg
[logrotate-entry-nginx] [logrotate-entry-nginx]
<= logrotate-entry-base <= logrotate-entry-base
name = nginx name = nginx
...@@ -468,6 +620,7 @@ recipe = slapos.cookbook:publish-early ...@@ -468,6 +620,7 @@ recipe = slapos.cookbook:publish-early
monitor-password monitor-htpasswd:passwd monitor-password monitor-htpasswd:passwd
db-root-password gen-root-password:passwd db-root-password gen-root-password:passwd
repman-password repman-password:passwd repman-password repman-password:passwd
database-list = {{ db_list }}
[publish-connection-parameter] [publish-connection-parameter]
<= monitor-publish <= monitor-publish
...@@ -476,6 +629,7 @@ recipe = slapos.cookbook:publish ...@@ -476,6 +629,7 @@ recipe = slapos.cookbook:publish
backend-url = ${nginx-parameter:backend-ssl-url} backend-url = ${nginx-parameter:backend-ssl-url}
url = ${repman-frontend:connection-secure_access} url = ${repman-frontend:connection-secure_access}
username = ${repman-parameter:username} username = ${repman-parameter:username}
database-list = {{ db_list }}
{% for name, value in publish_dict.items() -%} {% for name, value in publish_dict.items() -%}
{{ name }} = {{ value }} {{ name }} = {{ value }}
{% endfor %} {% endfor %}
...@@ -545,6 +699,15 @@ command = ${template-proxysql-need-stop-start:rendered} ...@@ -545,6 +699,15 @@ command = ${template-proxysql-need-stop-start:rendered}
# Deploy replication-manager instance # Deploy replication-manager instance
# #
############################# #############################
[slap_connection]
# Kept for backward compatibility
computer_id = ${slap-connection:computer-id}
partition_id = ${slap-connection:partition-id}
server_url = ${slap-connection:server-url}
software_release_url = ${slap-connection:software-release-url}
key_file = ${slap-connection:key-file}
cert_file = ${slap-connection:cert-file}
[buildout] [buildout]
extends = extends =
{{ template_monitor }} {{ template_monitor }}
...@@ -552,7 +715,7 @@ parts = ...@@ -552,7 +715,7 @@ parts =
replication-manager replication-manager
monitor-base monitor-base
logrotate-entry-nginx logrotate-entry-nginx
ca-nginx-service nginx-launcher
publish-connection-parameter publish-connection-parameter
repman-frontend-promise repman-frontend-promise
repman-backend-promise repman-backend-promise
......
software/repman/instance.cfg.in
View file @ 17bf5b6a
...@@ -28,8 +28,10 @@ mode = 0644 ...@@ -28,8 +28,10 @@ mode = 0644
extensions = jinja2.ext.do extensions = jinja2.ext.do
rendered= ${buildout:directory}/${:_buildout_section_name_} rendered= ${buildout:directory}/${:_buildout_section_name_}
supervisord-lib = {{ supervisord_lib }} supervisord-lib = {{ supervisord_lib }}
causace-lib = {{ caucase_library }}
import-list = import-list =
file supervisord_lib :supervisord-lib file supervisord_lib :supervisord-lib
file caucase :causace-lib
context = context =
key slapparameter_dict slap-configuration:configuration key slapparameter_dict slap-configuration:configuration
key computer_id slap-configuration:computer key computer_id slap-configuration:computer
...@@ -48,6 +50,8 @@ context = ...@@ -48,6 +50,8 @@ context =
raw bash_bin {{ bash_location }}/bin/bash raw bash_bin {{ bash_location }}/bin/bash
raw jq_bin {{ jq_location }}/bin/jq raw jq_bin {{ jq_location }}/bin/jq
raw curl_bin {{ curl_location }}/bin/curl raw curl_bin {{ curl_location }}/bin/curl
raw openssl_bin {{ openssl_location }}/bin/openssl
raw caucase_bin_client {{ caucase_bin_client }}
${:extra-context} ${:extra-context}
extra-context = extra-context =
...@@ -56,6 +60,7 @@ extra-context = ...@@ -56,6 +60,7 @@ extra-context =
<= jinja2-template-base <= jinja2-template-base
template= {{ template_repman_cfg }} template= {{ template_repman_cfg }}
extra-context = extra-context =
key slave_instance_list slap-configuration:slave-instance-list
raw gowork_bin {{ gowork_bin }} raw gowork_bin {{ gowork_bin }}
raw haproxy_location {{ haproxy_location }} raw haproxy_location {{ haproxy_location }}
raw nginx_bin {{ nginx_location }}/sbin/nginx raw nginx_bin {{ nginx_location }}/sbin/nginx
...@@ -71,6 +76,7 @@ extra-context = ...@@ -71,6 +76,7 @@ extra-context =
raw proxysql_location {{ proxysql_location }} raw proxysql_location {{ proxysql_location }}
raw template_repman_manager_sh {{ template_repman_manager_sh }} raw template_repman_manager_sh {{ template_repman_manager_sh }}
raw template_proxy_need_stopstart {{ proxy_need_stop_start_template }} raw template_proxy_need_stopstart {{ proxy_need_stop_start_template }}
raw template_publish_slave_information {{ template_publish_slave }}
[template-mariadb-parameters] [template-mariadb-parameters]
bash = {{ bash_location }} bash = {{ bash_location }}
...@@ -83,6 +89,7 @@ template-mariadb-init-root = {{ template_init_root_sql }} ...@@ -83,6 +89,7 @@ template-mariadb-init-root = {{ template_init_root_sql }}
template-init-root-wrapper = {{ template_init_root_wrapper }} template-init-root-wrapper = {{ template_init_root_wrapper }}
template-mysqld-wrapper = {{ template_mysqld_wrapper }} template-mysqld-wrapper = {{ template_mysqld_wrapper }}
template-mysqld-need-start = {{ mysqld_start_template }} template-mysqld-need-start = {{ mysqld_start_template }}
template-manage-db = {{ template_slave_db_manage }}
link-binary = {{ dumps(mariadb_link_binary) }} link-binary = {{ dumps(mariadb_link_binary) }}
check-computer-memory-binary = {{ bin_directory }}/check-computer-memory check-computer-memory-binary = {{ bin_directory }}/check-computer-memory
bin-directory = {{ bin_directory }} bin-directory = {{ bin_directory }}
......
software/repman/publish-database-slave-parameters.cfg.in 0 → 100644
View file @ 17bf5b6a
{% set part_list = [] -%}
[slap_connection]
# Kept for backward compatibility
computer_id = ${slap-connection:computer-id}
partition_id = ${slap-connection:partition-id}
server_url = ${slap-connection:server-url}
software_release_url = ${slap-connection:software-release-url}
key_file = ${slap-connection:key-file}
cert_file = ${slap-connection:cert-file}
{% for _, database in slave_dict.items() -%}
{% set slave_reference = database['slave_reference'] -%}
{% do part_list.append('publish-slave-' ~ slave_reference) -%}
[publish-slave-{{ slave_reference }}]
recipe = slapos.cookbook:publish
-slave-reference = {{ slave_reference }}
{% if database.get('user') -%}
url_v4 = {{ "mysql://" ~ database['user'] ~ ":" ~ database['password'] ~ "@" ~ ipv4 ~ ":" ~ proxy_port ~ '/' ~ database['name'] }}
url = {{ "mysql://" ~ database['user'] ~ ":" ~ database['password'] ~ "@[" ~ ipv6 ~ "]:" ~ proxy_port ~ '/' ~ database['name'] }}
{% else -%}
url = {{ "mysql://" ~ ipv4 ~ ":" ~ proxy_port ~ database['name'] }}
url_v6 = {{ "mysql://[" ~ ip ~ "]:" ~ proxy_port ~ database['name'] }}
{% endif -%}
{% endfor %}
[buildout]
eggs-directory = {{ eggs_directory }}
develop-eggs-directory = {{ develop_eggs_directory }}
offline = true
parts =
{% for part in part_list -%}
{{ ' %s' % part }}
{% endfor -%}
software/repman/software.cfg
View file @ 17bf5b6a
...@@ -18,9 +18,10 @@ extends = ...@@ -18,9 +18,10 @@ extends =
../../component/socat/buildout.cfg ../../component/socat/buildout.cfg
../../component/rsync/buildout.cfg ../../component/rsync/buildout.cfg
../../component/jq/buildout.cfg ../../component/jq/buildout.cfg
../../component/openssl/buildout.cfg
../../stack/supervisord/buildout.cfg ../../stack/supervisord/buildout.cfg
../../stack/monitor/buildout.cfg ../../stack/monitor/buildout.cfg
../neoppod/software-common.cfg ../../stack/caucase/buildout.cfg
parts = parts =
slapos-cookbook slapos-cookbook
...@@ -28,14 +29,43 @@ parts = ...@@ -28,14 +29,43 @@ parts =
instance.cfg instance.cfg
template-mariadb.cfg template-mariadb.cfg
template-mysqld-wrapper template-mysqld-wrapper
caucase-eggs
gowork gowork
[mariadb]
location = ${mariadb-10.4:location}
[python] [python]
part = python3 part = python3
[gowork] [template-mysqld-wrapper]
# replication-manager does not build on golang 1.17 recipe = slapos.recipe.template:jinja2
golang = ${golang1.16:location} rendered = ${buildout:parts-directory}/${:_buildout_section_name_}/mysqld.in
mode = 644
template =
inline:{% raw %}#!/bin/sh -e
basedir='${mariadb:location}'
datadir='{{datadir}}'
marker=$datadir/.slapos_initializing
system=$datadir/.system
[ -d "$datadir/mysql" ] && [ ! -f "$marker" ] || {
find "$datadir/" -mindepth 1 ! \( -path $marker -o -path $system -o -path "*$system/*" \) -delete || mkdir "$datadir"
touch "$marker"
"$basedir/scripts/mysql_install_db" \
--defaults-file='{{defaults_file}}' \
--skip-name-resolve \
--auth-root-authentication-method=normal \
--basedir="$basedir" --plugin_dir="$basedir/lib/plugin" \
--datadir="$datadir"
rm "$marker"
}
{%- if environ is defined %}
{%- for variable in environ.splitlines() %}
{{ variable }} \
{%- endfor %}
{%- endif %}
exec "$basedir/bin/mysqld" --defaults-file='{{defaults_file}}' "$@"
{% endraw %}
[instance.cfg] [instance.cfg]
recipe = slapos.recipe.template:jinja2 recipe = slapos.recipe.template:jinja2
...@@ -45,6 +75,7 @@ mode = 0644 ...@@ -45,6 +75,7 @@ mode = 0644
context = context =
key bash_location bash:location key bash_location bash:location
key bin_directory buildout:bin-directory key bin_directory buildout:bin-directory
key caucase_library caucase-jinja2-library:target
key config_toml_in config-toml.in:target key config_toml_in config-toml.in:target
key config_cluster_toml_in config-cluster-toml.in:target key config_cluster_toml_in config-cluster-toml.in:target
key coreutils_location coreutils:location key coreutils_location coreutils:location
...@@ -70,6 +101,7 @@ context = ...@@ -70,6 +101,7 @@ context =
key groonga_mysql_normalizer_plugin_dir groonga-normalizer-mysql:groonga-plugin-dir key groonga_mysql_normalizer_plugin_dir groonga-normalizer-mysql:groonga-plugin-dir
key nginx_conf_in nginx.conf.in:target key nginx_conf_in nginx.conf.in:target
key nginx_location nginx:location key nginx_location nginx:location
key openssl_location openssl:location
key percona_toolkit_location percona-toolkit:location key percona_toolkit_location percona-toolkit:location
key proxy_need_stop_start_template proxy-need-start-stop.sh.in:target key proxy_need_stop_start_template proxy-need-start-stop.sh.in:target
key repman_src_location git.signal18.io_signal18_repman:location key repman_src_location git.signal18.io_signal18_repman:location
...@@ -87,10 +119,12 @@ context = ...@@ -87,10 +119,12 @@ context =
key template_init_root_sql mariadb-init-root-sql:target key template_init_root_sql mariadb-init-root-sql:target
key template_init_root_wrapper init-root-wrapper-in:target key template_init_root_wrapper init-root-wrapper-in:target
key template_repman_cfg instance-repman.cfg:target key template_repman_cfg instance-repman.cfg:target
key template_publish_slave template-publish-slave-information:target
key template_slave_db_manage slave-db-manage.in:target
key unixodbc_location unixodbc:location key unixodbc_location unixodbc:location
key sysbench_location sysbench:location key sysbench_location sysbench:location
key proxysql_location proxysql:location key proxysql_location proxysql:location
raw caucase_bin_client ${buildout:bin-directory}/caucase
[download-file] [download-file]
recipe = slapos.recipe.build:download recipe = slapos.recipe.build:download
...@@ -126,6 +160,9 @@ link-binary = ...@@ -126,6 +160,9 @@ link-binary =
[template-mariadb-initial-setup] [template-mariadb-initial-setup]
<= download-file <= download-file
[template-publish-slave-information]
<= download-file
[template-my-cnf] [template-my-cnf]
<= download-file <= download-file
...@@ -146,3 +183,6 @@ link-binary = ...@@ -146,3 +183,6 @@ link-binary =
[proxy-need-start-stop.sh.in] [proxy-need-start-stop.sh.in]
<= download-file <= download-file
[slave-db-manage.in]
<= download-file
software/repman/software.cfg.json
View file @ 17bf5b6a
...@@ -10,6 +10,15 @@ ...@@ -10,6 +10,15 @@
"request": "instance-repman-input-schema.json", "request": "instance-repman-input-schema.json",
"response": "instance-repman-output-schema.json", "response": "instance-repman-output-schema.json",
"index": 0 "index": 0
},
"default-slave": {
"title": "Replication Manager Database",
"description": "Replication Manager Database",
"software-type": "default",
"request": "instance-repman-slave-input-schema.json",
"response": "instance-repman-output-schema.json",
"shared": true,
"index": 1
} }
} }
} }
software/repman/templates/cluster-config.toml.in
View file @ 17bf5b6a
...@@ -40,6 +40,7 @@ scheduler-db-servers-physical-backup-cron = "0 {{ parameter_dict['physical-backu ...@@ -40,6 +40,7 @@ scheduler-db-servers-physical-backup-cron = "0 {{ parameter_dict['physical-backu
backup-physical-type = "mariabackup" backup-physical-type = "mariabackup"
backup-logical-type = "mysqldump" backup-logical-type = "mysqldump"
scheduler-db-servers-receiver-ports= "{{ parameter_dict['receiver-port-list'] }}" scheduler-db-servers-receiver-ports= "{{ parameter_dict['receiver-port-list'] }}"
#scheduler-db-servers-receiver-use-ssl ## set to true when ssl enabled on socat
prov-proxy-cpu-cores = {{ parameter_dict['proxy-cpu-cores'] }} prov-proxy-cpu-cores = {{ parameter_dict['proxy-cpu-cores'] }}
prov-proxy-memory = {{ parameter_dict['proxy-memory'] }} prov-proxy-memory = {{ parameter_dict['proxy-memory'] }}
...@@ -49,6 +50,7 @@ prov-db-memory = {{ parameter_dict['db-memory'] }} ...@@ -49,6 +50,7 @@ prov-db-memory = {{ parameter_dict['db-memory'] }}
prov-db-memory-shared-pct = "{{ parameter_dict['db-memory-shared-pct'] }}" prov-db-memory-shared-pct = "{{ parameter_dict['db-memory-shared-pct'] }}"
prov-db-memory-threaded-pct = "{{ parameter_dict['db-memory-threaded-pct'] }}" prov-db-memory-threaded-pct = "{{ parameter_dict['db-memory-threaded-pct'] }}"
replication-use-ssl = true
test-inject-traffic = true test-inject-traffic = true
# failover # failover
failover-mode = "{{ parameter_dict['failover-mode'] }}" failover-mode = "{{ parameter_dict['failover-mode'] }}"
......
software/repman/templates/config.toml.in
View file @ 17bf5b6a
...@@ -21,7 +21,9 @@ db-servers-binary-path = "{{ parameter_dict['mysql-bin-dir'] }}" ...@@ -21,7 +21,9 @@ db-servers-binary-path = "{{ parameter_dict['mysql-bin-dir'] }}"
#db-servers-ignored-hosts = #db-servers-ignored-hosts =
# Database hosts list to monitor, IP and port (optional), specified in the host:[port] format and separated by commas # Database hosts list to monitor, IP and port (optional), specified in the host:[port] format and separated by commas
monitoring-address = "{{ parameter_dict['ipv4'] }}" monitoring-address = "{{ parameter_dict['ipv4'] }}"
monitoring-wait-retry = 40 monitoring-wait-retry = 1200
monitoring-ssl-cert = "{{ parameter_dict['ssl-cert'] }}"
monitoring-ssl-key = "{{ parameter_dict['ssl-key'] }}"
#haproxy = true #haproxy = true
#haproxy-binary-path = "{{ parameter_dict['haproxy-bin'] }}" #haproxy-binary-path = "{{ parameter_dict['haproxy-bin'] }}"
...@@ -48,6 +50,7 @@ backup-restic-password = "{{ parameter_dict['password'] }}" ...@@ -48,6 +50,7 @@ backup-restic-password = "{{ parameter_dict['password'] }}"
backup-mysqlclient-path = "{{ parameter_dict['mysqlclient-path'] }}" backup-mysqlclient-path = "{{ parameter_dict['mysqlclient-path'] }}"
backup-mysqlbinlog-path = "{{ parameter_dict['mysqlbinlog-path'] }}" backup-mysqlbinlog-path = "{{ parameter_dict['mysqlbinlog-path'] }}"
backup-mysqldump-path = "{{ parameter_dict['mysqldump-path'] }}" backup-mysqldump-path = "{{ parameter_dict['mysqldump-path'] }}"
backup-mysqldump-options = "--hex-blob --single-transaction --verbose --all-databases --add-drop-database"
# Mail configuration # Mail configuration
# Alert email sender (default "mrm@localhost") # Alert email sender (default "mrm@localhost")
......
software/repman/templates/dbjobs.in
View file @ 17bf5b6a
#!/bin/bash #!/bin/bash
USER={{ parameter_dict['db-user'] }}
DBUSER={{ parameter_dict['db-user'] }}
PASSWORD={{ parameter_dict['db-password'] }} PASSWORD={{ parameter_dict['db-password'] }}
ERROLOG={{ parameter_dict['mysql-dir'] }}/.system/logs/errors.log ERROLOG={{ parameter_dict['mysql-dir'] }}/.system/logs/errors.log
SLOWLOG={{ parameter_dict['mysql-dir']}}/.system/logs/sql-slow SLOWLOG={{ parameter_dict['mysql-dir']}}/.system/logs/sql-slow
BACKUPDIR={{ parameter_dict['mysql-dir'] }}/.system/backup BACKUPDIR={{ parameter_dict['mysql-dir'] }}/.system/backup
DATADIR={{ parameter_dict['mysql-dir'] }}/ DATADIR={{ parameter_dict['mysql-dir'] }}/
MYSQL_CLIENT={{ parameter_dict['mysql-client'] }}
{% if parameter_dict['use-ipv6'] == True -%} {% if parameter_dict['use-ipv6'] == True -%}
{% set listen = "TCP6-LISTEN" -%} {% set listen = "TCP6-LISTEN" -%}
{% else -%} {% else -%}
...@@ -16,45 +18,45 @@ JOBS=( "xtrabackup" "mariabackup" "error" "slowquery" "zfssnapback" "optimize" " ...@@ -16,45 +18,45 @@ JOBS=( "xtrabackup" "mariabackup" "error" "slowquery" "zfssnapback" "optimize" "
doneJob() doneJob()
{ {
mysql --defaults-file={{ parameter_dict['dbjob-cnf'] }} -e "set sql_log_bin=0;UPDATE replication_manager_schema.jobs set end=NOW(), result=LOAD_FILE('{{ parameter_dict['log-dir'] }}/dbjob.out') WHERE id='$ID';" & $MYSQL_CLIENT -e "set sql_log_bin=0;UPDATE replication_manager_schema.jobs set end=NOW(), result=LOAD_FILE('{{ parameter_dict['log-dir'] }}/dbjob.out') WHERE id='$ID';" &
} }
pauseJob() pauseJob()
{ {
mysql --defaults-file={{ parameter_dict['dbjob-cnf'] }} -e "select sleep(6);set sql_log_bin=0;UPDATE replication_manager_schema.jobs set result=LOAD_FILE('{{ parameter_dict['log-dir'] }}/dbjob.out') WHERE id='$ID';" & $MYSQL_CLIENT -e "select sleep(6);set sql_log_bin=0;UPDATE replication_manager_schema.jobs set result=LOAD_FILE('{{ parameter_dict['log-dir'] }}/dbjob.out') WHERE id='$ID';" &
} }
partialRestore() partialRestore()
{ {
mysql --defaults-file={{ parameter_dict['dbjob-cnf'] }} -e "set sql_log_bin=0;install plugin BLACKHOLE soname 'ha_blackhole.so'" $MYSQL_CLIENT -e "set sql_log_bin=0;install plugin BLACKHOLE soname 'ha_blackhole.so'"
for dir in $(ls -d $BACKUPDIR/*/ | xargs -n 1 basename | grep -vE 'mysql|performance_schema|replication_manager_schema') ; do for dir in $(ls -d $BACKUPDIR/*/ | xargs -n 1 basename | grep -vE 'mysql|performance_schema|replication_manager_schema') ; do
mysql --defaults-file={{ parameter_dict['dbjob-cnf'] }} -e "set sql_log_bin=0;drop database IF EXISTS $dir; CREATE DATABASE $dir;" $MYSQL_CLIENT -e "set sql_log_bin=0;drop database IF EXISTS $dir; CREATE DATABASE $dir;"
for file in $(find $BACKUPDIR/$dir/ -name "*.exp" | xargs -n 1 basename | cut -d'.' --complement -f2-) ; do for file in $(find $BACKUPDIR/$dir/ -name "*.exp" | xargs -n 1 basename | cut -d'.' --complement -f2-) ; do
cat $BACKUPDIR/$dir/$file.frm | sed -e 's/\x06\x00\x49\x6E\x6E\x6F\x44\x42\x00\x00\x00/\x09\x00\x42\x4C\x41\x43\x4B\x48\x4F\x4C\x45/g' > $DATADIR/$dir/mrm_pivo.frm cat $BACKUPDIR/$dir/$file.frm | sed -e 's/\x06\x00\x49\x6E\x6E\x6F\x44\x42\x00\x00\x00/\x09\x00\x42\x4C\x41\x43\x4B\x48\x4F\x4C\x45/g' > $DATADIR/$dir/mrm_pivo.frm
mysql --defaults-file={{ parameter_dict['dbjob-cnf'] }} -e "set sql_log_bin=0;ALTER TABLE $dir.mrm_pivo engine=innodb;RENAME TABLE $dir.mrm_pivo TO $dir.$file; ALTER TABLE $dir.$file DISCARD TABLESPACE;" $MYSQL_CLIENT -e "set sql_log_bin=0;ALTER TABLE $dir.mrm_pivo engine=innodb;RENAME TABLE $dir.mrm_pivo TO $dir.$file; ALTER TABLE $dir.$file DISCARD TABLESPACE;"
mv $BACKUPDIR/$dir/$file.ibd $DATADIR/$dir/$file.ibd mv $BACKUPDIR/$dir/$file.ibd $DATADIR/$dir/$file.ibd
mv $BACKUPDIR/$dir/$file.exp $DATADIR/$dir/$file.exp mv $BACKUPDIR/$dir/$file.exp $DATADIR/$dir/$file.exp
mv $BACKUPDIR/$dir/$file.cfg $DATADIR/$dir/$file.cfg mv $BACKUPDIR/$dir/$file.cfg $DATADIR/$dir/$file.cfg
mv $BACKUPDIR/$dir/$file.TRG $DATADIR/$dir/$file.TRG mv $BACKUPDIR/$dir/$file.TRG $DATADIR/$dir/$file.TRG
mysql --defaults-file={{ parameter_dict['dbjob-cnf'] }} -e "set sql_log_bin=0;ALTER TABLE $dir.$file IMPORT TABLESPACE" $MYSQL_CLIENT -e "set sql_log_bin=0;ALTER TABLE $dir.$file IMPORT TABLESPACE"
done done
for file in $(find $BACKUPDIR/$dir/ -name "*.MYD" | xargs -n 1 basename | cut -d'.' --complement -f2-) ; do for file in $(find $BACKUPDIR/$dir/ -name "*.MYD" | xargs -n 1 basename | cut -d'.' --complement -f2-) ; do
mv $BACKUPDIR/$dir/$file.* $DATADIR/$dir/ mv $BACKUPDIR/$dir/$file.* $DATADIR/$dir/
mysql --defaults-file=/etc/mysql/dbjob.cnf -e "set sql_log_bin=0;FLUSH TABLE $dir.$file" $MYSQL_CLIENT -e "set sql_log_bin=0;FLUSH TABLE $dir.$file"
done done
for file in $(find $BACKUPDIR/$dir/ -name "*.CSV" | xargs -n 1 basename | cut -d'.' --complement -f2-) ; do for file in $(find $BACKUPDIR/$dir/ -name "*.CSV" | xargs -n 1 basename | cut -d'.' --complement -f2-) ; do
mv $BACKUPDIR/$dir/$file.* $DATADIR/$dir/ mv $BACKUPDIR/$dir/$file.* $DATADIR/$dir/
mysql --defaults-file={{ parameter_dict['dbjob-cnf'] }} -e "set sql_log_bin=0;FLUSH TABLE $dir.$file" $MYSQL_CLIENT -e "set sql_log_bin=0;FLUSH TABLE $dir.$file"
done done
done done
for file in $(find $BACKUPDIR/mysql/ -name "*.MYD" | xargs -n 1 basename | cut -d'.' --complement -f2-) ; do for file in $(find $BACKUPDIR/mysql/ -name "*.MYD" | xargs -n 1 basename | cut -d'.' --complement -f2-) ; do
mv $BACKUPDIR/mysql/$file.* $DATADIR/mysql/ mv $BACKUPDIR/mysql/$file.* $DATADIR/mysql/
mysql --defaults-file={{ parameter_dict['dbjob-cnf'] }} -e "set sql_log_bin=0;FLUSH TABLE mysql.$file" $MYSQL_CLIENT -e "set sql_log_bin=0;FLUSH TABLE mysql.$file"
done done
cat $BACKUPDIR/xtrabackup_info | grep binlog_pos | awk -F, '{ print $3 }' | sed -e 's/GTID of the last change/set sql_log_bin=0;set global gtid_slave_pos=/g' | mysql -h{{ parameter_dict['ip'] }} -P{{ parameter_dict['port'] }} -p$PASSWORD -u$USER cat $BACKUPDIR/xtrabackup_info | grep binlog_pos | awk -F, '{ print $3 }' | sed -e 's/GTID of the last change/set sql_log_bin=0;set global gtid_slave_pos=/g' | mysql -h{{ parameter_dict['ip'] }} -P{{ parameter_dict['port'] }} -p$PASSWORD -u$DBUSER
mysql --defaults-file={{ parameter_dict['dbjob-cnf'] }} -e"flush privileges;start slave;" $MYSQL_CLIENT -e "flush privileges;start slave;"
} }
kill -9 $(lsof -t -i:{{ parameter_dict['socat-port'] }} -sTCP:LISTEN) kill -9 $(lsof -t -i:{{ parameter_dict['socat-port'] }} -sTCP:LISTEN)
...@@ -62,12 +64,12 @@ kill -9 $(lsof -t -i:{{ parameter_dict['socat-port'] }} -sTCP:LISTEN) ...@@ -62,12 +64,12 @@ kill -9 $(lsof -t -i:{{ parameter_dict['socat-port'] }} -sTCP:LISTEN)
for job in "${JOBS[@]}" for job in "${JOBS[@]}"
do do
TASK=($(echo "select concat(id,'@',server,':',port) from replication_manager_schema.jobs WHERE task='$job' and done=0 order by task desc limit 1" | mysql -h{{ parameter_dict['ip'] }} -P{{ parameter_dict['port'] }} -p$PASSWORD -u$USER -N)) TASK=($(echo "select concat(id,'@',server,':',port) from replication_manager_schema.jobs WHERE task='$job' and done=0 order by task desc limit 1" | mysql -h{{ parameter_dict['ip'] }} -P{{ parameter_dict['port'] }} -p$PASSWORD -u$DBUSER -N))
ADDRESS=($(echo $TASK | awk -F@ '{ print $2 }')) ADDRESS=($(echo $TASK | awk -F@ '{ print $2 }'))
ID=($(echo $TASK | awk -F@ '{ print $1 }')) ID=($(echo $TASK | awk -F@ '{ print $1 }'))
#purge de past #purge de past
mysql --defaults-file={{ parameter_dict['dbjob-cnf'] }} -e "set sql_log_bin=0;UPDATE replication_manager_schema.jobs set done=1 WHERE done=0 AND task='$job';" $MYSQL_CLIENT -e "set sql_log_bin=0;UPDATE replication_manager_schema.jobs set done=1 WHERE done=0 AND task='$job';"
if [ "$ADDRESS" == "" ]; then if [ "$ADDRESS" == "" ]; then
echo "No $job needed" echo "No $job needed"
...@@ -77,14 +79,14 @@ do ...@@ -77,14 +79,14 @@ do
reseedmysqldump) reseedmysqldump)
echo "Waiting backup." > {{ parameter_dict['log-dir'] }}/dbjob.out echo "Waiting backup." > {{ parameter_dict['log-dir'] }}/dbjob.out
pauseJob pauseJob
socat -u {{ listen }}:{{ parameter_dict['socat-port'] }},bind={{ parameter_dict['host'] }},reuseaddr STDOUT | gunzip | mysql -h{{ parameter_dict['ip'] }} -P{{ parameter_dict['port'] }} -p$PASSWORD -u$USER --init-command="reset master;set sql_log_bin=0" > {{ parameter_dict['log-dir'] }}/dbjob.out 2>&1 socat -u {{ listen }}:{{ parameter_dict['socat-port'] }},bind={{ parameter_dict['host'] }},reuseaddr STDOUT | gunzip | mysql -h{{ parameter_dict['ip'] }} -P{{ parameter_dict['port'] }} -p$PASSWORD -u$DBUSER --init-command="reset master;set sql_log_bin=0" &>{{ parameter_dict['log-dir'] }}/dbjob.out
mysql --defaults-file={{ parameter_dict['dbjob-cnf'] }} -e 'start slave;' $MYSQL_CLIENT -e 'start slave;'
;; ;;
flashbackmysqldump) flashbackmysqldump)
echo "Waiting backup." > {{ parameter_dict['log-dir'] }}/dbjob.out echo "Waiting backup." > {{ parameter_dict['log-dir'] }}/dbjob.out
pauseJob pauseJob
socat -u {{ listen }}:{{ parameter_dict['socat-port'] }},bind={{ parameter_dict['host'] }},reuseaddr STDOUT | gunzip | mysql -h{{ parameter_dict['ip'] }} -P{{ parameter_dict['port'] }} -p$PASSWORD -u$USER --init-command="set sql_log_bin=0" > {{ parameter_dict['log-dir'] }}/dbjob.out 2>&1 socat -u {{ listen }}:{{ parameter_dict['socat-port'] }},bind={{ parameter_dict['host'] }},reuseaddr STDOUT | gunzip | mysql -h{{ parameter_dict['ip'] }} -P{{ parameter_dict['port'] }} -p$PASSWORD -u$DBUSER &>{{ parameter_dict['log-dir'] }}/dbjob.out
mysql --defaults-file={{ parameter_dict['dbjob-cnf'] }} -e 'start slave;' $MYSQL_CLIENT -e 'start slave;'
;; ;;
reseedmariabackup) reseedmariabackup)
rm -rf $BACKUPDIR rm -rf $BACKUPDIR
...@@ -107,7 +109,7 @@ do ...@@ -107,7 +109,7 @@ do
;; ;;
mariabackup) mariabackup)
cd {{ parameter_dict['tmp-dir'] }} cd {{ parameter_dict['tmp-dir'] }}
mariadb-backup --innobackupex --defaults-file={{ parameter_dict['dbjob-cnf'] }} --socket='{{ parameter_dict["mysqld-socket"] }}' --no-version-check --user=$USER --password=$PASSWORD --stream=xbstream {{ parameter_dict['tmp-dir'] }}/ | socat -u stdio TCP:$ADDRESS &>{{ parameter_dict['log-dir'] }}/dbjob.out mariadb-backup --innobackupex --defaults-file={{ parameter_dict['dbjob-cnf'] }} --socket='{{ parameter_dict["mysqld-socket"] }}' --no-version-check --user=root --stream=xbstream {{ parameter_dict['tmp-dir'] }}/ | socat -u stdio TCP:$ADDRESS &>{{ parameter_dict['log-dir'] }}/dbjob.out
;; ;;
error) error)
cat $ERROLOG| socat -u stdio TCP:$ADDRESS &>{{ parameter_dict['log-dir'] }}/dbjob.out cat $ERROLOG| socat -u stdio TCP:$ADDRESS &>{{ parameter_dict['log-dir'] }}/dbjob.out
...@@ -116,10 +118,10 @@ do ...@@ -116,10 +118,10 @@ do
cat $SLOWLOG| socat -u stdio TCP:$ADDRESS &>{{ parameter_dict['log-dir'] }}/dbjob.out cat $SLOWLOG| socat -u stdio TCP:$ADDRESS &>{{ parameter_dict['log-dir'] }}/dbjob.out
;; ;;
optimize) optimize)
mysqlcheck --defaults-file={{ parameter_dict['dbjob-cnf'] }} -o --all-databases --skip-write-binlog &>{{ parameter_dict['log-dir'] }}/dbjob.out $MYSQL_CLIENT -o --all-databases --skip-write-binlog &>{{ parameter_dict['log-dir'] }}/dbjob.out
;; ;;
restart) restart)
{{ parameter_dict['restart-script'] }} > {{ parameter_dict['log-dir'] }}/dbjob.out {{ parameter_dict['restart-script'] }} &>{{ parameter_dict['log-dir'] }}/dbjob.out
;; ;;
esac esac
doneJob doneJob
......
software/repman/templates/init_root_wrapper.in
View file @ 17bf5b6a
...@@ -2,7 +2,7 @@ ...@@ -2,7 +2,7 @@
run_mysql () { run_mysql () {
{{ mysql_bin }} --defaults-file="{{ mysql_conf }}" \ {{ mysql_bin }} --defaults-file="{{ mysql_conf }}" \
--protocol=socket -uroot -hlocalhost $@ --protocol=socket -hlocalhost "$@"
} }
if [ ! -f "{{ init_password_done }}" ]; then if [ ! -f "{{ init_password_done }}" ]; then
...@@ -19,6 +19,9 @@ if [ ! -f "{{ init_password_done }}" ]; then ...@@ -19,6 +19,9 @@ if [ ! -f "{{ init_password_done }}" ]; then
fi fi
echo "Setting mariabdb root password..."; echo "Setting mariabdb root password...";
run_mysql < {{ init_root_sql }} && touch {{ init_password_done }} || exit 1; run_mysql < {{ init_root_sql }} && touch {{ init_password_done }} || exit 1;
echo "remove unwanted users...";
run_mysql -e "DROP USER IF EXISTS 'root'@'$HOSTNAME';"
run_mysql -e "DROP USER IF EXISTS ''@'$HOSTNAME';"
echo "done" echo "done"
fi fi
...@@ -29,4 +32,7 @@ if [ ! -f "{{ upgrade_done }}" ]; then ...@@ -29,4 +32,7 @@ if [ ! -f "{{ upgrade_done }}" ]; then
if [ $? -eq 0 ]; then if [ $? -eq 0 ]; then
touch {{ upgrade_done }}; touch {{ upgrade_done }};
fi fi
else
# only create databases now
run_mysql < {{ init_database_sql }} || exit 1;
fi fi
software/repman/templates/mariadb_init_root.sql.in
View file @ 17bf5b6a
...@@ -2,16 +2,18 @@ ...@@ -2,16 +2,18 @@
-- or products like mysql-fabric won't work -- or products like mysql-fabric won't work
SET @@SESSION.SQL_LOG_BIN=0; SET @@SESSION.SQL_LOG_BIN=0;
# {{ parameter_dict["require-ssl"] }}
{% set ssl = parameter_dict["require-ssl"] -%}
CREATE USER '{{ parameter_dict["root-user"] }}'@'localhost' IDENTIFIED BY '{{ parameter_dict["password"] }}' ; CREATE USER '{{ parameter_dict["root-user"] }}'@'localhost' IDENTIFIED BY '{{ parameter_dict["password"] }}' ;
GRANT ALL ON *.* TO '{{ parameter_dict["root-user"] }}'@'localhost' WITH GRANT OPTION ; GRANT ALL ON *.* TO '{{ parameter_dict["root-user"] }}'@'localhost' WITH GRANT OPTION ;
CREATE USER '{{ parameter_dict["root-user"] }}'@'%' IDENTIFIED BY '{{ parameter_dict["password"] }}' ; CREATE USER '{{ parameter_dict["root-user"] }}'@'%' IDENTIFIED BY '{{ parameter_dict["password"] }}' {% if ssl %} REQUIRE SSL{% endif %};
GRANT ALL ON *.* TO '{{ parameter_dict["root-user"] }}'@'%' WITH GRANT OPTION ; GRANT ALL ON *.* TO '{{ parameter_dict["root-user"] }}'@'%' WITH GRANT OPTION ;
CREATE USER '{{ parameter_dict["root-user"] }}'@'::' IDENTIFIED BY '{{ parameter_dict["password"] }}' ; CREATE USER '{{ parameter_dict["root-user"] }}'@'::' IDENTIFIED BY '{{ parameter_dict["password"] }}' ;
GRANT ALL ON *.* TO '{{ parameter_dict["root-user"] }}'@'::' WITH GRANT OPTION ; GRANT ALL ON *.* TO '{{ parameter_dict["root-user"] }}'@'::' WITH GRANT OPTION ;
CREATE USER '{{ parameter_dict["heartbeat-user"] }}'@'localhost' IDENTIFIED BY '{{ parameter_dict["password"] }}' ; CREATE USER '{{ parameter_dict["heartbeat-user"] }}'@'localhost' IDENTIFIED BY '{{ parameter_dict["password"] }}' ;
GRANT ALL ON *.* TO '{{ parameter_dict["heartbeat-user"] }}'@'localhost' WITH GRANT OPTION ; GRANT ALL ON *.* TO '{{ parameter_dict["heartbeat-user"] }}'@'localhost' WITH GRANT OPTION ;
CREATE USER '{{ parameter_dict["heartbeat-user"] }}'@'%' IDENTIFIED BY '{{ parameter_dict["password"] }}' ; CREATE USER '{{ parameter_dict["heartbeat-user"] }}'@'%' IDENTIFIED BY '{{ parameter_dict["password"] }}' {% if ssl %} REQUIRE SSL{% endif %};
GRANT ALL ON *.* TO '{{ parameter_dict["heartbeat-user"] }}'@'%' WITH GRANT OPTION ; GRANT ALL ON *.* TO '{{ parameter_dict["heartbeat-user"] }}'@'%' WITH GRANT OPTION ;
DROP DATABASE IF EXISTS test ; DROP DATABASE IF EXISTS test ;
......
software/repman/templates/mariadb_initial_setup.sql.in
View file @ 17bf5b6a
SET @@SESSION.SQL_LOG_BIN=0; SET @@SESSION.SQL_LOG_BIN=0;
USE mysql; USE mysql;
{% set mroonga = parameter_dict.get('mroonga', 'ha_mroonga.so') -%}
{% if mroonga %}
SOURCE {{ parameter_dict['mroonga-mariadb-install-sql'] }};
{% endif %}
DROP FUNCTION IF EXISTS sphinx_snippets; DROP FUNCTION IF EXISTS sphinx_snippets;
#CREATE FUNCTION sphinx_snippets RETURNS STRING SONAME 'ha_sphinx.so'; #CREATE FUNCTION sphinx_snippets RETURNS STRING SONAME 'ha_sphinx.so';
{% macro database(name, user, password) -%} CREATE DATABASE IF NOT EXISTS `repman_slave_definition`;
CREATE DATABASE IF NOT EXISTS `{{ name }}`;
{% macro database(name, user, password, ssl, charset="") -%}
{% set charset_collate = "" -%}
{% set charset_dict = {'utf8_general_ci': 'utf8',
'utf8_unicode_ci': 'utf8',
'utf8mb4_bin': 'utf8mb4',
'utf8mb4_general_ci': 'utf8mb4',
'utf8mb4_unicode_ci': 'utf8mb4',
'latin1_general_ci': 'latin1',
'latin1_bin': 'latin1',
'ascii_general_ci': 'ascii',
'ascii_bin': 'ascii'} -%}
{% if charset %}
{% set charset_collate = "DEFAULT CHARACTER SET %s" % charset -%}
{% if charset_dict.get(charset) -%}
{% set charset_collate = "DEFAULT CHARACTER SET %s COLLATE %s" % (charset_dict[charset], charset) -%}
{% endif -%}
{% endif -%}
CREATE DATABASE IF NOT EXISTS `{{ name }}` {{ charset_collate }};
{% if user -%} {% if user -%}
GRANT ALL PRIVILEGES ON `{{ name }}`.* TO `{{ user }}`@`%` IDENTIFIED BY '{{ password }}'; GRANT ALL PRIVILEGES ON `{{ name }}`.* TO `{{ user }}`@`%` IDENTIFIED BY '{{ password }}'{% if ssl %} REQUIRE SSL{% endif %};
GRANT ALL PRIVILEGES ON `{{ name }}`.* TO `{{ user }}`@localhost IDENTIFIED BY '{{ password }}'; GRANT ALL PRIVILEGES ON `{{ name }}`.* TO `{{ user }}`@localhost IDENTIFIED BY '{{ password }}';
GRANT ALL PRIVILEGES ON `{{ name }}`.* TO `{{ user }}`@'::' IDENTIFIED BY '{{ password }}'; GRANT ALL PRIVILEGES ON `{{ name }}`.* TO `{{ user }}`@'::' IDENTIFIED BY '{{ password }}';
{%- endif %} {%- endif %}
{% endmacro -%} {% endmacro -%}
{% for entry in parameter_dict['database-list'] -%} {% for entry in parameter_dict['database-list'] -%}
{{ database(entry['name'], entry.get('user'), entry.get('password')) }} {{ database(entry['name'], entry.get('user'), entry.get('password'), entry.get('require_ssl', True) ~ '', entry.get('charset')) }}
{% endfor -%} {% endfor -%}
software/repman/templates/mysqld-need-start.sh.in
View file @ 17bf5b6a
...@@ -4,13 +4,11 @@ curl () { ...@@ -4,13 +4,11 @@ curl () {
{{ curl_bin }} -k --silent -H "Accept: application/json" "$@" {{ curl_bin }} -k --silent -H "Accept: application/json" "$@"
} }
# TOKEN=$(curl -s -X POST --data '{"username":"{{ username }}","password":"XXXXX"}' {{ repman_url }}/api/login | {{ jq_bin }} -r '.token')
# Checking if mariadb start is needed # Checking if mariadb start is needed
#CODE=$(curl -H "Authorization: Bearer ${TOKEN}" -o /dev/null -w "%{http_code}" {{ repman_url }}/api/clusters/{{ cluster }}/servers/{{ db_host }}/{{ db_port }}/need-start)
CODE=$(curl -o /dev/null -w "%{http_code}" {{ repman_url }}/api/clusters/{{ cluster }}/servers/{{ db_host }}/{{ db_port }}/need-start) CODE=$(curl -o /dev/null -w "%{http_code}" {{ repman_url }}/api/clusters/{{ cluster }}/servers/{{ db_host }}/{{ db_port }}/need-start)
if [ $CODE -eq 200 ]; then if [ $CODE -eq 200 ]; then
sleep 5
echo "$CODE: Updating mysql configuration..." echo "$CODE: Updating mysql configuration..."
# update mysql configuration # update mysql configuration
{{ update_config }} {{ update_config }}
...@@ -21,4 +19,6 @@ if [ $CODE -eq 200 ]; then ...@@ -21,4 +19,6 @@ if [ $CODE -eq 200 ]; then
sleep 5 sleep 5
# check again if the service is still up... # check again if the service is still up...
{{ mariadb_controller }} status mariadb {{ mariadb_controller }} status mariadb
else
echo "CODE: $CODE. No need start!"
fi fi
software/repman/templates/slave-db-manage.in 0 → 100644
View file @ 17bf5b6a
#!/bin/bash
set -e
curl () {
{{ curl_bin }} -k --silent -H "Accept: application/json" "$@"
}
get_token () {
curl -s -X POST --data '{"username":"{{ username }}","password":"{{ password}}"}' {{ secure_url }}/api/login
}
run_mysql () {
{{ mysql_bin }} --defaults-file="{{ mysql_conf }}" --protocol=socket "$@"
}
TOKEN=$(get_token | {{ jq_bin }} -r '.token')
DATADIR=$(curl -H "Authorization: Bearer ${TOKEN}" \ {{ secure_url }}/api/clusters/{{ cluster_name }}/topology/master | {{ jq_bin }} -r '.slaposDatadir')
# Only write or delete on master database else, we break replication.
if [ "$DATADIR" = "{{ partition_dir }}" ]; then
cat << EOF > {{ tmp_dir }}/.script.sql
use repman_slave_definition;
CREATE TABLE IF NOT EXISTS \`slave\` (
\`name\` varchar(80) NOT NULL,
\`state\` tinyint(1) DEFAULT NULL,
PRIMARY KEY (\`name\`)
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4;
UPDATE \`slave\` set \`state\`=false;
{% for name in database_name.split(' ') -%}
{% if name -%}
REPLACE INTO \`slave\` VALUES ('{{ name }}', true);
{% endif -%}
{% endfor -%}
EOF
# Update requested slaves database
run_mysql < {{ tmp_dir }}/.script.sql
rm -f {{ tmp_dir }}/.script.sql
DBNAME=$(run_mysql --skip-column-names -Be "SELECT name FROM repman_slave_definition.slave WHERE state=false");
RET=$?
if [ ! "$RET" = "0" ]; then
echo "Mysql command failed: $DBNAME"
exit $RET
fi
if [ -z "$DBNAME" ]; then
echo "No database for slave to remove.";
fi
for NAME in $DBNAME; do
if [ ! -z "$NAME" ]; then
echo "Deleting database $NAME..."
run_mysql -e "DROP DATABASE IF EXISTS $NAME";
run_mysql -e "DELETE FROM repman_slave_definition.slave WHERE name='$NAME'";
echo "Done."
fi
done
fi
software/repman/test/test.py
View file @ 17bf5b6a
...@@ -62,12 +62,6 @@ class TestRepman(SlapOSInstanceTestCase): ...@@ -62,12 +62,6 @@ class TestRepman(SlapOSInstanceTestCase):
token = resp.json()['token'] token = resp.json()['token']
headers = {"authorization": "Bearer " + token} headers = {"authorization": "Bearer " + token}
resp = requests.get(
urljoin(self.url, '/api/monitor'),
headers=headers,
verify=False,
)
self.assertEqual(resp.status_code, requests.codes.ok)
for i in range(20): for i in range(20):
resp = requests.get( resp = requests.get(
...@@ -87,3 +81,11 @@ class TestRepman(SlapOSInstanceTestCase): ...@@ -87,3 +81,11 @@ class TestRepman(SlapOSInstanceTestCase):
self.assertTrue(cluster['isProvision']) self.assertTrue(cluster['isProvision'])
self.assertTrue(cluster['isFailable']) self.assertTrue(cluster['isFailable'])
self.assertFalse(cluster['isDown']) self.assertFalse(cluster['isDown'])
resp = requests.get(
urljoin(self.url, '/api/clusters/cluster1/status'),
headers=headers,
verify=False,
)
self.assertEqual(resp.status_code, requests.codes.ok)
self.assertEqual(resp.json(), {"alive": "running"})
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment
GitLab Nexedi Edition | About GitLab | About Nexedi | 沪ICP备2021021310号-2 | 沪ICP备2021021310号-7