Skip to content

slapos
slapos
Commit 17bf5b6a authored by Thomas Gambier's avatar Thomas Gambier 🚴🏼
Browse files
Options

Replication Manager Upgrade 

See merge request nexedi/slapos!1092
parents ad04d19a bee8ef72
Pipeline #19058 failed with stage
    Hide whitespace changes
    Inline Side-by-side
    Showing with 659 additions and 164 deletions
    component/replication-manager/buildout.cfg
    View file @ 17bf5b6a
    ...@@ -28,5 +28,6 @@ command = set -e ...@@ -28,5 +28,6 @@ command = set -e
    <= go-git-package <= go-git-package
    go.importpath = github.com/signal18/replication-manager go.importpath = github.com/signal18/replication-manager
    repository = https://github.com/signal18/replication-manager repository = https://github.com/signal18/replication-manager
    branch = 2.1 #branch = 2.1
    revision = 9167a82c81af8f7be41cf51bc9be8a37dc3d8c03 branch = develop
    revision = 838ffeb205ea4477f1c1bda607940fa07d8bcf51
    \ No newline at end of file
    slapos/recipe/generic_mysql/mysql.py
    View file @ 17bf5b6a
    from __future__ import print_function
    import os import os
    import subprocess import subprocess
    import time import time
    ...@@ -16,35 +18,39 @@ def updateMysql(mysql_upgrade_binary, mysql_binary, mysql_script_file): ...@@ -16,35 +18,39 @@ def updateMysql(mysql_upgrade_binary, mysql_binary, mysql_script_file):
    while True: while True:
    while True: while True:
    mysql_upgrade = subprocess.Popen(mysql_upgrade_binary, mysql_upgrade = subprocess.Popen(mysql_upgrade_binary,
    stdout=subprocess.PIPE, stderr=subprocess.STDOUT) stdout=subprocess.PIPE, stderr=subprocess.STDOUT,
    universal_newlines=True)
    result = mysql_upgrade.communicate()[0] result = mysql_upgrade.communicate()[0]
    if mysql_upgrade.returncode: if mysql_upgrade.returncode:
    print "Command %r failed with result:\n%s" % (mysql_upgrade_binary, result) print("Command %r failed with result:\n%s" % (mysql_upgrade_binary, result))
    break break
    print "MySQL database upgraded with result:\n%s" % result print("MySQL database upgraded with result:\n%s" % result)
    mysql = subprocess.Popen(mysql_list, stdin=subprocess.PIPE, mysql = subprocess.Popen(mysql_list, stdin=subprocess.PIPE,
    stdout=subprocess.PIPE, stderr=subprocess.STDOUT) stdout=subprocess.PIPE, stderr=subprocess.STDOUT,
    universal_newlines=True)
    result = mysql.communicate(mysql_script)[0] result = mysql.communicate(mysql_script)[0]
    if mysql.returncode: if mysql.returncode:
    print 'Command %r failed with:\n%s' % (mysql_list, result) print('Command %r failed with:\n%s' % (mysql_list, result))
    break break
    # import timezone database # import timezone database
    mysql_tzinfo_to_sql = subprocess.Popen(mysql_tzinfo_to_sql_list, stdin=subprocess.PIPE, mysql_tzinfo_to_sql = subprocess.Popen(mysql_tzinfo_to_sql_list, stdin=subprocess.PIPE,
    stdout=subprocess.PIPE, stderr=subprocess.PIPE) stdout=subprocess.PIPE, stderr=subprocess.PIPE,
    universal_newlines=True)
    timezone_sql = mysql_tzinfo_to_sql.communicate()[0] timezone_sql = mysql_tzinfo_to_sql.communicate()[0]
    if mysql_tzinfo_to_sql.returncode != 0: if mysql_tzinfo_to_sql.returncode != 0:
    print 'Command %r failed with:\n%s' % (mysql_tzinfo_to_sql_list, result) print('Command %r failed with:\n%s' % (mysql_tzinfo_to_sql_list, result))
    break break
    mysql = subprocess.Popen(mysql_list + ('mysql',), stdin=subprocess.PIPE, mysql = subprocess.Popen(mysql_list + ('mysql',), stdin=subprocess.PIPE,
    stdout=subprocess.PIPE, stderr=subprocess.STDOUT) stdout=subprocess.PIPE, stderr=subprocess.STDOUT,
    universal_newlines=True)
    result = mysql.communicate(timezone_sql)[0] result = mysql.communicate(timezone_sql)[0]
    if mysql.returncode: if mysql.returncode:
    print 'Command %r failed with:\n%s' % (mysql_list, result) print('Command %r failed with:\n%s' % (mysql_list, result))
    break break
    print 'SlapOS initialisation script succesfully applied on database.' print('SlapOS initialisation script succesfully applied on database.')
    return return
    sleep = min(sleep+1, 30) sleep = min(sleep+1, 30)
    print 'Sleeping for %ss and retrying' % sleep print('Sleeping for %ss and retrying' % sleep)
    sys.stdout.flush() sys.stdout.flush()
    sys.stderr.flush() sys.stderr.flush()
    time.sleep(sleep) time.sleep(sleep)
    software/repman/buildout.hash.cfg
    View file @ 17bf5b6a
    ...@@ -14,19 +14,19 @@ ...@@ -14,19 +14,19 @@
    # not need these here). # not need these here).
    [instance.cfg] [instance.cfg]
    filename = instance.cfg.in filename = instance.cfg.in
    md5sum = 8a08be95a04f1a47098c4fdef80bdfed md5sum = af2fc4a7a0f782fed2cb1112ef3cb397
    [instance-repman.cfg] [instance-repman.cfg]
    _update_hash_filename_ = instance-repman.cfg.jinja2.in _update_hash_filename_ = instance-repman.cfg.jinja2.in
    md5sum = 697a1b546c883da45c14dbcd2d73b2b9 md5sum = c6e41783b8e3e844b6319953a8d665ec
    [config-toml.in] [config-toml.in]
    _update_hash_filename_ = templates/config.toml.in _update_hash_filename_ = templates/config.toml.in
    md5sum = 2ee2c4bc5f985c11c2167b819d261256 md5sum = dc9fdcf6c3c775da6b0b72f83c4a4105
    [config-cluster-toml.in] [config-cluster-toml.in]
    _update_hash_filename_ = templates/cluster-config.toml.in _update_hash_filename_ = templates/cluster-config.toml.in
    md5sum = c78c6be9537d3dffbb526b0e819bacb5 md5sum = ff44b81dc1177e32019ddbf4f1c6b616
    [nginx.conf.in] [nginx.conf.in]
    _update_hash_filename_ = templates/nginx.conf.in _update_hash_filename_ = templates/nginx.conf.in
    ...@@ -34,7 +34,7 @@ md5sum = 0eeb24c6aa0760f0d33c4cc2828ddf30 ...@@ -34,7 +34,7 @@ md5sum = 0eeb24c6aa0760f0d33c4cc2828ddf30
    [template-mariadb.cfg] [template-mariadb.cfg]
    _update_hash_filename_ = instance-mariadb.cfg.jinja2.in _update_hash_filename_ = instance-mariadb.cfg.jinja2.in
    md5sum = a5c204cac552754520aee0570d379723 md5sum = 938f1e8087395757f5ae861d20500658
    [template-my-cnf] [template-my-cnf]
    _update_hash_filename_ = templates/my.cnf.in _update_hash_filename_ = templates/my.cnf.in
    ...@@ -42,15 +42,19 @@ md5sum = f3661b788099bb31d71ba6e7d36836d9 ...@@ -42,15 +42,19 @@ md5sum = f3661b788099bb31d71ba6e7d36836d9
    [template-mariadb-initial-setup] [template-mariadb-initial-setup]
    _update_hash_filename_ = templates/mariadb_initial_setup.sql.in _update_hash_filename_ = templates/mariadb_initial_setup.sql.in
    md5sum = df44c2d6fb1971df582345daedad280c md5sum = 47b76144e1b116580c8acf08274af976
    [template-publish-slave-information]
    _update_hash_filename_ = publish-database-slave-parameters.cfg.in
    md5sum = 9616477ff993c55c2b43bf8797db6994
    [mariadb-init-root-sql] [mariadb-init-root-sql]
    _update_hash_filename_ = templates/mariadb_init_root.sql.in _update_hash_filename_ = templates/mariadb_init_root.sql.in
    md5sum = d8a0a7d72b02f911dafda655941b805d md5sum = 6e5c02542568f250aacd84694249efc0
    [init-root-wrapper-in] [init-root-wrapper-in]
    _update_hash_filename_ = templates/init_root_wrapper.in _update_hash_filename_ = templates/init_root_wrapper.in
    md5sum = 7661e3b2f66982b9b17bd2ab73e2e8ef md5sum = c203f40a58386310a433b58fd345a341
    [repman-manager-sh.in] [repman-manager-sh.in]
    _update_hash_filename_ = templates/repman-manager.sh.in _update_hash_filename_ = templates/repman-manager.sh.in
    ...@@ -58,12 +62,16 @@ md5sum = 852dfab6d798aa1382eec4de2fd624f9 ...@@ -58,12 +62,16 @@ md5sum = 852dfab6d798aa1382eec4de2fd624f9
    [dbjobs-in] [dbjobs-in]
    _update_hash_filename_ = templates/dbjobs.in _update_hash_filename_ = templates/dbjobs.in
    md5sum = d2ebd2ec55bf8489789a52c808729925 md5sum = d5c19d866c5a76d3bc2f434a91b1553a
    [mysqld-need-start.sh.in] [mysqld-need-start.sh.in]
    _update_hash_filename_ = templates/mysqld-need-start.sh.in _update_hash_filename_ = templates/mysqld-need-start.sh.in
    md5sum = e9bcee5dc1318fe3acda2663472214f5 md5sum = 8f14054b3761b167cf8a86f4f084a574
    [proxy-need-start-stop.sh.in] [proxy-need-start-stop.sh.in]
    _update_hash_filename_ = templates/proxy-need-start-stop.sh.in _update_hash_filename_ = templates/proxy-need-start-stop.sh.in
    md5sum = 455aaf369bf5141758dc57f2c0e67b08 md5sum = 455aaf369bf5141758dc57f2c0e67b08
    [slave-db-manage.in]
    _update_hash_filename_ = templates/slave-db-manage.in
    md5sum = b45313ae5fb06972cc7fc945e34e434a
    software/repman/instance-mariadb.cfg.jinja2.in
    View file @ 17bf5b6a
    ...@@ -17,17 +17,11 @@ ...@@ -17,17 +17,11 @@
    [{{ section('publish') }}] [{{ section('publish') }}]
    recipe = slapos.cookbook:publish.serialised recipe = slapos.cookbook:publish.serialised
    -extends = publish-early
    database-host = {{ host }}:{{ port }} database-host = {{ host }}:{{ port }}
    monitor-base-url = ${monitor-publish-parameters:monitor-base-url} monitor-base-url = ${monitor-publish-parameters:monitor-base-url}
    partition-path = ${buildout:directory} partition-path = ${buildout:directory}
    receiver-port = ${dbjob-parameter:socat-port} receiver-port = ${dbjob-parameter:socat-port}
    csr-id = ${caucase-csr-id:csr-id}
    [publish-early]
    recipe = slapos.cookbook:publish-early
    -init =
    database-list = {{ dumps(database_list) }}
    [jinja2-template-base] [jinja2-template-base]
    recipe = slapos.recipe.template:jinja2 recipe = slapos.recipe.template:jinja2
    ...@@ -37,6 +31,22 @@ mode = 644 ...@@ -37,6 +31,22 @@ mode = 644
    < = jinja2-template-base < = jinja2-template-base
    mode = 755 mode = 755
    [jinja2-template-script-base]
    < = jinja2-template-executable
    context =
    key tmp_dir directory:tmp
    key partition_dir buildout:directory
    raw username {{ slapparameter_dict['repman-user'] }}
    raw password {{ slapparameter_dict['repman-passwd'] }}
    raw cluster_name {{ slapparameter_dict['cluster'] }}
    raw secure_url {{ slapparameter_dict['repman-secure-url'] }}
    raw mysql_conf ${directory:etc}/mysql/my.cnf
    raw dash_bin {{ dash }}
    raw jq_bin {{ jq_bin }}
    raw curl_bin {{ curl_bin }}
    raw mysql_bin {{ parameter_dict['mariadb-location'] }}/bin/mysql
    ${:extra-context}
    [simplefile] [simplefile]
    < = jinja2-template-base < = jinja2-template-base
    template = inline:{{ '{{ content }}' }} template = inline:{{ '{{ content }}' }}
    ...@@ -53,10 +63,54 @@ context = key content {{content_section_name}}:content ...@@ -53,10 +63,54 @@ context = key content {{content_section_name}}:content
    mode = {{ mode }} mode = {{ mode }}
    {%- endmacro %} {%- endmacro %}
    {% import "caucase" as caucase with context %}
    {{ caucase.updater(
    prefix='caucase-updater',
    buildout_bin_directory=buildout_bin_directory,
    updater_path='${directory:services}/caucase-updater',
    url=slapparameter_dict['caucase-url'],
    data_dir='${mysql-directory:ssl}',
    crt_path='${mysql-directory:ssl}/server-cert.pem',
    ca_path='${mysql-directory:ssl}/ca-cert.pem',
    crl_path='${mysql-directory:ssl}/crl.pem',
    key_path='${mysql-directory:ssl}/server-key.pem',
    on_renew=None,
    max_sleep=None,
    template_csr_pem=None,
    openssl=openssl_bin,
    )}}
    {% do part_list.append('caucase-updater') -%}
    {% do part_list.append('caucase-updater-promise') -%}
    [get-csr-id]
    recipe = plone.recipe.command
    output = ${directory:tmp}/csr_id
    command =
    if [ -s "${mysql-directory:ssl}/server-cert.pem" ]; then
    RESULT="None";
    else
    if [ -f "${caucase-updater-csr:csr}" ]; then
    RESULT=$({{ caucase_bin_client }} --ca-url {{ slapparameter_dict['caucase-url'] }} --send-csr ${caucase-updater-csr:csr} | cut -d ' ' -f1)
    if [ ! $? -eq 0 ]; then
    RESULT="None";
    fi
    fi
    fi
    cat <<EOF > ${:output}
    [caucase]
    csr-id = $(echo $RESULT)
    EOF
    update-command = ${:command}
    [caucase-csr-id]
    recipe = slapos.cookbook:zero-knowledge.read
    file-path = ${get-csr-id:output}
    csr-id =
    [my-cnf-parameters] [my-cnf-parameters]
    socket = ${directory:run}/mariadb.sock socket = ${directory:run}/mariadb.sock
    ip = {{ ip }} ip = {{ ip }}
    data-directory = ${directory:srv}/mariadb data-directory = ${mysql-directory:data}
    pid-file = ${directory:run}/mariadb.pid pid-file = ${directory:run}/mariadb.pid
    plugin-directory = {{ dumps(parameter_dict['mroonga-mariadb-plugin-dir']) }} plugin-directory = {{ dumps(parameter_dict['mroonga-mariadb-plugin-dir']) }}
    groonga-plugins-path = {{ parameter_dict['groonga-plugins-path'] }} groonga-plugins-path = {{ parameter_dict['groonga-plugins-path'] }}
    ...@@ -66,9 +120,6 @@ innodb-log-file-size = {{ dumps(slapparameter_dict.get('innodb-log-file-size', 0 ...@@ -66,9 +120,6 @@ innodb-log-file-size = {{ dumps(slapparameter_dict.get('innodb-log-file-size', 0
    innodb-file-per-table = {{ dumps(slapparameter_dict.get('innodb-file-per-table', 0)) }} innodb-file-per-table = {{ dumps(slapparameter_dict.get('innodb-file-per-table', 0)) }}
    innodb-log-buffer-size = {{ dumps(slapparameter_dict.get('innodb-log-buffer-size', 0)) }} innodb-log-buffer-size = {{ dumps(slapparameter_dict.get('innodb-log-buffer-size', 0)) }}
    relaxed-writes = {{ dumps(slapparameter_dict.get('relaxed-writes', False)) }} relaxed-writes = {{ dumps(slapparameter_dict.get('relaxed-writes', False)) }}
    ssl-crt = ${directory:mariadb-ssl}/crt.pem
    ssl-key = ${directory:mariadb-ssl}/key.pem
    ssl-ca-crt = ${certificate-authority:ca-dir}/cacert.pem
    [my-cnf] [my-cnf]
    < = jinja2-template-base < = jinja2-template-base
    ...@@ -89,9 +140,11 @@ database-list = {{ dumps(database_list) }} ...@@ -89,9 +140,11 @@ database-list = {{ dumps(database_list) }}
    mroonga-mariadb-install-sql = {{ dumps(parameter_dict['mroonga-mariadb-install-sql']) }} mroonga-mariadb-install-sql = {{ dumps(parameter_dict['mroonga-mariadb-install-sql']) }}
    root-user = repman root-user = repman
    heartbeat-user = {{ slapparameter_dict['heartbeat-user'] }} heartbeat-user = {{ slapparameter_dict['heartbeat-user'] }}
    require-ssl = {{ dumps(slapparameter_dict['require-ssl']) }}
    [init-script] [init-script]
    < = jinja2-template-executable < = jinja2-template-executable
    extensions = jinja2.ext.do
    # XXX: is there a better location ? # XXX: is there a better location ?
    rendered = ${directory:etc}/mariadb_initial_setup.sql rendered = ${directory:etc}/mariadb_initial_setup.sql
    template = {{ parameter_dict['template-mariadb-initial-setup'] }} template = {{ parameter_dict['template-mariadb-initial-setup'] }}
    ...@@ -106,20 +159,26 @@ init-script = ${init-script:rendered} ...@@ -106,20 +159,26 @@ init-script = ${init-script:rendered}
    mysql_tzinfo_to_sql = ${binary-wrap-mysql_tzinfo_to_sql:wrapper-path} mysql_tzinfo_to_sql = ${binary-wrap-mysql_tzinfo_to_sql:wrapper-path}
    [{{ section('update-mysql-script') }}] [{{ section('update-mysql-script') }}]
    < = jinja2-template-executable < = jinja2-template-script-base
    rendered = ${directory:scripts}/mariadb_update rendered = ${directory:scripts}/mariadb_update
    init-password = ${directory:etc}/.init-passwd.done init-password = ${directory:etc}/.init-passwd.done
    upgrade-done = ${directory:lib}/mariadb-update-done upgrade-done = ${directory:lib}/mariadb-update-done
    context = extra-context =
    key init_password_done :init-password key init_password_done :init-password
    key upgrade_done :upgrade-done key upgrade_done :upgrade-done
    key init_root_sql init-root-sql:rendered key init_root_sql init-root-sql:rendered
    key mysql_update update-mysql:output key mysql_update update-mysql:output
    raw mysql_conf ${directory:etc}/mysql/my.cnf key init_database_sql init-script:rendered
    raw dash_bin {{ dash }}
    raw mysql_bin {{ parameter_dict['mariadb-location'] }}/bin/mysql
    template = {{ parameter_dict['template-init-root-wrapper'] }} template = {{ parameter_dict['template-init-root-wrapper'] }}
    [{{ section('mysql-slave-db-cleanup') }}]
    < = jinja2-template-script-base
    rendered = ${directory:scripts}/manage-slave-db
    db-name = {{ slapparameter_dict['database-name'] }}
    extra-context =
    key database_name :db-name
    template = {{ parameter_dict['template-manage-db'] }}
    [mysqld] [mysqld]
    < = jinja2-template-executable < = jinja2-template-executable
    rendered = ${directory:bin}/mysqld rendered = ${directory:bin}/mysqld
    ...@@ -136,19 +195,19 @@ environ = ...@@ -136,19 +195,19 @@ environ =
    {{ variable }} {{ variable }}
    {%- endfor %} {%- endfor %}
    [ca-mysqld] [mysqld-launcher]
    <= certificate-authority recipe = slapos.cookbook:wrapper
    recipe = slapos.cookbook:certificate_authority.request command-line = ${mysqld:rendered}
    key-file = ${my-cnf-parameters:ssl-key} wrapper-path = ${directory:controller}/mariadb
    cert-file = ${my-cnf-parameters:ssl-crt} wait-for-files =
    executable = ${mysqld:rendered} ${mysql-directory:mysql}/my.cnf
    wrapper = ${directory:controller}/mariadb ${mysql-directory:ssl}/server-cert.pem
    {% import "supervisord_lib" as supervisord_lib with context %} {% import "supervisord_lib" as supervisord_lib with context %}
    {{ supervisord_lib.supervisord("mariadb-ctl", buildout_bin_directory, supervisord_conf, use_service_hash=False) }} {{ supervisord_lib.supervisord("mariadb-ctl", buildout_bin_directory, supervisord_conf, use_service_hash=False) }}
    {% do part_list.append("supervisord-mariadb-ctl") -%} {% do part_list.append("supervisord-mariadb-ctl") -%}
    {% set maradb_program_dict = {"name": "mariadb", "command": "${ca-mysqld:wrapper}", {% set maradb_program_dict = {"name": "mariadb", "command": "${mysqld-launcher:wrapper-path}",
    "stopwaitsecs": 86400, "environment": [], "stopwaitsecs": 86400, "startretries": 10, "autorestart": True, "environment": [],
    "stdout_logfile": "${directory:log}/mariadb_stdout.log", "stdout_logfile": "${directory:log}/mariadb_stdout.log",
    "stderr_logfile": "${directory:log}/mariadb_stdout.log" } %} "stderr_logfile": "${directory:log}/mariadb_stdout.log" } %}
    ...@@ -178,13 +237,18 @@ target-directory = ${directory:bin} ...@@ -178,13 +237,18 @@ target-directory = ${directory:bin}
    link-binary = {{ dumps(parameter_dict['link-binary']) }} link-binary = {{ dumps(parameter_dict['link-binary']) }}
    [binary-wrap-base] [binary-wrap-base]
    recipe = slapos.cookbook:wrapper
    # Note: --defaults-file must be the first argument, otherwise wrapped binary # Note: --defaults-file must be the first argument, otherwise wrapped binary
    # will reject it. # will reject it.
    command-line = recipe = slapos.recipe.template:jinja2
    "{{ parameter_dict['mariadb-location'] }}/bin/${:command}" rendered = ${directory:bin}/${:command}
    --defaults-file="${directory:etc}/mysql/my.cnf" --protocol=socket ${:extra-args} mode = 755
    wrapper-path = ${directory:bin}/${:command} template =
    inline:#!/bin/sh
    exec {{ parameter_dict['mariadb-location'] }}/bin/${:command} \
    --defaults-file="${directory:etc}/mysql/my.cnf" \
    --protocol=socket \
    ${:extra-args} "$@"
    wrapper-path = ${:rendered}
    extra-args = extra-args =
    [binary-wrap-mysql] [binary-wrap-mysql]
    ...@@ -208,6 +272,11 @@ command = mysqladmin ...@@ -208,6 +272,11 @@ command = mysqladmin
    <= binary-wrap-base <= binary-wrap-base
    command-line = "{{ parameter_dict['mariadb-location'] }}/bin/${:command}" --skip-write-binlog command-line = "{{ parameter_dict['mariadb-location'] }}/bin/${:command}" --skip-write-binlog
    command = mysql_tzinfo_to_sql command = mysql_tzinfo_to_sql
    template =
    inline:#!/bin/sh
    exec {{ parameter_dict['mariadb-location'] }}/bin/${:command} \
    --defaults-file="${directory:etc}/mysql/my.cnf" \
    --skip-write-binlog "$@"
    [binary-wrap-pt-digest] [binary-wrap-pt-digest]
    <= binary-wrap-base <= binary-wrap-base
    ...@@ -226,26 +295,29 @@ plugin = ${:etc}/plugin ...@@ -226,26 +295,29 @@ plugin = ${:etc}/plugin
    srv = ${buildout:directory}/srv srv = ${buildout:directory}/srv
    tmp = ${buildout:directory}/tmp tmp = ${buildout:directory}/tmp
    backup = ${:srv}/backup backup = ${:srv}/backup
    mariadb-backup-full = ${:backup}/mariadb-full
    mariadb-backup-incremental = ${:backup}/mariadb-incremental
    mariadb-ssl = ${:etc}/mariadb-ssl
    var = ${buildout:directory}/var var = ${buildout:directory}/var
    lib = ${:var}/lib lib = ${:var}/lib
    mysql = ${:lib}/mysql
    log = ${:var}/log log = ${:var}/log
    run = ${:var}/run run = ${:var}/run
    config-tmp = ${:tmp}/config config-tmp = ${:tmp}/config
    custom = ${directory:etc}/mysql/custom
    [mysql-directory]
    recipe = slapos.cookbook:mkdirectory
    mysql = ${directory:etc}/mysql
    ssl = ${:mysql}/ssl
    data = ${directory:lib}/mysql
    custom = ${:mysql}/custom.d
    [dbjob-parameter] [dbjob-parameter]
    bash-bin = {{ bash_bin }} bash-bin = {{ bash_bin }}
    db-user = ${init-script-parameters:root-user} db-user = ${init-script-parameters:root-user}
    db-password = ${init-script-parameters:password} db-password = ${init-script-parameters:password}
    mysql-dir = ${directory:mysql} mysql-dir = ${mysql-directory:data}
    dbjob-cnf = ${directory:etc}/mysql/my.cnf dbjob-cnf = ${mysql-directory:mysql}/dbjob.cnf
    log-dir = ${directory:mysql}/.system/logs log-dir = ${mysql-directory:data}/.system/logs
    tmp-dir = ${directory:tmp} tmp-dir = ${directory:tmp}
    mysqld-socket = ${my-cnf-parameters:socket} mysqld-socket = ${my-cnf-parameters:socket}
    mysql-client = ${binary-wrap-mysql:wrapper-path}
    socat-port = {{ int(port) + 9 }} socat-port = {{ int(port) + 9 }}
    restart-script = ${mysqld-restart-script:rendered} restart-script = ${mysqld-restart-script:rendered}
    socat-location = {{ parameter_dict['socat-location'] }} socat-location = {{ parameter_dict['socat-location'] }}
    ...@@ -309,11 +381,11 @@ template = inline:#!{{ bash_bin }} ...@@ -309,11 +381,11 @@ template = inline:#!{{ bash_bin }}
    cd ${directory:config-tmp} && cd ${directory:config-tmp} &&
    {{ curl_bin }} -o config.tar.gz {{ slapparameter_dict['repman-url'] }}/api/clusters/${:cluster}/servers/{{ host }}/{{ port }}/config {{ curl_bin }} -o config.tar.gz {{ slapparameter_dict['repman-url'] }}/api/clusters/${:cluster}/servers/{{ host }}/{{ port }}/config
    tar -xzf config.tar.gz tar -xzf config.tar.gz
    cp -r data/.system ${directory:mysql} rm -r etc/mysql/ssl # we don't need generated ssl files
    rm -rf ${directory:etc}/mysql cp -r data/.system ${mysql-directory:data}
    cp -r etc/mysql ${directory:etc} cp -r etc/mysql/* ${mysql-directory:mysql}
    ln -sf ${directory:mysql}/.system ${directory:var}/system ln -sf ${mysql-directory:data}/.system ${directory:var}/system
    ln -sf ${my-cnf:rendered} ${directory:etc}/mysql/custom/01_mariadb.cnf ln -sf ${my-cnf:rendered} ${mysql-directory:custom}/01_mariadb.cnf
    [{{ section('install-mysql-config') }}] [{{ section('install-mysql-config') }}]
    recipe = plone.recipe.command recipe = plone.recipe.command
    ......
    software/repman/instance-repman-input-schema.json
    View file @ 17bf5b6a
    ...@@ -113,7 +113,8 @@ ...@@ -113,7 +113,8 @@
    "mroonga", "mroonga",
    "utctime", "utctime",
    "readcommitted", "readcommitted",
    "nohandshake" "nohandshake",
    "ssl"
    ] ]
    }, },
    "http-session-lifetime": { "http-session-lifetime": {
    ...@@ -140,6 +141,12 @@ ...@@ -140,6 +141,12 @@
    "type": "boolean", "type": "boolean",
    "default": true "default": true
    }, },
    "require-ssl": {
    "title": "Enable REQUIRE SSL for repman database users",
    "description": "Enable secure connection between repman and mariadb databases. This option is an initial value, used only when creating database users.",
    "type": "boolean",
    "default": false
    },
    "repman-cluster-dict": { "repman-cluster-dict": {
    "title": "Replication Manager clusters definition", "title": "Replication Manager clusters definition",
    "description": "Replication Manager clusters definition", "description": "Replication Manager clusters definition",
    ...@@ -197,7 +204,7 @@ ...@@ -197,7 +204,7 @@
    }, },
    "proxy-tags": { "proxy-tags": {
    "title": "Proxy tag list", "title": "Proxy tag list",
    "description": "playbook configuration tags. Default: [\"pkg\", \"masterslave\", \"linux\", \"noreadwritesplit\"]", "description": "playbook configuration tags. Default: [\"pkg\", \"masterslave\", \"linux\", \"noreadwritesplit\", \"ssl\"]",
    "type": "array", "type": "array",
    "items": { "items": {
    "type": "string" "type": "string"
    ...@@ -206,7 +213,8 @@ ...@@ -206,7 +213,8 @@
    "pkg", "pkg",
    "masterslave", "masterslave",
    "linux", "linux",
    "noreadwritesplit" "noreadwritesplit",
    "ssl"
    ] ]
    }, },
    "logical-backup-cron": { "logical-backup-cron": {
    ......
    software/repman/instance-repman-slave-input-schema.json 0 → 100644
    View file @ 17bf5b6a
    {
    "$schema": "http://json-schema.org/draft-04/schema",
    "properties": {
    "db_user": {
    "description": "Database User, default is 'user'. This parameter is set only if database is not created yet.",
    "title": "Database User",
    "type": "string",
    "default": "user"
    },
    "db_password": {
    "description": "Database password. If no password set, a password will be generated. This parameter is set only if database is not created yet.",
    "title": "Initial database password",
    "type": "string",
    "default": ""
    },
    "db_charset": {
    "description": "Database Character set information. This parameter is set only if database is not created yet.",
    "title": "Database Character set information",
    "type": "string",
    "enum": [
    "latin1",
    "latin2",
    "latin5",
    "latin7",
    "ascii",
    "latin5",
    "utf8",
    "utf8mb4",
    "utf16",
    "utf16le",
    "utf32",
    "binary",
    "utf8_general_ci",
    "utf8_unicode_ci",
    "utf8_unicode_ci",
    "utf8mb4_bin",
    "utf8mb4_general_ci",
    "utf8mb4_unicode_ci",
    "latin1_general_ci",
    "latin1_bin",
    "ascii_general_ci",
    "ascii_bin"
    ],
    "default": ""
    }
    },
    "title": "Input Parameters",
    "type": "object"
    }
    software/repman/instance-repman.cfg.jinja2.in
    View file @ 17bf5b6a
    ...@@ -8,8 +8,23 @@ ...@@ -8,8 +8,23 @@
    {% set ip = (ipv6_set | list)[0] -%} {% set ip = (ipv6_set | list)[0] -%}
    {% set ipv4 = (ipv4_set | list)[0] -%} {% set ipv4 = (ipv4_set | list)[0] -%}
    {% set cluster_list = [] -%} {% set cluster_list = [] -%}
    {% set tag_list = ["gtidstrict", "bind", "pkg", "innodb", "noquerycache", "slow", "pfs", "linux", "readonly", "diskmonitor", "sqlerror", "compressbinlog", "bm4ci", "mroonga", "utctime", "readcommitted", "nohandshake"] -%} {% set tag_list = ["gtidstrict", "bind", "pkg", "innodb", "noquerycache", "slow",
    "pfs", "linux", "readonly", "diskmonitor", "sqlerror", "compressbinlog", "bm4ci",
    "mroonga", "utctime", "readcommitted", "nohandshake", "ssl"] -%}
    {% set frontend_parameter_dict = slapparameter_dict.get('slave-frontend', {}) -%} {% set frontend_parameter_dict = slapparameter_dict.get('slave-frontend', {}) -%}
    {% set database_slave_list = [] -%}
    {% set db_name_list = [] -%}
    {% set count = namespace(value=2) %}
    {% set caucase_bind = '[' ~ ip ~ ']:8890' -%}
    {% set caucase_url = 'http://' ~ caucase_bind -%}
    {% set csrid_list = [] -%}
    {% macro password(name) -%}
    [{{ name }}-password]
    recipe = slapos.cookbook:generate.password
    bytes = 12
    {% endmacro -%}
    [directory] [directory]
    recipe = slapos.cookbook:mkdirectory recipe = slapos.cookbook:mkdirectory
    ...@@ -25,11 +40,25 @@ log = ${:var}/log ...@@ -25,11 +40,25 @@ log = ${:var}/log
    data = ${:var}/lib data = ${:var}/lib
    nginx-prefix = ${:var}/nginx nginx-prefix = ${:var}/nginx
    tmp = ${:home}/tmp tmp = ${:home}/tmp
    backup-caucased = ${:srv}/backup-caucased
    caucased = ${:srv}/caucased
    ssl = ${:etc}/ssl
    [proxysql-directory]
    recipe = slapos.cookbook:mkdirectory
    config = ${directory:etc}/proxysql
    ssl = ${:config}/ssl
    {% import "supervisord_lib" as supervisord_lib with context %} {% import "supervisord_lib" as supervisord_lib with context %}
    {% set proxysql_controller = "proxysql-ctl" -%} {% set proxysql_controller = "proxysql-ctl" -%}
    {{ supervisord_lib.supervisord(proxysql_controller, buildout_bin_directory, supervisord_conf, use_service_hash=False) }} {{ supervisord_lib.supervisord(proxysql_controller, buildout_bin_directory, supervisord_conf, use_service_hash=False) }}
    [proxysql-controller-update]
    recipe = slapos.cookbook:wrapper
    command-line = ${proxysql-ctl-bin:wrapper-path} update
    wrapper-path = ${directory:scripts}/proxysql-ctl-update
    {% do part_list.append("supervisord-proxysql-ctl") -%} {% do part_list.append("supervisord-proxysql-ctl") -%}
    {% do part_list.append("proxysql-controller-update") -%}
    [request-common] [request-common]
    recipe = slapos.cookbook:request.serialised recipe = slapos.cookbook:request.serialised
    ...@@ -39,6 +68,7 @@ key-file = ${slap-connection:key-file} ...@@ -39,6 +68,7 @@ key-file = ${slap-connection:key-file}
    cert-file = ${slap-connection:cert-file} cert-file = ${slap-connection:cert-file}
    computer-id = ${slap-connection:computer-id} computer-id = ${slap-connection:computer-id}
    partition-id = ${slap-connection:partition-id} partition-id = ${slap-connection:partition-id}
    config-caucase-url = {{ caucase_url }}
    [download-proxy-config] [download-proxy-config]
    recipe = slapos.recipe.template:jinja2 recipe = slapos.recipe.template:jinja2
    ...@@ -48,16 +78,35 @@ template = inline:#!{{ bash_bin }} ...@@ -48,16 +78,35 @@ template = inline:#!{{ bash_bin }}
    PORT=$3 PORT=$3
    CONFIG=$4 CONFIG=$4
    if [ -z "$CONFIG" ]; then if [ -z "$CONFIG" ]; then
    CONFIG="${repman:proxies}/proxysql-$NAME.cnf" CONFIG="${proxysql-directory:config}/proxysql.cnf"
    fi fi
    mkdir -p ${repman:config-tmp}/proxies mkdir -p ${repman:config-tmp}/proxies
    cd ${repman:config-tmp}/proxies cd ${repman:config-tmp}/proxies
    {{ curl_bin }} -o proxies-$NAME.tar.gz ${nginx-parameter:repman-url}/api/clusters/$NAME/servers/$HOST/$PORT/config {{ curl_bin }} -o proxies-$NAME.tar.gz ${nginx-parameter:repman-url}/api/clusters/$NAME/servers/$HOST/$PORT/config
    tar -xzf proxies-$NAME.tar.gz tar -xzf proxies-$NAME.tar.gz
    cp conf/proxysql.cnf $CONFIG cp etc/proxysql/proxysql.cnf $CONFIG
    rendered = ${directory:bin}/update-proxysql-config rendered = ${directory:bin}/update-proxysql-config
    mode = 755 mode = 755
    {% for instance_dict in slave_instance_list -%}
    {% set slave_dict = {
    'name': 'db_%s' % instance_dict['slave_reference'].replace('-', '_').lower(),
    'user': instance_dict.get('db_user', 'user'),
    'password': instance_dict.get('db_password', '${' ~ instance_dict['slave_reference'] ~ '-password:passwd}'),
    'slave_reference': instance_dict['slave_reference'],
    'charset': instance_dict.get('db_charset', ''),
    'require_ssl': True
    } -%}
    {% do database_slave_list.append(slave_dict) -%}
    {% do db_name_list.append(slave_dict['name']) -%}
    {{ password(instance_dict['slave_reference']) }}
    {% endfor %}
    [database-slave-information]
    {% for slave_dict in database_slave_list -%}
    {{ slave_dict['name'] }} = !py!{{ slave_dict }}
    {% endfor %}
    {% set db_list = db_name_list | join(' ') -%}
    {% do mariadb_dict.__setitem__('computer-memory-percent-threshold', 80) -%} {% do mariadb_dict.__setitem__('computer-memory-percent-threshold', 80) -%}
    {% set default_parameter_dict = {"cluster1": {"name": "cluster1", "db-prefered-master": "", {% set default_parameter_dict = {"cluster1": {"name": "cluster1", "db-prefered-master": "",
    "database-amount": 2, "proxysql-user": "external", "database-amount": 2, "proxysql-user": "external",
    ...@@ -67,22 +116,23 @@ mode = 755 ...@@ -67,22 +116,23 @@ mode = 755
    {% do mariadb_dict.__setitem__('innodb-file-per-table', parameter_dict.get('innodb-file-per-table', 1)) -%} {% do mariadb_dict.__setitem__('innodb-file-per-table', parameter_dict.get('innodb-file-per-table', 1)) -%}
    {% do mariadb_dict.__setitem__('use-ipv6', parameter_dict.get('use-ipv6', True)) -%} {% do mariadb_dict.__setitem__('use-ipv6', parameter_dict.get('use-ipv6', True)) -%}
    {% set database_list = parameter_dict.get('database-list', [{'name': 'repdb', 'user': 'user', 'password': 'insecure'}]) -%}
    # Request mariadb instances # Request mariadb instances
    {% set db_amount = parameter_dict.get('database-amount', 2) -%} {% set db_amount = parameter_dict.get('database-amount', 2) -%}
    {% if db_amount < 2 -%} {% if db_amount < 2 -%}
    {% set db_amount = 2 -%} {% set db_amount = 2 -%}
    {% endif -%} {% endif -%}
    {% set count.value = count.value + db_amount %}
    {% for i in range(0, db_amount) -%} {% for i in range(0, db_amount) -%}
    {% do mariadb_dict.__setitem__('tcp-port', 2099 + (i * 100)) -%} {% do mariadb_dict.__setitem__('tcp-port', 2099 + (i * 100)) -%}
    {% set section = 'request-mariadb-' ~ i -%} {% set section = 'request-mariadb-' ~ i -%}
    {% set dbname = 'Mariadb-' ~ i -%} {% set dbname = 'Mariadb-' ~ i %}
    [{{ section }}] [{{ section }}]
    <= request-common <= request-common
    software-type = mariadb software-type = mariadb
    name = {{ dbname }} name = {{ dbname ~ '-' ~ name}}
    sla-computer_guid = {{ dumps(parameter_dict.get('-sla-' ~ i ~'-computer_guid', '')) }} sla-computer_guid = {{ dumps(parameter_dict.get('-sla-' ~ i ~'-computer_guid', '')) }}
    {% for key, value in mariadb_dict.items() -%} {% for key, value in mariadb_dict.items() -%}
    config-{{ key }} = {{ dumps(value) }} config-{{ key }} = {{ dumps(value) }}
    ...@@ -91,23 +141,27 @@ config-monitor-passwd = ${publish-early:monitor-password} ...@@ -91,23 +141,27 @@ config-monitor-passwd = ${publish-early:monitor-password}
    config-root-password = ${publish-early:db-root-password} config-root-password = ${publish-early:db-root-password}
    config-repman-user = ${repman-parameter:username} config-repman-user = ${repman-parameter:username}
    config-heartbeat-user = ${repman-parameter:heartbeat-user} config-heartbeat-user = ${repman-parameter:heartbeat-user}
    #config-repman-passwd = ${repman-parameter:password} config-repman-passwd = ${repman-parameter:password}
    config-repman-url = ${nginx-parameter:backend-url} config-repman-url = ${nginx-parameter:backend-url}
    config-repman-secure-url = ${nginx-parameter:backend-ssl-url} config-repman-secure-url = ${nginx-parameter:backend-ssl-url}
    config-cluster = {{ name }} config-cluster = {{ name }}
    config-name = {{ dbname }} config-name = {{ dbname }}
    config-database-list = {{ dumps(database_list) }} config-database-list = !py!{{ database_slave_list }}
    return = config-database-name = {{ dumps(db_list) }}
    config-require-ssl = {{ dumps(slapparameter_dict.get('require-ssl', False)) }}
    return =
    database-host database-host
    receiver-port receiver-port
    monitor-base-url monitor-base-url
    partition-path partition-path
    csr-id
    {% do part_list.append(section) -%} {% do part_list.append(section) -%}
    {% do mariadb_server_list.append('${' ~ section ~ ':connection-database-host}') -%} {% do mariadb_server_list.append('${' ~ section ~ ':connection-database-host}') -%}
    {% do receiver_port_list.append('${' ~ section ~ ':connection-receiver-port}') -%} {% do receiver_port_list.append('${' ~ section ~ ':connection-receiver-port}') -%}
    {% do mariadb_path_list.append('${' ~ section ~ ':connection-partition-path}') -%} {% do mariadb_path_list.append('${' ~ section ~ ':connection-partition-path}') -%}
    {% do monitor_base_url_dict.__setitem__('mariadb' ~ i, '${' ~ section ~ ':connection-monitor-base-url}') -%} {% do monitor_base_url_dict.__setitem__('mariadb' ~ i, '${' ~ section ~ ':connection-monitor-base-url}') -%}
    {% do csrid_list.append('${' ~ section ~ ':connection-csr-id}') -%}
    {% endfor -%} {% endfor -%}
    ...@@ -129,6 +183,7 @@ maximum = 7132 ...@@ -129,6 +183,7 @@ maximum = 7132
    {% for key, value in parameter_dict.items() -%} {% for key, value in parameter_dict.items() -%}
    {{ key }} = {{ value }} {{ key }} = {{ value }}
    {% endfor -%} {% endfor -%}
    name = {{ name }}
    proxysql-user = {{ parameter_dict.get("proxysql-user", "external") }} proxysql-user = {{ parameter_dict.get("proxysql-user", "external") }}
    proxy-port = {{ '${' ~ name ~ '-port:port}' }} proxy-port = {{ '${' ~ name ~ '-port:port}' }}
    proxy-admin-port = {{ '${' ~ name ~ '-admin-port:port}' }} proxy-admin-port = {{ '${' ~ name ~ '-admin-port:port}' }}
    ...@@ -151,7 +206,7 @@ password = ${repman-parameter:password} ...@@ -151,7 +206,7 @@ password = ${repman-parameter:password}
    proxysql-partition = ${buildout:directory} proxysql-partition = ${buildout:directory}
    receiver-port-list = {{ receiver_port_list | join(',') }} receiver-port-list = {{ receiver_port_list | join(',') }}
    enabled-tags = {{ slapparameter_dict.get("tag-list", tag_list) | join(',') }} enabled-tags = {{ slapparameter_dict.get("tag-list", tag_list) | join(',') }}
    proxy-tags = {{ parameter_dict.get("proxy-tags", ["pkg", "masterslave", "linux", "noreadwritesplit"]) | join(',') }} proxy-tags = {{ parameter_dict.get("proxy-tags", ["pkg", "masterslave", "linux", "noreadwritesplit", "ssl"]) | join(',') }}
    logical-backup-cron = {{ parameter_dict.get("logical-backup-cron", "0 22 * * *") }} logical-backup-cron = {{ parameter_dict.get("logical-backup-cron", "0 22 * * *") }}
    physical-backup-cron = {{ parameter_dict.get("physical-backup-cron", "0 0 * * *") }} physical-backup-cron = {{ parameter_dict.get("physical-backup-cron", "0 0 * * *") }}
    proxy-cpu-cores = {{ parameter_dict.get("proxy-cpu-cores", 2) }} proxy-cpu-cores = {{ parameter_dict.get("proxy-cpu-cores", 2) }}
    ...@@ -191,7 +246,7 @@ context = ...@@ -191,7 +246,7 @@ context =
    recipe = plone.recipe.command recipe = plone.recipe.command
    # if Repman is not started, cannot download config from server # if Repman is not started, cannot download config from server
    stop-on-error = false stop-on-error = false
    config = ${repman:proxies}/proxysql-{{ name }}.cnf config = ${proxysql-directory:config}/proxysql.cnf
    data = ${repman:proxy-data}/{{ name }} data = ${repman:proxy-data}/{{ name }}
    command = command =
    mkdir -p ${:data} && mkdir -p ${:data} &&
    ...@@ -209,6 +264,7 @@ command-line = ...@@ -209,6 +264,7 @@ command-line =
    wrapper-path = ${directory:controller}/proxysql-{{ name }} wrapper-path = ${directory:controller}/proxysql-{{ name }}
    wait-for-files = wait-for-files =
    ${repman:bootstrap}/{{ name }}_bootstrapped ${repman:bootstrap}/{{ name }}_bootstrapped
    ${config-proxysql-{{ name }}:config}
    depends = depends =
    {{ '${proxysql-' ~ name ~ '-admin-promise:recipe}' }} {{ '${proxysql-' ~ name ~ '-admin-promise:recipe}' }}
    {{ '${proxysql-' ~ name ~ '-promise:recipe}' }} {{ '${proxysql-' ~ name ~ '-promise:recipe}' }}
    ...@@ -237,7 +293,7 @@ config-port = {{ '${' ~ name ~ '-cluster-parameter:proxy-port}' }} ...@@ -237,7 +293,7 @@ config-port = {{ '${' ~ name ~ '-cluster-parameter:proxy-port}' }}
    {% set service_name = "proxysql-" ~ name -%} {% set service_name = "proxysql-" ~ name -%}
    {% set proxysql_dict = {"name": service_name, "command": "${" ~ service_name ~ "-wrapper:wrapper-path}", {% set proxysql_dict = {"name": service_name, "command": "${" ~ service_name ~ "-wrapper:wrapper-path}",
    "stopwaitsecs": 60, "environment": [], "stopwaitsecs": 60, "startretries": 10, "autorestart": True, "environment": [],
    "stdout_logfile": "${repman:proxies-log}/" ~ service_name ~ ".log", "stdout_logfile": "${repman:proxies-log}/" ~ service_name ~ ".log",
    "stderr_logfile": "${repman:proxies-log}/" ~ service_name ~ ".log" } %} "stderr_logfile": "${repman:proxies-log}/" ~ service_name ~ ".log" } %}
    ...@@ -247,22 +303,126 @@ config-port = {{ '${' ~ name ~ '-cluster-parameter:proxy-port}' }} ...@@ -247,22 +303,126 @@ config-port = {{ '${' ~ name ~ '-cluster-parameter:proxy-port}' }}
    {% do part_list.append('config-' ~ name) -%} {% do part_list.append('config-' ~ name) -%}
    {% do cluster_list.append("{'name': '" ~ name ~ "', 'host': '" ~ ipv4 ~ "', 'port': '${" ~ name ~ "-cluster-parameter:proxy-admin-port}'}") -%} {% do cluster_list.append("{'name': '" ~ name ~ "', 'host': '" ~ ipv4 ~ "', 'port': '${" ~ name ~ "-cluster-parameter:proxy-admin-port}'}") -%}
    {% set publish_database_list = [] -%}
    {% set publish_database_v6_list = [] -%}
    {% for database in database_list -%}
    {% if database.get('user') -%}
    {% do publish_database_list.append("mysql://" ~ database['user'] ~ ":" ~ database['password'] ~ "@" ~ ipv4 ~ ":${" ~ name ~ "-cluster-parameter:proxy-port}/" ~ database['name']) -%}
    {% do publish_database_v6_list.append("mysql://" ~ database['user'] ~ ":" ~ database['password'] ~ "@[" ~ ip ~ "]:${" ~ name ~ "-cluster-parameter:proxy-port}/" ~ database['name']) -%}
    {% else -%}
    {% do publish_database_list.append("mysql://" ~ ipv4 ~ ":${" ~ name ~ "-cluster-parameter:proxy-port}/" ~ database['name']) -%}
    {% do publish_database_v6_list.append("mysql://[" ~ ip ~ "]:${" ~ name ~ "-cluster-parameter:proxy-port}/" ~ database['name']) -%}
    {% endif -%}
    {% endfor -%}
    {% do publish_dict.__setitem__(name ~ '-database-list', "!py!['" ~ publish_database_list | join("', '") ~ "']") -%} [{{ name}}-publish-slave-information]
    {% do publish_dict.__setitem__(name ~ '-database-list-v6', "!py!['" ~ publish_database_v6_list | join("', '") ~ "']") -%} recipe = slapos.cookbook:switch-softwaretype
    default = {{ 'dynamic-' ~name ~ '-publish-slave-information:rendered' }}
    RootSoftwareInstance = ${:default}
    [dynamic-{{ name}}-publish-slave-information]
    recipe = slapos.recipe.template:jinja2
    rendered = ${buildout:directory}/database-publish-slave-information.cfg
    extensions = jinja2.ext.do
    template = {{ template_publish_slave_information }}
    context =
    import json_module json
    key proxy_port {{ name }}-cluster-parameter:proxy-port
    raw eggs_directory {{ eggs_directory }}
    raw develop_eggs_directory {{ develop_eggs_directory }}
    raw ipv6 {{ ip }}
    raw ipv4 {{ ipv4 }}
    section slave_dict database-slave-information
    {% do part_list.append(name ~ '-publish-slave-information') -%}
    {% endfor -%} {% endfor -%}
    # deploy caucase
    {% import "caucase" as caucase with context %}
    {{ caucase.caucased(
    prefix='caucased',
    buildout_bin_directory=buildout_bin_directory,
    caucased_path='${directory:service}/caucased',
    backup_dir='${directory:backup-caucased}',
    data_dir='${directory:caucased}',
    netloc=caucase_bind,
    tmp='${directory:tmp}',
    service_auto_approve_count=count.value,
    user_auto_approve_count=1,
    key_len=2048,
    )}}
    {% do part_list.append('caucased') -%}
    {% do part_list.append('caucased-promise') -%}
    {% do publish_dict.__setitem__('caucase-http-url', caucase_url) -%}
    {{ caucase.updater(
    prefix='caucase-updater',
    buildout_bin_directory=buildout_bin_directory,
    updater_path='${directory:services}/caucase-updater',
    url=caucase_url,
    data_dir='${directory:srv}/caucase-updater',
    crt_path='${directory:ssl}/repman-cert.crt',
    ca_path='${directory:srv}/caucase-updater/ca.crt',
    crl_path='${directory:srv}/caucase-updater/crl.pem',
    key_path='${directory:ssl}/repman-cert.key',
    on_renew=None,
    max_sleep=None,
    template_csr_pem=None,
    openssl=openssl_bin,
    )}}
    {% do part_list.append('caucase-updater') -%}
    {% do part_list.append('caucase-updater-promise') -%}
    #caucase user certificate
    {{ caucase.updater(
    prefix='caucase-user-updater',
    buildout_bin_directory=buildout_bin_directory,
    updater_path='${directory:services}/caucase-user-updater',
    url=caucase_url,
    data_dir='${directory:srv}/caucase-user-updater',
    crt_path='${directory:ssl}/caucase.user.crt',
    ca_path='${directory:srv}/caucase-user-updater/ca.crt',
    crl_path='${directory:srv}/caucase-user-updater/crl.pem',
    key_path='${directory:ssl}/caucase.user.key',
    on_renew=None,
    max_sleep=None,
    template_csr_pem=None,
    openssl=openssl_bin,
    mode='user',
    )}}
    {% do part_list.append('caucase-user-updater') -%}
    {% do part_list.append('caucase-user-updater-promise') -%}
    #caucase proxysql certificate
    {{ caucase.updater(
    prefix='caucase-proxysql-updater',
    buildout_bin_directory=buildout_bin_directory,
    updater_path='${directory:services}/caucase-proxysql-updater',
    url=caucase_url,
    data_dir='${proxysql-directory:ssl}',
    crt_path='${proxysql-directory:ssl}/client-cert.pem',
    ca_path='${proxysql-directory:ssl}/ca-cert.pem',
    crl_path='${proxysql-directory:ssl}/crl.pem',
    key_path='${proxysql-directory:ssl}/client-key.pem',
    on_renew=None,
    max_sleep=None,
    template_csr_pem=None,
    openssl=openssl_bin,
    )}}
    {% do part_list.append('caucase-proxysql-updater') -%}
    {% do part_list.append('caucase-proxysql-updater-promise') -%}
    [caucase-sign-csr]
    recipe = slapos.recipe.template:jinja2
    mode = 755
    rendered = ${directory:bin}/caucase-sign
    template =
    inline:#!/bin/sh
    cp ${directory:ssl}/caucase.user.key ${directory:ssl}/caucase-full.key
    cat ${directory:ssl}/caucase.user.crt >> ${directory:ssl}/caucase-full.key
    for csr_id in {{ csrid_list | join(' ') }}; do
    if [ "$csr_id" = "None" ] || [ -z "$csr_id"]; then
    continue
    fi
    {{ buildout_bin_directory }}/caucase --ca-url {{ caucase_url }} --user-key ${directory:ssl}/caucase-full.key --sign-csr $csr_id
    done
    [cron-caucase-sign-csr]
    recipe = slapos.cookbook:cron.d
    cron-entries = ${cron:cron-entries}
    name = caucase-sign-csr
    frequency = * * * * *
    command = ${caucase-sign-csr:rendered}
    {% do part_list.append('cron-caucase-sign-csr') -%}
    [slap-configuration] [slap-configuration]
    recipe = slapos.cookbook:slapconfiguration recipe = slapos.cookbook:slapconfiguration
    computer = ${slap-connection:computer-id} computer = ${slap-connection:computer-id}
    ...@@ -286,7 +446,6 @@ etc = ${directory:etc}/repman ...@@ -286,7 +446,6 @@ etc = ${directory:etc}/repman
    data-dir = ${directory:var}/lib data-dir = ${directory:var}/lib
    root-dir = ${directory:srv}/repman root-dir = ${directory:srv}/repman
    clusters = ${:etc}/cluster.d clusters = ${:etc}/cluster.d
    proxies = ${:etc}/proxy
    proxy-data = ${:data-dir}/proxy proxy-data = ${:data-dir}/proxy
    config-tmp = ${directory:tmp}/config config-tmp = ${directory:tmp}/config
    bootstrap = ${:etc}/bootstrap bootstrap = ${:etc}/bootstrap
    ...@@ -296,8 +455,9 @@ proxies-log = ${directory:log}/proxy ...@@ -296,8 +455,9 @@ proxies-log = ${directory:log}/proxy
    ipv6 = ${instance-parameter:ipv6-random} ipv6 = ${instance-parameter:ipv6-random}
    port = ${instance-parameter:nginx-port} port = ${instance-parameter:nginx-port}
    ssl-port = ${instance-parameter:nginx-ssl-port} ssl-port = ${instance-parameter:nginx-ssl-port}
    ssl-certificate = ${ca-nginx:cert-file} ssl-certificate = ${directory:ssl}/repman-cert.crt
    ssl-key = ${ca-nginx:key-file} ca-certificate = ${directory:srv}/caucase-updater/ca.crt
    ssl-key = ${directory:ssl}/repman-cert.key
    pid-file = ${directory:run}/nginx.pid pid-file = ${directory:run}/nginx.pid
    access-log = ${directory:log}/nginx_access.log access-log = ${directory:log}/nginx_access.log
    error-log = ${directory:log}/nginx_error.log error-log = ${directory:log}/nginx_error.log
    ...@@ -345,6 +505,8 @@ mail-to = {{ slapparameter_dict.get("mail-to", "") }} ...@@ -345,6 +505,8 @@ mail-to = {{ slapparameter_dict.get("mail-to", "") }}
    http-session-lifetime = {{ slapparameter_dict.get("http-session-lifetime", 86400) }} http-session-lifetime = {{ slapparameter_dict.get("http-session-lifetime", 86400) }}
    http-refresh-interval = {{ slapparameter_dict.get("http-refresh-interval", 4) }} http-refresh-interval = {{ slapparameter_dict.get("http-refresh-interval", 4) }}
    ssl-cert = ${directory:ssl}/repman-cert.crt
    ssl-key = ${directory:ssl}/repman-cert.key
    [repman-config-folder] [repman-config-folder]
    recipe = plone.recipe.command recipe = plone.recipe.command
    ...@@ -392,6 +554,9 @@ depends = ...@@ -392,6 +554,9 @@ depends =
    ${replication-manager-reload:recipe} ${replication-manager-reload:recipe}
    ${repman-listen-promise:recipe} ${repman-listen-promise:recipe}
    ${repman-listen-ssl-promise:recipe} ${repman-listen-ssl-promise:recipe}
    wait-for-files =
    ${directory:ssl}/repman-cert.crt
    ${directory:ssl}/repman-cert.key
    [repman-config.toml] [repman-config.toml]
    recipe = slapos.recipe.template:jinja2 recipe = slapos.recipe.template:jinja2
    ...@@ -428,11 +593,12 @@ command-line = ...@@ -428,11 +593,12 @@ command-line =
    {{ nginx_bin }} {{ nginx_bin }}
    -p ${directory:nginx-prefix} -p ${directory:nginx-prefix}
    -c ${nginx-conf:rendered} -c ${nginx-conf:rendered}
    wrapper-path = ${directory:bin}/nginx-start wrapper-path = ${directory:services}/nginx
    wait-for-files = wait-for-files =
    ${ca-directory:certs}/nginx.key ${directory:ssl}/repman-cert.crt
    ${ca-directory:certs}/nginx.crt ${directory:srv}/caucase-updater/ca.crt
    ${nginx-graceful-wrapper:rendered} ${directory:ssl}/repman-cert.key
    [nginx-graceful-wrapper] [nginx-graceful-wrapper]
    recipe = slapos.recipe.template:jinja2 recipe = slapos.recipe.template:jinja2
    ...@@ -442,20 +608,6 @@ rendered = ${directory:scripts}/nginx-graceful ...@@ -442,20 +608,6 @@ rendered = ${directory:scripts}/nginx-graceful
    context = context =
    mode = 755 mode = 755
    [ca-nginx]
    <= certificate-authority
    recipe = slapos.cookbook:certificate_authority.request
    cert-file = ${ca-directory:certs}/nginx.crt
    key-file = ${ca-directory:certs}/nginx.key
    executable = ${nginx-launcher:wrapper-path}
    wrapper = ${directory:bin}/ca-nginx
    [ca-nginx-service]
    recipe = slapos.cookbook:wrapper
    command-line = ${ca-nginx:wrapper}
    wrapper-path = ${directory:services}/nginx
    hash-existing-files = ${buildout:directory}/software_release/buildout.cfg
    [logrotate-entry-nginx] [logrotate-entry-nginx]
    <= logrotate-entry-base <= logrotate-entry-base
    name = nginx name = nginx
    ...@@ -468,6 +620,7 @@ recipe = slapos.cookbook:publish-early ...@@ -468,6 +620,7 @@ recipe = slapos.cookbook:publish-early
    monitor-password monitor-htpasswd:passwd monitor-password monitor-htpasswd:passwd
    db-root-password gen-root-password:passwd db-root-password gen-root-password:passwd
    repman-password repman-password:passwd repman-password repman-password:passwd
    database-list = {{ db_list }}
    [publish-connection-parameter] [publish-connection-parameter]
    <= monitor-publish <= monitor-publish
    ...@@ -476,6 +629,7 @@ recipe = slapos.cookbook:publish ...@@ -476,6 +629,7 @@ recipe = slapos.cookbook:publish
    backend-url = ${nginx-parameter:backend-ssl-url} backend-url = ${nginx-parameter:backend-ssl-url}
    url = ${repman-frontend:connection-secure_access} url = ${repman-frontend:connection-secure_access}
    username = ${repman-parameter:username} username = ${repman-parameter:username}
    database-list = {{ db_list }}
    {% for name, value in publish_dict.items() -%} {% for name, value in publish_dict.items() -%}
    {{ name }} = {{ value }} {{ name }} = {{ value }}
    {% endfor %} {% endfor %}
    ...@@ -545,6 +699,15 @@ command = ${template-proxysql-need-stop-start:rendered} ...@@ -545,6 +699,15 @@ command = ${template-proxysql-need-stop-start:rendered}
    # Deploy replication-manager instance # Deploy replication-manager instance
    # #
    ############################# #############################
    [slap_connection]
    # Kept for backward compatibility
    computer_id = ${slap-connection:computer-id}
    partition_id = ${slap-connection:partition-id}
    server_url = ${slap-connection:server-url}
    software_release_url = ${slap-connection:software-release-url}
    key_file = ${slap-connection:key-file}
    cert_file = ${slap-connection:cert-file}
    [buildout] [buildout]
    extends = extends =
    {{ template_monitor }} {{ template_monitor }}
    ...@@ -552,7 +715,7 @@ parts = ...@@ -552,7 +715,7 @@ parts =
    replication-manager replication-manager
    monitor-base monitor-base
    logrotate-entry-nginx logrotate-entry-nginx
    ca-nginx-service nginx-launcher
    publish-connection-parameter publish-connection-parameter
    repman-frontend-promise repman-frontend-promise
    repman-backend-promise repman-backend-promise
    ......
    software/repman/instance.cfg.in
    View file @ 17bf5b6a
    ...@@ -28,8 +28,10 @@ mode = 0644 ...@@ -28,8 +28,10 @@ mode = 0644
    extensions = jinja2.ext.do extensions = jinja2.ext.do
    rendered= ${buildout:directory}/${:_buildout_section_name_} rendered= ${buildout:directory}/${:_buildout_section_name_}
    supervisord-lib = {{ supervisord_lib }} supervisord-lib = {{ supervisord_lib }}
    causace-lib = {{ caucase_library }}
    import-list = import-list =
    file supervisord_lib :supervisord-lib file supervisord_lib :supervisord-lib
    file caucase :causace-lib
    context = context =
    key slapparameter_dict slap-configuration:configuration key slapparameter_dict slap-configuration:configuration
    key computer_id slap-configuration:computer key computer_id slap-configuration:computer
    ...@@ -48,6 +50,8 @@ context = ...@@ -48,6 +50,8 @@ context =
    raw bash_bin {{ bash_location }}/bin/bash raw bash_bin {{ bash_location }}/bin/bash
    raw jq_bin {{ jq_location }}/bin/jq raw jq_bin {{ jq_location }}/bin/jq
    raw curl_bin {{ curl_location }}/bin/curl raw curl_bin {{ curl_location }}/bin/curl
    raw openssl_bin {{ openssl_location }}/bin/openssl
    raw caucase_bin_client {{ caucase_bin_client }}
    ${:extra-context} ${:extra-context}
    extra-context = extra-context =
    ...@@ -56,6 +60,7 @@ extra-context = ...@@ -56,6 +60,7 @@ extra-context =
    <= jinja2-template-base <= jinja2-template-base
    template= {{ template_repman_cfg }} template= {{ template_repman_cfg }}
    extra-context = extra-context =
    key slave_instance_list slap-configuration:slave-instance-list
    raw gowork_bin {{ gowork_bin }} raw gowork_bin {{ gowork_bin }}
    raw haproxy_location {{ haproxy_location }} raw haproxy_location {{ haproxy_location }}
    raw nginx_bin {{ nginx_location }}/sbin/nginx raw nginx_bin {{ nginx_location }}/sbin/nginx
    ...@@ -71,6 +76,7 @@ extra-context = ...@@ -71,6 +76,7 @@ extra-context =
    raw proxysql_location {{ proxysql_location }} raw proxysql_location {{ proxysql_location }}
    raw template_repman_manager_sh {{ template_repman_manager_sh }} raw template_repman_manager_sh {{ template_repman_manager_sh }}
    raw template_proxy_need_stopstart {{ proxy_need_stop_start_template }} raw template_proxy_need_stopstart {{ proxy_need_stop_start_template }}
    raw template_publish_slave_information {{ template_publish_slave }}
    [template-mariadb-parameters] [template-mariadb-parameters]
    bash = {{ bash_location }} bash = {{ bash_location }}
    ...@@ -83,6 +89,7 @@ template-mariadb-init-root = {{ template_init_root_sql }} ...@@ -83,6 +89,7 @@ template-mariadb-init-root = {{ template_init_root_sql }}
    template-init-root-wrapper = {{ template_init_root_wrapper }} template-init-root-wrapper = {{ template_init_root_wrapper }}
    template-mysqld-wrapper = {{ template_mysqld_wrapper }} template-mysqld-wrapper = {{ template_mysqld_wrapper }}
    template-mysqld-need-start = {{ mysqld_start_template }} template-mysqld-need-start = {{ mysqld_start_template }}
    template-manage-db = {{ template_slave_db_manage }}
    link-binary = {{ dumps(mariadb_link_binary) }} link-binary = {{ dumps(mariadb_link_binary) }}
    check-computer-memory-binary = {{ bin_directory }}/check-computer-memory check-computer-memory-binary = {{ bin_directory }}/check-computer-memory
    bin-directory = {{ bin_directory }} bin-directory = {{ bin_directory }}
    ......
    software/repman/publish-database-slave-parameters.cfg.in 0 → 100644
    View file @ 17bf5b6a
    {% set part_list = [] -%}
    [slap_connection]
    # Kept for backward compatibility
    computer_id = ${slap-connection:computer-id}
    partition_id = ${slap-connection:partition-id}
    server_url = ${slap-connection:server-url}
    software_release_url = ${slap-connection:software-release-url}
    key_file = ${slap-connection:key-file}
    cert_file = ${slap-connection:cert-file}
    {% for _, database in slave_dict.items() -%}
    {% set slave_reference = database['slave_reference'] -%}
    {% do part_list.append('publish-slave-' ~ slave_reference) -%}
    [publish-slave-{{ slave_reference }}]
    recipe = slapos.cookbook:publish
    -slave-reference = {{ slave_reference }}
    {% if database.get('user') -%}
    url_v4 = {{ "mysql://" ~ database['user'] ~ ":" ~ database['password'] ~ "@" ~ ipv4 ~ ":" ~ proxy_port ~ '/' ~ database['name'] }}
    url = {{ "mysql://" ~ database['user'] ~ ":" ~ database['password'] ~ "@[" ~ ipv6 ~ "]:" ~ proxy_port ~ '/' ~ database['name'] }}
    {% else -%}
    url = {{ "mysql://" ~ ipv4 ~ ":" ~ proxy_port ~ database['name'] }}
    url_v6 = {{ "mysql://[" ~ ip ~ "]:" ~ proxy_port ~ database['name'] }}
    {% endif -%}
    {% endfor %}
    [buildout]
    eggs-directory = {{ eggs_directory }}
    develop-eggs-directory = {{ develop_eggs_directory }}
    offline = true
    parts =
    {% for part in part_list -%}
    {{ ' %s' % part }}
    {% endfor -%}
    software/repman/software.cfg
    View file @ 17bf5b6a
    ...@@ -18,9 +18,10 @@ extends = ...@@ -18,9 +18,10 @@ extends =
    ../../component/socat/buildout.cfg ../../component/socat/buildout.cfg
    ../../component/rsync/buildout.cfg ../../component/rsync/buildout.cfg
    ../../component/jq/buildout.cfg ../../component/jq/buildout.cfg
    ../../component/openssl/buildout.cfg
    ../../stack/supervisord/buildout.cfg ../../stack/supervisord/buildout.cfg
    ../../stack/monitor/buildout.cfg ../../stack/monitor/buildout.cfg
    ../neoppod/software-common.cfg ../../stack/caucase/buildout.cfg
    parts = parts =
    slapos-cookbook slapos-cookbook
    ...@@ -28,14 +29,43 @@ parts = ...@@ -28,14 +29,43 @@ parts =
    instance.cfg instance.cfg
    template-mariadb.cfg template-mariadb.cfg
    template-mysqld-wrapper template-mysqld-wrapper
    caucase-eggs
    gowork gowork
    [mariadb]
    location = ${mariadb-10.4:location}
    [python] [python]
    part = python3 part = python3
    [gowork] [template-mysqld-wrapper]
    # replication-manager does not build on golang 1.17 recipe = slapos.recipe.template:jinja2
    golang = ${golang1.16:location} rendered = ${buildout:parts-directory}/${:_buildout_section_name_}/mysqld.in
    mode = 644
    template =
    inline:{% raw %}#!/bin/sh -e
    basedir='${mariadb:location}'
    datadir='{{datadir}}'
    marker=$datadir/.slapos_initializing
    system=$datadir/.system
    [ -d "$datadir/mysql" ] && [ ! -f "$marker" ] || {
    find "$datadir/" -mindepth 1 ! \( -path $marker -o -path $system -o -path "*$system/*" \) -delete || mkdir "$datadir"
    touch "$marker"
    "$basedir/scripts/mysql_install_db" \
    --defaults-file='{{defaults_file}}' \
    --skip-name-resolve \
    --auth-root-authentication-method=normal \
    --basedir="$basedir" --plugin_dir="$basedir/lib/plugin" \
    --datadir="$datadir"
    rm "$marker"
    }
    {%- if environ is defined %}
    {%- for variable in environ.splitlines() %}
    {{ variable }} \
    {%- endfor %}
    {%- endif %}
    exec "$basedir/bin/mysqld" --defaults-file='{{defaults_file}}' "$@"
    {% endraw %}
    [instance.cfg] [instance.cfg]
    recipe = slapos.recipe.template:jinja2 recipe = slapos.recipe.template:jinja2
    ...@@ -45,6 +75,7 @@ mode = 0644 ...@@ -45,6 +75,7 @@ mode = 0644
    context = context =
    key bash_location bash:location key bash_location bash:location
    key bin_directory buildout:bin-directory key bin_directory buildout:bin-directory
    key caucase_library caucase-jinja2-library:target
    key config_toml_in config-toml.in:target key config_toml_in config-toml.in:target
    key config_cluster_toml_in config-cluster-toml.in:target key config_cluster_toml_in config-cluster-toml.in:target
    key coreutils_location coreutils:location key coreutils_location coreutils:location
    ...@@ -70,6 +101,7 @@ context = ...@@ -70,6 +101,7 @@ context =
    key groonga_mysql_normalizer_plugin_dir groonga-normalizer-mysql:groonga-plugin-dir key groonga_mysql_normalizer_plugin_dir groonga-normalizer-mysql:groonga-plugin-dir
    key nginx_conf_in nginx.conf.in:target key nginx_conf_in nginx.conf.in:target
    key nginx_location nginx:location key nginx_location nginx:location
    key openssl_location openssl:location
    key percona_toolkit_location percona-toolkit:location key percona_toolkit_location percona-toolkit:location
    key proxy_need_stop_start_template proxy-need-start-stop.sh.in:target key proxy_need_stop_start_template proxy-need-start-stop.sh.in:target
    key repman_src_location git.signal18.io_signal18_repman:location key repman_src_location git.signal18.io_signal18_repman:location
    ...@@ -87,10 +119,12 @@ context = ...@@ -87,10 +119,12 @@ context =
    key template_init_root_sql mariadb-init-root-sql:target key template_init_root_sql mariadb-init-root-sql:target
    key template_init_root_wrapper init-root-wrapper-in:target key template_init_root_wrapper init-root-wrapper-in:target
    key template_repman_cfg instance-repman.cfg:target key template_repman_cfg instance-repman.cfg:target
    key template_publish_slave template-publish-slave-information:target
    key template_slave_db_manage slave-db-manage.in:target
    key unixodbc_location unixodbc:location key unixodbc_location unixodbc:location
    key sysbench_location sysbench:location key sysbench_location sysbench:location
    key proxysql_location proxysql:location key proxysql_location proxysql:location
    raw caucase_bin_client ${buildout:bin-directory}/caucase
    [download-file] [download-file]
    recipe = slapos.recipe.build:download recipe = slapos.recipe.build:download
    ...@@ -126,6 +160,9 @@ link-binary = ...@@ -126,6 +160,9 @@ link-binary =
    [template-mariadb-initial-setup] [template-mariadb-initial-setup]
    <= download-file <= download-file
    [template-publish-slave-information]
    <= download-file
    [template-my-cnf] [template-my-cnf]
    <= download-file <= download-file
    ...@@ -146,3 +183,6 @@ link-binary = ...@@ -146,3 +183,6 @@ link-binary =
    [proxy-need-start-stop.sh.in] [proxy-need-start-stop.sh.in]
    <= download-file <= download-file
    [slave-db-manage.in]
    <= download-file
    software/repman/software.cfg.json
    View file @ 17bf5b6a
    ...@@ -10,6 +10,15 @@ ...@@ -10,6 +10,15 @@
    "request": "instance-repman-input-schema.json", "request": "instance-repman-input-schema.json",
    "response": "instance-repman-output-schema.json", "response": "instance-repman-output-schema.json",
    "index": 0 "index": 0
    },
    "default-slave": {
    "title": "Replication Manager Database",
    "description": "Replication Manager Database",
    "software-type": "default",
    "request": "instance-repman-slave-input-schema.json",
    "response": "instance-repman-output-schema.json",
    "shared": true,
    "index": 1
    } }
    } }
    } }
    software/repman/templates/cluster-config.toml.in
    View file @ 17bf5b6a
    ...@@ -40,6 +40,7 @@ scheduler-db-servers-physical-backup-cron = "0 {{ parameter_dict['physical-backu ...@@ -40,6 +40,7 @@ scheduler-db-servers-physical-backup-cron = "0 {{ parameter_dict['physical-backu
    backup-physical-type = "mariabackup" backup-physical-type = "mariabackup"
    backup-logical-type = "mysqldump" backup-logical-type = "mysqldump"
    scheduler-db-servers-receiver-ports= "{{ parameter_dict['receiver-port-list'] }}" scheduler-db-servers-receiver-ports= "{{ parameter_dict['receiver-port-list'] }}"
    #scheduler-db-servers-receiver-use-ssl ## set to true when ssl enabled on socat
    prov-proxy-cpu-cores = {{ parameter_dict['proxy-cpu-cores'] }} prov-proxy-cpu-cores = {{ parameter_dict['proxy-cpu-cores'] }}
    prov-proxy-memory = {{ parameter_dict['proxy-memory'] }} prov-proxy-memory = {{ parameter_dict['proxy-memory'] }}
    ...@@ -49,6 +50,7 @@ prov-db-memory = {{ parameter_dict['db-memory'] }} ...@@ -49,6 +50,7 @@ prov-db-memory = {{ parameter_dict['db-memory'] }}
    prov-db-memory-shared-pct = "{{ parameter_dict['db-memory-shared-pct'] }}" prov-db-memory-shared-pct = "{{ parameter_dict['db-memory-shared-pct'] }}"
    prov-db-memory-threaded-pct = "{{ parameter_dict['db-memory-threaded-pct'] }}" prov-db-memory-threaded-pct = "{{ parameter_dict['db-memory-threaded-pct'] }}"
    replication-use-ssl = true
    test-inject-traffic = true test-inject-traffic = true
    # failover # failover
    failover-mode = "{{ parameter_dict['failover-mode'] }}" failover-mode = "{{ parameter_dict['failover-mode'] }}"
    ......
    software/repman/templates/config.toml.in
    View file @ 17bf5b6a
    ...@@ -21,7 +21,9 @@ db-servers-binary-path = "{{ parameter_dict['mysql-bin-dir'] }}" ...@@ -21,7 +21,9 @@ db-servers-binary-path = "{{ parameter_dict['mysql-bin-dir'] }}"
    #db-servers-ignored-hosts = #db-servers-ignored-hosts =
    # Database hosts list to monitor, IP and port (optional), specified in the host:[port] format and separated by commas # Database hosts list to monitor, IP and port (optional), specified in the host:[port] format and separated by commas
    monitoring-address = "{{ parameter_dict['ipv4'] }}" monitoring-address = "{{ parameter_dict['ipv4'] }}"
    monitoring-wait-retry = 40 monitoring-wait-retry = 1200
    monitoring-ssl-cert = "{{ parameter_dict['ssl-cert'] }}"
    monitoring-ssl-key = "{{ parameter_dict['ssl-key'] }}"
    #haproxy = true #haproxy = true
    #haproxy-binary-path = "{{ parameter_dict['haproxy-bin'] }}" #haproxy-binary-path = "{{ parameter_dict['haproxy-bin'] }}"
    ...@@ -48,6 +50,7 @@ backup-restic-password = "{{ parameter_dict['password'] }}" ...@@ -48,6 +50,7 @@ backup-restic-password = "{{ parameter_dict['password'] }}"
    backup-mysqlclient-path = "{{ parameter_dict['mysqlclient-path'] }}" backup-mysqlclient-path = "{{ parameter_dict['mysqlclient-path'] }}"
    backup-mysqlbinlog-path = "{{ parameter_dict['mysqlbinlog-path'] }}" backup-mysqlbinlog-path = "{{ parameter_dict['mysqlbinlog-path'] }}"
    backup-mysqldump-path = "{{ parameter_dict['mysqldump-path'] }}" backup-mysqldump-path = "{{ parameter_dict['mysqldump-path'] }}"
    backup-mysqldump-options = "--hex-blob --single-transaction --verbose --all-databases --add-drop-database"
    # Mail configuration # Mail configuration
    # Alert email sender (default "mrm@localhost") # Alert email sender (default "mrm@localhost")
    ......
    software/repman/templates/dbjobs.in
    View file @ 17bf5b6a
    #!/bin/bash #!/bin/bash
    USER={{ parameter_dict['db-user'] }}
    DBUSER={{ parameter_dict['db-user'] }}
    PASSWORD={{ parameter_dict['db-password'] }} PASSWORD={{ parameter_dict['db-password'] }}
    ERROLOG={{ parameter_dict['mysql-dir'] }}/.system/logs/errors.log ERROLOG={{ parameter_dict['mysql-dir'] }}/.system/logs/errors.log
    SLOWLOG={{ parameter_dict['mysql-dir']}}/.system/logs/sql-slow SLOWLOG={{ parameter_dict['mysql-dir']}}/.system/logs/sql-slow
    BACKUPDIR={{ parameter_dict['mysql-dir'] }}/.system/backup BACKUPDIR={{ parameter_dict['mysql-dir'] }}/.system/backup
    DATADIR={{ parameter_dict['mysql-dir'] }}/ DATADIR={{ parameter_dict['mysql-dir'] }}/
    MYSQL_CLIENT={{ parameter_dict['mysql-client'] }}
    {% if parameter_dict['use-ipv6'] == True -%} {% if parameter_dict['use-ipv6'] == True -%}
    {% set listen = "TCP6-LISTEN" -%} {% set listen = "TCP6-LISTEN" -%}
    {% else -%} {% else -%}
    ...@@ -16,45 +18,45 @@ JOBS=( "xtrabackup" "mariabackup" "error" "slowquery" "zfssnapback" "optimize" " ...@@ -16,45 +18,45 @@ JOBS=( "xtrabackup" "mariabackup" "error" "slowquery" "zfssnapback" "optimize" "
    doneJob() doneJob()
    { {
    mysql --defaults-file={{ parameter_dict['dbjob-cnf'] }} -e "set sql_log_bin=0;UPDATE replication_manager_schema.jobs set end=NOW(), result=LOAD_FILE('{{ parameter_dict['log-dir'] }}/dbjob.out') WHERE id='$ID';" & $MYSQL_CLIENT -e "set sql_log_bin=0;UPDATE replication_manager_schema.jobs set end=NOW(), result=LOAD_FILE('{{ parameter_dict['log-dir'] }}/dbjob.out') WHERE id='$ID';" &
    } }
    pauseJob() pauseJob()
    { {
    mysql --defaults-file={{ parameter_dict['dbjob-cnf'] }} -e "select sleep(6);set sql_log_bin=0;UPDATE replication_manager_schema.jobs set result=LOAD_FILE('{{ parameter_dict['log-dir'] }}/dbjob.out') WHERE id='$ID';" & $MYSQL_CLIENT -e "select sleep(6);set sql_log_bin=0;UPDATE replication_manager_schema.jobs set result=LOAD_FILE('{{ parameter_dict['log-dir'] }}/dbjob.out') WHERE id='$ID';" &
    } }
    partialRestore() partialRestore()
    { {
    mysql --defaults-file={{ parameter_dict['dbjob-cnf'] }} -e "set sql_log_bin=0;install plugin BLACKHOLE soname 'ha_blackhole.so'" $MYSQL_CLIENT -e "set sql_log_bin=0;install plugin BLACKHOLE soname 'ha_blackhole.so'"
    for dir in $(ls -d $BACKUPDIR/*/ | xargs -n 1 basename | grep -vE 'mysql|performance_schema|replication_manager_schema') ; do for dir in $(ls -d $BACKUPDIR/*/ | xargs -n 1 basename | grep -vE 'mysql|performance_schema|replication_manager_schema') ; do
    mysql --defaults-file={{ parameter_dict['dbjob-cnf'] }} -e "set sql_log_bin=0;drop database IF EXISTS $dir; CREATE DATABASE $dir;" $MYSQL_CLIENT -e "set sql_log_bin=0;drop database IF EXISTS $dir; CREATE DATABASE $dir;"
    for file in $(find $BACKUPDIR/$dir/ -name "*.exp" | xargs -n 1 basename | cut -d'.' --complement -f2-) ; do for file in $(find $BACKUPDIR/$dir/ -name "*.exp" | xargs -n 1 basename | cut -d'.' --complement -f2-) ; do
    cat $BACKUPDIR/$dir/$file.frm | sed -e 's/\x06\x00\x49\x6E\x6E\x6F\x44\x42\x00\x00\x00/\x09\x00\x42\x4C\x41\x43\x4B\x48\x4F\x4C\x45/g' > $DATADIR/$dir/mrm_pivo.frm cat $BACKUPDIR/$dir/$file.frm | sed -e 's/\x06\x00\x49\x6E\x6E\x6F\x44\x42\x00\x00\x00/\x09\x00\x42\x4C\x41\x43\x4B\x48\x4F\x4C\x45/g' > $DATADIR/$dir/mrm_pivo.frm
    mysql --defaults-file={{ parameter_dict['dbjob-cnf'] }} -e "set sql_log_bin=0;ALTER TABLE $dir.mrm_pivo engine=innodb;RENAME TABLE $dir.mrm_pivo TO $dir.$file; ALTER TABLE $dir.$file DISCARD TABLESPACE;" $MYSQL_CLIENT -e "set sql_log_bin=0;ALTER TABLE $dir.mrm_pivo engine=innodb;RENAME TABLE $dir.mrm_pivo TO $dir.$file; ALTER TABLE $dir.$file DISCARD TABLESPACE;"
    mv $BACKUPDIR/$dir/$file.ibd $DATADIR/$dir/$file.ibd mv $BACKUPDIR/$dir/$file.ibd $DATADIR/$dir/$file.ibd
    mv $BACKUPDIR/$dir/$file.exp $DATADIR/$dir/$file.exp mv $BACKUPDIR/$dir/$file.exp $DATADIR/$dir/$file.exp
    mv $BACKUPDIR/$dir/$file.cfg $DATADIR/$dir/$file.cfg mv $BACKUPDIR/$dir/$file.cfg $DATADIR/$dir/$file.cfg
    mv $BACKUPDIR/$dir/$file.TRG $DATADIR/$dir/$file.TRG mv $BACKUPDIR/$dir/$file.TRG $DATADIR/$dir/$file.TRG
    mysql --defaults-file={{ parameter_dict['dbjob-cnf'] }} -e "set sql_log_bin=0;ALTER TABLE $dir.$file IMPORT TABLESPACE" $MYSQL_CLIENT -e "set sql_log_bin=0;ALTER TABLE $dir.$file IMPORT TABLESPACE"
    done done
    for file in $(find $BACKUPDIR/$dir/ -name "*.MYD" | xargs -n 1 basename | cut -d'.' --complement -f2-) ; do for file in $(find $BACKUPDIR/$dir/ -name "*.MYD" | xargs -n 1 basename | cut -d'.' --complement -f2-) ; do
    mv $BACKUPDIR/$dir/$file.* $DATADIR/$dir/ mv $BACKUPDIR/$dir/$file.* $DATADIR/$dir/
    mysql --defaults-file=/etc/mysql/dbjob.cnf -e "set sql_log_bin=0;FLUSH TABLE $dir.$file" $MYSQL_CLIENT -e "set sql_log_bin=0;FLUSH TABLE $dir.$file"
    done done
    for file in $(find $BACKUPDIR/$dir/ -name "*.CSV" | xargs -n 1 basename | cut -d'.' --complement -f2-) ; do for file in $(find $BACKUPDIR/$dir/ -name "*.CSV" | xargs -n 1 basename | cut -d'.' --complement -f2-) ; do
    mv $BACKUPDIR/$dir/$file.* $DATADIR/$dir/ mv $BACKUPDIR/$dir/$file.* $DATADIR/$dir/
    mysql --defaults-file={{ parameter_dict['dbjob-cnf'] }} -e "set sql_log_bin=0;FLUSH TABLE $dir.$file" $MYSQL_CLIENT -e "set sql_log_bin=0;FLUSH TABLE $dir.$file"
    done done
    done done
    for file in $(find $BACKUPDIR/mysql/ -name "*.MYD" | xargs -n 1 basename | cut -d'.' --complement -f2-) ; do for file in $(find $BACKUPDIR/mysql/ -name "*.MYD" | xargs -n 1 basename | cut -d'.' --complement -f2-) ; do
    mv $BACKUPDIR/mysql/$file.* $DATADIR/mysql/ mv $BACKUPDIR/mysql/$file.* $DATADIR/mysql/
    mysql --defaults-file={{ parameter_dict['dbjob-cnf'] }} -e "set sql_log_bin=0;FLUSH TABLE mysql.$file" $MYSQL_CLIENT -e "set sql_log_bin=0;FLUSH TABLE mysql.$file"
    done done
    cat $BACKUPDIR/xtrabackup_info | grep binlog_pos | awk -F, '{ print $3 }' | sed -e 's/GTID of the last change/set sql_log_bin=0;set global gtid_slave_pos=/g' | mysql -h{{ parameter_dict['ip'] }} -P{{ parameter_dict['port'] }} -p$PASSWORD -u$USER cat $BACKUPDIR/xtrabackup_info | grep binlog_pos | awk -F, '{ print $3 }' | sed -e 's/GTID of the last change/set sql_log_bin=0;set global gtid_slave_pos=/g' | mysql -h{{ parameter_dict['ip'] }} -P{{ parameter_dict['port'] }} -p$PASSWORD -u$DBUSER
    mysql --defaults-file={{ parameter_dict['dbjob-cnf'] }} -e"flush privileges;start slave;" $MYSQL_CLIENT -e "flush privileges;start slave;"
    } }
    kill -9 $(lsof -t -i:{{ parameter_dict['socat-port'] }} -sTCP:LISTEN) kill -9 $(lsof -t -i:{{ parameter_dict['socat-port'] }} -sTCP:LISTEN)
    ...@@ -62,12 +64,12 @@ kill -9 $(lsof -t -i:{{ parameter_dict['socat-port'] }} -sTCP:LISTEN) ...@@ -62,12 +64,12 @@ kill -9 $(lsof -t -i:{{ parameter_dict['socat-port'] }} -sTCP:LISTEN)
    for job in "${JOBS[@]}" for job in "${JOBS[@]}"
    do do
    TASK=($(echo "select concat(id,'@',server,':',port) from replication_manager_schema.jobs WHERE task='$job' and done=0 order by task desc limit 1" | mysql -h{{ parameter_dict['ip'] }} -P{{ parameter_dict['port'] }} -p$PASSWORD -u$USER -N)) TASK=($(echo "select concat(id,'@',server,':',port) from replication_manager_schema.jobs WHERE task='$job' and done=0 order by task desc limit 1" | mysql -h{{ parameter_dict['ip'] }} -P{{ parameter_dict['port'] }} -p$PASSWORD -u$DBUSER -N))
    ADDRESS=($(echo $TASK | awk -F@ '{ print $2 }')) ADDRESS=($(echo $TASK | awk -F@ '{ print $2 }'))
    ID=($(echo $TASK | awk -F@ '{ print $1 }')) ID=($(echo $TASK | awk -F@ '{ print $1 }'))
    #purge de past #purge de past
    mysql --defaults-file={{ parameter_dict['dbjob-cnf'] }} -e "set sql_log_bin=0;UPDATE replication_manager_schema.jobs set done=1 WHERE done=0 AND task='$job';" $MYSQL_CLIENT -e "set sql_log_bin=0;UPDATE replication_manager_schema.jobs set done=1 WHERE done=0 AND task='$job';"
    if [ "$ADDRESS" == "" ]; then if [ "$ADDRESS" == "" ]; then
    echo "No $job needed" echo "No $job needed"
    ...@@ -77,14 +79,14 @@ do ...@@ -77,14 +79,14 @@ do
    reseedmysqldump) reseedmysqldump)
    echo "Waiting backup." > {{ parameter_dict['log-dir'] }}/dbjob.out echo "Waiting backup." > {{ parameter_dict['log-dir'] }}/dbjob.out
    pauseJob pauseJob
    socat -u {{ listen }}:{{ parameter_dict['socat-port'] }},bind={{ parameter_dict['host'] }},reuseaddr STDOUT | gunzip | mysql -h{{ parameter_dict['ip'] }} -P{{ parameter_dict['port'] }} -p$PASSWORD -u$USER --init-command="reset master;set sql_log_bin=0" > {{ parameter_dict['log-dir'] }}/dbjob.out 2>&1 socat -u {{ listen }}:{{ parameter_dict['socat-port'] }},bind={{ parameter_dict['host'] }},reuseaddr STDOUT | gunzip | mysql -h{{ parameter_dict['ip'] }} -P{{ parameter_dict['port'] }} -p$PASSWORD -u$DBUSER --init-command="reset master;set sql_log_bin=0" &>{{ parameter_dict['log-dir'] }}/dbjob.out
    mysql --defaults-file={{ parameter_dict['dbjob-cnf'] }} -e 'start slave;' $MYSQL_CLIENT -e 'start slave;'
    ;; ;;
    flashbackmysqldump) flashbackmysqldump)
    echo "Waiting backup." > {{ parameter_dict['log-dir'] }}/dbjob.out echo "Waiting backup." > {{ parameter_dict['log-dir'] }}/dbjob.out
    pauseJob pauseJob
    socat -u {{ listen }}:{{ parameter_dict['socat-port'] }},bind={{ parameter_dict['host'] }},reuseaddr STDOUT | gunzip | mysql -h{{ parameter_dict['ip'] }} -P{{ parameter_dict['port'] }} -p$PASSWORD -u$USER --init-command="set sql_log_bin=0" > {{ parameter_dict['log-dir'] }}/dbjob.out 2>&1 socat -u {{ listen }}:{{ parameter_dict['socat-port'] }},bind={{ parameter_dict['host'] }},reuseaddr STDOUT | gunzip | mysql -h{{ parameter_dict['ip'] }} -P{{ parameter_dict['port'] }} -p$PASSWORD -u$DBUSER &>{{ parameter_dict['log-dir'] }}/dbjob.out
    mysql --defaults-file={{ parameter_dict['dbjob-cnf'] }} -e 'start slave;' $MYSQL_CLIENT -e 'start slave;'
    ;; ;;
    reseedmariabackup) reseedmariabackup)
    rm -rf $BACKUPDIR rm -rf $BACKUPDIR
    ...@@ -107,7 +109,7 @@ do ...@@ -107,7 +109,7 @@ do
    ;; ;;
    mariabackup) mariabackup)
    cd {{ parameter_dict['tmp-dir'] }} cd {{ parameter_dict['tmp-dir'] }}
    mariadb-backup --innobackupex --defaults-file={{ parameter_dict['dbjob-cnf'] }} --socket='{{ parameter_dict["mysqld-socket"] }}' --no-version-check --user=$USER --password=$PASSWORD --stream=xbstream {{ parameter_dict['tmp-dir'] }}/ | socat -u stdio TCP:$ADDRESS &>{{ parameter_dict['log-dir'] }}/dbjob.out mariadb-backup --innobackupex --defaults-file={{ parameter_dict['dbjob-cnf'] }} --socket='{{ parameter_dict["mysqld-socket"] }}' --no-version-check --user=root --stream=xbstream {{ parameter_dict['tmp-dir'] }}/ | socat -u stdio TCP:$ADDRESS &>{{ parameter_dict['log-dir'] }}/dbjob.out
    ;; ;;
    error) error)
    cat $ERROLOG| socat -u stdio TCP:$ADDRESS &>{{ parameter_dict['log-dir'] }}/dbjob.out cat $ERROLOG| socat -u stdio TCP:$ADDRESS &>{{ parameter_dict['log-dir'] }}/dbjob.out
    ...@@ -116,10 +118,10 @@ do ...@@ -116,10 +118,10 @@ do
    cat $SLOWLOG| socat -u stdio TCP:$ADDRESS &>{{ parameter_dict['log-dir'] }}/dbjob.out cat $SLOWLOG| socat -u stdio TCP:$ADDRESS &>{{ parameter_dict['log-dir'] }}/dbjob.out
    ;; ;;
    optimize) optimize)
    mysqlcheck --defaults-file={{ parameter_dict['dbjob-cnf'] }} -o --all-databases --skip-write-binlog &>{{ parameter_dict['log-dir'] }}/dbjob.out $MYSQL_CLIENT -o --all-databases --skip-write-binlog &>{{ parameter_dict['log-dir'] }}/dbjob.out
    ;; ;;
    restart) restart)
    {{ parameter_dict['restart-script'] }} > {{ parameter_dict['log-dir'] }}/dbjob.out {{ parameter_dict['restart-script'] }} &>{{ parameter_dict['log-dir'] }}/dbjob.out
    ;; ;;
    esac esac
    doneJob doneJob
    ......
    software/repman/templates/init_root_wrapper.in
    View file @ 17bf5b6a
    ...@@ -2,7 +2,7 @@ ...@@ -2,7 +2,7 @@
    run_mysql () { run_mysql () {
    {{ mysql_bin }} --defaults-file="{{ mysql_conf }}" \ {{ mysql_bin }} --defaults-file="{{ mysql_conf }}" \
    --protocol=socket -uroot -hlocalhost $@ --protocol=socket -hlocalhost "$@"
    } }
    if [ ! -f "{{ init_password_done }}" ]; then if [ ! -f "{{ init_password_done }}" ]; then
    ...@@ -19,6 +19,9 @@ if [ ! -f "{{ init_password_done }}" ]; then ...@@ -19,6 +19,9 @@ if [ ! -f "{{ init_password_done }}" ]; then
    fi fi
    echo "Setting mariabdb root password..."; echo "Setting mariabdb root password...";
    run_mysql < {{ init_root_sql }} && touch {{ init_password_done }} || exit 1; run_mysql < {{ init_root_sql }} && touch {{ init_password_done }} || exit 1;
    echo "remove unwanted users...";
    run_mysql -e "DROP USER IF EXISTS 'root'@'$HOSTNAME';"
    run_mysql -e "DROP USER IF EXISTS ''@'$HOSTNAME';"
    echo "done" echo "done"
    fi fi
    ...@@ -29,4 +32,7 @@ if [ ! -f "{{ upgrade_done }}" ]; then ...@@ -29,4 +32,7 @@ if [ ! -f "{{ upgrade_done }}" ]; then
    if [ $? -eq 0 ]; then if [ $? -eq 0 ]; then
    touch {{ upgrade_done }}; touch {{ upgrade_done }};
    fi fi
    else
    # only create databases now
    run_mysql < {{ init_database_sql }} || exit 1;
    fi fi
    software/repman/templates/mariadb_init_root.sql.in
    View file @ 17bf5b6a
    ...@@ -2,16 +2,18 @@ ...@@ -2,16 +2,18 @@
    -- or products like mysql-fabric won't work -- or products like mysql-fabric won't work
    SET @@SESSION.SQL_LOG_BIN=0; SET @@SESSION.SQL_LOG_BIN=0;
    # {{ parameter_dict["require-ssl"] }}
    {% set ssl = parameter_dict["require-ssl"] -%}
    CREATE USER '{{ parameter_dict["root-user"] }}'@'localhost' IDENTIFIED BY '{{ parameter_dict["password"] }}' ; CREATE USER '{{ parameter_dict["root-user"] }}'@'localhost' IDENTIFIED BY '{{ parameter_dict["password"] }}' ;
    GRANT ALL ON *.* TO '{{ parameter_dict["root-user"] }}'@'localhost' WITH GRANT OPTION ; GRANT ALL ON *.* TO '{{ parameter_dict["root-user"] }}'@'localhost' WITH GRANT OPTION ;
    CREATE USER '{{ parameter_dict["root-user"] }}'@'%' IDENTIFIED BY '{{ parameter_dict["password"] }}' ; CREATE USER '{{ parameter_dict["root-user"] }}'@'%' IDENTIFIED BY '{{ parameter_dict["password"] }}' {% if ssl %} REQUIRE SSL{% endif %};
    GRANT ALL ON *.* TO '{{ parameter_dict["root-user"] }}'@'%' WITH GRANT OPTION ; GRANT ALL ON *.* TO '{{ parameter_dict["root-user"] }}'@'%' WITH GRANT OPTION ;
    CREATE USER '{{ parameter_dict["root-user"] }}'@'::' IDENTIFIED BY '{{ parameter_dict["password"] }}' ; CREATE USER '{{ parameter_dict["root-user"] }}'@'::' IDENTIFIED BY '{{ parameter_dict["password"] }}' ;
    GRANT ALL ON *.* TO '{{ parameter_dict["root-user"] }}'@'::' WITH GRANT OPTION ; GRANT ALL ON *.* TO '{{ parameter_dict["root-user"] }}'@'::' WITH GRANT OPTION ;
    CREATE USER '{{ parameter_dict["heartbeat-user"] }}'@'localhost' IDENTIFIED BY '{{ parameter_dict["password"] }}' ; CREATE USER '{{ parameter_dict["heartbeat-user"] }}'@'localhost' IDENTIFIED BY '{{ parameter_dict["password"] }}' ;
    GRANT ALL ON *.* TO '{{ parameter_dict["heartbeat-user"] }}'@'localhost' WITH GRANT OPTION ; GRANT ALL ON *.* TO '{{ parameter_dict["heartbeat-user"] }}'@'localhost' WITH GRANT OPTION ;
    CREATE USER '{{ parameter_dict["heartbeat-user"] }}'@'%' IDENTIFIED BY '{{ parameter_dict["password"] }}' ; CREATE USER '{{ parameter_dict["heartbeat-user"] }}'@'%' IDENTIFIED BY '{{ parameter_dict["password"] }}' {% if ssl %} REQUIRE SSL{% endif %};
    GRANT ALL ON *.* TO '{{ parameter_dict["heartbeat-user"] }}'@'%' WITH GRANT OPTION ; GRANT ALL ON *.* TO '{{ parameter_dict["heartbeat-user"] }}'@'%' WITH GRANT OPTION ;
    DROP DATABASE IF EXISTS test ; DROP DATABASE IF EXISTS test ;
    ......
    software/repman/templates/mariadb_initial_setup.sql.in
    View file @ 17bf5b6a
    SET @@SESSION.SQL_LOG_BIN=0; SET @@SESSION.SQL_LOG_BIN=0;
    USE mysql; USE mysql;
    {% set mroonga = parameter_dict.get('mroonga', 'ha_mroonga.so') -%}
    {% if mroonga %}
    SOURCE {{ parameter_dict['mroonga-mariadb-install-sql'] }};
    {% endif %}
    DROP FUNCTION IF EXISTS sphinx_snippets; DROP FUNCTION IF EXISTS sphinx_snippets;
    #CREATE FUNCTION sphinx_snippets RETURNS STRING SONAME 'ha_sphinx.so'; #CREATE FUNCTION sphinx_snippets RETURNS STRING SONAME 'ha_sphinx.so';
    {% macro database(name, user, password) -%} CREATE DATABASE IF NOT EXISTS `repman_slave_definition`;
    CREATE DATABASE IF NOT EXISTS `{{ name }}`;
    {% macro database(name, user, password, ssl, charset="") -%}
    {% set charset_collate = "" -%}
    {% set charset_dict = {'utf8_general_ci': 'utf8',
    'utf8_unicode_ci': 'utf8',
    'utf8mb4_bin': 'utf8mb4',
    'utf8mb4_general_ci': 'utf8mb4',
    'utf8mb4_unicode_ci': 'utf8mb4',
    'latin1_general_ci': 'latin1',
    'latin1_bin': 'latin1',
    'ascii_general_ci': 'ascii',
    'ascii_bin': 'ascii'} -%}
    {% if charset %}
    {% set charset_collate = "DEFAULT CHARACTER SET %s" % charset -%}
    {% if charset_dict.get(charset) -%}
    {% set charset_collate = "DEFAULT CHARACTER SET %s COLLATE %s" % (charset_dict[charset], charset) -%}
    {% endif -%}
    {% endif -%}
    CREATE DATABASE IF NOT EXISTS `{{ name }}` {{ charset_collate }};
    {% if user -%} {% if user -%}
    GRANT ALL PRIVILEGES ON `{{ name }}`.* TO `{{ user }}`@`%` IDENTIFIED BY '{{ password }}'; GRANT ALL PRIVILEGES ON `{{ name }}`.* TO `{{ user }}`@`%` IDENTIFIED BY '{{ password }}'{% if ssl %} REQUIRE SSL{% endif %};
    GRANT ALL PRIVILEGES ON `{{ name }}`.* TO `{{ user }}`@localhost IDENTIFIED BY '{{ password }}'; GRANT ALL PRIVILEGES ON `{{ name }}`.* TO `{{ user }}`@localhost IDENTIFIED BY '{{ password }}';
    GRANT ALL PRIVILEGES ON `{{ name }}`.* TO `{{ user }}`@'::' IDENTIFIED BY '{{ password }}'; GRANT ALL PRIVILEGES ON `{{ name }}`.* TO `{{ user }}`@'::' IDENTIFIED BY '{{ password }}';
    {%- endif %} {%- endif %}
    {% endmacro -%} {% endmacro -%}
    {% for entry in parameter_dict['database-list'] -%} {% for entry in parameter_dict['database-list'] -%}
    {{ database(entry['name'], entry.get('user'), entry.get('password')) }} {{ database(entry['name'], entry.get('user'), entry.get('password'), entry.get('require_ssl', True) ~ '', entry.get('charset')) }}
    {% endfor -%} {% endfor -%}
    software/repman/templates/mysqld-need-start.sh.in
    View file @ 17bf5b6a
    ...@@ -4,13 +4,11 @@ curl () { ...@@ -4,13 +4,11 @@ curl () {
    {{ curl_bin }} -k --silent -H "Accept: application/json" "$@" {{ curl_bin }} -k --silent -H "Accept: application/json" "$@"
    } }
    # TOKEN=$(curl -s -X POST --data '{"username":"{{ username }}","password":"XXXXX"}' {{ repman_url }}/api/login | {{ jq_bin }} -r '.token')
    # Checking if mariadb start is needed # Checking if mariadb start is needed
    #CODE=$(curl -H "Authorization: Bearer ${TOKEN}" -o /dev/null -w "%{http_code}" {{ repman_url }}/api/clusters/{{ cluster }}/servers/{{ db_host }}/{{ db_port }}/need-start)
    CODE=$(curl -o /dev/null -w "%{http_code}" {{ repman_url }}/api/clusters/{{ cluster }}/servers/{{ db_host }}/{{ db_port }}/need-start) CODE=$(curl -o /dev/null -w "%{http_code}" {{ repman_url }}/api/clusters/{{ cluster }}/servers/{{ db_host }}/{{ db_port }}/need-start)
    if [ $CODE -eq 200 ]; then if [ $CODE -eq 200 ]; then
    sleep 5
    echo "$CODE: Updating mysql configuration..." echo "$CODE: Updating mysql configuration..."
    # update mysql configuration # update mysql configuration
    {{ update_config }} {{ update_config }}
    ...@@ -21,4 +19,6 @@ if [ $CODE -eq 200 ]; then ...@@ -21,4 +19,6 @@ if [ $CODE -eq 200 ]; then
    sleep 5 sleep 5
    # check again if the service is still up... # check again if the service is still up...
    {{ mariadb_controller }} status mariadb {{ mariadb_controller }} status mariadb
    else
    echo "CODE: $CODE. No need start!"
    fi fi
    software/repman/templates/slave-db-manage.in 0 → 100644
    View file @ 17bf5b6a
    #!/bin/bash
    set -e
    curl () {
    {{ curl_bin }} -k --silent -H "Accept: application/json" "$@"
    }
    get_token () {
    curl -s -X POST --data '{"username":"{{ username }}","password":"{{ password}}"}' {{ secure_url }}/api/login
    }
    run_mysql () {
    {{ mysql_bin }} --defaults-file="{{ mysql_conf }}" --protocol=socket "$@"
    }
    TOKEN=$(get_token | {{ jq_bin }} -r '.token')
    DATADIR=$(curl -H "Authorization: Bearer ${TOKEN}" \ {{ secure_url }}/api/clusters/{{ cluster_name }}/topology/master | {{ jq_bin }} -r '.slaposDatadir')
    # Only write or delete on master database else, we break replication.
    if [ "$DATADIR" = "{{ partition_dir }}" ]; then
    cat << EOF > {{ tmp_dir }}/.script.sql
    use repman_slave_definition;
    CREATE TABLE IF NOT EXISTS \`slave\` (
    \`name\` varchar(80) NOT NULL,
    \`state\` tinyint(1) DEFAULT NULL,
    PRIMARY KEY (\`name\`)
    ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4;
    UPDATE \`slave\` set \`state\`=false;
    {% for name in database_name.split(' ') -%}
    {% if name -%}
    REPLACE INTO \`slave\` VALUES ('{{ name }}', true);
    {% endif -%}
    {% endfor -%}
    EOF
    # Update requested slaves database
    run_mysql < {{ tmp_dir }}/.script.sql
    rm -f {{ tmp_dir }}/.script.sql
    DBNAME=$(run_mysql --skip-column-names -Be "SELECT name FROM repman_slave_definition.slave WHERE state=false");
    RET=$?
    if [ ! "$RET" = "0" ]; then
    echo "Mysql command failed: $DBNAME"
    exit $RET
    fi
    if [ -z "$DBNAME" ]; then
    echo "No database for slave to remove.";
    fi
    for NAME in $DBNAME; do
    if [ ! -z "$NAME" ]; then
    echo "Deleting database $NAME..."
    run_mysql -e "DROP DATABASE IF EXISTS $NAME";
    run_mysql -e "DELETE FROM repman_slave_definition.slave WHERE name='$NAME'";
    echo "Done."
    fi
    done
    fi
    software/repman/test/test.py
    View file @ 17bf5b6a
    ...@@ -62,12 +62,6 @@ class TestRepman(SlapOSInstanceTestCase): ...@@ -62,12 +62,6 @@ class TestRepman(SlapOSInstanceTestCase):
    token = resp.json()['token'] token = resp.json()['token']
    headers = {"authorization": "Bearer " + token} headers = {"authorization": "Bearer " + token}
    resp = requests.get(
    urljoin(self.url, '/api/monitor'),
    headers=headers,
    verify=False,
    )
    self.assertEqual(resp.status_code, requests.codes.ok)
    for i in range(20): for i in range(20):
    resp = requests.get( resp = requests.get(
    ...@@ -87,3 +81,11 @@ class TestRepman(SlapOSInstanceTestCase): ...@@ -87,3 +81,11 @@ class TestRepman(SlapOSInstanceTestCase):
    self.assertTrue(cluster['isProvision']) self.assertTrue(cluster['isProvision'])
    self.assertTrue(cluster['isFailable']) self.assertTrue(cluster['isFailable'])
    self.assertFalse(cluster['isDown']) self.assertFalse(cluster['isDown'])
    resp = requests.get(
    urljoin(self.url, '/api/clusters/cluster1/status'),
    headers=headers,
    verify=False,
    )
    self.assertEqual(resp.status_code, requests.codes.ok)
    self.assertEqual(resp.json(), {"alive": "running"})
    Markdown is supported
    0% or
    You are about to add 0 people to the discussion. Proceed with caution.
    Finish editing this message first!
    Please register or to comment
    GitLab Nexedi Edition | About GitLab | About Nexedi | 沪ICP备2021021310号-2 | 沪ICP备2021021310号-7