Skip to content

slapos
slapos
Commit c5d93b74 authored by Alain Takoudjou's avatar Alain Takoudjou
Browse files
Options

Replication manager: Correctly handle request database slaves and multi clusters 

See merge request !1209
parents 65bfd447 4ded1e20
Pipeline #22834 failed with stage
    in 0 seconds
    Hide whitespace changes
    Inline Side-by-side
    Showing with 135 additions and 59 deletions
    component/replication-manager/buildout.cfg
    View file @ c5d93b74
    ...@@ -30,4 +30,4 @@ go.importpath = github.com/signal18/replication-manager ...@@ -30,4 +30,4 @@ go.importpath = github.com/signal18/replication-manager
    repository = https://github.com/signal18/replication-manager repository = https://github.com/signal18/replication-manager
    #branch = 2.1 #branch = 2.1
    branch = develop branch = develop
    revision = 774fde2ce67942a31b99200fea6301189344ecd9 revision = ecd7fa29cb517b292e70ca557757882cd748016e
    \ No newline at end of file \ No newline at end of file
    software/repman/buildout.hash.cfg
    View file @ c5d93b74
    ...@@ -18,7 +18,7 @@ md5sum = 55c7fd4dd6a39b31878889fbfb00f995 ...@@ -18,7 +18,7 @@ md5sum = 55c7fd4dd6a39b31878889fbfb00f995
    [instance-repman.cfg] [instance-repman.cfg]
    _update_hash_filename_ = instance-repman.cfg.jinja2.in _update_hash_filename_ = instance-repman.cfg.jinja2.in
    md5sum = 4ce8808677773a033f4b59bdf6b8e653 md5sum = 6fc82c4b645bceb97ba7b6f5b56d161b
    [config-toml.in] [config-toml.in]
    _update_hash_filename_ = templates/config.toml.in _update_hash_filename_ = templates/config.toml.in
    ...@@ -34,7 +34,7 @@ md5sum = 0eeb24c6aa0760f0d33c4cc2828ddf30 ...@@ -34,7 +34,7 @@ md5sum = 0eeb24c6aa0760f0d33c4cc2828ddf30
    [template-mariadb.cfg] [template-mariadb.cfg]
    _update_hash_filename_ = instance-mariadb.cfg.jinja2.in _update_hash_filename_ = instance-mariadb.cfg.jinja2.in
    md5sum = 34eed1b6d7fc45dc078f30235e22d04f md5sum = bd5b0e72e8b5ddcd691e7f90a2947e50
    [template-my-cnf] [template-my-cnf]
    _update_hash_filename_ = templates/my.cnf.in _update_hash_filename_ = templates/my.cnf.in
    ...@@ -46,7 +46,7 @@ md5sum = 47b76144e1b116580c8acf08274af976 ...@@ -46,7 +46,7 @@ md5sum = 47b76144e1b116580c8acf08274af976
    [template-publish-slave-information] [template-publish-slave-information]
    _update_hash_filename_ = publish-database-slave-parameters.cfg.in _update_hash_filename_ = publish-database-slave-parameters.cfg.in
    md5sum = 9616477ff993c55c2b43bf8797db6994 md5sum = 2e9f3afde619e20f08803013a4225e94
    [mariadb-init-root-sql] [mariadb-init-root-sql]
    _update_hash_filename_ = templates/mariadb_init_root.sql.in _update_hash_filename_ = templates/mariadb_init_root.sql.in
    ...@@ -58,7 +58,7 @@ md5sum = c203f40a58386310a433b58fd345a341 ...@@ -58,7 +58,7 @@ md5sum = c203f40a58386310a433b58fd345a341
    [repman-manager-sh.in] [repman-manager-sh.in]
    _update_hash_filename_ = templates/repman-manager.sh.in _update_hash_filename_ = templates/repman-manager.sh.in
    md5sum = 852dfab6d798aa1382eec4de2fd624f9 md5sum = 70201395116d79b5bb4aabef13079c10
    [dbjobs-in] [dbjobs-in]
    _update_hash_filename_ = templates/dbjobs.in _update_hash_filename_ = templates/dbjobs.in
    ...@@ -74,4 +74,4 @@ md5sum = 455aaf369bf5141758dc57f2c0e67b08 ...@@ -74,4 +74,4 @@ md5sum = 455aaf369bf5141758dc57f2c0e67b08
    [slave-db-manage.in] [slave-db-manage.in]
    _update_hash_filename_ = templates/slave-db-manage.in _update_hash_filename_ = templates/slave-db-manage.in
    md5sum = b45313ae5fb06972cc7fc945e34e434a md5sum = cefcb8c7d17367b14414314ffd325c26
    software/repman/instance-mariadb.cfg.jinja2.in
    View file @ c5d93b74
    ...@@ -166,9 +166,9 @@ url = {{ parameter_dict['template-init-root-wrapper'] }} ...@@ -166,9 +166,9 @@ url = {{ parameter_dict['template-init-root-wrapper'] }}
    [{{ section('mysql-slave-db-cleanup') }}] [{{ section('mysql-slave-db-cleanup') }}]
    < = jinja2-template-script-base < = jinja2-template-script-base
    output = ${directory:scripts}/manage-slave-db output = ${directory:scripts}/manage-slave-db
    db-name = {{ slapparameter_dict['database-name'] }} db-name = {{ dumps(slapparameter_dict['database-name']) }}
    extra-context = extra-context =
    key database_name :db-name key database_name_list :db-name
    url = {{ parameter_dict['template-manage-db'] }} url = {{ parameter_dict['template-manage-db'] }}
    [mysqld] [mysqld]
    ...@@ -199,7 +199,7 @@ wait-for-files = ...@@ -199,7 +199,7 @@ wait-for-files =
    {{ supervisord_lib.supervisord("mariadb-ctl", buildout_bin_directory, supervisord_conf, use_service_hash=False) }} {{ supervisord_lib.supervisord("mariadb-ctl", buildout_bin_directory, supervisord_conf, use_service_hash=False) }}
    {% do part_list.append("supervisord-mariadb-ctl") -%} {% do part_list.append("supervisord-mariadb-ctl") -%}
    {% set maradb_program_dict = {"name": "mariadb", "command": "${mysqld-launcher:wrapper-path}", {% set maradb_program_dict = {"name": "mariadb", "command": "${mysqld-launcher:wrapper-path}",
    "stopwaitsecs": 86400, "startretries": 10, "autorestart": True, "environment": [], "stopwaitsecs": 86400, "startretries": 10, "autorestart": False, "environment": [],
    "stdout_logfile": "${directory:log}/mariadb_stdout.log", "stdout_logfile": "${directory:log}/mariadb_stdout.log",
    "stderr_logfile": "${directory:log}/mariadb_stdout.log" } %} "stderr_logfile": "${directory:log}/mariadb_stdout.log" } %}
    ...@@ -381,9 +381,6 @@ config = ${directory:etc}/mysql/my.cnf ...@@ -381,9 +381,6 @@ config = ${directory:etc}/mysql/my.cnf
    command = ${mysql-get-config:output} command = ${mysql-get-config:output}
    update-command = ${:command} update-command = ${:command}
    [dash]
    dash = {{ dumps(dash) }}
    [{{ section('promise-check-computer-memory') }}] [{{ section('promise-check-computer-memory') }}]
    <= monitor-promise-base <= monitor-promise-base
    promise = check_command_execute promise = check_command_execute
    ......
    software/repman/instance-repman-input-schema.json
    View file @ c5d93b74
    ...@@ -113,12 +113,6 @@ ...@@ -113,12 +113,6 @@
    "patternProperties": { "patternProperties": {
    ".*": { ".*": {
    "properties": { "properties": {
    "name": {
    "title": "Name of the cluster",
    "description": "Name of the cluster: Should not contains spaces or any special characters.",
    "type": "string",
    "default": ""
    },
    "database-amount": { "database-amount": {
    "title": "Amount of databases for cluster", "title": "Amount of databases for cluster",
    "description": "Database amount to deploy with this cluster. Minimal amount is 2 required to enable replication.", "description": "Database amount to deploy with this cluster. Minimal amount is 2 required to enable replication.",
    ......
    software/repman/instance-repman-slave-input-schema.json
    View file @ c5d93b74
    { {
    "$schema": "http://json-schema.org/draft-04/schema", "$schema": "http://json-schema.org/draft-04/schema",
    "properties": { "properties": {
    "db_user": {
    "description": "Database User, default is 'user'. This parameter is set only if database is not created yet.",
    "title": "Database User",
    "type": "string",
    "default": "user"
    },
    "db_password": { "db_password": {
    "description": "Database password. If no password set, a password will be generated. This parameter is set only if database is not created yet.", "description": "Database password. If no password set, a password will be generated. This parameter is set only if database is not created yet.",
    "title": "Initial database password", "title": "Initial database password",
    ...@@ -42,6 +36,12 @@ ...@@ -42,6 +36,12 @@
    "ascii_bin" "ascii_bin"
    ], ],
    "default": "" "default": ""
    },
    "cluster_id": {
    "title": "Replication Manager Cluster Name",
    "description": "When a cluster name is given, try to request the slave in the specified cluster name. If cluster name is not provided or does not exist, the slave is requested in the first cluster.",
    "type": "string",
    "default": ""
    } }
    }, },
    "title": "Input Parameters", "title": "Input Parameters",
    ......
    software/repman/instance-repman.cfg.jinja2.in
    View file @ c5d93b74
    ...@@ -2,9 +2,6 @@ ...@@ -2,9 +2,6 @@
    {% set part_list = [] -%} {% set part_list = [] -%}
    {% set monitor_base_url_dict = {} -%} {% set monitor_base_url_dict = {} -%}
    {% set mariadb_dict = {} -%} {% set mariadb_dict = {} -%}
    {% set mariadb_server_list = [] -%}
    {% set receiver_port_list = [] -%}
    {% set mariadb_path_list = [] -%}
    {% set ip = (ipv6_set | list)[0] -%} {% set ip = (ipv6_set | list)[0] -%}
    {% set ipv4 = (ipv4_set | list)[0] -%} {% set ipv4 = (ipv4_set | list)[0] -%}
    {% set cluster_list = [] -%} {% set cluster_list = [] -%}
    ...@@ -12,12 +9,17 @@ ...@@ -12,12 +9,17 @@
    "pfs", "linux", "readonly", "diskmonitor", "sqlerror", "compressbinlog", "bm4ci", "pfs", "linux", "readonly", "diskmonitor", "sqlerror", "compressbinlog", "bm4ci",
    "mroonga", "utctime", "readcommitted", "nohandshake", "ssl"] -%} "mroonga", "utctime", "readcommitted", "nohandshake", "ssl"] -%}
    {% set frontend_parameter_dict = slapparameter_dict.get('slave-frontend', {}) -%} {% set frontend_parameter_dict = slapparameter_dict.get('slave-frontend', {}) -%}
    {% set database_slave_list = [] -%}
    {% set db_name_list = [] -%}
    {% set count = namespace(value=2) %} {% set count = namespace(value=2) %}
    {% set caucase_bind = '[' ~ ip ~ ']:8890' -%} {% set caucase_bind = '[' ~ ip ~ ']:8890' -%}
    {% set caucase_url = 'http://' ~ caucase_bind -%} {% set caucase_url = 'http://' ~ caucase_bind -%}
    {% set csrid_list = [] -%} {% set csrid_list = [] -%}
    {% set default_parameter_dict = {"cluster1": {"db-prefered-master": "",
    "database-amount": 2, "proxysql-user": "external",
    "logical-backup-cron": "0 21 * * *", "physical-backup-cron": "0 1 * * *"}} -%}
    {% set cluster_name_list = slapparameter_dict.get('repman-cluster-dict', default_parameter_dict).keys() -%}
    {% set database_slave_dict = {} -%}
    {% set db_name_dict = {} -%}
    {% set default_slave_cluster_id = list(cluster_name_list)[0] -%}
    {% macro password(name) -%} {% macro password(name) -%}
    [{{ name }}-password] [{{ name }}-password]
    ...@@ -46,8 +48,14 @@ ssl = ${:etc}/ssl ...@@ -46,8 +48,14 @@ ssl = ${:etc}/ssl
    [proxysql-directory] [proxysql-directory]
    recipe = slapos.cookbook:mkdirectory recipe = slapos.cookbook:mkdirectory
    home = ${buildout:directory}/proxy
    config = ${directory:etc}/proxysql config = ${directory:etc}/proxysql
    ssl = ${:config}/ssl ssl = ${:config}/default-ssl
    {% for cluster in cluster_name_list -%}
    {{ cluster }} = ${:home}/{{ cluster }}
    {% do database_slave_dict.__setitem__(cluster, []) -%}
    {% do db_name_dict.__setitem__(cluster, []) -%}
    {% endfor -%}
    {% import "supervisord_lib" as supervisord_lib with context %} {% import "supervisord_lib" as supervisord_lib with context %}
    {% set proxysql_controller = "proxysql-ctl" -%} {% set proxysql_controller = "proxysql-ctl" -%}
    ...@@ -81,40 +89,44 @@ inline = ...@@ -81,40 +89,44 @@ inline =
    if [ -z "$CONFIG" ]; then if [ -z "$CONFIG" ]; then
    CONFIG="${proxysql-directory:config}/proxysql.cnf" CONFIG="${proxysql-directory:config}/proxysql.cnf"
    fi fi
    ETCDIR=`dirname $CONFIG`
    mkdir -p ${repman:config-tmp}/proxies mkdir -p ${repman:config-tmp}/proxies
    mkdir -p $ETCDIR/proxysql
    ln -sf ${proxysql-directory:ssl} $ETCDIR/proxysql/ssl
    cd ${repman:config-tmp}/proxies cd ${repman:config-tmp}/proxies
    {{ curl_bin }} -o proxies-$NAME.tar.gz ${nginx-parameter:repman-url}/api/clusters/$NAME/servers/$HOST/$PORT/config {{ curl_bin }} -o proxies-$NAME.tar.gz ${nginx-parameter:repman-url}/api/clusters/$NAME/servers/$HOST/$PORT/config
    tar -xzf proxies-$NAME.tar.gz tar -xzf proxies-$NAME.tar.gz
    cp etc/proxysql/proxysql.cnf $CONFIG cp etc/proxysql/proxysql.cnf $CONFIG
    output = ${directory:bin}/update-proxysql-config output = ${directory:bin}/update-proxysql-config
    {% set slave_information_list = [] -%}
    {% for instance_dict in slave_instance_list -%} {% for instance_dict in slave_instance_list -%}
    {% set slave_dict = { {% set slave_dict = {
    'name': 'db_%s' % instance_dict['slave_reference'].replace('-', '_').lower(), 'name': 'db_%s' % instance_dict['slave_reference'].replace('-', '_').lower(),
    'user': instance_dict.get('db_user', 'user'), 'user': instance_dict['slave_reference'].replace('_', '').replace('-', '').lower(),
    'password': instance_dict.get('db_password', '${' ~ instance_dict['slave_reference'] ~ '-password:passwd}'), 'password': instance_dict.get('db_password', '${' ~ instance_dict['slave_reference'] ~ '-password:passwd}'),
    'slave_reference': instance_dict['slave_reference'], 'slave_reference': instance_dict['slave_reference'],
    'charset': instance_dict.get('db_charset', ''), 'charset': instance_dict.get('db_charset', ''),
    'cluster_id': instance_dict.get('cluster_id', default_slave_cluster_id),
    'require_ssl': True 'require_ssl': True
    } -%} } -%}
    {% do database_slave_list.append(slave_dict) -%} {% do slave_information_list.append(slave_dict) -%}
    {% do db_name_list.append(slave_dict['name']) -%} {% do database_slave_dict[slave_dict['cluster_id']].append(slave_dict) -%}
    {% do db_name_dict[slave_dict['cluster_id']].append([slave_dict['name'], slave_dict['user']]) -%}
    {{ password(instance_dict['slave_reference']) }} {{ password(instance_dict['slave_reference']) }}
    {% endfor %} {% endfor %}
    # set each database dict which will be used to publish slave information
    [database-slave-information] [database-slave-information]
    {% for slave_dict in database_slave_list -%} {% for slave_dict in slave_information_list -%}
    {{ slave_dict['name'] }} = !py!{{ slave_dict }} {{ slave_dict['name'] }} = !py!{{ slave_dict }}
    {% endfor %} {% endfor %}
    {% set db_list = db_name_list | join(' ') -%}
    {% do mariadb_dict.__setitem__('computer-memory-percent-threshold', 80) -%} {% do mariadb_dict.__setitem__('computer-memory-percent-threshold', 80) -%}
    {% set default_parameter_dict = {"cluster1": {"name": "cluster1", "db-prefered-master": "",
    "database-amount": 2, "proxysql-user": "external",
    "logical-backup-cron": "0 21 * * *", "physical-backup-cron": "0 1 * * *"}} -%}
    {% for name, parameter_dict in slapparameter_dict.get('repman-cluster-dict', default_parameter_dict).items() -%} {% for name, parameter_dict in slapparameter_dict.get('repman-cluster-dict', default_parameter_dict).items() -%}
    {% do mariadb_dict.__setitem__('innodb-file-per-table', parameter_dict.get('innodb-file-per-table', 1)) -%} {% do mariadb_dict.__setitem__('innodb-file-per-table', parameter_dict.get('innodb-file-per-table', 1)) -%}
    {% do mariadb_dict.__setitem__('use-ipv6', parameter_dict.get('use-ipv6', True)) -%} {% do mariadb_dict.__setitem__('use-ipv6', parameter_dict.get('use-ipv6', True)) -%}
    {% set db_list = db_name_dict[name] -%}
    # Request mariadb instances # Request mariadb instances
    {% set db_amount = parameter_dict.get('database-amount', 2) -%} {% set db_amount = parameter_dict.get('database-amount', 2) -%}
    ...@@ -123,16 +135,19 @@ output = ${directory:bin}/update-proxysql-config ...@@ -123,16 +135,19 @@ output = ${directory:bin}/update-proxysql-config
    {% endif -%} {% endif -%}
    {% set count.value = count.value + db_amount %} {% set count.value = count.value + db_amount %}
    {% set mariadb_path_list = [] -%}
    {% set mariadb_server_list = [] -%}
    {% set receiver_port_list = [] -%}
    {% for i in range(0, db_amount) -%} {% for i in range(0, db_amount) -%}
    {% do mariadb_dict.__setitem__('tcp-port', 2099 + (i * 100)) -%} {% do mariadb_dict.__setitem__('tcp-port', 2099 + (i * 100)) -%}
    {% set section = 'request-mariadb-' ~ i -%} {% set section = 'request-mariadb-' ~ i ~ '-' ~ name -%}
    {% set dbname = 'Mariadb-' ~ i %} {% set dbname = 'Mariadb-' ~ i %}
    {% set instance_name = dbname ~ '-' ~ name-%}
    [{{ section }}] [{{ section }}]
    <= request-common <= request-common
    software-type = mariadb software-type = mariadb
    name = {{ dbname ~ '-' ~ name}} name = {{ instance_name }}
    sla-computer_guid = {{ dumps(parameter_dict.get('-sla-' ~ i ~'-computer_guid', '')) }} sla-computer_guid = {{ dumps(parameter_dict.get('-sla-' ~ i ~'-computer_guid', '')) }}
    {% for key, value in mariadb_dict.items() -%} {% for key, value in mariadb_dict.items() -%}
    config-{{ key }} = {{ dumps(value) }} config-{{ key }} = {{ dumps(value) }}
    ...@@ -145,8 +160,8 @@ config-repman-passwd = ${repman-parameter:password} ...@@ -145,8 +160,8 @@ config-repman-passwd = ${repman-parameter:password}
    config-repman-url = ${nginx-parameter:backend-url} config-repman-url = ${nginx-parameter:backend-url}
    config-repman-secure-url = ${nginx-parameter:backend-ssl-url} config-repman-secure-url = ${nginx-parameter:backend-ssl-url}
    config-cluster = {{ name }} config-cluster = {{ name }}
    config-name = {{ dbname }} config-name = {{ instance_name }}
    config-database-list = !py!{{ database_slave_list }} config-database-list = !py!{{ database_slave_dict[name] }}
    config-database-name = {{ dumps(db_list) }} config-database-name = {{ dumps(db_list) }}
    config-require-ssl = {{ dumps(slapparameter_dict.get('require-ssl', False)) }} config-require-ssl = {{ dumps(slapparameter_dict.get('require-ssl', False)) }}
    return = return =
    ...@@ -160,7 +175,7 @@ return = ...@@ -160,7 +175,7 @@ return =
    {% do mariadb_server_list.append('${' ~ section ~ ':connection-database-host}') -%} {% do mariadb_server_list.append('${' ~ section ~ ':connection-database-host}') -%}
    {% do receiver_port_list.append('${' ~ section ~ ':connection-receiver-port}') -%} {% do receiver_port_list.append('${' ~ section ~ ':connection-receiver-port}') -%}
    {% do mariadb_path_list.append('${' ~ section ~ ':connection-partition-path}') -%} {% do mariadb_path_list.append('${' ~ section ~ ':connection-partition-path}') -%}
    {% do monitor_base_url_dict.__setitem__('mariadb' ~ i, '${' ~ section ~ ':connection-monitor-base-url}') -%} {% do monitor_base_url_dict.__setitem__(instance_name, '${' ~ section ~ ':connection-monitor-base-url}') -%}
    {% do csrid_list.append('${' ~ section ~ ':connection-csr-id}') -%} {% do csrid_list.append('${' ~ section ~ ':connection-csr-id}') -%}
    {% endfor -%} {% endfor -%}
    ...@@ -205,7 +220,7 @@ db-prefered-master = {{ mariadb_server_list[0] }} ...@@ -205,7 +220,7 @@ db-prefered-master = {{ mariadb_server_list[0] }}
    proxysql-servers = {{ ipv4 }} proxysql-servers = {{ ipv4 }}
    proxysql-servers-ipv6 = [{{ ip }}] proxysql-servers-ipv6 = [{{ ip }}]
    password = ${repman-parameter:password} password = ${repman-parameter:password}
    proxysql-partition = ${buildout:directory} proxysql-partition = {{ '${proxysql-directory:' ~ name ~ '}' }}
    receiver-port-list = {{ receiver_port_list | join(',') }} receiver-port-list = {{ receiver_port_list | join(',') }}
    enabled-tags = {{ parameter_dict.get("tag-list", tag_list) | join(',') }} enabled-tags = {{ parameter_dict.get("tag-list", tag_list) | join(',') }}
    proxy-tags = {{ parameter_dict.get("proxy-tags", ["pkg", "masterslave", "linux", "noreadwritesplit", "ssl"]) | join(',') }} proxy-tags = {{ parameter_dict.get("proxy-tags", ["pkg", "masterslave", "linux", "noreadwritesplit", "ssl"]) | join(',') }}
    ...@@ -248,7 +263,7 @@ context = ...@@ -248,7 +263,7 @@ context =
    recipe = plone.recipe.command recipe = plone.recipe.command
    # if Repman is not started, cannot download config from server # if Repman is not started, cannot download config from server
    stop-on-error = false stop-on-error = false
    config = ${proxysql-directory:config}/proxysql.cnf config = ${proxysql-directory:home}/{{ name }}/etc/proxysql.cnf
    data = ${repman:proxy-data}/{{ name }} data = ${repman:proxy-data}/{{ name }}
    command = command =
    mkdir -p ${:data} && mkdir -p ${:data} &&
    ...@@ -295,7 +310,7 @@ config-port = {{ '${' ~ name ~ '-cluster-parameter:proxy-port}' }} ...@@ -295,7 +310,7 @@ config-port = {{ '${' ~ name ~ '-cluster-parameter:proxy-port}' }}
    {% set service_name = "proxysql-" ~ name -%} {% set service_name = "proxysql-" ~ name -%}
    {% set proxysql_dict = {"name": service_name, "command": "${" ~ service_name ~ "-wrapper:wrapper-path}", {% set proxysql_dict = {"name": service_name, "command": "${" ~ service_name ~ "-wrapper:wrapper-path}",
    "stopwaitsecs": 60, "startretries": 10, "autorestart": True, "environment": [], "stopwaitsecs": 60, "startretries": 10, "autorestart": False, "environment": [],
    "stdout_logfile": "${repman:proxies-log}/" ~ service_name ~ ".log", "stdout_logfile": "${repman:proxies-log}/" ~ service_name ~ ".log",
    "stderr_logfile": "${repman:proxies-log}/" ~ service_name ~ ".log" } %} "stderr_logfile": "${repman:proxies-log}/" ~ service_name ~ ".log" } %}
    ...@@ -313,7 +328,7 @@ RootSoftwareInstance = ${:default} ...@@ -313,7 +328,7 @@ RootSoftwareInstance = ${:default}
    [dynamic-{{ name}}-publish-slave-information] [dynamic-{{ name}}-publish-slave-information]
    recipe = slapos.recipe.template:jinja2 recipe = slapos.recipe.template:jinja2
    output = ${buildout:directory}/database-publish-slave-information.cfg output = ${buildout:directory}/database-{{ name}}-publish-slave-information.cfg
    extensions = jinja2.ext.do extensions = jinja2.ext.do
    url = {{ template_publish_slave_information }} url = {{ template_publish_slave_information }}
    context = context =
    ...@@ -321,6 +336,7 @@ context = ...@@ -321,6 +336,7 @@ context =
    key proxy_port {{ name }}-cluster-parameter:proxy-port key proxy_port {{ name }}-cluster-parameter:proxy-port
    raw eggs_directory {{ eggs_directory }} raw eggs_directory {{ eggs_directory }}
    raw develop_eggs_directory {{ develop_eggs_directory }} raw develop_eggs_directory {{ develop_eggs_directory }}
    raw cluster {{ name }}
    raw ipv6 {{ ip }} raw ipv6 {{ ip }}
    raw ipv4 {{ ipv4 }} raw ipv4 {{ ipv4 }}
    section slave_dict database-slave-information section slave_dict database-slave-information
    ...@@ -617,7 +633,7 @@ recipe = slapos.cookbook:publish-early ...@@ -617,7 +633,7 @@ recipe = slapos.cookbook:publish-early
    monitor-password monitor-htpasswd:passwd monitor-password monitor-htpasswd:passwd
    db-root-password gen-root-password:passwd db-root-password gen-root-password:passwd
    repman-password repman-password:passwd repman-password repman-password:passwd
    database-list = {{ db_list }} database-dict = {{ db_name_dict }}
    [publish-connection-parameter] [publish-connection-parameter]
    <= monitor-publish <= monitor-publish
    ...@@ -626,7 +642,7 @@ recipe = slapos.cookbook:publish ...@@ -626,7 +642,7 @@ recipe = slapos.cookbook:publish
    backend-url = ${nginx-parameter:backend-ssl-url} backend-url = ${nginx-parameter:backend-ssl-url}
    url = ${repman-frontend:connection-secure_access} url = ${repman-frontend:connection-secure_access}
    username = ${repman-parameter:username} username = ${repman-parameter:username}
    database-list = {{ db_list }} database-dict = {{ db_name_dict }}
    {% for name, value in publish_dict.items() -%} {% for name, value in publish_dict.items() -%}
    {{ name }} = {{ value }} {{ name }} = {{ value }}
    {% endfor %} {% endfor %}
    ......
    software/repman/publish-database-slave-parameters.cfg.in
    View file @ c5d93b74
    ...@@ -11,6 +11,7 @@ cert_file = ${slap-connection:cert-file} ...@@ -11,6 +11,7 @@ cert_file = ${slap-connection:cert-file}
    {% for _, database in slave_dict.items() -%} {% for _, database in slave_dict.items() -%}
    {% if database['cluster_id'] == cluster -%}
    {% set slave_reference = database['slave_reference'] -%} {% set slave_reference = database['slave_reference'] -%}
    {% do part_list.append('publish-slave-' ~ slave_reference) -%} {% do part_list.append('publish-slave-' ~ slave_reference) -%}
    ...@@ -24,6 +25,8 @@ url = {{ "mysql://" ~ database['user'] ~ ":" ~ database['password'] ~ "@[" ~ ipv ...@@ -24,6 +25,8 @@ url = {{ "mysql://" ~ database['user'] ~ ":" ~ database['password'] ~ "@[" ~ ipv
    url = {{ "mysql://" ~ ipv4 ~ ":" ~ proxy_port ~ database['name'] }} url = {{ "mysql://" ~ ipv4 ~ ":" ~ proxy_port ~ database['name'] }}
    url_v6 = {{ "mysql://[" ~ ip ~ "]:" ~ proxy_port ~ database['name'] }} url_v6 = {{ "mysql://[" ~ ip ~ "]:" ~ proxy_port ~ database['name'] }}
    {% endif -%} {% endif -%}
    cluster = {{ database['cluster_id'] }}
    {% endif -%}
    {% endfor %} {% endfor %}
    [buildout] [buildout]
    ......
    software/repman/templates/repman-manager.sh.in
    View file @ c5d93b74
    #!{{ bash_bin }} #!{{ bash_bin }}
    #set -e set -e
    curl () { curl () {
    {{ curl_bin }} -k --silent -H "Accept: application/json" "$@" {{ curl_bin }} -k --silent -H "Accept: application/json" "$@"
    ...@@ -33,6 +33,19 @@ wait_database () { ...@@ -33,6 +33,19 @@ wait_database () {
    done done
    } }
    check_cluster () {
    # Check if cluster is boostrapped
    NAME=$1
    TOKEN=$(get_token | {{ jq_bin }} -r '.token')
    ERRORS=$(curl -H "Authorization: Bearer ${TOKEN}" {{ secure_url }}/api/clusters/$NAME/topology/alerts | {{ jq_bin }} -r '.errors')
    if [ "$ERRORS" != "null" ] && [ ! -z "$ERRORS" ]; then
    echo "ERROR: Bootstrap replication of cluster $NAME failed!";
    echo $ERRORS;
    return 1;
    fi
    return 0
    }
    activate_proxy () { activate_proxy () {
    NAME=$1 NAME=$1
    URL="{{ secure_url }}/api/clusters/$NAME/settings/actions/switch/database-hearbeat" URL="{{ secure_url }}/api/clusters/$NAME/settings/actions/switch/database-hearbeat"
    ...@@ -46,14 +59,42 @@ activate_proxy () { ...@@ -46,14 +59,42 @@ activate_proxy () {
    fi fi
    } }
    check_add () {
    NAME=$1
    STATUS=$(curl -H "Authorization: Bearer ${TOKEN}" -o /dev/null -w "%{http_code}" {{ secure_url }}/api/clusters/$NAME)
    if [ $STATUS != 200 ]; then
    # if cluster doesn't exists we add it
    curl -H "Authorization: Bearer ${TOKEN}" \
    {{ secure_url }}/api/clusters/actions/add/$NAME
    fi
    }
    reload_config () {
    # check if etc/repman/cluster.d/config-clusterXX.toml has changed and call reload
    NAME=$1
    FILE="{{ parameter_dict['config-tmp'] }}/$NAME-config-cheksum.txt"
    CHECKSUM=$(md5sum {{ parameter_dict['clusters'] }}/config-$NAME.toml)
    if [ -s $FILE ]; then
    OLDSUM=$(cat $FILE);
    fi
    echo "$CHECKSUM" > $FILE
    if [ "$CHECKSUM" != "$OLDSUM" ]; then
    echo "Reloading settings for $NAME to apply changes..."
    curl -H "Authorization: Bearer ${TOKEN}" \
    {{ secure_url }}/api/clusters/$NAME/settings/actions/reload
    else
    echo "$NAME config is up to date."
    fi
    }
    TOKEN=$(get_token | {{ jq_bin }} -r '.token') TOKEN=$(get_token | {{ jq_bin }} -r '.token')
    # Always reload cluster configuration to apply recent changes # Always reload cluster configuration to apply recent changes
    {% for name in cluster_name_list -%} {% for name in cluster_name_list -%}
    # reload {{ name }} settings
    echo "Reloading settings for {{ name }}..." check_add {{ name }}
    curl -H "Authorization: Bearer ${TOKEN}" \
    {{ secure_url }}/api/clusters/{{ name }}/settings/actions/reload reload_config {{ name }}
    # Start Replication on {{ name }} # Start Replication on {{ name }}
    if [ ! -f "{{ parameter_dict['bootstrap'] }}/{{ name }}_bootstrapped" ]; then if [ ! -f "{{ parameter_dict['bootstrap'] }}/{{ name }}_bootstrapped" ]; then
    ...@@ -66,7 +107,6 @@ if [ ! -f "{{ parameter_dict['bootstrap'] }}/{{ name }}_bootstrapped" ]; then ...@@ -66,7 +107,6 @@ if [ ! -f "{{ parameter_dict['bootstrap'] }}/{{ name }}_bootstrapped" ]; then
    curl -H "Authorization: Bearer ${TOKEN}" \ curl -H "Authorization: Bearer ${TOKEN}" \
    {{ secure_url }}/api/clusters/{{ name }}/actions/replication/cleanup {{ secure_url }}/api/clusters/{{ name }}/actions/replication/cleanup
    CODE=$(curl -H "Authorization: Bearer ${TOKEN}" -o /dev/null -w "%{http_code}" {{ secure_url }}/api/clusters/{{ name }}/actions/replication/bootstrap/master-slave) CODE=$(curl -H "Authorization: Bearer ${TOKEN}" -o /dev/null -w "%{http_code}" {{ secure_url }}/api/clusters/{{ name }}/actions/replication/bootstrap/master-slave)
    SUCCESS=0
    if [ $CODE -eq 200 ]; then if [ $CODE -eq 200 ]; then
    activate_proxy {{ name }} activate_proxy {{ name }}
    if [ $? -eq 0 ]; then if [ $? -eq 0 ]; then
    ...@@ -77,6 +117,9 @@ if [ ! -f "{{ parameter_dict['bootstrap'] }}/{{ name }}_bootstrapped" ]; then ...@@ -77,6 +117,9 @@ if [ ! -f "{{ parameter_dict['bootstrap'] }}/{{ name }}_bootstrapped" ]; then
    else else
    echo "ERROR: Failed to bootstrap cluster {{ name }}... http_code $CODE" echo "ERROR: Failed to bootstrap cluster {{ name }}... http_code $CODE"
    fi fi
    else
    # Check cluster health
    check_cluster {{ name }}
    fi fi
    {% endfor %} {% endfor %}
    software/repman/templates/slave-db-manage.in
    View file @ c5d93b74
    ...@@ -17,6 +17,21 @@ run_mysql () { ...@@ -17,6 +17,21 @@ run_mysql () {
    TOKEN=$(get_token | {{ jq_bin }} -r '.token') TOKEN=$(get_token | {{ jq_bin }} -r '.token')
    DATADIR=$(curl -H "Authorization: Bearer ${TOKEN}" \ {{ secure_url }}/api/clusters/{{ cluster_name }}/topology/master | {{ jq_bin }} -r '.slaposDatadir') DATADIR=$(curl -H "Authorization: Bearer ${TOKEN}" \ {{ secure_url }}/api/clusters/{{ cluster_name }}/topology/master | {{ jq_bin }} -r '.slaposDatadir')
    revoke_user () {
    DB=$1
    UNAME=$2
    if [ ! -z "$UNAME" ]; then
    echo "Revoking all grants for user '$USER'";
    run_mysql -Be "
    REVOKE ALL PRIVILEGES, GRANT OPTION FROM '$UNAME';
    DROP USER IF EXISTS '$UNAME'@'%';
    DROP USER IF EXISTS '$UNAME'@'localhost';
    DROP USER IF EXISTS '$UNAME'@'::';
    FLUSH PRIVILEGES;
    "
    fi
    }
    # Only write or delete on master database else, we break replication. # Only write or delete on master database else, we break replication.
    if [ "$DATADIR" = "{{ partition_dir }}" ]; then if [ "$DATADIR" = "{{ partition_dir }}" ]; then
    ...@@ -26,21 +41,24 @@ use repman_slave_definition; ...@@ -26,21 +41,24 @@ use repman_slave_definition;
    CREATE TABLE IF NOT EXISTS \`slave\` ( CREATE TABLE IF NOT EXISTS \`slave\` (
    \`name\` varchar(80) NOT NULL, \`name\` varchar(80) NOT NULL,
    \`state\` tinyint(1) DEFAULT NULL, \`state\` tinyint(1) DEFAULT NULL,
    \`user\` varchar(80) NOT NULL,
    PRIMARY KEY (\`name\`) PRIMARY KEY (\`name\`)
    ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4; ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4;
    UPDATE \`slave\` set \`state\`=false; UPDATE \`slave\` set \`state\`=false;
    {% for name in database_name.split(' ') -%} {% for name, user in database_name_list -%}
    {% if name -%} {% if name -%}
    REPLACE INTO \`slave\` VALUES ('{{ name }}', true); REPLACE INTO \`slave\` VALUES ('{{ name }}', true, '{{ user }}');
    {% endif -%} {% endif -%}
    {% endfor -%} {% endfor -%}
    EOF EOF
    # Update requested slaves database # Update requested slaves database
    OUTPUT="{{ tmp_dir }}/removed_db.txt"
    run_mysql < {{ tmp_dir }}/.script.sql run_mysql < {{ tmp_dir }}/.script.sql
    rm -f {{ tmp_dir }}/.script.sql rm -f {{ tmp_dir }}/.script.sql
    run_mysql -NBe "SELECT name, user FROM repman_slave_definition.slave WHERE state=false" > $OUTPUT
    DBNAME=$(run_mysql --skip-column-names -Be "SELECT name FROM repman_slave_definition.slave WHERE state=false"); DBNAME=$(run_mysql --skip-column-names -Be "SELECT name FROM repman_slave_definition.slave WHERE state=false");
    RET=$? RET=$?
    ...@@ -49,12 +67,17 @@ EOF ...@@ -49,12 +67,17 @@ EOF
    echo "Mysql command failed: $DBNAME" echo "Mysql command failed: $DBNAME"
    exit $RET exit $RET
    fi fi
    if [ -z "$DBNAME" ]; then if [ -z "$DBNAME" ]; then
    echo "No database for slave to remove."; echo "No database for slave to remove.";
    fi fi
    for NAME in $DBNAME; do for NAME in $DBNAME; do
    if [ ! -z "$NAME" ]; then if [ ! -z "$NAME" ]; then
    USER=$(grep -oP "$NAME\s*\K\w+" $OUTPUT);
    if [ ! -z "$USER" ]; then
    revoke_user $NAME $USER;
    fi
    echo "Deleting database $NAME..." echo "Deleting database $NAME..."
    run_mysql -e "DROP DATABASE IF EXISTS $NAME"; run_mysql -e "DROP DATABASE IF EXISTS $NAME";
    run_mysql -e "DELETE FROM repman_slave_definition.slave WHERE name='$NAME'"; run_mysql -e "DELETE FROM repman_slave_definition.slave WHERE name='$NAME'";
    ......
    Markdown is supported
    0% or
    You are about to add 0 people to the discussion. Proceed with caution.
    Finish editing this message first!
    Please register or to comment
    GitLab Nexedi Edition | About GitLab | About Nexedi | 沪ICP备2021021310号-2 | 沪ICP备2021021310号-7