Skip to content
Projects
Groups
Snippets
Help
Loading...
Help
Support
Keyboard shortcuts
?
Submit feedback
Contribute to GitLab
Sign in / Register
Toggle navigation
slapos
Project overview
Project overview
Details
Activity
Releases
Repository
Repository
Files
Commits
Branches
Tags
Contributors
Graph
Compare
Labels
Merge Requests
104
Merge Requests
104
CI / CD
CI / CD
Pipelines
Jobs
Schedules
Analytics
Analytics
CI / CD
Repository
Value Stream
Snippets
Snippets
Members
Members
Collapse sidebar
Close sidebar
Activity
Graph
Jobs
Commits
Open sidebar
nexedi
slapos
Commits
10e03fe5
Commit
10e03fe5
authored
Apr 20, 2020
by
Alain Takoudjou
Browse files
Options
Browse Files
Download
Email Patches
Plain Diff
repman: fixup, add more parameters
parent
44684450
Changes
5
Show whitespace changes
Inline
Side-by-side
Showing
5 changed files
with
66 additions
and
26 deletions
+66
-26
software/repman/buildout.hash.cfg
software/repman/buildout.hash.cfg
+4
-4
software/repman/instance-repman.cfg.jinja2.in
software/repman/instance-repman.cfg.jinja2.in
+32
-7
software/repman/templates/cluster-config.toml.in
software/repman/templates/cluster-config.toml.in
+3
-3
software/repman/templates/config.toml.in
software/repman/templates/config.toml.in
+7
-8
software/repman/templates/repman-manager.sh.in
software/repman/templates/repman-manager.sh.in
+20
-4
No files found.
software/repman/buildout.hash.cfg
View file @
10e03fe5
...
@@ -18,15 +18,15 @@ md5sum = b38fb4c5cdba2223c84d62ef60513e49
...
@@ -18,15 +18,15 @@ md5sum = b38fb4c5cdba2223c84d62ef60513e49
[instance-repman.cfg]
[instance-repman.cfg]
_update_hash_filename_ = instance-repman.cfg.jinja2.in
_update_hash_filename_ = instance-repman.cfg.jinja2.in
md5sum =
17431810792121fa5c9dc1ccfeec9d13
md5sum =
c682174b1aecf6a079e02d353e339967
[config-toml.in]
[config-toml.in]
_update_hash_filename_ = templates/config.toml.in
_update_hash_filename_ = templates/config.toml.in
md5sum =
66108ddfda7169695d7b876a4e3b246c
md5sum =
446044949547efc66b89aa9f20ef7a2e
[config-cluster-toml.in]
[config-cluster-toml.in]
_update_hash_filename_ = templates/cluster-config.toml.in
_update_hash_filename_ = templates/cluster-config.toml.in
md5sum =
d883337a75140c08853960cd6a8fdc5f
md5sum =
4d8d38dadda6cf31e078f5fc4918da7d
[nginx.conf.in]
[nginx.conf.in]
_update_hash_filename_ = templates/nginx.conf.in
_update_hash_filename_ = templates/nginx.conf.in
...
@@ -54,7 +54,7 @@ md5sum = 3ba02b99c3fcbc24b3659917980b310f
...
@@ -54,7 +54,7 @@ md5sum = 3ba02b99c3fcbc24b3659917980b310f
[repman-manager-sh.in]
[repman-manager-sh.in]
_update_hash_filename_ = templates/repman-manager.sh.in
_update_hash_filename_ = templates/repman-manager.sh.in
md5sum =
8ad6565e7b26e57b41bda4023a9d29ed
md5sum =
fca71cf71a94e612b8c9bd69899c568b
[dbjobs-in]
[dbjobs-in]
_update_hash_filename_ = templates/dbjobs.in
_update_hash_filename_ = templates/dbjobs.in
...
...
software/repman/instance-repman.cfg.jinja2.in
View file @
10e03fe5
...
@@ -6,6 +6,7 @@
...
@@ -6,6 +6,7 @@
{% set mariadb_path_list = [] -%}
{% set mariadb_path_list = [] -%}
{% set ip = (ipv6_set | list)[0] -%}
{% set ip = (ipv6_set | list)[0] -%}
{% set ipv4 = (ipv4_set | list)[0] -%}
{% set ipv4 = (ipv4_set | list)[0] -%}
{% set tags = "gtidstrict,bind,pkg,innodb,noquerycache,slow,pfs,linux,readonly,diskmonitor,sqlerror,compressbinlog" -%}
[directory]
[directory]
recipe = slapos.cookbook:mkdirectory
recipe = slapos.cookbook:mkdirectory
...
@@ -31,11 +32,29 @@ cert-file = ${slap-connection:cert-file}
...
@@ -31,11 +32,29 @@ cert-file = ${slap-connection:cert-file}
computer-id = ${slap-connection:computer-id}
computer-id = ${slap-connection:computer-id}
partition-id = ${slap-connection:partition-id}
partition-id = ${slap-connection:partition-id}
[proxy-admin-port]
recipe = slapos.cookbook:free_port
minimum = 6032
maximum = 6132
[proxy-port]
recipe = slapos.cookbook:free_port
minimum = 6033
maximum = 6133
{% do mariadb_dict.__setitem__('computer-memory-percent-threshold', 80) -%}
{% do mariadb_dict.__setitem__('computer-memory-percent-threshold', 80) -%}
{% do mariadb_dict.__setitem__('innodb-file-per-table', slapparameter_dict.get('innodb-file-per-table', 1)) -%}
{% do mariadb_dict.__setitem__('innodb-file-per-table', slapparameter_dict.get('innodb-file-per-table', 1)) -%}
{% set default_parameter_dict = {"cluster1": {"name": "cluster1", "db-prefered-master": "",
"database-amount": 2, "proxysql-user": "external", "proxy-tags": "pkg,masterslave,linux,noreadwritesplit",
"logical-backup-cron": "0 1 * * 6", "physical-backup-cron": "0 0 * * *"}} -%}
{% for name, parameter_dict in slapparameter_dict.get('cluster-partition-dict', default_parameter_dict).items() -%}
# Request mariadb instances
# Request mariadb instances
{% for i in range(0, slapparameter_dict.get('database-amount', 2)) -%}
{% set db_amount = 2 if parameter_dict.get('database-amount', 2) < 3 else set db_amount = parameter_dict['database-amount'] -%}
{% for i in range(0, db_amount) -%}
{% do mariadb_dict.__setitem__('tcp-port', 2099 + (i * 100)) -%}
{% do mariadb_dict.__setitem__('tcp-port', 2099 + (i * 100)) -%}
{% set section = 'request-mariadb-' ~ i -%}
{% set section = 'request-mariadb-' ~ i -%}
{% set name = 'Mariadb-' ~ i -%}
{% set name = 'Mariadb-' ~ i -%}
...
@@ -44,7 +63,7 @@ partition-id = ${slap-connection:partition-id}
...
@@ -44,7 +63,7 @@ partition-id = ${slap-connection:partition-id}
<= request-common
<= request-common
software-type = mariadb
software-type = mariadb
name = {{ name }}
name = {{ name }}
sla-computer_guid = {{ dumps(
slapparameter_dict.get('mariadb-computer-
guid', '')) }}
sla-computer_guid = {{ dumps(
parameter_dict.get('-sla-' ~ i ~'-computer_
guid', '')) }}
{% for key, value in mariadb_dict.items() -%}
{% for key, value in mariadb_dict.items() -%}
config-{{ key }} = {{ dumps(value) }}
config-{{ key }} = {{ dumps(value) }}
{% endfor -%}
{% endfor -%}
...
@@ -66,13 +85,14 @@ return =
...
@@ -66,13 +85,14 @@ return =
{% endfor -%}
{% endfor -%}
# Manage Replication Manager clusters
# Manage Replication Manager clusters
{% set default_parameter_dict = {"cluster1": {"name": "cluster1", "db-prefered-master": "", "proxy-admin-port": 6032, "proxy-port": 6033, "proxysql-user": "external"}} -%}
{% set prefered_master = parameter_dict.get("db-prefered-master") -%}
{% for name, parameter_dict in slapparameter_dict.get('cluster-partition-dict', default_parameter_dict).items() -%}
{% set prefered_master = parameter_dict.pop("db-prefered-master") -%}
[{{ name ~ '-cluster-parameter' }}]
[{{ name ~ '-cluster-parameter' }}]
{% for key, value in parameter_dict.items() -%}
{% for key, value in parameter_dict.items() -%}
{{ key }} = {{ dumps(value) }}
{{ key }} = {{ dumps(value) }}
{% endfor -%}
{% endfor -%}
proxysql-user = {{ parameter_dict.get("proxysql-user", "external") }}
proxy-port = ${proxy-port:port}
proxy-admin-port = ${proxy-admin-port:port}
db-credential = repman:${publish-early:root-password}
db-credential = repman:${publish-early:root-password}
db-list = {{ mariadb_server_list | join(',') }}
db-list = {{ mariadb_server_list | join(',') }}
partition-list = {{ mariadb_path_list | join(',') }}
partition-list = {{ mariadb_path_list | join(',') }}
...
@@ -85,6 +105,9 @@ db-prefered-master = {{ mariadb_server_list[0] }}
...
@@ -85,6 +105,9 @@ db-prefered-master = {{ mariadb_server_list[0] }}
proxysql-servers = {{ ipv4 }}
proxysql-servers = {{ ipv4 }}
password = ${repman-parameter:password}
password = ${repman-parameter:password}
proxysql-partition = ${buildout:directory}
proxysql-partition = ${buildout:directory}
proxy-tags = {{ parameter_dict.get("proxy-tags", "pkg,masterslave,linux,noreadwritesplit") }}
logical-backup-cron = {{ parameter_dict.get("logical-backup-cron", "0 1 * * 6") }}
physical-backup-cron = {{ parameter_dict.get("physical-backup-cron", "0 0 * * *") }}
[{{ 'config-' ~ name }}]
[{{ 'config-' ~ name }}]
recipe = slapos.recipe.template:jinja2
recipe = slapos.recipe.template:jinja2
...
@@ -185,7 +208,7 @@ secure-port = ${instance-parameter:repman-secure-port}
...
@@ -185,7 +208,7 @@ secure-port = ${instance-parameter:repman-secure-port}
port = ${instance-parameter:repman-port}
port = ${instance-parameter:repman-port}
ipv4 = ${instance-parameter:ipv4-random}
ipv4 = ${instance-parameter:ipv4-random}
username = admin
username = admin
password = ${
repman-password:passw
d}
password = ${
publish-early:passwor
d}
cluster-d = ${repman:clusters}
cluster-d = ${repman:clusters}
mysql-bin-dir = {{ mariadb_location }}/bin
mysql-bin-dir = {{ mariadb_location }}/bin
mysqlbinlog-path = {{ mariadb_location }}/bin/mysqlbinlog
mysqlbinlog-path = {{ mariadb_location }}/bin/mysqlbinlog
...
@@ -200,6 +223,7 @@ mail-smtp-addr = {{ slapparameter_dict.get("mail-smtp-addr", "localhost:25") }}
...
@@ -200,6 +223,7 @@ mail-smtp-addr = {{ slapparameter_dict.get("mail-smtp-addr", "localhost:25") }}
mail-smtp-password = {{ slapparameter_dict.get("mail-smtp-password", "") }}
mail-smtp-password = {{ slapparameter_dict.get("mail-smtp-password", "") }}
mail-smtp-user = {{ slapparameter_dict.get("mail-smtp-user", "") }}
mail-smtp-user = {{ slapparameter_dict.get("mail-smtp-user", "") }}
mail-to = {{ slapparameter_dict.get("mail-to", "") }}
mail-to = {{ slapparameter_dict.get("mail-to", "") }}
enabled-tags = {{ slapparameter_dict.get("tags", tags) }}
[repman-config-folder]
[repman-config-folder]
recipe = plone.recipe.command
recipe = plone.recipe.command
...
@@ -322,14 +346,15 @@ recipe = slapos.cookbook:publish-early
...
@@ -322,14 +346,15 @@ recipe = slapos.cookbook:publish-early
-init =
-init =
monitor-password monitor-htpasswd:passwd
monitor-password monitor-htpasswd:passwd
root-password gen-root-password:passwd
root-password gen-root-password:passwd
password repman-password:passwd
[publish-connection-parameter]
[publish-connection-parameter]
<= monitor-publish
<= monitor-publish
-extends = publish-early
recipe = slapos.cookbook:publish
recipe = slapos.cookbook:publish
backend-url = ${nginx-parameter:backend-ssl-url}
backend-url = ${nginx-parameter:backend-ssl-url}
url = ${repman-frontend:connection-secure_access}
url = ${repman-frontend:connection-secure_access}
username = ${repman-parameter:username}
username = ${repman-parameter:username}
password = ${repman-parameter:password}
{% for name, value in publish_dict.items() -%}
{% for name, value in publish_dict.items() -%}
{{ name }} = {{ value }}
{{ name }} = {{ value }}
{% endfor %}
{% endfor %}
...
...
software/repman/templates/cluster-config.toml.in
View file @
10e03fe5
...
@@ -16,11 +16,11 @@ proxysql-bootstrap = true
...
@@ -16,11 +16,11 @@ proxysql-bootstrap = true
proxysql-admin-port = {{ parameter_dict['proxy-admin-port'] }}
proxysql-admin-port = {{ parameter_dict['proxy-admin-port'] }}
proxysql-password = "{{ parameter_dict['password'] }}"
proxysql-password = "{{ parameter_dict['password'] }}"
prov-proxy-tags = "
pkg,masterslave,linux,noreadwritesplit
"
prov-proxy-tags = "
{{ parameter_dict['proxy-tags'] }}
"
monitoring-scheduler = true
monitoring-scheduler = true
scheduler-db-servers-logical-backup = true
scheduler-db-servers-logical-backup = true
scheduler-db-servers-logical-backup-cron
= "0 0 1 * * 6
"
scheduler-db-servers-logical-backup-cron
= "0 {{ parameter_dict['logical-backup-cron'] }}
"
scheduler-db-servers-logs = true
scheduler-db-servers-logs = true
scheduler-db-servers-logs-cron = "0 0 * * * *"
scheduler-db-servers-logs-cron = "0 0 * * * *"
#scheduler-db-servers-logs-table-keep = 12
#scheduler-db-servers-logs-table-keep = 12
...
@@ -31,7 +31,7 @@ scheduler-db-servers-logs-table-rotate-cron = "0 0 23 * * *"
...
@@ -31,7 +31,7 @@ scheduler-db-servers-logs-table-rotate-cron = "0 0 23 * * *"
scheduler-db-servers-optimize = true
scheduler-db-servers-optimize = true
scheduler-db-servers-optimize-cron = "0 0 3 1 * 5"
scheduler-db-servers-optimize-cron = "0 0 3 1 * 5"
scheduler-db-servers-physical-backup = true
scheduler-db-servers-physical-backup = true
scheduler-db-servers-physical-backup-cron = "0
0 0 * * *
"
scheduler-db-servers-physical-backup-cron = "0
{{ parameter_dict['physical-backup-cron'] }}
"
backup-physical-type = "mariabackup"
backup-physical-type = "mariabackup"
backup-logical-type = "mysqldump"
backup-logical-type = "mysqldump"
# scheduler-db-servers-receiver-ports="4444,4445"
# scheduler-db-servers-receiver-ports="4444,4445"
\ No newline at end of file
software/repman/templates/config.toml.in
View file @
10e03fe5
...
@@ -10,8 +10,6 @@ db-servers-binary-path = "{{ parameter_dict['mysql-bin-dir'] }}"
...
@@ -10,8 +10,6 @@ db-servers-binary-path = "{{ parameter_dict['mysql-bin-dir'] }}"
# Database list of hosts to ignore in election
# Database list of hosts to ignore in election
#db-servers-ignored-hosts =
#db-servers-ignored-hosts =
# Database hosts list to monitor, IP and port (optional), specified in the host:[port] format and separated by commas
# Database hosts list to monitor, IP and port (optional), specified in the host:[port] format and separated by commas
# XXX - check if mandatory (as present in cluster)
#db-servers-hosts =
monitoring-address = "{{ parameter_dict['ipv4'] }}"
monitoring-address = "{{ parameter_dict['ipv4'] }}"
#haproxy = true
#haproxy = true
...
@@ -30,10 +28,15 @@ haproxy-binary-path = "{{ parameter_dict['haproxy-bin'] }}"
...
@@ -30,10 +28,15 @@ haproxy-binary-path = "{{ parameter_dict['haproxy-bin'] }}"
#haproxy-write-port =
#haproxy-write-port =
# Use restic to archive and restore backups
# Use restic to archive and restore backups
backup = true
backup-restic = true
backup-restic = true
backup-restic-binary-path = "{{ parameter_dict['restic-bin'] }}"
backup-restic-binary-path = "{{ parameter_dict['restic-bin'] }}"
backup-restic-aws = false
backup-restic-aws = false
backup-restic-password = ""
backup-restic-password = "{{ parameter_dict['password'] }}"
backup-mysqlclient-path = "{{ parameter_dict['mysqlclient-path'] }}"
backup-mysqlbinlog-path = "{{ parameter_dict['mysqlbinlog-path'] }}"
backup-mysqldump-path = "{{ parameter_dict['mysqldump-path'] }}"
# Mail configuration
# Mail configuration
# Alert email sender (default "mrm@localhost")
# Alert email sender (default "mrm@localhost")
...
@@ -45,12 +48,8 @@ mail-smtp-user = "{{ parameter_dict['mail-smtp-user'] }}"
...
@@ -45,12 +48,8 @@ mail-smtp-user = "{{ parameter_dict['mail-smtp-user'] }}"
# Alert email recipients, separated by commas
# Alert email recipients, separated by commas
mail-to = "{{ parameter_dict['mail-to'] }}"
mail-to = "{{ parameter_dict['mail-to'] }}"
mysqlbinlog-path = "{{ parameter_dict['mysqlbinlog-path'] }}"
mysqlclient-path = "{{ parameter_dict['mysqlclient-path'] }}"
mysqldump-path = "{{ parameter_dict['mysqldump-path'] }}"
prov-orchestrator = "slapos"
prov-orchestrator = "slapos"
prov-db-tags
="gtidstrict,bind,pkg,innodb,noquerycache,slow,pfs,linux,readonly,diskmonitor,sqlerror,compressbinlog"
prov-db-tags
= {{ parameter_dict['enabled-tags') }}
sysbench-binary-path = "{{ parameter_dict['sysbench-bin'] }}"
sysbench-binary-path = "{{ parameter_dict['sysbench-bin'] }}"
# Number of threads to run benchmark (default 4)
# Number of threads to run benchmark (default 4)
...
...
software/repman/templates/repman-manager.sh.in
View file @
10e03fe5
...
@@ -25,6 +25,18 @@ wait_database () {
...
@@ -25,6 +25,18 @@ wait_database () {
done
done
}
}
activate_proxy () {
NAME=$1
$URL="{{ secure_url }}/api/clusters/$NAME/settings/actions/switch/database-hearbeat"
CODE=$(curl -H "Authorization: Bearer ${TOKEN}" -o /dev/null -w "%{http_code}" $URL)
if [ $CODE -eq 200 ]; then
exit 0;
else
echo "ERROR: failed to activate proxy: $URL returned code $CODE"
exit 1;
fi
}
TOKEN=$(curl -s -X POST --data '{"username":"{{ username }}","password":"{{ password}}"}' {{ secure_url }}/api/login | {{ jq_bin }} -r '.token')
TOKEN=$(curl -s -X POST --data '{"username":"{{ username }}","password":"{{ password}}"}' {{ secure_url }}/api/login | {{ jq_bin }} -r '.token')
# Always reload cluster configuration to apply recent changes
# Always reload cluster configuration to apply recent changes
...
@@ -40,15 +52,19 @@ if [ ! -f "{{ parameter_dict['bootstrap'] }}/{{ name }}_bootstrapped" ]; then
...
@@ -40,15 +52,19 @@ if [ ! -f "{{ parameter_dict['bootstrap'] }}/{{ name }}_bootstrapped" ]; then
wait_database {{ name }}
wait_database {{ name }}
echo "Bootstrap replication on {{ name }}..."
echo "Bootstrap replication on {{ name }}..."
TOKEN=$(curl -s -X POST --data '{"username":"{{ username }}","password":"{{ password}}"}' {{ secure_url }}/api/login | {{ jq_bin }} -r '.token')
#
TOKEN=$(curl -s -X POST --data '{"username":"{{ username }}","password":"{{ password}}"}' {{ secure_url }}/api/login | {{ jq_bin }} -r '.token')
curl -H "Authorization: Bearer ${TOKEN}" \
curl -H "Authorization: Bearer ${TOKEN}" \
{{ secure_url }}/api/clusters/{{ name }}/actions/replication/cleanup
{{ secure_url }}/api/clusters/{{ name }}/actions/replication/cleanup
CODE=$(curl -H "Authorization: Bearer ${TOKEN}" -o /dev/null -w "%{http_code}" {{ secure_url }}/api/clusters/{{ name }}/actions/replication/bootstrap/master-slave)
CODE=$(curl -H "Authorization: Bearer ${TOKEN}" -o /dev/null -w "%{http_code}" {{ secure_url }}/api/clusters/{{ name }}/actions/replication/bootstrap/master-slave)
SUCCESS=0
if [ $CODE -eq 200 ]; then
if [ $CODE -eq 200 ]; then
activate_proxy {{ name }}
if [ $? -eq 0 ]; then
# Mark boostrap done!
# Mark boostrap done!
echo "Cluster {{ name }} replication bootstrapped"
echo "Cluster {{ name }} replication bootstrapped"
echo "DO NOT REMOVE THIS FILE" > {{ parameter_dict['bootstrap'] }}/{{ name }}_bootstrapped
echo "DO NOT REMOVE THIS FILE" > {{ parameter_dict['bootstrap'] }}/{{ name }}_bootstrapped
fi
else
else
echo "ERROR: Failed to bootstrap cluster {{ name }}... http_code $CODE"
echo "ERROR: Failed to bootstrap cluster {{ name }}... http_code $CODE"
fi
fi
...
...
Write
Preview
Markdown
is supported
0%
Try again
or
attach a new file
Attach a file
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment