Skip to content
Projects
Groups
Snippets
Help
Loading...
Help
Support
Keyboard shortcuts
?
Submit feedback
Contribute to GitLab
Sign in / Register
Toggle navigation
slapos
Project overview
Project overview
Details
Activity
Releases
Repository
Repository
Files
Commits
Branches
Tags
Contributors
Graph
Compare
Labels
Merge Requests
104
Merge Requests
104
CI / CD
CI / CD
Pipelines
Jobs
Schedules
Analytics
Analytics
CI / CD
Repository
Value Stream
Snippets
Snippets
Members
Members
Collapse sidebar
Close sidebar
Activity
Graph
Jobs
Commits
Open sidebar
nexedi
slapos
Commits
10e03fe5
Commit
10e03fe5
authored
Apr 20, 2020
by
Alain Takoudjou
Browse files
Options
Browse Files
Download
Email Patches
Plain Diff
repman: fixup, add more parameters
parent
44684450
Changes
5
Hide whitespace changes
Inline
Side-by-side
Showing
5 changed files
with
66 additions
and
26 deletions
+66
-26
software/repman/buildout.hash.cfg
software/repman/buildout.hash.cfg
+4
-4
software/repman/instance-repman.cfg.jinja2.in
software/repman/instance-repman.cfg.jinja2.in
+32
-7
software/repman/templates/cluster-config.toml.in
software/repman/templates/cluster-config.toml.in
+3
-3
software/repman/templates/config.toml.in
software/repman/templates/config.toml.in
+7
-8
software/repman/templates/repman-manager.sh.in
software/repman/templates/repman-manager.sh.in
+20
-4
No files found.
software/repman/buildout.hash.cfg
View file @
10e03fe5
...
...
@@ -18,15 +18,15 @@ md5sum = b38fb4c5cdba2223c84d62ef60513e49
[instance-repman.cfg]
_update_hash_filename_ = instance-repman.cfg.jinja2.in
md5sum =
17431810792121fa5c9dc1ccfeec9d13
md5sum =
c682174b1aecf6a079e02d353e339967
[config-toml.in]
_update_hash_filename_ = templates/config.toml.in
md5sum =
66108ddfda7169695d7b876a4e3b246c
md5sum =
446044949547efc66b89aa9f20ef7a2e
[config-cluster-toml.in]
_update_hash_filename_ = templates/cluster-config.toml.in
md5sum =
d883337a75140c08853960cd6a8fdc5f
md5sum =
4d8d38dadda6cf31e078f5fc4918da7d
[nginx.conf.in]
_update_hash_filename_ = templates/nginx.conf.in
...
...
@@ -54,7 +54,7 @@ md5sum = 3ba02b99c3fcbc24b3659917980b310f
[repman-manager-sh.in]
_update_hash_filename_ = templates/repman-manager.sh.in
md5sum =
8ad6565e7b26e57b41bda4023a9d29ed
md5sum =
fca71cf71a94e612b8c9bd69899c568b
[dbjobs-in]
_update_hash_filename_ = templates/dbjobs.in
...
...
software/repman/instance-repman.cfg.jinja2.in
View file @
10e03fe5
...
...
@@ -6,6 +6,7 @@
{% set mariadb_path_list = [] -%}
{% set ip = (ipv6_set | list)[0] -%}
{% set ipv4 = (ipv4_set | list)[0] -%}
{% set tags = "gtidstrict,bind,pkg,innodb,noquerycache,slow,pfs,linux,readonly,diskmonitor,sqlerror,compressbinlog" -%}
[directory]
recipe = slapos.cookbook:mkdirectory
...
...
@@ -31,11 +32,29 @@ cert-file = ${slap-connection:cert-file}
computer-id = ${slap-connection:computer-id}
partition-id = ${slap-connection:partition-id}
[proxy-admin-port]
recipe = slapos.cookbook:free_port
minimum = 6032
maximum = 6132
[proxy-port]
recipe = slapos.cookbook:free_port
minimum = 6033
maximum = 6133
{% do mariadb_dict.__setitem__('computer-memory-percent-threshold', 80) -%}
{% do mariadb_dict.__setitem__('innodb-file-per-table', slapparameter_dict.get('innodb-file-per-table', 1)) -%}
{% set default_parameter_dict = {"cluster1": {"name": "cluster1", "db-prefered-master": "",
"database-amount": 2, "proxysql-user": "external", "proxy-tags": "pkg,masterslave,linux,noreadwritesplit",
"logical-backup-cron": "0 1 * * 6", "physical-backup-cron": "0 0 * * *"}} -%}
{% for name, parameter_dict in slapparameter_dict.get('cluster-partition-dict', default_parameter_dict).items() -%}
# Request mariadb instances
{% for i in range(0, slapparameter_dict.get('database-amount', 2)) -%}
{% set db_amount = 2 if parameter_dict.get('database-amount', 2) < 3 else set db_amount = parameter_dict['database-amount'] -%}
{% for i in range(0, db_amount) -%}
{% do mariadb_dict.__setitem__('tcp-port', 2099 + (i * 100)) -%}
{% set section = 'request-mariadb-' ~ i -%}
{% set name = 'Mariadb-' ~ i -%}
...
...
@@ -44,7 +63,7 @@ partition-id = ${slap-connection:partition-id}
<= request-common
software-type = mariadb
name = {{ name }}
sla-computer_guid = {{ dumps(
slapparameter_dict.get('mariadb-computer-
guid', '')) }}
sla-computer_guid = {{ dumps(
parameter_dict.get('-sla-' ~ i ~'-computer_
guid', '')) }}
{% for key, value in mariadb_dict.items() -%}
config-{{ key }} = {{ dumps(value) }}
{% endfor -%}
...
...
@@ -66,13 +85,14 @@ return =
{% endfor -%}
# Manage Replication Manager clusters
{% set default_parameter_dict = {"cluster1": {"name": "cluster1", "db-prefered-master": "", "proxy-admin-port": 6032, "proxy-port": 6033, "proxysql-user": "external"}} -%}
{% for name, parameter_dict in slapparameter_dict.get('cluster-partition-dict', default_parameter_dict).items() -%}
{% set prefered_master = parameter_dict.pop("db-prefered-master") -%}
{% set prefered_master = parameter_dict.get("db-prefered-master") -%}
[{{ name ~ '-cluster-parameter' }}]
{% for key, value in parameter_dict.items() -%}
{{ key }} = {{ dumps(value) }}
{% endfor -%}
proxysql-user = {{ parameter_dict.get("proxysql-user", "external") }}
proxy-port = ${proxy-port:port}
proxy-admin-port = ${proxy-admin-port:port}
db-credential = repman:${publish-early:root-password}
db-list = {{ mariadb_server_list | join(',') }}
partition-list = {{ mariadb_path_list | join(',') }}
...
...
@@ -85,6 +105,9 @@ db-prefered-master = {{ mariadb_server_list[0] }}
proxysql-servers = {{ ipv4 }}
password = ${repman-parameter:password}
proxysql-partition = ${buildout:directory}
proxy-tags = {{ parameter_dict.get("proxy-tags", "pkg,masterslave,linux,noreadwritesplit") }}
logical-backup-cron = {{ parameter_dict.get("logical-backup-cron", "0 1 * * 6") }}
physical-backup-cron = {{ parameter_dict.get("physical-backup-cron", "0 0 * * *") }}
[{{ 'config-' ~ name }}]
recipe = slapos.recipe.template:jinja2
...
...
@@ -185,7 +208,7 @@ secure-port = ${instance-parameter:repman-secure-port}
port = ${instance-parameter:repman-port}
ipv4 = ${instance-parameter:ipv4-random}
username = admin
password = ${
repman-password:passw
d}
password = ${
publish-early:passwor
d}
cluster-d = ${repman:clusters}
mysql-bin-dir = {{ mariadb_location }}/bin
mysqlbinlog-path = {{ mariadb_location }}/bin/mysqlbinlog
...
...
@@ -200,6 +223,7 @@ mail-smtp-addr = {{ slapparameter_dict.get("mail-smtp-addr", "localhost:25") }}
mail-smtp-password = {{ slapparameter_dict.get("mail-smtp-password", "") }}
mail-smtp-user = {{ slapparameter_dict.get("mail-smtp-user", "") }}
mail-to = {{ slapparameter_dict.get("mail-to", "") }}
enabled-tags = {{ slapparameter_dict.get("tags", tags) }}
[repman-config-folder]
recipe = plone.recipe.command
...
...
@@ -322,14 +346,15 @@ recipe = slapos.cookbook:publish-early
-init =
monitor-password monitor-htpasswd:passwd
root-password gen-root-password:passwd
password repman-password:passwd
[publish-connection-parameter]
<= monitor-publish
-extends = publish-early
recipe = slapos.cookbook:publish
backend-url = ${nginx-parameter:backend-ssl-url}
url = ${repman-frontend:connection-secure_access}
username = ${repman-parameter:username}
password = ${repman-parameter:password}
{% for name, value in publish_dict.items() -%}
{{ name }} = {{ value }}
{% endfor %}
...
...
software/repman/templates/cluster-config.toml.in
View file @
10e03fe5
...
...
@@ -16,11 +16,11 @@ proxysql-bootstrap = true
proxysql-admin-port = {{ parameter_dict['proxy-admin-port'] }}
proxysql-password = "{{ parameter_dict['password'] }}"
prov-proxy-tags = "
pkg,masterslave,linux,noreadwritesplit
"
prov-proxy-tags = "
{{ parameter_dict['proxy-tags'] }}
"
monitoring-scheduler = true
scheduler-db-servers-logical-backup = true
scheduler-db-servers-logical-backup-cron
= "0 0 1 * * 6
"
scheduler-db-servers-logical-backup-cron
= "0 {{ parameter_dict['logical-backup-cron'] }}
"
scheduler-db-servers-logs = true
scheduler-db-servers-logs-cron = "0 0 * * * *"
#scheduler-db-servers-logs-table-keep = 12
...
...
@@ -31,7 +31,7 @@ scheduler-db-servers-logs-table-rotate-cron = "0 0 23 * * *"
scheduler-db-servers-optimize = true
scheduler-db-servers-optimize-cron = "0 0 3 1 * 5"
scheduler-db-servers-physical-backup = true
scheduler-db-servers-physical-backup-cron = "0
0 0 * * *
"
scheduler-db-servers-physical-backup-cron = "0
{{ parameter_dict['physical-backup-cron'] }}
"
backup-physical-type = "mariabackup"
backup-logical-type = "mysqldump"
# scheduler-db-servers-receiver-ports="4444,4445"
\ No newline at end of file
software/repman/templates/config.toml.in
View file @
10e03fe5
...
...
@@ -10,8 +10,6 @@ db-servers-binary-path = "{{ parameter_dict['mysql-bin-dir'] }}"
# Database list of hosts to ignore in election
#db-servers-ignored-hosts =
# Database hosts list to monitor, IP and port (optional), specified in the host:[port] format and separated by commas
# XXX - check if mandatory (as present in cluster)
#db-servers-hosts =
monitoring-address = "{{ parameter_dict['ipv4'] }}"
#haproxy = true
...
...
@@ -30,10 +28,15 @@ haproxy-binary-path = "{{ parameter_dict['haproxy-bin'] }}"
#haproxy-write-port =
# Use restic to archive and restore backups
backup = true
backup-restic = true
backup-restic-binary-path = "{{ parameter_dict['restic-bin'] }}"
backup-restic-aws = false
backup-restic-password = ""
backup-restic-password = "{{ parameter_dict['password'] }}"
backup-mysqlclient-path = "{{ parameter_dict['mysqlclient-path'] }}"
backup-mysqlbinlog-path = "{{ parameter_dict['mysqlbinlog-path'] }}"
backup-mysqldump-path = "{{ parameter_dict['mysqldump-path'] }}"
# Mail configuration
# Alert email sender (default "mrm@localhost")
...
...
@@ -45,12 +48,8 @@ mail-smtp-user = "{{ parameter_dict['mail-smtp-user'] }}"
# Alert email recipients, separated by commas
mail-to = "{{ parameter_dict['mail-to'] }}"
mysqlbinlog-path = "{{ parameter_dict['mysqlbinlog-path'] }}"
mysqlclient-path = "{{ parameter_dict['mysqlclient-path'] }}"
mysqldump-path = "{{ parameter_dict['mysqldump-path'] }}"
prov-orchestrator = "slapos"
prov-db-tags
="gtidstrict,bind,pkg,innodb,noquerycache,slow,pfs,linux,readonly,diskmonitor,sqlerror,compressbinlog"
prov-db-tags
= {{ parameter_dict['enabled-tags') }}
sysbench-binary-path = "{{ parameter_dict['sysbench-bin'] }}"
# Number of threads to run benchmark (default 4)
...
...
software/repman/templates/repman-manager.sh.in
View file @
10e03fe5
...
...
@@ -25,6 +25,18 @@ wait_database () {
done
}
activate_proxy () {
NAME=$1
$URL="{{ secure_url }}/api/clusters/$NAME/settings/actions/switch/database-hearbeat"
CODE=$(curl -H "Authorization: Bearer ${TOKEN}" -o /dev/null -w "%{http_code}" $URL)
if [ $CODE -eq 200 ]; then
exit 0;
else
echo "ERROR: failed to activate proxy: $URL returned code $CODE"
exit 1;
fi
}
TOKEN=$(curl -s -X POST --data '{"username":"{{ username }}","password":"{{ password}}"}' {{ secure_url }}/api/login | {{ jq_bin }} -r '.token')
# Always reload cluster configuration to apply recent changes
...
...
@@ -40,15 +52,19 @@ if [ ! -f "{{ parameter_dict['bootstrap'] }}/{{ name }}_bootstrapped" ]; then
wait_database {{ name }}
echo "Bootstrap replication on {{ name }}..."
TOKEN=$(curl -s -X POST --data '{"username":"{{ username }}","password":"{{ password}}"}' {{ secure_url }}/api/login | {{ jq_bin }} -r '.token')
#
TOKEN=$(curl -s -X POST --data '{"username":"{{ username }}","password":"{{ password}}"}' {{ secure_url }}/api/login | {{ jq_bin }} -r '.token')
curl -H "Authorization: Bearer ${TOKEN}" \
{{ secure_url }}/api/clusters/{{ name }}/actions/replication/cleanup
CODE=$(curl -H "Authorization: Bearer ${TOKEN}" -o /dev/null -w "%{http_code}" {{ secure_url }}/api/clusters/{{ name }}/actions/replication/bootstrap/master-slave)
SUCCESS=0
if [ $CODE -eq 200 ]; then
# Mark boostrap done!
echo "Cluster {{ name }} replication bootstrapped"
echo "DO NOT REMOVE THIS FILE" > {{ parameter_dict['bootstrap'] }}/{{ name }}_bootstrapped
activate_proxy {{ name }}
if [ $? -eq 0 ]; then
# Mark boostrap done!
echo "Cluster {{ name }} replication bootstrapped"
echo "DO NOT REMOVE THIS FILE" > {{ parameter_dict['bootstrap'] }}/{{ name }}_bootstrapped
fi
else
echo "ERROR: Failed to bootstrap cluster {{ name }}... http_code $CODE"
fi
...
...
Write
Preview
Markdown
is supported
0%
Try again
or
attach a new file
Attach a file
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment