Skip to content
Projects
Groups
Snippets
Help
Loading...
Help
Support
Keyboard shortcuts
?
Submit feedback
Contribute to GitLab
Sign in / Register
Toggle navigation
slapos
Project overview
Project overview
Details
Activity
Releases
Repository
Repository
Files
Commits
Branches
Tags
Contributors
Graph
Compare
Issues
0
Issues
0
List
Boards
Labels
Milestones
Merge Requests
0
Merge Requests
0
CI / CD
CI / CD
Pipelines
Jobs
Schedules
Analytics
Analytics
CI / CD
Repository
Value Stream
Wiki
Wiki
Snippets
Snippets
Members
Members
Collapse sidebar
Close sidebar
Activity
Graph
Create a new issue
Jobs
Commits
Issue Boards
Open sidebar
Lukas Niegsch
slapos
Commits
fc87834c
Commit
fc87834c
authored
May 05, 2020
by
Alain Takoudjou
Browse files
Options
Browse Files
Download
Email Patches
Plain Diff
repman: fix conflict with socat port and inform repman which port to uses
parent
0502172c
Changes
5
Show whitespace changes
Inline
Side-by-side
Showing
5 changed files
with
28 additions
and
12 deletions
+28
-12
software/repman/buildout.hash.cfg
software/repman/buildout.hash.cfg
+4
-4
software/repman/instance-mariadb.cfg.jinja2.in
software/repman/instance-mariadb.cfg.jinja2.in
+5
-1
software/repman/instance-repman.cfg.jinja2.in
software/repman/instance-repman.cfg.jinja2.in
+5
-1
software/repman/templates/cluster-config.toml.in
software/repman/templates/cluster-config.toml.in
+1
-1
software/repman/templates/dbjobs.in
software/repman/templates/dbjobs.in
+13
-5
No files found.
software/repman/buildout.hash.cfg
View file @
fc87834c
...
@@ -18,7 +18,7 @@ md5sum = da50540b1c0fc69ffbf8f6e345a3baad
...
@@ -18,7 +18,7 @@ md5sum = da50540b1c0fc69ffbf8f6e345a3baad
[instance-repman.cfg]
[instance-repman.cfg]
_update_hash_filename_ = instance-repman.cfg.jinja2.in
_update_hash_filename_ = instance-repman.cfg.jinja2.in
md5sum =
482c9f2cad5a35fd0e556dd353a0d39c
md5sum =
2b5552d587e7735b4ff6daec24fcb8c1
[config-toml.in]
[config-toml.in]
_update_hash_filename_ = templates/config.toml.in
_update_hash_filename_ = templates/config.toml.in
...
@@ -26,7 +26,7 @@ md5sum = 5cfa75ca5a0048a050c0041dfe541f3d
...
@@ -26,7 +26,7 @@ md5sum = 5cfa75ca5a0048a050c0041dfe541f3d
[config-cluster-toml.in]
[config-cluster-toml.in]
_update_hash_filename_ = templates/cluster-config.toml.in
_update_hash_filename_ = templates/cluster-config.toml.in
md5sum =
45f4628a5660b663dfdd9037e803d884
md5sum =
9be37f86d0fad81cc13e64c5a0557c59
[nginx.conf.in]
[nginx.conf.in]
_update_hash_filename_ = templates/nginx.conf.in
_update_hash_filename_ = templates/nginx.conf.in
...
@@ -34,7 +34,7 @@ md5sum = 0eeb24c6aa0760f0d33c4cc2828ddf30
...
@@ -34,7 +34,7 @@ md5sum = 0eeb24c6aa0760f0d33c4cc2828ddf30
[template-mariadb.cfg]
[template-mariadb.cfg]
_update_hash_filename_ = instance-mariadb.cfg.jinja2.in
_update_hash_filename_ = instance-mariadb.cfg.jinja2.in
md5sum =
7049767810d8b8a6a16668512672f5a3
md5sum =
63af2f4fc184cfc9d09f8b72dee5165a
[template-my-cnf]
[template-my-cnf]
_update_hash_filename_ = templates/my.cnf.in
_update_hash_filename_ = templates/my.cnf.in
...
@@ -58,7 +58,7 @@ md5sum = 99324b56192003254081ef336dcee94c
...
@@ -58,7 +58,7 @@ md5sum = 99324b56192003254081ef336dcee94c
[dbjobs-in]
[dbjobs-in]
_update_hash_filename_ = templates/dbjobs.in
_update_hash_filename_ = templates/dbjobs.in
md5sum =
d623a4c684578602b9d8ee49034aeb
fa
md5sum =
9731eff8918fa3e68198de6b1d45ec
fa
[mysqld-need-start.sh.in]
[mysqld-need-start.sh.in]
_update_hash_filename_ = templates/mysqld-need-start.sh.in
_update_hash_filename_ = templates/mysqld-need-start.sh.in
...
...
software/repman/instance-mariadb.cfg.jinja2.in
View file @
fc87834c
...
@@ -21,6 +21,7 @@ recipe = slapos.cookbook:publish.serialised
...
@@ -21,6 +21,7 @@ recipe = slapos.cookbook:publish.serialised
database-host = {{ host }}:{{ port }}
database-host = {{ host }}:{{ port }}
monitor-base-url = ${monitor-publish-parameters:monitor-base-url}
monitor-base-url = ${monitor-publish-parameters:monitor-base-url}
partition-path = ${buildout:directory}
partition-path = ${buildout:directory}
receiver-port = ${dbjob-parameter:socat-port}
[publish-early]
[publish-early]
recipe = slapos.cookbook:publish-early
recipe = slapos.cookbook:publish-early
...
@@ -250,12 +251,15 @@ dbjob-cnf = ${directory:etc}/mysql/my.cnf
...
@@ -250,12 +251,15 @@ dbjob-cnf = ${directory:etc}/mysql/my.cnf
log-dir = ${directory:mysql}/.system/logs
log-dir = ${directory:mysql}/.system/logs
tmp-dir = ${directory:tmp}
tmp-dir = ${directory:tmp}
mysqld-socket = ${my-cnf-parameters:socket}
mysqld-socket = ${my-cnf-parameters:socket}
socat-port = {{
port + 1
}}
socat-port = {{
int(port) + 9
}}
restart-script = ${mysqld-restart-script:rendered}
restart-script = ${mysqld-restart-script:rendered}
socat-location = {{ parameter_dict['socat-location'] }}
socat-location = {{ parameter_dict['socat-location'] }}
mysql-location = {{ parameter_dict['mariadb-location'] }}
mysql-location = {{ parameter_dict['mariadb-location'] }}
gzip-location = {{ parameter_dict['gzip-location'] }}
ip = {{ ip }}
ip = {{ ip }}
host = {{ host }}
port = {{ port }}
port = {{ port }}
use-ipv6 = {{ dumps(use_ipv6) }}
[dbjobs-executable]
[dbjobs-executable]
< = jinja2-template-executable
< = jinja2-template-executable
...
...
software/repman/instance-repman.cfg.jinja2.in
View file @
fc87834c
...
@@ -3,6 +3,7 @@
...
@@ -3,6 +3,7 @@
{% set monitor_base_url_dict = {} -%}
{% set monitor_base_url_dict = {} -%}
{% set mariadb_dict = {} -%}
{% set mariadb_dict = {} -%}
{% set mariadb_server_list = [] -%}
{% set mariadb_server_list = [] -%}
{% set receiver_port_list = [] -%}
{% set mariadb_path_list = [] -%}
{% set mariadb_path_list = [] -%}
{% set ip = (ipv6_set | list)[0] -%}
{% set ip = (ipv6_set | list)[0] -%}
{% set ipv4 = (ipv4_set | list)[0] -%}
{% set ipv4 = (ipv4_set | list)[0] -%}
...
@@ -95,11 +96,13 @@ config-cluster = {{ name }}
...
@@ -95,11 +96,13 @@ config-cluster = {{ name }}
config-name = {{ dbname }}
config-name = {{ dbname }}
return =
return =
database-host
database-host
receiver-port
monitor-base-url
monitor-base-url
partition-path
partition-path
{% do part_list.append(section) -%}
{% do part_list.append(section) -%}
{% do mariadb_server_list.append('${' ~ section ~ ':connection-database-host}') -%}
{% do mariadb_server_list.append('${' ~ section ~ ':connection-database-host}') -%}
{% do receiver_port_list.append('${' ~ section ~ ':connection-receiver-port}') -%}
{% do mariadb_path_list.append('${' ~ section ~ ':connection-partition-path}') -%}
{% do mariadb_path_list.append('${' ~ section ~ ':connection-partition-path}') -%}
{% do monitor_base_url_dict.__setitem__('mariadb' ~ i, '${' ~ section ~ ':connection-monitor-base-url}') -%}
{% do monitor_base_url_dict.__setitem__('mariadb' ~ i, '${' ~ section ~ ':connection-monitor-base-url}') -%}
...
@@ -138,6 +141,7 @@ db-prefered-master = {{ mariadb_server_list[0] }}
...
@@ -138,6 +141,7 @@ db-prefered-master = {{ mariadb_server_list[0] }}
proxysql-servers = {{ ipv4 }}
proxysql-servers = {{ ipv4 }}
password = ${repman-parameter:password}
password = ${repman-parameter:password}
proxysql-partition = ${buildout:directory}
proxysql-partition = ${buildout:directory}
receiver-port-list = {{ receiver_port_list | join(',') }}
proxy-tags = {{ parameter_dict.get("proxy-tags", "pkg,masterslave,linux,noreadwritesplit") }}
proxy-tags = {{ parameter_dict.get("proxy-tags", "pkg,masterslave,linux,noreadwritesplit") }}
logical-backup-cron = {{ parameter_dict.get("logical-backup-cron", "0 22 * * *") }}
logical-backup-cron = {{ parameter_dict.get("logical-backup-cron", "0 22 * * *") }}
physical-backup-cron = {{ parameter_dict.get("physical-backup-cron", "0 0 * * *") }}
physical-backup-cron = {{ parameter_dict.get("physical-backup-cron", "0 0 * * *") }}
...
@@ -476,7 +480,7 @@ config-check-secure = 1
...
@@ -476,7 +480,7 @@ config-check-secure = 1
[repman-backend-promise]
[repman-backend-promise]
<= monitor-promise-base
<= monitor-promise-base
module = check_url_available
module = check_url_available
name = check_repman_
front
end.py
name = check_repman_
back
end.py
config-url = ${nginx-parameter:backend-ssl-url}
config-url = ${nginx-parameter:backend-ssl-url}
config-check-secure = 1
config-check-secure = 1
...
...
software/repman/templates/cluster-config.toml.in
View file @
fc87834c
...
@@ -36,7 +36,7 @@ scheduler-db-servers-physical-backup = true
...
@@ -36,7 +36,7 @@ scheduler-db-servers-physical-backup = true
scheduler-db-servers-physical-backup-cron = "0 {{ parameter_dict['physical-backup-cron'] }}"
scheduler-db-servers-physical-backup-cron = "0 {{ parameter_dict['physical-backup-cron'] }}"
backup-physical-type = "mariabackup"
backup-physical-type = "mariabackup"
backup-logical-type = "mysqldump"
backup-logical-type = "mysqldump"
# scheduler-db-servers-receiver-ports="4444,4445
"
scheduler-db-servers-receiver-ports= "{{ parameter_dict['receiver-port-list'] }}
"
prov-proxy-cpu-cores = {{ parameter_dict['proxy-cpu-cores'] }}
prov-proxy-cpu-cores = {{ parameter_dict['proxy-cpu-cores'] }}
prov-proxy-memory = {{ parameter_dict['proxy-memory'] }}
prov-proxy-memory = {{ parameter_dict['proxy-memory'] }}
...
...
software/repman/templates/dbjobs.in
View file @
fc87834c
...
@@ -6,7 +6,12 @@ ERROLOG={{ parameter_dict['mysql-dir'] }}/.system/logs/errors.log
...
@@ -6,7 +6,12 @@ ERROLOG={{ parameter_dict['mysql-dir'] }}/.system/logs/errors.log
SLOWLOG={{ parameter_dict['mysql-dir']}}/.system/logs/sql-slow
SLOWLOG={{ parameter_dict['mysql-dir']}}/.system/logs/sql-slow
BACKUPDIR={{ parameter_dict['mysql-dir'] }}/.system/backup
BACKUPDIR={{ parameter_dict['mysql-dir'] }}/.system/backup
DATADIR={{ parameter_dict['mysql-dir'] }}/
DATADIR={{ parameter_dict['mysql-dir'] }}/
export PATH={{ parameter_dict['socat-location'] }}/bin:{{ parameter_dict['mysql-location'] }}/bin:$PATH
{% if parameter_dict['use-ipv6'] == True -%}
{% set listen = "TCP6-LISTEN" -%}
{% else -%}
{% set listen = "TCP-LISTEN" -%}
{% endif -%}
export PATH={{ parameter_dict['socat-location'] }}/bin:{{ parameter_dict['mysql-location'] }}/bin:{{ parameter_dict['gzip-location'] }}/bin:$PATH
JOBS=( "xtrabackup" "mariabackup" "error" "slowquery" "zfssnapback" "optimize" "reseedxtrabackup" "reseedmariabackup" "reseedmysqldump" "flashbackxtrabackup" "flashbackmariadbackup" "flashbackmysqldump" "stop" "start")
JOBS=( "xtrabackup" "mariabackup" "error" "slowquery" "zfssnapback" "optimize" "reseedxtrabackup" "reseedmariabackup" "reseedmysqldump" "flashbackxtrabackup" "flashbackmariadbackup" "flashbackmysqldump" "stop" "start")
doneJob()
doneJob()
...
@@ -52,6 +57,9 @@ partialRestore()
...
@@ -52,6 +57,9 @@ partialRestore()
mysql --defaults-file={{ parameter_dict['dbjob-cnf'] }} -e"flush privileges;start slave;"
mysql --defaults-file={{ parameter_dict['dbjob-cnf'] }} -e"flush privileges;start slave;"
}
}
kill -9 $(lsof -t -i:{{ parameter_dict['socat-port'] }} -sTCP:LISTEN)
kill -9 $(lsof -t -i:{{ parameter_dict['socat-port'] }} -sTCP6:LISTEN)
for job in "${JOBS[@]}"
for job in "${JOBS[@]}"
do
do
...
@@ -70,13 +78,13 @@ do
...
@@ -70,13 +78,13 @@ do
reseedmysqldump)
reseedmysqldump)
echo "Waiting backup." > {{ parameter_dict['log-dir'] }}/dbjob.out
echo "Waiting backup." > {{ parameter_dict['log-dir'] }}/dbjob.out
pauseJob
pauseJob
socat -u
TCP-LISTEN:{{ parameter_dict['socat-port'] }},reuseaddr STDOUT | gunzip | mysql -h{{ parameter_dict['ip'] }} -P{{ parameter_dict['port'] }} -p$PASSWORD -u$USER
> {{ parameter_dict['log-dir'] }}/dbjob.out 2>&1
socat -u
{{ listen }}:{{ parameter_dict['socat-port'] }},bind={{ parameter_dict['host'] }},reuseaddr STDOUT | gunzip | mysql -h{{ parameter_dict['ip'] }} -P{{ parameter_dict['port'] }} -p$PASSWORD -u$USER --init-command="set sql_log_bin=0"
> {{ parameter_dict['log-dir'] }}/dbjob.out 2>&1
mysql --defaults-file={{ parameter_dict['dbjob-cnf'] }} -e 'start slave;'
mysql --defaults-file={{ parameter_dict['dbjob-cnf'] }} -e 'start slave;'
;;
;;
flashbackmysqldump)
flashbackmysqldump)
echo "Waiting backup." > {{ parameter_dict['log-dir'] }}/dbjob.out
echo "Waiting backup." > {{ parameter_dict['log-dir'] }}/dbjob.out
pauseJob
pauseJob
socat -u
TCP-LISTEN:{{ parameter_dict['socat-port'] }},reuseaddr STDOUT | gunzip | mysql -h{{ parameter_dict['ip'] }} -P{{ parameter_dict['port'] }} -p$PASSWORD -u$USER
> {{ parameter_dict['log-dir'] }}/dbjob.out 2>&1
socat -u
{{ listen }}:{{ parameter_dict['socat-port'] }},bind={{ parameter_dict['host'] }},reuseaddr STDOUT | gunzip | mysql -h{{ parameter_dict['ip'] }} -P{{ parameter_dict['port'] }} -p$PASSWORD -u$USER --init-command="set sql_log_bin=0"
> {{ parameter_dict['log-dir'] }}/dbjob.out 2>&1
mysql --defaults-file={{ parameter_dict['dbjob-cnf'] }} -e 'start slave;'
mysql --defaults-file={{ parameter_dict['dbjob-cnf'] }} -e 'start slave;'
;;
;;
reseedmariabackup)
reseedmariabackup)
...
@@ -84,7 +92,7 @@ do
...
@@ -84,7 +92,7 @@ do
mkdir $BACKUPDIR
mkdir $BACKUPDIR
echo "Waiting backup." > {{ parameter_dict['log-dir'] }}/dbjob.out
echo "Waiting backup." > {{ parameter_dict['log-dir'] }}/dbjob.out
pauseJob
pauseJob
socat -u
TCP-LISTEN:{{ parameter_dict['socat-por
t'] }},reuseaddr STDOUT | mbstream -x -C $BACKUPDIR
socat -u
{{ listen }}:{{ parameter_dict['socat-port'] }},bind={{ parameter_dict['hos
t'] }},reuseaddr STDOUT | mbstream -x -C $BACKUPDIR
# mbstream -p, --parallel
# mbstream -p, --parallel
mariabackup --prepare --export --target-dir=$BACKUPDIR
mariabackup --prepare --export --target-dir=$BACKUPDIR
partialRestore
partialRestore
...
@@ -94,7 +102,7 @@ do
...
@@ -94,7 +102,7 @@ do
mkdir $BACKUPDIR
mkdir $BACKUPDIR
echo "Waiting backup." > {{ parameter_dict['log-dir'] }}/dbjob.out
echo "Waiting backup." > {{ parameter_dict['log-dir'] }}/dbjob.out
pauseJob
pauseJob
socat -u
TCP-LISTEN:{{ parameter_dict['socat-por
t'] }},reuseaddr STDOUT | xbstream -x -C $BACKUPDIR
socat -u
{{ listen }}:{{ parameter_dict['socat-port'] }},bind={{ parameter_dict['hos
t'] }},reuseaddr STDOUT | xbstream -x -C $BACKUPDIR
mariabackup --prepare --export --target-dir=$BACKUPDIR
mariabackup --prepare --export --target-dir=$BACKUPDIR
partialRestore
partialRestore
;;
;;
...
...
Write
Preview
Markdown
is supported
0%
Try again
or
attach a new file
Attach a file
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment