repman: check start and stop needed for all proxysql and mariadb services

software/repman/buildout.hash.cfg

@@ -14,15 +14,15 @@
 # not need these here).
 [instance.cfg]
 filename = instance.cfg.in
-md5sum = 8f3e5d9c0de16006cb36f0b8121fc11a
+md5sum = da50540b1c0fc69ffbf8f6e345a3baad
 
 [instance-repman.cfg]
 _update_hash_filename_ = instance-repman.cfg.jinja2.in
-md5sum = 2688b8aa946d4878d3f1c4fa08375cfe
+md5sum = 3b858324e8940a08379956bf76fb88bd
 
 [config-toml.in]
 _update_hash_filename_ = templates/config.toml.in
-md5sum = 1efa65af63b14ca2dd1b8c692b2af80e
+md5sum = 3adc842de5c0e8c94e90ac5f41ecab6c
 
 [config-cluster-toml.in]
 _update_hash_filename_ = templates/cluster-config.toml.in
@@ -34,7 +34,7 @@ md5sum = 0eeb24c6aa0760f0d33c4cc2828ddf30
 
 [template-mariadb.cfg]
 _update_hash_filename_ = instance-mariadb.cfg.jinja2.in
-md5sum = 10dea9ec48ce889315202d112a66f121
+md5sum = 0c6e9a3220cf6e8b096e523d0d1f407a
 
 [template-my-cnf]
 _update_hash_filename_ = templates/my.cnf.in
@@ -59,3 +59,11 @@ md5sum = 99324b56192003254081ef336dcee94c
 [dbjobs-in]
 _update_hash_filename_ = templates/dbjobs.in
 md5sum = d623a4c684578602b9d8ee49034aebfa
+
+[mysqld-need-start.sh.in]
+_update_hash_filename_ = templates/mysqld-need-start.sh.in
+md5sum = 76b984ef5e37248bc3a8010a74b24de5
+
+[proxy-need-start-stop.sh.in]
+_update_hash_filename_ = templates/proxy-need-start-stop.sh.in
+md5sum = 1ee3d69f0866605b33eac7ea596b21c3

software/repman/instance-mariadb.cfg.jinja2.in

@@ -275,16 +275,38 @@ command = ${dbjobs-executable:rendered}
 < = jinja2-template-executable
 rendered = ${directory:bin}/mysqld_restart
 template = inline:#!/bin/sh
+  ${mariadb-controller-bin:wrapper-path} status
   ${mariadb-controller-bin:wrapper-path} restart mariadb
   sleep 1
   ${mariadb-controller-bin:wrapper-path} status
 
+[{{ section('mariadb-need-start') }}]
+recipe = slapos.cookbook:cron.d
+cron-entries = ${cron:cron-entries}
+name = mariadb-need-start
+frequency = * * * * *
+command = ${template-mysqld-need-start:rendered}
+
+[template-mysqld-need-start]
+< = jinja2-template-executable
+rendered = ${directory:bin}/mysqld_need_start
+template = {{ parameter_dict['template-mysqld-need-start'] }}
+context =
+  key mariadb_controller mariadb-controller-bin:wrapper-path
+  raw username           {{ slapparameter_dict['repman-user'] }}
+  raw repman_url         {{ slapparameter_dict['repman-url'] }}
+  raw jq_bin             {{ jq_bin }}
+  raw cluster            {{ slapparameter_dict['cluster'] }}
+  raw db_host            {{ host }}
+  raw db_port            {{ port }}
+  raw bash_bin           {{ bash_bin }}
+  raw curl_bin           {{ curl_bin }}
+
 # Donwnload mariadb configuration from repman
 [{{ section('install-mysql-config') }}]
 recipe = plone.recipe.command
 stop-on-error = true
-# XXX - for now, we make only one cluster
-cluster = cluster1
+cluster = {{ slapparameter_dict['cluster'] }}
 config = ${directory:etc}/mysql/my.cnf
 command =
   cd ${directory:config-tmp} &&

software/repman/instance-repman-input-schema.json

@@ -237,9 +237,9 @@
 						},
 						"failover-mode": {
 							"title": "Failover mode",
-							"description": "Failover is manual or automatic (default \"automatic\").",
+							"description": "Failover is manual or automatic (default \"manual\").",
 							"type": "string",
-							"default": "automatic",
+							"default": "manual",
               "enum": [
                 "manual",
                 "automatic"

software/repman/instance-repman.cfg.jinja2.in

@@ -6,6 +6,7 @@
 {% set mariadb_path_list = [] -%}
 {% set ip = (ipv6_set | list)[0] -%}
 {% set ipv4 = (ipv4_set | list)[0] -%}
+{% set cluster_list = [] -%}
 {% set tags = "gtidstrict,bind,pkg,innodb,noquerycache,slow,pfs,linux,readonly,diskmonitor,sqlerror,compressbinlog" -%}
 {% set frontend_parameter_dict = slapparameter_dict.get('slave-frontend', {}) -%}
 
@@ -17,12 +18,17 @@ var = ${:home}/var
 run = ${:var}/run
 scripts = ${:etc}/run
 service = ${:etc}/service
+controller = ${:etc}/controller
 promise = ${:etc}/promise
 log = ${:var}/log
 data = ${:var}/lib
 nginx-prefix = ${:var}/nginx
 tmp = ${:home}/tmp
 
+{% import "supervisord_lib" as supervisord_lib with context %}
+{% set proxysql_controller = "proxysql-controller" -%}
+{{ supervisord_lib.supervisord(proxysql_controller, buildout_bin_directory, supervisord_conf, use_service_hash=False) }}
+{% do part_list.append("proxysql-controller-service") -%}
 
 [request-common]
 recipe = slapos.cookbook:request.serialised
@@ -33,18 +39,6 @@ cert-file = ${slap-connection:cert-file}
 computer-id = ${slap-connection:computer-id}
 partition-id = ${slap-connection:partition-id}
 
-[proxy-admin-port]
-recipe = slapos.cookbook:free_port
-ip = {{ ipv4 }}
-minimum = 6032
-maximum = 6132
-
-[proxy-port]
-recipe = slapos.cookbook:free_port
-ip = {{ ipv4 }}
-minimum = 7032
-maximum = 7132
-
 
 {% do mariadb_dict.__setitem__('computer-memory-percent-threshold', 80) -%}
 {% set default_parameter_dict = {"cluster1": {"name": "cluster1", "db-prefered-master": "",
@@ -64,21 +58,24 @@ maximum = 7132
 {% for i in range(0, db_amount) -%}
 {% do mariadb_dict.__setitem__('tcp-port', 2099 + (i * 100)) -%}
 {% set section = 'request-mariadb-' ~ i -%}
-{% set name = 'Mariadb-' ~ i -%}
+{% set dbname = 'Mariadb-' ~ i -%}
 
 [{{ section }}]
 <= request-common
 software-type = mariadb
-name = {{ name }}
+name = {{ dbname }}
 sla-computer_guid = {{ dumps(parameter_dict.get('-sla-' ~ i ~'-computer_guid', '')) }}
 {% for key, value in mariadb_dict.items() -%}
 config-{{ key }} = {{ dumps(value) }}
 {% endfor -%}
 config-monitor-passwd = ${publish-early:monitor-password}
 config-root-password = ${publish-early:db-root-password}
+config-repman-user = ${repman-parameter:username}
+#config-repman-passwd =  ${repman-parameter:password}
 config-repman-url = ${nginx-parameter:backend-url}
-config-repman-secure-url = ${nginx-parameter:backend-url}
-config-name = {{ name }}
+config-repman-secure-url = ${nginx-parameter:backend-ssl-url}
+config-cluster = {{ name }}
+config-name = {{ dbname }}
 return = 
   database-host
   monitor-base-url
@@ -92,14 +89,26 @@ return =
 {% endfor -%}
 
 # Manage Replication Manager clusters
+[{{name}}-admin-port]
+recipe = slapos.cookbook:free_port
+ip = {{ ipv4 }}
+minimum = 6032
+maximum = 6132
+
+[{{name}}-port]
+recipe = slapos.cookbook:free_port
+ip = {{ ipv4 }}
+minimum = 7032
+maximum = 7132
+
 {% set prefered_master = parameter_dict.get("db-prefered-master") -%}
 [{{ name ~ '-cluster-parameter' }}]
 {% for key, value in parameter_dict.items() -%}
 {{ key }} = {{ dumps(value) }}
 {% endfor -%}
 proxysql-user = {{ parameter_dict.get("proxysql-user", "external") }}
-proxy-port = ${proxy-port:port}
-proxy-admin-port = ${proxy-admin-port:port}
+proxy-port = {{ '${' ~ name ~ '-port:port}' }}
+proxy-admin-port = {{ '${' ~ name ~ '-admin-port:port}' }}
 db-credential = repman:${publish-early:db-root-password}
 db-list = {{ mariadb_server_list | join(',') }}
 partition-list = {{ mariadb_path_list | join(',') }}
@@ -123,7 +132,7 @@ db-memory = {{ parameter_dict.get("db-memory", 256) }}
 db-memory-shared-pct = {{ parameter_dict.get("db-memory-shared-pct", "threads:16,innodb:60,myisam:10,aria:10,rocksdb:1,tokudb:1,s3:1,archive:1,querycache:0") }}
 db-memory-threaded-pct = {{ parameter_dict.get("db-memory-threaded-pct", "tmp:70,join:20,sort:10") }}
 # failover
-failover-mode = {{ parameter_dict.get('failover-mode', 'automatic') }}
+failover-mode = {{ parameter_dict.get('failover-mode', 'manual') }}
 failover-limit = {{ parameter_dict.get('failover-limit', 5) }}
 failover-falsepositive-heartbeat = {{ parameter_dict.get('failover-falsepositive-heartbeat', True) }}
 failover-falsepositive-heartbeat-timeout = {{ parameter_dict.get('failover-falsepositive-heartbeat-timeout', 3) }}
@@ -158,12 +167,12 @@ command =
   mkdir -p ${repman:config-tmp}/proxies &&
   mkdir -p ${:data} &&
   cd ${repman:config-tmp}/proxies &&
-  {{ curl_bin }} -o proxies.tar.gz ${nginx-parameter:repman-url}/api/clusters/{{ name }}/servers/{{ ipv4 }}/${proxy-admin-port:port}/config &&
+  {{ curl_bin }} -o proxies.tar.gz ${nginx-parameter:repman-url}/api/clusters/{{ name }}/servers/{{ ipv4 }}/{{ '${' ~ name ~ '-cluster-parameter:proxy-admin-port}' }}/config &&
   tar -xzf proxies.tar.gz &&
   cp conf/proxysql.cnf  ${:config}
 update-command = ${:command}
 
-[service-proxysql-{{ name }}]
+[proxysql-{{ name }}-wrapper]
 recipe = slapos.cookbook:wrapper
 command-line =
   {{ proxysql_location }}/bin/proxysql -f
@@ -171,12 +180,19 @@ command-line =
   -D ${config-proxysql-{{ name }}:data}
   --reload
 #  -S /tmp/proxysql_admin.sock
-wrapper-path = ${directory:service}/proxysql-{{ name }}
+wrapper-path = ${directory:controller}/proxysql-{{ name }}
 wait-for-files =
   ${repman:bootstrap}/{{ name }}_bootstrapped
 
+{% set service_name = "proxysql-" ~ name -%}
+{% set proxysql_dict = {"name": service_name, "command": "${" ~ service_name ~ "-wrapper:wrapper-path}",
+  "stopwaitsecs": 60, "environment": []} %}
+
+{{ supervisord_lib.supervisord_program(proxysql_controller, service_name, proxysql_dict) }}
+{% do part_list.append(proxysql_controller ~ "-" ~ service_name) %}
+
 {% do part_list.append('config-' ~ name) -%}
-{% do part_list.append('service-proxysql-' ~ name) -%}
+{% do cluster_list.append("{'name': '" ~ name ~ "', 'host': '" ~ ipv4 ~ "', 'port': '${" ~ name ~ "-cluster-parameter:proxy-admin-port}'}") -%}
 {% endfor -%}
 
 [slap-configuration]
@@ -430,6 +446,26 @@ name = check_repman_frontend.py
 config-url = ${nginx-parameter:backend-ssl-url}
 config-check-secure = 1
 
+[template-proxysql-need-stop-start]
+recipe = slapos.recipe.template:jinja2
+rendered = ${directory:bin}/proxysql_check_stop_start
+template = {{ template_proxy_need_stopstart }}
+mode = 755
+cluster-list = !py![{{ cluster_list | join(', ') }}]
+context =
+  key proxysql_controller {{proxysql_controller}}-bin:wrapper-path
+  key repman_url          nginx-parameter:backend-url
+  key cluster_list        :cluster-list
+  raw jq_bin              {{ jq_bin }}
+  raw bash_bin           {{ bash_bin }}
+  raw curl_bin            {{ curl_bin }}
+
+[proxy-need-stop-start]
+recipe = slapos.cookbook:cron.d
+cron-entries = ${cron:cron-entries}
+name = proxysql-need-stop-start
+frequency = * * * * *
+command = ${template-proxysql-need-stop-start:rendered}
 
 #############################
 #
@@ -447,6 +483,7 @@ parts =
   publish-connection-parameter
   repman-frontend-promise
   repman-backend-promise
+  proxy-need-stop-start
 
 # Complete parts with sections
   {{ part_list | join('\n  ') }}

software/repman/instance.cfg.in

@@ -70,6 +70,7 @@ extra-context =
     raw sysbench_location          {{ sysbench_location }}
     raw proxysql_location          {{ proxysql_location }}
     raw template_repman_manager_sh {{ template_repman_manager_sh }}
+    raw template_proxy_need_stopstart {{ proxy_need_stop_start_template }}
 
 [template-mariadb-parameters]
 bash = {{ bash_location }}
@@ -81,6 +82,7 @@ template-mariadb-initial-setup = {{ template_mariadb_initial_setup }}
 template-mariadb-init-root = {{ template_init_root_sql }}
 template-init-root-wrapper = {{ template_init_root_wrapper }}
 template-mysqld-wrapper = {{ template_mysqld_wrapper }}
+template-mysqld-need-start = {{ mysqld_start_template }}
 link-binary = {{ dumps(mariadb_link_binary) }}
 check-computer-memory-binary = {{ bin_directory }}/check-computer-memory
 bin-directory = {{ bin_directory }}

software/repman/software.cfg

@@ -62,9 +62,11 @@ context =
     key template_monitor monitor2-template:rendered
     key mariadb_link_binary template-mariadb.cfg:link-binary
     key mariadb_location mariadb:location
+    key mysqld_start_template mysqld-need-start.sh.in:target
     key nginx_conf_in nginx.conf.in:target
     key nginx_location nginx:location
     key percona_toolkit_location percona-toolkit:location
+    key proxy_need_stop_start_template proxy-need-start-stop.sh.in:target
     key repman_src_location git.signal18.io_signal18_repman:location
     key rsync_location rsync:location
     key restic_bin_location restic:location
@@ -142,6 +144,12 @@ link-binary =
 [dbjobs-in]
 <= download-file
 
+[mysqld-need-start.sh.in]
+<= download-file
+
+[proxy-need-start-stop.sh.in]
+<= download-file
+
 # Pin versions of eggs used that are not already pinned by stack/slapos.cfg
 [versions]
 slapos.recipe.template = 4.3

software/repman/templates/config.toml.in

@@ -14,6 +14,7 @@ db-servers-binary-path = "{{ parameter_dict['mysql-bin-dir'] }}"
 #db-servers-ignored-hosts =
 # Database hosts list to monitor, IP and port (optional), specified in the host:[port] format and separated by commas
 monitoring-address = "{{ parameter_dict['ipv4'] }}"
+monitoring-wait-retry = 40
 
 #haproxy = true
 #haproxy-binary-path = "{{ parameter_dict['haproxy-bin'] }}"

software/repman/templates/mysqld-need-start.sh.in

+#!{{ bash_bin }}
+
+curl () {
+  {{ curl_bin }} -k --silent -H "Accept: application/json" "$@"
+}
+
+# TOKEN=$(curl -s -X POST --data '{"username":"{{ username }}","password":"XXXXX"}' {{ repman_url }}/api/login | {{ jq_bin }} -r '.token')
+
+# Checking if mariadb start is needed
+#CODE=$(curl -H "Authorization: Bearer ${TOKEN}" -o /dev/null -w "%{http_code}" {{ repman_url }}/api/clusters/{{ cluster }}/servers/{{ db_host }}/{{ db_port }}/need-start)
+CODE=$(curl -o /dev/null -w "%{http_code}" {{ repman_url }}/api/clusters/{{ cluster }}/servers/{{ db_host }}/{{ db_port }}/need-start)
+
+if [ $CODE -eq 200 ]; then
+  echo "$CODE: Starting mariadb service..."
+  # print current status, can be useful for debug...
+  {{ mariadb_controller }} status mariadb
+  {{ mariadb_controller }} start mariadb
+  sleep 5
+  # check again if the service is still up...
+  {{ mariadb_controller }} status mariadb
+fi

software/repman/templates/proxy-need-start-stop.sh.in

+#!{{ bash_bin }}
+
+curl () {
+  {{ curl_bin }} -k --silent "$@"
+}
+
+check_start_cluster () {
+  NAME=$1
+  HOST=$2
+  PORT=$3
+
+  CODE=$(curl -o /dev/null -w "%{http_code}" {{ repman_url }}/api/clusters/$NAME/servers/$HOST/$PORT/need-start)
+
+  if [ $CODE -eq 200 ]; then
+    echo "$CODE: Starting proxysql $HOST:$PORT..."
+    {{ proxysql_controller }} start proxysql-$NAME
+    sleep 1
+    # check again if the service is still up...
+    {{ proxysql_controller }} status proxysql-$NAME
+  fi
+}
+
+check_stop_cluster () {
+  NAME=$1
+  HOST=$2
+  PORT=$3
+
+  CODE=$(curl -o /dev/null -w "%{http_code}" {{ repman_url }}/api/clusters/$NAME/servers/$HOST/$PORT/need-stop)
+
+  if [ $CODE -eq 200 ]; then
+    echo "$CODE: Stoping proxysql $HOST:$PORT..."
+    {{ proxysql_controller }} stop proxysql-$NAME
+    sleep 1
+    # check again if the service is stopped...
+    {{ proxysql_controller }} status proxysql-$NAME
+  fi
+}
+
+{% for cluster_dict in cluster_list -%}
+check_start_cluster {{ cluster_dict['name'] }} {{ cluster_dict['host'] }} {{ cluster_dict['port'] }}
+check_stop_cluster {{ cluster_dict['name'] }} {{ cluster_dict['host'] }} {{ cluster_dict['port'] }}
+{% endfor -%}