repman: check start and stop needed for all proxysql services

software/repman/buildout.hash.cfg

@@ -14,15 +14,15 @@
 # not need these here).
 [instance.cfg]
 filename = instance.cfg.in
-md5sum = bc4f2f33c18d5372f2ecfa6940cf1b0d
+md5sum = da50540b1c0fc69ffbf8f6e345a3baad
 
 [instance-repman.cfg]
 _update_hash_filename_ = instance-repman.cfg.jinja2.in
-md5sum = dc86b3d180cb002919d1b4faf680b256
+md5sum = 3b858324e8940a08379956bf76fb88bd
 
 [config-toml.in]
 _update_hash_filename_ = templates/config.toml.in
-md5sum = 1efa65af63b14ca2dd1b8c692b2af80e
+md5sum = 3adc842de5c0e8c94e90ac5f41ecab6c
 
 [config-cluster-toml.in]
 _update_hash_filename_ = templates/cluster-config.toml.in
@@ -34,7 +34,7 @@ md5sum = 0eeb24c6aa0760f0d33c4cc2828ddf30
 
 [template-mariadb.cfg]
 _update_hash_filename_ = instance-mariadb.cfg.jinja2.in
-md5sum = 7d7738da2974578f393130b97acdcab7
+md5sum = 0c6e9a3220cf6e8b096e523d0d1f407a
 
 [template-my-cnf]
 _update_hash_filename_ = templates/my.cnf.in
@@ -62,4 +62,8 @@ md5sum = d623a4c684578602b9d8ee49034aebfa
 
 [mysqld-need-start.sh.in]
 _update_hash_filename_ = templates/mysqld-need-start.sh.in
-md5sum = ec15a9ca52ee3497885c85aa2a7a9008
+md5sum = 76b984ef5e37248bc3a8010a74b24de5
+
+[proxy-need-start-stop.sh.in]
+_update_hash_filename_ = templates/proxy-need-start-stop.sh.in
+md5sum = 1ee3d69f0866605b33eac7ea596b21c3

software/repman/instance-mariadb.cfg.jinja2.in

@@ -294,8 +294,7 @@ template = {{ parameter_dict['template-mysqld-need-start'] }}
 context =
   key mariadb_controller mariadb-controller-bin:wrapper-path
   raw username           {{ slapparameter_dict['repman-user'] }}
-  raw password           {{ slapparameter_dict['repman-passwd'] }}
-  raw secure_url         {{ slapparameter_dict['repman-secure-url'] }}
+  raw repman_url         {{ slapparameter_dict['repman-url'] }}
   raw jq_bin             {{ jq_bin }}
   raw cluster            {{ slapparameter_dict['cluster'] }}
   raw db_host            {{ host }}

software/repman/instance-repman.cfg.jinja2.in

@@ -6,6 +6,7 @@
 {% set mariadb_path_list = [] -%}
 {% set ip = (ipv6_set | list)[0] -%}
 {% set ipv4 = (ipv4_set | list)[0] -%}
+{% set cluster_list = [] -%}
 {% set tags = "gtidstrict,bind,pkg,innodb,noquerycache,slow,pfs,linux,readonly,diskmonitor,sqlerror,compressbinlog" -%}
 {% set frontend_parameter_dict = slapparameter_dict.get('slave-frontend', {}) -%}
 
@@ -17,12 +18,17 @@ var = ${:home}/var
 run = ${:var}/run
 scripts = ${:etc}/run
 service = ${:etc}/service
+controller = ${:etc}/controller
 promise = ${:etc}/promise
 log = ${:var}/log
 data = ${:var}/lib
 nginx-prefix = ${:var}/nginx
 tmp = ${:home}/tmp
 
+{% import "supervisord_lib" as supervisord_lib with context %}
+{% set proxysql_controller = "proxysql-controller" -%}
+{{ supervisord_lib.supervisord(proxysql_controller, buildout_bin_directory, supervisord_conf, use_service_hash=False) }}
+{% do part_list.append("proxysql-controller-service") -%}
 
 [request-common]
 recipe = slapos.cookbook:request.serialised
@@ -33,18 +39,6 @@ cert-file = ${slap-connection:cert-file}
 computer-id = ${slap-connection:computer-id}
 partition-id = ${slap-connection:partition-id}
 
-[proxy-admin-port]
-recipe = slapos.cookbook:free_port
-ip = {{ ipv4 }}
-minimum = 6032
-maximum = 6132
-
-[proxy-port]
-recipe = slapos.cookbook:free_port
-ip = {{ ipv4 }}
-minimum = 7032
-maximum = 7132
-
 
 {% do mariadb_dict.__setitem__('computer-memory-percent-threshold', 80) -%}
 {% set default_parameter_dict = {"cluster1": {"name": "cluster1", "db-prefered-master": "",
@@ -77,7 +71,7 @@ config-{{ key }} = {{ dumps(value) }}
 config-monitor-passwd = ${publish-early:monitor-password}
 config-root-password = ${publish-early:db-root-password}
 config-repman-user = ${repman-parameter:username}
-config-repman-passwd =  ${repman-parameter:password}
+#config-repman-passwd =  ${repman-parameter:password}
 config-repman-url = ${nginx-parameter:backend-url}
 config-repman-secure-url = ${nginx-parameter:backend-ssl-url}
 config-cluster = {{ name }}
@@ -95,14 +89,26 @@ return =
 {% endfor -%}
 
 # Manage Replication Manager clusters
+[{{name}}-admin-port]
+recipe = slapos.cookbook:free_port
+ip = {{ ipv4 }}
+minimum = 6032
+maximum = 6132
+
+[{{name}}-port]
+recipe = slapos.cookbook:free_port
+ip = {{ ipv4 }}
+minimum = 7032
+maximum = 7132
+
 {% set prefered_master = parameter_dict.get("db-prefered-master") -%}
 [{{ name ~ '-cluster-parameter' }}]
 {% for key, value in parameter_dict.items() -%}
 {{ key }} = {{ dumps(value) }}
 {% endfor -%}
 proxysql-user = {{ parameter_dict.get("proxysql-user", "external") }}
-proxy-port = ${proxy-port:port}
-proxy-admin-port = ${proxy-admin-port:port}
+proxy-port = {{ '${' ~ name ~ '-port:port}' }}
+proxy-admin-port = {{ '${' ~ name ~ '-admin-port:port}' }}
 db-credential = repman:${publish-early:db-root-password}
 db-list = {{ mariadb_server_list | join(',') }}
 partition-list = {{ mariadb_path_list | join(',') }}
@@ -161,12 +167,12 @@ command =
   mkdir -p ${repman:config-tmp}/proxies &&
   mkdir -p ${:data} &&
   cd ${repman:config-tmp}/proxies &&
-  {{ curl_bin }} -o proxies.tar.gz ${nginx-parameter:repman-url}/api/clusters/{{ name }}/servers/{{ ipv4 }}/${proxy-admin-port:port}/config &&
+  {{ curl_bin }} -o proxies.tar.gz ${nginx-parameter:repman-url}/api/clusters/{{ name }}/servers/{{ ipv4 }}/{{ '${' ~ name ~ '-cluster-parameter:proxy-admin-port}' }}/config &&
   tar -xzf proxies.tar.gz &&
   cp conf/proxysql.cnf  ${:config}
 update-command = ${:command}
 
-[service-proxysql-{{ name }}]
+[proxysql-{{ name }}-wrapper]
 recipe = slapos.cookbook:wrapper
 command-line =
   {{ proxysql_location }}/bin/proxysql -f
@@ -174,12 +180,19 @@ command-line =
   -D ${config-proxysql-{{ name }}:data}
   --reload
 #  -S /tmp/proxysql_admin.sock
-wrapper-path = ${directory:service}/proxysql-{{ name }}
+wrapper-path = ${directory:controller}/proxysql-{{ name }}
 wait-for-files =
   ${repman:bootstrap}/{{ name }}_bootstrapped
 
+{% set service_name = "proxysql-" ~ name -%}
+{% set proxysql_dict = {"name": service_name, "command": "${" ~ service_name ~ "-wrapper:wrapper-path}",
+  "stopwaitsecs": 60, "environment": []} %}
+
+{{ supervisord_lib.supervisord_program(proxysql_controller, service_name, proxysql_dict) }}
+{% do part_list.append(proxysql_controller ~ "-" ~ service_name) %}
+
 {% do part_list.append('config-' ~ name) -%}
-{% do part_list.append('service-proxysql-' ~ name) -%}
+{% do cluster_list.append("{'name': '" ~ name ~ "', 'host': '" ~ ipv4 ~ "', 'port': '${" ~ name ~ "-cluster-parameter:proxy-admin-port}'}") -%}
 {% endfor -%}
 
 [slap-configuration]
@@ -433,6 +446,26 @@ name = check_repman_frontend.py
 config-url = ${nginx-parameter:backend-ssl-url}
 config-check-secure = 1
 
+[template-proxysql-need-stop-start]
+recipe = slapos.recipe.template:jinja2
+rendered = ${directory:bin}/proxysql_check_stop_start
+template = {{ template_proxy_need_stopstart }}
+mode = 755
+cluster-list = !py![{{ cluster_list | join(', ') }}]
+context =
+  key proxysql_controller {{proxysql_controller}}-bin:wrapper-path
+  key repman_url          nginx-parameter:backend-url
+  key cluster_list        :cluster-list
+  raw jq_bin              {{ jq_bin }}
+  raw bash_bin           {{ bash_bin }}
+  raw curl_bin            {{ curl_bin }}
+
+[proxy-need-stop-start]
+recipe = slapos.cookbook:cron.d
+cron-entries = ${cron:cron-entries}
+name = proxysql-need-stop-start
+frequency = * * * * *
+command = ${template-proxysql-need-stop-start:rendered}
 
 #############################
 #
@@ -450,6 +483,7 @@ parts =
   publish-connection-parameter
   repman-frontend-promise
   repman-backend-promise
+  proxy-need-stop-start
 
 # Complete parts with sections
   {{ part_list | join('\n  ') }}

software/repman/instance.cfg.in

@@ -70,6 +70,7 @@ extra-context =
     raw sysbench_location          {{ sysbench_location }}
     raw proxysql_location          {{ proxysql_location }}
     raw template_repman_manager_sh {{ template_repman_manager_sh }}
+    raw template_proxy_need_stopstart {{ proxy_need_stop_start_template }}
 
 [template-mariadb-parameters]
 bash = {{ bash_location }}

software/repman/software.cfg

@@ -66,6 +66,7 @@ context =
     key nginx_conf_in nginx.conf.in:target
     key nginx_location nginx:location
     key percona_toolkit_location percona-toolkit:location
+    key proxy_need_stop_start_template proxy-need-start-stop.sh.in:target
     key repman_src_location git.signal18.io_signal18_repman:location
     key rsync_location rsync:location
     key restic_bin_location restic:location
@@ -146,6 +147,9 @@ link-binary =
 [mysqld-need-start.sh.in]
 <= download-file
 
+[proxy-need-start-stop.sh.in]
+<= download-file
+
 # Pin versions of eggs used that are not already pinned by stack/slapos.cfg
 [versions]
 slapos.recipe.template = 4.3

software/repman/templates/config.toml.in

@@ -14,6 +14,7 @@ db-servers-binary-path = "{{ parameter_dict['mysql-bin-dir'] }}"
 #db-servers-ignored-hosts =
 # Database hosts list to monitor, IP and port (optional), specified in the host:[port] format and separated by commas
 monitoring-address = "{{ parameter_dict['ipv4'] }}"
+monitoring-wait-retry = 40
 
 #haproxy = true
 #haproxy-binary-path = "{{ parameter_dict['haproxy-bin'] }}"

software/repman/templates/mysqld-need-start.sh.in

@@ -4,10 +4,11 @@ curl () {
   {{ curl_bin }} -k --silent -H "Accept: application/json" "$@"
 }
 
-TOKEN=$(curl -s -X POST --data '{"username":"{{ username }}","password":"{{ password }}"}' {{ secure_url }}/api/login | {{ jq_bin }} -r '.token')
+# TOKEN=$(curl -s -X POST --data '{"username":"{{ username }}","password":"XXXXX"}' {{ repman_url }}/api/login | {{ jq_bin }} -r '.token')
 
 # Checking if mariadb start is needed
-CODE=$(curl -H "Authorization: Bearer ${TOKEN}" -o /dev/null -w "%{http_code}" {{ secure_url }}/api/clusters/{{ cluster }}/servers/{{ db_host }}/{{ db_port }}/need-start)
+#CODE=$(curl -H "Authorization: Bearer ${TOKEN}" -o /dev/null -w "%{http_code}" {{ repman_url }}/api/clusters/{{ cluster }}/servers/{{ db_host }}/{{ db_port }}/need-start)
+CODE=$(curl -o /dev/null -w "%{http_code}" {{ repman_url }}/api/clusters/{{ cluster }}/servers/{{ db_host }}/{{ db_port }}/need-start)
 
 if [ $CODE -eq 200 ]; then
   echo "$CODE: Starting mariadb service..."

software/repman/templates/proxy-need-start-stop.sh.in

+#!{{ bash_bin }}
+
+curl () {
+  {{ curl_bin }} -k --silent "$@"
+}
+
+check_start_cluster () {
+  NAME=$1
+  HOST=$2
+  PORT=$3
+
+  CODE=$(curl -o /dev/null -w "%{http_code}" {{ repman_url }}/api/clusters/$NAME/servers/$HOST/$PORT/need-start)
+
+  if [ $CODE -eq 200 ]; then
+    echo "$CODE: Starting proxysql $HOST:$PORT..."
+    {{ proxysql_controller }} start proxysql-$NAME
+    sleep 1
+    # check again if the service is still up...
+    {{ proxysql_controller }} status proxysql-$NAME
+  fi
+}
+
+check_stop_cluster () {
+  NAME=$1
+  HOST=$2
+  PORT=$3
+
+  CODE=$(curl -o /dev/null -w "%{http_code}" {{ repman_url }}/api/clusters/$NAME/servers/$HOST/$PORT/need-stop)
+
+  if [ $CODE -eq 200 ]; then
+    echo "$CODE: Stoping proxysql $HOST:$PORT..."
+    {{ proxysql_controller }} stop proxysql-$NAME
+    sleep 1
+    # check again if the service is stopped...
+    {{ proxysql_controller }} status proxysql-$NAME
+  fi
+}
+
+{% for cluster_dict in cluster_list -%}
+check_start_cluster {{ cluster_dict['name'] }} {{ cluster_dict['host'] }} {{ cluster_dict['port'] }}
+check_stop_cluster {{ cluster_dict['name'] }} {{ cluster_dict['host'] }} {{ cluster_dict['port'] }}
+{% endfor -%}