Merge branch 'master' into master+ZODB4-wc2

* master:
  Revert "stack/monitor: cleanup deprecated (and confusing) config"
  software/caddy/test: prevent tracebacks during tests
  software/theia/test: adjust test for slapos.core 1.6.15
  software/restic: keep using check-secure = 1
  stack/monitor: don't enable check-monitor-password-promise yet
  component/tesseract: declare a minimal gcc version
  *: review promises for slapos.toolbox 0.123 compatibility
  stack/monitor: repair check-monitor-password-promise
  stack/slapos: version up slapos.core (1.6.17)
  stack/erp5: Make replicating mariadb write to their own binlogs.
  stack/slapos: version up slapos.core (1.6.15)

component/tesseract/buildout.cfg

@@ -14,6 +14,10 @@ extends =
 parts =
   tesseract
 
+[gcc]
+# tesseract uses some C++ 17 features which needs gcc >= 7
+min_version = 7
+
 [tesseract]
 recipe = slapos.recipe.cmmi
 shared = true

software/caddy-frontend/test/test.py

@@ -531,12 +531,12 @@ class TestHandler(BaseHTTPRequestHandler):
       timeout = int(config.pop('Timeout', '0'))
       compress = int(config.pop('Compress', '0'))
       drop_header_list = []
-      for header in config.pop('X-Drop-Header', '').split():
+      for header in (config.pop('X-Drop-Header') or '').split():
         drop_header_list.append(header)
       header_dict = config
     else:
       drop_header_list = []
-      for header in self.headers.dict.get('x-drop-header', '').split():
+      for header in (self.headers.dict.get('x-drop-header') or '').split():
         drop_header_list.append(header)
       response = None
       status_code = 200

software/html5as/buildout.hash.cfg

@@ -21,7 +21,7 @@ md5sum = 9e486efe4ab1aba8cb72b04f6c6da8ad
 
 [instance_html5as]
 _update_hash_filename_ = instance_html5as.cfg.in
-md5sum = 115918e0f8854e957ea8388bb064f457
+md5sum = 283440057c659bde2ae7fcc2c4c5b781
 
 [template_nginx_conf]
 _update_hash_filename_ = templates/nginx_conf.in

software/html5as/instance_html5as.cfg.in

@@ -239,4 +239,3 @@ module = check_url_available
 name = html5as-http-frontend.py
 url = ${html5as-frontend:connection-secure_access}
 config-url = ${:url}
-config-http-code = 401

software/jscrawler/buildout.hash.cfg

@@ -19,7 +19,7 @@ md5sum = 6c17361a49cfc47564063b867aab6e8c
 
 [template-jscrawler]
 filename = instance-jscrawler.cfg.jinja2.in
-md5sum = e76e35d9070bdeca014063e0a00879da
+md5sum = f61e0507717447e47c76a2b2712f17f4
 
 [template-jscrawler-builder]
 filename = template-jscrawler.builder.sh.in

software/jscrawler/instance-jscrawler.cfg.jinja2.in

@@ -50,7 +50,6 @@ return = secure_access domain
 module = check_url_available
 name = jscrawler_frontend.py
 config-url = ${request-jscrawler-frontend:connection-secure_access}
-config-http-code = 401
 
 [logrotate-entry-httpd]
 <= logrotate-entry-base

software/repman/buildout.hash.cfg

@@ -18,7 +18,7 @@ md5sum = 8a08be95a04f1a47098c4fdef80bdfed
 
 [instance-repman.cfg]
 _update_hash_filename_ = instance-repman.cfg.jinja2.in
-md5sum = b2273a47198d38f74c0bf84163b2ea1a
+md5sum = 839642d7a56447b3f08fa69729faca61
 
 [config-toml.in]
 _update_hash_filename_ = templates/config.toml.in

software/repman/instance-repman.cfg.jinja2.in

@@ -511,14 +511,12 @@ return = domain secure_access
 module = check_url_available
 name = check_repman_frontend.py
 config-url = https://${repman-frontend:connection-domain}
-config-http-code = 401
 
 [repman-backend-promise]
 <= monitor-promise-base
 module = check_url_available
 name = check_repman_backend.py
 config-url = ${nginx-parameter:backend-ssl-url}
-config-http-code = 401
 
 [template-proxysql-need-stop-start]
 recipe = slapos.recipe.template:jinja2

software/restic-rest-server/buildout.hash.cfg

@@ -15,4 +15,4 @@
 
 [instance.cfg.in]
 filename = instance.cfg.in
-md5sum = 0e5521cbbf78be884241e8cf585646ae
+md5sum = 0bd2594778cc5f29abfee7cd015c1e7d

software/restic-rest-server/instance.cfg.in

@@ -152,7 +152,7 @@ stop-on-error = true
 recipe = slapos.cookbook:wrapper
 command-line =
   {{ gowork_bin }}/rest-server \
-    --listen [${instance-parameter:ipv6-random}]:${:port}
+    --listen [${:ip}]:${:port}
     --log ${directory:var-log}/${:_buildout_section_name_}-access.log
     --path ${directory:rest-server-data-dir}
     --tls
@@ -185,7 +185,7 @@ return = domain secure_access
 [frontend-available-promise]
 <= check-url-available-promise
 url = ${frontend:connection-secure_access}
-config-http-code = 401
+check-secure = 1
 
 
 [promises]

software/slaprunner/buildout.hash.cfg

@@ -18,7 +18,7 @@ md5sum = 8d6878ff1d2e75010c50a1a2b0c13b24
 
 [template-runner]
 filename = instance-runner.cfg
-md5sum = 60b4d2025eace8da7de14b5bae3772d9
+md5sum = 2a09b11c7dbade65d50e66287bf4c7b9
 
 [template-runner-import-script]
 filename = template/runner-import.sh.jinja2

software/slaprunner/instance-runner.cfg

@@ -440,7 +440,6 @@ module = check_url_available
 name = $${:filename}.py
 filename = apache-httpd-listening-on-tcp
 config-url = $${apache-httpd:access-url}
-config-http-code = 401
 
 [slaprunner-httpd-cors]
 recipe = plone.recipe.command
@@ -541,7 +540,6 @@ return = site_url domain
 module = check_url_available
 name = slaprunner_frontend.py
 config-url = https://$${request-frontend:connection-domain}/login
-config-http-code = 401
 
 [request-httpd-frontend]
 <= slap-connection
@@ -561,7 +559,6 @@ return = secure_access domain
 module = check_url_available
 name = slaprunner-apache-http-frontend.py
 config-url = $${request-httpd-frontend:connection-secure_access}
-config-http-code = 401
 
 {% endif %}
 

software/theia/test/test.py

@@ -281,6 +281,8 @@ class TestTheiaEnv(TheiaTestCase):
     }
 
   def test_theia_env(self):
+    """Make sure environment variables are the same wether we use shell or supervisor services.
+    """
     # The path of the env.json file expected to be generated by building the dummy software release
     env_json_path = os.path.join(self.computer_partition_root_path, 'srv', 'runner', 'software', 'env.json')
 
@@ -315,18 +317,21 @@ class TestTheiaEnv(TheiaTestCase):
     # Remove the env.json file to later be sure that a new one has been generated
     os.remove(env_json_path)
 
-    # Launch slapos-node-software from the embedded supervisord
+    # Launch slapos node software service from the embedded supervisord.
+    # Note that we have two services, slapos-not-software and slapos-not-software-all
+    # The later uses --all which is what we want to use here, because the software
+    # is already installed and we want to install it again, this time from supervisor
     embedded_run_path = os.path.join(self.computer_partition_root_path, 'srv', 'runner', 'var', 'run')
     embedded_supervisord_socket_path = _getSupervisordSocketPath(embedded_run_path, self.logger)
     with getSupervisorRPC(embedded_supervisord_socket_path) as embedded_supervisor:
-      previous_stop_time = embedded_supervisor.getProcessInfo('slapos-node-software')['stop']
-      embedded_supervisor.startProcess('slapos-node-software')
+      previous_stop_time = embedded_supervisor.getProcessInfo('slapos-node-software-all')['stop']
+      embedded_supervisor.startProcess('slapos-node-software-all')
       for _retries in range(20):
         time.sleep(1)
-        if embedded_supervisor.getProcessInfo('slapos-node-software')['stop'] != previous_stop_time:
+        if embedded_supervisor.getProcessInfo('slapos-node-software-all')['stop'] != previous_stop_time:
           break
       else:
-        self.fail("the supervisord service 'slapos-node-software' takes too long to finish")
+        self.fail("the supervisord service 'slapos-node-software-all' takes too long to finish")
 
     # Get the supervisord environment
     with open(env_json_path) as f:

stack/erp5/buildout.hash.cfg

@@ -42,7 +42,7 @@ md5sum = d32417746fcf671d4e86a70379815039
 
 [template-my-cnf]
 filename = my.cnf.in
-md5sum = 7944ec58a2c6ee74a56219bacebfd145
+md5sum = 1de449e8c0c4a85c5ce2b447785b7654
 
 [template-mariadb-initial-setup]
 filename = mariadb_initial_setup.sql.in

stack/erp5/my.cnf.in

@@ -56,6 +56,16 @@ log_bin = {{ log_bin }}
 {% set binlog_expire_days = parameter_dict['binlog-expire-days'] -%}
 {% if binlog_expire_days > 0 %}expire_logs_days = {{ binlog_expire_days }}{% endif %}
 server_id = {{ parameter_dict['server-id'] }}
+# Note: increases writes on replica (it writes to relay log and to database as
+# usual, but then also to binlog), but with the advantage of allowing such node
+# to switch role later, or to allow another replica to catch up with it if the
+# primary is not available anymore. This also increases the disk usage on the
+# replica, as the binlogs are stored as usual.
+log_slave_updates = 1
+# Note: as of at least mariadb 10.3.27, the relay log name is not based on the
+# hostname, but still there is a line in mariadb_error.log on every start
+# warning about this missing option. So provide it.
+relay-log = mariadb-relay-bin
 {% endif %}
 
 # Some dangerous settings you may want to uncomment temporarily

stack/lamp/buildout.hash.cfg

@@ -22,7 +22,7 @@ md5sum = 4afee4377fa9cbc1e4ff80647b2f279c
 
 [instance-lamp]
 filename = instance-lamp.cfg.jinja2.in
-md5sum = 437ec82ac37b28626091000ba7a991a6
+md5sum = 79f562260895df2665a85df5cb442193
 
 [template-apache.conf]
 filename = apache.conf.in

stack/lamp/instance-lamp.cfg.jinja2.in

@@ -82,7 +82,6 @@ name = lamp-http-frontend.py
 url = ${request-frontend:connection-secure_access}
 config-url = ${:url}
 config-custom-domain = {{ slapparameter_dict.get('custom-domain', '') }}
-config-http-code = 401
 
 {% do publish_dict.__setitem__('url', '${lamp-frontend-promise:url}') -%}
 

stack/monitor/buildout.hash.cfg

@@ -14,7 +14,7 @@
 # not need these here).
 [monitor2-template]
 filename = instance-monitor.cfg.jinja2.in
-md5sum = fce342a392141f91c2e1a7b40efa9896
+md5sum = d4185c191e8b9df20e1f98cd8c556b1d
 
 [monitor-httpd-conf]
 _update_hash_filename_ = templates/monitor-httpd.conf.in

stack/monitor/instance-monitor.cfg.jinja2.in

@@ -97,6 +97,7 @@ parameter-file-path = ${monitor-instance-parameter:configuration-file-path}
 parameter-list =
   raw monitor-user ${monitor-instance-parameter:username}
   htpasswd monitor-password ${httpd-monitor-htpasswd:password-file} ${monitor-instance-parameter:username} ${httpd-monitor-htpasswd:htpasswd-path}
+  file min-free-disk-MB ${promise-check-free-disk-space:config-threshold-file}
   ${monitor-instance-parameter:instance-configuration}
 # htpasswd entry:  htpasswd key password-file username htpasswd-file
 
@@ -364,7 +365,7 @@ name = check-monitor-frontend-password.py
 url = ${monitor-frontend:connection-secure_access}
 config-url = ${:url}
 config-username = ${monitor-instance-parameter:username}
-config-password = ${monitor-instance-parameter:username}
+config-password = ${monitor-instance-parameter:password}
 
 # Requests to the frontend URL should fail when no credentials are
 # supplied.
@@ -395,6 +396,7 @@ config-monitor-url = ${monitor-instance-parameter:monitor-base-url}
 module = check_free_disk_space
 name = check-free-disk-space.py
 config-collectordb = ${monitor-instance-parameter:collector-db}
+config-threshold-file = ${directory:etc}/min-free-disk-size
 
 [monitor-base]
 # create dependencies between required monitor parts
@@ -414,7 +416,9 @@ depends =
   ${ca-monitor-httpd-service:wrapper-path}
   ${monitor-httpd-promise:name}
   ${monitor-frontend-promise:name}
-  ${check-monitor-password-promise:name}
+# XXX this is not enabled yet because this conflicts with testnode's own
+# monitoring.
+#  ${check-monitor-password-promise:name}
   ${monitor-bootstrap-promise:name}
   ${monitor-symlink:recipe}
   ${promise-check-slapgrid:recipe}

stack/slapos.cfg

@@ -190,7 +190,7 @@ setuptools-dso = 1.7
 rubygemsrecipe  = 0.3.0
 six = 1.12.0
 slapos.cookbook = 1.0.197
-slapos.core = 1.6.14
+slapos.core = 1.6.17
 slapos.extension.strip = 0.4
 slapos.extension.shared = 1.0
 slapos.libnetworkcache = 0.20