Merge remote-tracking branch 'origin' into zope4

component/lxml-python/buildout.cfg

@@ -8,7 +8,8 @@ parts =
   lxml-python
 
 [lxml-python-env]
-XSLT_CONFIG = ${libxslt:location}/bin/xslt-config
+WITH_XSLT_CONFIG = ${libxslt:location}/bin/xslt-config
+WITH_XML2_CONFIG = ${libxml2:location}/bin/xml2-config
 
 [lxml-python]
 recipe = zc.recipe.egg:custom

component/qemu-kvm/buildout.cfg

@@ -72,11 +72,11 @@ md5sum = 6097fdb9cbab47c96471274b9044e983
 # XXX: This is not the latest version because
 #      Debian does not provide a stable URL for it.
 <= debian-amd64-netinst-base
-version = 10.5.0
-md5sum = a3ebc76aec372808ad80000108a2593a
+version = 10.7.0
+md5sum = 7227c779619e6c8a0a1b0f55d10c6270
 
 [debian-amd64-testing-netinst.iso]
 <= debian-amd64-netinst-base
-release = bullseye_di_alpha2
-version = bullseye-DI-alpha2
-md5sum = 3d7f45ac47e36212f5f7924b41c47f61
+release = bullseye_di_alpha3
+version = bullseye-DI-alpha3
+md5sum = bff147077791586fa7c102267da9f2d2

software/caddy-frontend/CHANGES.caddy_frontend.rst

@@ -3,6 +3,25 @@ Changes
 
 Here are listed the most important changes, which might affect upgrades.
 
+1.0.XXX (XXXX-XX-XX)
+--------------------
+
+ * fix: exposed log file names are stabilised
+ * feature: in case of not found instance more information are provided
+ * feature: telemetry is fully disabled
+ * feature: Apache Traffic Server 8.0 is used
+ * feature: backend-haproxy statistic for haproxy's frontend is available
+ * fix: slave publication has been fixed in case of mixed case slave reference
+ * feature: running test/test.py resolves with starting backend used in tests
+ * fix: automatic caucase-updater usage has been fixed
+ * fix/workaround: reconnect to backend-haproxy from Caddy and Apache Traffic Server
+ * fix/feature: use explicitly Apache Traffic Server simulation of stale-if-error, as in reality Apache Traffic Server does not support it
+ * feature: dropped not used parameters
+ * feature: Strict-Transport-Security aka HSTS
+ * fix: use kedifa with with for file with multiple CAs
+ * feature: support query string (the characters after ? in the url) in url and https-url
+ * fix: by having unique acl names fix rare bug of directing traffic to https-url instead of url or otherwise
+
 1.0.164 (2020-09-24)
 --------------------
 

software/caddy-frontend/README.caddy_frontend.rst

@@ -239,14 +239,14 @@ Necessary to activate cache.
 
 ``enable_cache`` is an optional parameter.
 
-backend-active-check-*
-~~~~~~~~~~~~~~~~~~~~~~
+health-check-*
+~~~~~~~~~~~~~~
 
 This set of parameters is used to control the way how the backend checks will be done. Such active checks can be really useful for `stale-if-error` caching technique and especially in case if backend is very slow to reply or to connect to.
 
-`backend-active-check-http-method` can be used to configure the HTTP method used to check the backend. Special method `CONNECT` can be used to check only for connection attempt.
+`health-check-http-method` can be used to configure the HTTP method used to check the backend. Special method `CONNECT` can be used to check only for connection attempt.
 
-Please be aware that the `backend-active-check-timeout` is really short by default, so in case if `/` of the backend is slow to reply configure proper path with `backend-active-check-http-path` to not mark such backend down too fast, before increasing the check timeout.
+Please be aware that the `health-check-timeout` is really short by default, so in case if `/` of the backend is slow to reply configure proper path with `health-check-http-path` to not mark such backend down too fast, before increasing the check timeout.
 
 Examples
 ========

software/caddy-frontend/buildout.hash.cfg

@@ -26,11 +26,11 @@ md5sum = a6a626fd1579fd1d4b80ea67433ca16a
 
 [profile-caddy-replicate]
 filename = instance-apache-replicate.cfg.in
-md5sum = 7cb8157d2b368ab3b281ea42f743eb9c
+md5sum = 9cc78e7ce1960691e37f103855ff0dc9
 
 [profile-slave-list]
 _update_hash_filename_ = templates/apache-custom-slave-list.cfg.in
-md5sum = 772c04c165fdae91299fd909e061f926
+md5sum = eb98ffd96b2768cc6a5cf664b23aabd3
 
 [profile-replicate-publish-slave-information]
 _update_hash_filename_ = templates/replicate-publish-slave-information.cfg.in
@@ -50,7 +50,7 @@ md5sum = a0ae858a3db8825c22d33d323392f588
 
 [template-backend-haproxy-configuration]
 _update_hash_filename_ = templates/backend-haproxy.cfg.in
-md5sum = 0923a9227c131d2f1e11d7ddd5b15673
+md5sum = 8c4e2548a12c8fd7dba74f940201745a
 
 [template-empty]
 _update_hash_filename_ = templates/empty.in

software/caddy-frontend/instance-apache-replicate.cfg.in

@@ -123,33 +123,33 @@ context =
 {%   elif slave_type not in [None, '', 'default', 'zope', 'redirect', 'notebook', 'websocket'] %}
 {%     do slave_error_list.append('type:%s is not supported' % (slave_type,)) %}
 {%   endif %}
-{#   Check backend-active-check-* #}
-{%   set backend_active_check = (str(slave.get('backend-active-check', False)) or 'false').lower() %}
-{%   if backend_active_check in TRUE_VALUES %}
-{%     set backend_active_check_http_method = slave.get('backend-active-check-http-method') or 'GET' %}
-{%     if backend_active_check_http_method not in ['GET', 'OPTIONS', 'CONNECT', 'POST'] %}
-{%       do slave_error_list.append('Wrong backend-active-check-http-method %s' % (backend_active_check_http_method,)) %}
+{#   Check health-check-* #}
+{%   set health_check = (str(slave.get('health-check', False)) or 'false').lower() %}
+{%   if health_check in TRUE_VALUES %}
+{%     set health_check_http_method = slave.get('health-check-http-method') or 'GET' %}
+{%     if health_check_http_method not in ['GET', 'OPTIONS', 'CONNECT', 'POST'] %}
+{%       do slave_error_list.append('Wrong health-check-http-method %s' % (health_check_http_method,)) %}
 {%     endif %}
-{%     set backend_active_check_http_path = slave.get('backend-active-check-http-path') or '/' %}
-{%     set backend_active_check_http_version = slave.get('backend-active-check-http-version') or 'HTTP/1.1' %}
-{%     if backend_active_check_http_version not in ['HTTP/1.1', 'HTTP/1.0'] %}
-{%       do slave_error_list.append('Wrong backend-active-check-http-version %s' % (backend_active_check_http_version,)) %}
+{%     set health_check_http_path = slave.get('health-check-http-path') or '/' %}
+{%     set health_check_http_version = slave.get('health-check-http-version') or 'HTTP/1.1' %}
+{%     if health_check_http_version not in ['HTTP/1.1', 'HTTP/1.0'] %}
+{%       do slave_error_list.append('Wrong health-check-http-version %s' % (health_check_http_version,)) %}
 {%     endif %}
-{%     set backend_active_check_timeout = (slave.get('backend-active-check-timeout') or '2') | int(false) %}
-{%     if backend_active_check_timeout is false or backend_active_check_timeout <= 0 %}
-{%       do slave_error_list.append('Wrong backend-active-check-timeout %s' % (slave.get('backend-active-check-timeout'),)) %}
+{%     set health_check_timeout = (slave.get('health-check-timeout') or '2') | int(false) %}
+{%     if health_check_timeout is false or health_check_timeout <= 0 %}
+{%       do slave_error_list.append('Wrong health-check-timeout %s' % (slave.get('health-check-timeout'),)) %}
 {%     endif %}
-{%     set backend_active_check_interval = (slave.get('backend-active-check-interval') or '5') | int(false) %}
-{%     if backend_active_check_interval is false or backend_active_check_interval <= 0 %}
-{%       do slave_error_list.append('Wrong backend-active-check-interval %s' % (slave.get('backend-active-check-interval'),)) %}
+{%     set health_check_interval = (slave.get('health-check-interval') or '5') | int(false) %}
+{%     if health_check_interval is false or health_check_interval <= 0 %}
+{%       do slave_error_list.append('Wrong health-check-interval %s' % (slave.get('health-check-interval'),)) %}
 {%     endif %}
-{%     set backend_active_check_rise = (slave.get('backend-active-check-rise') or '1') | int(false) %}
-{%     if backend_active_check_rise is false or backend_active_check_rise <= 0 %}
-{%       do slave_error_list.append('Wrong backend-active-check-rise %s' % (slave.get('backend-active-check-rise'),)) %}
+{%     set health_check_rise = (slave.get('health-check-rise') or '1') | int(false) %}
+{%     if health_check_rise is false or health_check_rise <= 0 %}
+{%       do slave_error_list.append('Wrong health-check-rise %s' % (slave.get('health-check-rise'),)) %}
 {%     endif %}
-{%     set backend_active_check_fall = (slave.get('backend-active-check-fall') or '1') | int(false) %}
-{%     if backend_active_check_fall is false or backend_active_check_fall <= 0 %}
-{%       do slave_error_list.append('Wrong backend-active-check-fall %s' % (slave.get('backend-active-check-fall'),)) %}
+{%     set health_check_fall = (slave.get('health-check-fall') or '1') | int(false) %}
+{%     if health_check_fall is false or health_check_fall <= 0 %}
+{%       do slave_error_list.append('Wrong health-check-fall %s' % (slave.get('health-check-fall'),)) %}
 {%     endif %}
 {%   endif %}
 {#   Check virtualhostroot-http-port and virtualhostroot-https-port #}

software/caddy-frontend/instance-slave-caddy-input-schema.json

@@ -8,14 +8,14 @@
       "type": "string"
     },
     "url": {
-      "description": "Url of the backend",
+      "description": "URL of the backend",
       "pattern": "^(http|https|ftp)://",
       "title": "Backend URL",
       "type": "string"
     },
     "type": {
       "default": "",
-      "description": "Type of slave. If redirect, the slave will redirect to the given url. If zope, the rewrite rules will be compatible with Virtual Host Monster. Implemented are default, zope, redirect, notebook and websocket, not implemneted is eventsource.",
+      "description": "Type of slave. If redirect, the slave will redirect to the given URL. If zope, the rewrite rules will be compatible with Virtual Host Monster. Implemented are default, zope, redirect, notebook and websocket, not implemneted is eventsource.",
       "enum": [
         "",
         "zope",
@@ -96,7 +96,7 @@
       "type": "string"
     },
     "https-url": {
-      "description": "HTTPS Url of the backend if it is diferent from url parameter",
+      "description": "HTTPS URL of the backend if it is different from url parameter",
       "pattern": "^(http|https|ftp)://",
       "title": "HTTPS Backend URL",
       "type": "string"
@@ -224,8 +224,8 @@
       "title": "Authenticate to backend",
       "type": "string"
     },
-    "backend-active-check": {
-      "title": "Backend Active Check",
+    "health-check": {
+      "title": "Health Check",
       "description": "Enables active checks of the backend. For HTTP level checks the HTTP code shall be 2xx or 3xx, otherwise backend will be considered down.",
       "enum": [
         "false",
@@ -234,8 +234,8 @@
       "default": "false",
       "type": "string"
     },
-    "backend-active-check-http-method": {
-      "title": "Backend Active Check HTTP Metod",
+    "health-check-http-method": {
+      "title": "Health Check HTTP Metod",
       "description": "Selects method to do the active check. CONNECT means that connection will be enough for the check, otherwise it's HTTP method.",
       "enum": [
         "GET",
@@ -246,14 +246,14 @@
       "default": "GET",
       "type": "string"
     },
-    "backend-active-check-http-path": {
-      "title": "Backend Active Check HTTP Path",
+    "health-check-http-path": {
+      "title": "Health Check HTTP Path",
       "description": "A path on which do the active check, unused in case of CONNECT.",
       "default": "/",
       "type": "string"
     },
-    "backend-active-check-http-version": {
-      "title": "Backend Active Check HTTP Version",
+    "health-check-http-version": {
+      "title": "Health Check HTTP Version",
       "description": "A HTTP version to use to check the backend, unused in case of CONNECT.",
       "enum": [
         "HTTP/1.1",
@@ -262,26 +262,26 @@
       "default": "HTTP/1.1",
       "type": "string"
     },
-    "backend-active-check-timeout": {
-      "title": "Backend Active Check Timeout (seconds)",
+    "health-check-timeout": {
+      "title": "Health Check Timeout (seconds)",
       "description": "A timeout to for the request to be fulfilled, after connection happen.",
       "default": "2",
       "type": "integer"
     },
-    "backend-active-check-interval": {
-      "title": "Backend Active Check Interval (seconds)",
-      "description": "An interval of backend active check.",
+    "health-check-interval": {
+      "title": "Health Check Interval (seconds)",
+      "description": "An interval of health check.",
       "default": "5",
       "type": "integer"
     },
-    "backend-active-check-rise": {
-      "title": "Backend Active Check Rise",
+    "health-check-rise": {
+      "title": "Health Check Rise",
       "description": "Amount of correct responses from the backend to consider it up.",
       "default": "1",
       "type": "integer"
     },
-    "backend-active-check-fall": {
-      "title": "Backend Active Check Fall",
+    "health-check-fall": {
+      "title": "Health Check Fall",
       "description": "Amount of bad responses from the backend to consider it down.",
       "default": "1",
       "type": "integer"

software/caddy-frontend/software.cfg

@@ -228,15 +228,6 @@ mode = 0644
 [versions]
 # Modern KeDiFa requires zc.lockfile
 zc.lockfile = 1.4
-# Versions pinned for kedifa need urllib3 >= 1.18
-urllib3 = 1.24
-requests = 2.20.0
-certifi = 2018.10.15
-idna = 2.7
-chardet = 3.0.4
-# ipaddress is patching IPAddress so IPv6 match works
-ipaddress = 1.0.22
-# Versions pinned for kedifa need urllib3 >= 1.18
 
 validators = 0.12.2
 PyRSS2Gen = 1.1

software/caddy-frontend/templates/apache-custom-slave-list.cfg.in

@@ -54,7 +54,7 @@ context =
 {#-     * stabilise values for backend #}
 {%-   for key, prefix in [('url', 'http_backend'), ('https-url', 'https_backend')] %}
 {%-     set parsed = urlparse_module.urlparse(slave_instance.get(key, '').strip()) %}
-{%-     set info_dict = {'scheme': parsed.scheme, 'hostname': parsed.hostname, 'port': parsed.port or DEFAULT_PORT[parsed.scheme], 'path': parsed.path, 'fragment': parsed.fragment} %}
+{%-     set info_dict = {'scheme': parsed.scheme, 'hostname': parsed.hostname, 'port': parsed.port or DEFAULT_PORT[parsed.scheme], 'path': parsed.path, 'fragment': parsed.fragment, 'query': parsed.query} %}
 {%-     do slave_instance.__setitem__(prefix, info_dict) %}
 {%-   endfor %}
 {%-   do slave_instance.__setitem__('ssl_proxy_verify', ('' ~ slave_instance.get('ssl-proxy-verify', '')).lower() in TRUE_VALUES) %}
@@ -136,32 +136,32 @@ context =
 {%-   do slave_instance.__setitem__('strict-transport-security', int(slave_instance['strict-transport-security'])) %}
 {%-   do slave_instance.__setitem__('authenticate-to-backend', ('' ~ slave_instance.get('authenticate-to-backend', '')).lower() in TRUE_VALUES) %}
 {#-   Setup active check #}
-{%-   do slave_instance.__setitem__('backend-active-check',  ('' ~ slave_instance.get('backend-active-check', '')).lower() in TRUE_VALUES) %}
-{%-   if slave_instance['backend-active-check']  %}
-{%-     if 'backend-active-check-http-method' not in slave_instance %}
-{%-       do slave_instance.__setitem__('backend-active-check-http-method', 'GET') %}
+{%-   do slave_instance.__setitem__('health-check',  ('' ~ slave_instance.get('health-check', '')).lower() in TRUE_VALUES) %}
+{%-   if slave_instance['health-check']  %}
+{%-     if 'health-check-http-method' not in slave_instance %}
+{%-       do slave_instance.__setitem__('health-check-http-method', 'GET') %}
 {%-     endif %}
-{%-     if 'backend-active-check-http-version' not in slave_instance %}
-{%-       do slave_instance.__setitem__('backend-active-check-http-version', 'HTTP/1.1') %}
+{%-     if 'health-check-http-version' not in slave_instance %}
+{%-       do slave_instance.__setitem__('health-check-http-version', 'HTTP/1.1') %}
 {%-     endif %}
-{%-     if 'backend-active-check-interval' not in slave_instance %}
-{%-       do slave_instance.__setitem__('backend-active-check-interval', '5') %}
+{%-     if 'health-check-interval' not in slave_instance %}
+{%-       do slave_instance.__setitem__('health-check-interval', '5') %}
 {%-     endif %}
-{%-     if 'backend-active-check-rise' not in slave_instance %}
-{%-       do slave_instance.__setitem__('backend-active-check-rise', '1') %}
+{%-     if 'health-check-rise' not in slave_instance %}
+{%-       do slave_instance.__setitem__('health-check-rise', '1') %}
 {%-     endif %}
-{%-     if 'backend-active-check-fall' not in slave_instance %}
-{%-       do slave_instance.__setitem__('backend-active-check-fall', '2') %}
+{%-     if 'health-check-fall' not in slave_instance %}
+{%-       do slave_instance.__setitem__('health-check-fall', '2') %}
 {%-     endif %}
-{%-     if 'backend-active-check-timeout' not in slave_instance %}
-{%-       do slave_instance.__setitem__('backend-active-check-timeout', '2') %}
+{%-     if 'health-check-timeout' not in slave_instance %}
+{%-       do slave_instance.__setitem__('health-check-timeout', '2') %}
 {%-     endif %}
-{%-     do slave_instance.__setitem__('backend-active-check-http-path', slave_instance.get('backend-active-check-http-path') or '/') %}
+{%-     do slave_instance.__setitem__('health-check-http-path', slave_instance.get('health-check-http-path') or '/') %}
 {%-   else %}
-{%-     do slave_instance.__setitem__('backend-active-check-http-method', '') %}
-{%-     do slave_instance.__setitem__('backend-active-check-http-version', '') %}
-{%-     do slave_instance.__setitem__('backend-active-check-http-path', '') %}
-{%-   endif %} {# if backend_active_check #}
+{%-     do slave_instance.__setitem__('health-check-http-method', '') %}
+{%-     do slave_instance.__setitem__('health-check-http-version', '') %}
+{%-     do slave_instance.__setitem__('health-check-http-path', '') %}
+{%-   endif %} {# if slave_instance['health-check'] #}
 {#-   Set Up log files #}
 {%-   do slave_parameter_dict.__setitem__('access_log', '/'.join([caddy_log_directory, '%s_access_log' % slave_reference])) %}
 {%-   do slave_parameter_dict.__setitem__('error_log', '/'.join([caddy_log_directory, '%s_error_log' % slave_reference])) %}

software/caddy-frontend/templates/backend-haproxy.cfg.in

@@ -25,14 +25,14 @@ defaults
 {%-       if wildcard and host.startswith('*.') %}
 {%-         do matched.__setitem__('count', matched['count'] + 1) %}
 # match wildcard {{ host }}
-  acl is_{{ slave_instance['slave_reference'] }} hdr_reg(host) -i {{ host[2:] }}($|:.*)
+  acl is_{{ slave_instance['slave_reference'] }}_{{ scheme }} hdr_reg(host) -i {{ host[2:] }}($|:.*)
 {%-       elif not wildcard and not host.startswith('*.') %}
 {%-         do matched.__setitem__('count', matched['count'] + 1) %}
-  acl is_{{ slave_instance['slave_reference'] }} hdr_reg(host) -i ^{{ host }}($|:.*)
+  acl is_{{ slave_instance['slave_reference'] }}_{{ scheme }} hdr_reg(host) -i ^{{ host }}($|:.*)
 {%-       endif %}
 {%-     endfor %}
 {%-     if matched['count'] > 0 %}
-  use_backend {{ slave_instance['slave_reference'] }}-{{ scheme }} if is_{{ slave_instance['slave_reference'] }}
+  use_backend {{ slave_instance['slave_reference'] }}-{{ scheme }} if is_{{ slave_instance['slave_reference'] }}_{{ scheme }}
 {%-     endif %}
 {%-   endif %}
 {%- endmacro %}
@@ -92,22 +92,27 @@ frontend https-backend
 backend {{ slave_instance['slave_reference'] }}-{{ scheme }}
 {%-       set hostname = info_dict['hostname'] %}
 {%-       set port = info_dict['port'] %}
-{%-       set path = info_dict['path'].rstrip('/') %}
+{%-       set path_list = [info_dict['path'].rstrip('/')] %}
+{%-       set query = info_dict['query'] %}
+{%-       if query %}
+{%-         do path_list.append(query) %}
+{%-       endif %}
+{%-       set path = '?'.join(path_list) %}
 {%-       if hostname and port %}
   timeout server {{ slave_instance['request-timeout'] }}s
   timeout connect {{ slave_instance['backend-connect-timeout'] }}s
   retries {{ slave_instance['backend-connect-retries'] }}
 {%-         set active_check_list = [] %}
 {%-         set active_check_option_list = [] %}
-{%-         if slave_instance['backend-active-check'] %}
+{%-         if slave_instance['health-check'] %}
 {%-           do active_check_list.append('check') %}
-{%-           do active_check_list.append('inter %ss' % (slave_instance['backend-active-check-interval'])) %}
-{%-           do active_check_list.append('rise %s' % (slave_instance['backend-active-check-rise'])) %}
-{%-           do active_check_list.append('fall %s' % (slave_instance['backend-active-check-fall'])) %}
-{%-           if slave_instance['backend-active-check-http-method'] != 'CONNECT' %}
-{%-             do active_check_option_list.append('option httpchk %s %s %s' % (slave_instance['backend-active-check-http-method'], slave_instance['backend-active-check-http-path'] | urlencode, slave_instance['backend-active-check-http-version'])) %}
+{%-           do active_check_list.append('inter %ss' % (slave_instance['health-check-interval'])) %}
+{%-           do active_check_list.append('rise %s' % (slave_instance['health-check-rise'])) %}
+{%-           do active_check_list.append('fall %s' % (slave_instance['health-check-fall'])) %}
+{%-           if slave_instance['health-check-http-method'] != 'CONNECT' %}
+{%-             do active_check_option_list.append('option httpchk %s %s %s' % (slave_instance['health-check-http-method'], slave_instance['health-check-http-path'] | urlencode, slave_instance['health-check-http-version'])) %}
 {%-           endif %}
-{%-           do active_check_option_list.append('timeout check %ss' % (slave_instance['backend-active-check-timeout'])) %}
+{%-           do active_check_option_list.append('timeout check %ss' % (slave_instance['health-check-timeout'])) %}
 {%-         endif %}
   server {{ slave_instance['slave_reference'] }}-backend {{ hostname }}:{{ port }} {{ ' '.join(ssl_list) }} {{ ' ' + ' '.join(active_check_list)}}
 {%-         for active_check_option in active_check_option_list %}

software/caddy-frontend/test/test.py

@@ -37,6 +37,7 @@ from unittest import skip
 import ssl
 from BaseHTTPServer import HTTPServer
 from BaseHTTPServer import BaseHTTPRequestHandler
+from SocketServer import ThreadingMixIn
 import time
 import tempfile
 import ipaddress
@@ -198,6 +199,10 @@ def createCSR(common_name, ip=None):
   return key, key_pem, csr, csr_pem
 
 
+class ThreadedHTTPServer(ThreadingMixIn, HTTPServer):
+  pass
+
+
 class CertificateAuthority(object):
   def __init__(self, common_name):
     self.key, self.key_pem = createKey()
@@ -419,13 +424,21 @@ def fakeHTTPSResult(domain, path, port=HTTPS_PORT,
   try:
     add_custom_dns(domain, port, TEST_IP)
     socket.getaddrinfo = new_getaddrinfo
-    return session.get(
-      'https://%s:%s/%s' % (domain, port, path),
-      verify=False,
-      allow_redirects=False,
-      headers=headers,
-      cookies=cookies
-    )
+    # Use a prepared request, to disable path normalization.
+    # We need this because some test checks requests with paths like
+    # /test-path/deep/.././deeper but we don't want the client to send
+    # /test-path/deeper
+    # See also https://github.com/psf/requests/issues/5289
+    url = 'https://%s:%s/%s' % (domain, port, path)
+    req = requests.Request(
+        method='GET',
+        url=url,
+        headers=headers,
+        cookies=cookies,
+    )
+    prepped = req.prepare()
+    prepped.url = url
+    return session.send(prepped, verify=False, allow_redirects=False)
   finally:
     socket.getaddrinfo = socket_getaddrinfo
 
@@ -447,17 +460,25 @@ def fakeHTTPResult(domain, path, port=HTTP_PORT,
     new_source = source.SourceAddressAdapter(source_ip)
     session.mount('http://', new_source)
     session.mount('https://', new_source)
-  return session.get(
-    'http://%s:%s/%s' % (TEST_IP, port, path),
-    headers=headers,
-    allow_redirects=False,
-  )
+
+  # Use a prepared request, to disable path normalization.
+  url = 'http://%s:%s/%s' % (TEST_IP, port, path)
+  req = requests.Request(method='GET', url=url, headers=headers)
+  prepped = req.prepare()
+  prepped.url = url
+  return session.send(prepped, allow_redirects=False)
 
 
 class TestHandler(BaseHTTPRequestHandler):
   identification = None
   configuration = {}
 
+  def log_message(self, *args):
+    if os.environ.get('SLAPOS_TEST_DEBUG'):
+      return BaseHTTPRequestHandler.log_message(self, *args)
+    else:
+      return
+
   def do_DELETE(self):
     config = self.configuration.pop(self.path, None)
     if config is None:
@@ -471,7 +492,7 @@ class TestHandler(BaseHTTPRequestHandler):
 
   def do_PUT(self):
     config = {
-      'status_code': self.headers.dict.get('status-code', '200')
+      'status_code': self.headers.dict.get('x-reply-status-code', '200')
     }
     prefix = 'x-reply-header-'
     length = len(prefix)
@@ -583,11 +604,11 @@ class HttpFrontendTestCase(SlapOSInstanceTestCase):
 
   @classmethod
   def startServerProcess(cls):
-    server = HTTPServer(
+    server = ThreadedHTTPServer(
       (cls._ipv4_address, cls._server_http_port),
       TestHandler)
 
-    server_https = HTTPServer(
+    server_https = ThreadedHTTPServer(
       (cls._ipv4_address, cls._server_https_port),
       TestHandler)
 
@@ -626,6 +647,48 @@ class HttpFrontendTestCase(SlapOSInstanceTestCase):
           cls.logger.warning(
             'Process %s still alive' % (process, ))
 
+  def startAuthenticatedServerProcess(self):
+    master_parameter_dict = self.parseConnectionParameterDict()
+    caucase_url = master_parameter_dict['backend-client-caucase-url']
+    ca_certificate = requests.get(caucase_url + '/cas/crt/ca.crt.pem')
+    assert ca_certificate.status_code == httplib.OK
+    ca_certificate_file = os.path.join(
+      self.working_directory, 'ca-backend-client.crt.pem')
+    with open(ca_certificate_file, 'w') as fh:
+      fh.write(ca_certificate.text)
+
+    class OwnTestHandler(TestHandler):
+      identification = 'Auth Backend'
+
+    server_https_auth = ThreadedHTTPServer(
+      (self._ipv4_address, self._server_https_auth_port),
+      OwnTestHandler)
+
+    server_https_auth.socket = ssl.wrap_socket(
+      server_https_auth.socket,
+      certfile=self.test_server_certificate_file.name,
+      cert_reqs=ssl.CERT_REQUIRED,
+      ca_certs=ca_certificate_file,
+      server_side=True)
+
+    self.backend_https_auth_url = 'https://%s:%s/' \
+        % server_https_auth.server_address
+
+    self.server_https_auth_process = multiprocessing.Process(
+      target=server_https_auth.serve_forever, name='HTTPSServerAuth')
+    self.server_https_auth_process.start()
+    self.logger.debug('Started process %s' % (self.server_https_auth_process,))
+
+  def stopAuthenticatedServerProcess(self):
+    self.logger.debug('Stopping process %s' % (
+      self.server_https_auth_process,))
+    self.server_https_auth_process.join(10)
+    self.server_https_auth_process.terminate()
+    time.sleep(0.1)
+    if self.server_https_auth_process.is_alive():
+      self.logger.warning(
+        'Process %s still alive' % (self.server_https_auth_process, ))
+
   @classmethod
   def setUpMaster(cls):
     # run partition until AIKC finishes
@@ -1220,7 +1283,7 @@ class TestSlave(SlaveHttpFrontendTestCase, TestDataMixin):
       'Url': {
         # make URL "incorrect", with whitespace, nevertheless it shall be
         # correctly handled
-        'url': ' ' + cls.backend_url + ' ',
+        'url': ' ' + cls.backend_url + '/?a=b&c=' + ' ',
         # authenticating to http backend shall be no-op
         'authenticate-to-backend': True,
       },
@@ -1340,23 +1403,6 @@ class TestSlave(SlaveHttpFrontendTestCase, TestDataMixin):
         'type': 'zope',
         'https-only': 'false',
       },
-      'type-zope-ssl-proxy-verify_ssl_proxy_ca_crt': {
-        'url': cls.backend_https_url,
-        'type': 'zope',
-        'ssl-proxy-verify': True,
-        'ssl_proxy_ca_crt': cls.test_server_ca.certificate_pem,
-      },
-      'type-zope-ssl-proxy-verify_ssl_proxy_ca_crt-unverified': {
-        'url': cls.backend_https_url,
-        'type': 'zope',
-        'ssl-proxy-verify': True,
-        'ssl_proxy_ca_crt': cls.another_server_ca.certificate_pem,
-      },
-      'type-zope-ssl-proxy-verify-unverified': {
-        'url': cls.backend_https_url,
-        'type': 'zope',
-        'ssl-proxy-verify': True,
-      },
       'type-zope-virtualhostroot-http-port': {
         'url': cls.backend_url,
         'type': 'zope',
@@ -1443,23 +1489,6 @@ class TestSlave(SlaveHttpFrontendTestCase, TestDataMixin):
         'url': cls.backend_url,
         'enable-http2': False,
       },
-      'enable_cache-ssl-proxy-verify-unverified': {
-        'url': cls.backend_https_url,
-        'enable_cache': True,
-        'ssl-proxy-verify': True,
-      },
-      'enable_cache-ssl-proxy-verify_ssl_proxy_ca_crt': {
-        'url': cls.backend_https_url,
-        'enable_cache': True,
-        'ssl_proxy_ca_crt': cls.test_server_ca.certificate_pem,
-        'ssl-proxy-verify': True,
-      },
-      'enable_cache-ssl-proxy-verify_ssl_proxy_ca_crt-unverified': {
-        'url': cls.backend_https_url,
-        'enable_cache': True,
-        'ssl_proxy_ca_crt': cls.another_server_ca.certificate_pem,
-        'ssl-proxy-verify': True,
-      },
       'enable-http2-default': {
         'url': cls.backend_url,
       },
@@ -1652,15 +1681,15 @@ class TestSlave(SlaveHttpFrontendTestCase, TestDataMixin):
       'monitor-base-url': 'https://[%s]:8401' % self._ipv6_address,
       'backend-client-caucase-url': 'http://[%s]:8990' % self._ipv6_address,
       'domain': 'example.com',
-      'accepted-slave-amount': '56',
+      'accepted-slave-amount': '50',
       'rejected-slave-amount': '0',
-      'slave-amount': '56',
+      'slave-amount': '50',
       'rejected-slave-dict': {
       },
       'warning-slave-dict': {
         '_Url': [
-          "slave url ' %(backend)s ' has been converted to '%(backend)s'" % {
-            'backend': self.backend_url}]}
+          "slave url ' %(backend)s/?a=b&c= ' has been converted to "
+          "'%(backend)s/?a=b&c='" % {'backend': self.backend_url}]}
     }
 
     self.assertEqual(
@@ -1809,7 +1838,7 @@ class TestSlave(SlaveHttpFrontendTestCase, TestDataMixin):
         'secure_access': 'https://%s.example.com' % (hostname, ),
         'backend-client-caucase-url': 'http://[%s]:8990' % self._ipv6_address,
         'warning-list': [
-          "slave url ' %s ' has been converted to '%s'" % (
+          "slave url ' %s/?a=b&c= ' has been converted to '%s/?a=b&c='" % (
             self.backend_url, self.backend_url)],
       },
       parameter_dict
@@ -1821,6 +1850,7 @@ class TestSlave(SlaveHttpFrontendTestCase, TestDataMixin):
       headers={
         'Timeout': '10',  # more than default backend-connect-timeout == 5
         'Accept-Encoding': 'gzip',
+        'User-Agent': 'TEST USER AGENT',
       }
     )
 
@@ -1829,7 +1859,7 @@ class TestSlave(SlaveHttpFrontendTestCase, TestDataMixin):
       der2pem(result.peercert))
 
     self.assertNotIn('Strict-Transport-Security', result.headers)
-    self.assertEqualResultJson(result, 'Path', '/test-path/deeper')
+    self.assertEqualResultJson(result, 'Path', '?a=b&c=/test-path/deeper')
 
     try:
       j = result.json()
@@ -1854,7 +1884,7 @@ class TestSlave(SlaveHttpFrontendTestCase, TestDataMixin):
     log_regexp = r'^\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3} - - ' \
                  r'\[\d{2}\/.{3}\/\d{4}\:\d{2}\:\d{2}\:\d{2} \+\d{4}\] ' \
                  r'"GET \/test-path\/deep\/..\/.\/deeper HTTP\/1.1" \d{3} ' \
-                 r'\d+ "-" "python-requests.*" \d+'
+                 r'\d+ "-" "TEST USER AGENT" \d+'
 
     self.assertRegexpMatches(
       open(log_file, 'r').readlines()[-1],
@@ -1871,7 +1901,7 @@ class TestSlave(SlaveHttpFrontendTestCase, TestDataMixin):
                  r'http-backend _Url-http\/_Url-backend ' \
                  r'\d+/\d+\/\d+\/\d+\/\d+ ' \
                  r'200 \d+ - - ---- ' \
-                 r'\d\/\d\/\d\/\d\/\d \d\/\d ' \
+                 r'\d+\/\d+\/\d+\/\d+\/\d+ \d+\/\d+ ' \
                  r'"GET /test-path/deeper HTTP/1.1"'
 
     self.assertRegexpMatches(
@@ -1912,50 +1942,20 @@ class TestSlave(SlaveHttpFrontendTestCase, TestDataMixin):
 
   def test_auth_to_backend(self):
     parameter_dict = self.assertSlaveBase('auth-to-backend')
-    # 1. fetch certificate from backend-client-caucase-url
-    master_parameter_dict = self.parseConnectionParameterDict()
-    caucase_url = master_parameter_dict['backend-client-caucase-url']
-    ca_certificate = requests.get(caucase_url + '/cas/crt/ca.crt.pem')
-    assert ca_certificate.status_code == httplib.OK
-    ca_certificate_file = os.path.join(
-      self.working_directory, 'ca-backend-client.crt.pem')
-    with open(ca_certificate_file, 'w') as fh:
-      fh.write(ca_certificate.text)
 
-    # 2. start backend with this certificate
-    class OwnTestHandler(TestHandler):
-      identification = 'Auth Backend'
-
-    server_https_auth = HTTPServer(
-      (self._ipv4_address, self._server_https_auth_port),
-      OwnTestHandler)
-
-    server_https_auth.socket = ssl.wrap_socket(
-      server_https_auth.socket,
-      certfile=self.test_server_certificate_file.name,
-      cert_reqs=ssl.CERT_REQUIRED,
-      ca_certs=ca_certificate_file,
-      server_side=True)
-
-    backend_https_auth_url = 'https://%s:%s/' \
-        % server_https_auth.server_address
-
-    server_https_auth_process = multiprocessing.Process(
-      target=server_https_auth.serve_forever, name='HTTPSServerAuth')
-    server_https_auth_process.start()
-    self.logger.debug('Started process %s' % (server_https_auth_process,))
+    self.startAuthenticatedServerProcess()
     try:
-      # 3. assert that you can't fetch nothing without key
+      # assert that you can't fetch nothing without key
       try:
-        requests.get(backend_https_auth_url, verify=False)
+        requests.get(self.backend_https_auth_url, verify=False)
       except Exception:
         pass
       else:
         self.fail(
           'Access to %r shall be not possible without certificate' % (
-            backend_https_auth_url,))
-      # 4. check that you can access this backend via frontend
-      #    (so it means that auth to backend worked)
+            self.backend_https_auth_url,))
+      # check that you can access this backend via frontend
+      # (so it means that auth to backend worked)
       result = fakeHTTPSResult(
         parameter_dict['domain'],
         'test-path/deep/.././deeper',
@@ -1991,60 +1991,23 @@ class TestSlave(SlaveHttpFrontendTestCase, TestDataMixin):
         result.headers['X-Backend-Identification']
       )
     finally:
-      self.logger.debug('Stopping process %s' % (server_https_auth_process,))
-      server_https_auth_process.join(10)
-      server_https_auth_process.terminate()
-      time.sleep(0.1)
-      if server_https_auth_process.is_alive():
-        self.logger.warning(
-          'Process %s still alive' % (server_https_auth_process, ))
+      self.stopAuthenticatedServerProcess()
 
   def test_auth_to_backend_not_configured(self):
     parameter_dict = self.assertSlaveBase('auth-to-backend-not-configured')
-    # 1. fetch certificate from backend-client-caucase-url
-    master_parameter_dict = self.parseConnectionParameterDict()
-    caucase_url = master_parameter_dict['backend-client-caucase-url']
-    ca_certificate = requests.get(caucase_url + '/cas/crt/ca.crt.pem')
-    assert ca_certificate.status_code == httplib.OK
-    ca_certificate_file = os.path.join(
-      self.working_directory, 'ca-backend-client.crt.pem')
-    with open(ca_certificate_file, 'w') as fh:
-      fh.write(ca_certificate.text)
-
-    # 2. start backend with this certificate
-    class OwnTestHandler(TestHandler):
-      identification = 'Auth Backend'
-
-    server_https_auth = HTTPServer(
-      (self._ipv4_address, self._server_https_auth_port),
-      OwnTestHandler)
-
-    server_https_auth.socket = ssl.wrap_socket(
-      server_https_auth.socket,
-      certfile=self.test_server_certificate_file.name,
-      cert_reqs=ssl.CERT_REQUIRED,
-      ca_certs=ca_certificate_file,
-      server_side=True)
-
-    backend_https_auth_url = 'https://%s:%s/' \
-        % server_https_auth.server_address
-
-    server_https_auth_process = multiprocessing.Process(
-      target=server_https_auth.serve_forever, name='HTTPSServerAuth')
-    server_https_auth_process.start()
-    self.logger.debug('Started process %s' % (server_https_auth_process,))
+    self.startAuthenticatedServerProcess()
     try:
-      # 3. assert that you can't fetch nothing without key
+      # assert that you can't fetch nothing without key
       try:
-        requests.get(backend_https_auth_url, verify=False)
+        requests.get(self.backend_https_auth_url, verify=False)
       except Exception:
         pass
       else:
         self.fail(
           'Access to %r shall be not possible without certificate' % (
-            backend_https_auth_url,))
-      # 4. check that you can access this backend via frontend
-      #    (so it means that auth to backend worked)
+            self.backend_https_auth_url,))
+      # check that you can access this backend via frontend
+      # (so it means that auth to backend worked)
       result = fakeHTTPSResult(
         parameter_dict['domain'],
         'test-path/deep/.././deeper',
@@ -2063,13 +2026,7 @@ class TestSlave(SlaveHttpFrontendTestCase, TestDataMixin):
         httplib.BAD_GATEWAY
       )
     finally:
-      self.logger.debug('Stopping process %s' % (server_https_auth_process,))
-      server_https_auth_process.join(10)
-      server_https_auth_process.terminate()
-      time.sleep(0.1)
-      if server_https_auth_process.is_alive():
-        self.logger.warning(
-          'Process %s still alive' % (server_https_auth_process, ))
+      self.stopAuthenticatedServerProcess()
 
   def test_auth_to_backend_backend_ignore(self):
     parameter_dict = self.assertSlaveBase('auth-to-backend-backend-ignore')
@@ -2133,7 +2090,7 @@ class TestSlave(SlaveHttpFrontendTestCase, TestDataMixin):
         'secure_access': 'https://%s.example.com' % (hostname, ),
         'backend-client-caucase-url': 'http://[%s]:8990' % self._ipv6_address,
         'warning-list': [
-          "slave url ' %s ' has been converted to '%s'" % (
+          "slave url ' %s/?a=b&c= ' has been converted to '%s/?a=b&c='" % (
             self.backend_url, self.backend_url)],
       },
       parameter_dict
@@ -2185,7 +2142,7 @@ class TestSlave(SlaveHttpFrontendTestCase, TestDataMixin):
         'secure_access': 'https://%s.example.com' % (hostname, ),
         'backend-client-caucase-url': 'http://[%s]:8990' % self._ipv6_address,
         'warning-list': [
-          "slave url ' %s ' has been converted to '%s'" % (
+          "slave url ' %s/?a=b&c= ' has been converted to '%s/?a=b&c='" % (
             self.backend_url, self.backend_url)],
       },
       parameter_dict
@@ -3488,208 +3445,6 @@ class TestSlave(SlaveHttpFrontendTestCase, TestDataMixin):
       result.status_code
     )
 
-  def test_enable_cache_ssl_proxy_verify_ssl_proxy_ca_crt_unverified(self):
-    parameter_dict = self.parseSlaveParameterDict(
-      'enable_cache-ssl-proxy-verify_ssl_proxy_ca_crt-unverified')
-
-    self.assertLogAccessUrlWithPop(parameter_dict)
-    self.assertKedifaKeysWithPop(parameter_dict)
-    self.assertEqual(
-      {
-        'domain':
-        'enablecachesslproxyverifysslproxycacrtunverified.example.com',
-        'replication_number': '1',
-        'url':
-        'http://enablecachesslproxyverifysslproxycacrtunverified.example.com',
-        'site_url':
-        'http://enablecachesslproxyverifysslproxycacrtunverified.example.com',
-        'secure_access':
-        'https://enablecachesslproxyverifysslproxycacrtunverified.example.com',
-        'backend-client-caucase-url': 'http://[%s]:8990' % self._ipv6_address,
-      },
-      parameter_dict
-    )
-
-    result = fakeHTTPSResult(
-      parameter_dict['domain'],
-      'test-path/deep/.././deeper')
-
-    self.assertEqual(
-      self.certificate_pem,
-      der2pem(result.peercert))
-
-    self.assertEqual(
-      httplib.SERVICE_UNAVAILABLE,
-      result.status_code
-    )
-
-    result_http = fakeHTTPResult(
-      parameter_dict['domain'],
-      'test-path/deeper')
-
-    self.assertEqual(
-      httplib.FOUND,
-      result_http.status_code
-    )
-
-    self.assertEqual(
-      'https://enablecachesslproxyverifysslproxycacrtunverified.example.com'
-      ':%s/test-path/deeper' % (HTTP_PORT,),
-      result_http.headers['Location']
-    )
-
-  def test_enable_cache_ssl_proxy_verify_ssl_proxy_ca_crt(self):
-    parameter_dict = self.assertSlaveBase(
-      'enable_cache-ssl-proxy-verify_ssl_proxy_ca_crt')
-
-    result = fakeHTTPSResult(
-      parameter_dict['domain'],
-      'test-path/deep/.././deeper')
-
-    self.assertEqual(
-      self.certificate_pem,
-      der2pem(result.peercert))
-
-    self.assertEqualResultJson(result, 'Path', '/test-path/deeper')
-
-    headers = result.headers.copy()
-
-    self.assertKeyWithPop('Server', headers)
-    self.assertKeyWithPop('Date', headers)
-    self.assertKeyWithPop('Age', headers)
-
-    # drop keys appearing randomly in headers
-    headers.pop('Transfer-Encoding', None)
-    headers.pop('Content-Length', None)
-    headers.pop('Connection', None)
-    headers.pop('Keep-Alive', None)
-
-    self.assertEqual(
-      {
-        'Content-type': 'application/json',
-        'Set-Cookie': 'secured=value;secure, nonsecured=value',
-      },
-      headers
-    )
-
-  def test_enable_cache_ssl_proxy_verify_unverified(self):
-    parameter_dict = self.assertSlaveBase(
-      'enable_cache-ssl-proxy-verify-unverified')
-
-    result = fakeHTTPSResult(
-      parameter_dict['domain'], 'test-path')
-
-    self.assertEqual(
-      self.certificate_pem,
-      der2pem(result.peercert))
-
-    self.assertEqual(
-      httplib.SERVICE_UNAVAILABLE,
-      result.status_code
-    )
-
-  def test_type_zope_ssl_proxy_verify_ssl_proxy_ca_crt_unverified(self):
-    parameter_dict = self.parseSlaveParameterDict(
-      'type-zope-ssl-proxy-verify_ssl_proxy_ca_crt-unverified')
-
-    self.assertLogAccessUrlWithPop(parameter_dict)
-    self.assertKedifaKeysWithPop(parameter_dict)
-    self.assertEqual(
-      {
-        'domain': 'typezopesslproxyverifysslproxycacrtunverified.example.com',
-        'replication_number': '1',
-        'url':
-        'http://typezopesslproxyverifysslproxycacrtunverified.example.com',
-        'site_url':
-        'http://typezopesslproxyverifysslproxycacrtunverified.example.com',
-        'secure_access':
-        'https://typezopesslproxyverifysslproxycacrtunverified.example.com',
-        'backend-client-caucase-url': 'http://[%s]:8990' % self._ipv6_address,
-      },
-      parameter_dict
-    )
-
-    result = fakeHTTPSResult(
-      parameter_dict['domain'], 'test-path')
-
-    self.assertEqual(
-      self.certificate_pem,
-      der2pem(result.peercert))
-
-    self.assertEqual(
-      httplib.SERVICE_UNAVAILABLE,
-      result.status_code
-    )
-
-    result_http = fakeHTTPResult(
-      parameter_dict['domain'], 'test-path')
-
-    self.assertEqual(
-      httplib.FOUND,
-      result_http.status_code
-    )
-
-    self.assertEqual(
-      'https://typezopesslproxyverifysslproxycacrtunverified.example.com:%s/'
-      'test-path' % (HTTP_PORT,),
-      result_http.headers['Location']
-    )
-
-  def test_type_zope_ssl_proxy_verify_ssl_proxy_ca_crt(self):
-    parameter_dict = self.assertSlaveBase(
-      'type-zope-ssl-proxy-verify_ssl_proxy_ca_crt')
-
-    result = fakeHTTPSResult(
-      parameter_dict['domain'], 'test-path')
-
-    self.assertEqual(
-      self.certificate_pem,
-      der2pem(result.peercert))
-
-    try:
-      j = result.json()
-    except Exception:
-      raise ValueError('JSON decode problem in:\n%s' % (result.text,))
-    self.assertBackendHeaders(j['Incoming Headers'], parameter_dict['domain'])
-
-    self.assertEqualResultJson(
-      result,
-      'Path',
-      '/VirtualHostBase/https//'
-      'typezopesslproxyverifysslproxycacrt.example.com:443/'
-      '/VirtualHostRoot/test-path'
-    )
-
-    result = fakeHTTPResult(
-      parameter_dict['domain'], 'test-path')
-
-    self.assertEqual(
-      httplib.FOUND,
-      result.status_code
-    )
-
-    self.assertEqual(
-      'https://typezopesslproxyverifysslproxycacrt.example.com:'
-      '%s/test-path' % (HTTP_PORT,),
-      result.headers['Location']
-    )
-
-  def test_type_zope_ssl_proxy_verify_unverified(self):
-    parameter_dict = self.assertSlaveBase(
-      'type-zope-ssl-proxy-verify-unverified')
-
-    result = fakeHTTPSResult(
-      parameter_dict['domain'], 'test-path')
-
-    self.assertEqual(
-      self.certificate_pem,
-      der2pem(result.peercert))
-
-    self.assertEqual(
-      httplib.SERVICE_UNAVAILABLE,
-      result.status_code
-    )
-
   def test_monitor_ipv6_test(self):
     parameter_dict = self.assertSlaveBase('monitor-ipv6-test')
 
@@ -4061,7 +3816,7 @@ class TestSlave(SlaveHttpFrontendTestCase, TestDataMixin):
       backend_url = self.getSlaveParameterDictDict()['enable_cache']['url']
       result = requests.put(backend_url + path, headers={
           'X-Reply-Header-Cache-Control': 'max-age=%s, public' % (max_age,),
-          'Status-Code': status_code,
+          'X-Reply-Status-Code': status_code,
           'X-Reply-Body': base64.b64encode(body),
         })
       self.assertEqual(result.status_code, httplib.CREATED)
@@ -6454,45 +6209,45 @@ class TestSlaveRejectReportUnsafeDamaged(SlaveHttpFrontendTestCase):
         'ssl_key': '${section:option}ssl_keyunsafe\nunsafe',
         'ssl_crt': '${section:option}ssl_crtunsafe\nunsafe',
       },
-      'backend-active-check-http-method': {
-        'backend-active-check': True,
-        'backend-active-check-http-method': 'WRONG',
+      'health-check-http-method': {
+        'health-check': True,
+        'health-check-http-method': 'WRONG',
       },
-      'backend-active-check-http-version': {
-        'backend-active-check': True,
-        'backend-active-check-http-version': 'WRONG/1.1',
+      'health-check-http-version': {
+        'health-check': True,
+        'health-check-http-version': 'WRONG/1.1',
       },
-      'backend-active-check-timeout': {
-        'backend-active-check': True,
-        'backend-active-check-timeout': 'WRONG',
+      'health-check-timeout': {
+        'health-check': True,
+        'health-check-timeout': 'WRONG',
       },
-      'backend-active-check-timeout-negative': {
-        'backend-active-check': True,
-        'backend-active-check-timeout': '-2',
+      'health-check-timeout-negative': {
+        'health-check': True,
+        'health-check-timeout': '-2',
       },
-      'backend-active-check-interval': {
-        'backend-active-check': True,
-        'backend-active-check-interval': 'WRONG',
+      'health-check-interval': {
+        'health-check': True,
+        'health-check-interval': 'WRONG',
       },
-      'backend-active-check-interval-negative': {
-        'backend-active-check': True,
-        'backend-active-check-interval': '-2',
+      'health-check-interval-negative': {
+        'health-check': True,
+        'health-check-interval': '-2',
       },
-      'backend-active-check-rise': {
-        'backend-active-check': True,
-        'backend-active-check-rise': 'WRONG',
+      'health-check-rise': {
+        'health-check': True,
+        'health-check-rise': 'WRONG',
       },
-      'backend-active-check-rise-negative': {
-        'backend-active-check': True,
-        'backend-active-check-rise': '-2',
+      'health-check-rise-negative': {
+        'health-check': True,
+        'health-check-rise': '-2',
       },
-      'backend-active-check-fall': {
-        'backend-active-check': True,
-        'backend-active-check-fall': 'WRONG',
+      'health-check-fall': {
+        'health-check': True,
+        'health-check-fall': 'WRONG',
       },
-      'backend-active-check-fall-negative': {
-        'backend-active-check': True,
-        'backend-active-check-fall': '-2',
+      'health-check-fall-negative': {
+        'health-check': True,
+        'health-check-fall': '-2',
       }
     }
 
@@ -6548,26 +6303,26 @@ class TestSlaveRejectReportUnsafeDamaged(SlaveHttpFrontendTestCase):
         '_EMPTY-BACKEND': [
           "slave https-url '' invalid",
           "slave url '' invalid"],
-        '_backend-active-check-fall': [
-          'Wrong backend-active-check-fall WRONG'],
-        '_backend-active-check-fall-negative': [
-          'Wrong backend-active-check-fall -2'],
-        '_backend-active-check-http-method': [
-          'Wrong backend-active-check-http-method WRONG'],
-        '_backend-active-check-http-version': [
-          'Wrong backend-active-check-http-version WRONG/1.1'],
-        '_backend-active-check-interval': [
-          'Wrong backend-active-check-interval WRONG'],
-        '_backend-active-check-interval-negative': [
-          'Wrong backend-active-check-interval -2'],
-        '_backend-active-check-rise': [
-          'Wrong backend-active-check-rise WRONG'],
-        '_backend-active-check-rise-negative': [
-          'Wrong backend-active-check-rise -2'],
-        '_backend-active-check-timeout': [
-          'Wrong backend-active-check-timeout WRONG'],
-        '_backend-active-check-timeout-negative': [
-          'Wrong backend-active-check-timeout -2'],
+        '_health-check-fall': [
+          'Wrong health-check-fall WRONG'],
+        '_health-check-fall-negative': [
+          'Wrong health-check-fall -2'],
+        '_health-check-http-method': [
+          'Wrong health-check-http-method WRONG'],
+        '_health-check-http-version': [
+          'Wrong health-check-http-version WRONG/1.1'],
+        '_health-check-interval': [
+          'Wrong health-check-interval WRONG'],
+        '_health-check-interval-negative': [
+          'Wrong health-check-interval -2'],
+        '_health-check-rise': [
+          'Wrong health-check-rise WRONG'],
+        '_health-check-rise-negative': [
+          'Wrong health-check-rise -2'],
+        '_health-check-timeout': [
+          'Wrong health-check-timeout WRONG'],
+        '_health-check-timeout-negative': [
+          'Wrong health-check-timeout -2'],
       },
       'warning-slave-dict': {
         '_SSL_CA_CRT_ONLY': [
@@ -7269,7 +7024,7 @@ class TestPassedRequestParameter(HttpFrontendTestCase):
     )
 
 
-class TestSlaveBackendActiveCheck(SlaveHttpFrontendTestCase, TestDataMixin):
+class TestSlaveHealthCheck(SlaveHttpFrontendTestCase, TestDataMixin):
   @classmethod
   def getInstanceParameterDict(cls):
     return {
@@ -7286,28 +7041,28 @@ class TestSlaveBackendActiveCheck(SlaveHttpFrontendTestCase, TestDataMixin):
   def getSlaveParameterDictDict(cls):
     cls.setUpAssertionDict()
     return {
-      'backend-active-check-disabled': {
+      'health-check-disabled': {
         'url': cls.backend_url,
       },
-      'backend-active-check-default': {
+      'health-check-default': {
         'url': cls.backend_url,
-        'backend-active-check': True,
+        'health-check': True,
       },
-      'backend-active-check-connect': {
+      'health-check-connect': {
         'url': cls.backend_url,
-        'backend-active-check': True,
-        'backend-active-check-http-method': 'CONNECT',
+        'health-check': True,
+        'health-check-http-method': 'CONNECT',
       },
-      'backend-active-check-custom': {
+      'health-check-custom': {
         'url': cls.backend_url,
-        'backend-active-check': True,
-        'backend-active-check-http-method': 'POST',
-        'backend-active-check-http-path': '/POST-path to be encoded',
-        'backend-active-check-http-version': 'HTTP/1.0',
-        'backend-active-check-timeout': '7',
-        'backend-active-check-interval': '15',
-        'backend-active-check-rise': '3',
-        'backend-active-check-fall': '7',
+        'health-check': True,
+        'health-check-http-method': 'POST',
+        'health-check-http-path': '/POST-path to be encoded',
+        'health-check-http-version': 'HTTP/1.0',
+        'health-check-timeout': '7',
+        'health-check-interval': '15',
+        'health-check-rise': '3',
+        'health-check-fall': '7',
       },
     }
 
@@ -7315,35 +7070,35 @@ class TestSlaveBackendActiveCheck(SlaveHttpFrontendTestCase, TestDataMixin):
   def setUpAssertionDict(cls):
     backend = urlparse.urlparse(cls.backend_url).netloc
     cls.assertion_dict = {
-      'backend-active-check-disabled': """\
-backend _backend-active-check-disabled-http
+      'health-check-disabled': """\
+backend _health-check-disabled-http
   timeout server 12s
   timeout connect 5s
   retries 3
-  server _backend-active-check-disabled-backend %s""" % (backend,),
-      'backend-active-check-connect': """\
-backend _backend-active-check-connect-http
+  server _health-check-disabled-backend %s""" % (backend,),
+      'health-check-connect': """\
+backend _health-check-connect-http
   timeout server 12s
   timeout connect 5s
   retries 3
-  server _backend-active-check-connect-backend %s   check inter 5s"""
+  server _health-check-connect-backend %s   check inter 5s"""
       """ rise 1 fall 2
   timeout check 2s""" % (backend,),
-      'backend-active-check-custom': """\
-backend _backend-active-check-custom-http
+      'health-check-custom': """\
+backend _health-check-custom-http
   timeout server 12s
   timeout connect 5s
   retries 3
-  server _backend-active-check-custom-backend %s   check inter 15s"""
+  server _health-check-custom-backend %s   check inter 15s"""
       """ rise 3 fall 7
   option httpchk POST /POST-path%%20to%%20be%%20encoded HTTP/1.0
   timeout check 7s""" % (backend,),
-      'backend-active-check-default': """\
-backend _backend-active-check-default-http
+      'health-check-default': """\
+backend _health-check-default-http
   timeout server 12s
   timeout connect 5s
   retries 3
-  server _backend-active-check-default-backend %s   check inter 5s"""
+  server _health-check-default-backend %s   check inter 5s"""
       """ rise 1 fall 2
   option httpchk GET / HTTP/1.1
   timeout check 2s""" % (backend, )
@@ -7375,28 +7130,28 @@ backend _backend-active-check-default-http
 
     self.assertEqualResultJson(result, 'Path', '/test-path/deeper')
 
-  def test_backend_active_check_disabled(self):
-    self._test('backend-active-check-disabled')
+  def test_health_check_disabled(self):
+    self._test('health-check-disabled')
 
-  def test_backend_active_check_default(self):
-    self._test('backend-active-check-default')
+  def test_health_check_default(self):
+    self._test('health-check-default')
 
-  def test_backend_active_check_connect(self):
-    self._test('backend-active-check-connect')
+  def test_health_check_connect(self):
+    self._test('health-check-connect')
 
-  def test_backend_active_check_custom(self):
-    self._test('backend-active-check-custom')
+  def test_health_check_custom(self):
+    self._test('health-check-custom')
 
 
 if __name__ == '__main__':
-  class HTTP6Server(HTTPServer):
+  class HTTP6Server(ThreadedHTTPServer):
     address_family = socket.AF_INET6
   ip, port = sys.argv[1], int(sys.argv[2])
   if ':' in ip:
     klass = HTTP6Server
     url_template = 'http://[%s]:%s/'
   else:
-    klass = HTTPServer
+    klass = ThreadedHTTPServer
     url_template = 'http://%s:%s/'
 
   server = klass((ip, port), TestHandler)

software/caddy-frontend/test/test_data/test.TestSlave.test_file_list_log-CADDY.txt

@@ -55,15 +55,6 @@ T-2/var/log/httpd/_enable_cache-disable-no-cache-request_error_log
 T-2/var/log/httpd/_enable_cache-disable-via-header_access_log
 T-2/var/log/httpd/_enable_cache-disable-via-header_backend_log
 T-2/var/log/httpd/_enable_cache-disable-via-header_error_log
-T-2/var/log/httpd/_enable_cache-ssl-proxy-verify-unverified_access_log
-T-2/var/log/httpd/_enable_cache-ssl-proxy-verify-unverified_backend_log
-T-2/var/log/httpd/_enable_cache-ssl-proxy-verify-unverified_error_log
-T-2/var/log/httpd/_enable_cache-ssl-proxy-verify_ssl_proxy_ca_crt-unverified_access_log
-T-2/var/log/httpd/_enable_cache-ssl-proxy-verify_ssl_proxy_ca_crt-unverified_backend_log
-T-2/var/log/httpd/_enable_cache-ssl-proxy-verify_ssl_proxy_ca_crt-unverified_error_log
-T-2/var/log/httpd/_enable_cache-ssl-proxy-verify_ssl_proxy_ca_crt_access_log
-T-2/var/log/httpd/_enable_cache-ssl-proxy-verify_ssl_proxy_ca_crt_backend_log
-T-2/var/log/httpd/_enable_cache-ssl-proxy-verify_ssl_proxy_ca_crt_error_log
 T-2/var/log/httpd/_enable_cache_access_log
 T-2/var/log/httpd/_enable_cache_backend_log
 T-2/var/log/httpd/_enable_cache_custom_domain_access_log
@@ -150,15 +141,6 @@ T-2/var/log/httpd/_type-zope-prefer-gzip-encoding-to-backend-https-only_error_lo
 T-2/var/log/httpd/_type-zope-prefer-gzip-encoding-to-backend_access_log
 T-2/var/log/httpd/_type-zope-prefer-gzip-encoding-to-backend_backend_log
 T-2/var/log/httpd/_type-zope-prefer-gzip-encoding-to-backend_error_log
-T-2/var/log/httpd/_type-zope-ssl-proxy-verify-unverified_access_log
-T-2/var/log/httpd/_type-zope-ssl-proxy-verify-unverified_backend_log
-T-2/var/log/httpd/_type-zope-ssl-proxy-verify-unverified_error_log
-T-2/var/log/httpd/_type-zope-ssl-proxy-verify_ssl_proxy_ca_crt-unverified_access_log
-T-2/var/log/httpd/_type-zope-ssl-proxy-verify_ssl_proxy_ca_crt-unverified_backend_log
-T-2/var/log/httpd/_type-zope-ssl-proxy-verify_ssl_proxy_ca_crt-unverified_error_log
-T-2/var/log/httpd/_type-zope-ssl-proxy-verify_ssl_proxy_ca_crt_access_log
-T-2/var/log/httpd/_type-zope-ssl-proxy-verify_ssl_proxy_ca_crt_backend_log
-T-2/var/log/httpd/_type-zope-ssl-proxy-verify_ssl_proxy_ca_crt_error_log
 T-2/var/log/httpd/_type-zope-virtualhostroot-http-port_access_log
 T-2/var/log/httpd/_type-zope-virtualhostroot-http-port_backend_log
 T-2/var/log/httpd/_type-zope-virtualhostroot-http-port_error_log

software/caddy-frontend/test/test_data/test.TestSlaveGlobalDisableHttp2.test_file_list_log-CADDY.txt

@@ -55,15 +55,6 @@ T-2/var/log/httpd/_enable_cache-disable-no-cache-request_error_log
 T-2/var/log/httpd/_enable_cache-disable-via-header_access_log
 T-2/var/log/httpd/_enable_cache-disable-via-header_backend_log
 T-2/var/log/httpd/_enable_cache-disable-via-header_error_log
-T-2/var/log/httpd/_enable_cache-ssl-proxy-verify-unverified_access_log
-T-2/var/log/httpd/_enable_cache-ssl-proxy-verify-unverified_backend_log
-T-2/var/log/httpd/_enable_cache-ssl-proxy-verify-unverified_error_log
-T-2/var/log/httpd/_enable_cache-ssl-proxy-verify_ssl_proxy_ca_crt-unverified_access_log
-T-2/var/log/httpd/_enable_cache-ssl-proxy-verify_ssl_proxy_ca_crt-unverified_backend_log
-T-2/var/log/httpd/_enable_cache-ssl-proxy-verify_ssl_proxy_ca_crt-unverified_error_log
-T-2/var/log/httpd/_enable_cache-ssl-proxy-verify_ssl_proxy_ca_crt_access_log
-T-2/var/log/httpd/_enable_cache-ssl-proxy-verify_ssl_proxy_ca_crt_backend_log
-T-2/var/log/httpd/_enable_cache-ssl-proxy-verify_ssl_proxy_ca_crt_error_log
 T-2/var/log/httpd/_enable_cache_access_log
 T-2/var/log/httpd/_enable_cache_backend_log
 T-2/var/log/httpd/_enable_cache_custom_domain_access_log
@@ -150,15 +141,6 @@ T-2/var/log/httpd/_type-zope-prefer-gzip-encoding-to-backend-https-only_error_lo
 T-2/var/log/httpd/_type-zope-prefer-gzip-encoding-to-backend_access_log
 T-2/var/log/httpd/_type-zope-prefer-gzip-encoding-to-backend_backend_log
 T-2/var/log/httpd/_type-zope-prefer-gzip-encoding-to-backend_error_log
-T-2/var/log/httpd/_type-zope-ssl-proxy-verify-unverified_access_log
-T-2/var/log/httpd/_type-zope-ssl-proxy-verify-unverified_backend_log
-T-2/var/log/httpd/_type-zope-ssl-proxy-verify-unverified_error_log
-T-2/var/log/httpd/_type-zope-ssl-proxy-verify_ssl_proxy_ca_crt-unverified_access_log
-T-2/var/log/httpd/_type-zope-ssl-proxy-verify_ssl_proxy_ca_crt-unverified_backend_log
-T-2/var/log/httpd/_type-zope-ssl-proxy-verify_ssl_proxy_ca_crt-unverified_error_log
-T-2/var/log/httpd/_type-zope-ssl-proxy-verify_ssl_proxy_ca_crt_access_log
-T-2/var/log/httpd/_type-zope-ssl-proxy-verify_ssl_proxy_ca_crt_backend_log
-T-2/var/log/httpd/_type-zope-ssl-proxy-verify_ssl_proxy_ca_crt_error_log
 T-2/var/log/httpd/_type-zope-virtualhostroot-http-port_access_log
 T-2/var/log/httpd/_type-zope-virtualhostroot-http-port_backend_log
 T-2/var/log/httpd/_type-zope-virtualhostroot-http-port_error_log

software/caddy-frontend/test/test_data/test.TestSlaveBackendActiveCheck.test_file_list_log-CADDY.txt → software/caddy-frontend/test/test_data/test.TestSlaveHealthCheck.test_file_list_log-CADDY.txt

@@ -9,18 +9,18 @@ T-2/var/log/backend-haproxy.log
 T-2/var/log/expose-csr_id.log
 T-2/var/log/frontend-access.log
 T-2/var/log/frontend-error.log
-T-2/var/log/httpd/_backend-active-check-connect_access_log
-T-2/var/log/httpd/_backend-active-check-connect_backend_log
-T-2/var/log/httpd/_backend-active-check-connect_error_log
-T-2/var/log/httpd/_backend-active-check-custom_access_log
-T-2/var/log/httpd/_backend-active-check-custom_backend_log
-T-2/var/log/httpd/_backend-active-check-custom_error_log
-T-2/var/log/httpd/_backend-active-check-default_access_log
-T-2/var/log/httpd/_backend-active-check-default_backend_log
-T-2/var/log/httpd/_backend-active-check-default_error_log
-T-2/var/log/httpd/_backend-active-check-disabled_access_log
-T-2/var/log/httpd/_backend-active-check-disabled_backend_log
-T-2/var/log/httpd/_backend-active-check-disabled_error_log
+T-2/var/log/httpd/_health-check-connect_access_log
+T-2/var/log/httpd/_health-check-connect_backend_log
+T-2/var/log/httpd/_health-check-connect_error_log
+T-2/var/log/httpd/_health-check-custom_access_log
+T-2/var/log/httpd/_health-check-custom_backend_log
+T-2/var/log/httpd/_health-check-custom_error_log
+T-2/var/log/httpd/_health-check-default_access_log
+T-2/var/log/httpd/_health-check-default_backend_log
+T-2/var/log/httpd/_health-check-default_error_log
+T-2/var/log/httpd/_health-check-disabled_access_log
+T-2/var/log/httpd/_health-check-disabled_backend_log
+T-2/var/log/httpd/_health-check-disabled_error_log
 T-2/var/log/monitor-httpd-access.log
 T-2/var/log/monitor-httpd-error.log
 T-2/var/log/slave-introspection-access.log

software/caucase/buildout.hash.cfg

@@ -15,8 +15,8 @@
 # not need these here).
 [instance-caucased]
 filename = instance-caucased.cfg.jinja2
-md5sum = 2277c891a71534e00487468f5048d196
+md5sum = 8081efb41f12592033283db9841178ea
 
 [instance]
 filename = instance.cfg.jinja2
-md5sum = d40bed5ccc457ff7dc99b618bf29b189
+md5sum = 687ed460bebc18768ff7b7315e9b6de5

software/caucase/instance-caucased.cfg.jinja2

@@ -27,3 +27,4 @@ parts =
   publish
   caucased
   caucased-promise
+extends = {{ template_monitor }}

software/caucase/instance.cfg.jinja2

@@ -20,10 +20,12 @@ instance-caucased = {{ dumps(instance_caucased) }}
 recipe = slapos.recipe.template:jinja2
 template = ${context:instance-caucased}
 rendered = ${buildout:parts-directory}/instance-caucased.cfg
+monitor = {{ template_monitor }}
 context =
   key ipv6_set slap-configuration:ipv6
   key slapparameter_dict slap-configuration:configuration
   key bin_directory context:bin-directory
+  key template_monitor :monitor
 import-list =
   file caucase context:caucase-jinja2-library
 

software/caucase/software.cfg

@@ -2,6 +2,8 @@
 extends =
   buildout.hash.cfg
   ../../stack/caucase/buildout.cfg
+# Monitoring stack (keep on bottom)
+  ../../stack/monitor/buildout.cfg
 
 parts +=
   instance
@@ -24,3 +26,4 @@ context =
   key eggs_directory buildout:eggs-directory
   key caucase_jinja2_library caucase-jinja2-library:target
   key instance_caucased instance-caucased:target
+  key template_monitor monitor2-template:rendered

software/caucase/test/setup.py

+##############################################################################
+#
+# Copyright (c) 2018 Nexedi SA and Contributors. All Rights Reserved.
+#
+# WARNING: This program as such is intended to be used by professional
+# programmers who take the whole responsibility of assessing all potential
+# consequences resulting from its eventual inadequacies and bugs
+# End users who are looking for a ready-to-use solution with commercial
+# guarantees and support are strongly adviced to contract a Free Software
+# Service Company
+#
+# This program is Free Software; you can redistribute it and/or
+# modify it under the terms of the GNU General Public License
+# as published by the Free Software Foundation; either version 3
+# of the License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA  02111-1307, USA.
+#
+##############################################################################
+from setuptools import setup, find_packages
+
+version = '0.0.1.dev0'
+name = 'slapos.test.caucase'
+
+setup(
+    name=name,
+    version=version,
+    description="Test for SlapOS' Caucase",
+    maintainer="Nexedi",
+    maintainer_email="info@nexedi.com",
+    url="https://lab.nexedi.com/nexedi/slapos",
+    packages=find_packages(),
+    install_requires=[
+        'slapos.core',
+        'slapos.libnetworkcache',
+        'erp5.util',
+        'supervisor',
+        'pexpect',
+        'requests',
+    ],
+    zip_safe=True,
+    test_suite='test',
+)

software/caucase/test/test.py

+##############################################################################
+#
+# Copyright (c) 2019 Nexedi SA and Contributors. All Rights Reserved.
+#
+# WARNING: This program as such is intended to be used by professional
+# programmers who take the whole responsibility of assessing all potential
+# consequences resulting from its eventual inadequacies and bugs
+# End users who are looking for a ready-to-use solution with commercial
+# guarantees and support are strongly adviced to contract a Free Software
+# Service Company
+#
+# This program is Free Software; you can redistribute it and/or
+# modify it under the terms of the GNU General Public License
+# as published by the Free Software Foundation; either version 3
+# of the License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA  02111-1307, USA.
+#
+##############################################################################
+from __future__ import unicode_literals
+
+import json
+import os
+import requests
+import httplib
+
+from slapos.testing.testcase import makeModuleSetUpAndTestCaseClass
+
+setUpModule, SlapOSInstanceTestCase = makeModuleSetUpAndTestCaseClass(
+    os.path.abspath(
+        os.path.join(os.path.dirname(__file__), '..', 'software.cfg')))
+
+
+class TestCaucase(SlapOSInstanceTestCase):
+  def deserializeConnectionParameter(self):
+    return json.loads(
+      self.computer_partition.getConnectionParameter('_'))
+
+  def test(self):
+    connection_parameter_dict = self.deserializeConnectionParameter()
+    self.assertEqual(
+      connection_parameter_dict,
+      {'url': 'http://[%s]:8009' % (self._ipv6_address,)}
+    )
+    result = requests.get(connection_parameter_dict['url'])
+
+    self.assertEqual(result.status_code, httplib.OK)
+    self.assertEqual(
+      result.json(),
+      {
+        '_links': {
+          'self': {
+            'href': 'http://[%s]:8009' % (self._ipv6_address,)
+          },
+          'getCAUHAL': {
+            'href': 'http://[%s]:8009//cau' % (self._ipv6_address,),
+            'title': 'cau'
+          },
+          'getCASHAL': {
+            'href': 'http://[%s]:8009//cas' % (self._ipv6_address,),
+            'title': 'cas'
+          }
+        }
+      }
+    )

software/erp5/test/test/test_balancer.py

@@ -750,7 +750,7 @@ class TestFrontendXForwardedFor(BalancerTestCase):
     ).json()
     self.assertNotEqual(result['Incoming Headers'].get('x-forwarded-for', '').split(', ')[0], '1.2.3.4')
     balancer_url = json.loads(self.computer_partition.getConnectionParameterDict()['_'])['default-auth']
-    with self.assertRaises(OpenSSL.SSL.Error):
+    with self.assertRaisesRegexp(Exception, "certificate required"):
       requests.get(
         balancer_url,
         headers={'X-Forwarded-For': '1.2.3.4'},

software/erp5/test/test/test_mariadb.py

@@ -63,6 +63,7 @@ class MariaDBTestCase(ERP5InstanceTestCase):
     return {
         'tcpv4-port': 3306,
         'max-connection-count': 5,
+        'long-query-time': 3,
         'max-slowqueries-threshold': 1,
         'slowest-query-threshold': 0.1,
         # XXX what is this ? should probably not be needed here
@@ -115,12 +116,13 @@ class TestCrontabs(MariaDBTestCase, CrontabMixin):
     # run logrotate a first time so that it create state files
     self._executeCrontabAtDate('logrotate', '2000-01-01')
 
-    # make two slow queries
+    # make two slow queries. We are using long-query-time=3, so the queries
+    # must take more than 3 seconds to be logged.
     cnx = self.getDatabaseConnection()
     with contextlib.closing(cnx):
-      cnx.query("SELECT SLEEP(1.1)")
+      cnx.query("SELECT SLEEP(3.1)")
       cnx.store_result()
-      cnx.query("SELECT SLEEP(1.2)")
+      cnx.query("SELECT SLEEP(3.2)")
 
     # slow query crontab depends on crontab for log rotation
     # to be executed first.
@@ -171,7 +173,7 @@ class TestCrontabs(MariaDBTestCase, CrontabMixin):
 """\
 Threshold is lower than expected: 
 Expected total queries : 1.0 and current is: 2
-Expected slowest query : 0.1 and current is: 1
+Expected slowest query : 0.1 and current is: 3
 """)
 
 

software/jstestnode/buildout.hash.cfg

@@ -27,4 +27,4 @@ md5sum = 98faa5ad8cfb23a11d97a459078a1d05
 
 [template-runTestSuite]
 filename = runTestSuite.in
-md5sum = bb3f053b6cdb0a8888e9d32e63085ed5
+md5sum = 73ef758825563b7a6a1d660d4d5229b8

software/jstestnode/runTestSuite.in

@@ -97,8 +97,8 @@ def main():
             executable_path='${geckodriver:location}')
       else:
         assert target == 'selenium-server', "Unsupported target {}".format(test_runner['target'])
-        # use a remote connection which verifies TLS certificate
-        # workaround for https://github.com/SeleniumHQ/selenium/issues/6534
+        # use a remote connection which optionally verifies TLS certificate
+        # and optionally against a user provided CA.
         executor = RemoteConnection(test_runner['server-url'], keep_alive=True)
         cert_reqs = 'CERT_REQUIRED'
         ca_certs = certifi.where()

software/jstestnode/software.cfg

@@ -117,7 +117,4 @@ output = ${buildout:directory}/template-nginx.cfg.in
 output = ${buildout:directory}/runTestSuite.in
 
 [versions]
-
 selenium = 3.14.1
-urllib3 = 1.24
-certifi = 2018.10.15

software/kvm/buildout.hash.cfg

@@ -23,7 +23,7 @@ md5sum = 23493c541efef97ac5fe435114910b8e
 
 [template-kvm-cluster]
 filename = instance-kvm-cluster.cfg.jinja2.in
-md5sum = bdf8549a76ec61e125d51a05e611e004
+md5sum = 28a00c28a972f42627849b25c2792abb
 
 [template-kvm-resilient]
 filename = instance-kvm-resilient.cfg.jinja2

software/kvm/instance-kvm-cluster.cfg.jinja2.in

@@ -94,7 +94,15 @@ config-auto-ballooning = {{ dumps(kvm_parameter_dict.get('auto-ballooning', True
 {{ setconfig('disk-cache', kvm_parameter_dict.get('disk-cache', '')) }}
 {{ setconfig('disk-device-path', kvm_parameter_dict.get('disk-device-path', '')) }}
 
-{% set nat_rules_list = kvm_parameter_dict.get('nat-rules', []) -%}
+{# Note: dirty_nat_rules_list is cleaned up later, as the UI generated by JSON schema #}
+{#       gives freedom to the user to enter values separated by spaces and newlines #}
+{#       but on UI level they are only supported when separated by newlines, which #}
+{#       leads to cryptic failures of the cluster #}
+{% set dirty_nat_rules_list = kvm_parameter_dict.get('nat-rules', []) -%}
+{% set nat_rules_list = [] %}
+{% for nat_rule in dirty_nat_rules_list %}
+{%   do nat_rules_list.extend(nat_rule.split()) %}
+{% endfor %}
 {{ setconfig('nat-rules', nat_rules_list | join(' ')) }}
 config-publish-nat-url = True
 config-use-nat = {{ dumps(use_nat) }}

software/kvm/test/test.py

@@ -924,3 +924,78 @@ class TestCpuMemMaxDynamic(InstanceTestCase):
     self.assertIn('smp_max_count = 3', kvm_raw)
     self.assertIn('ram_size = 2048', kvm_raw)
     self.assertIn("ram_max_size = '2560'", kvm_raw)
+
+
+@skipUnlessKvm
+class TestNatRules(InstanceTestCase):
+  __partition_reference__ = 'nr'
+
+  @classmethod
+  def getInstanceParameterDict(cls):
+    return {
+      'nat-rules': '100 200',
+    }
+
+  def test(self):
+    connection_parameter_dict = self.computer_partition\
+      .getConnectionParameterDict()
+
+    self.assertIn('nat-rule-port-tcp-100', connection_parameter_dict)
+    self.assertIn('nat-rule-port-tcp-200', connection_parameter_dict)
+
+    self.assertEqual(
+      '%s : 10100' % (self._ipv6_address,),
+      connection_parameter_dict['nat-rule-port-tcp-100']
+    )
+    self.assertEqual(
+      '%s : 10200' % (self._ipv6_address,),
+      connection_parameter_dict['nat-rule-port-tcp-200']
+    )
+
+
+@skipUnlessKvm
+class TestNatRulesKvmCluster(InstanceTestCase):
+  __partition_reference__ = 'nrkc'
+
+  nat_rules = ["100", "200", "300"]
+  @classmethod
+  def getInstanceSoftwareType(cls):
+    return 'kvm-cluster'
+
+  @classmethod
+  def getInstanceParameterDict(cls):
+    return {'_': json.dumps({
+      "kvm-partition-dict": {
+        "KVM0": {
+            "nat-rules": cls.nat_rules,
+            "disable-ansible-promise": True,
+        }
+      }
+    })}
+
+  def getRunningHostFwd(self):
+    with self.slap.instance_supervisor_rpc as instance_supervisor:
+      kvm_pid = [q for q in instance_supervisor.getAllProcessInfo()
+                 if 'kvm-' in q['name']][0]['pid']
+      kvm_process = psutil.Process(kvm_pid)
+      for entry in kvm_process.cmdline():
+        if 'hostfwd' in entry:
+          return entry
+
+  def test(self):
+    host_fwd_entry = self.getRunningHostFwd()
+    self.assertIn(
+      'hostfwd=tcp:%s:10100-:100' % (self._ipv4_address,),
+      host_fwd_entry)
+    self.assertIn(
+      'hostfwd=tcp:%s:10200-:200' % (self._ipv4_address,),
+      host_fwd_entry)
+    self.assertIn(
+      'hostfwd=tcp:%s:10300-:300' % (self._ipv4_address,),
+      host_fwd_entry)
+
+
+@skipUnlessKvm
+class TestNatRulesKvmClusterComplex(TestNatRulesKvmCluster):
+  __partition_reference__ = 'nrkcc'
+  nat_rules = ["100", "200 300"]

software/monitor/software.cfg

@@ -81,12 +81,7 @@ surykatka = 0.5.0
 
 # For surykatka 0.5.0
 click = 7.0
-certifi = 2019.11.28
-chardet = 3.0.4
 dnspython = 1.16.0
 forcediphttpsadapter = 1.0.1
-idna = 2.8
 miniupnpc = 2.0.2
 peewee = 3.13.1
-requests = 2.22.0
-urllib3 = 1.25.8

software/seleniumserver/test/test.py

@@ -46,8 +46,10 @@ from PIL import Image
 from selenium import webdriver
 from selenium.webdriver.common.by import By
 from selenium.webdriver.common.desired_capabilities import DesiredCapabilities
+from selenium.webdriver.remote.remote_connection import RemoteConnection
 from selenium.webdriver.support import expected_conditions as EC
 from selenium.webdriver.support.ui import WebDriverWait
+import urllib3
 
 from slapos.testing.testcase import makeModuleSetUpAndTestCaseClass
 from slapos.testing.utils import findFreeTCPPort, ImageComparisonTestCase, ManagedHTTPServer
@@ -318,8 +320,13 @@ class TestFrontend(WebServerMixin, SeleniumServerTestCase):
     self.assertEqual('selenium', parsed.username)
     self.assertTrue(parsed.password)
 
+    # XXX we are using a self signed certificate, but selenium 3.141.0 does
+    # not expose API to ignore certificate verification
+    executor = RemoteConnection(webdriver_url, keep_alive=True)
+    executor._conn = urllib3.PoolManager(cert_reqs='CERT_NONE', ca_certs=None)
+
     driver = webdriver.Remote(
-        command_executor=webdriver_url,
+        command_executor=executor,
         desired_capabilities=DesiredCapabilities.CHROME)
 
     driver.get(self.server_url)

software/slapos-sr-testing/software.cfg

@@ -167,6 +167,11 @@ setup = ${slapos-repository:location}/software/repman/test/
 repository = https://lab.nexedi.com/nexedi/slapos.core.git
 branch = master
 
+[slapos.test.caucase-setup]
+<= setup-develop-egg
+egg = slapos.test.caucase
+setup = ${slapos-repository:location}/software/caucase/test/
+
 [slapos.core-setup]
 <= setup-develop-egg
 egg = slapos.core
@@ -209,6 +214,7 @@ extra-eggs =
   ${slapos.test.dream-setup:egg}
   ${slapos.test.metabase-setup:egg}
   ${slapos.test.repman-setup:egg}
+  ${slapos.test.caucase-setup:egg}
 
 # We don't name this interpreter `python`, so that when we run slapos node
 # software, installation scripts running `python` use a python without any
@@ -278,6 +284,7 @@ extra =
   gitlab ${slapos.test.gitlab-setup:setup}
   cloudooo ${slapos.test.cloudooo-setup:setup}
   dream ${slapos.test.dream-setup:setup}
+  caucase ${slapos.test.caucase-setup:setup}
 
 [versions]
 # slapos.core is used from the clone always
@@ -298,16 +305,7 @@ selenium = 3.141.0
 # Patched eggs
 PyPDF2 = 1.26.0+SlapOSPatched001
 
-# modern versions for SSL fixed support
-requests = 2.20.0
-certifi = 2018.10.15
-idna = 2.7
-chardet = 3.0.4
-
-# ipaddress is patching IPAddress so IPv6 match works
-ipaddress = 1.0.22
-
-# cacuase and its dependencies
+# caucase and its dependencies
 caucase = 0.9.4
 pem = 18.2.0
 PyJWT = 1.6.4
@@ -315,10 +313,6 @@ PyJWT = 1.6.4
 # Django 1.11 is python 2 compatible
 Django = 1.11
 
-# Required by:
-# selenium==3.141.0
-urllib3 = 1.24.1
-
 mock = 2.0.0
 testfixtures = 6.11
 funcsigs = 1.0.2

stack/slapos.cfg

@@ -145,8 +145,8 @@ hexagonit.recipe.download = 1.7.post4
 Jinja2 = 2.11.2
 Importing = 1.10
 MarkupSafe = 1.0
-PyYAML = 3.13
-Werkzeug = 0.12
+PyYAML = 5.4.1
+Werkzeug = 1.0.1
 asn1crypto = 1.3.0
 atomicwrites = 1.4.0
 backports.functools-lru-cache = 1.6.1
@@ -159,7 +159,7 @@ collective.recipe.shelloutput = 0.1
 collective.recipe.template = 2.0
 configparser = 4.0.2
 contextlib2 = 0.6.0.post1
-cryptography = 2.9.2
+cryptography = 3.3.1
 dateparser = 0.7.6
 decorator = 4.3.0
 funcsigs = 1.0.2
@@ -171,7 +171,7 @@ importlib-metadata = 1.7.0
 inotify-simple = 1.1.1
 itsdangerous = 0.24
 lock-file = 2.0
-lxml = 4.4.3
+lxml = 4.6.2
 meld3 = 1.0.2
 more-itertools = 5.0.0
 netaddr = 0.7.19
@@ -179,7 +179,7 @@ pathlib2 = 2.3.5
 pbr = 2.0.0
 plone.recipe.command = 1.1
 prettytable = 0.7.2
-psutil = 5.6.3
+psutil = 5.8.0
 pluggy = 0.13.1
 py = 1.9.0
 pyOpenSSL = 19.1.0
@@ -200,7 +200,7 @@ slapos.rebootstrap = 4.5
 slapos.recipe.build = 0.46
 slapos.recipe.cmmi = 0.16
 slapos.recipe.template = 4.5
-slapos.toolbox = 0.115
+slapos.toolbox = 0.116
 stevedore = 1.21.0
 subprocess32 = 3.5.3
 unicodecsv = 0.14.1
@@ -217,7 +217,7 @@ msgpack = 0.6.2
 
 # Required by:
 # slapos.core==1.5.0
-Flask = 0.12
+Flask = 1.1.2
 
 # Required by:
 # slapos.toolbox==0.94
@@ -251,8 +251,6 @@ croniter = 0.3.25
 # slapos.toolbox==0.94
 dnspython = 1.16.0
 
-# Required by:
-# cryptography==1.8.1
 enum34 = 1.1.10
 
 # Required by:
@@ -279,8 +277,6 @@ pyrsistent = 0.14.5
 # jsonschema==3.0.2
 setuptools-scm = 3.5.0
 
-# Required by:
-# cryptography==1.8.1
 ipaddress = 1.0.23
 
 # Required by:
@@ -296,8 +292,6 @@ lockfile = 0.12.2
 # XXX 'slapos node format' raises an exception with netifaces 0.10.5.
 netifaces = 0.10.7
 
-# Required by:
-# cryptography==1.8.1
 packaging = 16.8
 
 # Required by: