stack/erp5: implement Zope's rewrite rules in ERP5 balancer partition

The strategy for compatibility is that:
 - haproxy still listen on the same port as before, without rewrite rule.
   This is called "legacy" port.
 - for each frontend from request parameters, we introduce an haproxy
   frontend with a rewrite for the corresponding `internal-path`
   parameter.
 - the shared frontend instance is updated to use this new frontend
   entry from haproxy. This will cause a small downtime until the shared
   frontend is updated to the new URL on ERP5, but since this feature
   was not used, it's OK.

Technical details are that we:
 - split haproxy config to have frontends and backends.
 - introduce one frontend in haproxy for each frontend from request
   parameters.
 - routing-rule-list argument is still honored the same way, globally
   and after path from frontend.
 - change the shared frontend requests to use "" type, no longer "zope"
   type.
 - we don't do automatic detection of /VirtualHostRoot in URL but always
   add it, because it could be used to trick zope into thinking it
   serves requests for an arbitrary host and do open redirects
 - before using the request's host header in virtualhost path, we check
   that it does not contain /, to prevent injection of virutalhost path
   elements through the host header.
 - we don't use the "path" parameter from shared frontend, because we
   want the frontend to be simple, so we don't want it to rewrite the
   request path (which is also the reason why we deprecated "zope" type)
 - the tests have changed a lot, because they were using what's now the
   "legacy" URL types, so we updated it to use the new URL types with
   all the /VirtualHostRoot/../ in path and also because they use IPv6
   URL, no longer IPv4

software/erp5/instance-erp5-input-schema.json

@@ -5,7 +5,7 @@
   "additionalProperties": false,
   "definitions": {
     "routing-rule-list": {
-      "description": "Maps the path received in requests to given zope path. Rules are applied in the order they are given. This requires the path received from the outside world (typically: frontend) to have its root correspond to Zope's root (for frontend: 'path' parameter must be empty), with the customary VirtualHostMonster construct (for frontend: 'type' must be 'zope').",
+      "description": "Maps the path received in requests to given zope path. Rules are applied in the order they are given, after 'internal-path' from 'frontend' parameter. This also supports legacy frontends, using Rapid CDN with \"zope\" type.",
       "type": "array",
       "default": [
         [

software/erp5/test/test/__init__.py

@@ -197,3 +197,4 @@ class ERP5InstanceTestCase(SlapOSInstanceTestCase, metaclass=ERP5InstanceTestMet
   def getComputerPartitionPath(cls, partition_reference):
     partition_id = cls.getComputerPartition(partition_reference).getId()
     return os.path.join(cls.slap._instance_root, partition_id)
+

software/erp5/test/test/test_balancer.py

@@ -174,6 +174,12 @@ class BalancerTestCase(ERP5InstanceTestCase):
             'caucase-url': cls.getManagedResource("caucase", CaucaseService).url,
         },
         'timeout-dict': {'default': None},
+        'frontend-parameter-dict': {
+          'default': {
+            'internal-path': '',
+            'zope-family': 'default',
+          },
+        },
         'family-path-routing-dict': {},
         'path-routing-list': [],
       }
@@ -184,26 +190,72 @@ class BalancerTestCase(ERP5InstanceTestCase):
     return {'_': json.dumps(cls._getInstanceParameterDict())}
 
   def setUp(self):
-    # type: () -> None
-    self.default_balancer_url = json.loads(
+    self.default_balancer_direct_url = json.loads(
         self.computer_partition.getConnectionParameterDict()['_'])['default']
+    self.default_balancer_zope_url = json.loads(
+        self.computer_partition.getConnectionParameterDict()['_'])['url-backend-default']
+
+
+class TestURLRewrite(BalancerTestCase):
+  __partition_reference__ = 'ur'
+  def test_direct(self):
+    self.assertEqual(requests.get(self.default_balancer_direct_url, verify=False).json()['Path'], '/')
+    self.assertEqual(
+      requests.get(
+        urllib.parse.urljoin(
+          self.default_balancer_direct_url,
+          '/VirtualHostBase/https/example.com:443/VirtualHostRoot/path'),
+        verify=False
+      ).json()['Path'],
+      '/VirtualHostBase/https/example.com:443/VirtualHostRoot/path')
+
+  def test_zope(self):
+    netloc = urllib.parse.urlparse(self.default_balancer_zope_url).netloc
+    self.assertEqual(
+      requests.get(self.default_balancer_zope_url, verify=False).json()['Path'],
+      f'/VirtualHostBase/https/{netloc}/VirtualHostRoot/')
+    self.assertEqual(
+      requests.get(urllib.parse.urljoin(
+        self.default_balancer_zope_url, 'path'), verify=False).json()['Path'],
+      f'/VirtualHostBase/https/{netloc}/VirtualHostRoot/path')
+    self.assertEqual(
+      requests.get(
+        urllib.parse.urljoin(
+          self.default_balancer_zope_url,
+          '/VirtualHostBase/https/example.com:443/VirtualHostRoot/path'),
+        verify=False
+      ).json()['Path'],
+      f'/VirtualHostBase/https/{netloc}/VirtualHostRoot/VirtualHostBase/https/example.com:443/VirtualHostRoot/path')
+
+  def test_bad_host(self):
+    self.assertEqual(
+      requests.get(self.default_balancer_zope_url, headers={'Host': 'a/b'}, verify=False).status_code,
+      requests.codes.bad_request)
 
 
 class SlowHTTPServer(ManagedHTTPServer):
   """An HTTP Server which reply after a timeout.
 
-  Timeout is 2 seconds by default, and can be specified in the path of the URL
+  Timeout is 2 seconds by default, and can be specified in the path of the URL:
+
+    GET /{timeout}
+
+  but because balancer rewrites the URL, the actual URL used by this server is:
+
+    GET /VirtualHostBase/https/{host}/VirtualHostRoot/{timeout}
+
   """
   class RequestHandler(BaseHTTPRequestHandler):
     def do_GET(self):
-      # type: () -> None
-      self.send_response(200)
-      self.send_header("Content-Type", "text/plain")
       timeout = 2
+      if self.path == '/':  # for health checks
+        timeout = 0
       try:
-        timeout = int(self.path[1:])
-      except ValueError:
+        timeout = int(self.path.split('/')[5])
+      except (ValueError, IndexError):
         pass
+      self.send_response(200)
+      self.send_header("Content-Type", "text/plain")
       time.sleep(timeout)
       self.end_headers()
       self.wfile.write(b"OK\n")
@@ -227,12 +279,12 @@ class TestTimeout(BalancerTestCase, CrontabMixin):
     # type: () -> None
     self.assertEqual(
       requests.get(
-          six.moves.urllib.parse.urljoin(self.default_balancer_url, '/1'),
+          six.moves.urllib.parse.urljoin(self.default_balancer_zope_url, '/1'),
           verify=False).status_code,
       requests.codes.ok)
     self.assertEqual(
       requests.get(
-          six.moves.urllib.parse.urljoin(self.default_balancer_url, '/5'),
+          six.moves.urllib.parse.urljoin(self.default_balancer_zope_url, '/5'),
           verify=False).status_code,
       requests.codes.gateway_timeout)
 
@@ -252,7 +304,7 @@ class TestLog(BalancerTestCase, CrontabMixin):
   def test_access_log_format(self):
     # type: () -> None
     requests.get(
-        six.moves.urllib.parse.urljoin(self.default_balancer_url, '/url_path'),
+        six.moves.urllib.parse.urljoin(self.default_balancer_zope_url, '/url_path'),
         verify=False,
     )
     time.sleep(.5) # wait a bit more until access is logged
@@ -265,7 +317,7 @@ class TestLog(BalancerTestCase, CrontabMixin):
     # the request - but our test machines can be slow sometimes, so we tolerate
     # it can take up to 20 seconds.
     match = re.match(
-        r'([(\d\.)]+) - - \[(.*?)\] "(.*?)" (\d+) (\d+) "(.*?)" "(.*?)" (\d+)',
+        r'([(\da-fA-F:\.)]+) - - \[(.*?)\] "(.*?)" (\d+) (\d+) "(.*?)" "(.*?)" (\d+)',
         access_line
     )
     self.assertTrue(match)
@@ -277,7 +329,7 @@ class TestLog(BalancerTestCase, CrontabMixin):
   def test_access_log_apachedex_report(self):
     # type: () -> None
     # make a request so that we have something in the logs
-    requests.get(self.default_balancer_url, verify=False)
+    requests.get(self.default_balancer_zope_url, verify=False)
 
     # crontab for apachedex is executed
     self._executeCrontabAtDate('generate-apachedex-report', '23:59')
@@ -303,7 +355,7 @@ class TestLog(BalancerTestCase, CrontabMixin):
     self._executeCrontabAtDate('logrotate', '2000-01-01')
 
     # make a request so that we have something in the logs
-    requests.get(self.default_balancer_url, verify=False).raise_for_status()
+    requests.get(self.default_balancer_zope_url, verify=False).raise_for_status()
 
     # slow query crontab depends on crontab for log rotation
     # to be executed first.
@@ -318,7 +370,7 @@ class TestLog(BalancerTestCase, CrontabMixin):
     )
     self.assertTrue(os.path.exists(rotated_log_file))
 
-    requests.get(self.default_balancer_url, verify=False).raise_for_status()
+    requests.get(self.default_balancer_zope_url, verify=False).raise_for_status()
     # on next day execution of logrotate, log files are compressed
     self._executeCrontabAtDate('logrotate', '2050-01-02')
     self.assertTrue(os.path.exists(rotated_log_file + '.xz'))
@@ -333,11 +385,11 @@ class TestLog(BalancerTestCase, CrontabMixin):
     # after a while, balancer should detect and log this event in error log
     time.sleep(5)
     self.assertEqual(
-        requests.get(self.default_balancer_url, verify=False).status_code,
+        requests.get(self.default_balancer_zope_url, verify=False).status_code,
         requests.codes.service_unavailable)
     with open(os.path.join(self.computer_partition_root_path, 'var', 'log', 'apache-error.log')) as error_log_file:
       error_line = error_log_file.read().splitlines()[-1]
-    self.assertIn('proxy family_default has no server available!', error_line)
+    self.assertIn('backend default has no server available!', error_line)
     # this log also include a timestamp
     self.assertRegexpMatches(error_line, r'\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}')
 
@@ -345,7 +397,9 @@ class TestLog(BalancerTestCase, CrontabMixin):
 class BalancerCookieHTTPServer(ManagedHTTPServer):
   """An HTTP Server which can set balancer cookie.
 
-  This server set cookie when requested /set-cookie path.
+  This server set cookie when requested /set-cookie path (actually
+  /VirtualHostBase/https/{host}/VirtualHostRoot/set-cookie , which is
+  added by balancer proxy)
 
   The reply body is the name used when registering this resource
   using getManagedResource. This way we can assert which
@@ -360,7 +414,8 @@ class BalancerCookieHTTPServer(ManagedHTTPServer):
         # type: () -> None
         self.send_response(200)
         self.send_header("Content-Type", "text/plain")
-        if self.path == '/set_cookie':
+
+        if self.path != '/' and self.path.split('/')[5] == 'set_cookie':
           # the balancer tells the backend what's the name of the balancer cookie with
           # the X-Balancer-Current-Cookie header.
           self.send_header('Set-Cookie', '%s=anything' % self.headers['X-Balancer-Current-Cookie'])
@@ -393,7 +448,7 @@ class TestBalancer(BalancerTestCase):
     # type: () -> None
     # requests are by default balanced to both servers
     self.assertEqual(
-        {requests.get(self.default_balancer_url, verify=False).text for _ in range(10)},
+        {requests.get(self.default_balancer_zope_url, verify=False).text for _ in range(10)},
         {'backend_web_server1', 'backend_web_server2'}
     )
 
@@ -403,7 +458,7 @@ class TestBalancer(BalancerTestCase):
     self.getManagedResource("backend_web_server2", BalancerCookieHTTPServer).close()
     self.addCleanup(self.getManagedResource("backend_web_server2", BalancerCookieHTTPServer).open)
     self.assertEqual(
-        {requests.get(self.default_balancer_url, verify=False).text for _ in range(10)},
+        {requests.get(self.default_balancer_zope_url, verify=False).text for _ in range(10)},
         {'backend_web_server1',}
     )
 
@@ -412,7 +467,7 @@ class TestBalancer(BalancerTestCase):
     # if backend provides a "SERVERID" cookie, balancer will overwrite it with the
     # backend selected by balancing algorithm
     self.assertIn(
-        requests.get(six.moves.urllib.parse.urljoin(self.default_balancer_url, '/set_cookie'), verify=False).cookies['SERVERID'],
+        requests.get(six.moves.urllib.parse.urljoin(self.default_balancer_zope_url, '/set_cookie'), verify=False).cookies['SERVERID'],
         ('default-0', 'default-1'),
     )
 
@@ -421,7 +476,7 @@ class TestBalancer(BalancerTestCase):
     # if request is made with the sticky cookie, the client stick on one balancer
     cookies = dict(SERVERID='default-1')
     self.assertEqual(
-        {requests.get(self.default_balancer_url, verify=False, cookies=cookies).text for _ in range(10)},
+        {requests.get(self.default_balancer_zope_url, verify=False, cookies=cookies).text for _ in range(10)},
         {'backend_web_server2',}
     )
 
@@ -429,7 +484,7 @@ class TestBalancer(BalancerTestCase):
     self.getManagedResource("backend_web_server2", BalancerCookieHTTPServer).close()
     self.addCleanup(self.getManagedResource("backend_web_server2", BalancerCookieHTTPServer).open)
     self.assertEqual(
-        requests.get(self.default_balancer_url, verify=False, cookies=cookies).text,
+        requests.get(self.default_balancer_zope_url, verify=False, cookies=cookies).text,
         'backend_web_server1')
 
   def test_balancer_stats_socket(self):
@@ -450,7 +505,7 @@ class TestBalancer(BalancerTestCase):
       raise
     self.assertEqual(socat_process.poll(), 0)
     # output is a csv
-    self.assertIn(b'family_default,FRONTEND,', output)
+    self.assertIn(b'\ndefault,BACKEND,', output)
 
 
 class TestTestRunnerEntryPoints(BalancerTestCase):
@@ -556,7 +611,7 @@ class TestHTTP(BalancerTestCase):
             '--insecure',
             '--write-out',
             '%{http_version}',
-            self.default_balancer_url,
+            self.default_balancer_zope_url,
         ]),
         b'2',
     )
@@ -568,16 +623,16 @@ class TestHTTP(BalancerTestCase):
       session.verify = False
 
       # do a first request, which establish a first connection
-      session.get(self.default_balancer_url).raise_for_status()
+      session.get(self.default_balancer_zope_url).raise_for_status()
 
       # "break" new connection method and check we can make another request
       with mock.patch(
           "requests.packages.urllib3.connectionpool.HTTPSConnectionPool._new_conn",
       ) as new_conn:
-        session.get(self.default_balancer_url).raise_for_status()
+        session.get(self.default_balancer_zope_url).raise_for_status()
       new_conn.assert_not_called()
 
-      parsed_url = six.moves.urllib.parse.urlparse(self.default_balancer_url)
+      parsed_url = six.moves.urllib.parse.urlparse(self.default_balancer_zope_url)
 
       # check that we have an open file for the ip connection
       self.assertTrue([
@@ -592,6 +647,8 @@ class ContentTypeHTTPServer(ManagedHTTPServer):
 
   For example when requested http://host/text/plain it will reply
   with Content-Type: text/plain header.
+  This actually uses a URL like this to support zope style virtual host:
+  GET /VirtualHostBase/https/{host}/VirtualHostRoot/text/plain
 
   The body is always "OK"
   """
@@ -603,7 +660,7 @@ class ContentTypeHTTPServer(ManagedHTTPServer):
       if self.path == '/':
         self.send_header("Content-Length", '0')
         return self.end_headers()
-      content_type = self.path[1:]
+      content_type = '/'.join(self.path.split('/')[5:])
       body = b"OK"
       self.send_header("Content-Type", content_type)
       self.send_header("Content-Length", str(len(body)))
@@ -647,7 +704,7 @@ class TestContentEncoding(BalancerTestCase):
         'application/font-woff2',
         'application/x-font-opentype',
         'application/wasm',):
-      resp = requests.get(six.moves.urllib.parse.urljoin(self.default_balancer_url, content_type), verify=False)
+      resp = requests.get(six.moves.urllib.parse.urljoin(self.default_balancer_zope_url, content_type), verify=False)
       self.assertEqual(resp.headers['Content-Type'], content_type)
       self.assertEqual(
           resp.headers.get('Content-Encoding'),
@@ -657,7 +714,7 @@ class TestContentEncoding(BalancerTestCase):
 
   def test_no_gzip_encoding(self):
     # type: () -> None
-    resp = requests.get(six.moves.urllib.parse.urljoin(self.default_balancer_url, '/image/png'), verify=False)
+    resp = requests.get(six.moves.urllib.parse.urljoin(self.default_balancer_zope_url, '/image/png'), verify=False)
     self.assertNotIn('Content-Encoding', resp.headers)
     self.assertEqual(resp.text, 'OK')
 
@@ -846,7 +903,7 @@ class TestServerTLSProvidedCertificate(BalancerTestCase):
   def test_certificate_validates_with_provided_ca(self):
     # type: () -> None
     server_certificate = self.getManagedResource("server_certificate", CaucaseCertificate)
-    requests.get(self.default_balancer_url, verify=server_certificate.ca_crt_file)
+    requests.get(self.default_balancer_zope_url, verify=server_certificate.ca_crt_file)
 
 
 class TestClientTLS(BalancerTestCase):
@@ -895,7 +952,7 @@ class TestClientTLS(BalancerTestCase):
       def _make_request():
         # type: () -> dict
         return requests.get(
-            self.default_balancer_url,
+            self.default_balancer_zope_url,
             cert=(client_certificate.cert_file, client_certificate.key_file),
             verify=False,
         ).json()

software/erp5/test/test/test_erp5.py

@@ -58,15 +58,8 @@ class TestPublishedURLIsReachableMixin(object):
   """Mixin that checks that default page of ERP5 is reachable.
   """
 
-  def _checkERP5IsReachable(self, base_url, site_id, verify):
-    # We access ERP5 trough a "virtual host", which should make
-    # ERP5 produce URLs using https://virtual-host-name:1234/virtual_host_root
-    # as base.
-    virtual_host_url = six.moves.urllib.parse.urljoin(
-        base_url,
-        '/VirtualHostBase/https/virtual-host-name:1234/{}/VirtualHostRoot/_vh_virtual_host_root/'
-        .format(site_id))
-
+  @contextlib.contextmanager
+  def requestSession(self, base_url):
     # What happens is that instantiation just create the services, but does not
     # wait for ERP5 to be initialized. When this test run ERP5 instance is
     # instantiated, but zope is still busy creating the site and haproxy replies
@@ -82,7 +75,32 @@ class TestPublishedURLIsReachableMixin(object):
                   total=20,
                   backoff_factor=.5,
                   status_forcelist=(404, 500, 503))))
+      yield session
+
+  def _checkERP5IsReachableWithVirtualHost(self, url, verify):
+    with self.requestSession(six.moves.urllib.parse.urljoin(url, '/')) as session:
+      r = session.get(url, verify=verify, allow_redirects=True)
+      # access on / are redirected to login form
+      self.assertTrue(r.url.endswith('/login_form'))
+      self.assertEqual(r.status_code, requests.codes.ok)
+      self.assertIn("ERP5", r.text)
+
+      # host header is used in redirected URL. The URL is always https
+      r = session.get(url, verify=verify, allow_redirects=False, headers={'Host': 'www.example.com'})
+      self.assertEqual(r.headers.get('Location'), 'https://www.example.com/login_form')
+      r = session.get(url, verify=verify, allow_redirects=False, headers={'Host': 'www.example.com:1234'})
+      self.assertEqual(r.headers.get('Location'), 'https://www.example.com:1234/login_form')
 
+  def _checkERP5IsReachableWithoutVirtualHost(self, base_url, site_id, verify):
+    # We access ERP5 trough a "virtual host", which should make
+    # ERP5 produce URLs using https://virtual-host-name:1234/virtual_host_root
+    # as base.
+    virtual_host_url = six.moves.urllib.parse.urljoin(
+        base_url,
+        '/VirtualHostBase/https/virtual-host-name:1234/{}/VirtualHostRoot/_vh_virtual_host_root/'
+        .format(site_id))
+
+    with self.requestSession(base_url) as session:
       r = session.get(virtual_host_url, verify=verify, allow_redirects=False)
       self.assertEqual(r.status_code, requests.codes.found)
       # access on / are redirected to login form, with virtual host preserved
@@ -101,7 +119,7 @@ class TestPublishedURLIsReachableMixin(object):
     """Tests the IPv6 URL published by the root partition is reachable.
     """
     param_dict = self.getRootPartitionConnectionParameterDict()
-    self._checkERP5IsReachable(
+    self._checkERP5IsReachableWithoutVirtualHost(
       param_dict['family-default-v6'],
       param_dict['site-id'],
       verify=False,
@@ -111,7 +129,7 @@ class TestPublishedURLIsReachableMixin(object):
     """Tests the IPv4 URL published by the root partition is reachable.
     """
     param_dict = self.getRootPartitionConnectionParameterDict()
-    self._checkERP5IsReachable(
+    self._checkERP5IsReachableWithoutVirtualHost(
       param_dict['family-default'],
       param_dict['site-id'],
       verify=False,
@@ -121,10 +139,14 @@ class TestPublishedURLIsReachableMixin(object):
     """Tests the frontend URL published by the root partition is reachable.
     """
     param_dict = self.getRootPartitionConnectionParameterDict()
-    self._checkERP5IsReachable(
+    self._checkERP5IsReachableWithVirtualHost(
       param_dict['url-frontend-default'],
+<<<<<<< HEAD
       param_dict['site-id'],
       verify=False,
+=======
+      self._getCaucaseServiceCACertificate(),
+>>>>>>> 6e7358084 (stack/erp5: implement Zope's rewrite rules in ERP5 balancer partition)
     )
 
 
@@ -139,9 +161,8 @@ class TestDefaultParameters(ERP5InstanceTestCase, TestPublishedURLIsReachableMix
                            '.installed-switch-softwaretype.cfg')) as f:
       installed = zc.buildout.configparser.parse(f, 'installed')
     self.assertEqual(
-      installed['request-frontend-default']['config-type'], 'zope')
-    self.assertEqual(
-      installed['request-frontend-default']['config-path'], '/erp5')
+      installed['request-frontend-default']['config-type'], '')
+    self.assertNotIn('config-path', installed['request-frontend-default'])
     self.assertEqual(
       installed['request-frontend-default']['config-authenticate-to-backend'], 'true')
     self.assertEqual(installed['request-frontend-default']['shared'], 'true')
@@ -318,6 +339,8 @@ class TestBalancerPortsStable(ERP5InstanceTestCase):
 
   def test_same_balancer_ports_when_adding_zopes_or_frontends(self):
     param_dict_before = self.getRootPartitionConnectionParameterDict()
+    balancer_param_dict_before = json.loads(
+      self.getComputerPartition('balancer').getConnectionParameter('_'))
 
     # re-request with one more frontend and one more backend, that are before
     # the existing ones when sorting alphabetically
@@ -338,10 +361,15 @@ class TestBalancerPortsStable(ERP5InstanceTestCase):
     rerequest()
     self.slap.waitForInstance(max_retry=10)
     param_dict_after = json.loads(rerequest().getConnectionParameterDict()['_'])
+    balancer_param_dict_after = json.loads(
+      self.getComputerPartition('balancer').getConnectionParameter('_'))
+
     self.assertEqual(param_dict_before['family-zzz-v6'], param_dict_after['family-zzz-v6'])
     self.assertEqual(param_dict_before['url-frontend-zzz'], param_dict_after['url-frontend-zzz'])
+    self.assertEqual(balancer_param_dict_before['url-backend-zzz'], balancer_param_dict_after['url-backend-zzz'])
     self.assertNotEqual(param_dict_before['family-zzz-v6'], param_dict_after['family-aaa-v6'])
     self.assertNotEqual(param_dict_before['url-frontend-zzz'], param_dict_after['url-frontend-aaa'])
+    self.assertNotEqual(balancer_param_dict_before['url-backend-zzz'], balancer_param_dict_after['url-backend-aaa'])
 
 
 class TestSeleniumTestRunner(ERP5InstanceTestCase, TestPublishedURLIsReachableMixin):
@@ -404,14 +432,15 @@ class TestDisableTestRunner(ERP5InstanceTestCase, TestPublishedURLIsReachableMix
     self.assertNotIn('runTestSuite', bin_programs)
 
   def test_no_haproxy_testrunner_port(self):
-    # Haproxy only listen on two ports, there is no haproxy ports allocated for test runner
+    # Haproxy only listen on two ports for frontend, two ports for legacy entry points
+    # and there is no haproxy ports allocated for test runner
     with self.slap.instance_supervisor_rpc as supervisor:
       all_process_info = supervisor.getAllProcessInfo()
     process_info, = [p for p in all_process_info if p['name'].startswith('haproxy')]
     haproxy_master_process = psutil.Process(process_info['pid'])
     haproxy_worker_process, = haproxy_master_process.children()
     self.assertEqual(
-        sorted([socket.AF_INET, socket.AF_INET6]),
+        sorted([socket.AF_INET, socket.AF_INET6, socket.AF_INET, socket.AF_INET6]),
         sorted(
             c.family
             for c in haproxy_worker_process.connections()
@@ -1040,3 +1069,351 @@ class TestZopePublisherTimeout(ZopeSkinsMixin, ERP5InstanceTestCase):
         self._getAuthenticatedZopeUrl('ERP5Site_doSlowRequest', family_name='no-timeout'),
         verify=False,
         timeout=6)
+<<<<<<< HEAD
+=======
+
+
+class TestCloudooo(ZopeSkinsMixin, ERP5InstanceTestCase):
+  """Test ERP5 can be instantiated with cloudooo parameters
+  """
+  __partition_reference__ = 'c'
+
+  @classmethod
+  def getInstanceParameterDict(cls):
+    return {
+        '_':
+        json.dumps({
+            'cloudooo-url-list': [
+              'https://cloudooo1.example.com/',
+              'https://cloudooo2.example.com/',
+            ],
+            'cloudooo-retry-count': 123,
+        })
+    }
+
+  def test_cloudooo_url_list_preference(self):
+    self.assertEqual(
+      requests.get(
+          self._getAuthenticatedZopeUrl(
+            'portal_preferences/getPreferredDocumentConversionServerUrlList'),
+          verify=False).text,
+      "['https://cloudooo1.example.com/', 'https://cloudooo2.example.com/']")
+
+  @unittest.expectedFailure # setting "retry" is not implemented
+  def test_cloudooo_retry_count_preference(self):
+    self.assertEqual(
+      requests.get(
+          self._getAuthenticatedZopeUrl(
+            'portal_preferences/getPreferredDocumentConversionServerRetry'),
+          verify=False).text,
+      "123")
+
+
+class TestCloudoooDefaultParameter(ZopeSkinsMixin, ERP5InstanceTestCase):
+  """Test default ERP5 cloudooo parameters
+  """
+  __partition_reference__ = 'cd'
+
+  def test_cloudooo_url_list_preference(self):
+    self.assertIn(
+      requests.get(
+          self._getAuthenticatedZopeUrl(
+            'portal_preferences/getPreferredDocumentConversionServerUrlList'),
+          verify=False).text,
+      [
+        "['https://cloudooo1.erp5.net/', 'https://cloudooo.erp5.net/']",
+        "['https://cloudooo.erp5.net/', 'https://cloudooo1.erp5.net/']",
+      ])
+
+  @unittest.expectedFailure # default value of "retry" does not match schema
+  def test_cloudooo_retry_count_preference(self):
+    self.assertEqual(
+      requests.get(
+          self._getAuthenticatedZopeUrl(
+            'portal_preferences/getPreferredDocumentConversionServerRetry'),
+          verify=False).text,
+      "2")
+
+
+class TestNEO(ZopeSkinsMixin, CrontabMixin, ERP5InstanceTestCase):
+  """Tests specific to neo storage
+  """
+  __partition_reference__ = 'n'
+  __test_matrix__ = matrix((neo,))
+
+  def _getCrontabCommand(self, crontab_name: str) -> str:
+    """Read a crontab and return the command that is executed.
+
+    overloaded to use crontab from neo partition
+    """
+    with open(
+        os.path.join(
+            self.getComputerPartitionPath('neo-0'),
+            'etc',
+            'cron.d',
+            crontab_name,
+        )) as f:
+      crontab_spec, = f.readlines()
+    self.assertNotEqual(crontab_spec[0], '@', crontab_spec)
+    return crontab_spec.split(None, 5)[-1]
+
+  def test_log_rotation(self):
+    # first run to create state files
+    self._executeCrontabAtDate('logrotate', '2000-01-01')
+
+    def check_sqlite_log(path):
+      with self.subTest(path), contextlib.closing(sqlite3.connect(path)) as con:
+        con.execute('select * from log')
+
+    logfiles = ('neoadmin.log', 'neomaster.log', 'neostorage-0.log')
+    for f in logfiles:
+      check_sqlite_log(
+        os.path.join(
+          self.getComputerPartitionPath('neo-0'),
+          'var',
+          'log',
+          f))
+
+    self._executeCrontabAtDate('logrotate', '2050-01-01')
+
+    for f in logfiles:
+      check_sqlite_log(
+        os.path.join(
+          self.getComputerPartitionPath('neo-0'),
+          'srv',
+          'backup',
+          'logrotate',
+          f'{f}-20500101'))
+
+    self._executeCrontabAtDate('logrotate', '2050-01-02')
+    requests.get(self._getAuthenticatedZopeUrl('/'), verify=False).raise_for_status()
+
+    for f in logfiles:
+      check_sqlite_log(
+        os.path.join(
+          self.getComputerPartitionPath('neo-0'),
+          'var',
+          'log',
+          f))
+
+class TestPassword(ERP5InstanceTestCase, TestPublishedURLIsReachableMixin):
+  __partition_reference__ = 'p'
+
+  def test_no_plain_text_password_in_files(self):
+    inituser_password = self.getRootPartitionConnectionParameterDict()[
+      'inituser-password'].encode()
+    self.assertFalse(
+      [f for f in pathlib.Path(self.slap._instance_root).glob('**/*')
+        if f.is_file() and inituser_password in f.read_bytes()])
+    # the hashed password is present in some files
+    inituser_password_hashed = self.getRootPartitionConnectionParameterDict()[
+      'inituser-password-hashed'].encode()
+    self.assertTrue(
+      [f for f in pathlib.Path(self.slap._instance_root).glob('**/*')
+        if f.is_file() and inituser_password_hashed in f.read_bytes()])
+
+
+class TestWithMaxRlimitNofileParameter(ERP5InstanceTestCase, TestPublishedURLIsReachableMixin):
+  """Test setting the with-max-rlimit-nofile parameter sets the open fd soft limit to the hard limit.
+  """
+  __partition_reference__ = 'nf'
+
+  @classmethod
+  def getInstanceParameterDict(cls):
+    return {'_': json.dumps({'with-max-rlimit-nofile': True})}
+
+  def test_with_max_rlimit_nofile(self):
+    with self.slap.instance_supervisor_rpc as supervisor:
+      all_process_info = supervisor.getAllProcessInfo()
+    _, current_hard_limit = resource.getrlimit(resource.RLIMIT_NOFILE)
+    process_info, = (p for p in all_process_info if p['name'].startswith('zope-'))
+    self.assertEqual(
+      resource.prlimit(process_info['pid'], resource.RLIMIT_NOFILE),
+      (current_hard_limit, current_hard_limit))
+
+
+class TestUnsetWithMaxRlimitNofileParameter(ERP5InstanceTestCase, TestPublishedURLIsReachableMixin):
+  """Test not setting the with-max-rlimit-nofile parameter doesn't change the soft limit of erp5
+  """
+  __partition_reference__ = 'nnf'
+
+  def test_unset_with_max_rlimit_nofile(self) -> None:
+    with self.slap.instance_supervisor_rpc as supervisor:
+      all_process_info = supervisor.getAllProcessInfo()
+    limit = resource.getrlimit(resource.RLIMIT_NOFILE)
+    process_info, = (p for p in all_process_info if p['name'].startswith('zope-'))
+    self.assertEqual(
+      resource.prlimit(process_info['pid'], resource.RLIMIT_NOFILE), limit)
+
+
+class TestFrontend(ERP5InstanceTestCase):
+  __partition_reference__ = 'f'
+
+  @classmethod
+  def getInstanceParameterDict(cls):
+    return {
+      '_':
+      json.dumps(
+        {
+          "zope-partition-dict": {
+            "backoffice": {
+              "family": "default",
+            },
+            "web": {
+              "family": "web",
+              "port-base": 2300,
+            },
+            "activities": {
+              # this family will not have frontend
+              "family": "activities",
+              "port-base": 2400,
+
+            },
+          },
+          "frontend": {
+            "backoffice": {
+              "zope-family": "default",
+            },
+            "website": {
+              "zope-family": "web",
+              "internal-path": "/%(site-id)s/web_site_module/my_website",
+              "instance-parameters": {
+                # some extra frontend parameters
+                "enable_cache": "true",
+              }
+            }
+          },
+          "sla-dict": {
+            "computer_guid=COMP-1234": ["frontend-backoffice"]
+          }
+        })
+    }
+
+  def test_frontend_url_published(self):
+    param_dict = self.getRootPartitionConnectionParameterDict()
+    requests.get(
+      param_dict['url-frontend-backoffice'],
+      verify=False,
+      allow_redirects=False,
+    )
+    requests.get(
+      param_dict['url-frontend-website'],
+      verify=False,
+      allow_redirects=False,
+    )
+
+  def test_request_parameters(self):
+    param_dict = self.getRootPartitionConnectionParameterDict()
+    balancer_param_dict = json.loads(
+      self.getComputerPartition('balancer').getConnectionParameter('_'))
+
+    with open(os.path.join(self.computer_partition_root_path,
+                           '.installed-switch-softwaretype.cfg')) as f:
+      installed = zc.buildout.configparser.parse(f, 'installed')
+    self.assertEqual(
+      installed['request-frontend-backoffice']['config-type'], '')
+    self.assertEqual(
+      installed['request-frontend-backoffice']['shared'], 'true')
+    self.assertEqual(
+      installed['request-frontend-backoffice']['config-url'],
+      balancer_param_dict['url-backend-backoffice'])
+    self.assertNotIn('config-path', installed['request-frontend-backoffice'])
+    self.assertEqual(
+      installed['request-frontend-backoffice']['sla-computer_guid'],
+      'COMP-1234')
+    self.assertEqual(
+      installed['request-frontend-backoffice']['software-url'],
+      'http://git.erp5.org/gitweb/slapos.git/blob_plain/HEAD:/software/apache-frontend/software.cfg'
+    )
+    self.assertEqual(
+      installed['request-frontend-backoffice']['connection-secure_access'],
+      param_dict['url-frontend-backoffice'])
+
+    self.assertEqual(
+      installed['request-frontend-website']['config-type'], '')
+    # no SLA by default
+    self.assertFalse([k for k in installed['request-frontend-website'] if k.startswith('sla-')])
+    # instance parameters are propagated
+    self.assertEqual(
+      installed['request-frontend-website']['config-enable_cache'], 'true')
+    self.assertEqual(
+      installed['request-frontend-website']['config-url'],
+      balancer_param_dict['url-backend-website'])
+    self.assertNotIn('config-path', installed['request-frontend-website'])
+    self.assertEqual(
+      installed['request-frontend-website']['connection-secure_access'],
+      param_dict['url-frontend-website'])
+
+    # no frontend was requested for activities family
+    self.assertNotIn('request-frontend-activities', installed)
+    self.assertNotIn('url-frontend-activities', param_dict)
+    self.assertNotIn('url-backend-activities', balancer_param_dict)
+
+  def test_path_virtualhost(self):
+    balancer_param_dict = json.loads(
+      self.getComputerPartition('balancer').getConnectionParameter('_'))
+    found_line = False
+    retries = 10
+    while retries:
+      requests.get(balancer_param_dict['url-backend-website'], verify=False)
+      for logfile in glob.glob(os.path.join(self.getComputerPartitionPath('zope-web'), 'var/log/*Z2.log')):
+        with open(logfile) as f:
+          for line in f:
+            if 'GET /VirtualHost' in line:
+              found_line = True
+              break
+      if found_line:
+        break
+      time.sleep(1)
+      retries = retries - 1
+    self.assertTrue(found_line)
+
+    percent_encoded_netloc = six.moves.urllib.parse.quote(
+      six.moves.urllib.parse.urlparse(
+        balancer_param_dict['url-backend-website']).netloc)
+    self.assertIn(
+      f'/VirtualHostBase/https/{percent_encoded_netloc}/erp5/web_site_module/my_website/VirtualHostRoot/ HTTP', line)
+
+
+class TestDefaultFrontendWithZopePartitionDict(ERP5InstanceTestCase):
+  """Default frontend also is requested when only one zope family
+  is defined, but on multiple partitions
+  """
+  __partition_reference__ = 'fzpd'
+
+  @classmethod
+  def getInstanceParameterDict(cls):
+    return {
+      '_':
+      json.dumps(
+        {
+          "zope-partition-dict": {
+            "backoffice-0": {
+              "family": "backoffice",
+            },
+            "backoffice-1": {
+              "family": "backoffice",
+            }
+          }
+        }
+      )
+    }
+
+  def test_frontend_requested(self):
+    param_dict = self.getRootPartitionConnectionParameterDict()
+    balancer_param_dict = json.loads(
+      self.getComputerPartition('balancer').getConnectionParameter('_'))
+
+    with open(os.path.join(self.computer_partition_root_path,
+                           '.installed-switch-softwaretype.cfg')) as f:
+      installed = zc.buildout.configparser.parse(f, 'installed')
+    self.assertEqual(
+      installed['request-frontend-default']['config-url'],
+      balancer_param_dict['url-backend-default'])
+
+    requests.get(
+      param_dict['url-frontend-default'],
+      verify=False,
+      allow_redirects=False,
+    )
+>>>>>>> 6e7358084 (stack/erp5: implement Zope's rewrite rules in ERP5 balancer partition)

software/slapos-master/buildout.hash.cfg

@@ -14,7 +14,7 @@
 # not need these here).
 [template-erp5]
 filename = instance-erp5.cfg.in
-md5sum = ce9c231ec47eb8f528345add21cb7822
+md5sum = 60b68211c0a0d6bb87b5c45c5f36b11e
 
 [template-balancer]
 filename = instance-balancer.cfg.in

software/slapos-master/instance-erp5.cfg.in

@@ -430,6 +430,7 @@ return =
 {% endfor -%}
 {% do monitor_base_url_dict.__setitem__('request-balancer', '${' ~ 'request-balancer' ~ ':connection-monitor-base-url}') -%}
 config-zope-family-dict = {{ dumps(zope_family_parameter_dict) }}
+config-frontend-parameter-dict = {{ dumps({}) }}
 config-tcpv4-port = {{ dumps(balancer_dict.get('tcpv4-port', 2150)) }}
 {% for zope_section_id, name in zope_address_list_id_dict.items() -%}
 config-{{ name }} = {{ ' ${' ~ zope_section_id ~ ':connection-zope-address-list}' }}

stack/erp5/buildout.hash.cfg

@@ -74,7 +74,7 @@ md5sum = 33cf43b51d245f34fe7442eaac570f3f
 
 [template-erp5]
 filename = instance-erp5.cfg.in
-md5sum = 0006c02426e19d3fed2d4c96c0fd1691
+md5sum = 92b96409365304b68406cd49fde66de0
 
 [template-zeo]
 filename = instance-zeo.cfg.in
@@ -90,11 +90,11 @@ md5sum = c3e3f8cd985407931b705d15bdedc8d9
 
 [template-balancer]
 filename = instance-balancer.cfg.in
-md5sum = 2901ccefe10dd6ebe944c5fc69fb1ff7
+md5sum = 261074cf5d8e67eacdf0390ba9e1740e
 
 [template-haproxy-cfg]
 filename = haproxy.cfg.in
-md5sum = 9bb466897f845bef1c16c980ab10a32a
+md5sum = 2e342965d496392bc9449f53e008b8d6
 
 [template-rsyslogd-cfg]
 filename = rsyslogd.cfg.in

stack/erp5/haproxy.cfg.in

@@ -18,11 +18,11 @@
  #       "stats-socket": "<file_path>",
  #
  #       #  IPv4 to listen on
- #       #  All backends from `backend-dict` will listen on this IP.
+ #       #  All frontends from `frontend-dict` will listen on this IP.
  #       "ipv4": "0.0.0.0",
  #
  #       #  IPv6 to listen on
- #       #  All backends from `backend-dict` will listen on this IP.
+ #       #  All frontends from `frontend-dict` will listen on this IP.
  #       "ipv6": "::1",
  #
  #       #  Certificate and key in PEM format. All ports will serve TLS using
@@ -41,34 +41,59 @@
  #       #  Path to use for HTTP health check on backends from `backend-dict`.
  #       "server-check-path": "/",
  #
+ #       #  The mapping of frontend, keyed by frontend name
+ #       "frontend-dict": {
+ #         "frontend-default": {
+ #            "port": 8080,
+ #            "client-cert-required": False,
+ #            "backend-name": "family-default",
+ #            "request-path-prepend": "/erp5",
+ #         }
+ #         "legacy-frontend-family-secure": {
+ #            "port": 8000,
+ #            "client-cert-required": False,
+ #            "backend-name": "family-secure",
+ #            "request-path-prepend": None, # None means do not rewrite the request path
+ #         }
+ #         "legacy-frontend-family-default": {
+ #            "port": 8002,
+ #            "client-cert-required": False,
+ #            "backend-name": "family-default",
+ #            "request-path-prepend": None, # None means do not rewrite the request path
+ #         }
+ #       }
  #       #  The mapping of backends, keyed by family name
  #       "backend-dict": {
  #           "family-secure": {
- #            (  8000, # port int
- #               True, # ssl_required bool
- #               None,  # timeout (in seconds) int | None
- #               [  # backends
+ #               "timeout": None,  # in seconds
+ #               "backend-list": [
+ #                   [
  #                       '10.0.0.10:8001', # netloc str
  #                        1, # max_connection_count int
  #                        False, # is_web_dav bool
- #               ],
- #            ),
+ #                   ]
+ #               ]
  #           },
  #           "family-default": {
- #            (  8002, # port int
- #               False, # ssl_required bool
- #               None,  # timeout (in seconds) int | None
- #               [  # backends
+ #               "timeout": None,  # in seconds
+ #               "backend-list": [
+ #                   [
  #                      '10.0.0.10:8003', # netloc str
  #                       1, # max_connection_count int
  #                       False, # is_web_dav bool
  #                   ],
- #            ),
+ #                   [
+ #                      '10.0.0.10:8004', # netloc str
+ #                       1, # max_connection_count int
+ #                       False, # is_web_dav bool
+ #                   ],
+ #               ]
  #          },
  #
  #       # The mapping of zope paths.
- #       # This is a Zope specific feature.
- #       # `enable_authentication` has same meaning as for `backend-list`.
+ #       # This is a Zope specific feature used only to provide https while running
+ #       # ERP5 "unit test" suite.
+ #       # `enable_authentication` has same meaning as for `backend-dict`.
  #       "zope-virtualhost-monster-backend-dict": {
  #          # {(ip, port): ( enable_authentication, {frontend_path: ( internal_url ) }, ) }
  #          ('[::1]', 8004): (
@@ -81,15 +106,20 @@
  #     }
  #
  #  This sample of `parameter_dict` will make haproxy listening to :
- #  From to `backend-list`:
- #  For "family-secure":
+ #  For "frontend-default":
+ #   - 0.0.0.0:8080 redirecting internaly to http://10.0.0.10:8003 or http://10.0.0.10:8004
+ #   - [::1]:8080 redirecting internaly to http://10.0.0.10:8003 or http://10.0.0.10:8004
+ #  accepting requests from any client and rewriting the path to add a Zope rewrite rule
+ #  so that the a request on https://0.0.0.0:8080/path is rewritten to serve a Zope object at
+ #  path /erp5/path , visible as /path.
+ #  For "legacy-frontend-family-secure":
  #   - 0.0.0.0:8000 redirecting internaly to http://10.0.0.10:8001 and
  #   - [::1]:8000 redirecting internaly to http://10.0.0.10:8001
  #  only accepting requests from clients providing a verified TLS certificate
  #  emitted by a CA from `ca-cert` and not revoked in `crl`.
- #  For "family-default":
- #   - 0.0.0.0:8002 redirecting internaly to http://10.0.0.10:8003
- #   - [::1]:8002 redirecting internaly to http://10.0.0.10:8003
+ #  For "legacy-frontend-family-default":
+ #   - 0.0.0.0:8002 redirecting internaly to http://10.0.0.10:8003 or http://10.0.0.10:8004
+ #   - [::1]:8002 redirecting internaly to http://10.0.0.10:8003 or http://10.0.0.10:8004
  #  accepting requests from any client.
  #
  #  For both families, X-Forwarded-For header will be stripped unless
@@ -102,7 +132,7 @@
  #   with some VirtualHostMonster rewrite rules so zope writes URLs with
  #  [::1]:8004 as server name.
  #  For more details, refer to
- #  https://docs.zope.org/zope2/zope2book/VirtualHosting.html#using-virtualhostroot-and-virtualhostbase-together
+ #  https://zope.readthedocs.io/en/latest/zopebook/VirtualHosting.html#using-virtualhostroot-and-virtualhostbase-together
 -#}
 
 {% set server_check_path = parameter_dict['server-check-path'] -%}
@@ -148,26 +178,17 @@ defaults
 {% set family_path_routing_dict = parameter_dict['family-path-routing-dict'] %}
 {% set path_routing_list = parameter_dict['path-routing-list'] %}
 
-{% for name, (port, certificate_authentication, timeout, backend_list) in sorted(six.iteritems(parameter_dict['backend-dict'])) -%}
-listen family_{{ name }}
+
+{% for name, frontend in sorted(six.iteritems(parameter_dict['frontend-dict'])) %}
+listen {{ name }}
+
 {%-  if parameter_dict.get('ca-cert') -%}
-{%-    set ssl_auth = ' ca-file ' ~ parameter_dict['ca-cert'] ~ ' verify' ~ ( ' required' if certificate_authentication else ' optional' ) ~ ' crl-file ' ~ parameter_dict['crl'] %}
+{%-    set ssl_auth = ' ca-file ' ~ parameter_dict['ca-cert'] ~ ' verify' ~ ( ' required' if frontend['client-cert-required'] else ' optional' ) ~ ' crl-file ' ~ parameter_dict['crl'] %}
 {%-  else %}
 {%-    set ssl_auth = '' %}
 {%-  endif %}
-  bind {{ parameter_dict['ipv4'] }}:{{ port }} {{ bind_ssl_crt }} {{ ssl_auth }}
-  bind {{ parameter_dict['ipv6'] }}:{{ port }} {{ bind_ssl_crt }} {{ ssl_auth }}
-  cookie SERVERID rewrite
-  http-request set-header X-Balancer-Current-Cookie SERVERID
-
-{% if timeout %}
-  {#
-    Apply a slightly longer timeout than the zope timeout so that clients can see the
-    TimeoutReachedError from zope, that is a bit more informative than the 504 error
-    page from haproxy.
-  #}
-  timeout server {{ timeout + 3 }}s
-{%-  endif %}
+  bind {{ parameter_dict['ipv4'] }}:{{ frontend['port'] }} {{ bind_ssl_crt }} {{ ssl_auth }}
+  bind {{ parameter_dict['ipv6'] }}:{{ frontend['port'] }} {{ bind_ssl_crt }} {{ ssl_auth }}
 
   # remove X-Forwarded-For unless client presented a verified certificate
   acl client_cert_verified ssl_c_used ssl_c_verify 0
@@ -176,18 +197,43 @@ listen family_{{ name }}
   http-request del-header Remote-User
   http-request set-header Remote-User %{+Q}[ssl_c_s_dn(cn)] if client_cert_verified
 
+  # reject invalid host header before using it in path
+  http-request deny deny_status 400 if { req.hdr(host) -m sub / }
+
   # logs
   capture request header Referer len 512
   capture request header User-Agent len 512
   log-format "%{+Q}o %{-Q}ci - - [%trg] %r %ST %B %{+Q}[capture.req.hdr(0)] %{+Q}[capture.req.hdr(1)] %Ta"
 
-{%   for outer_prefix, inner_prefix in family_path_routing_dict.get(name, []) + path_routing_list %}
+{% if frontend['request-path-prepend'] is not none %}
+  http-request replace-path ^/(.*) /VirtualHostBase/https/%[req.hdr(Host)]{{ frontend['request-path-prepend'] }}/VirtualHostRoot/\1
+{% endif %}
+
+{%   for outer_prefix, inner_prefix in family_path_routing_dict.get(frontend['backend-name'], []) + path_routing_list %}
   {%   set outer_prefix = outer_prefix.strip('/') -%}
   http-request replace-path ^(/+VirtualHostBase/+[^/]+/+[^/]+)/+VirtualHostRoot/+{% if outer_prefix %}{{ outer_prefix }}($|/.*){% else %}(.*){% endif %} \1/{{ inner_prefix.strip('/') }}/VirtualHostRoot/{% if outer_prefix %}_vh_{{ outer_prefix.replace('/', '/_vh_') }}{% endif %}\2
 {%  endfor %}
 
+  use_backend {{ frontend['backend-name'] }}
+{% endfor %}
+
+
+{% for name, backend in sorted(six.iteritems(parameter_dict['backend-dict'])) %}
+backend {{ name }}
+  cookie SERVERID rewrite
+  http-request set-header X-Balancer-Current-Cookie SERVERID
+
+{% if backend['timeout'] %}
+  {#
+    Apply a slightly longer timeout than the zope timeout so that clients can see the
+    TimeoutReachedError from zope, that is a bit more informative than the 504 error
+    page from haproxy.
+  #}
+  timeout server {{ backend['timeout'] + 3 }}s
+{%-  endif %}
+
 {%   set has_webdav = [] -%}
-{%   for address, connection_count, webdav in backend_list -%}
+{%   for address, connection_count, webdav in backend['backend-list'] -%}
 {%     if webdav %}{% do has_webdav.append(None) %}{% endif -%}
 {%     set server_name = name ~ '-' ~ loop.index0 %}
   server {{ server_name }} {{ address }} cookie {{ server_name }} check inter 3s rise 1 fall 2 maxqueue 5 maxconn {{ connection_count }}

stack/erp5/instance-balancer.cfg.in

@@ -165,7 +165,8 @@ init =
     port_dict[name] = port
     return port
 
-  haproxy_dict = {}
+  backend_dict = {}
+  frontend_dict = {}
   zope_virtualhost_monster_backend_dict = {}
   for family_name, parameter_id_list in sorted(
       six.iteritems(slapparameter_dict['zope-family-dict'])):
@@ -198,17 +199,33 @@ init =
     # a port for monitoring promise (which port is not important, the promise checks
     # that haproxy is healthy enough to listen on a port)
     options['haproxy-promise-port'] = legacy_port
-    haproxy_dict[family_name] = (
-      legacy_port,
-      ssl_authentication,
-      slapparameter_dict['timeout-dict'][family_name],
-      zope_family_address_list,
-    )
+    frontend_dict['legacy-frontend-' + family_name] = {
+      'port': legacy_port,
+      'client-cert-required': ssl_authentication,
+      'backend-name': family_name,
+      'request-path-prepend': None,
+    }
+    backend_dict[family_name] = {
+      'timeout': slapparameter_dict['timeout-dict'][family_name],
+      'backend-list': zope_family_address_list,
+    }
     external_scheme = 'webdavs' if any(a[2] for a in zope_family_address_list) else 'https'
     self.buildout['publish'][family_name] = "{external_scheme}://{ipv4}:{legacy_port}".format(**locals())
     self.buildout['publish'][family_name + "-v6"] = "{external_scheme}://[{ipv6}]:{legacy_port}".format(**locals())
 
-  options['backend-dict'] = haproxy_dict
+  for frontend_name, frontend in six.iteritems(slapparameter_dict['frontend-parameter-dict']):
+    frontend_port = get_port('frontend-' + frontend_name)
+    family_name = frontend['zope-family']
+    frontend_dict['frontend-' + frontend_name] = {
+      'port': frontend_port,
+      'client-cert-required': slapparameter_dict['ssl-authentication-dict'][family_name],
+      'backend-name': family_name,
+      'request-path-prepend': frontend['internal-path'],
+    }
+    self.buildout['publish']['url-backend-' + frontend_name] = "https://[{ipv6}]:{frontend_port}".format(**locals())
+
+  options['backend-dict'] = backend_dict
+  options['frontend-dict'] = frontend_dict
   options['zope-virtualhost-monster-backend-dict'] = zope_virtualhost_monster_backend_dict
 
   if port_dict != previous_port_dict:

stack/erp5/instance-erp5.cfg.in

@@ -358,13 +358,13 @@ return =
 {%   set request_frontend_name = 'request-frontend-' ~ frontend_name -%}
 {%   set frontend_software_url = frontend_parameters.get('software-url', 'http://git.erp5.org/gitweb/slapos.git/blob_plain/HEAD:/software/apache-frontend/software.cfg') -%}
 {%   set frontend_software_type = frontend_parameters.get('software-type', '') -%}
+{%   do frontend_parameters.__setitem__('internal-path', frontend_parameters.get('internal-path', '/%(site-id)s') % {'site-id': site_id}) %}
 {%   set frontend_instance_parameters = frontend_parameters.get('instance-parameters', {}) -%}
-{%   if frontend_instance_parameters.setdefault('type', 'zope') == 'zope' -%}
+{%   if frontend_instance_parameters.setdefault('type', '') == '' -%}
 {%     do frontend_instance_parameters.setdefault('authenticate-to-backend', 'true') -%}
 {%     set zope_family_name = frontend_parameters['zope-family'] -%}
 {%     do assert(zope_family_name in zope_family_dict, 'Unknown family %s for frontend %s' % (zope_family_name, frontend_name)) -%}
-{%     do frontend_instance_parameters.setdefault('url', '${request-balancer:connection-' ~ zope_family_name ~ '-v6}') -%}
-{%     do frontend_instance_parameters.setdefault('path', frontend_parameters.get('internal-path', '/%(site-id)s') % {'site-id': site_id}) -%}
+{%     do frontend_instance_parameters.setdefault('url', '${request-balancer:connection-url-backend-' ~ frontend_name ~ '}') -%}
 {%   endif %}
 [{{ request_frontend_name }}]
 <= request-frontend-base
@@ -416,6 +416,9 @@ config-allow-redirects = 0
 {%     do balancer_ret_dict.__setitem__(family + '-test-runner-url-list', False) -%}
 {%   endif -%}
 {% endfor -%}
+{% for frontend_name in frontend_parameter_dict -%}
+{%   do balancer_ret_dict.__setitem__('url-backend-' ~ frontend_name, False) -%}
+{% endfor -%}
 {% set balancer_key_config_dict = {
   'monitor-passwd': 'monitor-htpasswd:passwd',
 } -%}
@@ -437,6 +440,7 @@ config-allow-redirects = 0
   config_key='balancer',
   config={
     'zope-family-dict': zope_family_parameter_dict,
+    'frontend-parameter-dict': frontend_parameter_dict,
     'ssl-authentication-dict': ssl_authentication_dict,
     'timeout-dict': balancer_timeout_dict,
     'apachedex-promise-threshold': monitor_dict.get('apachedex-promise-threshold', 70),