stack/erp5: use slapos.recipe.build to manage haproxy parameters

and save the already allocated ports in a state file, so that requesting
new families does not change already allocated ports.

software/erp5/test/test/test_erp5.py

@@ -305,6 +305,13 @@ class TestBalancerPorts(ERP5InstanceTestCase):
           param_dict[f'family-{family_name}'])
       self.checkValidHTTPSURL(
           param_dict[f'family-{family_name}-v6'])
+    # ports are allocated in alphabetical order and are "stable", ie. is not supposed
+    # to change after updating software release, because there is typically a rapid-cdn
+    # frontend pointing to this port.
+    self.assertEqual(urllib.parse.urlparse(param_dict['family-family1']).port, 2152)
+    self.assertEqual(urllib.parse.urlparse(param_dict['family-family1-v6']).port, 2152)
+    self.assertEqual(urllib.parse.urlparse(param_dict['family-family2']).port, 2154)
+    self.assertEqual(urllib.parse.urlparse(param_dict['family-family2-v6']).port, 2154)
 
   def test_published_test_runner_url(self):
     # each family's also a list of test test runner URLs, by default 3 per family
@@ -313,6 +320,7 @@ class TestBalancerPorts(ERP5InstanceTestCase):
       family_test_runner_url_list = param_dict[
           f'{family_name}-test-runner-url-list']
       self.assertEqual(3, len(family_test_runner_url_list))
+      self.assertEqual(3, len(set(family_test_runner_url_list)))
       for url in family_test_runner_url_list:
         self.checkValidHTTPSURL(url)
 
@@ -353,6 +361,61 @@ class TestBalancerPorts(ERP5InstanceTestCase):
       ])
 
 
+class TestBalancerPortsStable(ERP5InstanceTestCase):
+  """Instantiate with two one families and a frontend, then
+  re-request with one more family and one more frontend, the ports
+  should not change
+  """
+  __partition_reference__ = 'ap'
+
+  @classmethod
+  def getInstanceParameterDict(cls):
+    return {
+      '_':
+      json.dumps(
+        {
+          "frontend": {
+            "zzz": {
+              "zope-family": "zzz"
+            }
+          },
+          "zope-partition-dict": {
+            "zzz": {
+              "instance-count": 1,
+              "family": "zzz"
+            },
+          },
+        })
+    }
+
+  def test_same_balancer_ports_when_adding_zopes_or_frontends(self):
+    param_dict_before = self.getRootPartitionConnectionParameterDict()
+
+    # re-request with one more frontend and one more backend, that are before
+    # the existing ones when sorting alphabetically
+    instance_parameter_dict = json.loads(self.getInstanceParameterDict()['_'])
+    instance_parameter_dict['frontend']['aaa'] = {"zope-family": "aaa"}
+    instance_parameter_dict['zope-partition-dict']['aaa'] = {
+      "instance-count": 2,
+      "family": "aaa"
+    }
+    def rerequest():
+      return self.slap.request(
+        software_release=self.getSoftwareURL(),
+        software_type=self.getInstanceSoftwareType(),
+        partition_reference=self.default_partition_reference,
+        partition_parameter_kw={'_': json.dumps(instance_parameter_dict)},
+        state='started')
+
+    rerequest()
+    self.slap.waitForInstance(max_retry=10)
+    param_dict_after = json.loads(rerequest().getConnectionParameterDict()['_'])
+    self.assertEqual(param_dict_before['family-zzz-v6'], param_dict_after['family-zzz-v6'])
+    self.assertEqual(param_dict_before['url-frontend-zzz'], param_dict_after['url-frontend-zzz'])
+    self.assertNotEqual(param_dict_before['family-zzz-v6'], param_dict_after['family-aaa-v6'])
+    self.assertNotEqual(param_dict_before['url-frontend-zzz'], param_dict_after['url-frontend-aaa'])
+
+
 class TestSeleniumTestRunner(ERP5InstanceTestCase, TestPublishedURLIsReachableMixin):
   """Test ERP5 can be instantiated with selenium server for test runner.
   """

stack/erp5/buildout.hash.cfg

@@ -70,7 +70,7 @@ md5sum = b95084ae9eed95a68eada45e28ef0c04
 
 [template]
 filename = instance.cfg.in
-md5sum = 5e0e9565227fe190c420a7bbcd0f7b93
+md5sum = 55463b0abdbe0118ef1c27e6b71c3324
 
 [template-erp5]
 filename = instance-erp5.cfg.in
@@ -90,11 +90,11 @@ md5sum = 6178ba7b42848f9e2412ab898a7b026c
 
 [template-balancer]
 filename = instance-balancer.cfg.in
-md5sum = dbd17fbde7a1a3b1a12e3ea1db25baa1
+md5sum = 42cb68905f92e7df38cc5c64b94be3de
 
 [template-haproxy-cfg]
 filename = haproxy.cfg.in
-md5sum = 85a8c0dadf7b648ef9748b6199dcfeb6
+md5sum = 9988a14c4108e3bce3f871e34673cdd5
 
 [template-rsyslogd-cfg]
 filename = rsyslogd.cfg.in

stack/erp5/haproxy.cfg.in

-
 {# This file configures haproxy to redirect requests from ports to specific urls.
  # It provides TLS support for server and optionnaly for client.
  #
@@ -46,7 +45,6 @@
  #       "backend-dict": {
  #          "family-secure": {
  #            (  8000, # port int
- #              'https', # proto str
  #               True, # ssl_required bool
  #               None,  # timeout (in seconds) int | None
  #               [  # backends
@@ -58,7 +56,6 @@
  #          },
  #          "family-default": {
  #            (  8002, # port int
- #              'https', # proto str
  #               False, # ssl_required bool
  #               None,  # timeout (in seconds) int | None
  #               [  # backends
@@ -151,7 +148,7 @@ defaults
 {% set family_path_routing_dict = parameter_dict['family-path-routing-dict'] %}
 {% set path_routing_list = parameter_dict['path-routing-list'] %}
 
-{% for name, (port, _, certificate_authentication, timeout, backend_list) in sorted(six.iteritems(parameter_dict['backend-dict'])) -%}
+{% for name, (port, certificate_authentication, timeout, backend_list) in sorted(six.iteritems(parameter_dict['backend-dict'])) -%}
 listen family_{{ name }}
 {%-  if parameter_dict.get('ca-cert') -%}
 {%-    set ssl_auth = ' ca-file ' ~ parameter_dict['ca-cert'] ~ ' verify' ~ ( ' required' if certificate_authentication else ' optional crt-ignore-err all' ) ~ ' crl-file ' ~ parameter_dict['crl'] %}

stack/erp5/instance-balancer.cfg.in

@@ -208,70 +208,8 @@ command = ${caucase-updater-housekeeper:output}
 update-command = ${:command}
 {% endif -%}
 
-{% set haproxy_dict = {} -%}
-{% set zope_virtualhost_monster_backend_dict = {} %}
-{% set test_runner_url_dict = {} %} {# family_name => list of URLs #}
-{% set next_port = functools.partial(next, itertools.count(slapparameter_dict['tcpv4-port'])) -%}
-{% for family_name, parameter_id_list in sorted(
-  six.iteritems(slapparameter_dict['zope-family-dict'])) -%}
-{%   set zope_family_address_list = [] -%}
-{%   set ssl_authentication = slapparameter_dict['ssl-authentication-dict'][family_name] -%}
-{%   set has_webdav = [] -%}
-{%   for parameter_id in parameter_id_list -%}
-{%     set zope_address_list = slapparameter_dict[parameter_id] -%}
-{%     for zope_address, maxconn, webdav in zope_address_list -%}
-{%       if webdav -%}
-{%         do has_webdav.append(None) %}
-{%       endif -%}
-{%       set zope_effective_address = zope_address -%}
-{%       do zope_family_address_list.append((zope_effective_address, maxconn, webdav)) -%}
-{%     endfor -%}
-
-{#     # Generate entries with rewrite rule for test runnners #}
-{%     set test_runner_address_list = slapparameter_dict.get(parameter_id ~ '-test-runner-address-list', []) %}
-{%     if test_runner_address_list -%}
-{%       set test_runner_backend_mapping = {} %}
-{%       set test_runner_balancer_url_list = [] %}
-{%       set test_runner_external_port = next_port() %}
-{%       for i, (test_runner_internal_ip, test_runner_internal_port) in enumerate(test_runner_address_list) %}
-{%         do test_runner_backend_mapping.__setitem__(
-                'unit_test_' ~ i,
-                'http://' ~ test_runner_internal_ip ~ ':' ~ test_runner_internal_port ) %}
-{%         do test_runner_balancer_url_list.append(
-                'https://' ~ ipv4 ~ ':' ~ test_runner_external_port ~ '/unit_test_' ~ i ~ '/' ) %}
-{%       endfor %}
-{%       do zope_virtualhost_monster_backend_dict.__setitem__(
-              (ipv4, test_runner_external_port),
-              ( ssl_authentication, test_runner_backend_mapping ) ) -%}
-{%       do test_runner_url_dict.__setitem__(family_name, test_runner_balancer_url_list) -%}
-{%     endif -%}
-{%   endfor -%}
-
-{# Make rendering fail artificially if any family has no known backend.
- # This is useful as haproxy's hot-reconfiguration mechanism is
- # supervisord-incompatible.
- # As jinja2 postpones KeyError until place-holder value is actually used,
- # do a no-op getitem.
--#}
-{%   do zope_family_address_list[0][0] -%}
-
-{#
- # We use to have haproxy then apache, now haproxy is playing apache's role
- # To keep port stable, we consume one port so that haproxy use the same port
- # that apache was using before.
--#}
-{%   set _ = next_port() -%}
-
-{%   set haproxy_port = next_port() -%}
-{%   if has_webdav -%}
-{%     set external_scheme = 'webdavs' -%}
-{%   else %}
-{%     set external_scheme = 'https' -%}
-{%   endif -%}
-{%   do haproxy_dict.__setitem__(family_name, (haproxy_port, external_scheme, slapparameter_dict['ssl-authentication-dict'][family_name], slapparameter_dict['timeout-dict'][family_name], zope_family_address_list)) -%}
-{% endfor -%}
-
 [haproxy-cfg-parameter-dict]
+recipe = slapos.recipe.build
 ipv4 = {{ ipv4 }}
 ipv6 = {{ ipv6 }}
 cert = ${haproxy-conf-ssl:certificate}
@@ -285,9 +223,90 @@ family-path-routing-dict = {{ dumps(slapparameter_dict['family-path-routing-dict
 pidfile = ${directory:run}/haproxy.pid
 log-socket = ${rsyslogd-cfg-parameter-dict:log-socket}
 server-check-path = {{ dumps(slapparameter_dict['haproxy-server-check-path']) }}
-backend-dict = {{ dumps(haproxy_dict) }}
-zope-virtualhost-monster-backend-dict = {{ dumps(zope_virtualhost_monster_backend_dict) }}
+slapparameter-dict = {{ dumps(slapparameter_dict) }}
+ports-state-file = ${buildout:directory}/.${:_buildout_section_name_}-ports.json
 
+init =
+  import functools
+  import itertools
+  import json
+  import os
+  import shutil
+  import six
+  from zc.buildout import UserError
+
+  slapparameter_dict = options['slapparameter-dict']
+  ipv4 = options['ipv4']
+  ipv6 = options['ipv6']
+
+  # read port state file
+  port_dict = {}
+  previous_port_dict = None
+  if os.path.exists(options['ports-state-file']):
+    with open(options['ports-state-file']) as f:
+      port_dict = json.load(f)
+      previous_port_dict = dict(port_dict)
+
+  _next_port = functools.partial(next, itertools.count(slapparameter_dict['tcpv4-port']))
+  def get_port(name):
+    if name in port_dict:
+      return port_dict[name]
+    port = _next_port()
+    while port in port_dict.values():
+      port = _next_port()
+    port_dict[name] = port
+    return port
+
+  haproxy_dict = {}
+  zope_virtualhost_monster_backend_dict = {}
+  for family_name, parameter_id_list in sorted(
+      six.iteritems(slapparameter_dict['zope-family-dict'])):
+    zope_family_address_list = []
+    ssl_authentication = slapparameter_dict['ssl-authentication-dict'][family_name]
+    for parameter_id in parameter_id_list:
+      zope_family_address_list.extend(slapparameter_dict[parameter_id])
+
+      # Generate entries with rewrite rule for test runnners
+      test_runner_address_list = slapparameter_dict.get(parameter_id + '-test-runner-address-list', [])
+      if test_runner_address_list:
+        test_runner_backend_mapping = {}
+        test_runner_balancer_url_list = []
+        for i, (test_runner_internal_ip, test_runner_internal_port) in enumerate(test_runner_address_list):
+          test_runner_backend_mapping['unit_test_%s' % i] = \
+            'http://%s:%s' % (test_runner_internal_ip, test_runner_internal_port)
+          test_runner_balancer_url_list.append(
+            'https://%s:%s/unit_test_%s/' % (ipv4, get_port('test-runner-' + family_name), i))
+        zope_virtualhost_monster_backend_dict[(ipv4, get_port('test-runner-' + family_name))] =\
+          ( ssl_authentication, test_runner_backend_mapping )
+        self.buildout['publish'][family_name + '-test-runner-url-list'] = test_runner_balancer_url_list
+
+    if not zope_family_address_list:
+      raise UserError('No zope defined for family %s (maybe not ready)' % family_name)
+
+    # consume a port for compatibility when were using apache + haproxy
+    get_port('apache-compatibility-' + family_name)
+
+    legacy_port = get_port('legacy-' + family_name)
+    # a port for monitoring promise (which port is not important, the promise checks
+    # that haproxy is healthy enough to listen on a port)
+    options['haproxy-promise-port'] = legacy_port
+    haproxy_dict[family_name] = (
+      legacy_port,
+      ssl_authentication,
+      slapparameter_dict['timeout-dict'][family_name],
+      zope_family_address_list,
+    )
+    external_scheme = 'webdavs' if any(a[2] for a in zope_family_address_list) else 'https'
+    self.buildout['publish'][family_name] = "{external_scheme}://{ipv4}:{legacy_port}".format(**locals())
+    self.buildout['publish'][family_name + "-v6"] = "{external_scheme}://[{ipv6}]:{legacy_port}".format(**locals())
+
+  options['backend-dict'] = haproxy_dict
+  options['zope-virtualhost-monster-backend-dict'] = zope_virtualhost_monster_backend_dict
+
+  if port_dict != previous_port_dict:
+    with open(options['ports-state-file'] + '.tmp', 'w') as f:
+      json.dump(port_dict, f, indent=True)
+    shutil.move(options['ports-state-file'] + '.tmp', options['ports-state-file'])
 
 [haproxy-cfg]
 < = jinja2-template-base
@@ -371,17 +390,11 @@ certificate = ${haproxy-conf-ssl-certificate-and-key-from-parameters:output}
 promise = check_socket_listening
 name = haproxy.py
 config-host = {{ ipv4 }}
-config-port = {{ next(six.itervalues(haproxy_dict))[0] }}
+config-port = ${haproxy-cfg-parameter-dict:haproxy-promise-port}
 
 [{{ section('publish') }}]
 recipe = slapos.cookbook:publish.serialised
-{% for family_name, (port, scheme, _, _, _) in haproxy_dict.items() -%}
-{{   family_name ~ '-v6' }} = {% if ipv6_set %}{{ scheme ~ '://[' ~ ipv6 ~ ']:' ~ port }}{% endif %}
-{{   family_name }} = {{ scheme ~ '://' ~ ipv4 ~ ':' ~ port }}
-{% endfor -%}
-{% for family_name, test_runner_url_list in test_runner_url_dict.items() -%}
-{{    family_name ~ '-test-runner-url-list' }} = {{ dumps(test_runner_url_list) }}
-{% endfor -%}
+# note: some values are pushed by haproxy-cfg-parameter-dict
 caucase-http-url = {{ caucase_url }}
 monitor-base-url = ${monitor-publish-parameters:monitor-base-url}
 
@@ -466,7 +479,7 @@ config-command = "{{ parameter_dict["check-computer-memory-binary"] }}" -db ${mo
 
 [monitor-instance-parameter]
 monitor-httpd-ipv6 = {{ (ipv6_set | list)[0] }}
-monitor-httpd-port = {{ next_port() }}
+monitor-httpd-port = 2197
 monitor-title = {{ slapparameter_dict['name'] }}
 password = {{ slapparameter_dict['monitor-passwd'] }}
 

stack/erp5/instance.cfg.in

@@ -91,8 +91,9 @@ url = {{ template_balancer }}
 filename = instance-balancer.cfg
 extra-context =
     section parameter_dict dynamic-template-balancer-parameters
-    import itertools itertools
     import hashlib hashlib
+# XXX: only used in software/slapos-master:
+    import itertools itertools
     import functools functools
 import-list =
     file caucase context:caucase-jinja2-library