erp5: start runUnitTest's zserver on the partition IP

We don't want runUnitTest is to listen on 127.0.0.1, as it is not
allowed for SlapOS instances, runUnitTest wrapper is extended to include
`--zserver` with a pre-assigned ip:port.

runTestSuite starts several test in parrallel (controlled by
`--node_quantity` argument, which is passed by erp5testnode), so we need
to make sure that we provide it with enough ip:port. For this, we
extended runTestSuite with a `--zserver_address_list` argument and we
generate a wrapper with a list of pre-assigned ip:ports. The length of
this list is also controlled by `mariadb.test-database-amount` instance
parameter (eventhough it's slightly different).

Because zelenium tests needs to access this zserver over a secure origin
(otherwise modern browser features such as service worker are not
available), add an https proxy in the apache from the balancer
partition.

runUnitTest and runTestSuite have been extended with resp.
`--zserver_frontend_url` and `--zserver_frontend_url_list` arguments and
the URLs published by the balancer paritions are set in the wrappers.

This introduces a circular depencency, balancer partition needs to know
the address of the testrunners and zope partitions needs to know the
URLs of the corresponding http proxies on the apache. This is is handled
by `slapos.recipe:publish-early`:

 1. request zope family with an empty `test-runner-apache-url-list`.
    zope is instanciated a first time.
    zope returns `test-runner-address-list` ( a list of (host, port)
    tuples )
 2. request balancer with `test-runner-address-list`
    balancer is instanciated.
    balancer returns `{{ family_name }}-test-runner-url-list` ( a list
    of apache URLs ), which are published in the root partition.
 3. zope family is re-requested with
    updated`test-runner-apache-url-list` information

instance-erp5.cfg.in template was also reorganised to move
`[publish-early]` next to `[publish]` at the bottom of the file.

Document the existing behavior that setting mariadb.test-database-amount
to 0 disable the creation of testrunner.

software/erp5/instance-mariadb-schema.json

@@ -38,7 +38,7 @@
       "type": "array"
     },
     "test-database-amount": {
-      "description": "The number of test databases to create, adding auto-generated entries to database-list",
+      "description": "The number of test databases to create, adding auto-generated entries to database-list. The same number of testrunners will be created in Zope partitions. Setting this to 0 disable the creation of test runners.",
       "default": 1,
       "minimum": 0,
       "type": "integer"

stack/erp5/buildout.hash.cfg

@@ -74,7 +74,7 @@ md5sum = d41d8cd98f00b204e9800998ecf8427e
 
 [template-erp5]
 filename = instance-erp5.cfg.in
-md5sum = 83e0f627633ec3e394dc27d06c627b3b
+md5sum = b326c11f013b459c3d1b49c5f1c68e8b
 
 [template-zeo]
 filename = instance-zeo.cfg.in
@@ -82,11 +82,11 @@ md5sum = 3e650915959ff31c9c13c84069bbcd35
 
 [template-zope]
 filename = instance-zope.cfg.in
-md5sum = e73678921067506e710ae11e41f0a9a8
+md5sum = 61899bbe151ce7d86ff1c08d280c5689
 
 [template-balancer]
 filename = instance-balancer.cfg.in
-md5sum = 89872075cd2f31d824369447aaf0532f
+md5sum = f3d38628911a221fdff544452afdb34c
 
 [template-haproxy-cfg]
 filename = haproxy.cfg.in

stack/erp5/instance-balancer.cfg.in

@@ -97,10 +97,13 @@ ipv4 = {{ ipv4 }}
 {% endif -%}
 {% set haproxy_dict = {} -%}
 {% set apache_dict = {} -%}
+{% set zope_virtualhost_monster_backend_dict = {} %}
+{% set test_runner_url_dict = {} %} {# family_name => list of apache URLs #}
 {% set next_port = itertools.count(slapparameter_dict['tcpv4-port']).next -%}
 {% for family_name, parameter_id_list in sorted(
   slapparameter_dict['zope-family-dict'].iteritems()) -%}
 {%   set zope_family_address_list = [] -%}
+{%   set ssl_authentication = slapparameter_dict['ssl-authentication-dict'].get(family_name, False) -%}
 {%   set has_webdav = [] -%}
 {%   for parameter_id in parameter_id_list -%}
 {%     set zope_address_list = slapparameter_dict[parameter_id] -%}
@@ -121,8 +124,27 @@ ipv6 = {{ zope_address.split(']:')[0][1:] }}
 {%         set zope_effective_address = zope_address -%}
 {%       endif -%}
 {%       do zope_family_address_list.append((zope_effective_address, maxconn, webdav)) -%}
+
+{#       # Generate entries with rewrite rule for test runnners #}
+{%       set test_runner_backend_mapping = {} %}
+{%       set test_runner_apache_url_list = [] %}
+{%       set test_runner_external_port = next_port() %}
+{%       for i, (test_runner_internal_ip, test_runner_internal_port) in
+             enumerate(slapparameter_dict[parameter_id ~ '-test-runner-address-list']) %}
+{%         do test_runner_backend_mapping.__setitem__(
+                'unit_test_' ~ i,
+                'http://' ~ test_runner_internal_ip ~ ':' ~ test_runner_internal_port ) %}
+{%         do test_runner_apache_url_list.append(
+                'https://' ~ ipv4 ~ ':' ~ test_runner_external_port ~ '/unit_test_' ~ i ~ '/' ) %}
+{%       endfor %}
+{%       do zope_virtualhost_monster_backend_dict.__setitem__(
+              test_runner_external_port,
+              ( ssl_authentication, test_runner_backend_mapping ) ) -%}
+{%       do test_runner_url_dict.__setitem__(family_name, test_runner_apache_url_list) -%}
+
 {%     endfor -%}
 {%   endfor -%}
+
 {# Make rendering fail artificially if any family has no known backend.
  # This is useful as haproxy's hot-reconfiguration mechanism is
  # supervisord-incompatible.
@@ -140,7 +162,6 @@ ipv6 = {{ zope_address.split(']:')[0][1:] }}
 {%     set internal_scheme = 'http' -%}
 {%     set external_scheme = 'https' -%}
 {%   endif -%}
-{%   set ssl_authentication = slapparameter_dict['ssl-authentication-dict'].get(family_name, False) -%}
 {%   do apache_dict.__setitem__(family_name, (next_port(), external_scheme, internal_scheme ~ '://' ~ ipv4 ~ ':' ~ haproxy_port ~ backend_path, ssl_authentication)) -%}
 {% endfor -%}
 
@@ -173,6 +194,7 @@ crl = ${directory:apache-conf}/crl.pem
 
 [apache-conf-parameter-dict]
 backend-list = {{ dumps(apache_dict.values()) }}
+zope-virtualhost-monster-backend-dict = {{ dumps(zope_virtualhost_monster_backend_dict) }}
 ip-list = {{ dumps(apache_ip_list) }}
 pid-file = ${directory:run}/apache.pid
 log-dir = ${directory:log}
@@ -225,6 +247,10 @@ recipe = slapos.cookbook:publish.serialised
 {{   family_name ~ '-v6' }} = {% if ipv6_set %}{{ scheme ~ '://[' ~ ipv6 ~ ']:' ~ apache_port }}{% endif %}
 {{   family_name }} = {{ scheme ~ '://' ~ ipv4 ~ ':' ~ apache_port }}
 {% endfor -%}
+{% for family_name, test_runner_url_list in test_runner_url_dict.items() -%}
+{{    family_name ~ '-test-runner-url-list' }} = {{ dumps(test_runner_url_list) }}
+{% endfor -%}
+
 monitor-base-url = ${monitor-publish-parameters:monitor-base-url}
 
 [apache-ssl]

stack/erp5/instance-erp5.cfg.in

@@ -8,6 +8,7 @@
 {% set jupyter_dict = slapparameter_dict.get('jupyter', {}) -%}
 {% set has_jupyter = jupyter_dict.get('enable', jupyter_enable_default.lower() in ('true', 'yes')) -%}
 {% set jupyter_zope_family = jupyter_dict.get('zope-family', '') -%}
+{% set test_runner_count = slapparameter_dict.get('mariadb', {}).get('test-database-amount', 1) -%}
 {% set monitor_base_url_dict = {} -%}
 {% set caucase_url = slapparameter_dict.get('caucase', {}).pop('url', '') -%}
 {% set monitor_dict = slapparameter_dict.get('monitor', {}) %}
@@ -93,44 +94,6 @@ connection-http-url = {{ caucase_url }}
 {%   endif -%}
 {% endfor -%}
 
-[publish-early]
-recipe = slapos.cookbook:publish-early
--init =
-  inituser-password gen-password:passwd
-  deadlock-debugger-password gen-deadlock-debugger-password:passwd
-{%- if has_posftix %}
-  smtpd-sasl-password gen-smtpd-sasl-password:passwd
-{%- endif %}
-{%- if neo %}
-  neo-cluster gen-neo-cluster:name
-{%-  if neo[0] %}
-neo-cluster = {{ dumps(neo[0]) }}
-{%-  endif %}
-{%- endif %}
-{%- set inituser_password = slapparameter_dict.get('inituser-password') %}
-{%- if inituser_password %}
-inituser-password = {{ dumps(inituser_password) }}
-{%- endif %}
-{%- set deadlock_debugger_password = slapparameter_dict.get('deadlock-debugger-password') -%}
-{%- if deadlock_debugger_password %}
-deadlock-debugger-password = {{ dumps(deadlock_debugger_password) }}
-{%- endif %}
-
-[gen-password]
-recipe = slapos.cookbook:generate.password
-storage-path =
-
-[gen-deadlock-debugger-password]
-<= gen-password
-
-[gen-neo-cluster-base]
-<= gen-password
-
-[gen-neo-cluster]
-name = neo-${gen-neo-cluster-base:passwd}
-
-[gen-smtpd-sasl-password]
-< = gen-password
 
 {% set zope_partition_dict = slapparameter_dict.get('zope-partition-dict', {'1': {}}) -%}
 {% set zope_address_list_id_dict = {} -%}
@@ -142,6 +105,9 @@ return =
   zope-address-list
   hosts-dict
   monitor-base-url
+{%- if test_runner_count %}
+  test-runner-address-list
+{% endif %}
 {% set bt5_default_list = 'erp5_full_text_myisam_catalog erp5_configurator_standard erp5_configurator_maxma_demo erp5_configurator_run_my_doc' -%}
 {% if has_jupyter -%}
 {%   set bt5_default_list = bt5_default_list + ' erp5_data_notebook' -%}
@@ -169,6 +135,7 @@ config-timezone = {{ dumps(slapparameter_dict.get('timezone', 'UTC')) }}
 config-cloudooo-retry-count = {{ slapparameter_dict.get('cloudooo-retry-count', 2) }}
 config-wendelin-core-zblk-fmt = {{ dumps(slapparameter_dict.get('wendelin-core-zblk-fmt', '')) }}
 config-zodb-dict = {{ dumps(zodb_dict) }}
+config-test-runner-count = {{ dumps(test_runner_count) }}
 {% for server_type, server_dict in storage_dict.iteritems() -%}
 {%   if server_type == 'neo' -%}
 config-neo-cluster = ${publish-early:neo-cluster}
@@ -183,6 +150,7 @@ config-tidstorage-port = ${request-zodb:connection-tidstorage-port}
 software-type = zope
 
 {% set zope_family_dict = {} -%}
+{% set zope_family_name_list = [] -%}
 {% set zope_backend_path_dict = {} -%}
 {% set ssl_authentication_dict = {} -%}
 {% set jupyter_zope_family_default = [] -%}
@@ -190,6 +158,7 @@ software-type = zope
 {%   set partition_name = 'zope-' ~ custom_name -%}
 {%   set section_name = 'request-' ~ partition_name -%}
 {%   set zope_family = zope_parameter_dict.get('family', 'default') -%}
+{%   do zope_family_name_list.append(zope_family) %}
 {%   set backend_path = zope_parameter_dict.get('backend-path', '/') % {'site-id': site_id} %}
 {#   # default jupyter zope family is first zope family. -#}
 {#   # use list.append() to update it, because in jinja2 set changes only local scope. -#}
@@ -214,6 +183,7 @@ config-longrequest-logger-timeout = {{ dumps(zope_parameter_dict.get('longreques
 config-large-file-threshold = {{ dumps(zope_parameter_dict.get('large-file-threshold', "10MB")) }}
 config-port-base = {{ dumps(zope_parameter_dict.get('port-base', 2200)) }}
 config-webdav = {{ dumps(zope_parameter_dict.get('webdav', False)) }}
+config-test-runner-apache-url-list = ${publish-early:{{ zope_family }}-test-runner-url-list}
 {% endfor -%}
 
 {# if not explicitly configured, connect jupyter to first zope family, which  -#}
@@ -268,6 +238,9 @@ return =
 {%- for family in zope_family_dict %}
   {{ family }}
   {{ family }}-v6
+  {% if test_runner_count %}
+  {{ family }}-test-runner-url-list
+  {% endif %}
 {% endfor -%}
 {% do monitor_base_url_dict.__setitem__('request-balancer', '${' ~ 'request-balancer' ~ ':connection-monitor-base-url}') -%}
 
@@ -275,6 +248,9 @@ config-zope-family-dict = {{ dumps(zope_family_parameter_dict) }}
 config-tcpv4-port = {{ dumps(balancer_dict.get('tcpv4-port', 2150)) }}
 {% for zope_section_id, name in zope_address_list_id_dict.items() -%}
 config-{{ name }} = {{ ' ${' ~ zope_section_id ~ ':connection-zope-address-list}' }}
+{%   if test_runner_count -%}
+config-{{ name }}-test-runner-address-list = {{ ' ${' ~ zope_section_id ~ ':connection-test-runner-address-list}' }}
+{%   endif -%}
 {% endfor -%}
 # XXX: should those really be same for all families ?
 config-haproxy-server-check-path = {{ dumps(balancer_dict.get('haproxy-server-check-path', '/') % {'site-id': site_id}) }}
@@ -335,6 +311,57 @@ hosts-dict = {{ '${' ~ zope_address_list_id_dict.keys()[0] ~ ':connection-hosts-
 {% for name, value in publish_dict.items() -%}
 {{   name }} = {{ value }}
 {% endfor -%}
+{% for zope_family_name in zope_family_name_list -%}
+{{ zope_family_name }}-test-runner-url-list = ${request-balancer:connection-{{ zope_family_name }}-test-runner-url-list}
+{% endfor -%}
+
+
+[publish-early]
+recipe = slapos.cookbook:publish-early
+-init =
+  inituser-password gen-password:passwd
+  deadlock-debugger-password gen-deadlock-debugger-password:passwd
+{%- if has_posftix %}
+  smtpd-sasl-password gen-smtpd-sasl-password:passwd
+{%- endif %}
+{% for zope_family_name in zope_family_name_list %}
+  {{ zope_family_name }}-test-runner-url-list default-balancer-test-runner-url-list:default
+{% endfor -%}
+{%- if neo %}
+  neo-cluster gen-neo-cluster:name
+{%-  if neo[0] %}
+neo-cluster = {{ dumps(neo[0]) }}
+{%-  endif %}
+{%- endif %}
+{%- set inituser_password = slapparameter_dict.get('inituser-password') %}
+{%- if inituser_password %}
+inituser-password = {{ dumps(inituser_password) }}
+{%- endif %}
+{%- set deadlock_debugger_password = slapparameter_dict.get('deadlock-debugger-password') -%}
+{%- if deadlock_debugger_password %}
+deadlock-debugger-password = {{ dumps(deadlock_debugger_password) }}
+{%- endif %}
+
+
+[default-balancer-test-runner-url-list]
+recipe =
+default = {{ dumps(["ERROR: not published yet"]) }}
+
+[gen-password]
+recipe = slapos.cookbook:generate.password
+storage-path =
+
+[gen-deadlock-debugger-password]
+<= gen-password
+
+[gen-neo-cluster-base]
+<= gen-password
+
+[gen-neo-cluster]
+name = neo-${gen-neo-cluster-base:passwd}
+
+[gen-smtpd-sasl-password]
+< = gen-password
 
 [monitor-instance-parameter]
 monitor-httpd-port = 8386

stack/erp5/instance-zope.cfg.in

@@ -8,6 +8,8 @@
 {% set node_id_index_format = '-%%0%ii' % (len(str(instance_index_list[-1])), ) -%}
 {% set part_list = [] -%}
 {% set publish_list = [] -%}
+{% set test_runner_address_list = [] -%}
+{% set test_runner_count = slapparameter_dict['test-runner-count'] -%}
 {% set longrequest_logger_base_path = buildout_directory ~ '/var/log/longrequest_logger_' -%}
 {% macro section(name) %}{% do part_list.append(name) %}{{ name }}{% endmacro -%}
 {% set bin_directory = parameter_dict['buildout-bin-directory'] -%}
@@ -87,7 +89,11 @@ context =
     key password test-zelenium-runner-parameter:password
     key bin_path test-zelenium-runner-parameter:bin-path
 {% else -%}
-{% if slapparameter_dict['mysql-test-url-list'] -%}
+{% if test_runner_count -%}
+{% for _ in range(test_runner_count) %}
+{%   do test_runner_address_list.append((ipv4, next_port())) %}
+{% endfor %}
+
 [{{ section('run-unit-test-userhosts-wrapper') }}]
 <= userhosts-wrapper-base
 wrapped-command-line = ${runUnitTest:wrapper-path}
@@ -138,12 +144,22 @@ command-name = runUnitTest
 command-line-extra =
   --erp5_sql_connection_string '{{ connection_string_list[0] }}'
   --extra_sql_connection_string_list '{{ ','.join(connection_string_list[1:]) }}'
+  --zserver {{ test_runner_address_list[0][0] ~ ':' ~ test_runner_address_list[0][1] }}
+  --zserver_frontend_url {{ slapparameter_dict['test-runner-apache-url-list'][0] }}
 
 [{{ section('runTestSuite') }}]
 < = run-test-common
 command-name = runTestSuite
 command-line-extra =
   --db_list '{{ ','.join(connection_string_list) }}'
+
+ {#- turn a list of (ip, port) in a list of 'ip:port' #}
+ {% set zserver_address_list = [] -%}
+ {% for ip, port in test_runner_address_list %}
+ {%   do zserver_address_list.append(ip ~ ':' ~ port) %}
+ {% endfor -%}
+  --zserver_address_list '{{ ','.join(zserver_address_list) }}'
+  --zserver_frontend_url_list '{{ ','.join(slapparameter_dict['test-runner-apache-url-list']) }}'
 {%- endif %}
 {%- endif %}
 
@@ -484,6 +500,7 @@ hard to guess.
 -#}
 hosts-dict = {{ dumps(hosts_dict) }}
 monitor-base-url = ${monitor-publish-parameters:monitor-base-url}
+test-runner-address-list = {{ dumps(test_runner_address_list) }}
 
 [monitor-instance-parameter]
 monitor-httpd-ipv6 = {{ (ipv6_set | list)[0] }}