ERP5: rework frontend instance parameter

This change the format or the (mostly) unused frontend parameter to
support requesting more than one frontend and also enable the request of
a frontend by default, so that requesting a frontend separately is no
longer needed.

The `frontend` parameter now also supports requesting frontends for
specific paths on the ERP5 backend, the example below requests a
frontend serving directly a web site, with the necessary rewrite rules:

```js
{
  "frontend": {
    "default": {
      "internal-path": "/erp5/web_site_module/renderjs_runner/"
    }
  }
}
```

The example below requests a default frontend to the erp5 root, to
access the ZMI or erp5_xhtml_style interface and two web sites:

```js
{
  "frontend": {
    "default": {},
    "erp5js": {
      "internal-path": "/erp5/web_site_module/renderjs_runner/"
    },
    "crm": {
      "internal-path": "/erp5/web_site_module/erp5_officejs_support_request_ui/"
    }
  }
}
```

The example below has an explicit definition of the zope families using
`zope-partition-dict` parameter, because there is more than one zope
family, no frontend is requested by default:

```js
{
  "zope-partition-dict": {
    "backoffice": {
      "family": "backoffice"
    },
    "web": {
      "family": "web"
    },
    "activities": {
      "family": "activities"
    }
  }
}
```

Continuing this example, to have frontends for backoffice and web
families, the frontend request can specify the families, like it is
demonstrated in the example below. In this example, we don't specify an
entry for "activities" family, so no frontend will be requested for
this family.

```js
{
  "frontend": {
    "backoffice": {
      "zope-family": "backoffice"
    },
    "web": {
      "zope-family": "web",
      "internal-path": "/erp5/web_site_module/web_site/"
    }
  }
  "zope-partition-dict": {
    "backoffice": {
      "family": "backoffice"
    },
    "web": {
      "family": "web"
    },
    "activities": {
      "family": "activities"
    }
  }
}
```

software/erp5/instance-erp5-input-schema.json

@@ -37,7 +37,7 @@
   },
   "properties": {
     "sla-dict": {
-      "description": "Where to request instances. Each key is a query string for criterions (e.g. \"computer_guid=foo\"), and each value is a list of partition references (note: Zope partitions reference must be prefixed with \"zope-\").",
+      "description": "Where to request instances. Each key is a query string for criterions (e.g. \"computer_guid=foo\"), and each value is a list of partition references (notes: Zope partitions reference must be prefixed with \"zope-\", frontends must be prefixed with \"frontend-\").",
       "additionalProperties": {
         "type": "array",
         "items": {
@@ -155,33 +155,41 @@
       "type": "object"
     },
     "frontend": {
-      "description": "Front-end slave instance request parameters",
-      "properties": {
-        "software-url": {
-          "description": "Front-end's software type. If this parameter is empty, no front-end instance is requested. Else, sla-dict must specify 'frontend' which is a special value matching all frontends (e.g. {\"instance_guid=bar\": [\"frontend\"]}).",
-          "default": "",
-          "type": "string",
-          "format": "uri"
-        },
-        "domain": {
-          "description": "The domain name to request front-end to respond as.",
-          "default": "",
-          "type": "string"
-        },
-        "software-type": {
-          "description": "Request a front-end slave instance of this software type.",
-          "default": "RootSoftwareInstance",
-          "type": "string"
-        },
-        "virtualhostroot-http-port": {
-          "description": "Front-end slave http port. Port where http requests to frontend will be redirected.",
-          "default": 80,
-          "type": "integer"
-        },
-        "virtualhostroot-https-port": {
-          "description": "Front-end slave https port. Port where https requests to frontend will be redirected.",
-          "default": 443,
-          "type": "integer"
+      "description": "Frontend shared instances requests parameters. When this parameter is unset, the system defaults to requesting a frontend, but only when exactly one family exists in `zope-partition-dict`. For more complex zope partition layout, the frontend layout also have to be explicitly defined.",
+      "default": {
+        "default": {}
+      },
+      "patternProperties": {
+        ".*": {
+          "required": [
+            "zope-family"
+          ],
+          "properties": {
+            "zope-family": {
+              "description": "The zope family to which the requests will be routed.",
+              "type": "string"
+            },
+            "internal-path": {
+              "description": "Internal path from the backend. `%(site-id)s` is substituted by the site id.",
+              "type": "string",
+              "default": "/%(site-id)s"
+            },
+            "software-url": {
+              "description": "Software URL of the frontend shared instance.",
+              "type": "string",
+              "format": "uri",
+              "default": "http://git.erp5.org/gitweb/slapos.git/blob_plain/HEAD:/software/apache-frontend/software.cfg"
+            },
+            "software-type": {
+              "description": "Software type of the frontend shared instance.",
+              "type": "string"
+            },
+            "instance-parameters": {
+              "description": "Instance parameters for the frontend shared instance.",
+              "$ref": "../rapid-cdn/instance-slave-input-schema.json"
+            }
+          },
+          "type": "object"
         }
       },
       "type": "object"

software/erp5/instance-erp5-output-schema.json

@@ -79,6 +79,11 @@
       "description": "Zope family access information",
       "pattern": "^https://",
       "type": "string"
+    },
+    "url-frontend-.*": {
+      "description": "Frontend URL, following `url-frontend-{frontend_name}` naming scheme",
+      "pattern": "^https://",
+      "type": "string"
     }
   },
   "type": "object"

software/erp5/test/test/test_erp5.py

@@ -47,6 +47,7 @@ import psutil
 import requests
 import urllib3
 from slapos.testing.utils import CrontabMixin
+import zc.buildout.configparser
 
 from . import ERP5InstanceTestCase, default, matrix, neo, setUpModule
 
@@ -116,6 +117,16 @@ class TestPublishedURLIsReachableMixin:
       verify=False,
     )
 
+  def test_published_frontend_default_is_reachable(self):
+    """Tests the frontend URL published by the root partition is reachable.
+    """
+    param_dict = self.getRootPartitionConnectionParameterDict()
+    self._checkERP5IsReachable(
+      param_dict['url-frontend-default'],
+      param_dict['site-id'],
+      verify=False,
+    )
+
 
 class TestDefaultParameters(ERP5InstanceTestCase, TestPublishedURLIsReachableMixin):
   """Test ERP5 can be instantiated with no parameters
@@ -123,6 +134,28 @@ class TestDefaultParameters(ERP5InstanceTestCase, TestPublishedURLIsReachableMix
   __partition_reference__ = 'defp'
   __test_matrix__ = matrix((default,))
 
+  def test_frontend_request(self):
+    with open(os.path.join(self.computer_partition_root_path,
+                           '.installed-switch-softwaretype.cfg')) as f:
+      installed = zc.buildout.configparser.parse(f, 'installed')
+    self.assertEqual(
+      installed['request-frontend-default']['config-type'], 'zope')
+    self.assertEqual(
+      installed['request-frontend-default']['config-path'], '/erp5')
+    self.assertEqual(
+      installed['request-frontend-default']['config-authenticate-to-backend'], 'true')
+    self.assertEqual(installed['request-frontend-default']['shared'], 'true')
+    self.assertEqual(
+      installed['request-frontend-default']['name'], 'frontend-default')
+    self.assertEqual(
+      installed['request-frontend-default']['software-url'],
+      'http://git.erp5.org/gitweb/slapos.git/blob_plain/HEAD:/software/apache-frontend/software.cfg'
+    )
+
+    self.assertEqual(
+      installed['request-frontend-default']['connection-secure_access'],
+      self.getRootPartitionConnectionParameterDict()['url-frontend-default'])
+
 
 class TestJupyter(ERP5InstanceTestCase, TestPublishedURLIsReachableMixin):
   """Test ERP5 Jupyter notebook
@@ -153,6 +186,8 @@ class TestBalancerPorts(ERP5InstanceTestCase):
   """Instantiate with two zope families, this should create for each family:
    - a balancer entry point with corresponding haproxy
    - a balancer entry point for test runner
+
+  and no frontend at all, because more than one family exist.
   """
   __partition_reference__ = 'ap'
 
@@ -223,6 +258,18 @@ class TestBalancerPorts(ERP5InstanceTestCase):
             if c.status == 'LISTEN'
         ))
 
+  def test_no_frontend_request(self):
+    with open(os.path.join(self.computer_partition_root_path,
+                           '.installed-switch-softwaretype.cfg')) as f:
+      installed = zc.buildout.configparser.parse(f, 'installed')
+    self.assertFalse(
+      [section for section in installed if 'request-frontend' in section])
+    self.assertFalse(
+      [
+        param for param in self.getRootPartitionConnectionParameterDict()
+        if 'frontend' in param
+      ])
+
 
 class TestSeleniumTestRunner(ERP5InstanceTestCase, TestPublishedURLIsReachableMixin):
   """Test ERP5 can be instantiated with selenium server for test runner.
@@ -1100,3 +1147,144 @@ class TestNEO(ZopeSkinsMixin, CrontabMixin, ERP5InstanceTestCase):
           'var',
           'log',
           f))
+
+
+class TestFrontend(ERP5InstanceTestCase):
+  __partition_reference__ = 'f'
+
+  @classmethod
+  def getInstanceParameterDict(cls):
+    return {
+      '_':
+      json.dumps(
+        {
+          "zope-partition-dict": {
+            "backoffice": {
+              "family": "default",
+            },
+            "web": {
+              "family": "web",
+            },
+            "activities": {
+              # this family will not have frontend
+              "family": "activities"
+            },
+          },
+          "frontend": {
+            "backoffice": {
+              "zope-family": "default",
+            },
+            "website": {
+              "zope-family": "web",
+              "internal-path": "/%(site-id)s/web_site_module/my_website/",
+              "instance-parameters": {
+                # some extra frontend parameters
+                "enable_cache": "true",
+              }
+            }
+          },
+          "sla-dict": {
+            "computer_guid=COMP-1234": ["frontend-backoffice"]
+          }
+        })
+    }
+
+  def test_frontend_url_published(self):
+    param_dict = self.getRootPartitionConnectionParameterDict()
+    requests.get(
+      param_dict['url-frontend-backoffice'],
+      verify=False,
+      allow_redirects=False,
+    )
+    requests.get(
+      param_dict['url-frontend-website'],
+      verify=False,
+      allow_redirects=False,
+    )
+
+  def test_request_parameters(self):
+    param_dict = self.getRootPartitionConnectionParameterDict()
+
+    with open(os.path.join(self.computer_partition_root_path,
+                           '.installed-switch-softwaretype.cfg')) as f:
+      installed = zc.buildout.configparser.parse(f, 'installed')
+    self.assertEqual(
+      installed['request-frontend-backoffice']['config-type'], 'zope')
+    self.assertEqual(
+      installed['request-frontend-backoffice']['shared'], 'true')
+    self.assertEqual(
+      installed['request-frontend-backoffice']['config-url'],
+      param_dict['family-default-v6'])
+    self.assertEqual(
+      installed['request-frontend-backoffice']['config-path'], '/erp5')
+    self.assertEqual(
+      installed['request-frontend-backoffice']['sla-computer_guid'],
+      'COMP-1234')
+    self.assertEqual(
+      installed['request-frontend-backoffice']['software-url'],
+      'http://git.erp5.org/gitweb/slapos.git/blob_plain/HEAD:/software/apache-frontend/software.cfg'
+    )
+    self.assertEqual(
+      installed['request-frontend-backoffice']['connection-secure_access'],
+      param_dict['url-frontend-backoffice'])
+
+    self.assertEqual(
+      installed['request-frontend-website']['config-type'], 'zope')
+    # no SLA by default
+    self.assertFalse([k for k in installed['request-frontend-website'] if k.startswith('sla-')])
+    # instance parameters are propagated
+    self.assertEqual(
+      installed['request-frontend-website']['config-enable_cache'], 'true')
+    self.assertEqual(
+      installed['request-frontend-website']['config-url'],
+      param_dict['family-web-v6'])
+    self.assertEqual(
+      installed['request-frontend-website']['config-path'],
+      '/erp5/web_site_module/my_website/')
+    self.assertEqual(
+      installed['request-frontend-website']['connection-secure_access'],
+      param_dict['url-frontend-website'])
+
+    # no frontend was requested for activities family
+    self.assertNotIn('request-frontend-activities', installed)
+    self.assertNotIn('url-frontend-activities', param_dict)
+
+
+class TestDefaultFrontendWithZopePartitionDict(ERP5InstanceTestCase):
+  """Default frontend also is requested when only one zope family
+  is defined, but on multiple partitions
+  """
+  __partition_reference__ = 'fzpd'
+
+  @classmethod
+  def getInstanceParameterDict(cls):
+    return {
+      '_':
+      json.dumps(
+        {
+          "zope-partition-dict": {
+            "backoffice-0": {
+              "family": "backoffice",
+            },
+            "backoffice-1": {
+              "family": "backoffice",
+            }
+          }
+        }
+      )
+    }
+
+  def test_frontend_requested(self):
+    param_dict = self.getRootPartitionConnectionParameterDict()
+    with open(os.path.join(self.computer_partition_root_path,
+                           '.installed-switch-softwaretype.cfg')) as f:
+      installed = zc.buildout.configparser.parse(f, 'installed')
+    self.assertEqual(
+      installed['request-frontend-default']['config-url'],
+      param_dict['family-backoffice-v6'])
+
+    requests.get(
+      param_dict['url-frontend-default'],
+      verify=False,
+      allow_redirects=False,
+    )

software/neoppod/buildout.hash.cfg

@@ -18,7 +18,7 @@ md5sum = e000e7134113b9d1c63d40861eaf0489
 
 [root-common]
 filename = root-common.cfg.in
-md5sum = ae00507d9e69209a0babd725cf6be536
+md5sum = c91b5540f94ce76af31f84584df7a3ef
 
 [instance-neo-admin]
 filename = instance-neo-admin.cfg.in

software/neoppod/root-common.cfg.in

@@ -6,12 +6,12 @@
 {%   do sla_dict.update(dict.fromkeys(ref_list, sla)) -%}
 {% endfor -%}
 
-{% macro sla(name, required=False) -%}
+{% macro sla(name, required=False, default_to_same_computer=True) -%}
 {% if required or name in sla_dict -%}
 {%  for k, (v,) in six.iteritems(urllib_parse.parse_qs(sla_dict.pop(name), strict_parsing=1)) -%}
 sla-{{ k }} = {{ v }}
 {%  endfor -%}
-{% else -%}
+{% elif default_to_same_computer -%}
 sla-computer_guid = ${slap-connection:computer-id}
 {% endif -%}
 {% endmacro -%}

stack/erp5/buildout.hash.cfg

@@ -74,7 +74,7 @@ md5sum = 3f7b28085ceff321a3cb785db60f7c3e
 
 [template-erp5]
 filename = instance-erp5.cfg.in
-md5sum = 098e1d02159aeca9b36f2a0726b7b230
+md5sum = 2cecadd50525f2c517fb19bd82a8d3e7
 
 [template-zeo]
 filename = instance-zeo.cfg.in

stack/erp5/instance-erp5.cfg.in

 {% import "root_common" as root_common with context -%}
 {% import "caucase" as caucase with context %}
-{% set frontend_dict = slapparameter_dict.get('frontend', {}) -%}
-{% set has_frontend = frontend_dict.get('software-url', '') != '' -%}
 {% set site_id = slapparameter_dict.get('site-id', 'erp5') -%}
 {% set inituser_login = slapparameter_dict.get('inituser-login', 'zope') -%}
 {% set publish_dict = {'site-id': site_id, 'inituser-login': inituser_login} -%}
@@ -370,31 +368,67 @@ config-command = {{ '${' ~ check_software_url_section_name ~ ':path}' }}
 {%     do zope_address_list_id_dict.__setitem__(zope_section_id, parameter_name) -%}
 {%     do zope_family_parameter_dict.setdefault(family_name, []).append(parameter_name) -%}
 {%   endfor -%}
-{%   if has_frontend -%}
-{%     set frontend_name = 'frontend-' ~ family_name -%}
-{%     do publish_dict.__setitem__('family-' ~ family_name, '${' ~ frontend_name ~ ':connection-site_url}' ) -%}
-[{{ frontend_name }}]
+{%   do publish_dict.__setitem__('family-' ~ family_name, '${request-balancer:connection-' ~ family_name ~ '}' ) -%}
+{%   do publish_dict.__setitem__('family-' ~ family_name ~ '-v6', '${request-balancer:connection-' ~ family_name ~ '-v6}' ) -%}
+{% endfor -%}
+
+
+{# We request a default frontend when exactly only one zope family exists #}
+{% set frontend_parameters_dict = slapparameter_dict.get(
+  'frontend',
+  {'default': { 'zope-family': list(zope_family_dict)[0] }} if len(zope_family_dict) == 1 else {}
+) -%}
+
+[request-frontend-base]
+<= request-common
+recipe = slapos.cookbook:request
+shared = true
+return =
+  secure_access
+{{ root_common.sla('frontend', default_to_same_computer=False) }}
+
+{% for frontend_name, frontend_parameters in frontend_parameters_dict.items() -%}
+{%   set frontend_full_name = 'frontend-' ~ frontend_name -%}
+{%   set request_frontend_name = 'request-frontend-' ~ frontend_name -%}
+{%   set frontend_software_url = frontend_parameters.get('software-url', 'http://git.erp5.org/gitweb/slapos.git/blob_plain/HEAD:/software/apache-frontend/software.cfg') -%}
+{%   set frontend_software_type = frontend_parameters.get('software-type', '') -%}
+{%   set frontend_instance_parameters = frontend_parameters.get('instance-parameters', {}) -%}
+{%   if frontend_instance_parameters.setdefault('type', 'zope') == 'zope' -%}
+{%     do frontend_instance_parameters.setdefault('authenticate-to-backend', 'true') -%}
+{%     set zope_family_name = frontend_parameters['zope-family'] -%}
+{%     do assert(zope_family_name in zope_family_dict, 'Unknown family %s for frontend %s' % (zope_family_name, frontend_name)) -%}
+{%     do frontend_instance_parameters.setdefault('url', '${request-balancer:connection-' ~ zope_family_name ~ '-v6}') -%}
+{%     do frontend_instance_parameters.setdefault('path', frontend_parameters.get('internal-path', '/%(site-id)s') % {'site-id': site_id}) -%}
+{%   endif %}
+[{{ request_frontend_name }}]
 <= request-frontend-base
-name = {{ frontend_name }}
-config-url = ${request-balancer:connection-{{ family_name }}-v6}
-{%   else -%}
-{%     do publish_dict.__setitem__('family-' ~ family_name, '${request-balancer:connection-' ~ family_name ~ '}' ) -%}
-{%     do publish_dict.__setitem__('family-' ~ family_name ~ '-v6', '${request-balancer:connection-' ~ family_name ~ '-v6}' ) -%}
-{%   endif -%}
+name = {{ frontend_full_name }}
+software-url = {{ frontend_software_url }}
+{%   if frontend_software_type %}
+software-type = {{ frontend_software_type }}
+{%   endif %}
+{{ root_common.sla(frontend_full_name, default_to_same_computer=False) }}
+{%   for name, value in frontend_instance_parameters.items() -%}
+config-{{ name }} = {{ value }}
+{%   endfor -%}
+
+{%   set promise_frontend_section_name = 'promise-' ~ request_frontend_name %}
+[{{ promise_frontend_section_name }}]
+<= monitor-promise-base
+promise = check_url_available
+name = ${:_buildout_section_name_}.py
+config-url = {{ "${" }}{{ request_frontend_name }}:connection-secure_access}
+config-ignore-code = 1
+config-allow-redirects = 0
+
+{%   do root_common.section(promise_frontend_section_name) -%}
+{%   do publish_dict.__setitem__('url-frontend-' ~ frontend_name, '${' ~ request_frontend_name ~ ':connection-secure_access}' ) -%}
 {% endfor -%}
 
+
 {% if has_jupyter -%}
 {# request jupyter connected to balancer of proper zope family -#}
 {{   request('jupyter', 'jupyter', 'jupyter', {}, key_config={'erp5-url': 'request-balancer:connection-' ~ jupyter_zope_family}) }}
-
-{%   if has_frontend -%}
-[frontend-jupyter]
-<= request-frontend-base
-name = frontend-jupyter
-config-url = ${request-jupyter:connection-url}
-{#     # override jupyter-url in publish_dict with frontend address -#}
-{%     do publish_dict.__setitem__('jupyter-url', '${frontend-jupyter:connection-site_url}') -%}
-{%   endif -%}
 {%- endif %}
 
 {% if wcfs_enable -%}
@@ -458,31 +492,6 @@ config-url = ${request-jupyter:connection-url}
   key_config=balancer_key_config_dict,
 ) }}
 
-[request-frontend-base]
-{% if has_frontend -%}
-<= request-common
-recipe = slapos.cookbook:request
-software-url = {{ dumps(frontend_dict['software-url']) }}
-software-type = {{ dumps(frontend_dict.get('software-type', 'RootSoftwareInstance')) }}
-{{ root_common.sla('frontend', True) }}
-shared = true
-{% set config_dict = {
-  'type': 'zope',
-} -%}
-{%   if frontend_dict.get('domain') -%}
-{%     do config_dict.__setitem__('custom_domain', frontend_dict['domain']) -%}
-{%   endif -%}
-{%   if frontend_dict.get('virtualhostroot-http-port') -%}
-{%     do config_dict.__setitem__('virtualhostroot-http-port', frontend_dict['virtualhostroot-http-port']) -%}
-{%   endif -%}
-{%   if frontend_dict.get('virtualhostroot-https-port') -%}
-{%     do config_dict.__setitem__('virtualhostroot-https-port', frontend_dict['virtualhostroot-https-port']) -%}
-{%   endif -%}
-{%   for name, value in config_dict.items() -%}
-config-{{ name }} = {{ value }}
-{%   endfor -%}
-return = site_url
-{% endif -%}
 
 {% endif -%}{# if zope_partition_dict -#}