Commit cb78214e authored by Jérome Perrin's avatar Jérome Perrin

ERP5: rework frontend instance parameter

This change the format or the (mostly) unused frontend parameter to
support requesting more than one frontend and also enable the request of
a frontend by default, so that requesting a frontend separately is no
longer needed.

The `frontend` parameter now also supports requesting frontends for
specific paths on the ERP5 backend, the example below requests a
frontend serving directly a web site, with the necessary rewrite rules:

```js
{
  "frontend": {
    "default": {
      "internal-path": "/erp5/web_site_module/renderjs_runner/"
    }
  }
}
```

The example below requests a default frontend to the erp5 root, to
access the ZMI or erp5_xhtml_style interface and two web sites:

```js
{
  "frontend": {
    "default": {},
    "erp5js": {
      "internal-path": "/erp5/web_site_module/renderjs_runner/"
    },
    "crm": {
      "internal-path": "/erp5/web_site_module/erp5_officejs_support_request_ui/"
    }
  }
}
```

The example below has an explicit definition of the zope families using
`zope-partition-dict` parameter, because there is more than one zope
family, no frontend is requested by default:

```js
{
  "zope-partition-dict": {
    "backoffice": {
      "family": "backoffice"
    },
    "web": {
      "family": "web"
    },
    "activities": {
      "family": "activities"
    }
  }
}
```

Continuing this example, to have frontends for backoffice and web
families, the frontend request can specify the families, like it is
demonstrated in the example below. In this example, we don't specify an
entry for "activities" family, so no frontend will be requested for
this family.

```js
{
  "frontend": {
    "backoffice": {
      "zope-family": "backoffice"
    },
    "web": {
      "zope-family": "web",
      "internal-path": "/erp5/web_site_module/web_site/"
    }
  }
  "zope-partition-dict": {
    "backoffice": {
      "family": "backoffice"
    },
    "web": {
      "family": "web"
    },
    "activities": {
      "family": "activities"
    }
  }
}
```
parent 33d36fdb
...@@ -37,7 +37,7 @@ ...@@ -37,7 +37,7 @@
}, },
"properties": { "properties": {
"sla-dict": { "sla-dict": {
"description": "Where to request instances. Each key is a query string for criterions (e.g. \"computer_guid=foo\"), and each value is a list of partition references (note: Zope partitions reference must be prefixed with \"zope-\").", "description": "Where to request instances. Each key is a query string for criterions (e.g. \"computer_guid=foo\"), and each value is a list of partition references (notes: Zope partitions reference must be prefixed with \"zope-\", frontends must be prefixed with \"frontend-\").",
"additionalProperties": { "additionalProperties": {
"type": "array", "type": "array",
"items": { "items": {
...@@ -159,33 +159,41 @@ ...@@ -159,33 +159,41 @@
"type": "object" "type": "object"
}, },
"frontend": { "frontend": {
"description": "Front-end slave instance request parameters", "description": "Frontend shared instances requests parameters. When this parameter is unset, the system defaults to requesting a frontend, but only when exactly one family exists in `zope-partition-dict`. For more complex zope partition layout, the frontend layout also have to be explicitly defined.",
"properties": { "default": {
"software-url": { "default": {}
"description": "Front-end's software type. If this parameter is empty, no front-end instance is requested. Else, sla-dict must specify 'frontend' which is a special value matching all frontends (e.g. {\"instance_guid=bar\": [\"frontend\"]}).", },
"default": "", "patternProperties": {
"type": "string", ".*": {
"format": "uri" "required": [
}, "zope-family"
"domain": { ],
"description": "The domain name to request front-end to respond as.", "properties": {
"default": "", "zope-family": {
"type": "string" "description": "The zope family to which the requests will be routed.",
}, "type": "string"
"software-type": { },
"description": "Request a front-end slave instance of this software type.", "internal-path": {
"default": "RootSoftwareInstance", "description": "Internal path from the backend. `%(site-id)s` is substituted by the site id.",
"type": "string" "type": "string",
}, "default": "/%(site-id)s"
"virtualhostroot-http-port": { },
"description": "Front-end slave http port. Port where http requests to frontend will be redirected.", "software-url": {
"default": 80, "description": "Software URL of the frontend shared instance.",
"type": "integer" "type": "string",
}, "format": "uri",
"virtualhostroot-https-port": { "default": "http://git.erp5.org/gitweb/slapos.git/blob_plain/HEAD:/software/apache-frontend/software.cfg"
"description": "Front-end slave https port. Port where https requests to frontend will be redirected.", },
"default": 443, "software-type": {
"type": "integer" "description": "Software type of the frontend shared instance.",
"type": "string"
},
"instance-parameters": {
"description": "Instance parameters for the frontend shared instance.",
"$ref": "../rapid-cdn/instance-slave-input-schema.json"
}
},
"type": "object"
} }
}, },
"type": "object" "type": "object"
......
...@@ -79,6 +79,11 @@ ...@@ -79,6 +79,11 @@
"description": "Zope family access information", "description": "Zope family access information",
"pattern": "^https://", "pattern": "^https://",
"type": "string" "type": "string"
},
"url-frontend-.*": {
"description": "Frontend URL, following `url-frontend-{frontend_name}` naming scheme",
"pattern": "^https://",
"type": "string"
} }
}, },
"type": "object" "type": "object"
......
...@@ -49,6 +49,7 @@ import psutil ...@@ -49,6 +49,7 @@ import psutil
import requests import requests
import urllib3 import urllib3
from slapos.testing.utils import CrontabMixin from slapos.testing.utils import CrontabMixin
import zc.buildout.configparser
from . import ERP5InstanceTestCase, default, matrix, neo, setUpModule from . import ERP5InstanceTestCase, default, matrix, neo, setUpModule
...@@ -118,6 +119,16 @@ class TestPublishedURLIsReachableMixin: ...@@ -118,6 +119,16 @@ class TestPublishedURLIsReachableMixin:
verify=False, verify=False,
) )
def test_published_frontend_default_is_reachable(self):
"""Tests the frontend URL published by the root partition is reachable.
"""
param_dict = self.getRootPartitionConnectionParameterDict()
self._checkERP5IsReachable(
param_dict['url-frontend-default'],
param_dict['site-id'],
verify=False,
)
class TestDefaultParameters(ERP5InstanceTestCase, TestPublishedURLIsReachableMixin): class TestDefaultParameters(ERP5InstanceTestCase, TestPublishedURLIsReachableMixin):
"""Test ERP5 can be instantiated with no parameters """Test ERP5 can be instantiated with no parameters
...@@ -125,6 +136,28 @@ class TestDefaultParameters(ERP5InstanceTestCase, TestPublishedURLIsReachableMix ...@@ -125,6 +136,28 @@ class TestDefaultParameters(ERP5InstanceTestCase, TestPublishedURLIsReachableMix
__partition_reference__ = 'defp' __partition_reference__ = 'defp'
__test_matrix__ = matrix((default,)) __test_matrix__ = matrix((default,))
def test_frontend_request(self):
with open(os.path.join(self.computer_partition_root_path,
'.installed-switch-softwaretype.cfg')) as f:
installed = zc.buildout.configparser.parse(f, 'installed')
self.assertEqual(
installed['request-frontend-default']['config-type'], 'zope')
self.assertEqual(
installed['request-frontend-default']['config-path'], '/erp5')
self.assertEqual(
installed['request-frontend-default']['config-authenticate-to-backend'], 'true')
self.assertEqual(installed['request-frontend-default']['shared'], 'true')
self.assertEqual(
installed['request-frontend-default']['name'], 'frontend-default')
self.assertEqual(
installed['request-frontend-default']['software-url'],
'http://git.erp5.org/gitweb/slapos.git/blob_plain/HEAD:/software/apache-frontend/software.cfg'
)
self.assertEqual(
installed['request-frontend-default']['connection-secure_access'],
self.getRootPartitionConnectionParameterDict()['url-frontend-default'])
class TestJupyter(ERP5InstanceTestCase, TestPublishedURLIsReachableMixin): class TestJupyter(ERP5InstanceTestCase, TestPublishedURLIsReachableMixin):
"""Test ERP5 Jupyter notebook """Test ERP5 Jupyter notebook
...@@ -155,6 +188,8 @@ class TestBalancerPorts(ERP5InstanceTestCase): ...@@ -155,6 +188,8 @@ class TestBalancerPorts(ERP5InstanceTestCase):
"""Instantiate with two zope families, this should create for each family: """Instantiate with two zope families, this should create for each family:
- a balancer entry point with corresponding haproxy - a balancer entry point with corresponding haproxy
- a balancer entry point for test runner - a balancer entry point for test runner
and no frontend at all, because more than one family exist.
""" """
__partition_reference__ = 'ap' __partition_reference__ = 'ap'
...@@ -225,6 +260,18 @@ class TestBalancerPorts(ERP5InstanceTestCase): ...@@ -225,6 +260,18 @@ class TestBalancerPorts(ERP5InstanceTestCase):
if c.status == 'LISTEN' if c.status == 'LISTEN'
)) ))
def test_no_frontend_request(self):
with open(os.path.join(self.computer_partition_root_path,
'.installed-switch-softwaretype.cfg')) as f:
installed = zc.buildout.configparser.parse(f, 'installed')
self.assertFalse(
[section for section in installed if 'request-frontend' in section])
self.assertFalse(
[
param for param in self.getRootPartitionConnectionParameterDict()
if 'frontend' in param
])
class TestSeleniumTestRunner(ERP5InstanceTestCase, TestPublishedURLIsReachableMixin): class TestSeleniumTestRunner(ERP5InstanceTestCase, TestPublishedURLIsReachableMixin):
"""Test ERP5 can be instantiated with selenium server for test runner. """Test ERP5 can be instantiated with selenium server for test runner.
...@@ -1149,3 +1196,144 @@ class TestUnsetWithMaxRlimitNofileParameter(ERP5InstanceTestCase, TestPublishedU ...@@ -1149,3 +1196,144 @@ class TestUnsetWithMaxRlimitNofileParameter(ERP5InstanceTestCase, TestPublishedU
process_info, = (p for p in all_process_info if p['name'].startswith('zope-')) process_info, = (p for p in all_process_info if p['name'].startswith('zope-'))
self.assertEqual( self.assertEqual(
resource.prlimit(process_info['pid'], resource.RLIMIT_NOFILE), limit) resource.prlimit(process_info['pid'], resource.RLIMIT_NOFILE), limit)
class TestFrontend(ERP5InstanceTestCase):
__partition_reference__ = 'f'
@classmethod
def getInstanceParameterDict(cls):
return {
'_':
json.dumps(
{
"zope-partition-dict": {
"backoffice": {
"family": "default",
},
"web": {
"family": "web",
},
"activities": {
# this family will not have frontend
"family": "activities"
},
},
"frontend": {
"backoffice": {
"zope-family": "default",
},
"website": {
"zope-family": "web",
"internal-path": "/%(site-id)s/web_site_module/my_website/",
"instance-parameters": {
# some extra frontend parameters
"enable_cache": "true",
}
}
},
"sla-dict": {
"computer_guid=COMP-1234": ["frontend-backoffice"]
}
})
}
def test_frontend_url_published(self):
param_dict = self.getRootPartitionConnectionParameterDict()
requests.get(
param_dict['url-frontend-backoffice'],
verify=False,
allow_redirects=False,
)
requests.get(
param_dict['url-frontend-website'],
verify=False,
allow_redirects=False,
)
def test_request_parameters(self):
param_dict = self.getRootPartitionConnectionParameterDict()
with open(os.path.join(self.computer_partition_root_path,
'.installed-switch-softwaretype.cfg')) as f:
installed = zc.buildout.configparser.parse(f, 'installed')
self.assertEqual(
installed['request-frontend-backoffice']['config-type'], 'zope')
self.assertEqual(
installed['request-frontend-backoffice']['shared'], 'true')
self.assertEqual(
installed['request-frontend-backoffice']['config-url'],
param_dict['family-default-v6'])
self.assertEqual(
installed['request-frontend-backoffice']['config-path'], '/erp5')
self.assertEqual(
installed['request-frontend-backoffice']['sla-computer_guid'],
'COMP-1234')
self.assertEqual(
installed['request-frontend-backoffice']['software-url'],
'http://git.erp5.org/gitweb/slapos.git/blob_plain/HEAD:/software/apache-frontend/software.cfg'
)
self.assertEqual(
installed['request-frontend-backoffice']['connection-secure_access'],
param_dict['url-frontend-backoffice'])
self.assertEqual(
installed['request-frontend-website']['config-type'], 'zope')
# no SLA by default
self.assertFalse([k for k in installed['request-frontend-website'] if k.startswith('sla-')])
# instance parameters are propagated
self.assertEqual(
installed['request-frontend-website']['config-enable_cache'], 'true')
self.assertEqual(
installed['request-frontend-website']['config-url'],
param_dict['family-web-v6'])
self.assertEqual(
installed['request-frontend-website']['config-path'],
'/erp5/web_site_module/my_website/')
self.assertEqual(
installed['request-frontend-website']['connection-secure_access'],
param_dict['url-frontend-website'])
# no frontend was requested for activities family
self.assertNotIn('request-frontend-activities', installed)
self.assertNotIn('url-frontend-activities', param_dict)
class TestDefaultFrontendWithZopePartitionDict(ERP5InstanceTestCase):
"""Default frontend also is requested when only one zope family
is defined, but on multiple partitions
"""
__partition_reference__ = 'fzpd'
@classmethod
def getInstanceParameterDict(cls):
return {
'_':
json.dumps(
{
"zope-partition-dict": {
"backoffice-0": {
"family": "backoffice",
},
"backoffice-1": {
"family": "backoffice",
}
}
}
)
}
def test_frontend_requested(self):
param_dict = self.getRootPartitionConnectionParameterDict()
with open(os.path.join(self.computer_partition_root_path,
'.installed-switch-softwaretype.cfg')) as f:
installed = zc.buildout.configparser.parse(f, 'installed')
self.assertEqual(
installed['request-frontend-default']['config-url'],
param_dict['family-backoffice-v6'])
requests.get(
param_dict['url-frontend-default'],
verify=False,
allow_redirects=False,
)
...@@ -18,7 +18,7 @@ md5sum = e000e7134113b9d1c63d40861eaf0489 ...@@ -18,7 +18,7 @@ md5sum = e000e7134113b9d1c63d40861eaf0489
[root-common] [root-common]
filename = root-common.cfg.in filename = root-common.cfg.in
md5sum = ae00507d9e69209a0babd725cf6be536 md5sum = c91b5540f94ce76af31f84584df7a3ef
[instance-neo-admin] [instance-neo-admin]
filename = instance-neo-admin.cfg.in filename = instance-neo-admin.cfg.in
......
...@@ -6,12 +6,12 @@ ...@@ -6,12 +6,12 @@
{% do sla_dict.update(dict.fromkeys(ref_list, sla)) -%} {% do sla_dict.update(dict.fromkeys(ref_list, sla)) -%}
{% endfor -%} {% endfor -%}
{% macro sla(name, required=False) -%} {% macro sla(name, required=False, default_to_same_computer=True) -%}
{% if required or name in sla_dict -%} {% if required or name in sla_dict -%}
{% for k, (v,) in six.iteritems(urllib_parse.parse_qs(sla_dict.pop(name), strict_parsing=1)) -%} {% for k, (v,) in six.iteritems(urllib_parse.parse_qs(sla_dict.pop(name), strict_parsing=1)) -%}
sla-{{ k }} = {{ v }} sla-{{ k }} = {{ v }}
{% endfor -%} {% endfor -%}
{% else -%} {% elif default_to_same_computer -%}
sla-computer_guid = ${slap-connection:computer-id} sla-computer_guid = ${slap-connection:computer-id}
{% endif -%} {% endif -%}
{% endmacro -%} {% endmacro -%}
......
...@@ -74,7 +74,7 @@ md5sum = 5e0e9565227fe190c420a7bbcd0f7b93 ...@@ -74,7 +74,7 @@ md5sum = 5e0e9565227fe190c420a7bbcd0f7b93
[template-erp5] [template-erp5]
filename = instance-erp5.cfg.in filename = instance-erp5.cfg.in
md5sum = 2b91528d3a77a64714e4295a84c1d71b md5sum = 46c5d2173ab91f56b32e6eb2b544bc26
[template-zeo] [template-zeo]
filename = instance-zeo.cfg.in filename = instance-zeo.cfg.in
......
{% import "root_common" as root_common with context -%} {% import "root_common" as root_common with context -%}
{% import "caucase" as caucase with context %} {% import "caucase" as caucase with context %}
{% set frontend_dict = slapparameter_dict.get('frontend', {}) -%}
{% set has_frontend = frontend_dict.get('software-url', '') != '' -%}
{% set site_id = slapparameter_dict.get('site-id', 'erp5') -%} {% set site_id = slapparameter_dict.get('site-id', 'erp5') -%}
{% set inituser_login = slapparameter_dict.get('inituser-login', 'zope') -%} {% set inituser_login = slapparameter_dict.get('inituser-login', 'zope') -%}
{% set publish_dict = {'site-id': site_id, 'inituser-login': inituser_login} -%} {% set publish_dict = {'site-id': site_id, 'inituser-login': inituser_login} -%}
...@@ -375,31 +373,67 @@ config-command = {{ '${' ~ check_software_url_section_name ~ ':path}' }} ...@@ -375,31 +373,67 @@ config-command = {{ '${' ~ check_software_url_section_name ~ ':path}' }}
{% do zope_address_list_id_dict.__setitem__(zope_section_id, parameter_name) -%} {% do zope_address_list_id_dict.__setitem__(zope_section_id, parameter_name) -%}
{% do zope_family_parameter_dict.setdefault(family_name, []).append(parameter_name) -%} {% do zope_family_parameter_dict.setdefault(family_name, []).append(parameter_name) -%}
{% endfor -%} {% endfor -%}
{% if has_frontend -%} {% do publish_dict.__setitem__('family-' ~ family_name, '${request-balancer:connection-' ~ family_name ~ '}' ) -%}
{% set frontend_name = 'frontend-' ~ family_name -%} {% do publish_dict.__setitem__('family-' ~ family_name ~ '-v6', '${request-balancer:connection-' ~ family_name ~ '-v6}' ) -%}
{% do publish_dict.__setitem__('family-' ~ family_name, '${' ~ frontend_name ~ ':connection-site_url}' ) -%} {% endfor -%}
[{{ frontend_name }}]
{# We request a default frontend when exactly only one zope family exists #}
{% set frontend_parameter_dict = slapparameter_dict.get(
'frontend',
{'default': { 'zope-family': list(zope_family_dict)[0] }} if len(zope_family_dict) == 1 else {}
) -%}
[request-frontend-base]
<= request-common
recipe = slapos.cookbook:request
shared = true
return =
secure_access
{{ root_common.sla('frontend', default_to_same_computer=False) }}
{% for frontend_name, frontend_parameters in frontend_parameter_dict.items() -%}
{% set frontend_full_name = 'frontend-' ~ frontend_name -%}
{% set request_frontend_name = 'request-frontend-' ~ frontend_name -%}
{% set frontend_software_url = frontend_parameters.get('software-url', 'http://git.erp5.org/gitweb/slapos.git/blob_plain/HEAD:/software/apache-frontend/software.cfg') -%}
{% set frontend_software_type = frontend_parameters.get('software-type', '') -%}
{% set frontend_instance_parameters = frontend_parameters.get('instance-parameters', {}) -%}
{% if frontend_instance_parameters.setdefault('type', 'zope') == 'zope' -%}
{% do frontend_instance_parameters.setdefault('authenticate-to-backend', 'true') -%}
{% set zope_family_name = frontend_parameters['zope-family'] -%}
{% do assert(zope_family_name in zope_family_dict, 'Unknown family %s for frontend %s' % (zope_family_name, frontend_name)) -%}
{% do frontend_instance_parameters.setdefault('url', '${request-balancer:connection-' ~ zope_family_name ~ '-v6}') -%}
{% do frontend_instance_parameters.setdefault('path', frontend_parameters.get('internal-path', '/%(site-id)s') % {'site-id': site_id}) -%}
{% endif %}
[{{ request_frontend_name }}]
<= request-frontend-base <= request-frontend-base
name = {{ frontend_name }} name = {{ frontend_full_name }}
config-url = ${request-balancer:connection-{{ family_name }}-v6} software-url = {{ frontend_software_url }}
{% else -%} {% if frontend_software_type %}
{% do publish_dict.__setitem__('family-' ~ family_name, '${request-balancer:connection-' ~ family_name ~ '}' ) -%} software-type = {{ frontend_software_type }}
{% do publish_dict.__setitem__('family-' ~ family_name ~ '-v6', '${request-balancer:connection-' ~ family_name ~ '-v6}' ) -%} {% endif %}
{% endif -%} {{ root_common.sla(frontend_full_name, default_to_same_computer=False) }}
{% for name, value in frontend_instance_parameters.items() -%}
config-{{ name }} = {{ value }}
{% endfor -%}
{% set promise_frontend_section_name = 'promise-' ~ request_frontend_name %}
[{{ promise_frontend_section_name }}]
<= monitor-promise-base
promise = check_url_available
name = ${:_buildout_section_name_}.py
config-url = {{ "${" }}{{ request_frontend_name }}:connection-secure_access}
config-ignore-code = 1
config-allow-redirects = 0
{% do root_common.section(promise_frontend_section_name) -%}
{% do publish_dict.__setitem__('url-frontend-' ~ frontend_name, '${' ~ request_frontend_name ~ ':connection-secure_access}' ) -%}
{% endfor -%} {% endfor -%}
{% if has_jupyter -%} {% if has_jupyter -%}
{# request jupyter connected to balancer of proper zope family -#} {# request jupyter connected to balancer of proper zope family -#}
{{ request('jupyter', 'jupyter', 'jupyter', {}, key_config={'erp5-url': 'request-balancer:connection-' ~ jupyter_zope_family}) }} {{ request('jupyter', 'jupyter', 'jupyter', {}, key_config={'erp5-url': 'request-balancer:connection-' ~ jupyter_zope_family}) }}
{% if has_frontend -%}
[frontend-jupyter]
<= request-frontend-base
name = frontend-jupyter
config-url = ${request-jupyter:connection-url}
{# # override jupyter-url in publish_dict with frontend address -#}
{% do publish_dict.__setitem__('jupyter-url', '${frontend-jupyter:connection-site_url}') -%}
{% endif -%}
{%- endif %} {%- endif %}
{% if wcfs_enable -%} {% if wcfs_enable -%}
...@@ -463,31 +497,6 @@ config-url = ${request-jupyter:connection-url} ...@@ -463,31 +497,6 @@ config-url = ${request-jupyter:connection-url}
key_config=balancer_key_config_dict, key_config=balancer_key_config_dict,
) }} ) }}
[request-frontend-base]
{% if has_frontend -%}
<= request-common
recipe = slapos.cookbook:request
software-url = {{ dumps(frontend_dict['software-url']) }}
software-type = {{ dumps(frontend_dict.get('software-type', 'RootSoftwareInstance')) }}
{{ root_common.sla('frontend', True) }}
shared = true
{% set config_dict = {
'type': 'zope',
} -%}
{% if frontend_dict.get('domain') -%}
{% do config_dict.__setitem__('custom_domain', frontend_dict['domain']) -%}
{% endif -%}
{% if frontend_dict.get('virtualhostroot-http-port') -%}
{% do config_dict.__setitem__('virtualhostroot-http-port', frontend_dict['virtualhostroot-http-port']) -%}
{% endif -%}
{% if frontend_dict.get('virtualhostroot-https-port') -%}
{% do config_dict.__setitem__('virtualhostroot-https-port', frontend_dict['virtualhostroot-https-port']) -%}
{% endif -%}
{% for name, value in config_dict.items() -%}
config-{{ name }} = {{ value }}
{% endfor -%}
return = site_url
{% endif -%}
{% endif -%}{# if zope_partition_dict -#} {% endif -%}{# if zope_partition_dict -#}
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment