Commit a729a677 authored by Jérome Perrin's avatar Jérome Perrin

stack/erp5: also use backup cloudooo by default

We have two public cloudooo instances, https:///cloudooo.erp5.net/ and
https://cloudooo1.erp5.net/ but the software release uses only the first
one. This changes to use both instances by default, in a random order.
This happens for the ERP5 instance and also for the test runner.

This also includes a breaking change: the parameter is renamed from
`cloudooo-url` into `cloudooo-url-list` and instead of being a coma
delimited list of URLs, this is now a proper list of URLs.

Some obsolete code and comments about the old `erp5-cloudooo` partition
that used to part of ERP5 software release have also been removed.

On testing side, ZopeSkinsMixin._setUpClass is changed be made more
robust, by waiting that all activities are processed and using a new
xmlrpc client in every iteration ( to prevent issues like the one from
https://erp5js.nexedi.net/#/test_result_module/20230107-548523A7/10 )
parent 99b209eb
...@@ -145,7 +145,7 @@ ...@@ -145,7 +145,7 @@
"type": "object" "type": "object"
}, },
"hosts-dict": { "hosts-dict": {
"description": "Host entries to be used in addition to and/or overriding auto-generated ones (erp5-catalog-0, erp5-cloudooo, erp5-memcached-persistent, erp5-memcached-volatile and erp5-smtp)", "description": "Host entries to be used in addition to and/or overriding auto-generated ones (erp5-catalog-0, erp5-memcached-persistent, erp5-memcached-volatile and erp5-smtp)",
"patternProperties": { "patternProperties": {
".*": { ".*": {
"description": "An IP or domain name to which current entry will resolve", "description": "An IP or domain name to which current entry will resolve",
...@@ -294,11 +294,14 @@ ...@@ -294,11 +294,14 @@
], ],
"type": "object" "type": "object"
}, },
"cloudooo-url": { "cloudooo-url-list": {
"description": "Format conversion service URL", "description": "Format conversion service URLs",
"type": "array",
"items": {
"pattern": "^https?://", "pattern": "^https?://",
"type": "string", "type": "string",
"format": "uri" "format": "uri"
}
}, },
"cloudooo-retry-count": { "cloudooo-retry-count": {
"description": "Define retry count for cloudooo in network error case in test", "description": "Define retry count for cloudooo in network error case in test",
......
...@@ -28,6 +28,7 @@ ...@@ -28,6 +28,7 @@
import contextlib import contextlib
import glob import glob
import http.client
import json import json
import os import os
import shutil import shutil
...@@ -450,17 +451,20 @@ class ZopeSkinsMixin: ...@@ -450,17 +451,20 @@ class ZopeSkinsMixin:
def _setUpClass(cls): def _setUpClass(cls):
super()._setUpClass() super()._setUpClass()
param_dict = cls.getRootPartitionConnectionParameterDict() param_dict = cls.getRootPartitionConnectionParameterDict()
with cls.getXMLRPCClient() as erp5_xmlrpc_client: # wait for ERP5 to be ready and have processed all activities
# wait for ERP5 to be ready (TODO: this should probably be a promise) # from initial setup
for _ in range(120): for _ in range(120):
time.sleep(1) with cls.getXMLRPCClient() as erp5_xmlrpc_client:
try: try:
erp5_xmlrpc_client.getTitle() if erp5_xmlrpc_client.portal_activities.countMessage() == 0:
break
except (xmlrpc.client.ProtocolError, except (xmlrpc.client.ProtocolError,
xmlrpc.client.Fault): xmlrpc.client.Fault,
http.client.HTTPException):
pass pass
time.sleep(5)
else: else:
break raise AssertionError("ERP5 is not ready")
@classmethod @classmethod
def _getAuthenticatedZopeUrl(cls, path, family_name='default'): def _getAuthenticatedZopeUrl(cls, path, family_name='default'):
...@@ -924,3 +928,65 @@ class TestZopePublisherTimeout(ZopeSkinsMixin, ERP5InstanceTestCase): ...@@ -924,3 +928,65 @@ class TestZopePublisherTimeout(ZopeSkinsMixin, ERP5InstanceTestCase):
self._getAuthenticatedZopeUrl('ERP5Site_doSlowRequest', family_name='no-timeout'), self._getAuthenticatedZopeUrl('ERP5Site_doSlowRequest', family_name='no-timeout'),
verify=False, verify=False,
timeout=6) timeout=6)
class TestCloudooo(ZopeSkinsMixin, ERP5InstanceTestCase):
"""Test ERP5 can be instantiated with cloudooo parameters
"""
__partition_reference__ = 'c'
@classmethod
def getInstanceParameterDict(cls):
return {
'_':
json.dumps({
'cloudooo-url-list': [
'https://cloudooo1.example.com/',
'https://cloudooo2.example.com/',
],
'cloudooo-retry-count': 123,
})
}
def test_cloudooo_url_list_preference(self):
self.assertEqual(
requests.get(
self._getAuthenticatedZopeUrl(
'portal_preferences/getPreferredDocumentConversionServerUrlList'),
verify=False).text,
"['https://cloudooo1.example.com/', 'https://cloudooo2.example.com/']")
@unittest.expectedFailure # setting "retry" is not implemented
def test_cloudooo_retry_count_preference(self):
self.assertEqual(
requests.get(
self._getAuthenticatedZopeUrl(
'portal_preferences/getPreferredDocumentConversionServerRetry'),
verify=False).text,
"123")
class TestCloudoooDefaultParameter(ZopeSkinsMixin, ERP5InstanceTestCase):
"""Test default ERP5 cloudooo parameters
"""
__partition_reference__ = 'cd'
def test_cloudooo_url_list_preference(self):
self.assertIn(
requests.get(
self._getAuthenticatedZopeUrl(
'portal_preferences/getPreferredDocumentConversionServerUrlList'),
verify=False).text,
[
"['https://cloudooo1.erp5.net/', 'https://cloudooo.erp5.net/']",
"['https://cloudooo.erp5.net/', 'https://cloudooo1.erp5.net/']",
])
@unittest.expectedFailure # default value of "retry" does not match schema
def test_cloudooo_retry_count_preference(self):
self.assertEqual(
requests.get(
self._getAuthenticatedZopeUrl(
'portal_preferences/getPreferredDocumentConversionServerRetry'),
verify=False).text,
"2")
...@@ -14,7 +14,7 @@ ...@@ -14,7 +14,7 @@
# not need these here). # not need these here).
[template-erp5] [template-erp5]
filename = instance-erp5.cfg.in filename = instance-erp5.cfg.in
md5sum = 6d9b94b50ea46b0b98c9019d74780876 md5sum = 2491969d49f8bf9b172e89b1c0e9d98e
[template-balancer] [template-balancer]
filename = instance-balancer.cfg.in filename = instance-balancer.cfg.in
......
...@@ -21,6 +21,10 @@ ...@@ -21,6 +21,10 @@
{% set test_runner_total_database_count = mariadb_test_database_amount %} {% set test_runner_total_database_count = mariadb_test_database_amount %}
{% set test_runner_enabled = mariadb_test_database_amount > 0 %} {% set test_runner_enabled = mariadb_test_database_amount > 0 %}
{% endif -%} {% endif -%}
{# Backward compatibility for cloudooo-url #}
{% if slapparameter_dict.get('cloudooo-url') -%}
{% do slapparameter_dict.setdefault('cloudooo-url-list', slapparameter_dict['cloudooo-url'].split(',')) %}
{% endif -%}
{% set test_runner_random_activity_priority = slapparameter_dict.get('test-runner', {}).get('random-activity-priority') -%} {% set test_runner_random_activity_priority = slapparameter_dict.get('test-runner', {}).get('random-activity-priority') -%}
{% set monitor_base_url_dict = {} -%} {% set monitor_base_url_dict = {} -%}
{% set monitor_dict = slapparameter_dict.get('monitor', {}) %} {% set monitor_dict = slapparameter_dict.get('monitor', {}) %}
...@@ -231,7 +235,7 @@ return = ...@@ -231,7 +235,7 @@ return =
{% endif -%} {% endif -%}
config-bt5 = {{ dumps(slapparameter_dict.get('bt5', ' '.join(bt5_default_list))) }} config-bt5 = {{ dumps(slapparameter_dict.get('bt5', ' '.join(bt5_default_list))) }}
config-bt5-repository-url = {{ dumps(slapparameter_dict.get('bt5-repository-url', local_bt5_repository)) }} config-bt5-repository-url = {{ dumps(slapparameter_dict.get('bt5-repository-url', local_bt5_repository)) }}
config-cloudooo-url = {{ dumps(slapparameter_dict.get('cloudooo-url', default_cloudooo_url)) }} config-cloudooo-url-list = {{ dumps(slapparameter_dict.get('cloudooo-url-list', default_cloudooo_url_list)) }}
config-caucase-url = {{ dumps(caucase_url) }} config-caucase-url = {{ dumps(caucase_url) }}
config-deadlock-debugger-password = ${publish-early:deadlock-debugger-password} config-deadlock-debugger-password = ${publish-early:deadlock-debugger-password}
config-developer-list = {{ dumps(slapparameter_dict.get('developer-list', [inituser_login])) }} config-developer-list = {{ dumps(slapparameter_dict.get('developer-list', [inituser_login])) }}
......
...@@ -174,7 +174,7 @@ context = ...@@ -174,7 +174,7 @@ context =
key dash_location dash:location key dash_location dash:location
key bash_location bash:location key bash_location bash:location
key dcron_location dcron:location key dcron_location dcron:location
key default_cloudooo_url erp5-defaults:cloudooo-connection-url key default_cloudooo_url_list erp5-defaults:cloudooo-connection-url-list
key erp5_location erp5:location key erp5_location erp5:location
key findutils_location findutils:location key findutils_location findutils:location
key gzip_location gzip:location key gzip_location gzip:location
...@@ -331,7 +331,11 @@ repository_id_list = erp5 erp5-bin erp5-doc ...@@ -331,7 +331,11 @@ repository_id_list = erp5 erp5-bin erp5-doc
# ERP5 defaults, which can be overridden in inheriting recipes (e.g. wendelin) # ERP5 defaults, which can be overridden in inheriting recipes (e.g. wendelin)
[erp5-defaults] [erp5-defaults]
cloudooo-connection-url = https://cloudooo.erp5.net/ # Default cloudooo is https://cloudooo.erp5.net/ and https://cloudooo1.erp5.net/ in random order.
# The random order is applied in instance profile
cloudooo-connection-url-list =
https://cloudooo.erp5.net/
https://cloudooo1.erp5.net/
# Jupyter is by default disabled in ERP5 # Jupyter is by default disabled in ERP5
jupyter-enable-default = false jupyter-enable-default = false
# WCFS is by default disabled in ERP5 # WCFS is by default disabled in ERP5
......
...@@ -70,11 +70,11 @@ md5sum = b95084ae9eed95a68eada45e28ef0c04 ...@@ -70,11 +70,11 @@ md5sum = b95084ae9eed95a68eada45e28ef0c04
[template] [template]
filename = instance.cfg.in filename = instance.cfg.in
md5sum = 74979d6743cdc7ce3d2f17680115458d md5sum = 3f7b28085ceff321a3cb785db60f7c3e
[template-erp5] [template-erp5]
filename = instance-erp5.cfg.in filename = instance-erp5.cfg.in
md5sum = 52dfc0c9a4547b1e928707ef5caeecdb md5sum = 30a1e738a8211887e75a5e75820e5872
[template-zeo] [template-zeo]
filename = instance-zeo.cfg.in filename = instance-zeo.cfg.in
...@@ -86,7 +86,7 @@ md5sum = 0ac4b74436f554cd677f19275d18d880 ...@@ -86,7 +86,7 @@ md5sum = 0ac4b74436f554cd677f19275d18d880
[template-zope] [template-zope]
filename = instance-zope.cfg.in filename = instance-zope.cfg.in
md5sum = 57a19c90d281b8cbf3e36207db48f0bd md5sum = 7d3b3769b60f0cc2883beeb05cdf82d7
[template-balancer] [template-balancer]
filename = instance-balancer.cfg.in filename = instance-balancer.cfg.in
......
...@@ -21,6 +21,10 @@ ...@@ -21,6 +21,10 @@
{% set test_runner_total_database_count = mariadb_test_database_amount %} {% set test_runner_total_database_count = mariadb_test_database_amount %}
{% set test_runner_enabled = mariadb_test_database_amount > 0 %} {% set test_runner_enabled = mariadb_test_database_amount > 0 %}
{% endif -%} {% endif -%}
{# Backward compatibility for cloudooo-url #}
{% if slapparameter_dict.get('cloudooo-url') -%}
{% do slapparameter_dict.setdefault('cloudooo-url-list', slapparameter_dict['cloudooo-url'].split(',')) %}
{% endif -%}
{% set test_runner_random_activity_priority = slapparameter_dict.get('test-runner', {}).get('random-activity-priority') -%} {% set test_runner_random_activity_priority = slapparameter_dict.get('test-runner', {}).get('random-activity-priority') -%}
{% set monitor_base_url_dict = {} -%} {% set monitor_base_url_dict = {} -%}
{% set monitor_dict = slapparameter_dict.get('monitor', {}) %} {% set monitor_dict = slapparameter_dict.get('monitor', {}) %}
...@@ -234,7 +238,7 @@ return = ...@@ -234,7 +238,7 @@ return =
{% endif -%} {% endif -%}
config-bt5 = {{ dumps(slapparameter_dict.get('bt5', ' '.join(bt5_default_list))) }} config-bt5 = {{ dumps(slapparameter_dict.get('bt5', ' '.join(bt5_default_list))) }}
config-bt5-repository-url = {{ dumps(slapparameter_dict.get('bt5-repository-url', local_bt5_repository)) }} config-bt5-repository-url = {{ dumps(slapparameter_dict.get('bt5-repository-url', local_bt5_repository)) }}
config-cloudooo-url = {{ dumps(slapparameter_dict.get('cloudooo-url', default_cloudooo_url)) }} config-cloudooo-url-list = {{ dumps(slapparameter_dict.get('cloudooo-url-list', default_cloudooo_url_list)) }}
config-caucase-url = {{ dumps(caucase_url) }} config-caucase-url = {{ dumps(caucase_url) }}
config-deadlock-debugger-password = ${publish-early:deadlock-debugger-password} config-deadlock-debugger-password = ${publish-early:deadlock-debugger-password}
config-developer-list = {{ dumps(slapparameter_dict.get('developer-list', [inituser_login])) }} config-developer-list = {{ dumps(slapparameter_dict.get('developer-list', [inituser_login])) }}
......
...@@ -33,7 +33,6 @@ partition. No more (undefined result), no less (IndexError). ...@@ -33,7 +33,6 @@ partition. No more (undefined result), no less (IndexError).
{% for alias, url in ( {% for alias, url in (
('erp5-memcached-volatile', slapparameter_dict['memcached-url']), ('erp5-memcached-volatile', slapparameter_dict['memcached-url']),
('erp5-memcached-persistent', slapparameter_dict['kumofs-url']), ('erp5-memcached-persistent', slapparameter_dict['kumofs-url']),
('erp5-cloudooo', slapparameter_dict['cloudooo-url']),
('erp5-smtp', slapparameter_dict['smtp-url']), ('erp5-smtp', slapparameter_dict['smtp-url']),
) -%} ) -%}
{% set parsed_url = urllib_parse.urlparse(url) -%} {% set parsed_url = urllib_parse.urlparse(url) -%}
...@@ -282,12 +281,7 @@ bt5 = {{ slapparameter_dict['bt5'] }} ...@@ -282,12 +281,7 @@ bt5 = {{ slapparameter_dict['bt5'] }}
bt5-repository-url = {{ slapparameter_dict['bt5-repository-url'] }} bt5-repository-url = {{ slapparameter_dict['bt5-repository-url'] }}
id-store-interval = {{ dumps(slapparameter_dict['id-store-interval']) }} id-store-interval = {{ dumps(slapparameter_dict['id-store-interval']) }}
home = ${buildout:directory} home = ${buildout:directory}
# We only want to change the hostname to 'erp5-cloudooo' if we use the internal cloudooo-url = {{ ",".join(slapparameter_dict['cloudooo-url-list']) }}
# cloudooo. We plan to remove the ability to have an internal one, so this
# heuristic is enough.
{% set cloudooo = urllib_parse.urlsplit(slapparameter_dict['cloudooo-url']) -%}
cloudooo-url = {{ (cloudooo if cloudooo.port == None else
cloudooo._replace(netloc='erp5-cloudooo:%s' % cloudooo.port)).geturl() }}
{% endif -%} {% endif -%}
developer-list = {{ dumps(slapparameter_dict['developer-list']) }} developer-list = {{ dumps(slapparameter_dict['developer-list']) }}
...@@ -506,11 +500,8 @@ wrapper-path = ${directory:bin}/${:command-name}.real ...@@ -506,11 +500,8 @@ wrapper-path = ${directory:bin}/${:command-name}.real
command-line = command-line =
'{{ parameter_dict['bin-directory'] }}/${:command-name}' '{{ parameter_dict['bin-directory'] }}/${:command-name}'
${:command-line-extra} ${:command-line-extra}
--conversion_server_url={{ slapparameter_dict['cloudooo-url'] }} --conversion_server_url={{ ",".join(slapparameter_dict['cloudooo-url-list']) }}
--conversion_server_retry_count={{ slapparameter_dict.get('cloudooo-retry-count', 2) }} --conversion_server_retry_count={{ slapparameter_dict.get('cloudooo-retry-count', 2) }}
{#- BBB: We still have test suites that only accept the following 2 options. #}
--conversion_server_hostname=erp5-cloudooo
--conversion_server_port={{ port_dict['erp5-cloudooo'] }}
--volatile_memcached_server_hostname=erp5-memcached-volatile --volatile_memcached_server_hostname=erp5-memcached-volatile
--volatile_memcached_server_port={{ port_dict['erp5-memcached-volatile'] }} --volatile_memcached_server_port={{ port_dict['erp5-memcached-volatile'] }}
--persistent_memcached_server_hostname=erp5-memcached-persistent --persistent_memcached_server_hostname=erp5-memcached-persistent
......
...@@ -26,8 +26,21 @@ extra-context = ...@@ -26,8 +26,21 @@ extra-context =
section parameter_dict dynamic-template-postfix-parameters section parameter_dict dynamic-template-postfix-parameters
import urllib urllib import urllib urllib
[default-cloudooo-url-list]
recipe = slapos.recipe.build
default-url-list =
{{ default_cloudooo_url_list | indent }}
init =
# Expose the default cloudooo URLs, sorted using a pseudo-random seed,
# so that the same instance keep the same order
import six, random
seed = repr(sorted(six.iteritems(self.buildout['slap-connection'])))
default_cloudooo_url_list = [url.strip() for url in self.options['default-url-list'].splitlines()]
random.Random(seed).shuffle(default_cloudooo_url_list)
self.options['url-list'] = default_cloudooo_url_list
[dynamic-template-erp5-parameters] [dynamic-template-erp5-parameters]
default-cloudooo-url = {{ dumps(default_cloudooo_url) }} default-cloudooo-url-list = ${default-cloudooo-url-list:url-list}
jupyter-enable-default = {{ jupyter_enable_default }} jupyter-enable-default = {{ jupyter_enable_default }}
wcfs-enable-default = {{ wcfs_enable_default }} wcfs-enable-default = {{ wcfs_enable_default }}
local-bt5-repository = {{ ' '.join(local_bt5_repository.split()) }} local-bt5-repository = {{ ' '.join(local_bt5_repository.split()) }}
...@@ -42,7 +55,7 @@ template-zodb-base = {{ template_zodb_base }} ...@@ -42,7 +55,7 @@ template-zodb-base = {{ template_zodb_base }}
url = {{ template_erp5 }} url = {{ template_erp5 }}
filename = instance-erp5.cfg filename = instance-erp5.cfg
extra-context = extra-context =
key default_cloudooo_url dynamic-template-erp5-parameters:default-cloudooo-url key default_cloudooo_url_list dynamic-template-erp5-parameters:default-cloudooo-url-list
key jupyter_enable_default dynamic-template-erp5-parameters:jupyter-enable-default key jupyter_enable_default dynamic-template-erp5-parameters:jupyter-enable-default
key wcfs_enable_default dynamic-template-erp5-parameters:wcfs-enable-default key wcfs_enable_default dynamic-template-erp5-parameters:wcfs-enable-default
key local_bt5_repository dynamic-template-erp5-parameters:local-bt5-repository key local_bt5_repository dynamic-template-erp5-parameters:local-bt5-repository
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment