stack/erp5: also use backup cloudooo by default

We have two public cloudooo instances, https:///cloudooo.erp5.net/ and
https://cloudooo1.erp5.net/ but the software release uses only the first
one. This changes to use both instances by default, in a random order.
This happens for the ERP5 instance and also for the test runner.

This also includes a breaking change: the parameter is renamed from
`cloudooo-url` into `cloudooo-url-list` and instead of being a coma
delimited list of URLs, this is now a proper list of URLs.

Some obsolete code and comments about the old `erp5-cloudooo` partition
that used to part of ERP5 software release have also been removed.

On testing side, ZopeSkinsMixin._setUpClass is changed be made more
robust, by waiting that all activities are processed and using a new
xmlrpc client in every iteration ( to prevent issues like the one from
https://erp5js.nexedi.net/#/test_result_module/20230107-548523A7/10 )

software/erp5/instance-erp5-input-schema.json

@@ -145,7 +145,7 @@
       "type": "object"
     },
     "hosts-dict": {
-      "description": "Host entries to be used in addition to and/or overriding auto-generated ones (erp5-catalog-0, erp5-cloudooo, erp5-memcached-persistent, erp5-memcached-volatile and erp5-smtp)",
+      "description": "Host entries to be used in addition to and/or overriding auto-generated ones (erp5-catalog-0, erp5-memcached-persistent, erp5-memcached-volatile and erp5-smtp)",
       "patternProperties": {
         ".*": {
           "description": "An IP or domain name to which current entry will resolve",
@@ -294,11 +294,14 @@
       ],
       "type": "object"
     },
-    "cloudooo-url": {
-      "description": "Format conversion service URL",
-      "pattern": "^https?://",
-      "type": "string",
-      "format": "uri"
+    "cloudooo-url-list": {
+      "description": "Format conversion service URLs",
+      "type": "array",
+      "items": {
+        "pattern": "^https?://",
+        "type": "string",
+        "format": "uri"
+      }
     },
     "cloudooo-retry-count": {
       "description": "Define retry count for cloudooo in network error case in test",

software/erp5/test/test/test_erp5.py

@@ -28,6 +28,7 @@
 
 import contextlib
 import glob
+import http.client
 import json
 import os
 import shutil
@@ -450,17 +451,20 @@ class ZopeSkinsMixin:
   def _setUpClass(cls):
     super()._setUpClass()
     param_dict = cls.getRootPartitionConnectionParameterDict()
-    with cls.getXMLRPCClient() as erp5_xmlrpc_client:
-      # wait for ERP5 to be ready (TODO: this should probably be a promise)
-      for _ in range(120):
-        time.sleep(1)
+    # wait for ERP5 to be ready and have processed all activities
+    # from initial setup
+    for _ in range(120):
+      with cls.getXMLRPCClient() as erp5_xmlrpc_client:
         try:
-          erp5_xmlrpc_client.getTitle()
+          if erp5_xmlrpc_client.portal_activities.countMessage() == 0:
+            break
         except (xmlrpc.client.ProtocolError,
-                xmlrpc.client.Fault):
+                xmlrpc.client.Fault,
+                http.client.HTTPException):
           pass
-        else:
-          break
+      time.sleep(5)
+    else:
+      raise AssertionError("ERP5 is not ready")
 
   @classmethod
   def _getAuthenticatedZopeUrl(cls, path, family_name='default'):
@@ -924,3 +928,65 @@ class TestZopePublisherTimeout(ZopeSkinsMixin, ERP5InstanceTestCase):
         self._getAuthenticatedZopeUrl('ERP5Site_doSlowRequest', family_name='no-timeout'),
         verify=False,
         timeout=6)
+
+
+class TestCloudooo(ZopeSkinsMixin, ERP5InstanceTestCase):
+  """Test ERP5 can be instantiated with cloudooo parameters
+  """
+  __partition_reference__ = 'c'
+
+  @classmethod
+  def getInstanceParameterDict(cls):
+    return {
+        '_':
+        json.dumps({
+            'cloudooo-url-list': [
+              'https://cloudooo1.example.com/',
+              'https://cloudooo2.example.com/',
+            ],
+            'cloudooo-retry-count': 123,
+        })
+    }
+
+  def test_cloudooo_url_list_preference(self):
+    self.assertEqual(
+      requests.get(
+          self._getAuthenticatedZopeUrl(
+            'portal_preferences/getPreferredDocumentConversionServerUrlList'),
+          verify=False).text,
+      "['https://cloudooo1.example.com/', 'https://cloudooo2.example.com/']")
+
+  @unittest.expectedFailure # setting "retry" is not implemented
+  def test_cloudooo_retry_count_preference(self):
+    self.assertEqual(
+      requests.get(
+          self._getAuthenticatedZopeUrl(
+            'portal_preferences/getPreferredDocumentConversionServerRetry'),
+          verify=False).text,
+      "123")
+
+
+class TestCloudoooDefaultParameter(ZopeSkinsMixin, ERP5InstanceTestCase):
+  """Test default ERP5 cloudooo parameters
+  """
+  __partition_reference__ = 'cd'
+
+  def test_cloudooo_url_list_preference(self):
+    self.assertIn(
+      requests.get(
+          self._getAuthenticatedZopeUrl(
+            'portal_preferences/getPreferredDocumentConversionServerUrlList'),
+          verify=False).text,
+      [
+        "['https://cloudooo1.erp5.net/', 'https://cloudooo.erp5.net/']",
+        "['https://cloudooo.erp5.net/', 'https://cloudooo1.erp5.net/']",
+      ])
+
+  @unittest.expectedFailure # default value of "retry" does not match schema
+  def test_cloudooo_retry_count_preference(self):
+    self.assertEqual(
+      requests.get(
+          self._getAuthenticatedZopeUrl(
+            'portal_preferences/getPreferredDocumentConversionServerRetry'),
+          verify=False).text,
+      "2")

software/slapos-master/buildout.hash.cfg

@@ -14,7 +14,7 @@
 # not need these here).
 [template-erp5]
 filename = instance-erp5.cfg.in
-md5sum = 6d9b94b50ea46b0b98c9019d74780876
+md5sum = 2491969d49f8bf9b172e89b1c0e9d98e
 
 [template-balancer]
 filename = instance-balancer.cfg.in

software/slapos-master/instance-erp5.cfg.in

@@ -21,6 +21,10 @@
 {%   set test_runner_total_database_count = mariadb_test_database_amount %}
 {%   set test_runner_enabled = mariadb_test_database_amount > 0 %}
 {% endif -%}
+{# Backward compatibility for cloudooo-url #}
+{% if slapparameter_dict.get('cloudooo-url') -%}
+{%   do slapparameter_dict.setdefault('cloudooo-url-list', slapparameter_dict['cloudooo-url'].split(',')) %}
+{% endif -%}
 {% set test_runner_random_activity_priority = slapparameter_dict.get('test-runner', {}).get('random-activity-priority') -%}
 {% set monitor_base_url_dict = {} -%}
 {% set monitor_dict = slapparameter_dict.get('monitor', {}) %}
@@ -231,7 +235,7 @@ return =
 {% endif -%}
 config-bt5 = {{ dumps(slapparameter_dict.get('bt5', ' '.join(bt5_default_list))) }}
 config-bt5-repository-url = {{ dumps(slapparameter_dict.get('bt5-repository-url', local_bt5_repository)) }}
-config-cloudooo-url = {{ dumps(slapparameter_dict.get('cloudooo-url', default_cloudooo_url)) }}
+config-cloudooo-url-list = {{ dumps(slapparameter_dict.get('cloudooo-url-list', default_cloudooo_url_list)) }}
 config-caucase-url = {{ dumps(caucase_url) }}
 config-deadlock-debugger-password = ${publish-early:deadlock-debugger-password}
 config-developer-list = {{ dumps(slapparameter_dict.get('developer-list', [inituser_login])) }}

stack/erp5/buildout.cfg

@@ -174,7 +174,7 @@ context =
     key dash_location dash:location
     key bash_location bash:location
     key dcron_location dcron:location
-    key default_cloudooo_url erp5-defaults:cloudooo-connection-url
+    key default_cloudooo_url_list erp5-defaults:cloudooo-connection-url-list
     key erp5_location erp5:location
     key findutils_location findutils:location
     key gzip_location gzip:location
@@ -331,7 +331,11 @@ repository_id_list = erp5 erp5-bin erp5-doc
 
 # ERP5 defaults, which can be overridden in inheriting recipes (e.g. wendelin)
 [erp5-defaults]
-cloudooo-connection-url = https://cloudooo.erp5.net/
+# Default cloudooo is https://cloudooo.erp5.net/ and  https://cloudooo1.erp5.net/ in random order.
+# The random order is applied in instance profile
+cloudooo-connection-url-list =
+  https://cloudooo.erp5.net/
+  https://cloudooo1.erp5.net/
 # Jupyter is by default disabled in ERP5
 jupyter-enable-default = false
 # WCFS is by default disabled in ERP5

stack/erp5/buildout.hash.cfg

@@ -70,11 +70,11 @@ md5sum = b95084ae9eed95a68eada45e28ef0c04
 
 [template]
 filename = instance.cfg.in
-md5sum = 74979d6743cdc7ce3d2f17680115458d
+md5sum = 3f7b28085ceff321a3cb785db60f7c3e
 
 [template-erp5]
 filename = instance-erp5.cfg.in
-md5sum = 52dfc0c9a4547b1e928707ef5caeecdb
+md5sum = 30a1e738a8211887e75a5e75820e5872
 
 [template-zeo]
 filename = instance-zeo.cfg.in
@@ -86,7 +86,7 @@ md5sum = 0ac4b74436f554cd677f19275d18d880
 
 [template-zope]
 filename = instance-zope.cfg.in
-md5sum = 57a19c90d281b8cbf3e36207db48f0bd
+md5sum = 7d3b3769b60f0cc2883beeb05cdf82d7
 
 [template-balancer]
 filename = instance-balancer.cfg.in

stack/erp5/instance-erp5.cfg.in

@@ -21,6 +21,10 @@
 {%   set test_runner_total_database_count = mariadb_test_database_amount %}
 {%   set test_runner_enabled = mariadb_test_database_amount > 0 %}
 {% endif -%}
+{# Backward compatibility for cloudooo-url #}
+{% if slapparameter_dict.get('cloudooo-url') -%}
+{%   do slapparameter_dict.setdefault('cloudooo-url-list', slapparameter_dict['cloudooo-url'].split(',')) %}
+{% endif -%}
 {% set test_runner_random_activity_priority = slapparameter_dict.get('test-runner', {}).get('random-activity-priority') -%}
 {% set monitor_base_url_dict = {} -%}
 {% set monitor_dict = slapparameter_dict.get('monitor', {}) %}
@@ -234,7 +238,7 @@ return =
 {% endif -%}
 config-bt5 = {{ dumps(slapparameter_dict.get('bt5', ' '.join(bt5_default_list))) }}
 config-bt5-repository-url = {{ dumps(slapparameter_dict.get('bt5-repository-url', local_bt5_repository)) }}
-config-cloudooo-url = {{ dumps(slapparameter_dict.get('cloudooo-url', default_cloudooo_url)) }}
+config-cloudooo-url-list = {{ dumps(slapparameter_dict.get('cloudooo-url-list', default_cloudooo_url_list)) }}
 config-caucase-url = {{ dumps(caucase_url) }}
 config-deadlock-debugger-password = ${publish-early:deadlock-debugger-password}
 config-developer-list = {{ dumps(slapparameter_dict.get('developer-list', [inituser_login])) }}

stack/erp5/instance-zope.cfg.in

@@ -33,7 +33,6 @@ partition. No more (undefined result), no less (IndexError).
 {% for alias, url in (
     ('erp5-memcached-volatile', slapparameter_dict['memcached-url']),
     ('erp5-memcached-persistent', slapparameter_dict['kumofs-url']),
-    ('erp5-cloudooo', slapparameter_dict['cloudooo-url']),
     ('erp5-smtp', slapparameter_dict['smtp-url']),
   ) -%}
 {%   set parsed_url = urllib_parse.urlparse(url) -%}
@@ -282,12 +281,7 @@ bt5 = {{ slapparameter_dict['bt5'] }}
 bt5-repository-url = {{ slapparameter_dict['bt5-repository-url'] }}
 id-store-interval = {{ dumps(slapparameter_dict['id-store-interval']) }}
 home = ${buildout:directory}
-# We only want to change the hostname to 'erp5-cloudooo' if we use the internal
-# cloudooo. We plan to remove the ability to have an internal one, so this
-# heuristic is enough.
-{%  set cloudooo = urllib_parse.urlsplit(slapparameter_dict['cloudooo-url']) -%}
-cloudooo-url = {{ (cloudooo if cloudooo.port == None else
-  cloudooo._replace(netloc='erp5-cloudooo:%s' % cloudooo.port)).geturl() }}
+cloudooo-url = {{ ",".join(slapparameter_dict['cloudooo-url-list']) }}
 
 {% endif -%}
 developer-list = {{ dumps(slapparameter_dict['developer-list']) }}
@@ -506,11 +500,8 @@ wrapper-path = ${directory:bin}/${:command-name}.real
 command-line =
   '{{ parameter_dict['bin-directory'] }}/${:command-name}'
   ${:command-line-extra}
-  --conversion_server_url={{ slapparameter_dict['cloudooo-url'] }}
+  --conversion_server_url={{ ",".join(slapparameter_dict['cloudooo-url-list']) }}
   --conversion_server_retry_count={{ slapparameter_dict.get('cloudooo-retry-count', 2) }}
-{#- BBB: We still have test suites that only accept the following 2 options. #}
-  --conversion_server_hostname=erp5-cloudooo
-  --conversion_server_port={{ port_dict['erp5-cloudooo'] }}
   --volatile_memcached_server_hostname=erp5-memcached-volatile
   --volatile_memcached_server_port={{ port_dict['erp5-memcached-volatile'] }}
   --persistent_memcached_server_hostname=erp5-memcached-persistent

stack/erp5/instance.cfg.in

@@ -26,8 +26,21 @@ extra-context =
     section parameter_dict dynamic-template-postfix-parameters
     import urllib urllib
 
+[default-cloudooo-url-list]
+recipe = slapos.recipe.build
+default-url-list =
+  {{ default_cloudooo_url_list | indent }}
+init =
+  # Expose the default cloudooo URLs, sorted using a pseudo-random seed,
+  # so that the same instance keep the same order
+  import six, random
+  seed = repr(sorted(six.iteritems(self.buildout['slap-connection'])))
+  default_cloudooo_url_list = [url.strip() for url in self.options['default-url-list'].splitlines()]
+  random.Random(seed).shuffle(default_cloudooo_url_list)
+  self.options['url-list'] = default_cloudooo_url_list
+
 [dynamic-template-erp5-parameters]
-default-cloudooo-url =  {{ dumps(default_cloudooo_url) }}
+default-cloudooo-url-list = ${default-cloudooo-url-list:url-list}
 jupyter-enable-default = {{ jupyter_enable_default }}
 wcfs-enable-default = {{ wcfs_enable_default }}
 local-bt5-repository = {{ ' '.join(local_bt5_repository.split()) }}
@@ -42,7 +55,7 @@ template-zodb-base = {{ template_zodb_base }}
 url = {{ template_erp5 }}
 filename = instance-erp5.cfg
 extra-context =
-    key default_cloudooo_url dynamic-template-erp5-parameters:default-cloudooo-url
+    key default_cloudooo_url_list dynamic-template-erp5-parameters:default-cloudooo-url-list
     key jupyter_enable_default dynamic-template-erp5-parameters:jupyter-enable-default
     key wcfs_enable_default dynamic-template-erp5-parameters:wcfs-enable-default
     key local_bt5_repository dynamic-template-erp5-parameters:local-bt5-repository