ERP5 Software Release upgrade test + restart zopes on conf change

Introduce a new "software release upgrade test", which tests a scenario of keating
an ERP5 instance with an old version of software release and then update it to
current version.

To support this test and because we are using this technique for most of slapos
softwares nowadays, configure zopes to restart automatically when their
configuration change.

See merge request nexedi/slapos!875

software/erp5/upgrade_test/README.md

+Upgrade tests for ERP5 software release

software/erp5/upgrade_test/setup.py

+##############################################################################
+#
+# Copyright (c) 2020 Nexedi SA and Contributors. All Rights Reserved.
+#
+# WARNING: This program as such is intended to be used by professional
+# programmers who take the whole responsibility of assessing all potential
+# consequences resulting from its eventual inadequacies and bugs
+# End users who are looking for a ready-to-use solution with commercial
+# guarantees and support are strongly adviced to contract a Free Software
+# Service Company
+#
+# This program is Free Software; you can redistribute it and/or
+# modify it under the terms of the GNU General Public License
+# as published by the Free Software Foundation; either version 3
+# of the License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA  02111-1307, USA.
+#
+##############################################################################
+from setuptools import setup, find_packages
+
+version = '0.0.1.dev0'
+name = 'slapos.test.upgrade_erp5'
+with open("README.md") as f:
+  long_description = f.read()
+
+setup(name=name,
+      version=version,
+      description="Upgrade test for SlapOS' ERP5 software release",
+      long_description=long_description,
+      long_description_content_type='text/markdown',
+      maintainer="Nexedi",
+      maintainer_email="info@nexedi.com",
+      url="https://lab.nexedi.com/nexedi/slapos",
+      packages=find_packages(),
+      install_requires=[
+        'slapos.core',
+        'supervisor',
+        'slapos.libnetworkcache',
+        'typing; python_version<"3"',
+      ],
+      test_suite='test',
+    )

software/erp5/upgrade_test/test.py

+##############################################################################
+#
+# Copyright (c) 2020 Nexedi SA and Contributors. All Rights Reserved.
+#
+# WARNING: This program as such is intended to be used by professional
+# programmers who take the whole responsibility of assessing all potential
+# consequences resulting from its eventual inadequacies and bugs
+# End users who are looking for a ready-to-use solution with commercial
+# guarantees and support are strongly adviced to contract a Free Software
+# Service Company
+#
+# This program is Free Software; you can redistribute it and/or
+# modify it under the terms of the GNU General Public License
+# as published by the Free Software Foundation; either version 3
+# of the License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA  02111-1307, USA.
+#
+##############################################################################
+
+import glob
+import json
+import os
+import tempfile
+import time
+
+import requests
+import urlparse
+
+from slapos.testing.testcase import makeModuleSetUpAndTestCaseClass
+from slapos.testing.testcase import installSoftwareUrlList
+from slapos.testing.testcase import SlapOSNodeCommandError
+from slapos.grid.utils import md5digest
+
+old_software_release_url = 'https://lab.nexedi.com/nexedi/slapos/raw/1.0.167/software/erp5/software.cfg'
+new_software_release_url = os.path.abspath(
+    os.path.join(os.path.dirname(__file__), '..', 'software.cfg'))
+
+_, SlapOSInstanceTestCase = makeModuleSetUpAndTestCaseClass(
+    old_software_release_url, software_id="upgrade_erp5")
+
+
+def setUpModule():
+  installSoftwareUrlList(
+      SlapOSInstanceTestCase,
+      [old_software_release_url, new_software_release_url],
+      debug=SlapOSInstanceTestCase._debug,
+  )
+
+
+class ERP5UpgradeTestCase(SlapOSInstanceTestCase):
+  # use short partition names for unix sockets
+  __partition_reference__ = 'u'
+
+  @classmethod
+  def setUpOldInstance(cls):
+    """setUp hook executed while to old instance is running, before update
+    """
+    pass
+
+  _current_software_url = old_software_release_url
+
+  @classmethod
+  def getSoftwareURL(cls):
+    return cls._current_software_url
+
+  @classmethod
+  def setUpClass(cls):
+    # request and instanciate with old software url
+    super(ERP5UpgradeTestCase, cls).setUpClass()
+
+    cls.setUpOldInstance()
+
+    # request instance on new software
+    cls._current_software_url = new_software_release_url
+    cls.logger.debug('requesting instance on new software')
+    cls.requestDefaultInstance()
+
+    # wait for slapos node instance
+    snapshot_name = "{}.{}.setUpClass new instance".format(
+        cls.__module__, cls.__name__)
+    try:
+      if cls._debug and cls.instance_max_retry:
+        try:
+          cls.slap.waitForInstance(max_retry=cls.instance_max_retry - 1)
+        except SlapOSNodeCommandError:
+          cls.slap.waitForInstance(debug=True)
+      else:
+        cls.slap.waitForInstance(max_retry=cls.instance_max_retry,
+                                 debug=cls._debug)
+      cls.logger.debug("instance on new software done")
+    except BaseException:
+      cls.logger.exception("Error during instance on new software")
+      cls._storeSystemSnapshot(snapshot_name)
+      cls._cleanup(snapshot_name)
+      cls.setUp = lambda self: self.fail('Setup Class failed.')
+      raise
+    else:
+      cls._storeSystemSnapshot(snapshot_name)
+
+    cls.computer_partition = cls.requestDefaultInstance()
+
+
+class TestERP5Upgrade(ERP5UpgradeTestCase):
+  @classmethod
+  def setUpOldInstance(cls):
+    cls._default_instance_old_parameter_dict = json.loads(
+        cls.computer_partition.getConnectionParameterDict()['_'])
+
+  def test_published_url_is_same(self):
+    default_instance_new_parameter_dict = json.loads(
+        self.computer_partition.getConnectionParameterDict()['_'])
+    self.assertEqual(
+        default_instance_new_parameter_dict['family-default-v6'],
+        self._default_instance_old_parameter_dict['family-default-v6'],
+    )
+
+  def test_published_url_is_reachable(self):
+    default_instance_new_parameter_dict = json.loads(
+        self.computer_partition.getConnectionParameterDict()['_'])
+
+    # get certificate from caucase
+    with tempfile.NamedTemporaryFile(
+        prefix="ca.crt.pem",
+        mode="w",
+        delete=False,
+    ) as ca_cert:
+      ca_cert.write(
+          requests.get(
+              urlparse.urljoin(
+                  default_instance_new_parameter_dict['caucase-http-url'],
+                  '/cas/crt/ca.crt.pem',
+              )).text)
+      ca_cert.flush()
+
+      # use a session to retry on failures, when ERP5 is not ready.
+      # (see also TestPublishedURLIsReachableMixin)
+      session = requests.Session()
+      session.mount(
+        default_instance_new_parameter_dict['family-default-v6'],
+        requests.adapters.HTTPAdapter(
+            max_retries=requests.packages.urllib3.util.retry.Retry(
+                total=60,
+                backoff_factor=.5,
+                status_forcelist=(404, 500, 503))))
+
+      session.get(
+            '{}/{}/login_form'.format(
+                default_instance_new_parameter_dict['family-default-v6'],
+                default_instance_new_parameter_dict['site-id'],
+            ),
+            verify=False,
+            # TODO: we don't use caucase yet here.
+            # verify=ca_cert.name,
+        ).raise_for_status()
+
+
+  def test_all_instances_use_new_software_release(self):
+    self.assertEqual(
+        {
+            os.path.basename(os.readlink(sr))
+            for sr in glob.glob(
+                os.path.join(
+                    self.slap.instance_directory,
+                    '*',
+                    'software_release',
+                ))
+        },
+        {md5digest(self.getSoftwareURL())},)

software/slapos-sr-testing/software.cfg

@@ -47,6 +47,11 @@ setup = ${slapos-repository:location}/software/caddy-frontend/test/
 egg = slapos.test.erp5
 setup = ${slapos-repository:location}/software/erp5/test/
 
+[slapos.test.upgrade_erp5-setup]
+<= setup-develop-egg
+egg = slapos.test.upgrade_erp5
+setup = ${slapos-repository:location}/software/erp5/upgrade_test/
+
 [slapos.test.htmlvalidatorserver-setup]
 <= setup-develop-egg
 egg = slapos.test.htmlvalidatorserver
@@ -185,6 +190,7 @@ extra-eggs =
   ${slapos.test.backupserver-setup:egg}
   ${slapos.test.caddy-frontend-setup:egg}
   ${slapos.test.erp5-setup:egg}
+  ${slapos.test.upgrade_erp5-setup:egg}
   ${slapos.test.htmlvalidatorserver-setup:egg}
   ${slapos.test.slapos-master-setup:egg}
   ${slapos.test.jstestnode-setup:egg}
@@ -259,6 +265,7 @@ extra =
   backupserver ${slapos.test.backupserver-setup:setup}
   caddy-frontend ${slapos.test.caddy-frontend-setup:setup}
   erp5 ${slapos.test.erp5-setup:setup}
+  upgrade_erp5 ${slapos.test.upgrade_erp5-setup:setup}
   htmlvalidatorserver ${slapos.test.htmlvalidatorserver-setup:setup}
   slapos-master ${slapos.test.slapos-master-setup:setup}
   re6stnet ${slapos.test.re6stnet-setup:setup}

stack/erp5/buildout.hash.cfg

@@ -78,7 +78,7 @@ md5sum = 68b329da9893e34099c7d8ad5cb9c940
 
 [template-erp5]
 filename = instance-erp5.cfg.in
-md5sum = 0920a53b10d3811a5f49930adffb62d8
+md5sum = 5ea4bcdf74fb429f254af8e8fb7b38a3
 
 [template-zeo]
 filename = instance-zeo.cfg.in
@@ -86,7 +86,7 @@ md5sum = 0648e38bd5d3a15bb9f93264932740b9
 
 [template-zope]
 filename = instance-zope.cfg.in
-md5sum = fece4db89f72c16f8d71019746059469
+md5sum = d942854541f20faa57716b58959e1310
 
 [template-balancer]
 filename = instance-balancer.cfg.in

stack/erp5/instance-erp5.cfg.in

@@ -173,6 +173,7 @@ return =
   zope-address-list
   hosts-dict
   monitor-base-url
+  software-release-url
 {%- if test_runner_enabled %}
   test-runner-address-list
 {% endif %}
@@ -234,6 +235,7 @@ software-type = zope
 {% for custom_name, zope_parameter_dict in zope_partition_dict.items() -%}
 {%   set partition_name = 'zope-' ~ custom_name -%}
 {%   set section_name = 'request-' ~ partition_name -%}
+{%   set promise_software_url_section_name = 'promise-software-url' ~ partition_name -%}
 {%   set zope_family = zope_parameter_dict.get('family', 'default') -%}
 {%   do zope_family_name_list.append(zope_family) %}
 {%   set backend_path = zope_parameter_dict.get('backend-path', '') % {'site-id': site_id} %}
@@ -262,6 +264,18 @@ config-port-base = {{ dumps(zope_parameter_dict.get('port-base', 2200)) }}
 config-webdav = {{ dumps(zope_parameter_dict.get('webdav', False)) }}
 {%   if test_runner_enabled -%}
 config-test-runner-apache-url-list = ${publish-early:{{ zope_family }}-test-runner-url-list}
+
+[{{ promise_software_url_section_name }}]
+# Promise to wait for zope partition to use the expected software URL,
+# used on upgrades.
+recipe = slapos.cookbook:check_parameter
+value = {{ '${' ~ section_name ~ ':connection-software-release-url}' }}
+expected-not-value =
+expected-value = ${slap-connection:software-release-url}
+path = ${directory:bin}/${:_buildout_section_name_}
+
+{% do root_common.section(promise_software_url_section_name) -%}
+
 {%   endif -%}
 {% endfor -%}
 

stack/erp5/instance-zope.cfg.in

@@ -352,6 +352,10 @@ configuration-file = {{ '${' ~ conf_name ~ ':rendered}' }}
 {%- if wsgi %}
 port = {{ port }}
 {%- endif %}
+hash-files =
+  ${:configuration-file}
+hash-existing-files =
+  ${buildout:directory}/software_release/buildout.cfg
 
 [{{ section("promise-" ~ name) }}]
 <= monitor-promise-base
@@ -360,19 +364,6 @@ name = {{ name }}.py
 config-hostname = {{ ipv4 }}
 config-port = {{ port }}
 
-{% set extra_path_list = [] -%}
-{% set shell_escaped_extra_path_list = [] -%}
-{% for line in parameter_dict['extra-path-list'].splitlines() -%}
-{%   set line = line.strip() -%}
-{%   do extra_path_list.append(line) -%}
-{%   do shell_escaped_extra_path_list.append(line.replace("\x27", "\x27\\\x27\x27")) -%}
-{% endfor -%}
-[{{ section("promise-" ~ name ~ "-is-running-actual-product") }}]
-<= monitor-promise-base
-module = check_command_execute
-name = {{ name }}-is-running-actual-product.py
-config-command = '{{ parameter_dict['bin-directory'] }}/is-process-older-than-dependency-set' '{{ "${" ~ conf_parameter_name ~ ":pid-file}" }}' {{ " ".join(shell_escaped_extra_path_list) }}
-
 {% if use_ipv6 -%}
 [{{ zope_tunnel_section_name }}]
 < = ipv6toipv4-base
@@ -558,6 +549,7 @@ hard to guess.
 hosts-dict = {{ dumps(hosts_dict) }}
 monitor-base-url = ${monitor-publish-parameters:monitor-base-url}
 test-runner-address-list = {{ dumps(test_runner_address_list) }}
+software-release-url = ${slap-connection:software-release-url}
 
 [monitor-instance-parameter]
 monitor-httpd-ipv6 = {{ (ipv6_set | list)[0] }}