Update Release Candidate

component/egg-patch/Zope/PythonScript-2.13.patch

---- Zope2-2.13.30/src/Shared/DC/Scripts/Signature.py	2022-04-25 08:05:09.312966168 +0000
-+++ Zope2-2.13.30/src/Shared/DC/Scripts/Signature.py	2022-04-25 08:06:20.120743425 +0000
-@@ -35,7 +35,7 @@ def _setFuncSignature(self, defaults=None, varnames=(), argcount=-1):
-         argcount = len(varnames)
-     # Generate a change only if we have to.
-     if self.func_defaults != defaults:
--        self.func_defaults = defaults
-+        self.func_defaults = self.__defaults__ = defaults
-     code = FuncCode(varnames, argcount)
-     if self.func_code != code:
--        self.func_code = code
-+        self.func_code = self.__code__ = code

component/readline/buildout.cfg

@@ -31,8 +31,7 @@ md5sum = e39331f32ad14009b9ff49cc10c5e751
 configure-options =
   --enable-multibyte
   --disable-static
-environment =
-  PATH=${patch:location}/bin:%(PATH)s
+patch-binary = ${patch:location}/bin/patch
 
 [readline]
 recipe = slapos.recipe.cmmi
@@ -47,3 +46,4 @@ configure-options =
 environment =
   CPPFLAGS=-I${ncurses:location}/include
   LDFLAGS=-L${ncurses:location}/lib -Wl,-rpath=${ncurses:location}/lib
+patch-binary = ${patch:location}/bin/patch

component/slapos/obs.cfg

@@ -29,8 +29,8 @@ download-cache = download-cache
 init +=
   buildout = self.buildout['buildout']
   assert buildout['directory'] == buildout['destdir'] + buildout['rootdir'], (
-    "Buildout MUST BE launched in destdir/rootdir (currently launched in %s but should be launched in %s)",
-    buildout['directory'], buildout['destdir'] + buildout['rootdir'])
+    "Buildout MUST BE launched in destdir/rootdir (currently launched in %s but should be launched in %s)" %
+    (buildout['directory'], buildout['destdir'] + buildout['rootdir']))
 
 [python3-common]
 configure-options +=

setup.py

@@ -28,7 +28,7 @@ from setuptools import setup, find_packages
 import glob
 import os
 
-version = '1.0.244'
+version = '1.0.246'
 name = 'slapos.cookbook'
 long_description = open("README.rst").read()
 

slapos/recipe/librecipe/execute.py

@@ -67,7 +67,7 @@ def generic_exec(args, extra_environ=None, wait_list=None,
     else:
       # With chained shebangs, several paths may be inserted at the beginning.
       n = len(args)
-      for i in xrange(1+len(running)-n):
+      for i in six.moves.xrange(1+len(running)-n):
         if args == running[i:n+i]:
           sys.exit("Already running with pid %s." % pid)
     with open(pidfile, 'w') as f:
@@ -91,16 +91,19 @@ def generic_exec(args, extra_environ=None, wait_list=None,
     uid = os.getuid()
     gid = os.getgid()
     unshare(CLONE_NEWUSER |CLONE_NEWNS)
-    with open('/proc/self/setgroups', 'wb') as f: f.write('deny')
-    with open('/proc/self/uid_map',   'wb') as f: f.write('%s %s 1' % (uid, uid))
-    with open('/proc/self/gid_map',   'wb') as f: f.write('%s %s 1' % (gid, gid))
+    with open('/proc/self/setgroups', 'w') as f:
+      f.write('deny')
+    with open('/proc/self/uid_map', 'w') as f:
+      f.write('%s %s 1' % (uid, uid))
+    with open('/proc/self/gid_map', 'w') as f:
+      f.write('%s %s 1' % (gid, gid))
     for size, path in private_tmpfs:
       try:
         os.mkdir(path)
       except OSError as e:
         if e.errno != errno.EEXIST:
           raise
-      mount('tmpfs', path, 'tmpfs', 0, 'size=' + size)
+      mount(b'tmpfs', path.encode(), b'tmpfs', 0, ('size=' + size).encode())
 
   if extra_environ:
     env = os.environ.copy()

slapos/test/recipe/test_re6stnet.py

@@ -52,13 +52,15 @@ class Re6stnetTest(unittest.TestCase):
       return makeRecipe(
             re6stnet.Recipe,
             options=self.options,
-            slap_connection={
+            buildout={
+              'slap-connection': {
                    'computer-id': 'comp-test',
                    'partition-id': 'slappart0',
                    'server-url': 'http://server.com',
                    'software-release-url': 'http://software.com',
                    'key-file': '/path/to/key',
                    'cert-file': '/path/to/cert'
+              }
             },
             name='re6stnet')
 

slapos/test/recipe/test_wrapper.py

+import errno
+import functools
+import os
+import shutil
+import subprocess
+import sys
+import tempfile
+import textwrap
+import time
+import unittest
+
+from slapos.recipe import wrapper
+from slapos.test.utils import makeRecipe
+
+
+class WrapperTestCase(unittest.TestCase):
+  def getOptions(self):
+    raise NotImplementedError()
+
+  def setUp(self):
+    self.buildout_directory = tempfile.mkdtemp()
+    self.addCleanup(shutil.rmtree, self.buildout_directory)
+    self.getTempPath = functools.partial(os.path.join, self.buildout_directory)
+
+    self.wrapper_path = self.getTempPath('wrapper')
+
+    self.recipe = makeRecipe(
+      wrapper.Recipe,
+      options=self.getOptions(),
+      name="wrapper",
+      buildout={'buildout': {
+        'directory': self.buildout_directory,
+      }})
+
+  def terminate_process(self, process):
+    try:
+      process.terminate()
+    except OSError as e:
+      if e.errno != errno.ESRCH:
+        raise
+    process.wait()
+
+
+class TestSimpleCommandLineWrapper(WrapperTestCase):
+  def getOptions(self):
+    return {
+      'command-line': 'echo hello world',
+      'wrapper-path': self.wrapper_path,
+    }
+
+  def test_install_and_execute(self):
+    installed = self.recipe.install()
+    self.assertEqual(installed, self.wrapper_path)
+    self.assertEqual(
+      subprocess.check_output(installed, universal_newlines=True),
+      'hello world\n')
+
+
+class TestEscapeCommandLine(WrapperTestCase):
+  def getOptions(self):
+    return {
+      'command-line': "echo esca $PE",
+      'wrapper-path': self.wrapper_path,
+    }
+
+  def test_install_and_execute(self):
+    installed = self.recipe.install()
+    self.assertEqual(installed, self.wrapper_path)
+    self.assertEqual(
+      subprocess.check_output(installed, universal_newlines=True),
+      "esca $PE\n")
+
+
+class TestEnvironment(WrapperTestCase):
+  def getOptions(self):
+    return {
+      'command-line': 'sh -c "echo $FOO"',
+      'wrapper-path': self.wrapper_path,
+      'environment': 'FOO=bar',
+    }
+
+  def test_install_and_execute(self):
+    installed = self.recipe.install()
+    self.assertEqual(installed, self.wrapper_path)
+
+    output = subprocess.check_output(
+      installed, universal_newlines=True, env={'FOO': 'foo'})
+    self.assertEqual(output, 'bar\n')
+
+
+class TestHashFiles(WrapperTestCase):
+  def getOptions(self):
+    hashed_file = self.getTempPath('hashed_file')
+    with open(hashed_file, 'w') as f:
+      f.write('hello world')
+    return {
+      'command-line': "cat " + hashed_file,
+      'wrapper-path': self.wrapper_path,
+      'hash-files': hashed_file
+    }
+
+  def test_install_and_execute(self):
+    installed = self.recipe.install()
+    # 83af3240d992b2165abbd245a3e43368 is hashlib.md5(b'11\nhello world').hexdigest()
+    self.assertEqual(
+      installed, self.wrapper_path + '-83af3240d992b2165abbd245a3e43368')
+    self.assertEqual(
+      subprocess.check_output(installed, universal_newlines=True),
+      "hello world")
+
+
+class TestPidFile(WrapperTestCase):
+  def getOptions(self):
+    self.pidfile = self.getTempPath('hello.pid')
+    return {
+      'command-line': "/bin/sleep 10",
+      'wrapper-path': self.wrapper_path,
+      'pidfile': self.pidfile
+    }
+
+  def test_install_and_execute(self):
+    installed = self.recipe.install()
+    self.assertEqual(installed, self.wrapper_path)
+    process = subprocess.Popen(
+      installed,
+      stdout=subprocess.PIPE,
+      stderr=subprocess.STDOUT,
+      universal_newlines=True,
+    )
+    self.addCleanup(self.terminate_process, process)
+    if process.poll():
+      self.fail(process.stdout.read())
+
+    for _ in range(20):
+      time.sleep(0.1)
+      if os.path.exists(self.pidfile):
+        break
+
+    with open(self.pidfile) as f:
+      pid = int(f.read())
+    self.assertEqual(process.pid, pid)
+
+    with self.assertRaises(subprocess.CalledProcessError) as ctx:
+      subprocess.check_output(
+        installed, stderr=subprocess.STDOUT, universal_newlines=True)
+    self.assertEqual(
+      ctx.exception.output, 'Already running with pid %s.\n' % pid)
+
+  def test_stale_pidfile_is_ignored(self):
+    installed = self.recipe.install()
+    self.assertEqual(installed, self.wrapper_path)
+    with open(self.pidfile, 'w') as f:
+      f.write('1234')
+
+    process = subprocess.Popen(
+      installed,
+      stdout=subprocess.PIPE,
+      stderr=subprocess.STDOUT,
+      universal_newlines=True,
+    )
+    self.addCleanup(self.terminate_process, process)
+    if process.poll():
+      self.fail(process.stdout.read())
+
+    for _ in range(20):
+      time.sleep(0.1)
+      with open(self.pidfile) as f:
+        pid = int(f.read())
+      if process.pid == pid:
+        break
+    else:
+      self.fail('pidfile not updated', process.stdout.read())
+
+
+class TestWaitForFiles(WrapperTestCase):
+  def getOptions(self):
+    self.waitfile = self.getTempPath('wait')
+    return {
+      'command-line': "/bin/echo done",
+      'wrapper-path': self.wrapper_path,
+      'wait-for-files': self.waitfile,
+    }
+
+  def test_install_and_execute(self):
+    installed = self.recipe.install()
+    self.assertEqual(installed, self.wrapper_path)
+
+    process = subprocess.Popen(
+      installed,
+      stdout=subprocess.PIPE,
+      stderr=subprocess.STDOUT,
+      universal_newlines=True,
+    )
+    self.addCleanup(self.terminate_process, process)
+    if process.poll():
+      self.fail(process.stdout.read())
+
+    # nothing happens when file is not there
+    for _ in range(10):
+      time.sleep(0.1)
+      if process.poll():
+        self.fail(process.stdout.read())
+
+    open(self.waitfile, 'w').close()
+    for _ in range(20):
+      time.sleep(0.1)
+      if process.poll() is not None:
+        self.assertEqual(process.stdout.read(), 'done\n')
+        self.assertEqual(process.returncode, 0)
+        break
+    else:
+      self.fail('process did not start after file was created')
+
+
+class TestPrivateTmpFS(WrapperTestCase):
+  def getOptions(self):
+    self.tmpdir = self.getTempPath('tmpdir')
+    self.tmpfile = self.getTempPath('tmpdir', 'file')
+    self.program = self.getTempPath('program')
+    with open(self.program, 'w') as f:
+      f.write(
+        textwrap.dedent(
+          '''\
+          #!{sys_executable}
+          import os
+          with open({tmpfile!r}, 'w') as f:
+            f.write('ok')
+          with open({tmpfile!r}, 'r') as f:
+            print(f.read())
+        ''').format(sys_executable=sys.executable, tmpfile=self.tmpfile))
+    os.chmod(self.program, 0o700)
+    return {
+      'command-line': self.program,
+      'wrapper-path': self.wrapper_path,
+      'private-tmpfs': '1000 ' + self.tmpdir
+    }
+
+  def test_install_and_execute(self):
+    installed = self.recipe.install()
+    self.assertEqual(installed, self.wrapper_path)
+
+    output = subprocess.check_output(
+      installed,
+      universal_newlines=True,
+    )
+    self.assertEqual(output, 'ok\n')
+    self.assertFalse(os.path.exists(self.tmpfile))
+
+
+class TestReserveCPU(WrapperTestCase):
+  def getOptions(self):
+    self.slapos_cpu_exclusive = self.getTempPath('.slapos-cpu-exclusive')
+    self.program = self.getTempPath('program')
+    with open(self.program, 'w') as f:
+      f.write(
+        textwrap.dedent(
+          '''\
+          #!{sys_executable}
+          import os
+          with open({slapos_cpu_exclusive!r}, 'r') as f:
+            print('ok' if int(f.read()) == os.getpid() else 'error')
+        ''').format(
+            sys_executable=sys.executable,
+            slapos_cpu_exclusive=self.slapos_cpu_exclusive,
+          ))
+    os.chmod(self.program, 0o700)
+    return {
+      'command-line': self.program,
+      'wrapper-path': self.wrapper_path,
+      'reserve-cpu': 'true',
+    }
+
+  def test_install_and_execute(self):
+    installed = self.recipe.install()
+    self.assertEqual(installed, self.wrapper_path)
+
+    output = subprocess.check_output(
+      installed,
+      universal_newlines=True,
+      env={'HOME': self.buildout_directory})
+    self.assertEqual(output, 'ok\n')

slapos/test/utils.py

@@ -2,18 +2,18 @@
 """
 import os
 import sys
+import six
 
 
-def makeRecipe(recipe_class, options, name='test', slap_connection=None):
-  """Instanciate a recipe of `recipe_class` with `options` with a buildout
-  mapping containing a python and an empty `slapos-connection` mapping, unless
-  provided as `slap_connection`.
+def makeRecipe(recipe_class, options, name='test', buildout=None):
+  """Instantiate a recipe of `recipe_class` with `options` with a `buildout`
+  mapping containing by default a python and an empty slap-connection.
 
   This function expects the test suite to have set SLAPOS_TEST_EGGS_DIRECTORY
   and SLAPOS_TEST_DEVELOP_EGGS_DIRECTORY environment variables, so that the
   test recipe does not need to install eggs again when using working set.
   """
-  buildout = {
+  _buildout = {
     'buildout': {
       'bin-directory': '',
       'find-links': '',
@@ -32,15 +32,17 @@ def makeRecipe(recipe_class, options, name='test', slap_connection=None):
        'software-release-url': '',
      }
   }
-  if slap_connection is not None:
-    buildout['slap-connection'] = slap_connection
 
-  buildout['buildout']['eggs-directory'] = os.environ['SLAPOS_TEST_EGGS_DIRECTORY']
-  buildout['buildout']['develop-eggs-directory'] = os.environ['SLAPOS_TEST_DEVELOP_EGGS_DIRECTORY']
+  _buildout['buildout']['eggs-directory'] = os.environ['SLAPOS_TEST_EGGS_DIRECTORY']
+  _buildout['buildout']['develop-eggs-directory'] = os.environ['SLAPOS_TEST_DEVELOP_EGGS_DIRECTORY']
+
+  if buildout:
+    for section, _options in six.iteritems(buildout):
+      _buildout.setdefault(section, {}).update(**_options)
 
   # Prevent test from accidentally writing to the buildout's eggs
-  buildout['buildout']['newest'] = False
-  buildout['buildout']['offline'] = True
+  _buildout['buildout']['newest'] = False
+  _buildout['buildout']['offline'] = True
 
-  return recipe_class(buildout=buildout, name=name, options=options)
+  return recipe_class(buildout=_buildout, name=name, options=options)
 

software/erp5/upgrade_test/test.py

@@ -25,19 +25,26 @@
 #
 ##############################################################################
 
+import contextlib
 import glob
 import json
 import os
+import ssl
+import sys
 import tempfile
 import time
 
 import requests
-import urlparse
+import six.moves.urllib as urllib
+import six.moves.xmlrpc_client
+import urllib3
 
-from slapos.testing.testcase import makeModuleSetUpAndTestCaseClass
-from slapos.testing.testcase import installSoftwareUrlList
-from slapos.testing.testcase import SlapOSNodeCommandError
 from slapos.grid.utils import md5digest
+from slapos.testing.testcase import (
+  SlapOSNodeCommandError,
+  installSoftwareUrlList,
+  makeModuleSetUpAndTestCaseClass,
+)
 
 old_software_release_url = 'https://lab.nexedi.com/nexedi/slapos/raw/1.0.167.5/software/erp5/software.cfg'
 new_software_release_url = os.path.abspath(
@@ -49,6 +56,7 @@ _, SlapOSInstanceTestCase = makeModuleSetUpAndTestCaseClass(
   skip_software_check=True,
 )
 
+
 def setUpModule():
   installSoftwareUrlList(
     SlapOSInstanceTestCase,
@@ -95,8 +103,8 @@ class ERP5UpgradeTestCase(SlapOSInstanceTestCase):
         except SlapOSNodeCommandError:
           cls.slap.waitForInstance(debug=True)
       else:
-        cls.slap.waitForInstance(max_retry=cls.instance_max_retry,
-                                 debug=cls._debug)
+        cls.slap.waitForInstance(
+          max_retry=cls.instance_max_retry, debug=cls._debug)
       cls.logger.debug("instance on new software done")
     except BaseException:
       cls.logger.exception("Error during instance on new software")
@@ -113,9 +121,138 @@ class ERP5UpgradeTestCase(SlapOSInstanceTestCase):
 class TestERP5Upgrade(ERP5UpgradeTestCase):
   @classmethod
   def setUpOldInstance(cls):
-    cls._default_instance_old_parameter_dict = json.loads(
+    cls._default_instance_old_parameter_dict = param_dict = json.loads(
       cls.computer_partition.getConnectionParameterDict()['_'])
 
+    # use a session to retry on failures, when ERP5 is not ready.
+    # (see also TestPublishedURLIsReachableMixin)
+    cls.session = requests.Session()
+    cls.session.mount(
+      param_dict['family-default-v6'],
+      requests.adapters.HTTPAdapter(
+        max_retries=urllib3.util.retry.Retry(
+          total=20,
+          backoff_factor=.1,
+          status_forcelist=(404, 500, 503),
+        )))
+
+    # rebuild an url with user and password
+    parsed = urllib.parse.urlparse(param_dict['family-default'])
+    cls.authenticated_zope_base_url = parsed._replace(
+      netloc='{}:{}@{}:{}'.format(
+        param_dict['inituser-login'],
+        param_dict['inituser-password'],
+        parsed.hostname,
+        parsed.port,
+      ),
+      path=param_dict['site-id'] + '/',
+    ).geturl()
+
+    cls.zope_base_url = '{family_default_v6}/{site_id}'.format(
+      family_default_v6=param_dict['family-default-v6'],
+      site_id=param_dict['site-id'],
+    )
+
+    # wait for old site creation
+    cls.session.get(
+      '{zope_base_url}/person_module'.format(zope_base_url=cls.zope_base_url),
+      auth=requests.auth.HTTPBasicAuth(
+        username=param_dict['inituser-login'],
+        password=param_dict['inituser-password'],
+      ),
+      verify=False,
+      allow_redirects=False,
+    ).raise_for_status()
+
+    # Create scripts to create test data and search catalog for test data.
+    @contextlib.contextmanager
+    def getXMLRPCClient():
+      # don't verify certificate
+      ssl_context = ssl.create_default_context()
+      ssl_context.check_hostname = False
+      ssl_context.verify_mode = ssl.CERT_NONE
+      erp5_xmlrpc_client = six.moves.xmlrpc_client.ServerProxy(
+        cls.authenticated_zope_base_url,
+        context=ssl_context,
+      )
+      # BBB use as a context manager only on python3
+      if sys.version_info < (3, ):
+        yield erp5_xmlrpc_client
+      else:
+        with erp5_xmlrpc_client:
+          yield erp5_xmlrpc_client
+
+    def addPythonScript(script_id, params, body):
+      with getXMLRPCClient() as erp5_xmlrpc_client:
+        custom = erp5_xmlrpc_client.portal_skins.custom
+        try:
+          custom.manage_addProduct.PythonScripts.manage_addPythonScript(
+            script_id)
+        except six.moves.xmlrpc_client.ProtocolError as e:
+          if e.errcode != 302:
+            raise
+        getattr(custom, script_id).ZPythonScriptHTML_editAction(
+          '',
+          '',
+          params,
+          body,
+        )
+
+    # a python script to create a person with a name
+    addPythonScript(
+      script_id='ERP5Site_createTestPerson',
+      params='name',
+      body='''if 1:
+          portal = context.getPortalObject()
+          portal.person_module.newContent(
+            first_name=name,
+          )
+          return 'Done.'
+        ''',
+    )
+    # a python script to search for persons by name
+    addPythonScript(
+      script_id='ERP5Site_searchTestPerson',
+      params='name',
+      body='''if 1:
+          import json
+          portal = context.getPortalObject()
+          result = [brain.getObject().getTitle() for brain in portal.portal_catalog(
+              portal_type='Person',
+              title=name,)]
+          assert result # raise so that we retry until indexed
+          return json.dumps(result)
+        ''',
+    )
+
+    cls.session.post(
+      '{zope_base_url}/ERP5Site_createTestPerson'.format(
+        zope_base_url=cls.zope_base_url),
+      auth=requests.auth.HTTPBasicAuth(
+        username=param_dict['inituser-login'],
+        password=param_dict['inituser-password'],
+      ),
+      data={
+        'name': 'before upgrade'
+      },
+      verify=False,
+      allow_redirects=False,
+    ).raise_for_status()
+
+    assert cls.session.get(
+      '{zope_base_url}/ERP5Site_searchTestPerson'.format(
+        zope_base_url=cls.zope_base_url),
+      auth=requests.auth.HTTPBasicAuth(
+        username=param_dict['inituser-login'],
+        password=param_dict['inituser-password'],
+      ),
+      params={
+        'name': 'before upgrade'
+      },
+      verify=False,
+      allow_redirects=False,
+    ).json() == ['before upgrade']
+
   def test_published_url_is_same(self):
     default_instance_new_parameter_dict = json.loads(
       self.computer_partition.getConnectionParameterDict()['_'])
@@ -136,24 +273,13 @@ class TestERP5Upgrade(ERP5UpgradeTestCase):
     ) as ca_cert:
       ca_cert.write(
         requests.get(
-              urlparse.urljoin(
+          urllib.parse.urljoin(
             default_instance_new_parameter_dict['caucase-http-url'],
             '/cas/crt/ca.crt.pem',
           )).text)
       ca_cert.flush()
 
-      # use a session to retry on failures, when ERP5 is not ready.
-      # (see also TestPublishedURLIsReachableMixin)
-      session = requests.Session()
-      session.mount(
-        default_instance_new_parameter_dict['family-default-v6'],
-        requests.adapters.HTTPAdapter(
-            max_retries=requests.packages.urllib3.util.retry.Retry(
-                total=60,
-                backoff_factor=.5,
-                status_forcelist=(404, 500, 503))))
-
-      session.get(
+      self.session.get(
         '{}/{}/login_form'.format(
           default_instance_new_parameter_dict['family-default-v6'],
           default_instance_new_parameter_dict['site-id'],
@@ -163,7 +289,6 @@ class TestERP5Upgrade(ERP5UpgradeTestCase):
         # verify=ca_cert.name,
       ).raise_for_status()
 
-
   def test_all_instances_use_new_software_release(self):
     self.assertEqual(
       {
@@ -175,4 +300,56 @@ class TestERP5Upgrade(ERP5UpgradeTestCase):
             'software_release',
           ))
       },
-        {md5digest(self.getSoftwareURL())},)
+      {md5digest(self.getSoftwareURL())},
+    )
+
+  def test_catalog_available(self):
+    param_dict = json.loads(
+      self.computer_partition.getConnectionParameterDict()['_'])
+
+    # data created before upgrade is available
+    self.assertEqual(
+      self.session.get(
+        '{zope_base_url}/ERP5Site_searchTestPerson'.format(
+          zope_base_url=self.zope_base_url),
+        auth=requests.auth.HTTPBasicAuth(
+          username=param_dict['inituser-login'],
+          password=param_dict['inituser-password'],
+        ),
+        params={
+          'name': 'before upgrade'
+        },
+        verify=False,
+        allow_redirects=False,
+      ).json(), ['before upgrade'])
+
+    # create data after upgrade
+    self.session.post(
+      '{zope_base_url}/ERP5Site_createTestPerson'.format(
+        zope_base_url=self.zope_base_url),
+      auth=requests.auth.HTTPBasicAuth(
+        username=param_dict['inituser-login'],
+        password=param_dict['inituser-password'],
+      ),
+      data={
+        'name': 'after upgrade'
+      },
+      verify=False,
+      allow_redirects=False,
+    ).raise_for_status()
+
+    # new data can also be found
+    self.assertEqual(
+      self.session.get(
+        '{zope_base_url}/ERP5Site_searchTestPerson'.format(
+          zope_base_url=self.zope_base_url),
+        auth=requests.auth.HTTPBasicAuth(
+          username=param_dict['inituser-login'],
+          password=param_dict['inituser-password'],
+        ),
+        params={
+          'name': 'after upgrade'
+        },
+        verify=False,
+        allow_redirects=False,
+      ).json(), ['after upgrade'])

software/erp5testnode/testsuite/deploy-test/software.cfg

@@ -12,6 +12,7 @@ parts =
 recipe = zc.recipe.egg
 eggs =
   erp5.util
+  requests
 interpreter = ${:_buildout_section_name_}
 
 [python2.7-with-eggs]

software/kvm/buildout.hash.cfg

@@ -55,7 +55,7 @@ md5sum = a8cf453d20f01c707f02c4b4014580d8
 
 [template-kvm-run]
 filename = template/template-kvm-run.in
-md5sum = 395ee373ccda3382d257fde1ff4222b0
+md5sum = 6c100eec00de5e53f64b075dd69a9865
 
 [template-kvm-controller]
 filename = template/kvm-controller-run.in
@@ -79,11 +79,11 @@ md5sum = 438192aab9f11e40dc521b46a4854dcf
 
 [image-download-controller]
 filename = template/image-download-controller.py
-md5sum = 4d48b3da5bc611fc6533335b5953c840
+md5sum = 3cc10323fd4d2db4cfbac536b66eae7c
 
 [image-download-config-creator]
 filename = template/image-download-config-creator.py
-md5sum = 8fbe05c4175a7f31b6bffced9ad4e91d
+md5sum = 22ed19d9b8f7b983c97c52caa686bcd7
 
 [whitelist-firewall-download-controller]
 filename = template/whitelist-firewall-download-controller.py

software/kvm/instance-kvm-input-schema.json

@@ -204,7 +204,7 @@
       "title": "Size of additional disk to create for virtual machine, in Gigabytes",
       "description": "Specify the size of additional disk to create for virtual machine in data folder of SlapOS Node. Requires instance_storage_home to be configured on SlapOS Node.",
       "type": "integer",
-      "minimum": 10,
+      "minimum": 1,
       "default": 20
     },
     "external-disk-format": {

software/kvm/software-py3.cfg

-[buildout]
-extends =
-  software.cfg
-
-[python]
-part = python3

software/kvm/software.cfg

@@ -40,6 +40,9 @@ parts = ${:common-parts}
 
 #XXX-Cedric : add list of keyboard layouts (azerty/us querty/...) parameter to qemu
 
+[python]
+part = python3
+
 [python-with-eggs]
 recipe = zc.recipe.egg
 interpreter = ${:_buildout_section_name_}

software/kvm/template/image-download-config-creator.py

@@ -49,6 +49,10 @@ if __name__ == "__main__":
         image_list.append({
           'md5sum': md5sum,
           'url': url,
+          # Note: The destination here it's the waneted md5sum on purpose, as
+          #       it allows to assume, that correctly downloaded and hashed
+          #       image stored at this filename matches the md5sum, so it does
+          #       not have to be hashed on each download run.
           'destination': md5sum,
           'destination-tmp': md5sum + '_tmp',
           'image-number': '%03i' % (image_number,),

software/kvm/template/image-download-controller.py

@@ -68,13 +68,17 @@ if __name__ == "__main__":
     destination = os.path.join(
       config['destination-directory'], image['destination'])
     if os.path.exists(destination):
-      if md5Checksum(destination) == image['md5sum']:
+      # Note: There is no need to recheck md5sum here
+      #       The image name is its md5sum, so if it exists, it means it has
+      #       correct md5sum
+      #       Calculating md5sum of big images takes more time than processing
+      #       of the partition and running promises and this leads to endless
+      #       loop of never ending promise failures
+      #       Of course, someone nasty can come to the partition and damage
+      #       this image, but it's another story, and shall not be fixed
+      #       during download phase.
       print('INF: %s : already downloaded' % (image['url'],))
       continue
-      else:
-        print('INF: %s : Removed, as expected checksum does not match %s' % (
-          image['url'], image['md5sum']))
-        os.remove(destination)
     # key is str, as the dict is dumped to JSON which does not accept tuples
     md5sum_state_key = '%s#%s' % (image['url'], image['md5sum'])
     md5sum_state_amount = md5sum_state_dict.get(md5sum_state_key, 0)

software/kvm/template/template-kvm-run.in

@@ -130,7 +130,7 @@ def getMapStorageList(disk_storage_dict, external_disk_number):
       # ID are writen in one line: data1 data3 data2 ...
       content = mf.readline()
       for id in content.split(' '):
-        if disk_storage_dict.has_key(id):
+        if id in disk_storage_dict:
           id_list.append(id)
         else:
           # Mean that this disk path has been removed (disk unmounted)

software/matomo/software.cfg

@@ -34,6 +34,9 @@ md5sum = 8d592676bc2c0d51363ad7b2caf171fe
 [custom-application-deployment]
 path = ${template-matomo-instance:output}
 part-list = matomo-backup.sh matomo-backup-cron matomo-apache-httpd
+db-name = matomo
+db-user = matomo
+db-password = 12345678
 
 [template-matomo-instance]
 recipe = slapos.recipe.template:jinja2

software/metabase/software.cfg

@@ -22,8 +22,8 @@ part = python3
 
 [metabase.jar]
 recipe = slapos.recipe.build:download
-url = https://downloads.metabase.com/v0.41.4/metabase.jar
-md5sum = 9b81838e5c40302b552c66df5a767f8e
+url = https://downloads.metabase.com/v0.43.1/metabase.jar
+md5sum = 8033ba58825239e7dff29be8d4c885a7
 
 [instance-profile]
 recipe = slapos.recipe.template

software/nextcloud/software.cfg

@@ -41,6 +41,9 @@ context =
 [custom-application-deployment]
 path = ${template-nextcloud-instance:output}
 part-list = nextcloud-install.sh
+db-name = nextcloud
+db-user = nextcloud
+db-password = insecure
 
 [nc-download-unpacked]
 recipe = slapos.recipe.build:download-unpacked

software/slapos-sr-testing/software-py3.cfg

@@ -19,6 +19,7 @@ extra =
   helloworld ${slapos.test.helloworld-setup:setup}
   hugo ${slapos.test.hugo-setup:setup}
   jupyter ${slapos.test.jupyter-setup:setup}
+  kvm ${slapos.test.kvm-setup:setup}
   matomo ${slapos.test.matomo-setup:setup}
   monitor ${slapos.test.monitor-setup:setup}
   nextcloud ${slapos.test.nextcloud-setup:setup}

software/slapos-sr-testing/software.cfg

@@ -350,7 +350,6 @@ tests =
 # here, to check there's no promise issue when slapos node runs with Python 2.
   erp5 ${slapos.test.erp5-setup:setup}
   fluentd ${slapos.test.fluentd-setup:setup}
-  kvm ${slapos.test.kvm-setup:setup}
   metabase ${slapos.test.metabase-setup:setup}
 ###
   ${:extra}

stack/erp5/buildout.cfg

@@ -469,7 +469,6 @@ eggs = ${neoppod:eggs}
   pytz
   requests
   responses
-  threadframe
   urlnorm
   uuid
   xml_marshaller
@@ -524,9 +523,7 @@ eggs = ${neoppod:eggs}
   five.localsitemanager
 
 # Other products
-  Products.DCWorkflowGraph
   Products.MimetypesRegistry
-  Products.ExternalEditor
   Products.TIDStorage
   Products.LongRequestLogger
 
@@ -552,10 +549,8 @@ eggs = ${neoppod:eggs}
   ipykernel
 
 # Used by DiffTool
-  xmltodict
   deepdiff
   unidiff
-  jsonpickle
 
 # WSGI server
   zope.globalrequest
@@ -592,8 +587,6 @@ Acquisition-patches = ${:_profile_base_location_}/../../component/egg-patch/Acqu
 Acquisition-patch-options = -p1
 python-magic-patches = ${:_profile_base_location_}/../../component/egg-patch/python_magic/magic.patch#de0839bffac17801e39b60873a6c2068
 python-magic-patch-options = -p1
-Zope2-patches = ${:_profile_base_location_}/../../component/egg-patch/Zope/PythonScript-2.13.patch#124c0d37394dd5020c6fd241ad75cc29
-Zope2-patch-options = -p1
 
 [eggs-all-scripts]
 recipe = zc.recipe.egg
@@ -634,7 +627,6 @@ pysvn = 1.9.15+SlapOSPatched001
 python-ldap = 2.4.32+SlapOSPatched001
 python-magic = 0.4.12+SlapOSPatched001
 PyPDF2 = 1.26.0+SlapOSPatched001
-Zope2 = 2.13.30+SlapOSPatched001
 ## https://lab.nexedi.com/nexedi/slapos/merge_requests/648
 pylint = 1.4.4+SlapOSPatched002
 # astroid 1.4.1 breaks testDynamicClassGeneration
@@ -668,9 +660,6 @@ zope.app.testing = 3.8.1
 APacheDEX = 1.8
 Pillow = 6.2.2
 Products.CMFActionIcons = 2.1.3
-Products.DCWorkflowGraph = 0.4.1
-# Products.ExternalEditor 2.0.0's dtml is not based on Zope2 OFS's one.
-Products.ExternalEditor = 1.1.1
 Products.GenericSetup = 1.8.6
 Products.LongRequestLogger = 2.1.0
 # Products.MimetypesRegistry 2.1 requires AccessControl>=3.0.0Acquisition.
@@ -713,7 +702,6 @@ rsa = 3.4.2
 spyne = 2.12.14
 suds = 0.4
 facebook-sdk = 2.0.0
-threadframe = 0.2
 urlnorm = 1.1.4
 uuid = 1.30
 validictory = 1.1.0
@@ -732,8 +720,6 @@ zope.globalrequest = 1.5
 waitress = 1.4.4
 xlrd = 1.1.0
 
-# Re-add for as it is required to be there for uninstallation
-erp5.recipe.w3validator = 1.0.2
 Products.ZSQLMethods = 2.13.5
 fpconst = 0.7.2
 graphviz = 0.5.2
@@ -748,7 +734,6 @@ mpmath = 0.19
 openpyxl = 2.4.8
 sympy = 1.1.1
 jdcal = 1.3
-xmltodict = 0.11.0
 deepdiff = 3.3.0
 unidiff = 0.5.5
 jsonpickle = 0.9.6

stack/lamp/buildout.cfg

@@ -53,6 +53,10 @@ part = python3
 # See software/maarch/software.cfg for an example.
 path =
 part-list =
+# database information
+db-name = lamp
+db-user = lamp
+db-password = insecure
 
 #----------------
 #-- Instance-level buildout profiles.
@@ -100,6 +104,9 @@ context =
     key unixodbc_location unixodbc:location
     key openssl_location openssl:location
     key custom_application_template custom-application-deployment:path
+    key db_name custom-application-deployment:db-name
+    key db_user custom-application-deployment:db-user
+    key db_password custom-application-deployment:db-password
 
 [instance-apache-php]
 <= template-download-base

stack/lamp/buildout.hash.cfg

@@ -14,7 +14,7 @@
 # not need these here).
 [instance]
 filename = instance.cfg.in
-md5sum = 29df0dc24386ecb97dc52c9fb59108c8
+md5sum = a5a630377bfb0421d6993c9c2c411a23
 
 [instance-apache-php]
 filename = instance-apache-php.cfg.in
@@ -22,7 +22,7 @@ md5sum = 0952ef9f6cb5e259ad5519d2975d2f37
 
 [instance-lamp]
 filename = instance-lamp.cfg.jinja2.in
-md5sum = e0e2e88b6deeb011b998b78e4e468555
+md5sum = b3d68a13d7a7ffcac774f51f02a68359
 
 [template-apache.conf]
 filename = apache.conf.in

stack/lamp/instance-lamp.cfg.jinja2.in

@@ -37,8 +37,8 @@ return =
 {% do publish_dict.__setitem__('backend-url', '${request-apache:connection-backend-url}') -%}
 {% do monitor_base_url_dict.__setitem__('apache', '${request-apache:connection-monitor-base-url}') -%}
 
+{% do mariadb_dict.__setitem__('database-list', [{'name': db_name, 'user': db_user, 'password': db_password}]) -%}
 
-{% do mariadb_dict.__setitem__('database-list', [{'name': 'nextcloud', 'user': 'nextcloud', 'password': 'insecure'}]) -%}
 {% do mariadb_dict.__setitem__('test-database-amount', 0) -%}
 {% do mariadb_dict.__setitem__('tcpv4-port', 2099) -%}
 {% do mariadb_dict.__setitem__('max-slowqueries-threshold', 1000) -%}

stack/lamp/instance.cfg.in

@@ -49,6 +49,9 @@ url = {{ template_lamp }}
 filename = template-lamp.cfg
 extra-context =
   section parameter_dict dynamic-template-lamp-parameters
+  raw db_name {{ db_name }}
+  raw db_user {{ db_user }}
+  raw db_password {{ db_password }}
 
 [dynamic-template-apache-php-parameters]
 application-location = {{ application_location }}

stack/resilient/buildout.hash.cfg

@@ -30,7 +30,7 @@ md5sum = 44a3166048a81d0d76d69527b1934ef7
 
 [template-replicated]
 filename = template-replicated.cfg.in
-md5sum = c4012ccc2c473ae5c7cad9dcac61e0f1
+md5sum = 2eea3b0227c3ae9e44cfc41df9930fa7
 
 [template-parts]
 filename = template-parts.cfg.in

stack/resilient/template-replicated.cfg.in

@@ -6,7 +6,7 @@
 {% set monitor_url_list = [] -%}
 # prepare sla-parameters
 {% if slapparameter_dict is defined -%}
-{%   for key in slapparameter_dict.keys() -%}
+{%   for key in list(slapparameter_dict.keys()) -%}
 {%     if key.startswith('-sla-') -%}
 {%       do sla_parameter_dict.__setitem__(key, slapparameter_dict.pop(key)) -%}
 {%     endif -%}
@@ -59,7 +59,7 @@ sla-mode = unique_by_network
 {%   for key in sla_parameter_dict.keys() -%}
 {%     if key.startswith(sla_key_main) -%}
 {%       do sla_dict.__setitem__(key[sla_key_main_length:], sla_parameter_dict.get(key)) -%}
-{%     elif key.startswith(sla_key_secondary) and not sla_dict.has_key(key[sla_key_secondary_length:]) -%}
+{%     elif key.startswith(sla_key_secondary) and key[sla_key_secondary_length:] not in sla_dict -%}
 {%         do sla_dict.__setitem__(key[sla_key_secondary_length:], sla_parameter_dict.get(key)) -%}
 {%     endif -%}
 {%   endfor -%}
@@ -111,7 +111,7 @@ sla-mode = unique_by_network
 {%   for key in sla_parameter_dict.keys() -%}
 {%     if key.startswith(sla_key_main) -%}
 {%       do sla_dict.__setitem__(key[sla_key_main_length:], sla_parameter_dict.get(key)) -%}
-{%     elif key.startswith(sla_key_secondary) and not sla_dict.has_key(key[sla_key_secondary_length:]) -%}
+{%     elif key.startswith(sla_key_secondary) and key[sla_key_secondary_length:] not in sla_dict -%}
 {%         do sla_dict.__setitem__(key[sla_key_secondary_length:], sla_parameter_dict.get(key)) -%}
 {%     endif -%}
 {%   endfor -%}
@@ -214,7 +214,7 @@ sla-mode = unique_by_network
 {%   for key in sla_parameter_dict.keys() -%}
 {%     if key.startswith(sla_key_main) -%}
 {%       do sla_dict.__setitem__(key[sla_key_main_length:], sla_parameter_dict.get(key)) -%}
-{%     elif key.startswith(sla_key_secondary) and not sla_dict.has_key(key[sla_key_secondary_length:]) -%}
+{%     elif key.startswith(sla_key_secondary) and key[sla_key_secondary_length:] not in sla_dict -%}
 {%         do sla_dict.__setitem__(key[sla_key_secondary_length:], sla_parameter_dict.get(key)) -%}
 {%     endif -%}
 {%   endfor -%}