ERP5 py3: WIP ( all changes squashed )

software/slapos-sr-testing: add erp5-py3

---

WIP ERP5: XXX dumps() parameter for longrequest promise 🚧

Not sure why it was OK on python2 and not sure if this has to be done
or the promise code needs to cast the results of getConfig()

---

py3: do not enable NEO test yet 🚧

at this point they all fail after long timeouts, because neo does not
support py3 yet

stack/erp5: version up APacheDEX 2.0 (py3 only)

component/ZODB/buildout.cfg

@@ -57,6 +57,12 @@ egg-versions =
 
 [ZODB5]
 <= _ZODB
+egg-versions =
+  ZODB = 5.8.1
+  transaction = 4.0.0
+
+[ZODB5:python2]
+<= _ZODB
 egg-versions =
   ZODB = 5.8.1
   transaction = 3.0.1
@@ -94,11 +100,17 @@ setup-eggs = ${python-cffi:egg}
 
 # eggs that are common to ZODB4 and ZODB5.
 [versions]
-BTrees = 4.11.3
-persistent = 4.9.3
-zodbpickle = 2.6.0
+BTrees = 5.1.0
+persistent = 5.1.0
+zodbpickle = 3.1.0
 
 # Provide ZODB3 for those eggs that still care about ZODB3 compatibility -
 # for example wendelin.core. ZODB3 3.11 is just a dependency egg on _latest_
 # ZODB, persistent, BTrees and ZEO.
 ZODB3 = 3.11.0
+
+
+[versions:python2]
+BTrees = 4.11.3
+persistent = 4.9.3
+zodbpickle = 2.6.0

component/egg-patch/SOAPpy-py3/0001-backport-changes-from-0.52.29.patch

+From 21a91db138cca3ada0e4dff475b061066362410c Mon Sep 17 00:00:00 2001
+From: =?UTF-8?q?J=C3=A9rome=20Perrin?= <jerome@nexedi.com>
+Date: Sat, 17 Feb 2024 23:25:43 +0900
+Subject: [PATCH] backport changes from 0.52.29
+
+We can not use 0.52.29 directly because it does not have a setup.py
+and our buildout / setuptools tooling is too old.
+---
+ src/SOAPpy/Client.py | 3 ++-
+ src/SOAPpy/Types.py  | 2 ++
+ 2 files changed, 4 insertions(+), 1 deletion(-)
+
+diff --git a/src/SOAPpy/Client.py b/src/SOAPpy/Client.py
+index e86c5ec..d2bbefb 100644
+--- a/src/SOAPpy/Client.py
++++ b/src/SOAPpy/Client.py
+@@ -45,6 +45,7 @@
+ ident = '$Id: Client.py 1496 2010-03-04 23:46:17Z pooryorick $'
+ 
+ from .version import __version__
++from io import StringIO
+ 
+ #import xml.sax
+ import urllib.request, urllib.parse, urllib.error
+@@ -152,7 +153,7 @@ class HTTP:
+             return -1, e.line, None
+ 
+         self.headers = response.msg
+-        self.file = response.fp
++        self.file = StringIO(response.fp.read().decode('utf-8'))
+         return response.status, response.reason, response.msg
+ 
+     def close(self):
+diff --git a/src/SOAPpy/Types.py b/src/SOAPpy/Types.py
+index de9dcac..cf08d17 100644
+--- a/src/SOAPpy/Types.py
++++ b/src/SOAPpy/Types.py
+@@ -1451,6 +1451,8 @@ class arrayType(collections.UserList, compoundType):
+     def __getitem__(self, item):
+         try:
+             return self.data[int(item)]
++        except TypeError:
++            return self.data[item]
+         except ValueError:
+             return getattr(self, item)
+ 
+-- 
+2.42.0
+

component/egg-patch/Zope/1196.patch

+From c8c52a14d481403f1db252631d3cc4b8e86e1798 Mon Sep 17 00:00:00 2001
+From: =?UTF-8?q?J=C3=A9rome=20Perrin?= <perrinjerome@gmail.com>
+Date: Sat, 17 Feb 2024 05:32:46 +0000
+Subject: [PATCH] Fix authentication error viewing ZMI with a user defined
+ outside of zope root.
+
+Fixes https://github.com/zopefoundation/Zope/issues/1195
+
+ 
+ 5.9 (2023-11-24)
+ ----------------
+diff --git a/src/App/dtml/manage_page_header.dtml b/src/App/dtml/manage_page_header.dtml
+index 2dcb047552..e8b8659e10 100644
+--- a/src/App/dtml/manage_page_header.dtml
++++ b/src/App/dtml/manage_page_header.dtml
+@@ -10,24 +10,27 @@
+ </dtml-let>
+ 
+ <title><dtml-if title_or_id><dtml-var title_or_id><dtml-else>Zope</dtml-if></title>
++<dtml-let basepath="'/'.join([''] + [p for p in (REQUEST['BASEPATH1'], REQUEST.get('AUTHENTICATION_PATH')) if p])">
++
+ <dtml-in css_urls>
+-	<link rel="stylesheet" type="text/css" href="&dtml-BASEPATH1;&dtml-sequence-item;" />
++	<link rel="stylesheet" type="text/css" href="&dtml-basepath;&dtml-sequence-item;" />
+ </dtml-in>
+ <dtml-in js_urls>
+-	<script src="&dtml-BASEPATH1;&dtml-sequence-item;"></script>
++	<script src="&dtml-basepath;&dtml-sequence-item;"></script>
+ </dtml-in>
+ 
+-<link rel="shortcut icon" type="image/x-icon" href="&dtml-BASEPATH1;/++resource++zmi/logo/favicon/favicon.ico" />
+-<link rel="apple-touch-icon" sizes="180x180" href="&dtml-BASEPATH1;/++resource++zmi/logo/favicon/apple-touch-icon.png" />
+-<link rel="icon" type="image/png" sizes="32x32" href="&dtml-BASEPATH1;/++resource++zmi/logo/favicon/favicon-32x32.png" />
+-<link rel="icon" type="image/png" sizes="16x16" href="&dtml-BASEPATH1;/++resource++zmi/logo/favicon/favicon-16x16.png" />
+-<link rel="manifest" href="&dtml-BASEPATH1;/++resource++zmi/logo/favicon/site.webmanifest" />
+-<link rel="mask-icon" href="&dtml-BASEPATH1;/++resource++zmi/logo/favicon/safari-pinned-tab.svg" color="#5bbad5" />
+-<meta name="msapplication-config" content="&dtml-BASEPATH1;/++resource++zmi/logo/favicon/browserconfig.xml"/>
++<link rel="shortcut icon" type="image/x-icon" href="&dtml-basepath;/++resource++zmi/logo/favicon/favicon.ico" />
++<link rel="apple-touch-icon" sizes="180x180" href="&dtml-basepath;/++resource++zmi/logo/favicon/apple-touch-icon.png" />
++<link rel="icon" type="image/png" sizes="32x32" href="&dtml-basepath;/++resource++zmi/logo/favicon/favicon-32x32.png" />
++<link rel="icon" type="image/png" sizes="16x16" href="&dtml-basepath;/++resource++zmi/logo/favicon/favicon-16x16.png" />
++<link rel="manifest" href="&dtml-basepath;/++resource++zmi/logo/favicon/site.webmanifest" />
++<link rel="mask-icon" href="&dtml-basepath;/++resource++zmi/logo/favicon/safari-pinned-tab.svg" color="#5bbad5" />
++<meta name="msapplication-config" content="&dtml-basepath;/++resource++zmi/logo/favicon/browserconfig.xml"/>
+ <meta name="msapplication-TileColor" content="#2d89ef" />
+ <meta name="theme-color" content="#ffffff" />
+ 
+ </head>
++</dtml-let>
+ <!-- REFACT what is a better way to get the last part of the current URL? -->
+ <body id="nodeid-<dtml-var "getId()">" class="zmi zmi-<dtml-var "this().meta_type.replace(' ', '-').replace('(', '').replace(')', '')"> zmi-<dtml-var "URL0[_.len(URL1)+1:]">">
+ </dtml-unless>
+diff --git a/src/zmi/styles/tests.py b/src/zmi/styles/tests.py
+index 256edfa74b..36c7b65bec 100644
+--- a/src/zmi/styles/tests.py
++++ b/src/zmi/styles/tests.py
+@@ -21,6 +21,8 @@ def setupZCML():
+ class SubscriberTests(Testing.ZopeTestCase.FunctionalTestCase):
+     """Testing .subscriber.*"""
+ 
++    base_path = f'/{Testing.ZopeTestCase.folder_name}'
++
+     def call_manage_main(self):
+         """Call /folder/manage_main and return the HTML text."""
+         def _call_manage_main(self):
+@@ -29,7 +31,7 @@ def _call_manage_main(self):
+             # which the WSGI publisher does not expect.
+             endInteraction()
+             response = self.publish(
+-                f'/{Testing.ZopeTestCase.folder_name}/manage_main',
++                f'{self.base_path}/manage_main',
+                 basic=basic_auth)
+             return str(response)
+         return temporaryPlacelessSetUp(
+@@ -40,11 +42,11 @@ def test_subscriber__css_paths__1(self):
+         from .subscriber import css_paths
+         body = self.call_manage_main()
+         for path in css_paths(None):
+-            self.assertIn(path, body)
++            self.assertIn(f'href="{self.base_path}{path}"', body)
+ 
+     def test_subscriber__js_paths__1(self):
+         """The paths it returns are rendered in the ZMI."""
+         from .subscriber import js_paths
+         body = self.call_manage_main()
+         for path in js_paths(None):
+-            self.assertIn(path, body)
++            self.assertIn(f'src="{self.base_path}{path}"', body)

component/egg-patch/Zope/1198.patch

+From 7f4c61c0d7fe0751be93e80683659271fa0c65a3 Mon Sep 17 00:00:00 2001
+From: =?UTF-8?q?J=C3=A9rome=20Perrin?= <perrinjerome@gmail.com>
+Date: Sat, 17 Feb 2024 13:06:28 +0000
+Subject: [PATCH] Fix Content-Disposition heeader for clients without rfc6266
+ support.
+
+Since https://github.com/zopefoundation/Zope/pull/893 the
+Content-Disposition header supports non-ascii filenames, by containing
+the filename in ascii and the filename in UTF-8, but the ascii version
+was produced by applying `str` on a `bytes` instance, so it looks like
+`b'file.txt'` instead of `file.txt`.
+---
+ CHANGES.rst                              | 3 +++
+ src/ZPublisher/HTTPResponse.py           | 3 ++-
+ src/ZPublisher/tests/testHTTPResponse.py | 4 ++--
+ 3 files changed, 7 insertions(+), 3 deletions(-)
+
+diff --git a/CHANGES.rst b/CHANGES.rst
+index e8dedc6a66..30061e2093 100644
+--- a/CHANGES.rst
++++ b/CHANGES.rst
+@@ -18,6 +18,9 @@ https://github.com/zopefoundation/Zope/blob/4.x/CHANGES.rst
+ - Fix redirections to URLs with host given as IP-literal with brackets.
+   Fixes `#1191 <https://github.com/zopefoundation/Zope/issues/1191>`_.
+ 
++- Fix ``Content-Disposition`` filename for clients without rfc6266 support.
++  (`#1198 <https://github.com/zopefoundation/Zope/pull/1198>`_)
++
+ 
+ 5.9 (2023-11-24)
+ ----------------
+diff --git a/src/ZPublisher/HTTPResponse.py b/src/ZPublisher/HTTPResponse.py
+index ed6bc00a89..264a488a03 100644
+--- a/src/ZPublisher/HTTPResponse.py
++++ b/src/ZPublisher/HTTPResponse.py
+@@ -142,7 +142,8 @@ def make_content_disposition(disposition, file_name):
+         #
+         # a special header has to be crafted
+         # also see https://tools.ietf.org/html/rfc6266#appendix-D
+-        encoded_file_name = file_name.encode('us-ascii', errors='ignore')
++        encoded_file_name = file_name.encode(
++            'us-ascii', errors='ignore').decode()
+         header += f'; filename="{encoded_file_name}"'
+         quoted_file_name = quote(file_name)
+         header += f'; filename*=UTF-8\'\'{quoted_file_name}'
+diff --git a/src/ZPublisher/tests/testHTTPResponse.py b/src/ZPublisher/tests/testHTTPResponse.py
+index 1613b0ea59..5d64b5b3dd 100644
+--- a/src/ZPublisher/tests/testHTTPResponse.py
++++ b/src/ZPublisher/tests/testHTTPResponse.py
+@@ -1433,7 +1433,7 @@ def test_ascii(self):
+     def test_latin_one(self):
+         self.assertEqual(
+             make_content_disposition('inline', 'Dänemark.png'),
+-            'inline; filename="b\'Dnemark.png\'"; filename*=UTF-8\'\'D%C3%A4nemark.png'  # noqa: E501
++            'inline; filename="Dnemark.png"; filename*=UTF-8\'\'D%C3%A4nemark.png'  # noqa: E501
+         )
+ 
+     def test_unicode(self):
+@@ -1445,7 +1445,7 @@ def test_unicode(self):
+         """
+         self.assertEqual(
+             make_content_disposition('inline', 'ıq.png'),
+-            'inline; filename="b\'q.png\'"; filename*=UTF-8\'\'%C4%B1q.png'
++            'inline; filename="q.png"; filename*=UTF-8\'\'%C4%B1q.png'
+         )
+ 
+ 

component/pylint/buildout.cfg

@@ -13,6 +13,27 @@ patches =
 patch-options = -p1
 patch-binary = ${patch:location}/bin/patch
 
+[astroid:python3]
+setup-eggs = ${lazy-object-proxy:egg}
+
+[lazy-object-proxy]
+recipe = zc.recipe.egg:custom
+egg = lazy-object-proxy
+setup-eggs =
+  ${setuptools-scm:egg}
+  typing-extensions
+  tomli
+
+[setuptools-scm]
+recipe = zc.recipe.egg:custom
+egg = setuptools-scm
+setup-eggs = packaging
+
+[mccabe]
+recipe = zc.recipe.egg:custom
+egg = mccabe
+setup-eggs = pytest-runner
+
 [pylint]
 recipe = zc.recipe.egg:custom
 egg = pylint
@@ -21,3 +42,15 @@ patches =
   ${:_profile_base_location_}/pylint-redefining-builtins-modules.patch#043defc6e9002ac48b40e078797d4d17
 patch-options = -p1
 patch-binary = ${patch:location}/bin/patch
+
+[pylint:python3]
+recipe = zc.recipe.egg
+patches =
+
+[mccabe:python3]
+recipe = zc.recipe.egg
+patches =
+setup-eggs =
+
+[astroid:python3]
+recipe = zc.recipe.egg

software/erp5/software-py3.cfg

+[buildout]
+extends =
+  ../../stack/erp5/buildout-py3.cfg

software/erp5/test/test/__init__.py

@@ -46,10 +46,12 @@ from cryptography.x509.oid import NameOID
 from slapos.testing.testcase import ManagedResource, makeModuleSetUpAndTestCaseClass
 from slapos.testing.utils import findFreeTCPPort
 
+ERP5PY3 = os.environ['SLAPOS_SR_TEST_NAME'] == 'erp5-py3'
 
 _setUpModule, SlapOSInstanceTestCase = makeModuleSetUpAndTestCaseClass(
     os.path.abspath(
-        os.path.join(os.path.dirname(__file__), '..', '..', 'software.cfg')))
+        os.path.join(os.path.dirname(__file__), '..', '..', 'software%s.cfg' % (
+          '-py3' if ERP5PY3 else ''))))
 
 
 setup_module_executed = False
@@ -191,7 +193,10 @@ def neo(instance_parameter_dict):
 class ERP5InstanceTestCase(SlapOSInstanceTestCase, metaclass=ERP5InstanceTestMeta):
   """ERP5 base test case
   """
-  __test_matrix__ = matrix((zeo, neo))  # switch between NEO and ZEO mode
+  if ERP5PY3:
+    __test_matrix__ = matrix((zeo, ))  # TODO: NEO is not yet enabled for py3
+  else:
+    __test_matrix__ = matrix((zeo, neo))  # switch between NEO and ZEO mode
 
   @classmethod
   def isNEO(cls):

software/erp5/test/test/test_erp5.py

@@ -51,7 +51,7 @@ import urllib3
 from slapos.testing.utils import CrontabMixin
 import zc.buildout.configparser
 
-from . import CaucaseService, ERP5InstanceTestCase, default, matrix, neo, setUpModule
+from . import CaucaseService, ERP5InstanceTestCase, default, matrix, neo, setUpModule, ERP5PY3
 
 setUpModule # pyflakes
 
@@ -1260,60 +1260,80 @@ class TestNEO(ZopeSkinsMixin, CrontabMixin, ERP5InstanceTestCase):
   __partition_reference__ = 'n'
   __test_matrix__ = matrix((neo,))
 
-  def _getCrontabCommand(self, crontab_name: str) -> str:
-    """Read a crontab and return the command that is executed.
-
-    overloaded to use crontab from neo partition
-    """
-    with open(
-        os.path.join(
+  if ERP5PY3:
+    # NEO is not ready for python3 at this time, this test is here to become
+    # an unexpected success once it starts working, so that we remember to
+    # remove this and enable neo in ERP5InstanceTestCase.__test_matrix__
+    setup_failed_exception = None
+    @classmethod
+    def setUpClass(cls):
+      try:
+        super().setUpClass()
+      except BaseException as e:
+        cls.setup_failed_exception = e
+        cls.setUp = lambda self: None
+        cls.tearDownClass = classmethod(lambda cls: None)
+
+    @unittest.expectedFailure
+    def test_neo_py3(self):
+      self.assertIsNone(self.setup_failed_exception)
+
+  else:
+    def _getCrontabCommand(self, crontab_name: str) -> str:
+      """Read a crontab and return the command that is executed.
+
+      overloaded to use crontab from neo partition
+      """
+      with open(
+          os.path.join(
+              self.getComputerPartitionPath('neo-0'),
+              'etc',
+              'cron.d',
+              crontab_name,
+          )) as f:
+        crontab_spec, = f.readlines()
+      self.assertNotEqual(crontab_spec[0], '@', crontab_spec)
+      return crontab_spec.split(None, 5)[-1]
+
+    def test_log_rotation(self):
+      # first run to create state files
+      self._executeCrontabAtDate('logrotate', '2000-01-01')
+
+      def check_sqlite_log(path):
+        with self.subTest(path), contextlib.closing(sqlite3.connect(path)) as con:
+          con.execute('select * from log')
+
+      logfiles = ('neoadmin.log', 'neomaster.log', 'neostorage-0.log')
+      for f in logfiles:
+        check_sqlite_log(
+          os.path.join(
             self.getComputerPartitionPath('neo-0'),
-            'etc',
-            'cron.d',
-            crontab_name,
-        )) as f:
-      crontab_spec, = f.readlines()
-    self.assertNotEqual(crontab_spec[0], '@', crontab_spec)
-    return crontab_spec.split(None, 5)[-1]
-
-  def test_log_rotation(self):
-    # first run to create state files
-    self._executeCrontabAtDate('logrotate', '2000-01-01')
-
-    def check_sqlite_log(path):
-      with self.subTest(path), contextlib.closing(sqlite3.connect(path)) as con:
-        con.execute('select * from log')
+            'var',
+            'log',
+            f))
 
-    logfiles = ('neoadmin.log', 'neomaster.log', 'neostorage-0.log')
-    for f in logfiles:
-      check_sqlite_log(
-        os.path.join(
-          self.getComputerPartitionPath('neo-0'),
-          'var',
-          'log',
-          f))
+      self._executeCrontabAtDate('logrotate', '2050-01-01')
 
-    self._executeCrontabAtDate('logrotate', '2050-01-01')
+      for f in logfiles:
+        check_sqlite_log(
+          os.path.join(
+            self.getComputerPartitionPath('neo-0'),
+            'srv',
+            'backup',
+            'logrotate',
+            f'{f}-20500101'))
 
-    for f in logfiles:
-      check_sqlite_log(
-        os.path.join(
-          self.getComputerPartitionPath('neo-0'),
-          'srv',
-          'backup',
-          'logrotate',
-          f'{f}-20500101'))
+      self._executeCrontabAtDate('logrotate', '2050-01-02')
+      requests.get(self._getAuthenticatedZopeUrl('/'), verify=False).raise_for_status()
 
-    self._executeCrontabAtDate('logrotate', '2050-01-02')
-    requests.get(self._getAuthenticatedZopeUrl('/'), verify=False).raise_for_status()
+      for f in logfiles:
+        check_sqlite_log(
+          os.path.join(
+            self.getComputerPartitionPath('neo-0'),
+            'var',
+            'log',
+            f))
 
-    for f in logfiles:
-      check_sqlite_log(
-        os.path.join(
-          self.getComputerPartitionPath('neo-0'),
-          'var',
-          'log',
-          f))
 
 class TestPassword(ERP5InstanceTestCase, TestPublishedURLIsReachableMixin):
   __partition_reference__ = 'p'

software/neoppod/buildout.hash.cfg

@@ -22,7 +22,7 @@ md5sum = c91b5540f94ce76af31f84584df7a3ef
 
 [instance-neo-admin]
 filename = instance-neo-admin.cfg.in
-md5sum = b6e1ccb1d90160110202e5111eec2afa
+md5sum = a0ec1dce4c7a237fbeef3f8aee62e55a
 
 [instance-neo-master]
 filename = instance-neo-master.cfg.in
@@ -34,7 +34,7 @@ md5sum = fda911d5ef9efee365f1b0ff9843a50b
 
 [template-neo-my-cnf]
 filename = my.cnf.in
-md5sum = 56ea8f452d9e1526157ab9d03e631e1a
+md5sum = 3ae93702f3890a504cc8a93eb5ad52bc
 
 [template-neo]
 filename = instance.cfg.in

software/neoppod/instance-neo-admin.cfg.in

@@ -45,7 +45,7 @@ ssl = {{ dumps(bool(slapparameter_dict['ssl'])) }}
 cluster = {{ dumps(slapparameter_dict['cluster']) }}
 masters = {{ dumps(slapparameter_dict['masters']) }}
 extra-options =
-{%- for k, v in monitor_dict.iteritems() %}
+{%- for k, v in six.iteritems(monitor_dict) %}
 {%-   if k == 'backup' %}
 {%-     set k = 'monitor-backup' %}
 {%-   endif %}

software/neoppod/my.cnf.in

@@ -45,7 +45,7 @@ innodb_locks_unsafe_for_binlog = 1
 {{x}}sync_frm = 0
 
 # Extra parameters.
-{%- for k, v in extra_dict.iteritems() %}
+{%- for k, v in six.iteritems(extra_dict) %}
 {%- do assert('-' not in k) %}
 {{ k }} = {{ v }}
 {%- endfor %}

software/neoppod/software-common.cfg

@@ -130,8 +130,11 @@ inline =
 [versions]
 coverage = 5.5
 ecdsa = 0.13
-mysqlclient = 1.3.12
+mysqlclient = 2.0.1
 PyMySQL = 0.10.1
 pycrypto = 2.6.1
 cython-zstd = 0.2
 funcsigs = 1.0.2
+
+[versions:python2]
+mysqlclient = 1.3.12

software/slapos-sr-testing/buildout.hash.cfg

@@ -15,4 +15,4 @@
 
 [template]
 filename = instance.cfg
-md5sum = f10fbca22d1d30dd7a4f36e1cd521b97
+md5sum = 59a5b559b22ad0590e691226cea45055

software/slapos-sr-testing/instance.cfg

@@ -67,6 +67,7 @@ inline =
     command,
     cwd={{ repr(folder) }},
     summaryf=UnitTest.summary,
+    envadj={ 'SLAPOS_SR_TEST_NAME': {{ repr(name) }} },
   )
   {%-   endif %}
   {%- endfor %}

software/slapos-sr-testing/software.cfg

@@ -440,6 +440,7 @@ tests =
   dream ${slapos.test.dream-setup:setup}
   dufs ${slapos.test.dufs-setup:setup}
   erp5 ${slapos.test.erp5-setup:setup}
+  erp5-py3 ${slapos.test.erp5-setup:setup}
   erp5testnode ${slapos.test.erp5testnode-setup:setup}
   fluentd ${slapos.test.fluentd-setup:setup}
   galene ${slapos.test.galene-setup:setup}
@@ -495,7 +496,6 @@ pysftp = 0.2.9
 requests-toolbelt = 0.8.0
 testfixtures = 6.11.0
 mysqlclient = 2.1.1
-pexpect = 4.8.0
 ptyprocess = 0.6.0
 paho-mqtt = 1.5.0
 pcpp = 1.30

stack/erp5/buildout-py3.cfg

+[buildout]
+extends =
+# Required by PyStemmer 2.0.1
+  ../../component/cython/buildout.cfg
+  buildout.cfg
+
+
+parts -=
+# XXX: The following parts have no support for Python 3. Ignore for now.
+  rdiff-backup
+  ocropy
+  hookbox
+  scipy
+  scikit-learn
+  scikit-image
+  PyWavelets
+  jupyter-notebook-initialized-scripts
+  jupyter
+  matplotlibrc
+  h5py
+
+find-links +=
+  http://www.nexedi.org/static/packages/source/
+  http://www.nexedi.org/static/packages/source/hexagonit.recipe.download/
+  http://www.nexedi.org/static/packages/source/slapos.buildout/
+
+[python]
+part = python3
+
+[slapos.cookbook-repository]
+develop = true
+branch = zope4py3
+
+[PyStemmer]
+recipe = zc.recipe.egg:custom
+egg = PyStemmer
+setup-eggs =
+    ${cython:egg}
+    setuptools-dso
+
+[h5py]
+recipe =
+[matplotlibrc]
+recipe =
+[matplotlib]
+recipe =
+[jupyter]
+recipe =
+[rdiff-backup]
+recipe =
+[rdiff-backup-build]
+recipe =
+[ocropy]
+recipe =
+
+[hookbox]
+recipe =
+[scipy]
+recipe =
+[scikit-learn]
+recipe =
+[PyWavelets]
+recipe =
+[scikit-image]
+recipe =
+[statsmodels]
+recipe =
+[seaborn]
+recipe =
+
+[python-ldap-python]
+# No patch needed anymore
+patches =
+include-dirs -=
+  ${cyrus-sasl:location}/include/sasl
+include-dirs +=
+  ${cyrus-sasl:location}/include
+
+[eggs]
+
+eggs -=
+  ${ocropy:egg}
+#  fpconst
+  ${scipy:egg}
+  ${scikit-learn:egg}
+  ${scikit-image:egg}
+  ${matplotlib:egg}
+  h5py
+  ${h5py:egg}
+  ${statsmodels:egg}
+  astor
+  jupyter
+  jupyter_client
+  jupyter_console
+  jupyter_core
+  nbconvert
+  nbformat
+  notebook
+# additional eggs
+  astor
+  ${seaborn:egg}
+  ${statsmodels:egg}
+# additional eggs for jupyterlab
+  jupyterlab
+  jupyterlab-launcher
+  pytesseract
+
+
+
+[versions]
+pyasn1-modules = 0.2.8
+pyasn1 = 0.4.8
+
+google-api-core = 2.7.1
+google-auth-httplib2 = 0.1.0
+google-auth = 2.6.0
+protobuf = 3.19.4
+googleapis-common-protos = 1.55.0
+scipy = 1.8.1
+statsmodels = 0.10.2
+pyzmq = 19.0.2
+python-ldap = 3.1.0
+
+matplotlib-inline = 0.1.3
+backcall = 0.2.0
+# 2to3 fixers to port code to python3
+zope.fixers = 1.1.2
+defusedxml = 0.7.1
+suds-py3 = 1.4.5.0
+wstools-py3 = 0.54.4
+soupsieve = 2.3.2.post1:whl
+
+# WIP pylint
+[versions]
+pylint = 2.17.0:whl
+mccabe = 0.7.0:whl
+tomli = 2.0.1:whl
+lazy-object-proxy = 1.4.3
+isort = 4.2.5
+astroid = 2.15.0:whl
+wrapt = 1.11
+typed-ast = 1.4.0
+tomlkit = 0.11.6
+platformdirs = 3.1.1:whl
+dill = 0.3.6:whl
+
+[erp5]
+repository = https://lab.nexedi.com/nexedi/erp5.git
+branch = zope4py3
+develop = true

stack/erp5/buildout.hash.cfg

@@ -74,7 +74,7 @@ md5sum = ca0cb83950dd9079cc289891cce08e76
 
 [template-erp5]
 filename = instance-erp5.cfg.in
-md5sum = 6f57c834eb3f774d265c3fd6661429d8
+md5sum = 9f6e95f54c8bef0bbdfead015887a4e2
 
 [template-zeo]
 filename = instance-zeo.cfg.in
@@ -86,11 +86,11 @@ md5sum = 0ac4b74436f554cd677f19275d18d880
 
 [template-zope]
 filename = instance-zope.cfg.in
-md5sum = e025aefe55f3be982157dcd3748e7fe1
+md5sum = 28c0d1bab2e58ba10a23256fea934e6c
 
 [template-balancer]
 filename = instance-balancer.cfg.in
-md5sum = 0fad9497da12ed0186dca5236c23f3a7
+md5sum = 3bdf08bf6ef735647fa54ce2ef273201
 
 [template-haproxy-cfg]
 filename = haproxy.cfg.in

stack/erp5/instance-balancer.cfg.in

@@ -471,10 +471,9 @@ command = generate-apachedex-report
 recipe = slapos.recipe.template
 output = ${directory:etc}/${:_buildout_section_name_}
 inline =
-  {% for line in slapparameter_dict['apachedex-configuration'] %}
+  {% for line in slapparameter_dict['apachedex-configuration'] -%}
     {# apachedex config files use shlex.split, so we need to quote the arguments. #}
-    {# BBB: in python 3 we can use shlex.quote instead. #}
-    {{ repr(line.encode('utf-8')) }}
+    {{ six.moves.shlex_quote(line) }}
   {% endfor %}
 
 [apachedex-parameters]

stack/erp5/instance-erp5.cfg.in

@@ -200,7 +200,7 @@ config-zodb-dict = {{ dumps(zodb_dict) }}
 {% for server_type, server_dict in six.iteritems(storage_dict) -%}
 {%   if server_type == 'neo' -%}
 config-neo-cluster = ${publish-early:neo-cluster}
-config-neo-name = {{ server_dict.keys()[0] }}
+config-neo-name = {{ list(server_dict.keys())[0] }}
 config-neo-masters = ${publish-early:neo-masters}
 {%   else -%}
 config-zodb-zeo = ${request-zodb:connection-storage-dict}

stack/erp5/instance-zope.cfg.in

@@ -80,6 +80,8 @@ environment +=
   TZ={{ slapparameter_dict['timezone'] }}
   MATPLOTLIBRC={{ parameter_dict['matplotlibrc'] }}
   PYTHONUNBUFFERED=1
+  OFS_IMAGE_USE_DENYLIST=1
+  DISALLOWED_INLINE_MIMETYPES=
   INSTANCE_HOME=${:instance-home}
   FONTCONFIG_FILE=${fontconfig-conf:output}
   JUPYTER_PATH=${directory:jupyter-dir}
@@ -401,8 +403,8 @@ config-port = {{ '${' ~ zope_tunnel_section_name ~ ':ipv6-port}' }}
 promise = check_error_on_zope_longrequest_log
 name = {{'check-' ~ name ~ '-longrequest-error-log.py'}}
 config-log-file = {{ '${' ~ conf_parameter_name ~ ':longrequest-logger-file}' }}
-config-error-threshold = {{ slapparameter_dict["zope-longrequest-logger-error-threshold"] }}
-config-maximum-delay = {{ slapparameter_dict["zope-longrequest-logger-maximum-delay"] }}
+config-error-threshold = {{ dumps(slapparameter_dict["zope-longrequest-logger-error-threshold"]) }}
+config-maximum-delay = {{ dumps(slapparameter_dict["zope-longrequest-logger-maximum-delay"]) }}
 {% endif -%}
 
 [{{ section('logrotate-entry-' ~ name) }}]

stack/erp5/zope-versions.cfg

@@ -2,150 +2,94 @@
 # Version pins for required and commonly used dependencies.
 
 [versions]
-Zope = 4.8.9
+Zope = 5.9
 Zope2 = 4.0
-# AccessControl 5+ no longer supports Zope 4.
-AccessControl = 4.4
-Acquisition = 4.13
-AuthEncoding = 4.3
-BTrees = 4.11.3
-Chameleon = 3.10.2
-DateTime = 4.9
-DocumentTemplate = 4.1
-ExtensionClass = 4.9
-Missing = 4.2
-MultiMapping = 4.1
-Paste = 3.5.2
-PasteDeploy = 3.0.1
-Persistence = 3.6
-Products.BTreeFolder2 = 4.4
-# ZCatalog 6+ no longer supports Zope 4.
-Products.ZCatalog = 5.4
-Record = 3.6
-# RestrictedPython >= 6 no longer supports Zope 4
-RestrictedPython = 5.4
+AccessControl = 6.3
+Acquisition = 5.1
+AuthEncoding = 5.0
+BTrees = 5.1
+Chameleon = 4.2.0
+DateTime = 5.3
+DocumentTemplate = 4.6
+ExtensionClass = 5.1
+MultiMapping = 5.0
+Paste = 3.7.1
+PasteDeploy = 3.1.0
+Persistence = 4.1
+RestrictedPython = 7.0
+WebTest = 3.0.0
 WSGIProxy2 = 0.5.1
 WebOb = 1.8.7
-WebTest = 3.0.0
-ZConfig = 3.6.1
-ZEO = 5.3.0
-ZODB = 5.8.0
-five.globalrequest = 99.1
-five.localsitemanager = 3.4
-funcsigs = 1.0.2
-future = 0.18.2
-ipaddress = 1.0.23
-mock = 4.0.3
+ZConfig = 4.0
+ZODB = 5.8.1
+beautifulsoup4 = 4.12.2
+cffi = 1.16.0
 multipart = 0.2.4
-pbr = 5.11.0
-persistent = 4.9.3
-pytz = 2022.7
-roman = 3.3
-shutilwhich = 1.1.0
+persistent = 5.1
+pycparser = 2.21
+python-gettext = 5.0
+pytz = 2023.3.post1
 six = 1.16.0
-transaction = 3.0.1
+roman = 4.1
+soupsieve = 2.5
+transaction = 4.0
 waitress = 2.1.2
-z3c.pt = 3.3.1
-zExceptions = 4.3
-zc.lockfile = 2.0
-zdaemon = 4.4
-zodbpickle = 2.6
-zope.annotation = 4.8
-zope.browser = 2.4
-zope.browsermenu = 4.4
-zope.browserpage = 4.4.0
-zope.browserresource = 4.4
-zope.cachedescriptors = 4.4
-zope.component = 5.0.1
-zope.componentvocabulary = 2.3.0
-zope.configuration = 4.4.1
-zope.container = 5.1
-zope.contentprovider = 4.2.1
-zope.contenttype = 4.6
-zope.datetime = 4.3.0
-zope.deferredimport = 4.4
-zope.deprecation = 4.4.0
-zope.dottedname = 5.0
-zope.event = 4.6
-zope.exceptions = 4.6
-zope.filerepresentation = 5.0.0
-zope.formlib = 5.0.1
-zope.globalrequest = 1.6
-zope.hookable = 5.4
-zope.i18n = 4.9.0
-zope.i18nmessageid = 5.1.1
-zope.interface = 5.5.2
-zope.lifecycleevent = 4.4
-zope.location = 4.3
-zope.pagetemplate = 4.6.0
-zope.processlifetime = 2.4
-zope.proxy = 4.6.1
-zope.ptresource = 4.3.0
-zope.publisher = 6.1.0
-zope.ramcache = 2.4
-zope.schema = 6.2.1
-zope.security = 5.8
-zope.sendmail = 5.3
-zope.sequencesort = 4.2
-zope.site = 4.6.1
-zope.size = 4.4
-zope.structuredtext = 4.4
-zope.tal = 4.5
-zope.tales = 5.2
-zope.testbrowser = 5.6.1
-zope.testing = 4.10
-zope.testrunner = 5.6
-zope.traversing = 4.4.1
-zope.viewlet = 4.3
-
-[versions:python27]
-# Chameleon 3.10 doesn't work on Python 2.7
-Chameleon = 3.9.1
-# DocumentTemplate 4+ requires Python 3.5 or higher
-DocumentTemplate = 3.4
-# PasteDeploy >3 requires Python 3.7
-PasteDeploy = 2.1.1
-# WSGIProxy 0.5 and up requires Python 3.7 and up
-WSGIProxy2 = 0.4.6
-# WebTest 3.0 and up requires Python 3.6 and up
-WebTest = 2.0.35
-# ZServer is only available for Python 2
-ZServer = 4.0.2
-# mock 4.0 and up requires Python 3.6 or higher
-mock = 3.0.5
-# multipart 0.2 and up requires Python 3
-multipart = 0.1.1
-# waitress 2 requires Python 3.6 or higher
-waitress = 1.4.4
-# zope.dottedname >= 5 requires Python 3.6 or higher
-zope.dottedname = 4.3
-# zope.container 5.x requires Python 3.7 or higher
-zope.container = 4.10
+z3c.pt = 4.0
+zExceptions = 5.0
+zc.lockfile = 3.0.post1
+zc.recipe.egg = 2.0.7
+zodbpickle = 3.1
+zope.annotation = 5.0
+zope.browser = 3.0
+zope.browsermenu = 5.0
+zope.browserpage = 5.0
+zope.browserresource = 5.1
+zope.cachedescriptors = 5.0
+zope.component = 6.0
+zope.configuration = 5.0
+zope.container = 5.2
+zope.contentprovider = 5.0
+zope.contenttype = 5.1
+zope.datetime = 5.0.0
+zope.deferredimport = 5.0
+zope.deprecation = 5.0
+zope.dottedname = 6.0
+zope.event = 5.0
+zope.exceptions = 5.0.1
+zope.filerepresentation = 6.0
+zope.globalrequest = 2.0
+zope.hookable = 6.0
+zope.i18n = 5.1
+zope.i18nmessageid = 6.1.0
+zope.interface = 6.1
+zope.lifecycleevent = 5.0
+zope.location = 5.0
+zope.pagetemplate = 5.0
+zope.processlifetime = 3.0
+zope.proxy = 5.1
+zope.ptresource = 5.0
+zope.publisher = 7.0
+zope.schema = 7.0.1
+zope.security = 6.2
+zope.sequencesort = 5.0
+zope.site = 5.0
+zope.size = 5.0
+zope.structuredtext = 5.0
+zope.tal = 5.0.1
+zope.tales = 6.0
+zope.testbrowser = 6.0
+zope.testing = 5.0.1
+zope.traversing = 5.0
+zope.viewlet = 5.0
 
-[versions:python35]
-# DocumentTemplate 4+ cannot be installed on Zope 4 for Python 3.5
-DocumentTemplate = 3.4
-# PasteDeploy >3 requires Python 3.7
-PasteDeploy = 2.1.1
-# WSGIProxy 0.5 and up requires Python 3.7 and up
-WSGIProxy2 = 0.4.6
-# WebTest 3.0 and up requires Python 3.6 and up
-WebTest = 2.0.35
-# mock 4.0 and up requires Python 3.6 or higher
-mock = 3.0.5
-# waitress 2 requires Python 3.6 or higher
-waitress = 1.4.4
-# zope.dottedname >= 5 requires Python 3.6 or higher
-zope.dottedname = 4.3
-# zope.container 5.x requires Python 3.7 or higher
-zope.container = 4.10
 
-[versions:python36]
-# PasteDeploy >3 requires Python 3.7
-PasteDeploy = 2.1.1
-# WSGIProxy 0.5 and up requires Python 3.7 and up
-WSGIProxy2 = 0.4.6
-# waitress 2.1 requires Python 3.7 or higher
-waitress = 2.0.0
-# zope.container 5.x requires Python 3.7 or higher
-zope.container = 4.10
+# XXX this is commented out because slapos.buildout is based on a too old buildout
+# which does not understands :python37 yet. We target a more recent python version
+# so we don't use these versions.
+# [versions:python37]
+# # PasteDeploy 3.x works on Python 3.7 but pulls tons of dependencies
+# PasteDeploy = 2.1.1
+# # SoupSieve 2.5 and up requires Python 3.8
+# soupsieve = 2.4.1
+# # cffi 1.16.0 requires Python 3.8
+# cffi = 1.15.1

stack/slapos.cfg

@@ -356,9 +356,9 @@ zipp = 3.12.0:whl
 zodburi = 2.5.0
 zope.event = 4.6.0
 zope.exceptions = 4.6
-zope.interface = 5.4.0
-zope.testing = 4.7
-zope.testrunner = 5.2
+zope.interface = 5.5.2
+zope.testing = 4.10
+zope.testrunner = 5.6
 
 [versions:sys.version_info < (3,10)]
 # keep old statsmodels by default until slapos.toolbox is updated