component/ZODB

- factor bits that constitute ZODB stack into component/ZODB;
- add support for testing ZODB on Nexedi testing infrastructure;

This are preparatory steps to integrate
nexedi/ZODB!1.

Please see details in the individual commits.

/cc @nexedi
/proposed-for-review-on: nexedi/slapos!867

component/ZEO/buildout.cfg

@@ -3,22 +3,48 @@
 
 [buildout]
 extends =
+    ../ZODB/buildout.cfg
     ../git/buildout.cfg
 
 parts = ZEO/scripts
 
 
-# ZEO allows to use either ZEO4 or ZEO5
-# To select which version to use users should do:
+# ZEO provides ZEO<X> depending on ZODB major version.
 #
-#   [ZEO]
-#   egg = ${ZEO<version>:egg}
-#
-# By default ZEO4 is used.
+# - ZEO3 is part of ZODB3 distribution
+#   https://github.com/zopefoundation/ZODB/tree/3.10.7-4-gb8d7a8567/src
+# - ZEO4 works only with ZODB4
+#   https://github.com/zopefoundation/ZEO/blob/4.3.1-1-g47d3fbe8/setup.py#L122
+# - ZEO5 works only with ZODB5
+#   https://github.com/zopefoundation/ZEO/blob/5.2.2-1-g3d90ed42/setup.py#L20
 [ZEO]
-recipe  = zc.recipe.egg:eggs
-egg     = ${ZEO4:egg}
-eggs    = ${:egg}
+recipe  = slapos.recipe.build
+depends = ${ZODB:egg}
+init =
+  zodb = self.buildout['ZODB']
+  zmajor = zodb['major']
+
+  # ZODB3 -> just link ZEO to ZODB3 (ZEO3 is part of ZODB3)
+  if zmajor == '3':
+    options['egg'] = zodb['egg']
+
+  # ZODB{4,5}: link/depend to ZEO<ZODB.major>
+  else:
+    zeo_x = self.buildout['ZEO'+zmajor]
+    options['depends'] += '$${%s:egg}' % zeo_x.name
+    options['egg'] = zeo_x['egg']
+
+    # update [versions] from what is needed by ZEO<X>
+    for _ in zeo_x['egg_versions'].splitlines():
+      if _ == '' or _.startswith('#'):
+          continue
+      egg, eq, version = _.split() # 'transaction = 1.7.0'
+      assert eq == '='
+      self.buildout['versions'][egg] = version
+    # propagate updated [versions] -> easy_install
+    # (buildout does this in Buildout constructor)
+    import zc.buildout.easy_install
+    zc.buildout.easy_install.default_versions(self.buildout['versions'])
 
 
 # ZEO/scripts installs scripts from ZEO
@@ -32,6 +58,7 @@ eggs    = ${ZEO:egg}
 recipe  = zc.recipe.egg:develop
 setup   = ${ZEO4-repository:location}
 egg     = ZEO
+egg_versions =
 
 [ZEO4-repository]
 recipe  = slapos.recipe.build:gitclone
@@ -47,3 +74,8 @@ git-executable = ${git:location}/bin/git
 recipe  = zc.recipe.egg:eggs
 egg     = ZEO
 eggs    = ${:egg}
+
+egg_versions =
+  ZEO = 5.2.2
+  trollius = 2.2.post1
+  futures = 3.2.0

component/ZODB/buildout-dev.cfg

+# SlapOS component for ZODB development.
+
+[buildout]
+extends = buildout.cfg
+
+# unpin ZODB<X> from particular revisions; only branch is left.
+[ZODB3-repository]
+revision=
+[ZODB4-repository]
+revision=
+[ZODB5-repository]
+revision=

component/ZODB/buildout.cfg

+# SlapOS component for ZODB.
+# https://zodb.org/
+
+[buildout]
+extends =
+    ../git/buildout.cfg
+    ../python-cffi/buildout.cfg
+
+parts = ZODB/scripts
+
+
+# ZODB allows to use either ZODB3, ZODB4 or ZODB5.
+# To select which version to use users should do:
+#
+#   [ZODB]
+#   major = <ZODB-version-major>
+#
+# By default ZODB4 is used.
+[ZODB]
+recipe = slapos.recipe.build
+major  = 4
+init =
+  # link/depend ZODB -> ZODB<X>
+  zodb_x = 'ZODB'+options['major']
+  zodb_x = self.buildout[zodb_x]
+  options['depends'] = '$${%s:egg}' % zodb_x.name
+
+  options['egg'] = zodb_x['egg']
+  options['location'] = zodb_x['setup']
+
+  # for ZODB3 also depend on _ZODB3-{ZODB,persistent,BTrees,ZEO}.
+  # installing those eggs ensures forward compatibility with ZODB4/5 where
+  # instead of single ZODB3 egg, there are separate ZODB,persistent,BTrees,... distributions.
+  if options['major'] == '3':
+    for egg in ['ZODB','persistent','BTrees','ZEO']:
+      z3egg = self.buildout['_ZODB3-%s' % egg]
+      options['depends'] += '\n$${%s:egg}' % z3egg.name
+
+  # update [versions] from what is needed by ZODB<X>
+  for _ in zodb_x['egg_versions'].splitlines():
+    if _ == '' or _.startswith('#'):
+        continue
+    egg, eq, version = _.split() # 'transaction = 1.7.0'
+    assert eq == '='
+    self.buildout['versions'][egg] = version
+  # propagate updated [versions] -> easy_install
+  # (buildout does this in Buildout constructor)
+  import zc.buildout.easy_install
+  zc.buildout.easy_install.default_versions(self.buildout['versions'])
+
+
+# ZODB/scripts installs scripts from ZODB
+[ZODB/scripts]
+recipe  = zc.recipe.egg:scripts
+eggs    = ${ZODB:egg}
+
+
+[_ZODB-git-repository]
+recipe  = slapos.recipe.build:gitclone
+repository = https://lab.nexedi.com/nexedi/ZODB.git
+git-executable = ${git:location}/bin/git
+
+[_ZODB45]
+recipe  = zc.recipe.egg:develop
+egg     = ZODB
+depends = ${persistent:egg} ${BTrees:egg}
+
+# eggs versions that are used by all ZODB{4,5}.
+egg_versions =
+  BTrees = 4.5.1
+  zodbpickle = 1.0.4
+
+  # Provide ZODB3 for those eggs that still care about ZODB3 compatibility -
+  # for example wendelin.core. ZODB3 3.11 is just a dependency egg on _latest_
+  # ZODB, persistent, BTrees and ZEO.
+  ZODB3 = 3.11.0
+
+
+# ZODB4: we maintain our own 4-nxd branch with patches for wendelin.core to work
+# correctly. The patches are backports of what is in ZODB master (ZODB5).
+[ZODB4]
+<= _ZODB45
+setup = ${ZODB4-repository:location}
+
+egg_versions +=
+  transaction = 1.7.0
+
+[ZODB4-repository]
+<= _ZODB-git-repository
+location = ${buildout:parts-directory}/ZODB4
+branch   = 4-nxd
+revision = 4.4.5-8-g401163753
+
+
+# ZODB3: similarly to ZODB4 we maintain our 3.10-nxd branch.
+[ZODB3]
+recipe  = zc.recipe.egg:develop
+setup   = ${ZODB3-repository:location}
+egg     = ZODB3
+
+egg_versions =
+  transaction = 1.6.1
+  zdaemon = 2.0.7
+
+[ZODB3-repository]
+<= _ZODB-git-repository
+location = ${buildout:parts-directory}/ZODB3
+branch   = 3.10-nxd
+revision = 3.10.7-9-gc72883108
+
+# Stub eggs for ZODB3 to be forward-compatible with ZODB4/5.
+[_ZODB3-ZODB45-compat-egg]
+recipe  = zc.recipe.egg:develop
+setup   = ${ZODB3-repository:location}/ZODB45-compat/${:egg}/setup.py
+depends = ${ZODB3:recipe}
+
+[_ZODB3-ZODB]
+<= _ZODB3-ZODB45-compat-egg
+egg = ZODB
+
+[_ZODB3-persistent]
+<= _ZODB3-ZODB45-compat-egg
+egg = persistent
+
+[_ZODB3-BTrees]
+<= _ZODB3-ZODB45-compat-egg
+egg = BTrees
+
+[_ZODB3-ZEO]
+<= _ZODB3-ZODB45-compat-egg
+egg = ZEO
+
+
+# ZODB5: pristine upstream
+[ZODB5]
+<= _ZODB45
+setup = ${ZODB5-repository:location}
+
+egg_versions +=
+  transaction = 2.4.0
+
+[ZODB5-repository]
+<= _ZODB-git-repository
+location = ${buildout:parts-directory}/ZODB5
+branch   = master
+revision = 5.6.0-0-g3d06c0462
+
+
+# XXX: buildout fails to install properly eggs with setup_requires
+[BTrees]
+recipe = zc.recipe.egg:custom
+egg = ${:_buildout_section_name_}
+setup-eggs = ${persistent:egg}
+
+[persistent]
+recipe = zc.recipe.egg:develop
+egg = ${:_buildout_section_name_}
+setup-eggs = ${python-cffi:egg}
+setup = ${persistent-repository:location}
+
+[persistent-repository]
+<= _ZODB-git-repository
+location = ${buildout:parts-directory}/persistent
+repository = https://github.com/zopefoundation/persistent.git
+branch = master
+revision = 4.6.4-0-g7ed95cf

component/ZODB/test-common.cfg

+# SlapOS software release to test ZODB on Nexedi testing infrastructure.
+# Common parts.
+
+[buildout]
+extends =
+# test<X>.cfg configures ZODB.major=<X>.
+
+    ../../stack/nxdtest.cfg
+    ../pygolang/buildout.cfg
+    ../python-manuel/buildout.cfg
+    buildout-dev.cfg
+    ../ZEO/buildout.cfg
+
+parts =
+    ZODB
+    .nxdtest
+
+#   for instance
+    ZODB-python
+    slapos-cookbook
+    instance.cfg
+
+
+# test-dependent eggs that must come through in-tree recipes.
+[ZODB]
+depends += ${manuel:egg}
+
+
+# bin/python is preinstalled with sys.path to ZODB & friends.
+[ZODB-python]
+<= python-interpreter
+eggs +=
+  ${ZODB:egg}[test]
+  ${ZEO:egg}[test]
+  transaction[test]
+
+# env.sh for ZODB's python to be on $PATH.
+[ZODB-env.sh]
+recipe   = slapos.recipe.template:jinja2
+rendered = ${buildout:directory}/${:_buildout_section_name_}
+template = inline:
+  export PS1="(ZODB-env) $PS1"
+  export PATH=${buildout:bin-directory}:$PATH
+
+# .nxdtest to run ZODB tests
+[.nxdtest]
+<= jinja2-template
+template = inline:
+  # determine where ZODB & friends were installed
+  from subprocess import check_output
+  _ = check_output(['python', '-c',
+                    'from os.path import dirname\n'+
+                    'def X(mod): m=__import__(mod); print(dirname(m.__file__))\n\n'+
+                    'X("ZODB"); X("persistent"); X("BTrees"); X("transaction"), X("ZEO")'])
+  ZODB, persistent, BTrees, transaction, ZEO = _.split()
+
+  # run tests for whole ZODB stack
+  TestCase('ZODB/unit',       ['python', '-m', 'zope.testrunner', '-uv', '--package-path', ZODB, 'ZODB'])
+  TestCase('ZODB/functional', ['python', '-m', 'zope.testrunner', '-fv', '--package-path', ZODB, 'ZODB'])
+
+  TestCase('ZEO/unit',       ['python', '-m', 'zope.testrunner', '-uv', '--package-path', ZEO, 'ZEO'])
+  TestCase('ZEO/functional', ['python', '-m', 'zope.testrunner', '-fv', '--package-path', ZEO, 'ZEO'])
+
+  TestCase('persistent', ['python', '-m', 'zope.testrunner', '-v', '--package-path', persistent, 'persistent'])
+  TestCase('BTrees',     ['python', '-m', 'zope.testrunner', '-v', '--package-path', BTrees, 'BTrees'])
+
+  # transaction uses unittest instead of zope.testrunner
+  import os.path
+  def P(path): return os.path.join(path, '..')
+  TestCase('transaction',['python', '-m', 'unittest', 'discover', '-s', P(transaction)])
+
+
+# instance to run nxdtest.
+[instance.cfg]
+<= jinja2-template
+template = inline:
+  [buildout]
+  extends = ${nxdtest-instance.cfg:rendered}
+
+  [runTestSuite]
+  env.sh  = ${ZODB-env.sh:rendered}
+  workdir = ${buildout:directory}

component/ZODB/test-zodb3.cfg

+# SlapOS software release to test ZODB3 on Nexedi testing infrastructure.
+
+[buildout]
+extends = test-common.cfg
+
+[ZODB]
+major = 3

component/ZODB/test-zodb5.cfg

+# SlapOS software release to test ZODB5 on Nexedi testing infrastructure.
+
+[buildout]
+extends = test-common.cfg
+
+[ZODB]
+major = 5
+
+
+# NOTE with default zope.testing=3.9.7 it fails with
+#    setupstack.TestCase -> AttributeError: 'module' object has no attribute 'TestCase'
+[versions]
+zope.testing = 4.7

component/ZODB/test.cfg

+# SlapOS software release to test ZODB on Nexedi testing infrastructure.
+
+[buildout]
+extends = test-common.cfg
+
+[ZODB]
+major = 4

component/wendelin.core/buildout.cfg

@@ -3,6 +3,7 @@
 [buildout]
 extends =
   ../pygolang/buildout.cfg
+  ../ZODB/buildout.cfg
   ../zodbtools/buildout.cfg
   ../numpy/buildout.cfg
   ../golang/buildout.cfg
@@ -24,6 +25,7 @@ environment = wendelin.core-env
 
 # dependent eggs that must come through in-tree recipes
 depends =
+    ${ZODB:egg}
     ${zodbtools:egg}
     ${numpy:egg}
 

component/wendelin.core/test-common.cfg

@@ -56,6 +56,4 @@ template = inline:
 [versions]
 neoppod = 1.12.0
 
-zope.testrunner = 5.2
-random2 = 1.0.1
 manuel = 1.10.1

component/zodbtools/buildout.cfg

@@ -4,6 +4,7 @@
 [buildout]
 extends =
     ../pygolang/buildout.cfg
+    ../ZODB/buildout.cfg
     ../ZEO/buildout.cfg
 
 parts =
@@ -19,6 +20,7 @@ eggs    =
 # dependent eggs that must come through in-tree recipes
 depends =
     ${pygolang:egg}
+    ${ZODB:egg}
 # ZEO comes through zodbtools -> zodburi -> ZEO
     ${ZEO:egg}
 

component/zodbtools/test-common.cfg

@@ -3,8 +3,7 @@
 
 [buildout]
 extends =
-# test*.cfg first extend from neoppod/software<ZODB-flabour>.cfg to use
-# appropriate ZODB and versions of other components.
+# test<X>.cfg configures ZODB.major=<X>.
 
     ../../stack/nxdtest.cfg
     ../pytest/buildout.cfg

component/zodbtools/test-zodb5.cfg

 # SlapOS software release to test zodbtools/ZODB5 on Nexedi testing infrastructure.
 
 [buildout]
-extends =
-    ../../software/neoppod/software-zodb5.cfg
-    test-common.cfg
+extends = test-common.cfg
+
+[ZODB]
+major = 5

component/zodbtools/test.cfg

 # SlapOS software release to test zodbtools on Nexedi testing infrastructure.
 
 [buildout]
-extends =
-    ../../software/neoppod/software.cfg
-    test-common.cfg
+extends = test-common.cfg

software/neoppod/software-common.cfg

@@ -18,6 +18,7 @@ extends =
     ../../component/python-mysqlclient/buildout.cfg
     ../../component/python-cryptography/buildout.cfg
     ../../component/pycurl/buildout.cfg
+    ../../component/ZODB/buildout.cfg
     ../../component/ZEO/buildout.cfg
     ../../component/zodbtools/buildout.cfg
 
@@ -41,32 +42,23 @@ recipe = zc.recipe.egg:develop
 setup = ${neoppod-repository:location}
 environment = neoppod-setup-env
 
-[neoppod]
-recipe = zc.recipe.egg
-eggs = neoppod[admin, ctl, master]
+# dependent eggs that must come through in-tree recipes
+intree-eggs =
   ${cython-zstd:egg}
   ${msgpack-python:egg}
   ${:adapter-egg}
-  ${BTrees:egg}
+  ${ZODB:egg}
+  ${zodbtools:egg}
+adapter-egg = ${python-mysqlclient:egg}
+
+[neoppod]
+recipe = zc.recipe.egg
+eggs = neoppod[admin, ctl, master]
+  ${neoppod-develop:intree-eggs}
   psutil
-  ZODB
   zope.testing
-  ${zodbtools:egg}
   coverage
   setproctitle
-adapter-egg = ${python-mysqlclient:egg}
-
-# XXX: buildout fails to install properly eggs with setup_requires
-[BTrees]
-recipe = zc.recipe.egg:custom
-egg = ${:_buildout_section_name_}
-setup-eggs = ${persistent:egg}
-
-[persistent]
-recipe = zc.recipe.egg:custom
-egg = ${:_buildout_section_name_}
-setup-eggs = ${python-cffi:egg}
-###
 
 [download-base-neo]
 recipe = slapos.recipe.build:download
@@ -136,21 +128,11 @@ template =
   {% endraw %}
 
 [versions]
-BTrees = 4.5.1
-ZODB = 4.4.5
 coverage = 4.5.1
-mock = 3.0.5
 ecdsa = 0.13
 mysqlclient = 1.3.12
-persistent = 4.6.4
 pycrypto = 2.6.1
 pycurl = 7.43.0
 setproctitle = 1.1.10
-transaction = 1.7.0
-zodbpickle = 1.0.4
 cython-zstd = 0.2
 python-dateutil = 2.7.3
-
-# Required by:
-# mock = 3.0.5
-funcsigs = 1.0.2

software/neoppod/software-zodb3.cfg

 [buildout]
 extends = software.cfg
 
+[ZODB]
+major = 3
+
 [neoppod]
 eggs = neoppod[admin, ctl, master, storage-importer, storage-mysqldb, tests]
   ${python-mysqlclient:egg}
-  ZODB3
-patch-binary = ${patch:location}/bin/patch
-ZODB3-patch-options = -p1
-ZODB3-patches =
-  ${neoppod-repository:location}/ZODB3.patch
-
-[versions]
-ZODB3 = 3.10.7+SlapOSPatched001
-transaction = 1.1.1
-zdaemon = 2.0.7

software/neoppod/software-zodb5.cfg

 [buildout]
 extends = software.cfg
 
+[ZODB]
+major = 5
+
 [neoppod]
 eggs += mock
-
-[ZEO]
-egg = ${ZEO5:egg}
-
-[versions]
-ZODB = 5.6.0
-ZEO = 5.2.2
-transaction = 2.4.0
-
-# BBB: ZEO
-msgpack = 0.5.6
-
-# Required by:
-# ZEO==5.2.0
-# trollius==2.2.post1
-futures = 3.2.0
-
-# Required by:
-# ZEO==5.2.0
-trollius = 2.2.post1

software/neoppod/software.cfg

@@ -31,12 +31,3 @@ url = ${:_profile_base_location_}/${:_update_hash_filename_}
 recipe = zc.recipe.egg
 eggs = erp5.util
 interpreter = ${:_buildout_section_name_}
-
-[versions]
-# To match ERP5
-ZConfig = 2.9.3
-zc.lockfile = 1.0.2
-zope.event = 3.5.2
-zope.exceptions = 3.6.2
-zope.testing = 3.9.7
-##

software/neoppod/sqlite-mixin.cfg

 # To be extended after software-common.cfg if only SQLite backend is wanted.
 
-[neoppod]
+[neoppod-develop]
 adapter-egg =
 
 [instance-common]

software/neotest/software.cfg

@@ -129,22 +129,3 @@ eggs    =
     ${wendelin.core:egg}
 
 
-# ping eggs versions
-[versions]
-ZODB3 = 3.11.0
-zope.testing = 4.6.2
-
-# Required by:
-# ZEO==4.3.1
-# ZODB==4.4.5
-# zodburi==2.3.0
-ZConfig = 3.2.0
-
-# Required by:
-# ZEO==4.3.1
-# ZODB==4.4.5
-zc.lockfile = 1.2.1
-
-# Required by:
-# neoppod==1.8.1
-python-dateutil = 2.7.1

stack/erp5/buildout.cfg

@@ -664,7 +664,6 @@ PyXML = 0.8.5
 Pympler = 0.4.3
 StructuredText = 2.11.1
 WSGIUtils = 0.7
-ZODB3 = 3.11.0
 # astroid 1.4.1 breaks testDynamicClassGeneration
 astroid = 1.3.8
 erp5diff = 0.8.1.7

stack/slapos.cfg

@@ -155,6 +155,7 @@ Importing = 1.10
 MarkupSafe = 1.0
 PyYAML = 5.4.1
 Werkzeug = 1.0.1
+ZConfig = 2.9.3
 asn1crypto = 1.3.0
 atomicwrites = 1.4.0
 backports.functools-lru-cache = 1.6.1
@@ -181,6 +182,7 @@ itsdangerous = 0.24
 lock-file = 2.0
 lxml = 4.6.2
 meld3 = 1.0.2
+mock = 3.0.5
 more-itertools = 5.0.0
 netaddr = 0.7.19
 pathlib2 = 2.3.5
@@ -193,6 +195,7 @@ py = 1.9.0
 pyOpenSSL = 19.1.0
 pyparsing = 2.2.0
 pytz = 2016.10
+random2 = 1.0.1
 regex = 2020.9.27
 requests = 2.24.0
 scandir = 1.10.0
@@ -215,10 +218,14 @@ unicodecsv = 0.14.1
 wcwidth = 0.2.5
 wheel = 0.35.1
 xml-marshaller = 1.0.2
+zc.lockfile = 1.0.2
 zdaemon = 4.2.0
 zipp = 1.2.0
 zodburi = 2.4.0
 zope.event = 3.5.2
+zope.exceptions = 3.6.2
+zope.testrunner = 5.2
+zope.testing = 3.9.7
 paramiko = 2.1.3
 CacheControl = 0.12.5
 msgpack = 0.6.2