Merge branch 'y/ZODB' into y/wc2-next

* y/ZODB:
  ZODB: Provide ZODB4-wc2 which backports MVCC approach from ZODB5
  ZODB: Factor it to separate component

component/ZEO/buildout.cfg

@@ -3,22 +3,37 @@
 
 [buildout]
 extends =
+    ../ZODB/buildout.cfg
     ../git/buildout.cfg
 
 parts = ZEO/scripts
 
 
-# ZEO allows to use either ZEO4 or ZEO5
-# To select which version to use users should do:
+# ZEO provides ZEO<X> depending on ZODB major version.
 #
-#   [ZEO]
-#   egg = ${ZEO<version>:egg}
-#
-# By default ZEO4 is used.
+# - ZEO4 works only with ZODB4
+#   https://github.com/zopefoundation/ZEO/blob/4.3.1-1-g47d3fbe8/setup.py#L122
+# - ZEO5 works only with ZODB5
+#   https://github.com/zopefoundation/ZEO/blob/5.2.2-1-g3d90ed42/setup.py#L20
 [ZEO]
-recipe  = zc.recipe.egg:eggs
-egg     = ${ZEO4:egg}
-eggs    = ${:egg}
+recipe  = slapos.recipe.build
+depends = ${ZODB:egg}
+init =
+  # link/depend to ZEO<ZODB.major>
+  zodb = self.buildout['ZODB']
+  zmajor = zodb['major']
+  zeo_x = self.buildout['ZEO'+zmajor]
+  options['depends'] += '$${%s:egg}' % zeo_x.name
+  options['egg'] = zeo_x['egg']
+
+  # update [versions] from what is needed by ZEO<X>
+  self.buildout.parse('[_ZEO-versions]\n' + zeo_x['egg_versions'])
+  for (egg, version) in self.buildout['_ZEO-versions'].items():
+    self.buildout['versions'][egg] = version
+  # propagate updated [versions] -> easy_install
+  # (buildout does this in Buildout constructor)
+  import zc.buildout.easy_install
+  zc.buildout.easy_install.default_versions(self.buildout['versions'])
 
 
 # ZEO/scripts installs scripts from ZEO
@@ -32,6 +47,7 @@ eggs    = ${ZEO:egg}
 recipe  = zc.recipe.egg:develop
 setup   = ${ZEO4-repository:location}
 egg     = ZEO
+egg_versions =
 
 [ZEO4-repository]
 recipe  = slapos.recipe.build:gitclone
@@ -47,3 +63,18 @@ git-executable = ${git:location}/bin/git
 recipe  = zc.recipe.egg:eggs
 egg     = ZEO
 eggs    = ${:egg}
+
+egg_versions =
+  ZEO = 5.2.2
+  trollius = 2.2.post1
+  futures = 3.2.0
+
+
+# ZEO4-wc2 is ZEO4 version with patches for wendelin.core 2 to work correctly.
+# The patches are backports of what is in ZEO master (ZEO5), but since upstream
+# considers 4 branch to be "dead", we have to maintain it by ourselves.
+# See https://github.com/zopefoundation/ZEO/pull/161 for the reference.
+[ZEO4-wc2]
+<= ZEO4
+revisions = bf80d23d3506
+location = ${buildout:parts-directory}/ZEO4-wc2

component/ZODB/buildout.cfg

+# SlapOS component for ZODB.
+# https://zodb.org/
+
+[buildout]
+extends =
+    ../git/buildout.cfg
+    ../python-cffi/buildout.cfg
+
+parts = ZODB/scripts
+
+
+# ZODB allows to use either ZODB4, ZODB4-wc2 or ZODB5.
+# To select which version to use users should do:
+#
+#   [ZODB]
+#   major = <ZODB-version-major>
+#
+# By default ZODB4 is used.
+[ZODB]
+recipe = slapos.recipe.build
+major  = 4
+init =
+  # link/depend ZODB -> ZODB<X>
+  zodb_x = 'ZODB'+options['major']
+  zodb_x = self.buildout[zodb_x]
+  options['depends'] = '$${%s:egg}' % zodb_x.name
+
+  options['egg'] = zodb_x['egg']
+
+  # update [versions] from what is needed by ZODB<X>
+  self.buildout.parse('[_ZODB-versions]\n' + zodb_x['egg_versions'])
+  for (egg, version) in self.buildout['_ZODB-versions'].items():
+    self.buildout['versions'][egg] = version
+  # propagate updated [versions] -> easy_install
+  # (buildout does this in Buildout constructor)
+  import zc.buildout.easy_install
+  zc.buildout.easy_install.default_versions(self.buildout['versions'])
+
+
+# ZODB/scripts installs scripts from ZODB
+[ZODB/scripts]
+recipe  = zc.recipe.egg:scripts
+eggs    = ${ZODB:egg}
+
+# ZODB4 and ZODB5 are plain upstream eggs
+[_ZODB]
+recipe  = zc.recipe.egg:eggs
+egg     = ZODB
+eggs    = ${:egg}
+depends = ${persistent:egg} ${BTrees:egg}
+
+[ZODB4]
+<= _ZODB
+egg_versions =
+  ZODB = 4.4.5
+  transaction = 1.7.0
+
+[ZODB5]
+<= _ZODB
+egg_versions =
+  ZODB = 5.6.0
+  transaction = 2.4.0
+
+
+# ZODB4-wc2 is ZODB4 version with patches for wendelin.core 2 to work correctly.
+# The main change is backport of the way MVCC is handled by always calling
+# loadBefore instead of load.
+# See https://lab.nexedi.com/nexedi/ZODB/commit/8e7eab33 for details.
+[ZODB4-wc2]
+<= ZODB4
+recipe = zc.recipe.egg:develop
+setup = ${ZODB4-wc2-repository:location}
+
+[ZODB4-wc2-repository]
+recipe  = slapos.recipe.build:gitclone
+repository = https://lab.nexedi.com/nexedi/ZODB.git
+git-executable = ${git:location}/bin/git
+branch = 4-nxd+MVCC-load-before
+revision = 4.4.5-9-g8e7eab330
+location = ${buildout:parts-directory}/ZODB4-wc2
+
+
+# XXX: buildout fails to install properly eggs with setup_requires
+[BTrees]
+recipe = zc.recipe.egg:custom
+egg = ${:_buildout_section_name_}
+setup-eggs = ${persistent:egg}
+
+[persistent]
+recipe = zc.recipe.egg:custom
+egg = ${:_buildout_section_name_}
+setup-eggs = ${python-cffi:egg}
+
+
+# eggs that are common to ZODB4 and ZODB5.
+[versions]
+BTrees = 4.5.1
+persistent = 4.6.4
+zodbpickle = 1.0.4
+
+# Provide ZODB3 for those eggs that still care about ZODB3 compatibility -
+# for example wendelin.core. ZODB3 3.11 is just a dependency egg on _latest_
+# ZODB, persistent, BTrees and ZEO.
+ZODB3 = 3.11.0

component/ZODB/test-zodb4-wc2.cfg

+# SlapOS software release to test ZODB4-wc2 on Nexedi testing infrastructure.
+
+[buildout]
+extends =
+# test<X>.cfg configures ZODB.major=<X>.
+
+    ../../stack/nxdtest.cfg
+    ../pygolang/buildout.cfg
+    ../python-manuel/buildout.cfg
+    buildout.cfg
+    ../ZEO/buildout.cfg
+
+parts =
+    ZODB
+    .nxdtest
+
+#   for instance
+    ZODB-python
+    slapos-cookbook
+    instance.cfg
+
+[ZODB]
+major = 4-wc2
+[ZODB4-wc2-repository]
+revision =
+
+# we need persistent to be a git checkout because persistent tests want to
+# discover in-tree files that are not present in persistent egg when it is
+# installed in non-development mode:
+#
+#   https://erp5.nexedi.net/test_result_module/20201123-3F859E35/7
+#   (look for "AssertionError: could not find my setup.py")
+#
+#   https://github.com/zopefoundation/persistent/blob/4.6.4-0-g7ed95cf/persistent/tests/test_docs.py#L37-L43
+[persistent]
+recipe = zc.recipe.egg:develop
+setup = ${persistent-repository:location}
+
+[persistent-repository]
+recipe  = slapos.recipe.build:gitclone
+location = ${buildout:parts-directory}/persistent
+repository = https://github.com/zopefoundation/persistent.git
+branch = master
+revision = 4.6.4-0-g7ed95cf
+git-executable = ${git:location}/bin/git
+
+
+# test-dependent eggs that must come through in-tree recipes.
+[ZODB]
+depends += ${manuel:egg}
+
+
+# bin/python is preinstalled with sys.path to ZODB & friends.
+[ZODB-python]
+<= python-interpreter
+eggs +=
+  ${ZODB:egg}[test]
+  ${ZEO:egg}[test]
+  transaction[test]
+
+# env.sh for ZODB's python to be on $PATH.
+[ZODB-env.sh]
+recipe   = slapos.recipe.template:jinja2
+rendered = ${buildout:directory}/${:_buildout_section_name_}
+template = inline:
+  export PS1="(ZODB-env) $PS1"
+  export PATH=${buildout:bin-directory}:$PATH
+
+# .nxdtest to run ZODB tests
+[.nxdtest]
+<= jinja2-template
+template =
+  inline:
+  # determine where ZODB & friends were installed
+  from subprocess import check_output
+  _ = check_output(['python', '-c', """if 1:
+        from os.path import dirname
+        def X(mod):
+            m=__import__(mod)
+            print(dirname(m.__file__))
+        X("ZODB"); X("persistent"); X("BTrees"); X("transaction"), X("ZEO")
+  """])
+  ZODB, persistent, BTrees, transaction, ZEO = _.split()
+
+  # run tests for whole ZODB stack
+  TestCase('ZODB/unit',       ['python', '-m', 'zope.testrunner', '-uv', '--package-path', ZODB, 'ZODB'])
+  TestCase('ZODB/functional', ['python', '-m', 'zope.testrunner', '-fv', '--package-path', ZODB, 'ZODB'])
+
+  TestCase('ZEO/unit',       ['python', '-m', 'zope.testrunner', '-uv', '--package-path', ZEO, 'ZEO'])
+  TestCase('ZEO/functional', ['python', '-m', 'zope.testrunner', '-fv', '--package-path', ZEO, 'ZEO'])
+
+  TestCase('persistent', ['python', '-m', 'zope.testrunner', '-v', '--package-path', persistent, 'persistent'])
+  TestCase('BTrees',     ['python', '-m', 'zope.testrunner', '-v', '--package-path', BTrees, 'BTrees'])
+
+  # transaction uses unittest instead of zope.testrunner
+  import os.path
+  def P(path): return os.path.join(path, '..')
+  TestCase('transaction',['python', '-m', 'unittest', 'discover', '-s', P(transaction)])
+
+
+# instance to run nxdtest.
+[instance.cfg]
+<= jinja2-template
+template = inline:
+  [buildout]
+  extends = ${nxdtest-instance.cfg:rendered}
+
+  [runTestSuite]
+  env.sh  = ${ZODB-env.sh:rendered}
+  workdir = ${buildout:directory}
+
+
+[versions]
+mock = 3.0.5
+random2 = 1.0.1
+zope.testing = 4.7
+zope.testrunner = 5.2
+zope.exceptions = 4.4

component/wendelin.core/buildout.cfg

@@ -3,6 +3,7 @@
 [buildout]
 extends =
   ../pygolang/buildout.cfg
+  ../ZODB/buildout.cfg
   ../zodbtools/buildout.cfg
   ../numpy/buildout.cfg
   ../golang/buildout.cfg
@@ -24,6 +25,7 @@ environment = wendelin.core-env
 
 # dependent eggs that must come through in-tree recipes
 depends =
+    ${ZODB:egg}
     ${zodbtools:egg}
     ${numpy:egg}
 

component/zodbtools/buildout.cfg

@@ -4,6 +4,7 @@
 [buildout]
 extends =
     ../pygolang/buildout.cfg
+    ../ZODB/buildout.cfg
     ../ZEO/buildout.cfg
 
 parts =
@@ -19,6 +20,7 @@ eggs    =
 # dependent eggs that must come through in-tree recipes
 depends =
     ${pygolang:egg}
+    ${ZODB:egg}
 # ZEO comes through zodbtools -> zodburi -> ZEO
     ${ZEO:egg}
 

component/zodbtools/test-common.cfg

@@ -3,8 +3,7 @@
 
 [buildout]
 extends =
-# test*.cfg first extend from neoppod/software<ZODB-flabour>.cfg to use
-# appropriate ZODB and versions of other components.
+# test<X>.cfg configures ZODB.major=<X>.
 
     ../../stack/nxdtest.cfg
     ../pytest/buildout.cfg
@@ -45,4 +44,5 @@ template = inline:
 
 
 [versions]
+mock = 3.0.5
 freezegun = 1.0.0

component/zodbtools/test-zodb5.cfg

 # SlapOS software release to test zodbtools/ZODB5 on Nexedi testing infrastructure.
 
 [buildout]
-extends =
-    ../../software/neoppod/software-zodb5.cfg
-    test-common.cfg
+extends = test-common.cfg
+
+[ZODB]
+major = 5

component/zodbtools/test.cfg

 # SlapOS software release to test zodbtools on Nexedi testing infrastructure.
 
 [buildout]
-extends =
-    ../../software/neoppod/software.cfg
-    test-common.cfg
+extends = test-common.cfg

software/neoppod/software-common.cfg

@@ -18,6 +18,7 @@ extends =
     ../../component/python-mysqlclient/buildout.cfg
     ../../component/python-cryptography/buildout.cfg
     ../../component/pycurl/buildout.cfg
+    ../../component/ZODB/buildout.cfg
     ../../component/ZEO/buildout.cfg
     ../../component/zodbtools/buildout.cfg
 
@@ -41,32 +42,23 @@ recipe = zc.recipe.egg:develop
 setup = ${neoppod-repository:location}
 environment = neoppod-setup-env
 
-[neoppod]
-recipe = zc.recipe.egg
-eggs = neoppod[admin, ctl, master]
+# dependent eggs that must come through in-tree recipes
+intree-eggs =
   ${cython-zstd:egg}
   ${msgpack-python:egg}
   ${:adapter-egg}
-  ${BTrees:egg}
+  ${ZODB:egg}
+  ${zodbtools:egg}
+adapter-egg = ${python-mysqlclient:egg}
+
+[neoppod]
+recipe = zc.recipe.egg
+eggs = neoppod[admin, ctl, master]
+  ${neoppod-develop:intree-eggs}
   psutil
-  ZODB
   zope.testing
-  ${zodbtools:egg}
   coverage
   setproctitle
-adapter-egg = ${python-mysqlclient:egg}
-
-# XXX: buildout fails to install properly eggs with setup_requires
-[BTrees]
-recipe = zc.recipe.egg:custom
-egg = ${:_buildout_section_name_}
-setup-eggs = ${persistent:egg}
-
-[persistent]
-recipe = zc.recipe.egg:custom
-egg = ${:_buildout_section_name_}
-setup-eggs = ${python-cffi:egg}
-###
 
 [download-base-neo]
 recipe = slapos.recipe.build:download
@@ -136,18 +128,13 @@ template =
   {% endraw %}
 
 [versions]
-BTrees = 4.5.1
-ZODB = 4.4.5
 coverage = 4.5.1
 mock = 3.0.5
 ecdsa = 0.13
 mysqlclient = 1.3.12
-persistent = 4.6.4
 pycrypto = 2.6.1
 pycurl = 7.43.0
 setproctitle = 1.1.10
-transaction = 1.7.0
-zodbpickle = 1.0.4
 cython-zstd = 0.2
 python-dateutil = 2.7.3
 

software/neoppod/software-zodb5.cfg

 [buildout]
 extends = software.cfg
 
+[ZODB]
+major = 5
+
 [neoppod]
 eggs += mock
-
-[ZEO]
-egg = ${ZEO5:egg}
-
-[versions]
-ZODB = 5.6.0
-ZEO = 5.2.2
-transaction = 2.4.0
-
-# Required by:
-# ZEO==5.2.0
-# trollius==2.2.post1
-futures = 3.2.0
-
-# Required by:
-# ZEO==5.2.0
-trollius = 2.2.post1

software/neoppod/software.cfg

@@ -34,8 +34,6 @@ interpreter = ${:_buildout_section_name_}
 
 [versions]
 # To match ERP5
-ZConfig = 2.9.3
-zc.lockfile = 1.0.2
 zope.event = 3.5.2
 zope.exceptions = 3.6.2
 zope.testing = 3.9.7

software/neoppod/sqlite-mixin.cfg

 # To be extended after software-common.cfg if only SQLite backend is wanted.
 
-[neoppod]
+[neoppod-develop]
 adapter-egg =
 
 [instance-common]

software/neotest/software.cfg

@@ -131,20 +131,4 @@ eggs    =
 
 # ping eggs versions
 [versions]
-ZODB3 = 3.11.0
 zope.testing = 4.6.2
-
-# Required by:
-# ZEO==4.3.1
-# ZODB==4.4.5
-# zodburi==2.3.0
-ZConfig = 3.2.0
-
-# Required by:
-# ZEO==4.3.1
-# ZODB==4.4.5
-zc.lockfile = 1.2.1
-
-# Required by:
-# neoppod==1.8.1
-python-dateutil = 2.7.1

stack/erp5/buildout.cfg

@@ -664,7 +664,6 @@ PyXML = 0.8.5
 Pympler = 0.4.3
 StructuredText = 2.11.1
 WSGIUtils = 0.7
-ZODB3 = 3.11.0
 # astroid 1.4.1 breaks testDynamicClassGeneration
 astroid = 1.3.8
 erp5diff = 0.8.1.7

stack/slapos.cfg

@@ -156,6 +156,7 @@ Importing = 1.10
 MarkupSafe = 1.0
 PyYAML = 5.4.1
 Werkzeug = 1.0.1
+ZConfig = 2.9.3
 asn1crypto = 1.3.0
 atomicwrites = 1.4.0
 backports.functools-lru-cache = 1.6.1
@@ -216,6 +217,7 @@ unicodecsv = 0.14.1
 wcwidth = 0.2.5
 wheel = 0.35.1
 xml-marshaller = 1.0.2
+zc.lockfile = 1.0.2
 zdaemon = 4.2.0
 zipp = 1.2.0
 zodburi = 2.4.0