Update Release Candidate

component/ZEO/buildout.cfg

+# SlapOS component for ZEO.
+# https://zeo.readthedocs.io/
+
+[buildout]
+extends =
+    ../git/buildout.cfg
+
+parts = ZEO/scripts
+
+
+# ZEO allows to use either ZEO4 or ZEO5
+# To select which version to use users should do:
+#
+#   [ZEO]
+#   egg = ${ZEO<version>:egg}
+#
+# By default ZEO4 is used.
+[ZEO]
+recipe  = zc.recipe.egg:eggs
+egg     = ${ZEO4:egg}
+eggs    = ${:egg}
+
+
+# ZEO/scripts installs scripts from ZEO
+[ZEO/scripts]
+recipe  = zc.recipe.egg:scripts
+eggs    = ${ZEO:egg}
+
+
+# ZEO4: we maintain our own 4-nxd branch with patches
+[ZEO4]
+recipe  = zc.recipe.egg:develop
+setup   = ${ZEO4-repository:location}
+egg     = ZEO
+
+[ZEO4-repository]
+recipe  = slapos.recipe.build:gitclone
+repository = https://lab.nexedi.com/nexedi/ZEO.git
+branch  = 4-nxd
+revision= 5114f909e5a5
+location = ${buildout:parts-directory}/ZEO4
+git-executable = ${git:location}/bin/git
+
+
+# ZEO5 is plain upstream egg
+[ZEO5]
+recipe  = zc.recipe.egg:eggs
+egg     = ZEO
+eggs    = ${:egg}

component/caddy/buildout.cfg

@@ -11,7 +11,7 @@ parts =
 command = bash -c ". ${gowork:env.sh} && cd ${go_github.com_caddyserver_caddy:location} && GO111MODULE=on go install -v $(echo -n '${gowork:install}' |tr '\n' ' ')"
 
 [gowork]
-golang  = ${golang1.12:location}
+golang  = ${golang1.14:location}
 install =
   github.com/caddyserver/caddy/...
 

component/egg-patch/ZEO4/TCP_NODELAY.patch

-From 8b31ccec54584a287cc61501948283d7d6ee7073 Mon Sep 17 00:00:00 2001
-From: Julien Muchembled <jm@nexedi.com>
-Date: Mon, 26 Mar 2018 20:39:07 +0200
-Subject: [PATCH] Enable TCP_NODELAY for inet(6) sockets
-
-See commit 3d886d426243655b9f5a2528636e42b5c7662c19.
----
- src/ZEO/zrpc/client.py | 2 ++
- src/ZEO/zrpc/server.py | 1 +
- 2 files changed, 3 insertions(+)
-
-diff --git a/src/ZEO/zrpc/client.py b/src/ZEO/zrpc/client.py
-index 32a7a877..669f5962 100644
---- a/src/ZEO/zrpc/client.py
-+++ b/src/ZEO/zrpc/client.py
-@@ -568,6 +568,8 @@ def __init__(self, domain, addr, mgr, client):
-             self.close()
-             return
-         self.sock.setblocking(0)
-+        if domain in (socket.AF_INET, socket.AF_INET6):
-+            self.sock.setsockopt(socket.IPPROTO_TCP, socket.TCP_NODELAY, 1)
-         self.state = "opened"
- 
-     def connect_procedure(self):
-diff --git a/src/ZEO/zrpc/server.py b/src/ZEO/zrpc/server.py
-index b83cc004..af91e3e4 100644
---- a/src/ZEO/zrpc/server.py
-+++ b/src/ZEO/zrpc/server.py
-@@ -66,6 +66,7 @@ def _open_socket(self):
-                         socket.IPPROTO_IPV6, socket.IPV6_V6ONLY, True)
-             else:
-                 self.create_socket(socket.AF_INET, socket.SOCK_STREAM)
-+            self.socket.setsockopt(socket.IPPROTO_TCP, socket.TCP_NODELAY, 1)
-         else:
-             self.create_socket(socket.AF_UNIX, socket.SOCK_STREAM)
-         self.set_reuse_addr()
--- 
-2.14.1
-

component/fmtlib/buildout.cfg

+[buildout]
+extends =
+  ../cmake/buildout.cfg
+parts =
+  fmtlib
+
+[fmtlib]
+recipe = slapos.recipe.cmmi
+shared = true
+url = https://github.com/fmtlib/fmt/archive/7.0.3.tar.gz
+md5sum = 57392b7ea09592a2b237a02950f35bb0
+configure-command = ${cmake:location}/bin/cmake
+configure-options =
+  -DCMAKE_INSTALL_PREFIX=@@LOCATION@@
+  -DBUILD_SHARED_LIBS=ON
+  -DFMT_TEST=OFF

component/gdb/buildout.cfg

+# SlapOS component for GDB.
+# https://www.gnu.org/software/gdb
+
+[buildout]
+extends =
+  ../xz-utils/buildout.cfg
+  ../binutils/buildout.cfg
+  ../libexpat/buildout.cfg
+  ../texinfo/buildout.cfg
+
+parts = gdb
+
+[gdb]
+recipe = slapos.recipe.cmmi
+shared = true
+url = http://ftp.gnu.org/gnu/gdb/gdb-9.2.tar.xz
+md5sum = db95524e554870209ab7d9f8fd8dc557
+location = @@LOCATION@@
+# gdb refuses to build in-tree -> build it inside build/
+pre-configure =
+  mkdir -p build
+configure-command =
+  cd build && ../configure
+configure-options =
+  --prefix=${:location}
+  --disable-bootstrap
+  --with-mpc=${mpc:location}
+  --with-mpfr=${mpfr:location}
+  --with-gmp=${gmp:location}
+  --with-isl=${isl:location}
+  --with-expat=${libexpat:location}
+make-binary =
+make-targets = cd build && make && make install
+environment =
+  PATH=${texinfo:location}/bin:${xz-utils:location}/bin:%(PATH)s
+  LDFLAGS=-Wl,-rpath=${libexpat:location}/lib

component/golang/buildout.cfg

@@ -6,7 +6,7 @@ extends =
   ../git/buildout.cfg
   ../pkgconfig/buildout.cfg
 
-parts = gowork
+parts = gowork go
 
 # ---- Go builds itself ----
 
@@ -84,6 +84,18 @@ environment-extra =
 #
 #   [gowork]
 #   buildflags = -race
+#
+# ${go:exe} is standalone executable that runs go in activated gowork environment.
+[go]
+recipe  = slapos.recipe.template:jinja2
+exe     = ${buildout:bin-directory}/go
+rendered= ${:exe}
+mode    = 755
+template= inline:
+  #!/bin/sh -e
+  . ${gowork:env.sh}
+  exec go "$@"
+
 [gowork]
 directory = ${buildout:directory}/go.work
 src	= ${:directory}/src

component/numpy/buildout.cfg

@@ -4,3 +4,7 @@ parts = numpy
 [numpy]
 recipe = zc.recipe.egg:custom
 egg = numpy
+
+
+[versions]
+numpy = 1.16.4

component/pygolang/buildout-dev.cfg

+# SlapOS component for pygolang development.
+
+[buildout]
+extends =
+    buildout.cfg
+    ../git/buildout.cfg
+
+# override pygolang to install it from latest git version
+[pygolang]
+recipe  = zc.recipe.egg:develop
+setup   = ${pygolang-repository:location}
+
+[pygolang-repository]
+recipe  = slapos.recipe.build:gitclone
+repository = https://lab.nexedi.com/nexedi/pygolang.git
+branch  = master
+location = ${buildout:parts-directory}/pygolang-dev
+git-executable = ${git:location}/bin/git
+
+
+# unpin pygolang from versions, so that buildout does not fallback to
+# installing non-dev egg if dev one has changed its version.
+[versions]
+pygolang =

component/pygolang/buildout.cfg

+# SlapOS component for pygolang.
+# pygolang.nexedi.com
+
+[buildout]
+extends =
+    ../../component/cython/buildout.cfg
+
+parts =
+    pygolang
+    gpython
+
+
+# pygolang installed from released egg from pypi
+[pygolang]
+recipe  = zc.recipe.egg:custom
+egg     = pygolang
+setup-eggs =
+    ${cython:egg}
+    setuptools-dso
+    gevent
+
+# gpython program
+[gpython]
+recipe  = zc.recipe.egg:scripts
+eggs    = ${pygolang:egg}
+scripts = gpython
+# convenience for gpython users
+exe     = ${buildout:bin-directory}/gpython
+
+# python-interpreter provides python interpreter with all specified eggs.
+# eggs default to pygolang, but can be overwritten or changed in inherited section.
+# if eggs are changes, they must still have pygolang.
+[python-interpreter]
+recipe = zc.recipe.egg:scripts
+eggs   = ${pygolang:egg}
+interpreter = python
+# interpreter code that buildout generates cannot process `-m pytest --<pytest-option>`
+# -> use pymain from gpython to workaround that.
+initialization =
+  # set sys.executable to self, so that subprocess and friends go through us
+  # and this way spawn children with correct sys.path where all eggs that
+  # parent have are present. TODO consider migrating this into pymain
+  sys.executable = sys.argv[0]
+  # tail to pymain
+  from gpython import pymain
+  pymain(sys.argv[1:])
+  sys.exit(0)
+# don't install scripts from listed eggs (avoid conflict with other sections
+# that use zc.recipe.egg with eggs overlapping with ${:eggs} - ex neoppod)
+# (we cannot use zc.recipe.egg:eggs because interpreter does not work there)
+# NOTE with scripts=ø interpreter is not handled, so we use `scripts=python` as
+# a workaround.
+scripts = ${:interpreter}
+
+
+[versions]
+pygolang = 0.0.7.post1

component/pygolang/test.cfg

+# SlapOS software release to test pygolang on Nexedi testing infrastructure.
+
+[buildout]
+extends =
+    ../../stack/nxdtest.cfg
+    ../pytest/buildout.cfg
+    ../numpy/buildout.cfg
+    buildout-dev.cfg
+
+parts =
+    pygolang
+    gpython
+    pygolang-python
+
+#   for instance
+    slapos-cookbook
+    instance.cfg
+
+
+# tune pygolang to install with all and for-tests extras.
+[pygolang]
+egg = pygolang[all_test]
+
+# bin/python is preinstalled with sys.path to pygolang & friends.
+[pygolang-python]
+<= python-interpreter
+eggs = ${pygolang:egg}
+
+# env.sh for pygolang's python/gpython to be on $PATH.
+[pygolang-env.sh]
+recipe   = slapos.recipe.template:jinja2
+rendered = ${buildout:directory}/${:_buildout_section_name_}
+template = inline:
+  export PS1="(pygolang-env) $PS1"
+  export PATH=${buildout:bin-directory}:$PATH
+
+# instance to run nxdtest.
+[instance.cfg]
+<= jinja2-template
+template = inline:
+  [buildout]
+  extends = ${nxdtest-instance.cfg:rendered}
+
+  [runTestSuite]
+  env.sh  = ${pygolang-env.sh:rendered}
+  workdir = ${pygolang-repository:location}
+
+
+[versions]
+ipython = 5.10.0
+ipython-genutils = 0.2.0
+
+traitlets = 4.3.3
+simplegeneric = 0.8.1
+Pygments = 2.5.2
+prompt-toolkit = 1.0.18
+pickleshare = 0.7.5
+pexpect = 4.8.0
+backports.shutil-get-terminal-size = 1.0.0
+ptyprocess = 0.6.0

component/pytest/buildout.cfg

+# SlapOS component for pytest.
+# https://pytest.org
+
+[buildout]
+parts = pytest/scripts
+
+[pytest]
+recipe  = zc.recipe.egg:eggs
+eggs    = pytest
+
+[pytest/scripts]
+recipe  = zc.recipe.egg:scripts
+eggs    = ${pytest:eggs}
+
+
+[versions]
+pytest  = 4.6.11

component/wendelin.core/buildout.cfg

+# SlapOS component for wendelin.core
+
 [buildout]
-extends = ../git/buildout.cfg
+extends =
+  ../pygolang/buildout.cfg
+  ../git/buildout.cfg
+
 parts =
   wendelin.core
 
@@ -7,18 +12,23 @@ parts =
 # wendelin.core installed from released egg from pypi
 [wendelin.core]
 recipe = zc.recipe.egg:custom
-egg = wendelin.core
-
+<= wendelin.core-eggcommon
 
 # wendelin.core installed from latest git version
 [wendelin.core-dev]
 recipe = zc.recipe.egg:develop
-egg = wendelin.core
 setup = ${wendelin.core-repository:location}
-environment = wendelin.core-dev-env
+<= wendelin.core-eggcommon
+
+# common parts in between wendelin.core and wendelin.core-dev
+[wendelin.core-eggcommon]
+egg = wendelin.core
+setup-eggs =
+    ${pygolang:egg}[pyx.build]
+environment = wendelin.core-env
 
-[wendelin.core-dev-env]
-# wendelin.core-dev needs git to build
+[wendelin.core-env]
+# wendelin.core needs git(dev) to build
 PATH = ${git:location}/bin:%(PATH)s
 
 

component/wendelin.core/test-common.cfg

+# SlapOS software release to test wendelin.core on Nexedi testing infrastructure.
+# Common parts.
+
+[buildout]
+extends =
+# test*.cfg first extend from neoppod/software<ZODB-flavour>.cfg to use
+# appropriate ZODB and versions of other components.
+
+    ../pytest/buildout.cfg
+    ../gdb/buildout.cfg
+    ../../stack/nxdtest.cfg
+    buildout.cfg
+
+parts =
+# keep neoppod first and in parts so that ZODB is built correctly
+    neoppod
+
+    wendelin.core-dev
+
+#   for instance
+    wendelin.core-python
+    slapos-cookbook
+    instance.cfg
+
+
+# bin/python s python interpreter with wendelin.core and all other eggs.
+[wendelin.core-python]
+<= python-interpreter
+eggs =
+    wendelin.core[test]
+    pygolang[pyx.build]
+    neoppod[tests]
+    ZEO[test]
+
+# env.sh for that python to be on $PATH
+[wendelin.core-env.sh]
+recipe   = slapos.recipe.template:jinja2
+rendered = ${buildout:directory}/${:_buildout_section_name_}
+template = inline:
+  export PS1="(wendelin.core-env) $PS1"
+  export PATH=${buildout:bin-directory}:${gdb:location}/bin:$PATH
+
+# instance to run nxdtest.
+[instance.cfg]
+<= jinja2-template
+template = inline:
+  [buildout]
+  extends = ${nxdtest-instance.cfg:rendered}
+
+  [runTestSuite]
+  env.sh  = ${wendelin.core-env.sh:rendered}
+  workdir = ${wendelin.core-repository:location}
+
+
+[versions]
+neoppod = 1.12.0
+
+zope.testrunner = 5.2
+random2 = 1.0.1
+manuel = 1.10.1

component/wendelin.core/test-zodb5.cfg

+# SlapOS software release to test wendelin.core/ZODB5 on Nexedi testing infrastructure.
+
+[buildout]
+extends =
+    ../../stack/erp5/buildout.cfg
+    ../../software/neoppod/software-zodb5.cfg
+    test-common.cfg

component/wendelin.core/test.cfg

+# SlapOS software release to test wendelin.core on Nexedi testing infrastructure.
+
+[buildout]
+extends =
+    ../../stack/erp5/buildout.cfg
+    ../../software/neoppod/software.cfg
+    test-common.cfg

component/zodbtools/buildout-dev.cfg

+# SlapOS component for zodbtools development.
+
+[buildout]
+extends =
+    buildout.cfg
+    ../git/buildout.cfg
+
+# override zodbtools to install it from latest git version
+[zodbtools]
+recipe  = zc.recipe.egg:develop
+setup   = ${zodbtools-repository:location}
+
+[zodbtools-repository]
+recipe  = slapos.recipe.build:gitclone
+repository = https://lab.nexedi.com/nexedi/zodbtools.git
+branch  = master
+location = ${buildout:parts-directory}/zodbtools-dev
+git-executable = ${git:location}/bin/git
+
+
+# unpin zodbtools from versions, so that buildout does not fallback to
+# installing non-dev egg if dev one has changed its version.
+[versions]
+zodbtools =

component/zodbtools/buildout.cfg

+# SlapOS component for zodbtools.
+# https://lab.nexedi.com/nexedi/zodbtools
+
+[buildout]
+extends =
+    ../pygolang/buildout.cfg
+    ../ZEO/buildout.cfg
+
+parts =
+    zodbtools/scripts
+
+
+[zodbtools]
+recipe  = zc.recipe.egg:eggs
+egg     = zodbtools
+eggs    =
+    ${:egg}
+
+# dependent eggs that must come through in-tree recipes
+depends =
+    ${pygolang:egg}
+# ZEO comes through zodbtools -> zodburi -> ZEO
+    ${ZEO:egg}
+
+[zodbtools/scripts]
+recipe  = zc.recipe.egg:scripts
+eggs    = ${zodbtools:eggs}
+
+
+[versions]
+zodbtools = 0.0.0.dev8

component/zodbtools/test-common.cfg

+# SlapOS software release to test zodbtools on Nexedi testing infrastructure.
+# Common parts.
+
+[buildout]
+extends =
+# test*.cfg first extend from neoppod/software<ZODB-flabour>.cfg to use
+# appropriate ZODB and versions of other components.
+
+    ../../stack/nxdtest.cfg
+    ../pytest/buildout.cfg
+    buildout-dev.cfg
+
+parts =
+    zodbtools
+
+#   for instance
+    zodbtools-python
+    slapos-cookbook
+    instance.cfg
+
+
+# bin/python is preinstalled with sys.path to zodbtools & friends.
+[zodbtools-python]
+<= python-interpreter
+eggs = zodbtools[test]
+
+# env.sh for zodbtools's python to be on $PATH.
+[zodbtools-env.sh]
+recipe   = slapos.recipe.template:jinja2
+rendered = ${buildout:directory}/${:_buildout_section_name_}
+template = inline:
+  export PS1="(zodbtools-env) $PS1"
+  export PATH=${buildout:bin-directory}:$PATH
+
+# instance to run nxdtest.
+[instance.cfg]
+<= jinja2-template
+template = inline:
+  [buildout]
+  extends = ${nxdtest-instance.cfg:rendered}
+
+  [runTestSuite]
+  env.sh  = ${zodbtools-env.sh:rendered}
+  workdir = ${zodbtools-repository:location}
+
+
+[versions]
+freezegun = 1.0.0

component/zodbtools/test-zodb5.cfg

+# SlapOS software release to test zodbtools/ZODB5 on Nexedi testing infrastructure.
+
+[buildout]
+extends =
+    ../../software/neoppod/software-zodb5.cfg
+    test-common.cfg

component/zodbtools/test.cfg

+# SlapOS software release to test zodbtools on Nexedi testing infrastructure.
+
+[buildout]
+extends =
+    ../../software/neoppod/software.cfg
+    test-common.cfg

software/caddy-frontend/software.cfg

@@ -11,6 +11,7 @@ extends =
   ../../component/6tunnel/buildout.cfg
   ../../component/xz-utils/buildout.cfg
   ../../component/rsyslogd/buildout.cfg
+  ../../component/numpy/buildout.cfg
   ../../component/haproxy/buildout.cfg
   ../../component/nginx/buildout.cfg
 
@@ -251,7 +252,6 @@ pycrypto = 2.6.1
 rdiff-backup = 1.0.5+SlapOSPatched001
 slapos.recipe.template = 4.4
 smmap = 0.9.0
-numpy = 1.16.4
 
 websockify = 0.8.0
 furl = 2.1.0

software/cython-test/software.cfg

@@ -102,5 +102,4 @@ context =
 [versions]
 slapos.recipe.template = 4.4
 coverage = 4.5.1
-numpy = 1.16.4
 pycodestyle = 2.5.0

software/dream/software.cfg

@@ -40,7 +40,6 @@ rpy2 = 2.4.0
 pydot = 1.0.28
 xlrd = 0.9.3
 xlwt = 0.7.5
-numpy = 1.16.4
 scipy = 0.13.3
 simpy = 3.0.5
 zope.dottedname = 4.1.0

software/erp5/test/setup.py

@@ -33,7 +33,7 @@ with open("README.md") as f:
 
 setup(name=name,
       version=version,
-      description="Test for SlapOS' ERP5 software releae",
+      description="Test for SlapOS' ERP5 software release",
       long_description=long_description,
       long_description_content_type='text/markdown',
       maintainer="Nexedi",
@@ -50,8 +50,9 @@ setup(name=name,
         'mysqlclient',
         'backports.lzma',
         'cryptography',
+        'pexpect',
         'pyOpenSSL',
+        'typing; python_version<"3"',
       ],
-      zip_safe=True,
       test_suite='test',
     )

software/erp5/test/test/test_balancer.py

-from . import ERP5InstanceTestCase
-from . import setUpModule
-from slapos.testing.utils import findFreeTCPPort
-
-from BaseHTTPServer import HTTPServer
-from BaseHTTPServer import BaseHTTPRequestHandler
-import OpenSSL.SSL
-from cryptography.hazmat.backends import default_backend
-from cryptography.hazmat.primitives import serialization, hashes
-from cryptography.hazmat.primitives.asymmetric import rsa
-from cryptography import x509
-from cryptography.x509.oid import NameOID
+import glob
 import hashlib
 import json
-import multiprocessing
+import logging
 import os
-import requests
+import re
 import shutil
+import socket
 import subprocess
 import tempfile
 import time
+import urlparse
+from BaseHTTPServer import BaseHTTPRequestHandler
+from typing import Any, Dict, Optional
+
+import idna
+import mock
+import OpenSSL.SSL
+import pexpect
+import psutil
+import requests
+from cryptography import x509
+from cryptography.hazmat.backends import default_backend
+from cryptography.hazmat.primitives import hashes, serialization
+from cryptography.hazmat.primitives.asymmetric import rsa
+from cryptography.x509.oid import NameOID
+from slapos.testing.testcase import ManagedResource
+from slapos.testing.utils import (CrontabMixin, ManagedHTTPServer,
+                                  findFreeTCPPort)
+
+from . import ERP5InstanceTestCase, setUpModule
 
 setUpModule  # pyflakes
 
-class TestHandler(BaseHTTPRequestHandler):
+
+class EchoHTTPServer(ManagedHTTPServer):
+  """An HTTP Server responding with the request path and incoming headers,
+  encoded in json.
+  """
+  class RequestHandler(BaseHTTPRequestHandler):
     def do_GET(self):
+      # type: () -> None
       self.send_response(200)
       self.send_header("Content-Type", "application/json")
-    response = {
+      response = json.dumps(
+          {
               'Path': self.path,
               'Incoming Headers': self.headers.dict
-    }
-    response = json.dumps(response, indent=2)
+          },
+          indent=2,
+      )
       self.end_headers()
       self.wfile.write(response)
 
-class TestFrontendXForwardedFor(ERP5InstanceTestCase):
-  __partition_reference__ = 'xff'
-  http_server_process = None
-  frontend_caucase_dir = None
-  frontend_caucased_process = None
-  backend_caucase_dir = None
-  backend_caucased_process = None
+    log_message = logging.getLogger(__name__ + '.HeaderEchoHandler').info
 
-  @classmethod
-  def getInstanceSoftwareType(cls):
-    return 'balancer'
 
-  @classmethod
-  def setUpClass(cls):
-    # start a dummy web server echoing headers.
-    http_server_port = findFreeTCPPort(cls._ipv4_address)
-    server = HTTPServer(
-      (cls._ipv4_address, http_server_port),
-      TestHandler)
-    cls.http_server_process = multiprocessing.Process(
-      target=server.serve_forever, name='HTTPServer')
-    cls.http_server_process.start()
-    cls.http_server_netloc = '%s:%s' % (cls._ipv4_address, http_server_port)
-
-    # start a caucased and generate a valid client certificate.
-    cls.computer_partition_root_path = os.path.abspath(os.curdir)
-    cls.frontend_caucase_dir = tempfile.mkdtemp()
-    frontend_caucased_dir = os.path.join(cls.frontend_caucase_dir, 'caucased')
-    os.mkdir(frontend_caucased_dir)
-    frontend_user_dir = os.path.join(cls.frontend_caucase_dir, 'user')
-    os.mkdir(frontend_user_dir)
-    frontend_service_dir = os.path.join(cls.frontend_caucase_dir, 'service')
-    os.mkdir(frontend_service_dir)
-    frontend_caucased_netloc = '%s:%s' % (cls._ipv4_address, findFreeTCPPort(cls._ipv4_address))
-    cls.frontend_caucased_url = 'http://' + frontend_caucased_netloc
-
-    cls.user_certificate = frontend_user_key = os.path.join(frontend_user_dir, 'client.key.pem')
-    frontend_user_csr = os.path.join(frontend_user_dir, 'client.csr.pem')
+class CaucaseService(ManagedResource):
+  """A caucase service.
+  """
+  url = None # type: str
+  directory = None # type: str
+  _caucased_process = None # type: subprocess.Popen
 
-    key = rsa.generate_private_key(
-      public_exponent=65537,
-      key_size=2048,
-      backend=default_backend()
+  def open(self):
+    # type: () -> None
+    # start a caucased and server certificate.
+    software_release_root_path = os.path.join(
+        self._cls.slap._software_root,
+        hashlib.md5(self._cls.getSoftwareURL().encode()).hexdigest(),
     )
-    with open(frontend_user_key, 'wb') as f:
-      f.write(key.private_bytes(
-        encoding=serialization.Encoding.PEM,
-        format=serialization.PrivateFormat.TraditionalOpenSSL,
-        encryption_algorithm=serialization.NoEncryption(),
-      ))
+    caucased_path = os.path.join(software_release_root_path, 'bin', 'caucased')
 
-    csr = x509.CertificateSigningRequestBuilder().subject_name(x509.Name([
-      x509.NameAttribute(NameOID.COMMON_NAME, u'user'),
-    ])).sign(key, hashes.SHA256(), default_backend())
-    with open(frontend_user_csr, 'wb') as f:
-      f.write(csr.public_bytes(serialization.Encoding.PEM))
+    self.directory = tempfile.mkdtemp()
+    caucased_dir = os.path.join(self.directory, 'caucased')
+    os.mkdir(caucased_dir)
+    os.mkdir(os.path.join(caucased_dir, 'user'))
+    os.mkdir(os.path.join(caucased_dir, 'service'))
 
-    cls.software_release_root_path = os.path.join(
-       cls.slap._software_root,
-       hashlib.md5(cls.getSoftwareURL()).hexdigest(),
-    )
-    caucased_path = os.path.join(cls.software_release_root_path, 'bin', 'caucased')
-    caucase_path = os.path.join(cls.software_release_root_path, 'bin', 'caucase')
-    cls.frontend_caucased_process = subprocess.Popen(
+    backend_caucased_netloc = '%s:%s' % (self._cls._ipv4_address, findFreeTCPPort(self._cls._ipv4_address))
+    self.url = 'http://' + backend_caucased_netloc
+    self._caucased_process = subprocess.Popen(
         [
             caucased_path,
-        '--db', os.path.join(frontend_caucased_dir, 'caucase.sqlite'),
-        '--server-key', os.path.join(frontend_caucased_dir, 'server.key.pem'),
-        '--netloc', frontend_caucased_netloc,
+            '--db', os.path.join(caucased_dir, 'caucase.sqlite'),
+            '--server-key', os.path.join(caucased_dir, 'server.key.pem'),
+            '--netloc', backend_caucased_netloc,
             '--service-auto-approve-count', '1',
         ],
         stdout=subprocess.PIPE,
         stderr=subprocess.STDOUT,
     )
-    for _ in range(10):
+    for _ in range(30):
       try:
-        if requests.get(cls.frontend_caucased_url).status_code == 200:
+        if requests.get(self.url).status_code == 200:
           break
       except Exception:
         pass
@@ -118,173 +100,591 @@ class TestFrontendXForwardedFor(ERP5InstanceTestCase):
     else:
       raise RuntimeError('caucased failed to start.')
 
-    cau_args = [
-      caucase_path,
-      '--ca-url', cls.frontend_caucased_url,
-      '--ca-crt', os.path.join(frontend_user_dir, 'service-ca-crt.pem'),
-      '--crl', os.path.join(frontend_user_dir, 'service.crl'),
-      '--user-ca-crt', os.path.join(frontend_user_dir, 'user-ca-crt.pem'),
-      '--user-crl', os.path.join(frontend_user_dir, 'user.crl'),
-    ]
+  def close(self):
+    # type: () -> None
+    self._caucased_process.terminate()
+    self._caucased_process.wait()
+    shutil.rmtree(self.directory)
 
-    cas_args = [
-      caucase_path,
-      '--ca-url', cls.frontend_caucased_url,
-      '--ca-crt', os.path.join(frontend_service_dir, 'service-ca-crt.pem'),
-      '--crl', os.path.join(frontend_service_dir, 'service.crl'),
-      '--user-ca-crt', os.path.join(frontend_service_dir, 'user-ca-crt.pem'),
-      '--user-crl', os.path.join(frontend_service_dir, 'user.crl'),
+  @property
+  def ca_crt_path(self):
+    # type: () -> str
+    """Path of the CA certificate from this caucase.
+    """
+    ca_crt_path = os.path.join(self.directory, 'ca.crt.pem')
+    if not os.path.exists(ca_crt_path):
+      with open(ca_crt_path, 'w') as f:
+        f.write(
+            requests.get(urlparse.urljoin(
+                self.url,
+                '/cas/crt/ca.crt.pem',
+            )).text)
+    return ca_crt_path
+
+class BalancerTestCase(ERP5InstanceTestCase):
+
+  @classmethod
+  def getInstanceSoftwareType(cls):
+    return 'balancer'
+
+  @classmethod
+  def _getInstanceParameterDict(cls):
+    # type: () -> Dict
+    return {
+        'tcpv4-port': 8000,
+        'computer-memory-percent-threshold': 100,
+        # XXX what is this ? should probably not be needed here
+        'name': cls.__name__,
+        'monitor-passwd': 'secret',
+        'apachedex-configuration': '--erp5-base +erp5 .*/VirtualHostRoot/erp5(/|\\?|$) --base +other / --skip-user-agent Zabbix --error-detail --js-embed --quiet',
+        'apachedex-promise-threshold': 100,
+        'haproxy-server-check-path': '/',
+        'zope-family-dict': {
+            'default': ['dummy_http_server'],
+        },
+        'dummy_http_server': [[cls.getManagedResource("backend_web_server", EchoHTTPServer).netloc, 1, False]],
+        'backend-path-dict': {
+            'default': '',
+        },
+        'ssl-authentication-dict': {},
+        'ssl': {
+            'caucase-url': cls.getManagedResource("caucase", CaucaseService).url,
+        }
+      }
+
+  @classmethod
+  def getInstanceParameterDict(cls):
+    # type: () -> Dict
+    return {'_': json.dumps(cls._getInstanceParameterDict())}
+
+  def setUp(self):
+    self.default_balancer_url = json.loads(
+        self.computer_partition.getConnectionParameterDict()['_'])['default']
+
+
+class SlowHTTPServer(ManagedHTTPServer):
+  """An HTTP Server which reply after 3 seconds.
+  """
+  class RequestHandler(BaseHTTPRequestHandler):
+    def do_GET(self):
+      # type: () -> None
+      self.send_response(200)
+      self.send_header("Content-Type", "text/plain")
+      time.sleep(3)
+      self.end_headers()
+      self.wfile.write("OK\n")
+
+    log_message = logging.getLogger(__name__ + '.SlowHandler').info
+
+
+class TestAccessLog(BalancerTestCase, CrontabMixin):
+  """Check access logs emitted by balancer
+  """
+  __partition_reference__ = 'l'
+  @classmethod
+  def _getInstanceParameterDict(cls):
+    # type: () -> Dict
+    parameter_dict = super(TestAccessLog, cls)._getInstanceParameterDict()
+    # use a slow server instead
+    parameter_dict['dummy_http_server'] = [[cls.getManagedResource("slow_web_server", SlowHTTPServer).netloc, 1, False]]
+    return parameter_dict
+
+  def test_access_log_format(self):
+    # type: () -> None
+    requests.get(
+        urlparse.urljoin(self.default_balancer_url, '/url_path'),
+        verify=False,
+    )
+    with open(os.path.join(self.computer_partition_root_path, 'var', 'log', 'apache-access.log')) as access_log_file:
+      access_line = access_log_file.read()
+    self.assertIn('/url_path', access_line)
+
+    # last \d is the request time in micro seconds, since this SlowHTTPServer
+    # sleeps for 3 seconds, it should take between 3 and 4 seconds to process
+    # the request - but our test machines can be slow sometimes, so we tolerate
+    # it can take up to 20 seconds.
+    match = re.match(
+        r'([(\d\.)]+) - - \[(.*?)\] "(.*?)" (\d+) (\d+) "(.*?)" "(.*?)" (\d+)',
+        access_line
+    )
+    self.assertTrue(match)
+    assert match
+    request_time = int(match.groups()[-1])
+    self.assertGreater(request_time, 3 * 1000 * 1000)
+    self.assertLess(request_time, 20 * 1000 * 1000)
+
+  def test_access_log_apachedex_report(self):
+    # type: () -> None
+    # make a request so that we have something in the logs
+    requests.get(self.default_balancer_url, verify=False)
+
+    # crontab for apachedex is executed
+    self._executeCrontabAtDate('generate-apachedex-report', '23:59')
+    # it creates a report for the day
+    apachedex_report, = glob.glob(
+        os.path.join(
+            self.computer_partition_root_path,
+            'srv',
+            'monitor',
+            'private',
+            'apachedex',
+            'ApacheDex-*.html',
+        ))
+    with open(apachedex_report, 'r') as f:
+      report_text = f.read()
+    self.assertIn('APacheDEX', report_text)
+    # having this table means that apachedex could parse some lines.
+    self.assertIn('<h2>Hits per status code</h2>', report_text)
+
+  def test_access_log_rotation(self):
+    # type: () -> None
+    # run logrotate a first time so that it create state files
+    self._executeCrontabAtDate('logrotate', '2000-01-01')
+
+    # make a request so that we have something in the logs
+    requests.get(self.default_balancer_url, verify=False).raise_for_status()
+
+    # slow query crontab depends on crontab for log rotation
+    # to be executed first.
+    self._executeCrontabAtDate('logrotate', '2050-01-01')
+    # this logrotate leaves the log for the day as non compressed
+    rotated_log_file = os.path.join(
+        self.computer_partition_root_path,
+        'srv',
+        'backup',
+        'logrotate',
+        'apache-access.log-20500101',
+    )
+    self.assertTrue(os.path.exists(rotated_log_file))
+
+    requests.get(self.default_balancer_url, verify=False).raise_for_status()
+    # on next day execution of logrotate, log files are compressed
+    self._executeCrontabAtDate('logrotate', '2050-01-02')
+    self.assertTrue(os.path.exists(rotated_log_file + '.xz'))
+    self.assertFalse(os.path.exists(rotated_log_file))
+
+
+class BalancerCookieHTTPServer(ManagedHTTPServer):
+  """An HTTP Server which can set balancer cookie.
+
+  This server set cookie when requested /set-cookie path.
+
+  The reply body is the name used when registering this resource
+  using getManagedResource. This way we can assert which
+  backend replied.
+  """
+
+  @property
+  def RequestHandler(self):
+    server = self
+    class RequestHandler(BaseHTTPRequestHandler):
+      def do_GET(self):
+        # type: () -> None
+        self.send_response(200)
+        self.send_header("Content-Type", "text/plain")
+        if self.path == '/set_cookie':
+          # the balancer tells the backend what's the name of the balancer cookie with
+          # the X-Balancer-Current-Cookie header.
+          self.send_header('Set-Cookie', '%s=anything' % self.headers['X-Balancer-Current-Cookie'])
+          # The name of this cookie is SERVERID
+          assert self.headers['X-Balancer-Current-Cookie'] == 'SERVERID'
+        self.end_headers()
+        self.wfile.write(server._name)
+      log_message = logging.getLogger(__name__ + '.BalancerCookieHTTPServer').info
+
+    return RequestHandler
+
+
+class TestBalancer(BalancerTestCase):
+  """Check balancing capabilities
+  """
+  __partition_reference__ = 'b'
+  @classmethod
+  def _getInstanceParameterDict(cls):
+    # type: () -> Dict
+    parameter_dict = super(TestBalancer, cls)._getInstanceParameterDict()
+
+    # use two backend servers
+    parameter_dict['dummy_http_server'] = [
+        [cls.getManagedResource("backend_web_server1", BalancerCookieHTTPServer).netloc, 1, False],
+        [cls.getManagedResource("backend_web_server2", BalancerCookieHTTPServer).netloc, 1, False],
     ]
+    return parameter_dict
 
-    caucase_process = subprocess.Popen(
-      cau_args + [
-        '--mode', 'user',
-        '--send-csr', frontend_user_csr,
-      ],
-      stdout=subprocess.PIPE,
-      stderr=subprocess.STDOUT,
+  def test_balancer_round_robin(self):
+    # requests are by default balanced to both servers
+    self.assertEqual(
+        {requests.get(self.default_balancer_url, verify=False).text for _ in range(10)},
+        {'backend_web_server1', 'backend_web_server2'}
     )
-    result = caucase_process.communicate()
-    csr_id = result[0].split()[0]
 
-    subprocess.check_call(
-      cau_args + [
-        '--mode', 'user',
-        '--get-crt', csr_id, frontend_user_key,
-      ],
+  def test_balancer_server_down(self):
+    # if one backend is down, it is excluded from balancer
+    self.getManagedResource("backend_web_server2", BalancerCookieHTTPServer).close()
+    self.addCleanup(self.getManagedResource("backend_web_server2", BalancerCookieHTTPServer).open)
+    self.assertEqual(
+        {requests.get(self.default_balancer_url, verify=False).text for _ in range(10)},
+        {'backend_web_server1',}
+    )
+
+  def test_balancer_set_cookie(self):
+    # if backend provides a "SERVERID" cookie, balancer will overwrite it with the
+    # backend selected by balancing algorithm
+    self.assertIn(
+        requests.get(urlparse.urljoin(self.default_balancer_url, '/set_cookie'), verify=False).cookies['SERVERID'],
+        ('default-0', 'default-1'),
+    )
+
+  def test_balancer_respects_sticky_cookie(self):
+    # if request is made with the sticky cookie, the client stick on one balancer
+    cookies = dict(SERVERID='default-1')
+    self.assertEqual(
+        {requests.get(self.default_balancer_url, verify=False, cookies=cookies).text for _ in range(10)},
+        {'backend_web_server2',}
     )
 
-    cls.client_certificate = frontend_service_key = os.path.join(frontend_service_dir, 'crt.pem')
-    frontend_service_csr = os.path.join(frontend_service_dir, 'csr.pem')
+    # if that backend becomes down, requests are balanced to another server
+    self.getManagedResource("backend_web_server2", BalancerCookieHTTPServer).close()
+    self.addCleanup(self.getManagedResource("backend_web_server2", BalancerCookieHTTPServer).open)
+    self.assertEqual(
+        requests.get(self.default_balancer_url, verify=False, cookies=cookies).text,
+        'backend_web_server1')
+
+
+class TestHTTP(BalancerTestCase):
+  """Check HTTP protocol
+  """
+  __partition_reference__ = 'h'
+
+  def test_http_version(self):
+    # type: () -> None
+    # https://stackoverflow.com/questions/37012486/python-3-x-how-to-get-http-version-using-requests-library/37012810
+    self.assertEqual(
+        requests.get(self.default_balancer_url, verify=False).raw.version, 11)
+
+  def test_keep_alive(self):
+    # type: () -> None
+    # when doing two requests, connection is established only once
+    session = requests.Session()
+    session.verify = False
+
+    # do a first request, which establish a first connection
+    session.get(self.default_balancer_url).raise_for_status()
+
+    # "break" new connection method and check we can make another request
+    with mock.patch(
+        "requests.packages.urllib3.connectionpool.HTTPSConnectionPool._new_conn",
+    ) as new_conn:
+      session.get(self.default_balancer_url).raise_for_status()
+    new_conn.assert_not_called()
+
+    parsed_url = urlparse.urlparse(self.default_balancer_url)
+    # check that we have an open file for the ip connection
+    self.assertTrue([
+        c for c in psutil.Process(os.getpid()).connections()
+        if c.status == 'ESTABLISHED' and c.raddr.ip == parsed_url.hostname
+        and c.raddr.port == parsed_url.port
+    ])
+
+
+class TestTLS(BalancerTestCase):
+  """Check TLS
+  """
+  __partition_reference__ = 's'
+
+  def _getServerCertificate(self, hostname, port):
+    # type: (Optional[str], Optional[int]) -> Any
+    hostname_idna = idna.encode(hostname)
+    sock = socket.socket()
+
+    sock.connect((hostname, port))
+    ctx = OpenSSL.SSL.Context(OpenSSL.SSL.SSLv23_METHOD)
+    ctx.check_hostname = False
+    ctx.verify_mode = OpenSSL.SSL.VERIFY_NONE
+
+    sock_ssl = OpenSSL.SSL.Connection(ctx, sock)
+    sock_ssl.set_connect_state()
+    sock_ssl.set_tlsext_host_name(hostname_idna)
+    sock_ssl.do_handshake()
+    cert = sock_ssl.get_peer_certificate()
+    crypto_cert = cert.to_cryptography()
+    sock_ssl.close()
+    sock.close()
+    return crypto_cert
+
+  def test_certificate_validates_with_caucase_ca(self):
+    # type: () -> None
+    caucase = self.getManagedResource("caucase", CaucaseService)
+    requests.get(self.default_balancer_url, verify=caucase.ca_crt_path)
+
+  def test_certificate_renewal(self):
+    # type: () -> None
+    caucase = self.getManagedResource("caucase", CaucaseService)
+    balancer_parsed_url = urlparse.urlparse(self.default_balancer_url)
+    certificate_before_renewal = self._getServerCertificate(
+        balancer_parsed_url.hostname,
+        balancer_parsed_url.port)
+
+    # run caucase updater 90 days in the future, so that certificate is
+    # renewed.
+    caucase_updater = os.path.join(
+        self.computer_partition_root_path,
+        'etc',
+        'service',
+        'caucase-updater',
+    )
+    process = pexpect.spawnu(
+       "faketime +90days %s" % caucase_updater,
+        env=dict(os.environ, PYTHONPATH=''),
+    )
+    logger = self.logger
+    class DebugLogFile:
+      def write(self, msg):
+        logger.info("output from caucase_updater: %s", msg)
+      def flush(self):
+        pass
+    process.logfile = DebugLogFile()
+    process.expect(u"Renewing .*\nNext wake-up.*")
+    process.terminate()
+    process.wait()
+
+    # wait for server to use new certificate
+    for _ in range(30):
+      certificate_after_renewal = self._getServerCertificate(
+          balancer_parsed_url.hostname,
+          balancer_parsed_url.port)
+      if certificate_after_renewal.not_valid_before > certificate_before_renewal.not_valid_before:
+        break
+      time.sleep(.5)
+
+    self.assertGreater(
+        certificate_after_renewal.not_valid_before,
+        certificate_before_renewal.not_valid_before,
+    )
+
+    # requests are served properly after cert renewal
+    requests.get(self.default_balancer_url, verify=caucase.ca_crt_path).raise_for_status()
+
+
+class ContentTypeHTTPServer(ManagedHTTPServer):
+  """An HTTP Server which reply with content type from path.
+
+  For example when requested http://host/text/plain it will reply
+  with Content-Type: text/plain header.
+
+  The body is always "OK"
+  """
+
+  class RequestHandler(BaseHTTPRequestHandler):
+    def do_GET(self):
+      # type: () -> None
+      self.send_response(200)
+      if self.path == '/':
+        return self.end_headers()
+      content_type = self.path[1:]
+      self.send_header("Content-Type", content_type)
+      self.end_headers()
+      self.wfile.write("OK")
+
+    log_message = logging.getLogger(__name__ + '.ContentTypeHTTPServer').info
+
+
+class TestContentEncoding(BalancerTestCase):
+  """Test how responses are gzip encoded or not depending on content type header.
+  """
+  __partition_reference__ = 'ce'
+  @classmethod
+  def _getInstanceParameterDict(cls):
+    # type: () -> Dict
+    parameter_dict = super(TestContentEncoding, cls)._getInstanceParameterDict()
+    parameter_dict['dummy_http_server'] = [
+        [cls.getManagedResource("content_type_server", ContentTypeHTTPServer).netloc, 1, False],
+    ]
+    return parameter_dict
+
+  def test_gzip_encoding(self):
+    # type: () -> None
+    for content_type in (
+        'text/cache-manifest',
+        'text/html',
+        'text/plain',
+        'text/css',
+        'application/hal+json',
+        'application/json',
+        'application/x-javascript',
+        'text/xml',
+        'application/xml',
+        'application/rss+xml',
+        'text/javascript',
+        'application/javascript',
+        'image/svg+xml',
+        'application/x-font-ttf',
+        'application/font-woff',
+        'application/font-woff2',
+        'application/x-font-opentype',
+        'application/wasm',):
+      resp = requests.get(urlparse.urljoin(self.default_balancer_url, content_type), verify=False)
+      self.assertEqual(resp.headers['Content-Type'], content_type)
+      self.assertEqual(
+          resp.headers['Content-Encoding'],
+          'gzip',
+          '%s uses wrong encoding: %s' % (content_type, resp.headers['Content-Encoding']))
+      self.assertEqual(resp.text, 'OK')
+
+  def test_no_gzip_encoding(self):
+    # type: () -> None
+    resp = requests.get(urlparse.urljoin(self.default_balancer_url, '/image/png'), verify=False)
+    self.assertNotIn('Content-Encoding', resp.headers)
+    self.assertEqual(resp.text, 'OK')
+
+
+class CaucaseClientCertificate(ManagedResource):
+  """A client certificate issued by a caucase services.
+  """
+
+  ca_crt_file = None # type: str
+  crl_file = None # type: str
+  csr_file = None # type: str
+  cert_file = None # type: str
+  key_file = None # type: str
+
+  def open(self):
+    # type: () -> None
+    self.tmpdir = tempfile.mkdtemp()
+    self.ca_crt_file = os.path.join(self.tmpdir, 'ca-crt.pem')
+    self.crl_file = os.path.join(self.tmpdir, 'ca-crl.pem')
+    self.csr_file = os.path.join(self.tmpdir, 'csr.pem')
+    self.cert_file = os.path.join(self.tmpdir, 'crt.pem')
+    self.key_file = os.path.join(self.tmpdir, 'key.pem')
+
+  def close(self):
+    # type: () -> None
+    shutil.rmtree(self.tmpdir)
+
+  @property
+  def _caucase_path(self):
+    # type: () -> str
+    """path of caucase executable.
+    """
+    software_release_root_path = os.path.join(
+        self._cls.slap._software_root,
+        hashlib.md5(self._cls.getSoftwareURL().encode()).hexdigest(),
+    )
+    return os.path.join(software_release_root_path, 'bin', 'caucase')
+
+  def request(self, common_name, caucase):
+    # type: (str, CaucaseService) -> None
+    """Generate certificate and request signature to the caucase service.
+
+    This overwrite any previously requested certificate for this instance.
+    """
+    cas_args = [
+        self._caucase_path,
+        '--ca-url', caucase.url,
+        '--ca-crt', self.ca_crt_file,
+        '--crl', self.crl_file,
+    ]
 
     key = rsa.generate_private_key(
         public_exponent=65537,
         key_size=2048,
         backend=default_backend()
     )
-    with open(frontend_service_key, 'wb') as f:
-      f.write(key.private_bytes(
+    with open(self.key_file, 'wb') as f:
+      f.write(
+          key.private_bytes(
               encoding=serialization.Encoding.PEM,
               format=serialization.PrivateFormat.TraditionalOpenSSL,
               encryption_algorithm=serialization.NoEncryption(),
           ))
 
-    csr = x509.CertificateSigningRequestBuilder().subject_name(x509.Name([
-      x509.NameAttribute(NameOID.COMMON_NAME, u'service'),
-    ])).sign(key, hashes.SHA256(), default_backend())
-    with open(frontend_service_csr, 'wb') as f:
+    csr = x509.CertificateSigningRequestBuilder().subject_name(
+        x509.Name([
+            x509.NameAttribute(
+                NameOID.COMMON_NAME,
+                common_name,
+            ),
+        ])).sign(
+            key,
+            hashes.SHA256(),
+            default_backend(),
+        )
+    with open(self.csr_file, 'wb') as f:
       f.write(csr.public_bytes(serialization.Encoding.PEM))
 
-    caucase_process = subprocess.Popen(
+    csr_id = subprocess.check_output(
       cas_args + [
-        '--send-csr', frontend_service_csr,
+          '--send-csr', self.csr_file,
       ],
-      stdout=subprocess.PIPE,
-      stderr=subprocess.STDOUT,
-    )
-    result = caucase_process.communicate()
-    csr_id = result[0].split()[0]
+    ).split()[0]
+    assert csr_id
 
-    for _ in range(10):
+    for _ in range(30):
       if not subprocess.call(
         cas_args + [
-          '--get-crt', csr_id, frontend_service_key,
+            '--get-crt', csr_id, self.cert_file,
         ],
+        stdout=subprocess.PIPE,
+        stderr=subprocess.STDOUT,
       ) == 0:
         break
       else:
         time.sleep(1)
     else:
       raise RuntimeError('getting service certificate failed.')
+    with open(self.cert_file) as f:
+      assert 'BEGIN CERTIFICATE' in f.read()
 
-    # start a caucased and server certificate.
-    cls.backend_caucase_dir = tempfile.mkdtemp()
-    backend_caucased_dir = os.path.join(cls.backend_caucase_dir, 'caucased')
-    os.mkdir(backend_caucased_dir)
-    backend_caucased_netloc = '%s:%s' % (cls._ipv4_address, findFreeTCPPort(cls._ipv4_address))
-    cls.backend_caucased_url = 'http://' + backend_caucased_netloc
-    cls.backend_caucased_process = subprocess.Popen(
-      [
-        caucased_path,
-        '--db', os.path.join(backend_caucased_dir, 'caucase.sqlite'),
-        '--server-key', os.path.join(backend_caucased_dir, 'server.key.pem'),
-        '--netloc', backend_caucased_netloc,
-        '--service-auto-approve-count', '1',
-      ],
-      stdout=subprocess.PIPE,
-      stderr=subprocess.STDOUT,
-    )
-    for _ in range(10):
-      try:
-        if requests.get(cls.backend_caucased_url).status_code == 200:
-          break
-      except Exception:
-        pass
-      time.sleep(1)
-    else:
-      raise RuntimeError('caucased failed to start.')
+  def revoke(self, caucase):
+    # type: (str, CaucaseService) -> None
+    """Revoke the client certificate on this caucase instance.
+    """
+    subprocess.check_call([
+        self._caucase_path,
+        '--ca-url', caucase.url,
+        '--ca-crt', self.ca_crt_file,
+        '--crl', self.crl_file,
+        '--revoke-crt', self.cert_file, self.key_file,
+    ])
 
-    super(TestFrontendXForwardedFor, cls).setUpClass()
+
+class TestFrontendXForwardedFor(BalancerTestCase):
+  __partition_reference__ = 'xff'
 
   @classmethod
-  def getInstanceParameterDict(cls):
-    return {
-      '_': json.dumps({
-        'tcpv4-port': 3306,
-        'computer-memory-percent-threshold': 100,
-        # XXX what is this ? should probably not be needed here
-        'name': cls.__name__,
-        'monitor-passwd': 'secret',
-        'apachedex-configuration': '',
-        'apachedex-promise-threshold': 100,
-        'haproxy-server-check-path': '/',
-        'zope-family-dict': {
-          'default': ['dummy_http_server'],
-          'default-auth': ['dummy_http_server'],
-        },
-        'dummy_http_server': [[cls.http_server_netloc, 1, False]],
-        'backend-path-dict': {
-          'default': '/',
-          'default-auth': '/',
-        },
-        'ssl-authentication-dict': {
+  def _getInstanceParameterDict(cls):
+    # type: () -> Dict
+    frontend_caucase = cls.getManagedResource('frontend_caucase', CaucaseService)
+    certificate = cls.getManagedResource('client_certificate', CaucaseClientCertificate)
+    certificate.request(u'shared frontend', frontend_caucase)
+
+    parameter_dict = super(TestFrontendXForwardedFor, cls)._getInstanceParameterDict()
+    # add another "-auth" backend, that will have ssl-authentication enabled
+    parameter_dict['zope-family-dict']['default-auth'] = ['dummy_http_server']
+    parameter_dict['backend-path-dict']['default-auth'] = '/'
+    parameter_dict['ssl-authentication-dict'] = {
         'default': False,
         'default-auth': True,
-        },
-        'ssl': {
-          'caucase-url': cls.backend_caucased_url,
-          'frontend-caucase-url-list': [cls.frontend_caucased_url],
-        },
-      })
     }
-
-  @classmethod
-  def _cleanup(cls, snapshot_name):
-    if cls.http_server_process:
-      cls.http_server_process.terminate()
-    if cls.frontend_caucased_process:
-      cls.frontend_caucased_process.terminate()
-    if cls.frontend_caucase_dir:
-      shutil.rmtree(cls.frontend_caucase_dir)
-    if cls.backend_caucased_process:
-      cls.backend_caucased_process.terminate()
-    if cls.backend_caucase_dir:
-      shutil.rmtree(cls.backend_caucase_dir)
-
-    super(TestFrontendXForwardedFor, cls)._cleanup(snapshot_name)
+    parameter_dict['ssl']['frontend-caucase-url-list'] = [frontend_caucase.url]
+    return parameter_dict
 
   def test_x_forwarded_for_added_when_verified_connection(self):
+    # type: () -> None
+    client_certificate = self.getManagedResource('client_certificate', CaucaseClientCertificate)
+
     for backend in ('default', 'default-auth'):
       balancer_url = json.loads(self.computer_partition.getConnectionParameterDict()['_'])[backend]
       result = requests.get(
         balancer_url,
         headers={'X-Forwarded-For': '1.2.3.4'},
-        cert=self.client_certificate,
+        cert=(client_certificate.cert_file, client_certificate.key_file),
         verify=False,
       ).json()
       self.assertEqual(result['Incoming Headers'].get('x-forwarded-for').split(', ')[0], '1.2.3.4')
 
   def test_x_forwarded_for_stripped_when_not_verified_connection(self):
+    # type: () -> None
     balancer_url = json.loads(self.computer_partition.getConnectionParameterDict()['_'])['default']
     result = requests.get(
       balancer_url,
@@ -299,3 +699,103 @@ class TestFrontendXForwardedFor(ERP5InstanceTestCase):
         headers={'X-Forwarded-For': '1.2.3.4'},
         verify=False,
       )
+
+
+class TestClientTLS(BalancerTestCase):
+  __partition_reference__ = 'c'
+
+  @classmethod
+  def _getInstanceParameterDict(cls):
+    # type: () -> Dict
+    frontend_caucase1 = cls.getManagedResource('frontend_caucase1', CaucaseService)
+    certificate1 = cls.getManagedResource('client_certificate1', CaucaseClientCertificate)
+    certificate1.request(u'client_certificate1', frontend_caucase1)
+
+    frontend_caucase2 = cls.getManagedResource('frontend_caucase2', CaucaseService)
+    certificate2 = cls.getManagedResource('client_certificate2', CaucaseClientCertificate)
+    certificate2.request(u'client_certificate2', frontend_caucase2)
+
+    parameter_dict = super(TestClientTLS, cls)._getInstanceParameterDict()
+    parameter_dict['ssl-authentication-dict'] = {
+        'default': True,
+    }
+    parameter_dict['ssl']['frontend-caucase-url-list'] = [
+        frontend_caucase1.url,
+        frontend_caucase2.url,
+    ]
+    return parameter_dict
+
+  def test_refresh_crl(self):
+    # type: () -> None
+
+    logger = self.logger
+
+    class DebugLogFile:
+      def write(self, msg):
+        logger.info("output from caucase_updater: %s", msg)
+      def flush(self):
+        pass
+
+    for client_certificate_name, caucase_name in (
+        ('client_certificate1', 'frontend_caucase1'),
+        ('client_certificate2', 'frontend_caucase2'),
+    ):
+      client_certificate = self.getManagedResource(client_certificate_name,
+                                                   CaucaseClientCertificate)
+
+      # when client certificate can be authenticated, backend receive the CN of
+      # the client certificate in "remote-user" header
+      def _make_request():
+        return requests.get(
+            self.default_balancer_url,
+            cert=(client_certificate.cert_file, client_certificate.key_file),
+            verify=False,
+        ).json()
+
+      self.assertEqual(_make_request()['Incoming Headers'].get('remote-user'),
+                       client_certificate_name)
+
+      # when certificate is revoked, updater service should update the CRL
+      # used by balancer from the caucase service used for client certificates
+      # (ie. the one used by frontend).
+      caucase = self.getManagedResource(caucase_name, CaucaseService)
+      client_certificate.revoke(caucase)
+
+      # until the CRL is updated, the client certificate is still accepted.
+      self.assertEqual(_make_request()['Incoming Headers'].get('remote-user'),
+                       client_certificate_name)
+
+      # We have two services, in charge of updating CRL and CA certificates for
+      # each frontend CA
+      caucase_updater_list = glob.glob(
+          os.path.join(
+              self.computer_partition_root_path,
+              'etc',
+              'service',
+              'caucase-updater-*',
+          ))
+      self.assertEqual(len(caucase_updater_list), 2)
+
+      # find the one corresponding to this caucase
+      for caucase_updater_candidate in caucase_updater_list:
+        with open(caucase_updater_candidate) as f:
+          if caucase.url in f.read():
+            caucase_updater = caucase_updater_candidate
+            break
+      else:
+        self.fail("Could not find caucase updater script for %s" % caucase.url)
+
+      # simulate running updater service in the future, to confirm that it fetches
+      # the new CRL and make sure balancer uses that new CRL.
+      process = pexpect.spawnu(
+          "faketime +1day %s" % caucase_updater,
+          env=dict(os.environ, PYTHONPATH=''),
+      )
+
+      process.logfile = DebugLogFile()
+      process.expect(u"Got new CRL.*Next wake-up at.*")
+      process.terminate()
+      process.wait()
+
+      with self.assertRaisesRegexp(Exception, 'certificate revoked'):
+        _make_request()

software/erp5/test/test/test_erp5.py

@@ -31,6 +31,7 @@ import glob
 import urlparse
 import socket
 import time
+import tempfile
 
 import psutil
 import requests
@@ -43,7 +44,7 @@ setUpModule # pyflakes
 class TestPublishedURLIsReachableMixin(object):
   """Mixin that checks that default page of ERP5 is reachable.
   """
-  def _checkERP5IsReachable(self, url):
+  def _checkERP5IsReachable(self, url, verify):
     # What happens is that instanciation just create the services, but does not
     # wait for ERP5 to be initialized. When this test run ERP5 instance is
     # instanciated, but zope is still busy creating the site and haproxy replies
@@ -51,7 +52,7 @@ class TestPublishedURLIsReachableMixin(object):
     # erp5 site is not created, with 500 when mysql is not yet reachable, so we
     # retry in a loop until we get a succesful response.
     for i in range(1, 60):
-      r = requests.get(url, verify=False)  # XXX can we get CA from caucase already ?
+      r = requests.get(url, verify=verify)
       if r.status_code != requests.codes.ok:
         delay = i * 2
         self.logger.warn("ERP5 was not available, sleeping for %ds and retrying", delay)
@@ -62,19 +63,36 @@ class TestPublishedURLIsReachableMixin(object):
 
     self.assertIn("ERP5", r.text)
 
+  def _getCaucaseServiceCACertificate(self):
+    ca_cert = tempfile.NamedTemporaryFile(
+        prefix="ca.crt.pem",
+        mode="w",
+        delete=False,
+    )
+    ca_cert.write(
+        requests.get(
+            urlparse.urljoin(
+                self.getRootPartitionConnectionParameterDict()['caucase-http-url'],
+                '/cas/crt/ca.crt.pem',
+            )).text)
+    self.addCleanup(os.unlink, ca_cert.name)
+    return ca_cert.name
+
   def test_published_family_default_v6_is_reachable(self):
     """Tests the IPv6 URL published by the root partition is reachable.
     """
     param_dict = self.getRootPartitionConnectionParameterDict()
     self._checkERP5IsReachable(
-      urlparse.urljoin(param_dict['family-default-v6'], param_dict['site-id']))
+      urlparse.urljoin(param_dict['family-default-v6'], param_dict['site-id']),
+      self._getCaucaseServiceCACertificate())
 
   def test_published_family_default_v4_is_reachable(self):
     """Tests the IPv4 URL published by the root partition is reachable.
     """
     param_dict = self.getRootPartitionConnectionParameterDict()
     self._checkERP5IsReachable(
-      urlparse.urljoin(param_dict['family-default'], param_dict['site-id']))
+      urlparse.urljoin(param_dict['family-default'], param_dict['site-id']),
+      self._getCaucaseServiceCACertificate())
 
 
 class TestDefaultParameters(ERP5InstanceTestCase, TestPublishedURLIsReachableMixin):

software/fluentd/software.cfg

@@ -24,4 +24,3 @@ gems +=
 
 [versions]
 slapos.recipe.template = 4.4
-rubygemsrecipe = 0.2.2+slapos001

software/gitlab/software.cfg

@@ -405,6 +405,5 @@ strip-top-level-dir = true
 cns.recipe.symlink = 0.2.3
 docutils = 0.12
 plone.recipe.command = 1.1
-rubygemsrecipe = 0.2.2+slapos002
 slapos.recipe.template = 4.3
 z3c.recipe.scripts = 1.0.1

software/helloworld/software.cfg

@@ -29,6 +29,9 @@ parts =
   helloweb-go
 
 
+[python]
+part = python3
+
 # Macro for jinja templates. The filename is set in buildout.hash.cfg
 # in the section using this template
 [jinja-template]
@@ -54,4 +57,3 @@ context =
 # Pin versions of eggs used that are not already pinned by stack/slapos.cfg
 [versions]
 slapos.recipe.template = 4.4
-rubygemsrecipe  = 0.2.2+slapos001

software/kvm/buildout.hash.cfg

@@ -55,7 +55,7 @@ md5sum = b7e87479a289f472b634a046b44b5257
 
 [template-kvm-run]
 filename = template/template-kvm-run.in
-md5sum = 4a6f149177a453a13436f320f6841518
+md5sum = a97b8462955dd422a30fbe02d6906172
 
 [template-kvm-controller]
 filename = template/kvm-controller-run.in

software/kvm/instance-kvm-cluster-input-schema.json

@@ -505,9 +505,9 @@
             },
             "boot-image-url-list": {
               "title": "Boot image list",
-              "description": "The list shall be list of direct URLs to images, followed by hash (#), then by image MD5SUM. Each image shall appear on newline, like: \"https://example.com/image.iso#06226c7fac5bacfa385872a19bb99684<newline>https://example.com/another-image.iso#31b40d58b18e038498ddb46caea1361c\". They will be provided in KVM image list according to the order on the list. After updating the list, the instance has to be restarted to refresh it. Amount of images is limited to 4, and one image can be maximum 5G. Image will be downloaded and checked against its MD5SUM 4 times, then it will be considered as impossible to download with given MD5SUM. Each image has to be downloaded in time shorter than 4 hours, so in case of very slow images to access, it can take up to 16 hours to download all of them. Note: The instance has to be restarted in order to update the list of available images in the VM.",
+              "description": "The list shall be list of direct URLs to images, followed by hash (#), then by image MD5SUM. Each image shall appear on newline, like: \"https://example.com/image.iso#06226c7fac5bacfa385872a19bb99684<newline>https://example.com/another-image.iso#31b40d58b18e038498ddb46caea1361c\". They will be provided in KVM image list according to the order on the list. After updating the list, the instance has to be restarted to refresh it. Amount of images is limited to 4, and one image can be maximum 5G. Image will be downloaded and checked against its MD5SUM 4 times, then it will be considered as impossible to download with given MD5SUM. Each image has to be downloaded in time shorter than 4 hours, so in case of very slow images to access, it can take up to 16 hours to download all of them. Note: The instance has to be restarted in order to update the list of available images in the VM. Note: Maximum 3 ISOs are supported.",
               "type": "string",
-              "textarea": "true"
+              "textarea": true
             },
             "boot-image-url-select": {
               "title": "Boot image",

software/kvm/instance-kvm-input-schema.json

@@ -368,9 +368,9 @@
     },
     "boot-image-url-list": {
       "title": "Boot image list",
-      "description": "The list shall be list of direct URLs to images, followed by hash (#), then by image MD5SUM. Each image shall appear on newline, like: \"https://example.com/image.iso#06226c7fac5bacfa385872a19bb99684<newline>https://example.com/another-image.iso#31b40d58b18e038498ddb46caea1361c\". They will be provided in KVM image list according to the order on the list. After updating the list, the instance has to be restarted to refresh it. Amount of images is limited to 4, and one image can be maximum 5G. Image will be downloaded and checked against its MD5SUM 4 times, then it will be considered as impossible to download with given MD5SUM. Each image has to be downloaded in time shorter than 4 hours, so in case of very slow images to access, it can take up to 16 hours to download all of them. Note: The instance has to be restarted in order to update the list of available images in the VM.",
+      "description": "The list shall be list of direct URLs to images, followed by hash (#), then by image MD5SUM. Each image shall appear on newline, like: \"https://example.com/image.iso#06226c7fac5bacfa385872a19bb99684<newline>https://example.com/another-image.iso#31b40d58b18e038498ddb46caea1361c\". They will be provided in KVM image list according to the order on the list. After updating the list, the instance has to be restarted to refresh it. Amount of images is limited to 4, and one image can be maximum 5G. Image will be downloaded and checked against its MD5SUM 4 times, then it will be considered as impossible to download with given MD5SUM. Each image has to be downloaded in time shorter than 4 hours, so in case of very slow images to access, it can take up to 16 hours to download all of them. Note: The instance has to be restarted in order to update the list of available images in the VM. Note: Maximum 3 ISOs are supported.",
       "type": "string",
-      "textarea": "true"
+      "textarea": true
     },
     "boot-image-url-select": {
       "title": "Boot image",

software/kvm/software.cfg

@@ -10,6 +10,7 @@ extends =
   ../../component/netcat/buildout.cfg
   ../../component/lxml-python/buildout.cfg
   ../../component/pycurl/buildout.cfg
+  ../../component/numpy/buildout.cfg
   ../../component/gzip/buildout.cfg
   ../../stack/slapos.cfg
   ../../component/nodejs/buildout.cfg
@@ -206,4 +207,3 @@ gitdb = 0.6.4
 pycurl = 7.43.0
 slapos.recipe.template = 4.4
 smmap = 0.9.0
-numpy = 1.16.4

software/kvm/template/template-kvm-run.in

@@ -371,7 +371,8 @@ for nbd_ip, nbd_port in nbd_list:
           '-drive',
           'file=nbd:[%s]:%s,media=cdrom' % (nbd_ip, nbd_port)])
 else:
-  index = 0
+  # Note: Do not get tempted to use virtio-scsi-pci, as it does not work with
+  #       Debian installation CDs, rendering it uninstallable
   if boot_image_url_select_json_config:
     # Support boot-image-url-select
     with open(boot_image_url_select_json_config) as fh:
@@ -381,11 +382,9 @@ else:
         link = os.path.join(image_config['destination-directory'], image['link'])
         if os.path.exists(link) and os.path.islink(link):
           kvm_argument_list.extend([
-            '-drive', 'file=%s,media=cdrom,if=none,id=cdrom%s' % (link, index),
-            '-device', 'virtio-scsi-pci,id=scsi%s' % (index,),
-            '-device', 'scsi-cd,bus=scsi%s.0,drive=cdrom%s' % (index, index)
+            '-drive',
+            'file=%s,media=cdrom' % (link,)
           ])
-          index += 1
   if boot_image_url_list_json_config:
     # Support boot-image-url-list
     with open(boot_image_url_list_json_config) as fh:
@@ -395,16 +394,12 @@ else:
         link = os.path.join(image_config['destination-directory'], image['link'])
         if os.path.exists(link) and os.path.islink(link):
           kvm_argument_list.extend([
-            '-drive', 'file=%s,media=cdrom,if=none,id=cdrom%s' % (link, index),
-            '-device', 'virtio-scsi-pci,id=scsi%s' % (index,),
-            '-device', 'scsi-cd,bus=scsi%s.0,drive=cdrom%s' % (index, index)
+            '-drive',
+            'file=%s,media=cdrom' % (link,)
           ])
-          index += 1
   # Always add by default the default image
   kvm_argument_list.extend([
-      '-drive', 'file=%s,media=cdrom,if=none,id=cdrom%s' % (default_cdrom_iso, index),
-      '-device', 'virtio-scsi-pci,id=scsi%s' % (index,),
-      '-device', 'scsi-cd,bus=scsi%s.0,drive=cdrom%s' % (index, index)
+      '-drive', 'file=%s,media=cdrom' % default_cdrom_iso
   ])
 
 

software/kvm/test/test.py

@@ -604,7 +604,7 @@ class TestBootImageUrlList(InstanceTestCase):
     # check that the image is NOT YET available in kvm
     self.assertEqual(
       ['file=/parts/debian-amd64-netinst.iso/debian-amd64-netinst.iso,'
-       'media=cdrom,if=none,id=cdrom0'],
+       'media=cdrom'],
       getRunningImageList()
     )
 
@@ -618,12 +618,10 @@ class TestBootImageUrlList(InstanceTestCase):
     # now the image is available in the kvm, and its above default image
     self.assertEqual(
       [
-        'file=/srv/%s/image_001,media=cdrom,if=none,id=cdrom0' % (
-          self.image_directory,),
-        'file=/srv/%s/image_002,media=cdrom,if=none,id=cdrom1' % (
-          self.image_directory,),
+        'file=/srv/%s/image_001,media=cdrom' % (self.image_directory,),
+        'file=/srv/%s/image_002,media=cdrom' % (self.image_directory,),
         'file=/parts/debian-amd64-netinst.iso/debian-amd64-netinst.iso,'
-        'media=cdrom,if=none,id=cdrom2'
+        'media=cdrom'
       ],
       getRunningImageList()
     )
@@ -647,7 +645,7 @@ class TestBootImageUrlList(InstanceTestCase):
     # again only default image is available in the running process
     self.assertEqual(
       ['file=/parts/debian-amd64-netinst.iso/debian-amd64-netinst.iso,'
-       'media=cdrom,if=none,id=cdrom0'],
+       'media=cdrom'],
       getRunningImageList()
     )
 
@@ -786,7 +784,7 @@ class TestBootImageUrlSelect(TestBootImageUrlList):
     # check that the image is NOT YET available in kvm
     self.assertEqual(
       ['file=/parts/debian-amd64-netinst.iso/debian-amd64-netinst.iso,'
-       'media=cdrom,if=none,id=cdrom0'],
+       'media=cdrom'],
       getRunningImageList()
     )
 
@@ -800,12 +798,10 @@ class TestBootImageUrlSelect(TestBootImageUrlList):
     # now the image is available in the kvm, and its above default image
     self.assertEqual(
       [
-        'file=/srv/boot-image-url-select-repository/image_001,media=cdrom,'
-        'if=none,id=cdrom0',
-        'file=/srv/boot-image-url-list-repository/image_001,media=cdrom,'
-        'if=none,id=cdrom1',
+        'file=/srv/boot-image-url-select-repository/image_001,media=cdrom',
+        'file=/srv/boot-image-url-list-repository/image_001,media=cdrom',
         'file=/parts/debian-amd64-netinst.iso/debian-amd64-netinst.iso,'
-        'media=cdrom,if=none,id=cdrom2'
+        'media=cdrom'
       ],
       getRunningImageList()
     )
@@ -834,7 +830,7 @@ class TestBootImageUrlSelect(TestBootImageUrlList):
     # again only default image is available in the running process
     self.assertEqual(
       ['file=/parts/debian-amd64-netinst.iso/debian-amd64-netinst.iso,'
-       'media=cdrom,if=none,id=cdrom0'],
+       'media=cdrom'],
       getRunningImageList()
     )
 

software/neoppod/software-common.cfg

@@ -15,10 +15,11 @@ extends =
     ../../component/lxml-python/buildout.cfg
 #END LXML
     ../../component/msgpack-python/buildout.cfg
-    ../../component/patch/buildout.cfg
     ../../component/python-mysqlclient/buildout.cfg
     ../../component/python-cryptography/buildout.cfg
     ../../component/pycurl/buildout.cfg
+    ../../component/ZEO/buildout.cfg
+    ../../component/zodbtools/buildout.cfg
 
 parts =
 # keep neoppod first so that ZODB is built correctly,
@@ -50,14 +51,10 @@ eggs = neoppod[admin, ctl, master]
   psutil
   ZODB
   zope.testing
-  zodbtools
+  ${zodbtools:egg}
   coverage
   setproctitle
 adapter-egg = ${python-mysqlclient:egg}
-patch-binary = ${patch:location}/bin/patch
-ZEO-patch-options = -p1
-ZEO-patches =
-  ${:_profile_base_location_}/../../component/egg-patch/ZEO4/TCP_NODELAY.patch#b07288522d5c6857738240d948321df6
 
 # XXX: buildout fails to install properly eggs with setup_requires
 [BTrees]
@@ -152,23 +149,9 @@ setproctitle = 1.1.10
 slapos.recipe.template = 4.4
 transaction = 1.7.0
 zodbpickle = 1.0.4
-zodbtools = 0.0.0.dev4
 cython-zstd = 0.2
 python-dateutil = 2.7.3
 
-# Required by:
-# zodbtools==0.0.0dev4
-zodburi = 2.3.0
-
-# Required by:
-# zodburi==2.0
-# ZEO 5 requires transaction >= 2
-ZEO = 4.3.1+SlapOSPatched001
-
-# Required by:
-# ZEO==4.3.1
-zdaemon = 4.2.0
-
 # Required by:
 # mock = 3.0.5
 funcsigs = 1.0.2

software/neoppod/software-zodb5.cfg

@@ -3,7 +3,9 @@ extends = software.cfg
 
 [neoppod]
 eggs += mock
-ZEO-patches =
+
+[ZEO]
+egg = ${ZEO5:egg}
 
 [versions]
 ZODB = 5.6.0

software/neotest/buildout.hash.cfg

@@ -19,11 +19,3 @@ md5sum = bd46e95f1cea62c3b0082fe8c0c9c90b
 [neotest]
 filename = neotest.in
 md5sum = fb3b4109128c1db1739ef5bb6abd1d94
-
-[neotest-runTestSuite]
-filename = neotest-runTestSuite.in
-md5sum = 6a4281730b68cdba5c873817a6754428
-
-[instance.cfg]
-filename = instance.cfg.in
-md5sum = 8a40d365c85d70f057ce997405ac3e88

software/neotest/gowork.cfg

@@ -5,25 +5,34 @@
 depends_gitfetch  =
     ${go_github.com_DataDog_czlib:recipe}
     ${go_github.com_cznic_strutil:recipe}
+    ${go_github.com_davecgh_go-spew:recipe}
     ${go_github.com_fsnotify_fsnotify:recipe}
     ${go_github.com_golang_glog:recipe}
     ${go_github.com_gwenn_gosqlite:recipe}
     ${go_github.com_gwenn_yacr:recipe}
     ${go_github.com_kisielk_og-rek:recipe}
     ${go_github.com_kylelemons_godebug:recipe}
+    ${go_github.com_philhofer_fwd:recipe}
     ${go_github.com_pkg_errors:recipe}
     ${go_github.com_pkg_profile:recipe}
+    ${go_github.com_pmezard_go-difflib:recipe}
+    ${go_github.com_shamaton_msgpack:recipe}
     ${go_github.com_soheilhy_cmux:recipe}
     ${go_github.com_someonegg_gocontainer:recipe}
     ${go_github.com_someonegg_gox:recipe}
     ${go_github.com_stretchr_testify:recipe}
+    ${go_github.com_tinylib_msgp:recipe}
+    ${go_github.com_ttacon_chalk:recipe}
     ${go_golang.org_x_crypto:recipe}
+    ${go_golang.org_x_mod:recipe}
     ${go_golang.org_x_net:recipe}
     ${go_golang.org_x_perf:recipe}
     ${go_golang.org_x_sync:recipe}
     ${go_golang.org_x_sys:recipe}
     ${go_golang.org_x_text:recipe}
     ${go_golang.org_x_tools:recipe}
+    ${go_golang.org_x_xerrors:recipe}
+    ${go_gopkg.in_yaml.v3:recipe}
     ${go_lab.nexedi.com_kirr_go123:recipe}
     ${go_lab.nexedi.com_kirr_neo:recipe}
 
@@ -40,11 +49,17 @@ go.importpath = github.com/cznic/strutil
 repository    = https://github.com/cznic/strutil
 revision      = 529a34b1c1
 
+[go_github.com_davecgh_go-spew]
+<= go-git-package
+go.importpath = github.com/davecgh/go-spew
+repository    = https://github.com/davecgh/go-spew
+revision      = v1.1.1-1-gd8f796af33
+
 [go_github.com_fsnotify_fsnotify]
 <= go-git-package
 go.importpath = github.com/fsnotify/fsnotify
 repository    = https://github.com/fsnotify/fsnotify
-revision      = ccc981bf80
+revision      = 7f4cf4dd2b
 
 [go_github.com_golang_glog]
 <= go-git-package
@@ -56,112 +71,160 @@ revision      = 23def4e6c1
 <= go-git-package
 go.importpath = github.com/gwenn/gosqlite
 repository    = https://github.com/gwenn/gosqlite
-revision      = 29cd841087
+revision      = 24878be1a2
 
 [go_github.com_gwenn_yacr]
 <= go-git-package
 go.importpath = github.com/gwenn/yacr
 repository    = https://github.com/gwenn/yacr
-revision      = 77093bdc7e
+revision      = bbe82c1f4d
 
 [go_github.com_kisielk_og-rek]
 <= go-git-package
 go.importpath = github.com/kisielk/og-rek
 repository    = https://github.com/kisielk/og-rek
-revision      = 8b25c4cefd
+revision      = 24bb08c22e
 
 [go_github.com_kylelemons_godebug]
 <= go-git-package
 go.importpath = github.com/kylelemons/godebug
 repository    = https://github.com/kylelemons/godebug
-revision      = d65d576e93
+revision      = fa7b53cdfc
+
+[go_github.com_philhofer_fwd]
+<= go-git-package
+go.importpath = github.com/philhofer/fwd
+repository    = https://github.com/philhofer/fwd
+revision      = 414ae1bb9e
 
 [go_github.com_pkg_errors]
 <= go-git-package
 go.importpath = github.com/pkg/errors
 repository    = https://github.com/pkg/errors
-revision      = v0.8.0-12-g816c908556
+revision      = v0.8.1-31-g614d223910
 
 [go_github.com_pkg_profile]
 <= go-git-package
 go.importpath = github.com/pkg/profile
 repository    = https://github.com/pkg/profile
-revision      = v1.2.1-0-g5b67d42886
+revision      = v1.5.0-1-g3704c8d233
+
+[go_github.com_pmezard_go-difflib]
+<= go-git-package
+go.importpath = github.com/pmezard/go-difflib
+repository    = https://github.com/pmezard/go-difflib
+revision      = v1.0.0-4-g5d4384ee4f
+
+[go_github.com_shamaton_msgpack]
+<= go-git-package
+go.importpath = github.com/shamaton/msgpack
+repository    = https://github.com/shamaton/msgpack
+revision      = 29a4ba0bb9
 
 [go_github.com_soheilhy_cmux]
 <= go-git-package
 go.importpath = github.com/soheilhy/cmux
 repository    = https://github.com/soheilhy/cmux
-revision      = e09e9389d8
+revision      = 8a8ea3c539
 
 [go_github.com_someonegg_gocontainer]
 <= go-git-package
 go.importpath = github.com/someonegg/gocontainer
 repository    = https://github.com/someonegg/gocontainer
-revision      = fc2c7e84b5
+revision      = 24fb283ab9
 
 [go_github.com_someonegg_gox]
 <= go-git-package
 go.importpath = github.com/someonegg/gox
 repository    = https://github.com/someonegg/gox
-revision      = 4915b7fd7c
+revision      = c9bfdf8eb4
 
 [go_github.com_stretchr_testify]
 <= go-git-package
 go.importpath = github.com/stretchr/testify
 repository    = https://github.com/stretchr/testify
-revision      = 363ebb24d0
+revision      = v1.2.2-210-g54d05a4e18
+
+[go_github.com_tinylib_msgp]
+<= go-git-package
+go.importpath = github.com/tinylib/msgp
+repository    = https://github.com/tinylib/msgp
+revision      = 87c1ec45d5
+
+[go_github.com_ttacon_chalk]
+<= go-git-package
+go.importpath = github.com/ttacon/chalk
+repository    = https://github.com/ttacon/chalk
+revision      = v0.1-3-g22c06c80ed
 
 [go_golang.org_x_crypto]
 <= go-git-package
 go.importpath = golang.org/x/crypto
 repository    = https://go.googlesource.com/crypto
-revision      = 88942b9c40
+revision      = 5c72a88397
+
+[go_golang.org_x_mod]
+<= go-git-package
+go.importpath = golang.org/x/mod
+repository    = https://go.googlesource.com/mod
+revision      = ce943fd024
 
 [go_golang.org_x_net]
 <= go-git-package
 go.importpath = golang.org/x/net
 repository    = https://go.googlesource.com/net
-revision      = 6078986fec
+revision      = 62affa334b
 
 [go_golang.org_x_perf]
 <= go-git-package
 go.importpath = golang.org/x/perf
 repository    = https://go.googlesource.com/perf
-revision      = 8c788eb673
+revision      = d949658356
 
 [go_golang.org_x_sync]
 <= go-git-package
 go.importpath = golang.org/x/sync
 repository    = https://go.googlesource.com/sync
-revision      = 1d60e4601c
+revision      = 6e8e738ad2
 
 [go_golang.org_x_sys]
 <= go-git-package
 go.importpath = golang.org/x/sys
 repository    = https://go.googlesource.com/sys
-revision      = 91ee8cde43
+revision      = aee5d888a8
 
 [go_golang.org_x_text]
 <= go-git-package
 go.importpath = golang.org/x/text
 repository    = https://go.googlesource.com/text
-revision      = v0.3.0-42-gab48842968
+revision      = v0.3.3-2-ga8b4671254
 
 [go_golang.org_x_tools]
 <= go-git-package
 go.importpath = golang.org/x/tools
 repository    = https://go.googlesource.com/tools
-revision      = 77106db15f
+revision      = ba800b16d8
+
+[go_golang.org_x_xerrors]
+<= go-git-package
+go.importpath = golang.org/x/xerrors
+repository    = https://go.googlesource.com/xerrors
+revision      = 5ec99f83af
+
+[go_gopkg.in_yaml.v3]
+<= go-git-package
+go.importpath = gopkg.in/yaml.v3
+repository    = https://gopkg.in/yaml.v3
+revision      = v2.1.1-97-geeeca48fe7
 
 [go_lab.nexedi.com_kirr_go123]
 <= go-git-package
 go.importpath = lab.nexedi.com/kirr/go123
 repository    = https://lab.nexedi.com/kirr/go123.git
-revision      = 76f667ba6c
+revision      = 316617668e
 
 [go_lab.nexedi.com_kirr_neo]
 <= go-git-package
 go.importpath = lab.nexedi.com/kirr/neo
 repository    = https://lab.nexedi.com/kirr/neo.git
-revision      = v1.8.1-1634-g4000df14e4
+revision      = v1.9-2480-g5ee3c077b3

software/neotest/neotest-runTestSuite.in

-#!/bin/bash -e
-# neotest's runTestSuite wraper so it could be run without any environment preset
-
-. ${buildout:directory}/neotest-env.sh
-exec ${gowork:src}/lab.nexedi.com/kirr/neo/go/neo/t/nxd/runTestSuite "$@"

software/neotest/software.cfg

@@ -17,6 +17,10 @@ extends =
     ../../component/lmbench/buildout.cfg
     ../../component/coreutils/buildout.cfg
     ../../component/util-linux/buildout.cfg
+    ../../component/pygolang/buildout.cfg
+    ../../component/numpy/buildout.cfg
+    ../../stack/nxdtest.cfg
+
 
 parts =
     gowork
@@ -36,7 +40,6 @@ parts =
 
     neotest-env.sh
     neotest
-    neotest-runTestSuite
 
 #   for instance
     slapos-cookbook
@@ -73,23 +76,28 @@ output  = ${buildout:directory}/${:_buildout_section_name_}
 output  = ${buildout:bin-directory}/${:_buildout_section_name_}
 mode    = 0755
 
-[neotest-runTestSuite]
-<= buildout-template
-output  = ${buildout:bin-directory}/${:_buildout_section_name_}
-mode    = 0755
-
-
-# instance
-[jinja2-template]
-recipe  = slapos.recipe.template:jinja2
-template= ${:_profile_base_location_}/${:filename}
-rendered= ${buildout:directory}/${:_buildout_section_name_}
-mode    = 0644
-context =
-  section buildout  buildout
 
+# instance to run nxdtest.
 [instance.cfg]
 <= jinja2-template
+template = inline:
+  [buildout]
+  extends = ${nxdtest-instance.cfg:rendered}
+  parts += tnxdtest
+
+  [runTestSuite]
+  env.sh  = ${neotest-env.sh:output}
+  workdir = $${directory:t}
+
+  [directory]
+  t = $${:home}/t
+
+  # instance/t/.nxdtest -> .nxdtest inside go/neo
+  [tnxdtest]
+  recipe = plone.recipe.command
+  stop-on-error = yes
+  command = ln -s -t $${directory:t} ${gowork:src}/lab.nexedi.com/kirr/neo/go/neo/t/.nxdtest
+
 
 
 # eggs:
@@ -99,15 +107,15 @@ eggs    =
 # wendelin.core still requires ZODB3 but having ZODB4 or 5 installed satisfies latest ZODB3
     ZODB3
 # also for wc
-    numpy
+    ${numpy:egg}
 # to install not only wendelin.core modules but also scripts
     wendelin.core
 # for ZEO scripts (runzeo)
-    ZEO
-# for nxd/runTestSuite
-    erp5.util
+    ${ZEO:egg}
 # for e.g. tcpu.py
-    pygolang
+    ${pygolang:egg}
+# for instance
+    plone.recipe.command
 
 
 # wendelin.core: latest not yet released
@@ -117,9 +125,7 @@ revision= v0.11-15-gf785ac079b
 # ping eggs versions
 [versions]
 ZODB3 = 3.11.0
-numpy = 1.16.4
 zope.testing = 4.6.2
-pygolang = 0.0.0.dev4
 
 # Required by:
 # ZEO==4.3.1

software/repman/software.cfg

@@ -153,4 +153,3 @@ link-binary =
 # Pin versions of eggs used that are not already pinned by stack/slapos.cfg
 [versions]
 slapos.recipe.template = 4.4
-rubygemsrecipe  = 0.2.2+slapos001

software/seleniumserver/test/test.py

@@ -34,8 +34,10 @@ import unittest
 import urlparse
 import base64
 import hashlib
+import logging
 import contextlib
-from BaseHTTPServer import BaseHTTPRequestHandler, HTTPServer
+from BaseHTTPServer import BaseHTTPRequestHandler
+
 from io import BytesIO
 
 import paramiko
@@ -48,30 +50,22 @@ from selenium.webdriver.support import expected_conditions as EC
 from selenium.webdriver.support.ui import WebDriverWait
 
 from slapos.testing.testcase import makeModuleSetUpAndTestCaseClass
-from slapos.testing.utils import findFreeTCPPort, ImageComparisonTestCase
+from slapos.testing.utils import findFreeTCPPort, ImageComparisonTestCase, ManagedHTTPServer
 
 setUpModule, SeleniumServerTestCase = makeModuleSetUpAndTestCaseClass(
     os.path.abspath(
         os.path.join(os.path.dirname(__file__), '..', 'software.cfg')))
 
 
-class WebServerMixin(object):
-  """Mixin class which provides a simple web server reachable at self.server_url
-  """
-  def setUp(self):
-    """Start a minimal web server.
-    """
-    class TestHandler(BaseHTTPRequestHandler):
+
+class WebServer(ManagedHTTPServer):
+  class RequestHandler(BaseHTTPRequestHandler):
     """Request handler for our test server.
 
     The implemented server is:
       - submit q and you'll get a page with q as title
       - upload a file and the file content will be displayed in div.uploadedfile
     """
-      def log_message(self, *args, **kw):
-        if SeleniumServerTestCase._debug:
-          BaseHTTPRequestHandler.log_message(self, *args, **kw)
-
     def do_GET(self):
       self.send_response(200)
       self.end_headers()
@@ -111,18 +105,14 @@ class WebServerMixin(object):
         </html>
       ''' % (form['q'].value, file_data))
 
-    super(WebServerMixin, self).setUp()
-    ip = os.environ.get('SLAPOS_TEST_IPV4', '127.0.1.1')
-    port = findFreeTCPPort(ip)
-    server = HTTPServer((ip, port), TestHandler)
-    self.server_process = multiprocessing.Process(target=server.serve_forever)
-    self.server_process.start()
-    self.server_url = 'http://%s:%s/' % (ip, port)
+    log_message = logging.getLogger(__name__ + '.WebServer').info
 
-  def tearDown(self):
-    self.server_process.terminate()
-    self.server_process.join()
-    super(WebServerMixin, self).tearDown()
+
+class WebServerMixin(object):
+  """Mixin class which provides a simple web server reachable at self.server_url
+  """
+  def setUp(self):
+    self.server_url = self.getManagedResource('web_server', WebServer).url
 
 
 class BrowserCompatibilityMixin(WebServerMixin):

software/slapos-sr-testing/software-py3.cfg

@@ -12,6 +12,7 @@ eggs -=
 
 [template]
 extra =
+  ${slapos.test.helloworld-setup:setup}
   ${slapos.test.monitor-setup:setup}
   ${slapos.test.powerdns-setup:setup}
   ${slapos.test.proftpd-setup:setup}

software/slapos-sr-testing/software.cfg

@@ -259,7 +259,6 @@ extra =
   ${slapos.test.plantuml-setup:setup}
   ${slapos.test.re6stnet-setup:setup}
   ${slapos.test.seleniumserver-setup:setup}
-  ${slapos.test.helloworld-setup:setup}
   ${slapos.test.jstestnode-setup:setup}
   ${slapos.test.jupyter-setup:setup}
   ${slapos.test.nextcloud-setup:setup}
@@ -317,3 +316,4 @@ funcsigs = 1.0.2
 mysqlclient = 1.3.12
 pexpect = 4.8.0
 ptyprocess = 0.6.0
+typing = 3.7.4.3

software/theia/buildout.hash.cfg

@@ -19,7 +19,7 @@ md5sum = 397fcb3279029af3055b23525d147445
 
 [yarn.lock]
 filename = yarn.lock
-md5sum = d058e73c3d90ac3da44734c2d47eac95
+md5sum = c7aa84922a1b80fd8a4c3d96f6ac7e25
 
 [python-language-server-requirements.txt]
 filename = python-language-server-requirements.txt

stack/erp5/buildout.cfg

@@ -455,7 +455,7 @@ eggs = ${neoppod:eggs}
 # Zope acquisition patch
   Acquisition
 # for runzeo
-  ZEO
+  ${ZEO:egg}
 
 # Other Zope 2 packages
   Products.PluggableAuthService

stack/erp5/buildout.hash.cfg

@@ -78,7 +78,7 @@ md5sum = 68b329da9893e34099c7d8ad5cb9c940
 
 [template-erp5]
 filename = instance-erp5.cfg.in
-md5sum = 82dc695e212be124d60ceb1143e56b0d
+md5sum = 0920a53b10d3811a5f49930adffb62d8
 
 [template-zeo]
 filename = instance-zeo.cfg.in
@@ -90,7 +90,7 @@ md5sum = 2f3ddd328ac1c375e483ecb2ef5ffb57
 
 [template-balancer]
 filename = instance-balancer.cfg.in
-md5sum = bb9a953ce22f7d5188385f0171b6198e
+md5sum = ecf119142e6b5cd85a2ba397552d2142
 
 [template-haproxy-cfg]
 filename = haproxy.cfg.in

stack/erp5/instance-balancer.cfg.in

@@ -18,19 +18,52 @@ per partition. No more (undefined result), no less (IndexError).
 recipe = slapos.recipe.template:jinja2
 mode = 644
 
+[balancer-csr-request-config]
+< = jinja2-template-base
+template = inline:
+    [req]
+    prompt = no
+    req_extensions = req_ext
+    distinguished_name = dn
+    [ dn ]
+    CN = example.com
+    [ req_ext ]
+    subjectAltName = @alt_names
+    [ alt_names ]
+    IP.1 =  {{ ipv4 }}
+    {% if ipv6_set -%}
+    IP.2 =  {{ ipv6 }}
+    {% endif %}
+rendered = ${buildout:parts-directory}/${:_buildout_section_name_}/${:_buildout_section_name_}.txt
+
+[balancer-csr-request]
+recipe = plone.recipe.command
+command = {{ parameter_dict["openssl"] }}/bin/openssl req \
+  -newkey rsa:2048 \
+  -batch \
+  -new \
+  -nodes \
+  -keyout '${apache-conf-ssl:key}' \
+  -config '${balancer-csr-request-config:rendered}' \
+  -out '${:csr}'
+stop-on-error = true
+csr = ${directory:etc}/${:_buildout_section_name_}.csr.pem
+
+
 {{ caucase.updater(
      prefix='caucase-updater',
      buildout_bin_directory=parameter_dict['bin-directory'],
      updater_path='${directory:services-on-watch}/caucase-updater',
      url=ssl_parameter_dict['caucase-url'],
      data_dir='${directory:srv}/caucase-updater',
-     crt_path='${apache-conf-ssl:caucase-cert}',
+     crt_path='${apache-conf-ssl:cert}',
      ca_path='${directory:srv}/caucase-updater/ca.crt',
      crl_path='${directory:srv}/caucase-updater/crl.pem',
-     key_path='${apache-conf-ssl:caucase-key}',
+     key_path='${apache-conf-ssl:key}',
      on_renew='${apache-graceful:output}',
      max_sleep=ssl_parameter_dict.get('max-crl-update-delay', 1.0),
      template_csr_pem=ssl_parameter_dict.get('csr'),
+     template_csr=None if ssl_parameter_dict.get('csr') else '${balancer-csr-request:csr}',
      openssl=parameter_dict['openssl'] ~ '/bin/openssl',
 )}}
 {% do section('caucase-updater') -%}
@@ -176,9 +209,6 @@ hash-files = ${haproxy-cfg:rendered}
 [apache-conf-ssl]
 cert = ${directory:apache-conf}/apache.crt
 key = ${directory:apache-conf}/apache.pem
-# XXX caucase certificate is not supported by caddy for now
-caucase-cert = ${directory:apache-conf}/apache-caucase.crt
-caucase-key = ${directory:apache-conf}/apache-caucase.pem
 {% if frontend_caucase_url_list -%}
 depends = ${caucase-updater-housekeeper-run:recipe}
 ca-cert-dir = ${directory:apache-ca-cert-dir}
@@ -201,19 +231,6 @@ context = key content {{content_section_name}}:content
 mode = {{ mode }}
 {%- endmacro %}
 
-[apache-ssl]
-{% if ssl_parameter_dict.get('key') -%}
-key = ${apache-ssl-key:rendered}
-cert = ${apache-ssl-cert:rendered}
-{{ simplefile('apache-ssl-key', '${apache-conf-ssl:key}', ssl_parameter_dict['key']) }}
-{{ simplefile('apache-ssl-cert', '${apache-conf-ssl:cert}', ssl_parameter_dict['cert']) }}
-{% else %}
-recipe = plone.recipe.command
-command = "{{ parameter_dict['openssl'] }}/bin/openssl" req -newkey rsa -batch -new -x509 -days 3650 -nodes -keyout "${:key}" -out "${:cert}"
-key = ${apache-conf-ssl:key}
-cert = ${apache-conf-ssl:cert}
-{%- endif %}
-
 [apache-conf-parameter-dict]
 backend-list = {{ dumps(apache_dict.values()) }}
 zope-virtualhost-monster-backend-dict = {{ dumps(zope_virtualhost_monster_backend_dict) }}
@@ -225,8 +242,8 @@ access-log = ${directory:log}/apache-access.log
 # Apache 2.4's default value (60 seconds) can be a bit too short
 timeout = 300
 # Basic SSL server configuration
-cert = ${apache-ssl:cert}
-key = ${apache-ssl:key}
+cert = ${apache-conf-ssl:cert}
+key = ${apache-conf-ssl:key}
 cipher =
 ssl-session-cache = ${directory:log}/apache-ssl-session-cache
 {% if frontend_caucase_url_list -%}

stack/erp5/instance-erp5.cfg.in

@@ -176,11 +176,16 @@ return =
 {%- if test_runner_enabled %}
   test-runner-address-list
 {% endif %}
-{% set bt5_default_list = 'erp5_full_text_mroonga_catalog erp5_configurator_standard erp5_configurator_maxma_demo erp5_configurator_run_my_doc' -%}
+{% set bt5_default_list = [
+  'erp5_full_text_mroonga_catalog',
+  'erp5_configurator_standard',
+  'erp5_configurator_maxma_demo',
+  'erp5_configurator_run_my_doc',
+] -%}
 {% if has_jupyter -%}
-{%   set bt5_default_list = bt5_default_list + ' erp5_data_notebook' -%}
+{%   do bt5_default_list.append('erp5_data_notebook') -%}
 {% endif -%}
-config-bt5 = {{ dumps(slapparameter_dict.get('bt5', bt5_default_list)) }}
+config-bt5 = {{ dumps(slapparameter_dict.get('bt5', ' '.join(bt5_default_list))) }}
 config-bt5-repository-url = {{ dumps(slapparameter_dict.get('bt5-repository-url', local_bt5_repository)) }}
 config-cloudooo-url = {{ dumps(slapparameter_dict.get('cloudooo-url', default_cloudooo_url)) }}
 config-caucase-url = {{ dumps(caucase_url) }}

stack/nxdtest.cfg

+# Stack for creating testnode instances to be run via nxdtest on Nexedi testing
+# infrastructure.
+#
+# Usage:
+#
+#    ---- 8< ---- (<software>/test.cfg)
+#    [buildout]
+#    extends = .../stack/nxdtest.cfg
+#
+#    parts =
+#       ...
+#
+#    #  for instance
+#       slapos-cookbook
+#       instance.cfg
+#
+#    ...
+#
+#    [instance.cfg]
+#    <= jinja2-template
+#    template = inline:
+#      [buildout]
+#      extends = ${nxdtest-instance.cfg:rendered}
+#
+#      [runTestSuite]
+#      env.sh  = ...
+#      workdir = ...
+#
+# Created instance will have bin/runTestSuite that sources env.sh and runs
+# nxdtest in workdir.
+
+[buildout]
+extends =
+    slapos.cfg
+    ../component/git/buildout.cfg
+    nxdtest/buildout.hash.cfg
+
+[jinja2-template]
+recipe   = slapos.recipe.template:jinja2
+template = ${:_profile_base_location_}/${:filename}
+rendered = ${buildout:directory}/${:_buildout_section_name_}
+mode     = 0644
+context  =
+  section buildout  buildout
+
+[nxdtest-instance.cfg]
+<= jinja2-template
+template = ${:_profile_base_location_}/nxdtest/${:filename}
+# NOTE += does not work
+context =
+  section buildout  buildout
+  section nxdtest   nxdtest
+
+
+[nxdtest]
+recipe  = zc.recipe.egg:scripts
+eggs    = ${nxdtest-egg:egg}
+scripts = nxdtest
+# convenience for nxdtest users
+exe     = ${buildout:bin-directory}/nxdtest
+
+[nxdtest-egg]
+recipe  = zc.recipe.egg:develop
+setup   = ${nxdtest-repository:location}
+egg     = nxdtest
+
+[nxdtest-repository]
+recipe  = slapos.recipe.build:gitclone
+repository = https://lab.nexedi.com/nexedi/nxdtest.git
+revision = bd91f6f1579a
+location = ${buildout:parts-directory}/nxdtest
+git-executable = ${git:location}/bin/git
+
+
+[versions]
+slapos.recipe.template = 4.4

stack/nxdtest/buildout.hash.cfg

+# THIS IS NOT A BUILDOUT FILE ... -> use update-hash to update it.
+
+[nxdtest-instance.cfg]
+filename = instance.cfg.in
+md5sum = ea54ba85655f52ce491c84450400e5af

software/neotest/instance.cfg.in → stack/nxdtest/instance.cfg.in

-# NEO test instance: run neotest under Nexedi testing infrastructure
+# instance to test a software via nxdtest under Nexedi testing infrastructure.
 [buildout]
 parts = runTestSuite
 
@@ -17,12 +17,17 @@ bin     = {{ buildout['bin-directory'] }}
 recipe  = slapos.cookbook:mkdirectory
 home    = ${buildout:directory}
 bin     = ${:home}/bin
-neotest = ${:home}/neotest
 
-# script to run the testsuite from inside <instance>/neotest/
+# script to run the testsuite via nxdtest
 # located @ <instance>/bin/runTestSuite so testnode can see this as run tests entrypoint.
+#
+# instance that uses stack/nxdtest.cfg must provide
+#
+#   [runTestSuite]
+#   env.sh  = ...
+#   workdir = ...
 [runTestSuite]
 recipe  = slapos.cookbook:wrapper
 wrapper-path = ${directory:bin}/${:_buildout_section_name_}
-command-line = /bin/bash -c 'cd ${directory:neotest} && ${software:bin}/neotest-runTestSuite "$@"' runTestSuite
+command-line = /bin/bash -c '. ${:env.sh} && cd ${:workdir} && {{ nxdtest['exe'] }} "$@"' nxdtest
 # "$@" is implicitly added to argv

stack/slapos.cfg

@@ -143,10 +143,13 @@ zc.recipe.egg = 2.0.3+slapos003
 hexagonit.recipe.download = 1.7.post4
 
 Jinja2 = 2.9.5
+Importing = 1.10
 MarkupSafe = 1.0
 PyYAML = 3.13
 Werkzeug = 0.12
 asn1crypto = 1.3.0
+atomicwrites = 1.4.0
+backports.functools-lru-cache = 1.6.1
 backports.lzma = 0.0.14
 cffi = 1.14.0
 click = 6.7
@@ -154,23 +157,38 @@ cliff = 2.4.0
 cmd2 = 0.7.0
 collective.recipe.shelloutput = 0.1
 collective.recipe.template = 2.0
+configparser = 4.0.2
+contextlib2 = 0.6.0.post1
 cryptography = 2.9.2
+dateparser = 0.7.6
 decorator = 4.3.0
+funcsigs = 1.0.2
+gevent = 20.9.0
+greenlet = 0.4.17
 idna = 2.9
+importlib-metadata = 1.7.0
 inotify-simple = 1.1.1
 itsdangerous = 0.24
 lock-file = 2.0
 lxml = 4.4.3
 meld3 = 1.0.2
+more-itertools = 5.0.0
 netaddr = 0.7.19
+pathlib2 = 2.3.5
 pbr = 2.0.0
 plone.recipe.command = 1.1
 prettytable = 0.7.2
 psutil = 5.6.3
+pluggy = 0.13.1
+py = 1.9.0
 pyOpenSSL = 19.1.0
 pyparsing = 2.2.0
 pytz = 2016.10
+regex = 2020.9.27
 requests = 2.24.0
+scandir = 1.10.0
+setuptools-dso = 1.7
+rubygemsrecipe  = 0.3.0
 six = 1.12.0
 slapos.cookbook = 1.0.167
 slapos.core = 1.6.2
@@ -184,7 +202,13 @@ slapos.toolbox = 0.112
 stevedore = 1.21.0
 subprocess32 = 3.5.3
 unicodecsv = 0.14.1
+wcwidth = 0.2.5
+wheel = 0.35.1
 xml-marshaller = 1.0.2
+zdaemon = 4.2.0
+zipp = 1.2.0
+zodburi = 2.4.0
+zope.event = 3.5.2
 paramiko = 2.1.3
 CacheControl = 0.12.5
 msgpack = 0.6.2