Merge branch 'master' into master+ZODB4-wc2

* origin/master: (22 commits)
  component/perl-Image-ExifTool: version up 12.30
  fixup! stack/erp5: use %Ta for timing of haproxy requests
  stack/erp5: use %Ta for timing of haproxy requests
  Revert "kvm: Bang immediately in case of vnc_promise problem"
  kvm: Bang immediately in case of vnc_promise problem
  kvm: Switch mouse to tablet mode
  software/slapos-sr-testing: update doc
  software/slapos-sr-testing: run ERP5 SR test with a python3 slapos.core
  component/depot_tools: check out a fixed revision
  software/erp5/test: add python3 support
  software/galene: use python3
  stack/caucase: Version up caucase 0.9.12
  galene: version up 0.4
  kvm: Increase default VM parameters to match nowadays requirement
  hugo: create tempfiles in a dedicated temporary directory
  component/java-jdk: add missing rpaths
  component/alsa: enable slapos.recipe.cmmi's shared
  kvm: Upgrade default ISO for vm to debian bullseye
  componant/slapos/obs.cfg: update OBS build
  caddy-frontend: Pick up recent kedifa from the distribution
  ...

component/alsa/buildout.cfg

@@ -5,6 +5,7 @@ parts =
 [alsa]
 # Contains libasound
 recipe = slapos.recipe.cmmi
+shared = true
 url = ftp://ftp.alsa-project.org/pub/lib/alsa-lib-1.1.3.tar.bz2
 md5sum = eefe5992567ba00d6110a540657aaf5c
 configure-options =

component/depot_tools/buildout.cfg

 [buildout]
+extends =
+  ../git/buildout.cfg
 parts =
   depot_tools
 
 [depot_tools]
 recipe = slapos.recipe.build:gitclone
 repository = https://chromium.googlesource.com/chromium/tools/depot_tools.git
-branch = master
+revsion = e023d4482012d89690f6a483e877eceb47c4501e
 git-executable = ${git:location}/bin/git

component/firewalld/buildout.cfg

@@ -73,7 +73,7 @@ environment =
   PATH=${pkgconfig:location}/bin:${gettext:location}/bin:${glib:location}/bin:${xz-utils:location}/bin:${flex:location}/bin:${bison:location}/bin:%(PATH)s
   PKG_CONFIG_PATH=${glib:location}/lib/pkgconfig:${pcre:location}/lib/pkgconfig
   CPPFLAGS=-I${glib:location}/include/glib-2.0 -I${glib:location}/lib/glib-2.0/include
-  LDFLAGS=-L${glib:location}/lib -Wl,-rpath=${glib:location}/lib -L${libffi:location}/lib -Wl,-rpath=${libffi:location}/lib -lffi
+  LDFLAGS=-L${glib:location}/lib -Wl,-rpath=${glib:location}/lib -L${libffi:location}/lib -Wl,-rpath=${libffi:location}/lib -lffi -L${zlib:location}/lib/ -Wl,-rpath=${zlib:location}/lib/
   GLIB_CFLAGS=-I${glib:location}/include/glib-2.0 -I${glib:location}/lib/glib-2.0/include
   GLIB_LIBS=-L${glib:location}/lib -lglib-2.0 -lintl -lgobject-2.0
   FFI_CFLAGS=-I${libffi:location}/include

component/java-jdk/buildout.cfg

 [buildout]
-
 extends = 
   ../patchelf/buildout.cfg
   ../zlib/buildout.cfg
-
+  ../alsa/buildout.cfg
+  ../libpng/buildout.cfg
+  ../freetype/buildout.cfg
+  ../xorg/buildout.cfg
 parts = 
   java-jdk
 
@@ -24,8 +26,8 @@ configure-command = :
 make-binary = :
 post-install =
   mv * %(location)s
-  for file in %(location)s/bin/* ; do
+  for file in %(location)s/bin/* %(location)s/lib/*.so ; do
     echo appending rpath to $file
     ${patchelf:location}/bin/patchelf --set-rpath %(rpath)s $file
   done
-rpath = ${zlib:location}/lib:@@LOCATION@@/lib
+rpath = ${zlib:location}/lib:${alsa:location}/lib:${freetype:location}/lib:${libpng:location}/lib:${libXrender:location}/lib:${libXtst:location}/lib:${libX11:location}/lib:${libXau:location}/lib:${libXext:location}/lib:${libXdmcp:location}/lib:${libXi:location}/lib:${libxcb:location}/lib:@@LOCATION@@/lib:@@LOCATION@@/lib/server

component/perl-Image-ExifTool/buildout.cfg

@@ -7,5 +7,5 @@ parts =
 [perl-Image-ExifTool]
 <= perl-CPAN-package
 module = Image/Image-ExifTool
-version = 12.00
-md5sum = 8c3f99a9a8d110ba340ee3f7063c940e
+version = 12.30
+md5sum = 1f5d66d62418c8b29eb0c0b7fd272b28

component/slapos/obs.cfg

@@ -76,8 +76,6 @@ post-install =
 [dbus-glib]
 location = ${buildout:parts-directory}/${:_buildout_section_name_}
 environment +=
-  CPPFLAGS=-I${libexpat:location}/include -I${dbus:location}/include/dbus-1.0 -I${dbus:location}/lib/dbus-1.0/include
-  LDFLAGS=-L${libexpat:location}/lib -L${gettext:location}/lib -Wl,-rpath=${zlib:location}/lib -L${dbus:location}/lib -Wl,-rpath=${dbus:location}/lib
   LD_LIBRARY_PATH=${dbus:location}/lib
   DBUS_CFLAGS=-I${dbus:location}/include/dbus-1.0 -I${dbus:location}/lib/dbus-1.0/include
 
@@ -86,9 +84,7 @@ environment +=
 pre-configure =
   sed -i 's#/opt/slapos/parts/dbus/lib/libdbus-1.la#${dbus:location}/lib/libdbus-1.la#' ${dbus-glib:location}/lib/libdbus-glib-1.la
 environment +=
-  PATH=${pkgconfig:location}/bin:${python:location}/bin:%(PATH)s
   LD_LIBRARY_PATH=${dbus:location}/lib
-  LDFLAGS=-L${glib:location}/lib -Wl,-rpath=${glib:location}/lib -L${dbus:location}/lib -Wl,-rpath=${dbus:location}/lib
   PYTHON=${python:location}/bin/python${python:version}
   PYTHON_INCLUDES=-I${python:location}/include/python${python:version}
   PYTHON_LIBS=-L${python:location}/lib -lpython${python:version} -lpthread -ldl -lutil -lm
@@ -114,34 +110,21 @@ make-options =
 
 [gobject-introspection]
 pre-configure =
-  ln -s ${python:location}/bin/python${python:version} ${python:location}/bin/python2.
-  sed -i 's#!/opt/slapos/parts/python2.7/bin/python2.7#!${python2.7:location}/bin/python2.7#' ${python2.7:location}/bin/python-config
-  libtoolize -c -f
-  aclocal -I${pkgconfig:location}/share/aclocal -I${gettext:location}/share/aclocal -I${libtool:location}/share/aclocal -I${glib:location}/share/aclocal
-  ./autogen.sh
+  sed -i 's#!/opt/slapos/parts/python${python:version}/bin/python${python:version}#!${python:location}/bin/python${python:version}#' ${python:location}/bin/python${python:version}-config
 configure-options +=
   --enable-shared
 environment +=
-  PATH=${autoconf:location}/bin:${automake:location}/bin:${pkgconfig:location}/bin:${libtool:location}/bin:${intltool:location}/bin:${gettext:location}/bin:${glib:location}/bin:${flex:location}/bin:${bison-go:location}/bin:%(PATH)s
-  GIR_DIR=${buildout:parts-directory}/${:_buildout_section_name_}/share/gir-1.0
-  CPPFLAGS=-I${glib:location}/include/glib-2.0 -I${glib:location}/lib/glib-2.0/include -I${python:location}/include/python${python:version}
-  LDFLAGS=-L${glib:location}/lib -Wl,-rpath=${glib:location}/lib -L${libffi:location}/lib -Wl,-rpath=${libffi:location}/lib -lffi -L${python:location}/lib
-  ACLOCAL_PATH=${pkgconfig:location}/share/aclocal:${gettext:location}/share/aclocal:${libtool:location}/share/aclocal:${glib:location}/share/aclocal:${intltool:location}/share/aclocal
-  M4=${m4:location}/bin/m4
   PERL5LIB=${perl:location}/lib/5.28.1/
 
 post-install = 
-  sed -i 's#!${python:location}/bin/python${python:version}#!/opt/slapos/parts/python${python:version}/bin/python${python:version}#' ${python:location}/bin/python-config
+  sed -i 's#!${python:location}/bin/python${python:version}#!/opt/slapos/parts/python${python:version}/bin/python${python:version}#' ${python:location}/bin/python${python:version}-config
   rm -rf ${bison-go:location}
 
 [pygobject3]
 pre-configure +=
-  sed -i 's#!/opt/slapos/parts/python${python:version}/bin/python${python:version}#!${python:location}/bin/python${python:version}#' ${python:location}/bin/python-config
-environment +=
-  CPPFLAGS=-I${glib:location}/include/glib-2.0 -I${glib:location}/lib/glib-2.0/include -I${gettext:location}/include -I${libffi:location}/include -I${python:location}/include/python${python:version}
-  LDFLAGS=-L${glib:location}/lib -Wl,-rpath=${glib:location}/lib -L${gettext:location}/lib -Wl,-rpath=${gettext:location}/lib -L${python:location}/lib
+  sed -i 's#!/opt/slapos/parts/python${python:version}/bin/python${python:version}#!${python:location}/bin/python${python:version}#' ${python:location}/bin/python${python:version}-config
 post-install =
-  sed -i 's#!${python:location}/bin/python${python:version}#!/opt/slapos/parts/python${python:version}/bin/python${python:version}#' ${python:location}/bin/python-config
+  sed -i 's#!${python:location}/bin/python${python:version}#!/opt/slapos/parts/python${python:version}/bin/python${python:version}#' ${python:location}/bin/python${python:version}-config
 
 [ncurses]
 configure-options =
@@ -189,3 +172,9 @@ configure-options +=
   --prefix=${buildout:rootdir}/parts/${:_buildout_section_name_}
 environment +=
   DESTDIR=${buildout:destdir}
+
+[swig]
+configure-options +=
+  --prefix=${buildout:rootdir}/parts/${:_buildout_section_name_}
+environment +=
+  DESTDIR=${buildout:destdir}

slapos/recipe/promise_plugin.py

@@ -91,7 +91,7 @@ class Recipe(GenericBaseRecipe):
     if klass == 'RunPromise':
       klass = None
     elif klass and not isPythonName(klass):
-      raise UserError("%r is not valid class name" % klass)
+      raise UserError("%r is not a valid class name" % klass)
 
     if bool(module) == bool(filepath):
       raise UserError("Either 'module' or 'file' is required but not both")

slapos/test/recipe/test_plugin.py

 import os, shutil, tempfile, unittest
 from slapos.recipe import promise_plugin
 from slapos.test.utils import makeRecipe
+from zc.buildout import UserError
 from pprint import pformat
 import stat, json
 import six
@@ -10,7 +11,7 @@ class TestPromisePlugin(unittest.TestCase):
   def setUp(self):
     self.tmp = tempfile.mkdtemp()
     self.output = os.path.join(self.tmp, 'output.py')
-    self.options = options = {
+    self.options = {
       'output': self.output,
       'eggs': 'slapos.cookbook'
     }
@@ -18,9 +19,16 @@ class TestPromisePlugin(unittest.TestCase):
   def tearDown(self):
     shutil.rmtree(self.tmp)
 
-  def test_parameters(self):
-    self.options['mode'] = '0644'
-    self.options['import'] = 'slapos.promise.plugin.check_site_available'
+  def makeRecipe(self):
+    return makeRecipe(
+            promise_plugin.Recipe,
+            options=self.options,
+            name='plugin')
+
+  def installRecipe(self):
+    self.makeRecipe().install()
+
+  def setConfig(self):
     self.options['config-param1'] = "YY^@12"
     self.options['config-param2'] = "23'91'"
     self.options['config-param3'] = None
@@ -28,88 +36,102 @@ class TestPromisePlugin(unittest.TestCase):
 in multi line
 123444
 """
-    recipe = makeRecipe(
-            promise_plugin.Recipe,
-            options=self.options,
-            name='plugin')
-    recipe.install()
 
+  def assertOutput(self, *expect):
     self.assertTrue(os.path.exists(self.output))
     with open(self.output, 'r') as f:
       content = f.read()
-    self.assertIn("from slapos.promise.plugin.check_site_available import RunPromise", content)
-    self.assertEqual(stat.S_IMODE(os.stat(self.output).st_mode), int('644', 8))
-
-    expected_dict = dict(
-      param1=self.options['config-param1'],
-      param2=self.options['config-param2'],
-      param3=self.options['config-param3'],
-      param4=self.options['config-param4'],
-    )
-    self.assertIn('extra_config_dict = %s' % pformat(expected_dict, indent=2), content)
-
-  def test_no_module_set(self):
-    recipe = makeRecipe(
-            promise_plugin.Recipe,
-            options=self.options,
-            name='plugin')
-    with self.assertRaises(KeyError):
-      recipe.install()
-
-  def test_default(self):
-    self.options['import'] = 'slapos.promise.plugin.check_site_available'
-    recipe = makeRecipe(
-            promise_plugin.Recipe,
-            options=self.options,
-            name='plugin')
-    recipe.install()
-
-    self.assertTrue(os.path.exists(self.output))
+    for s in expect:
+      self.assertIn(s, content)
+
+  def assertConfig(self):
+    items = self.options.items()
+    expect = {k[7:] : v  for k, v in items if k.startswith('config-')}
+    self.assertOutput("extra_config_dict = %s" % pformat(expect, indent=2))
+
+  def assertEmptyConfig(self):
+    self.assertOutput("extra_config_dict = %s" % ('{}' if six.PY3 else '{ }'))
+
+  def test_module(self):
+    self.options['module'] = 'slapos.promise.plugin.check_site_available'
+    self.installRecipe()
+    self.assertOutput("from %s import RunPromise" % self.options['module'])
+    self.assertEmptyConfig()
+
+  def test_file(self):
+    self.options['file'] = __file__
+    self.installRecipe()
+    self.assertOutput("exec(_(%r))" % self.options['file'])
+    self.assertEmptyConfig()
+
+  def test_module_and_parameters(self):
+    self.options['module'] = 'slapos.promise.plugin.check_site_available'
+    self.setConfig()
+    self.installRecipe()
+    self.assertOutput("from %s import RunPromise" % self.options['module'])
+    self.assertConfig()
+
+  def test_file_and_parameters(self):
+    self.options['file'] = __file__
+    self.setConfig()
+    self.installRecipe()
+    self.assertOutput("exec(_(%r))" % self.options['file'])
+    self.assertConfig()
+
+  def test_mode(self):
+    self.options['mode'] = '0644'
+    self.options['module'] = 'slapos.promise.plugin.check_site_available'
+    self.installRecipe()
     self.assertEqual(stat.S_IMODE(os.stat(self.output).st_mode), int('644', 8))
-    with open(self.output) as f:
-      content = f.read()
-    self.assertIn("from slapos.promise.plugin.check_site_available import RunPromise", content)
-    self.assertIn('extra_config_dict = %s' % ('{}' if six.PY3 else '{ }'), content)
 
+  def test_module_and_class(self):
+    self.options['module'] = m = 'slapos.promise.plugin.check_site_available'
+    self.options['class']  = 'MyPromise'
+    self.installRecipe()
+    self.assertOutput("from %s import MyPromise as RunPromise" % m)
+
+  def test_file_and_class(self):
+    self.options['file'] = __file__
+    self.options['class'] = 'MyPromise'
+    self.installRecipe()
+    self.assertOutput("exec(_(%r))\n\nRunPromise = MyPromise" % __file__)
+
+  def test_no_module_or_file(self):
+    with self.assertRaises(UserError) as p:
+      self.makeRecipe()
+    msg = str(p.exception)
+    self.assertEqual(msg, "Either 'module' or 'file' is required but not both")
+
+  def test_module_and_file(self):
+    self.options['module'] = 'slapos.promise.plugin.check_site_available'
+    self.options['file'] = __file__
+    self.test_no_module_or_file()
 
   def test_bad_parameters(self):
-    self.options['import'] = 'slapos.promise.plugin.check_site_available'
-    self.options['config-param1; print "toto"'] = """#xxxx"\nimport os; os.stat(f)"""
-    self.options['config-param2\n@domething'] = '"#$$*PPP\n\n p = 2*5; print "result is %s" % p'
-    recipe = makeRecipe(
-            promise_plugin.Recipe,
-            options=self.options,
-            name='plugin')
-    recipe.install()
-
-    self.assertTrue(os.path.exists(self.output))
-    with open(self.output) as f:
-      content = f.read()
-
-    expected_param1 = r"""'param1; print "toto"': '#xxxx"\nimport os; os.stat(f)',"""
-    expected_param2 = r"""'param2\n@domething': '"#$$*PPP\n\n p = 2*5; print "result is %s" % p'"""
-    self.assertIn(expected_param1, content)
-    self.assertIn(expected_param2, content)
-
-  def test_bad_module_path(self):
-    self.options['import'] = 'slapos.promise.plugin.check_site_available; print "toto"'
-    recipe = makeRecipe(
-            promise_plugin.Recipe,
-            options=self.options,
-            name='plugin')
-    with self.assertRaises(ValueError) as p:
-      recipe.install()
-
-    self.assertEqual(str(p.exception), "Import path %r is not a valid" % self.options['import'])
-
-  def test_bad_content(self):
-    self.options['content'] = 'from slapos.plugin.check_site_available import toto; print "toto"'
-    recipe = makeRecipe(
-            promise_plugin.Recipe,
-            options=self.options,
-            name='plugin')
-    with self.assertRaises(ValueError) as p:
-      recipe.install()
-
-    self.assertEqual(str(p.exception), "Promise content %r is not valid" % self.options['content'])
+    self.options['module'] = 'slapos.promise.plugin.check_site_available'
+    self.options.update((
+      ('config-param1; print "toto"', '#xxxx"\nimport os; os.stat(f)'),
+      ('config-param2\n@domething', '"#$$*PPP\np = 2*5; print "result=%s" % p')
+    ))
+    self.installRecipe()
+    self.assertOutput(
+      r"""'param1; print "toto"': '#xxxx"\nimport os; os.stat(f)',""",
+      r"""'param2\n@domething': '"#$$*PPP\np = 2*5; print "result=%s" % p'"""
+    )
 
+  def test_bad_module(self):
+    self.options['module'] = 'slapos.promise.plugin.check_site_available; print "toto"'
+    with self.assertRaises(UserError) as p:
+      self.makeRecipe()
+    self.assertEqual(str(p.exception), "%r is not a valid module name" % self.options['module'])
+
+  def test_bad_file(self):
+    self.options['file'] = 'print "toto"'
+    self.installRecipe()
+    self.assertOutput(r"""exec(_('print "toto"'))""")
+
+  def test_bad_class(self):
+    self.options['class'] = 'MyPromise; print "toto"'
+    with self.assertRaises(UserError) as p:
+      self.makeRecipe()
+    self.assertEqual(str(p.exception), "%r is not a valid class name" % self.options['class'])

software/caddy-frontend/software.cfg

@@ -20,19 +20,8 @@ parts +=
   caucase-eggs
   template
   caddyprofiledeps
-  kedifa-develop
   kedifa
 
-[kedifa-repository]
-recipe = slapos.recipe.build:gitclone
-repository = https://lab.nexedi.com/nexedi/kedifa.git
-git-executable = ${git:location}/bin/git
-revision = 3fccc2ec945c59b644a12fa40225795abd61e0b0
-
-[kedifa-develop]
-recipe = zc.recipe.egg:develop
-setup = ${kedifa-repository:location}
-
 [kedifa]
 recipe = zc.recipe.egg
 eggs =
@@ -222,6 +211,7 @@ mode = 0644
 <=download-template
 
 [versions]
+kedifa = 0.0.6
 # Modern KeDiFa requires zc.lockfile
 zc.lockfile = 1.4
 

software/erp5/test/setup.py

@@ -24,6 +24,7 @@
 # Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA  02111-1307, USA.
 #
 ##############################################################################
+from __future__ import absolute_import
 from setuptools import setup, find_packages
 
 version = '0.0.1.dev0'
@@ -52,7 +53,6 @@ setup(name=name,
         'cryptography',
         'pexpect',
         'pyOpenSSL',
-        'typing; python_version<"3"',
       ],
       test_suite='test',
     )

software/erp5/test/test/__init__.py

@@ -25,6 +25,7 @@
 #
 ##############################################################################
 
+from __future__ import absolute_import
 import json
 import os
 

software/erp5/test/test/test_balancer.py

+from __future__ import absolute_import
 import glob
 import hashlib
 import json
@@ -8,10 +9,8 @@ import shutil
 import subprocess
 import tempfile
 import time
-import urllib
-import urlparse
-from BaseHTTPServer import BaseHTTPRequestHandler
-from typing import Dict
+import six.moves.urllib.request, six.moves.urllib.parse, six.moves.urllib.error
+from six.moves.BaseHTTPServer import BaseHTTPRequestHandler
 
 import mock
 import OpenSSL.SSL
@@ -28,6 +27,7 @@ from slapos.testing.utils import (CrontabMixin, ManagedHTTPServer,
                                   findFreeTCPPort)
 
 from . import ERP5InstanceTestCase, setUpModule
+from six.moves import range
 
 setUpModule  # pyflakes
 
@@ -44,10 +44,10 @@ class EchoHTTPServer(ManagedHTTPServer):
       response = json.dumps(
           {
               'Path': self.path,
-              'Incoming Headers': self.headers.dict
+              'Incoming Headers': dict(self.headers.items()),
           },
           indent=2,
-      )
+      ).encode('utf-8')
       self.end_headers()
       self.wfile.write(response)
 
@@ -67,11 +67,11 @@ class EchoHTTP11Server(ManagedHTTPServer):
       response = json.dumps(
           {
               'Path': self.path,
-              'Incoming Headers': self.headers.dict
+              'Incoming Headers': dict(self.headers.items()),
           },
           indent=2,
-      )
-      self.send_header("Content-Length", len(response))
+      ).encode('utf-8')
+      self.send_header("Content-Length", str(len(response)))
       self.end_headers()
       self.wfile.write(response)
 
@@ -139,7 +139,7 @@ class BalancerTestCase(ERP5InstanceTestCase):
 
   @classmethod
   def _getInstanceParameterDict(cls):
-    # type: () -> Dict
+    # type: () -> dict
     return {
         'tcpv4-port': 8000,
         'computer-memory-percent-threshold': 100,
@@ -174,10 +174,11 @@ class BalancerTestCase(ERP5InstanceTestCase):
 
   @classmethod
   def getInstanceParameterDict(cls):
-    # type: () -> Dict
+    # type: () -> dict
     return {'_': json.dumps(cls._getInstanceParameterDict())}
 
   def setUp(self):
+    # type: () -> None
     self.default_balancer_url = json.loads(
         self.computer_partition.getConnectionParameterDict()['_'])['default']
 
@@ -192,7 +193,7 @@ class SlowHTTPServer(ManagedHTTPServer):
       self.send_header("Content-Type", "text/plain")
       time.sleep(2)
       self.end_headers()
-      self.wfile.write("OK\n")
+      self.wfile.write(b"OK\n")
 
     log_message = logging.getLogger(__name__ + '.SlowHandler').info
 
@@ -203,7 +204,7 @@ class TestLog(BalancerTestCase, CrontabMixin):
   __partition_reference__ = 'l'
   @classmethod
   def _getInstanceParameterDict(cls):
-    # type: () -> Dict
+    # type: () -> dict
     parameter_dict = super(TestLog, cls)._getInstanceParameterDict()
     # use a slow server instead
     parameter_dict['dummy_http_server'] = [[cls.getManagedResource("slow_web_server", SlowHTTPServer).netloc, 1, False]]
@@ -212,7 +213,7 @@ class TestLog(BalancerTestCase, CrontabMixin):
   def test_access_log_format(self):
     # type: () -> None
     requests.get(
-        urlparse.urljoin(self.default_balancer_url, '/url_path'),
+        six.moves.urllib.parse.urljoin(self.default_balancer_url, '/url_path'),
         verify=False,
     )
     time.sleep(.5) # wait a bit more until access is logged
@@ -285,6 +286,7 @@ class TestLog(BalancerTestCase, CrontabMixin):
     self.assertFalse(os.path.exists(rotated_log_file))
 
   def test_error_log(self):
+    # type: () -> None
     # stop backend server
     backend_server = self.getManagedResource("slow_web_server", SlowHTTPServer)
     self.addCleanup(backend_server.open)
@@ -326,7 +328,7 @@ class BalancerCookieHTTPServer(ManagedHTTPServer):
           # The name of this cookie is SERVERID
           assert self.headers['X-Balancer-Current-Cookie'] == 'SERVERID'
         self.end_headers()
-        self.wfile.write(server._name)
+        self.wfile.write(server._name.encode('utf-8'))
       log_message = logging.getLogger(__name__ + '.BalancerCookieHTTPServer').info
 
     return RequestHandler
@@ -338,7 +340,7 @@ class TestBalancer(BalancerTestCase):
   __partition_reference__ = 'b'
   @classmethod
   def _getInstanceParameterDict(cls):
-    # type: () -> Dict
+    # type: () -> dict
     parameter_dict = super(TestBalancer, cls)._getInstanceParameterDict()
 
     # use two backend servers
@@ -349,6 +351,7 @@ class TestBalancer(BalancerTestCase):
     return parameter_dict
 
   def test_balancer_round_robin(self):
+    # type: () -> None
     # requests are by default balanced to both servers
     self.assertEqual(
         {requests.get(self.default_balancer_url, verify=False).text for _ in range(10)},
@@ -356,6 +359,7 @@ class TestBalancer(BalancerTestCase):
     )
 
   def test_balancer_server_down(self):
+    # type: () -> None
     # if one backend is down, it is excluded from balancer
     self.getManagedResource("backend_web_server2", BalancerCookieHTTPServer).close()
     self.addCleanup(self.getManagedResource("backend_web_server2", BalancerCookieHTTPServer).open)
@@ -365,14 +369,16 @@ class TestBalancer(BalancerTestCase):
     )
 
   def test_balancer_set_cookie(self):
+    # type: () -> None
     # if backend provides a "SERVERID" cookie, balancer will overwrite it with the
     # backend selected by balancing algorithm
     self.assertIn(
-        requests.get(urlparse.urljoin(self.default_balancer_url, '/set_cookie'), verify=False).cookies['SERVERID'],
+        requests.get(six.moves.urllib.parse.urljoin(self.default_balancer_url, '/set_cookie'), verify=False).cookies['SERVERID'],
         ('default-0', 'default-1'),
     )
 
   def test_balancer_respects_sticky_cookie(self):
+    # type: () -> None
     # if request is made with the sticky cookie, the client stick on one balancer
     cookies = dict(SERVERID='default-1')
     self.assertEqual(
@@ -388,6 +394,7 @@ class TestBalancer(BalancerTestCase):
         'backend_web_server1')
 
   def test_balancer_stats_socket(self):
+    # type: () -> None
     # real time statistics can be obtained by using the stats socket and there
     # is a wrapper which makes this a bit easier.
     socat_process = subprocess.Popen(
@@ -397,14 +404,14 @@ class TestBalancer(BalancerTestCase):
         stderr=subprocess.STDOUT
     )
     try:
-      output, _ = socat_process.communicate("show stat\n")
+      output, _ = socat_process.communicate(b"show stat\n")
     except:
       socat_process.kill()
       socat_process.wait()
       raise
     self.assertEqual(socat_process.poll(), 0)
     # output is a csv
-    self.assertIn('family_default,FRONTEND,', output)
+    self.assertIn(b'family_default,FRONTEND,', output)
 
 
 class TestTestRunnerEntryPoints(BalancerTestCase):
@@ -413,7 +420,7 @@ class TestTestRunnerEntryPoints(BalancerTestCase):
   __partition_reference__ = 't'
   @classmethod
   def _getInstanceParameterDict(cls):
-    # type: () -> Dict
+    # type: () -> dict
     parameter_dict = super(
         TestTestRunnerEntryPoints,
         cls,
@@ -436,23 +443,24 @@ class TestTestRunnerEntryPoints(BalancerTestCase):
     return parameter_dict
 
   def test_use_proper_backend(self):
+    # type: () -> None
     # requests are directed to proper backend based on URL path
     test_runner_url_list = self.getRootPartitionConnectionParameterDict(
     )['default-test-runner-url-list']
     url_0, url_1, url_2 = test_runner_url_list
     self.assertEqual(
-        urlparse.urlparse(url_0).netloc,
-        urlparse.urlparse(url_1).netloc)
+        six.moves.urllib.parse.urlparse(url_0).netloc,
+        six.moves.urllib.parse.urlparse(url_1).netloc)
     self.assertEqual(
-        urlparse.urlparse(url_0).netloc,
-        urlparse.urlparse(url_2).netloc)
+        six.moves.urllib.parse.urlparse(url_0).netloc,
+        six.moves.urllib.parse.urlparse(url_2).netloc)
 
     path_0 = '/VirtualHostBase/https/{netloc}/VirtualHostRoot/_vh_unit_test_0/something'.format(
-        netloc=urlparse.urlparse(url_0).netloc)
+        netloc=six.moves.urllib.parse.urlparse(url_0).netloc)
     path_1 = '/VirtualHostBase/https/{netloc}/VirtualHostRoot/_vh_unit_test_1/something'.format(
-        netloc=urlparse.urlparse(url_0).netloc)
+        netloc=six.moves.urllib.parse.urlparse(url_0).netloc)
     path_2 = '/VirtualHostBase/https/{netloc}/VirtualHostRoot/_vh_unit_test_2/something'.format(
-        netloc=urlparse.urlparse(url_0).netloc)
+        netloc=six.moves.urllib.parse.urlparse(url_0).netloc)
 
     self.assertEqual(
         {
@@ -489,7 +497,7 @@ class TestHTTP(BalancerTestCase):
   """
   @classmethod
   def _getInstanceParameterDict(cls):
-    # type: () -> Dict
+    # type: () -> dict
     parameter_dict = super(TestHTTP, cls)._getInstanceParameterDict()
     # use a HTTP/1.1 server instead
     parameter_dict['dummy_http_server'] = [[cls.getManagedResource("HTTP/1.1 Server", EchoHTTP11Server).netloc, 1, False]]
@@ -511,7 +519,7 @@ class TestHTTP(BalancerTestCase):
             '%{http_version}',
             self.default_balancer_url,
         ]),
-        '2',
+        b'2',
     )
 
   def test_keep_alive(self):
@@ -530,7 +538,7 @@ class TestHTTP(BalancerTestCase):
       session.get(self.default_balancer_url).raise_for_status()
     new_conn.assert_not_called()
 
-    parsed_url = urlparse.urlparse(self.default_balancer_url)
+    parsed_url = six.moves.urllib.parse.urlparse(self.default_balancer_url)
     # check that we have an open file for the ip connection
     self.assertTrue([
         c for c in psutil.Process(os.getpid()).connections()
@@ -553,12 +561,12 @@ class ContentTypeHTTPServer(ManagedHTTPServer):
       # type: () -> None
       self.send_response(200)
       if self.path == '/':
-        self.send_header("Content-Length", 0)
+        self.send_header("Content-Length", '0')
         return self.end_headers()
       content_type = self.path[1:]
-      body = "OK"
+      body = b"OK"
       self.send_header("Content-Type", content_type)
-      self.send_header("Content-Length", len(body))
+      self.send_header("Content-Length", str(len(body)))
       self.end_headers()
       self.wfile.write(body)
 
@@ -571,7 +579,7 @@ class TestContentEncoding(BalancerTestCase):
   __partition_reference__ = 'ce'
   @classmethod
   def _getInstanceParameterDict(cls):
-    # type: () -> Dict
+    # type: () -> dict
     parameter_dict = super(TestContentEncoding, cls)._getInstanceParameterDict()
     parameter_dict['dummy_http_server'] = [
         [cls.getManagedResource("content_type_server", ContentTypeHTTPServer).netloc, 1, False],
@@ -599,7 +607,7 @@ class TestContentEncoding(BalancerTestCase):
         'application/font-woff2',
         'application/x-font-opentype',
         'application/wasm',):
-      resp = requests.get(urlparse.urljoin(self.default_balancer_url, content_type), verify=False)
+      resp = requests.get(six.moves.urllib.parse.urljoin(self.default_balancer_url, content_type), verify=False)
       self.assertEqual(resp.headers['Content-Type'], content_type)
       self.assertEqual(
           resp.headers.get('Content-Encoding'),
@@ -609,7 +617,7 @@ class TestContentEncoding(BalancerTestCase):
 
   def test_no_gzip_encoding(self):
     # type: () -> None
-    resp = requests.get(urlparse.urljoin(self.default_balancer_url, '/image/png'), verify=False)
+    resp = requests.get(six.moves.urllib.parse.urljoin(self.default_balancer_url, '/image/png'), verify=False)
     self.assertNotIn('Content-Encoding', resp.headers)
     self.assertEqual(resp.text, 'OK')
 
@@ -692,7 +700,7 @@ class CaucaseCertificate(ManagedResource):
       cas_args + [
           '--send-csr', self.csr_file,
       ],
-    ).split()[0]
+    ).split()[0].decode()
     assert csr_id
 
     for _ in range(30):
@@ -708,11 +716,11 @@ class CaucaseCertificate(ManagedResource):
         time.sleep(1)
     else:
       raise RuntimeError('getting service certificate failed.')
-    with open(self.cert_file) as f:
-      assert 'BEGIN CERTIFICATE' in f.read()
+    with open(self.cert_file) as cert_file:
+      assert 'BEGIN CERTIFICATE' in cert_file.read()
 
   def revoke(self, caucase):
-    # type: (str, CaucaseService) -> None
+    # type: (CaucaseService) -> None
     """Revoke the client certificate on this caucase instance.
     """
     subprocess.check_call([
@@ -729,7 +737,7 @@ class TestFrontendXForwardedFor(BalancerTestCase):
 
   @classmethod
   def _getInstanceParameterDict(cls):
-    # type: () -> Dict
+    # type: () -> dict
     frontend_caucase = cls.getManagedResource('frontend_caucase', CaucaseService)
     certificate = cls.getManagedResource('client_certificate', CaucaseCertificate)
     certificate.request(u'shared frontend', frontend_caucase)
@@ -784,10 +792,10 @@ class TestServerTLSProvidedCertificate(BalancerTestCase):
 
   @classmethod
   def _getInstanceParameterDict(cls):
-    # type: () -> Dict
+    # type: () -> dict
     server_caucase = cls.getManagedResource('server_caucase', CaucaseService)
     server_certificate = cls.getManagedResource('server_certificate', CaucaseCertificate)
-    server_certificate.request(cls._ipv4_address.decode(), server_caucase)
+    server_certificate.request(six.ensure_text(cls._ipv4_address), server_caucase)
     parameter_dict = super(TestServerTLSProvidedCertificate, cls)._getInstanceParameterDict()
     with open(server_certificate.cert_file) as f:
       parameter_dict['ssl']['cert'] = f.read()
@@ -806,7 +814,7 @@ class TestClientTLS(BalancerTestCase):
 
   @classmethod
   def _getInstanceParameterDict(cls):
-    # type: () -> Dict
+    # type: () -> dict
     frontend_caucase1 = cls.getManagedResource('frontend_caucase1', CaucaseService)
     certificate1 = cls.getManagedResource('client_certificate1', CaucaseCertificate)
     certificate1.request(u'client_certificate1', frontend_caucase1)
@@ -827,7 +835,6 @@ class TestClientTLS(BalancerTestCase):
 
   def test_refresh_crl(self):
     # type: () -> None
-
     logger = self.logger
 
     class DebugLogFile:
@@ -846,6 +853,7 @@ class TestClientTLS(BalancerTestCase):
       # when client certificate can be authenticated, backend receive the CN of
       # the client certificate in "remote-user" header
       def _make_request():
+        # type: () -> dict
         return requests.get(
             self.default_balancer_url,
             cert=(client_certificate.cert_file, client_certificate.key_file),
@@ -897,6 +905,7 @@ class TestClientTLS(BalancerTestCase):
       with self.assertRaisesRegexp(Exception, 'certificate revoked'):
         _make_request()
 
+
 class TestPathBasedRouting(BalancerTestCase):
   """Check path-based routing rewrites URLs as expected.
   """
@@ -904,7 +913,7 @@ class TestPathBasedRouting(BalancerTestCase):
 
   @classmethod
   def _getInstanceParameterDict(cls):
-    # type: () -> Dict
+    # type: () -> dict
     parameter_dict = super(
       TestPathBasedRouting,
       cls,
@@ -936,14 +945,15 @@ class TestPathBasedRouting(BalancerTestCase):
     published_dict = json.loads(self.computer_partition.getConnectionParameterDict()['_'])
     scheme = 'scheme'
     netloc = 'example.com:8080'
-    prefix = '/VirtualHostBase/' + scheme + '//' + urllib.quote(
+    prefix = '/VirtualHostBase/' + scheme + '//' + six.moves.urllib.parse.quote(
       netloc,
       safe='',
     )
-    # For easier reading of test data, visualy separating the virtual host
+    # For easier reading of test data, visually separating the virtual host
     # base from the virtual host root
     vhr = '/VirtualHostRoot'
     def assertRoutingEqual(family, path, expected_path):
+      # type: (str, str, str) -> None
       # sanity check: unlike the rules, this test is sensitive to outermost
       # slashes, and paths must be absolute-ish for code simplicity.
       assert path.startswith('/')
@@ -959,7 +969,7 @@ class TestPathBasedRouting(BalancerTestCase):
       # test will need to be updated accordingly.
       self.assertEqual(
         requests.get(
-          urlparse.urljoin(published_dict[family], prefix + vhr + path),
+          six.moves.urllib.parse.urljoin(published_dict[family], prefix + vhr + path),
           verify=False,
         ).json()['Path'],
         expected_path,
@@ -978,7 +988,7 @@ class TestPathBasedRouting(BalancerTestCase):
     # Rule precedence: family rules applied before general rules.
     assertRoutingEqual('default', '/next',          prefix + '/erp5/web_site_module/another_next_website' + vhr + '/_vh_next')
     # Fallback on general rules when no family-specific rule matches
-    # Note: the root is special in that there is aways a trailing slash in the
+    # Note: the root is special in that there is always a trailing slash in the
     # produced URL.
     assertRoutingEqual('default', '/',              prefix + '/erp5/web_site_module/123' + vhr + '/')
     # Rule-less family reach general rules.

software/erp5/test/test/test_erp5.py

@@ -25,10 +25,11 @@
 #
 ##############################################################################
 
+from __future__ import absolute_import
 import os
 import json
 import glob
-import urlparse
+import six.moves.urllib.parse
 import socket
 import time
 
@@ -37,6 +38,9 @@ import requests
 
 from . import ERP5InstanceTestCase
 from . import setUpModule
+import six
+from six.moves import map
+from six.moves import range
 setUpModule # pyflakes
 
 
@@ -48,7 +52,7 @@ class TestPublishedURLIsReachableMixin(object):
     # We access ERP5 trough a "virtual host", which should make
     # ERP5 produce URLs using https://virtual-host-name:1234/virtual_host_root
     # as base.
-    virtual_host_url = urlparse.urljoin(
+    virtual_host_url = six.moves.urllib.parse.urljoin(
         base_url,
         '/VirtualHostBase/https/virtual-host-name:1234/{}/VirtualHostRoot/_vh_virtual_host_root/'
         .format(site_id))
@@ -76,7 +80,7 @@ class TestPublishedURLIsReachableMixin(object):
 
     # login page can be rendered and contain the text "ERP5"
     r = session.get(
-        urlparse.urljoin(base_url, '{}/login_form'.format(site_id)),
+        six.moves.urllib.parse.urljoin(base_url, '{}/login_form'.format(site_id)),
         verify=verify,
         allow_redirects=False,
     )
@@ -119,6 +123,7 @@ class TestMedusa(ERP5InstanceTestCase, TestPublishedURLIsReachableMixin):
   def getInstanceParameterDict(cls):
     return {'_': json.dumps({'wsgi': False})}
 
+
 class TestJupyter(ERP5InstanceTestCase, TestPublishedURLIsReachableMixin):
   """Test ERP5 Jupyter notebook
   """
@@ -143,6 +148,7 @@ class TestJupyter(ERP5InstanceTestCase, TestPublishedURLIsReachableMixin):
       [result.status_code, result.is_redirect, result.headers['Location']]
     )
 
+
 class TestBalancerPorts(ERP5InstanceTestCase):
   """Instantiate with two zope families, this should create for each family:
    - a balancer entry point with corresponding haproxy
@@ -169,7 +175,7 @@ class TestBalancerPorts(ERP5InstanceTestCase):
     }
 
   def checkValidHTTPSURL(self, url):
-    parsed = urlparse.urlparse(url)
+    parsed = six.moves.urllib.parse.urlparse(url)
     self.assertEqual(parsed.scheme, 'https')
     self.assertTrue(parsed.hostname)
     self.assertTrue(parsed.port)
@@ -254,7 +260,7 @@ class TestSeleniumTestRunner(ERP5InstanceTestCase, TestPublishedURLIsReachableMi
     with open(config_file.strip()) as f:
       self.assertEqual(
           f.read(),
-          json.dumps(json.loads(self.getInstanceParameterDict()['_'])['test-runner']))
+          json.dumps(json.loads(self.getInstanceParameterDict()['_'])['test-runner'], sort_keys=True))
 
 
 class TestDisableTestRunner(ERP5InstanceTestCase, TestPublishedURLIsReachableMixin):
@@ -270,8 +276,8 @@ class TestDisableTestRunner(ERP5InstanceTestCase, TestPublishedURLIsReachableMix
     """
     # self.computer_partition_root_path is the path of root partition.
     # we want to assert that no scripts exist in any partition.
-    bin_programs = map(os.path.basename,
-      glob.glob(self.computer_partition_root_path + "/../*/bin/*"))
+    bin_programs = list(map(os.path.basename,
+      glob.glob(self.computer_partition_root_path + "/../*/bin/*")))
 
     self.assertTrue(bin_programs) # just to check the glob was correct.
     self.assertNotIn('runUnitTest', bin_programs)
@@ -352,7 +358,7 @@ class TestZopeNodeParameterOverride(ERP5InstanceTestCase, TestPublishedURLIsReac
       storage["storage"] = "root"
       storage["server"] = zeo_addr
       with open('%s/etc/zope-%s.conf' % (partition, zope)) as f:
-        conf = map(str.strip, f.readlines())
+        conf = list(map(str.strip, f.readlines()))
       i = conf.index("<zodb_db root>") + 1
       conf = iter(conf[i:conf.index("</zodb_db>", i)])
       for line in conf:
@@ -361,23 +367,23 @@ class TestZopeNodeParameterOverride(ERP5InstanceTestCase, TestPublishedURLIsReac
             if line == '</zeoclient>':
               break
             checkParameter(line, storage)
-          for k, v in storage.iteritems():
+          for k, v in six.iteritems(storage):
             self.assertIsNone(v, k)
           del storage
         else:
           checkParameter(line, zodb)
-      for k, v in zodb.iteritems():
+      for k, v in six.iteritems(zodb):
         self.assertIsNone(v, k)
 
     partition = self.getComputerPartitionPath('zope-a')
-    for zope in xrange(3):
+    for zope in range(3):
       checkConf({
           "cache-size-bytes": "20MB",
         }, {
           "cache-size": "50MB",
         })
     partition = self.getComputerPartitionPath('zope-bb')
-    for zope in xrange(5):
+    for zope in range(5):
       checkConf({
           "cache-size-bytes": "500MB" if zope else 1<<20,
         }, {

software/erp5/test/test/test_mariadb.py

@@ -26,10 +26,11 @@
 #
 ##############################################################################
 
+from __future__ import absolute_import
 import os
 import json
 import glob
-import urlparse
+import six.moves.urllib.parse
 import socket
 import sys
 import time
@@ -60,6 +61,7 @@ class MariaDBTestCase(ERP5InstanceTestCase):
 
   @classmethod
   def _getInstanceParameterDict(cls):
+    # type: () -> dict
     return {
         'tcpv4-port': 3306,
         'max-connection-count': 5,
@@ -75,12 +77,14 @@ class MariaDBTestCase(ERP5InstanceTestCase):
 
   @classmethod
   def getInstanceParameterDict(cls):
+    # type: () -> dict
     return {'_': json.dumps(cls._getInstanceParameterDict())}
 
   def getDatabaseConnection(self):
+    # type: () -> MySQLdb.connections.Connection
     connection_parameter_dict = json.loads(
         self.computer_partition.getConnectionParameterDict()['_'])
-    db_url = urlparse.urlparse(connection_parameter_dict['database-list'][0])
+    db_url = six.moves.urllib.parse.urlparse(connection_parameter_dict['database-list'][0])
     self.assertEqual('mysql', db_url.scheme)
 
     self.assertTrue(db_url.path.startswith('/'))
@@ -91,12 +95,15 @@ class MariaDBTestCase(ERP5InstanceTestCase):
         host=db_url.hostname,
         port=db_url.port,
         db=database_name,
+        use_unicode=True,
+        charset='utf8mb4'
     )
 
 
 class TestCrontabs(MariaDBTestCase, CrontabMixin):
 
   def test_full_backup(self):
+    # type: () -> None
     self._executeCrontabAtDate('mariadb-backup', '2050-01-01')
     with gzip.open(
         os.path.join(
@@ -106,10 +113,11 @@ class TestCrontabs(MariaDBTestCase, CrontabMixin):
             'mariadb-full',
             '20500101000000.sql.gz',
         ),
-        'r') as dump:
+        'rt') as dump:
       self.assertIn('CREATE TABLE', dump.read())
 
   def test_logrotate_and_slow_query_digest(self):
+    # type: () -> None
     # slow query digest needs to run after logrotate, since it operates on the rotated
     # file, so this tests both logrotate and slow query digest.
 
@@ -148,7 +156,7 @@ class TestCrontabs(MariaDBTestCase, CrontabMixin):
         'slowquery_digest',
         'slowquery_digest.txt-2050-01-01.xz',
     )
-    with lzma.open(slow_query_report, 'r') as f:
+    with lzma.open(slow_query_report, 'rt') as f:
       # this is the hash for our "select sleep(n)" slow query
       self.assertIn("ID 0xF9A57DD5A41825CA", f.read())
 
@@ -170,7 +178,7 @@ class TestCrontabs(MariaDBTestCase, CrontabMixin):
       subprocess.check_output('faketime 2050-01-01 %s' % check_slow_query_promise_plugin['command'], shell=True)
     self.assertEqual(
         error_context.exception.output,
-"""\
+b"""\
 Threshold is lower than expected: 
 Expected total queries : 1.0 and current is: 2
 Expected slowest query : 0.1 and current is: 3
@@ -179,6 +187,7 @@ Expected slowest query : 0.1 and current is: 3
 
 class TestMariaDB(MariaDBTestCase):
   def test_utf8_collation(self):
+    # type: () -> None
     cnx = self.getDatabaseConnection()
     with contextlib.closing(cnx):
       cnx.query(
@@ -199,11 +208,12 @@ class TestMariaDB(MariaDBTestCase):
           """
           select * from test_utf8_collation where col1 = "a"
           """)
-      self.assertEqual((('à',),), cnx.store_result().fetch_row(maxrows=2))
+      self.assertEqual(((u'à',),), cnx.store_result().fetch_row(maxrows=2))
 
 
 class TestMroonga(MariaDBTestCase):
   def test_mroonga_plugin_loaded(self):
+    # type: () -> None
     cnx = self.getDatabaseConnection()
     with contextlib.closing(cnx):
       cnx.query("show plugins")
@@ -213,6 +223,7 @@ class TestMroonga(MariaDBTestCase):
           plugins)
 
   def test_mroonga_normalize_udf(self):
+    # type: () -> None
     # example from https://mroonga.org/docs/reference/udf/mroonga_normalize.html#usage
     cnx = self.getDatabaseConnection()
     with contextlib.closing(cnx):
@@ -220,7 +231,8 @@ class TestMroonga(MariaDBTestCase):
           """
           SELECT mroonga_normalize("ABCDあぃうぇ㍑")
           """)
-      self.assertEqual((('abcdあぃうぇリットル',),),
+      # XXX this is returned as bytes by mroonga/mariadb (this might be a bug)
+      self.assertEqual(((u'abcdあぃうぇリットル'.encode('utf-8'),),),
                        cnx.store_result().fetch_row(maxrows=2))
 
       if 0:
@@ -233,10 +245,11 @@ class TestMroonga(MariaDBTestCase):
             """
             SELECT mroonga_normalize("aBｃＤあぃウェ㍑", "NormalizerMySQLUnicodeCIExceptKanaCIKanaWithVoicedSoundMark")
             """)
-        self.assertEqual((('ABCDあぃうぇ㍑',),),
+        self.assertEqual(((u'ABCDあぃうぇ㍑'.encode('utf-8'),),),
                          cnx.store_result().fetch_row(maxrows=2))
 
   def test_mroonga_full_text_normalizer(self):
+    # type: () -> None
     # example from https://mroonga.org//docs/tutorial/storage.html#how-to-specify-the-normalizer
     cnx = self.getDatabaseConnection()
     with contextlib.closing(cnx):
@@ -269,11 +282,12 @@ class TestMroonga(MariaDBTestCase):
            WHERE MATCH (content) AGAINST ("+ﾌﾞﾗｯｸ" IN BOOLEAN MODE)
           """)
       self.assertEqual(
-          ((datetime.date(2013, 4, 23), 'ブラックコーヒーを飲んだ。'),),
+          ((datetime.date(2013, 4, 23), u'ブラックコーヒーを飲んだ。'),),
           cnx.store_result().fetch_row(maxrows=2),
       )
 
   def test_mroonga_full_text_normalizer_TokenBigramSplitSymbolAlphaDigit(self):
+    # type: () -> None
     # Similar to as ERP5's testI18NSearch with erp5_full_text_mroonga_catalog
     cnx = self.getDatabaseConnection()
     with contextlib.closing(cnx):
@@ -317,11 +331,12 @@ class TestMroonga(MariaDBTestCase):
       self.assertEqual(((1,),), cnx.store_result().fetch_row(maxrows=2))
 
   def test_mroonga_full_text_stem(self):
+    # type: () -> None
     # example from https://mroonga.org//docs/tutorial/storage.html#how-to-specify-the-token-filters
     cnx = self.getDatabaseConnection()
     with contextlib.closing(cnx):
       cnx.query("SELECT mroonga_command('register token_filters/stem')")
-      self.assertEqual((('true',),), cnx.store_result().fetch_row(maxrows=2))
+      self.assertEqual(((b'true',),), cnx.store_result().fetch_row(maxrows=2))
       cnx.query(
           """
           CREATE TABLE memos (

software/erp5/test/test/test_wcfs.py

@@ -16,6 +16,7 @@
 # See COPYING file for full licensing terms.
 # See https://www.nexedi.com/licensing for rationale and options.
 
+from __future__ import absolute_import
 import json
 import os.path
 import unittest

software/erp5testnode/software.cfg

@@ -49,7 +49,7 @@ WebTest = 2.0.33
 soupsieve = 1.8
 #gitdb2 = 2.0.4
 #smmap2 = 2.0.4
-waitress = 1.2.1
+waitress = 1.4.4
 z3c.etestbrowser = 2.0.1
 zope.testbrowser = 5.3.2
 WSGIProxy2 = 0.4.6

software/galene/software.cfg

@@ -10,6 +10,9 @@ parts =
   eggs
   instance-cfg
 
+[python]
+part = python3
+
 # eggs for instance.cfg
 [eggs]
 recipe  = zc.recipe.egg
@@ -22,7 +25,7 @@ eggs    =
 <= go-git-package
 go.importpath = lab.nexedi.com/nexedi/galene
 repository = https://lab.nexedi.com/nexedi/galene.git
-revision = galene-0.3.3
+revision = galene-0.4
 
 [gowork]
 install =

software/hugo/buildout.hash.cfg

@@ -15,7 +15,7 @@
 
 [template-cfg]
 filename = instance.cfg.in
-md5sum = 2e4bed8c7b78c410b28f8becf81da596
+md5sum = 6f5acc546a7e9ad502d5fe586a3c3072
 
 [template_nginx_conf]
 _update_hash_filename_ = templates/nginx_conf.in

software/hugo/instance.cfg.in

@@ -113,6 +113,7 @@ template =
   . ${hugo:go-environment}
   cd ${basedirectory:data}/${slap-configuration:configuration.site}
   if [ -d "public" ]; then rm -Rf public; fi
+  export TMPDIR=${hugo:path-tmp}
   hugo && hugo server --bind=${hugo:ip} --port=${hugo:hugo-port} --baseURL=${hugo-frontend:connection-secure_access} --appendPort=false  
 
 [hugo-server-service]

software/kvm/README.rst

@@ -43,12 +43,12 @@ KVM instance parameters:
 - nbd2-port (default: 1024)
 - nbd2-host
 
-- ram-size (default: 1024)
-- disk-size = (default: 10)
+- ram-size (default: 4096)
+- disk-size = (default: 40)
 - disk-type (default: virtio)
       Disk size and Disk type are used if no virtual hard drive is specified.
 
-- cpu-count (default: 1)
+- cpu-count (default: 2)
 - cpu-options
     cpu-option is a string: [cores=cores][,threads=threads][,sockets=sockets][,maxcpus=maxcpus]
 - numa

software/kvm/buildout.hash.cfg

@@ -15,15 +15,15 @@
 
 [template]
 filename = instance.cfg.in
-md5sum = b129c9b2a614563d3f7c3f9e906d59f2
+md5sum = f2b0f1ed27148504f220e06eaceff935
 
 [template-kvm]
 filename = instance-kvm.cfg.jinja2
-md5sum = 78c3b69a2b18106cd6e289732116881d
+md5sum = 17c58f74d1ed4cb7dce11bf9af71dd33
 
 [template-kvm-cluster]
 filename = instance-kvm-cluster.cfg.jinja2.in
-md5sum = 59b92e1300aad4e9b116c532caf7d042
+md5sum = fcb35c32ef985e3d69a7914711675dcc
 
 [template-kvm-resilient]
 filename = instance-kvm-resilient.cfg.jinja2
@@ -55,7 +55,7 @@ md5sum = b7e87479a289f472b634a046b44b5257
 
 [template-kvm-run]
 filename = template/template-kvm-run.in
-md5sum = a502782244d1be536b732ebb40725f47
+md5sum = f840e8b7af83982525f66d7ec12b7085
 
 [template-kvm-controller]
 filename = template/kvm-controller-run.in

software/kvm/instance-kvm-cluster-input-schema.json

@@ -225,7 +225,7 @@
               "title": "RAM size",
               "description": "RAM size, in MB.",
               "type": "integer",
-              "default": 1024,
+              "default": 4096,
               "minimum": 1024,
               "multipleOf": 512
             },
@@ -233,6 +233,7 @@
               "title": "Maximum RAM size, in MB",
               "description": "Define the maximum size of the memory. The size is in MB and should be a multiple of 512. Defaults to ram-size + 512",
               "type": "integer",
+              "default": 4608,
               "minimum": 1024,
               "multipleOf": 512
             },
@@ -254,7 +255,7 @@
               "title": "Disk size",
               "description": "Disk size, in GB.",
               "type": "integer",
-              "default": 10,
+              "default": 40,
               "minimum": 1
             },
             "disk-format": {
@@ -313,6 +314,7 @@
               "title": "CPU count",
               "description": "Number of CPU cores.",
               "type": "integer",
+              "default": 2,
               "minimum": 1
             },
             "cpu-max-count": {

software/kvm/instance-kvm-cluster.cfg.jinja2.in

@@ -77,14 +77,14 @@ config-authorized-key = {{ dumps(slapparameter_dict.get('authorized-keys') | joi
 {% endif -%}
 config-nbd-port = {{ dumps(kvm_parameter_dict.get('nbd-port', 1024)) }}
 config-nbd2-port = {{ dumps(kvm_parameter_dict.get('nbd-port2', 1024)) }}
-config-ram-size = {{ dumps(kvm_parameter_dict.get('ram-size', 1024)) }}
-config-ram-max-size = {{ dumps(kvm_parameter_dict.get('ram-max-size', int(kvm_parameter_dict.get('ram-size', 1024)) + 512)) }}
+config-ram-size = {{ dumps(kvm_parameter_dict.get('ram-size', 4096)) }}
+config-ram-max-size = {{ dumps(kvm_parameter_dict.get('ram-max-size', int(kvm_parameter_dict.get('ram-size', 4096)) + 512)) }}
 config-enable-device-hotplug = {{ dumps(kvm_parameter_dict.get('enable-device-hotplug', False)) }}
 config-ram-hotplug-slot-size = {{ dumps(kvm_parameter_dict.get('ram-hotplug-slot-size', 512)) }}
-config-disk-size = {{ dumps(kvm_parameter_dict.get('disk-size', 10)) }}
+config-disk-size = {{ dumps(kvm_parameter_dict.get('disk-size', 40)) }}
 config-disk-type = {{ dumps(kvm_parameter_dict.get('disk-type', 'virtio')) }}
 config-disk-format = {{ dumps(kvm_parameter_dict.get('disk-format', 'qcow2')) }}
-config-cpu-count = {{ dumps(kvm_parameter_dict.get('cpu-count', 1)) }}
+config-cpu-count = {{ dumps(kvm_parameter_dict.get('cpu-count', 2)) }}
 config-cpu-max-count = {{ dumps(kvm_parameter_dict.get('cpu-max-count', int(kvm_parameter_dict.get('cpu-count', 1)) + 1)) }}
 {{ setconfig('numa', kvm_parameter_dict.get('numa', '')) }}
 {{ setconfig('machine-options', kvm_parameter_dict.get('machine-options', '')) }}

software/kvm/instance-kvm-input-schema.json

@@ -13,7 +13,7 @@
       "title": "RAM size",
       "description": "RAM size, in MB.",
       "type": "integer",
-      "default": 1024,
+      "default": 4096,
       "minimum": 1024,
       "multipleOf": 512
     },
@@ -21,6 +21,7 @@
       "title": "Maximum RAM size, in MB",
       "description": "Define the maximum size of the memory. The size is in MB and should be a multiple of 512. Defaults to ram-size + 512",
       "type": "integer",
+      "default": 4608,
       "minimum": 1024,
       "multipleOf": 512
     },
@@ -42,7 +43,7 @@
       "title": "Disk size",
       "description": "Disk size, in GB.",
       "type": "integer",
-      "default": 10,
+      "default": 40,
       "minimum": 1
     },
     "disk-type": {
@@ -101,7 +102,8 @@
       "title": "CPU count",
       "description": "Number of CPU cores.",
       "type": "integer",
-      "minimum": 1
+      "minimum": 1,
+      "default": 2
     },
     "cpu-max-count": {
       "title": "Maximum CPU amount",

software/kvm/instance-kvm.cfg.jinja2

@@ -22,8 +22,8 @@
 {% set boot_image_url_list_enabled = 'boot-image-url-list' in slapparameter_dict %}
 {% set boot_image_url_select_enabled = 'boot-image-url-select' in slapparameter_dict %}
 {% set bootstrap_script_url = slapparameter_dict.get('bootstrap-script-url') -%}
-{% set cpu_max_count = dumps(slapparameter_dict.get('cpu-max-count', int(slapparameter_dict.get('cpu-count', 1)) + 1)) %}
-{% set ram_max_size = dumps(slapparameter_dict.get('ram-max-size', int(slapparameter_dict.get('ram-size', 1024)) + 512)) %}
+{% set cpu_max_count = dumps(slapparameter_dict.get('cpu-max-count', int(slapparameter_dict.get('cpu-count', 2)) + 1)) %}
+{% set ram_max_size = dumps(slapparameter_dict.get('ram-max-size', int(slapparameter_dict.get('ram-size', 4096)) + 512)) %}
 {% set extends_list = [] -%}
 {% set part_list = [] -%}
 
@@ -441,8 +441,8 @@ socket-path = ${kvm-controller-parameter-dict:socket-path}
 smp-max-count = {{ cpu_max_count }}
 ram-max-size = {{ ram_max_size }}
 {%- if enable_device_hotplug %}
-init-ram-size = 1024
-init-smp-count = 1
+init-ram-size = 4096
+init-smp-count = 2
 {%- else %}
 init-ram-size = ${kvm-controller-parameter-dict:ram-size}
 init-smp-count = ${kvm-controller-parameter-dict:cpu-count}
@@ -1068,14 +1068,14 @@ nbd2-host =
 boot-image-url-list =
 
 enable-device-hotplug = False
-ram-size = 1024
+ram-size = 4096
 ram-hotplug-slot-size = 512
-disk-size = 10
+disk-size = 40
 disk-type = virtio
 disk-format = qcow2
 disk-device-path =
 
-cpu-count = 1
+cpu-count = 2
 disk-cache =
 disk-aio =
 auto-ballooning = True

software/kvm/instance.cfg.in

@@ -82,7 +82,7 @@ extra-context =
   raw dash_executable_location ${dash:location}/bin/dash
   raw dnsresolver_executable ${buildout:bin-directory}/dnsresolver
   raw dcron_executable_location ${dcron:location}/sbin/crond
-  raw debian_amd64_netinst_location ${debian-amd64-buster-netinst.iso:target}
+  raw debian_amd64_netinst_location ${debian-amd64-bullseye-netinst.iso:target}
   raw whitelist_domains_default ${whitelist-domains-default:location}/${whitelist-domains-default:filename}
   raw whitelist_firewall_download_controller ${whitelist-firewall-download-controller:target}
   raw image_download_controller ${image-download-controller:target}

software/kvm/template/template-kvm-run.in

@@ -277,6 +277,8 @@ kvm_argument_list = [qemu_path,
   '-pidfile', pid_file_path, '-msg', 'timestamp=on',
   '-D', logfile,
   '-nodefaults',
+  # switch to tablet mode for the mouse to have it synced with a client, see https://wiki.gentoo.org/wiki/QEMU/Options#USB
+  '-usbdevice', 'tablet',
 ]
 for disk_info in disk_info_list:
   kvm_argument_list += (

software/kvm/test/test.py

@@ -230,7 +230,7 @@ class TestMemoryManagement(InstanceTestCase, KvmMixin):
   def test(self):
     kvm_pid_1, info_list = self.getKvmProcessInfo(['-smp', '-m'])
     self.assertEqual(
-      ['1,maxcpus=2', '1024M,slots=128,maxmem=1536M'],
+      ['2,maxcpus=3', '4096M,slots=128,maxmem=4608M'],
       info_list
     )
     self.rerequestInstance({
@@ -264,7 +264,7 @@ class TestMemoryManagement(InstanceTestCase, KvmMixin):
 
     kvm_pid_1, info_list = self.getKvmProcessInfo(['-smp', '-m'])
     self.assertEqual(
-      ['1,maxcpus=2', '1024M,slots=128,maxmem=1536M'],
+      ['2,maxcpus=3', '4096M,slots=128,maxmem=4608M'],
       info_list
     )
     self.assertEqual(
@@ -275,15 +275,15 @@ class TestMemoryManagement(InstanceTestCase, KvmMixin):
     parameter_dict = {
       'enable-device-hotplug': 'true',
       # to avoid restarts the max RAM and CPU has to be static
-      'ram-max-size': '2048',
-      'cpu-max-count': '4',
+      'ram-max-size': '8192',
+      'cpu-max-count': '6',
     }
     self.rerequestInstance(parameter_dict)
     self.slap.waitForInstance(max_retry=2)
     kvm_pid_2, info_list = self.getKvmProcessInfo(['-smp', '-m'])
 
     self.assertEqual(
-      ['1,maxcpus=4', '1024M,slots=128,maxmem=2048M'],
+      ['2,maxcpus=6', '4096M,slots=128,maxmem=8192M'],
       info_list
     )
     self.assertEqual(
@@ -292,21 +292,21 @@ class TestMemoryManagement(InstanceTestCase, KvmMixin):
     )
     self.assertNotEqual(kvm_pid_1, kvm_pid_2, "Unexpected: KVM not restarted")
     parameter_dict.update(**{
-      'ram-size': '1536',
-      'cpu-count': '2'
+      'ram-size': '5120',
+      'cpu-count': '4'
     })
     self.rerequestInstance(parameter_dict)
     self.slap.waitForInstance(max_retry=10)
     kvm_pid_3, info_list = self.getKvmProcessInfo(['-smp', '-m'])
 
     self.assertEqual(
-      ['1,maxcpus=4', '1024M,slots=128,maxmem=2048M'],
+      ['2,maxcpus=6', '4096M,slots=128,maxmem=8192M'],
       info_list
     )
     self.assertEqual(kvm_pid_2, kvm_pid_3, "Unexpected: KVM restarted")
     self.assertEqual(
       getHotpluggedCpuRamValue(),
-      {'cpu_count': 1, 'ram_mb': 512}
+      {'cpu_count': 2, 'ram_mb': 1024}
     )
 
 

software/slapos-sr-testing/README.md

@@ -2,7 +2,7 @@
 
 This software release is used to run integration test of slapos softwares.
 
-The approach is to use setuptools' integrated test runner, `python setup.py test`, to run tests.
+The approach is to use python unittest runner to run tests.
 
 The `python` used in this command will be a `zc.recipe.egg` interpreter with
 all eggs pre-installed by this software release.
@@ -45,22 +45,25 @@ SLAPOS_TEST_DEBUG=1 python_for_test -m unittest discover -v
 
 ## Environment variables
 
-The `environment-script` set all variabels except `SLAPOS_TEST_DEBUG` and `SLAPOS_TEST_VERBOSE` for you, but for reference, here is the list of variables which control the test runner:
+The `environment-script` set all variables except `SLAPOS_TEST_DEBUG`, `SLAPOS_TEST_VERBOSE`, `SLAPOS_TEST_SKIP_SOFTWARE_CHECK` and `SLAPOS_TEST_SKIP_SOFTWARE_REBUILD` for you, but for reference, here is the list of variables which control the test runner:
 
 | Variable | Description |
 | --- | --- |
+| `SLAPOS_TEST_VERBOSE` | If set to 1, debugging output will be printed on console. This also adjust the log level of python process running tests. When running on test nodes, this is not set, so keep this difference in mind if test rely on python logger |
+| `SLAPOS_TEST_DEBUG` | If set to 1, `slapos node instance` and `slapos node software` will run with `--buildout-debug` flag, which will invoke python debugger on error. |
+| `SLAPOS_TEST_SKIP_SOFTWARE_CHECK` | If set to 1, software checks will not be performed. This can be used to speed up running tests locally, as checking software is an expensive operation. |
+| `SLAPOS_TEST_SKIP_SOFTWARE_REBUILD` | If set to 1, software will not be rebuilt before running the test. The default is to use `slapos node software --all`, which unconditionally rebuild all softwares and to use `slapos node software`, which installs the software only if it's not yet available when the environment variable is set to 1. |
 | `SLAPOS_TEST_IPV6` | ipv6 used by this instance. Usually you want to use a global address here to be able to connect to this instance. |
 | `SLAPOS_TEST_IPV4` | ipv4 used by this instance. |
 | `SLAPOS_TEST_WORKING_DIR` | Path to use as a working directory to hold the standalone SlapOS. |
 | `SLAPOS_TEST_SHARED_PART_LIST` | A `:` separated of paths to look for already installed shared parts. The SlapOS used in the test will not write in these, but will use a dedicated directory in `$SLAPOS_TEST_WORKING_DIR` |
-| `SLAPOS_TEST_VERBOSE` | If set to 1, debugging output will be printed on console. This also adjust the log level of python process running tests. When running on test nodes, this is not set, so keep this difference in mind if test rely on python logger |
-| `SLAPOS_TEST_DEBUG` | If set to 1, `slapos node instance` and `slapos node software` will run with `--buildout-debug` flag, which will invoke python debugger on error. |
+
 
 ## Frequently Asked Questions
 
 ### Where to find docs about the testing framework ?
 
-Please refere to the docstrings from `slapos.testing` module, from `slapos.core` package.
+Please refer to the docstrings from `slapos.testing` module, from `slapos.core` package.
 
 This uses python unittest module from standard library, especially the setup hooks:
  - `setUpModule` installs the software and perform some static checks
@@ -70,7 +73,7 @@ This uses python unittest module from standard library, especially the setup hoo
 ### Can I run slapos commands to debug ?
 
 The standalone slapos is created in `$SLAPOS_TEST_WORKING_DIR`. In this directory you will have a `bin/slapos` that you can run to start or stop services.
-It's fine to use this command during development, but to programatically interract with the environment within the test, the recommended approach is to use supervisor XML-RPC API.
+It's fine to use this command during development, but to programmatically interact with the environment within the test, the recommended approach is to use supervisor XML-RPC API.
 
 ### How to use a development version of `slapos.cookbook` ?
 
@@ -83,3 +86,7 @@ At the end of the test, a snapshot of the slapos instances is created. Sometimes
 
 Most of the time, problems are because on test nodes paths are very long. One advanced technique to reproduce the problem in your development environment is to set `SLAPOS_TEST_WORKING_DIR` environment variable to a path with the same length as the ones on test nodes.
 One way to make instances uses a slightly shorter path is to define `__partition_reference__` class attribute, so that the instances uses this as prefix instead of the class name.
+
+### Can I run only specific test ?
+
+Yes, please refer to python unittest documentation. For example, you can use `python_for_test -m unittest -v test_module.TestClass.test_function` to run only test function.

software/slapos-sr-testing/software.cfg

@@ -337,6 +337,7 @@ tests =
   theia ${slapos.test.theia-setup:setup}
   metabase ${slapos.test.metabase-setup:setup}
   nginx-push-stream ${slapos.test.nginx-push-stream-setup:setup}
+  erp5 ${slapos.test.erp5-setup:setup}
 ###
   ${:extra}
 
@@ -345,7 +346,6 @@ extra =
 #          You should not add more lines here.
   backupserver ${slapos.test.backupserver-setup:setup}
   caddy-frontend ${slapos.test.caddy-frontend-setup:setup}
-  erp5 ${slapos.test.erp5-setup:setup}
   upgrade_erp5 ${slapos.test.upgrade_erp5-setup:setup}
   htmlvalidatorserver ${slapos.test.htmlvalidatorserver-setup:setup}
   slapos-master ${slapos.test.slapos-master-setup:setup}

stack/caucase/buildout.cfg

@@ -36,6 +36,6 @@ mode = 0644
 depends = ${caucase-jinja2-library-eggs:eggs}
 
 [versions]
-caucase = 0.9.10
+caucase = 0.9.12
 pem = 21.1.0
 PyJWT = 1.7.1

stack/erp5/buildout.hash.cfg

@@ -94,7 +94,7 @@ md5sum = d6166515fda7b09df754672536b131be
 
 [template-haproxy-cfg]
 filename = haproxy.cfg.in
-md5sum = 3f4f7e49c504cbf610fc5dc462713dfc
+md5sum = 9d61e05c8578e0f17e349603ccaaf52c
 
 [template-rsyslogd-cfg]
 filename = rsyslogd.cfg.in

stack/erp5/haproxy.cfg.in

@@ -172,7 +172,7 @@ listen family_{{ name }}
   # logs
   capture request header Referer len 512
   capture request header User-Agent len 512
-  log-format "%{+Q}o %{-Q}ci - - [%trg] %r %ST %B %{+Q}[capture.req.hdr(0)] %{+Q}[capture.req.hdr(1)] %Tt"
+  log-format "%{+Q}o %{-Q}ci - - [%trg] %r %ST %B %{+Q}[capture.req.hdr(0)] %{+Q}[capture.req.hdr(1)] %Ta"
 
 {%   for outer_prefix, inner_prefix in family_path_routing_dict.get(name, []) + path_routing_list %}
   {%   set outer_prefix = outer_prefix.strip('/') -%}