Commit b2640909 authored by Kirill Smelkov's avatar Kirill Smelkov

Merge branch 'master' into master+ZODB4-wc2

* origin/master: (22 commits)
  component/perl-Image-ExifTool: version up 12.30
  fixup! stack/erp5: use %Ta for timing of haproxy requests
  stack/erp5: use %Ta for timing of haproxy requests
  Revert "kvm: Bang immediately in case of vnc_promise problem"
  kvm: Bang immediately in case of vnc_promise problem
  kvm: Switch mouse to tablet mode
  software/slapos-sr-testing: update doc
  software/slapos-sr-testing: run ERP5 SR test with a python3 slapos.core
  component/depot_tools: check out a fixed revision
  software/erp5/test: add python3 support
  software/galene: use python3
  stack/caucase: Version up caucase 0.9.12
  galene: version up 0.4
  kvm: Increase default VM parameters to match nowadays requirement
  hugo: create tempfiles in a dedicated temporary directory
  component/java-jdk: add missing rpaths
  component/alsa: enable slapos.recipe.cmmi's shared
  kvm: Upgrade default ISO for vm to debian bullseye
  componant/slapos/obs.cfg: update OBS build
  caddy-frontend: Pick up recent kedifa from the distribution
  ...
parents ddc8115d 0d1018d8
......@@ -5,6 +5,7 @@ parts =
[alsa]
# Contains libasound
recipe = slapos.recipe.cmmi
shared = true
url = ftp://ftp.alsa-project.org/pub/lib/alsa-lib-1.1.3.tar.bz2
md5sum = eefe5992567ba00d6110a540657aaf5c
configure-options =
......
[buildout]
extends =
../git/buildout.cfg
parts =
depot_tools
[depot_tools]
recipe = slapos.recipe.build:gitclone
repository = https://chromium.googlesource.com/chromium/tools/depot_tools.git
branch = master
revsion = e023d4482012d89690f6a483e877eceb47c4501e
git-executable = ${git:location}/bin/git
......@@ -73,7 +73,7 @@ environment =
PATH=${pkgconfig:location}/bin:${gettext:location}/bin:${glib:location}/bin:${xz-utils:location}/bin:${flex:location}/bin:${bison:location}/bin:%(PATH)s
PKG_CONFIG_PATH=${glib:location}/lib/pkgconfig:${pcre:location}/lib/pkgconfig
CPPFLAGS=-I${glib:location}/include/glib-2.0 -I${glib:location}/lib/glib-2.0/include
LDFLAGS=-L${glib:location}/lib -Wl,-rpath=${glib:location}/lib -L${libffi:location}/lib -Wl,-rpath=${libffi:location}/lib -lffi
LDFLAGS=-L${glib:location}/lib -Wl,-rpath=${glib:location}/lib -L${libffi:location}/lib -Wl,-rpath=${libffi:location}/lib -lffi -L${zlib:location}/lib/ -Wl,-rpath=${zlib:location}/lib/
GLIB_CFLAGS=-I${glib:location}/include/glib-2.0 -I${glib:location}/lib/glib-2.0/include
GLIB_LIBS=-L${glib:location}/lib -lglib-2.0 -lintl -lgobject-2.0
FFI_CFLAGS=-I${libffi:location}/include
......
[buildout]
extends =
../patchelf/buildout.cfg
../zlib/buildout.cfg
../alsa/buildout.cfg
../libpng/buildout.cfg
../freetype/buildout.cfg
../xorg/buildout.cfg
parts =
java-jdk
......@@ -24,8 +26,8 @@ configure-command = :
make-binary = :
post-install =
mv * %(location)s
for file in %(location)s/bin/* ; do
for file in %(location)s/bin/* %(location)s/lib/*.so ; do
echo appending rpath to $file
${patchelf:location}/bin/patchelf --set-rpath %(rpath)s $file
done
rpath = ${zlib:location}/lib:@@LOCATION@@/lib
rpath = ${zlib:location}/lib:${alsa:location}/lib:${freetype:location}/lib:${libpng:location}/lib:${libXrender:location}/lib:${libXtst:location}/lib:${libX11:location}/lib:${libXau:location}/lib:${libXext:location}/lib:${libXdmcp:location}/lib:${libXi:location}/lib:${libxcb:location}/lib:@@LOCATION@@/lib:@@LOCATION@@/lib/server
......@@ -7,5 +7,5 @@ parts =
[perl-Image-ExifTool]
<= perl-CPAN-package
module = Image/Image-ExifTool
version = 12.00
md5sum = 8c3f99a9a8d110ba340ee3f7063c940e
version = 12.30
md5sum = 1f5d66d62418c8b29eb0c0b7fd272b28
......@@ -76,8 +76,6 @@ post-install =
[dbus-glib]
location = ${buildout:parts-directory}/${:_buildout_section_name_}
environment +=
CPPFLAGS=-I${libexpat:location}/include -I${dbus:location}/include/dbus-1.0 -I${dbus:location}/lib/dbus-1.0/include
LDFLAGS=-L${libexpat:location}/lib -L${gettext:location}/lib -Wl,-rpath=${zlib:location}/lib -L${dbus:location}/lib -Wl,-rpath=${dbus:location}/lib
LD_LIBRARY_PATH=${dbus:location}/lib
DBUS_CFLAGS=-I${dbus:location}/include/dbus-1.0 -I${dbus:location}/lib/dbus-1.0/include
......@@ -86,9 +84,7 @@ environment +=
pre-configure =
sed -i 's#/opt/slapos/parts/dbus/lib/libdbus-1.la#${dbus:location}/lib/libdbus-1.la#' ${dbus-glib:location}/lib/libdbus-glib-1.la
environment +=
PATH=${pkgconfig:location}/bin:${python:location}/bin:%(PATH)s
LD_LIBRARY_PATH=${dbus:location}/lib
LDFLAGS=-L${glib:location}/lib -Wl,-rpath=${glib:location}/lib -L${dbus:location}/lib -Wl,-rpath=${dbus:location}/lib
PYTHON=${python:location}/bin/python${python:version}
PYTHON_INCLUDES=-I${python:location}/include/python${python:version}
PYTHON_LIBS=-L${python:location}/lib -lpython${python:version} -lpthread -ldl -lutil -lm
......@@ -114,34 +110,21 @@ make-options =
[gobject-introspection]
pre-configure =
ln -s ${python:location}/bin/python${python:version} ${python:location}/bin/python2.
sed -i 's#!/opt/slapos/parts/python2.7/bin/python2.7#!${python2.7:location}/bin/python2.7#' ${python2.7:location}/bin/python-config
libtoolize -c -f
aclocal -I${pkgconfig:location}/share/aclocal -I${gettext:location}/share/aclocal -I${libtool:location}/share/aclocal -I${glib:location}/share/aclocal
./autogen.sh
sed -i 's#!/opt/slapos/parts/python${python:version}/bin/python${python:version}#!${python:location}/bin/python${python:version}#' ${python:location}/bin/python${python:version}-config
configure-options +=
--enable-shared
environment +=
PATH=${autoconf:location}/bin:${automake:location}/bin:${pkgconfig:location}/bin:${libtool:location}/bin:${intltool:location}/bin:${gettext:location}/bin:${glib:location}/bin:${flex:location}/bin:${bison-go:location}/bin:%(PATH)s
GIR_DIR=${buildout:parts-directory}/${:_buildout_section_name_}/share/gir-1.0
CPPFLAGS=-I${glib:location}/include/glib-2.0 -I${glib:location}/lib/glib-2.0/include -I${python:location}/include/python${python:version}
LDFLAGS=-L${glib:location}/lib -Wl,-rpath=${glib:location}/lib -L${libffi:location}/lib -Wl,-rpath=${libffi:location}/lib -lffi -L${python:location}/lib
ACLOCAL_PATH=${pkgconfig:location}/share/aclocal:${gettext:location}/share/aclocal:${libtool:location}/share/aclocal:${glib:location}/share/aclocal:${intltool:location}/share/aclocal
M4=${m4:location}/bin/m4
PERL5LIB=${perl:location}/lib/5.28.1/
post-install =
sed -i 's#!${python:location}/bin/python${python:version}#!/opt/slapos/parts/python${python:version}/bin/python${python:version}#' ${python:location}/bin/python-config
sed -i 's#!${python:location}/bin/python${python:version}#!/opt/slapos/parts/python${python:version}/bin/python${python:version}#' ${python:location}/bin/python${python:version}-config
rm -rf ${bison-go:location}
[pygobject3]
pre-configure +=
sed -i 's#!/opt/slapos/parts/python${python:version}/bin/python${python:version}#!${python:location}/bin/python${python:version}#' ${python:location}/bin/python-config
environment +=
CPPFLAGS=-I${glib:location}/include/glib-2.0 -I${glib:location}/lib/glib-2.0/include -I${gettext:location}/include -I${libffi:location}/include -I${python:location}/include/python${python:version}
LDFLAGS=-L${glib:location}/lib -Wl,-rpath=${glib:location}/lib -L${gettext:location}/lib -Wl,-rpath=${gettext:location}/lib -L${python:location}/lib
sed -i 's#!/opt/slapos/parts/python${python:version}/bin/python${python:version}#!${python:location}/bin/python${python:version}#' ${python:location}/bin/python${python:version}-config
post-install =
sed -i 's#!${python:location}/bin/python${python:version}#!/opt/slapos/parts/python${python:version}/bin/python${python:version}#' ${python:location}/bin/python-config
sed -i 's#!${python:location}/bin/python${python:version}#!/opt/slapos/parts/python${python:version}/bin/python${python:version}#' ${python:location}/bin/python${python:version}-config
[ncurses]
configure-options =
......@@ -189,3 +172,9 @@ configure-options +=
--prefix=${buildout:rootdir}/parts/${:_buildout_section_name_}
environment +=
DESTDIR=${buildout:destdir}
[swig]
configure-options +=
--prefix=${buildout:rootdir}/parts/${:_buildout_section_name_}
environment +=
DESTDIR=${buildout:destdir}
......@@ -91,7 +91,7 @@ class Recipe(GenericBaseRecipe):
if klass == 'RunPromise':
klass = None
elif klass and not isPythonName(klass):
raise UserError("%r is not valid class name" % klass)
raise UserError("%r is not a valid class name" % klass)
if bool(module) == bool(filepath):
raise UserError("Either 'module' or 'file' is required but not both")
......
import os, shutil, tempfile, unittest
from slapos.recipe import promise_plugin
from slapos.test.utils import makeRecipe
from zc.buildout import UserError
from pprint import pformat
import stat, json
import six
......@@ -10,7 +11,7 @@ class TestPromisePlugin(unittest.TestCase):
def setUp(self):
self.tmp = tempfile.mkdtemp()
self.output = os.path.join(self.tmp, 'output.py')
self.options = options = {
self.options = {
'output': self.output,
'eggs': 'slapos.cookbook'
}
......@@ -18,9 +19,16 @@ class TestPromisePlugin(unittest.TestCase):
def tearDown(self):
shutil.rmtree(self.tmp)
def test_parameters(self):
self.options['mode'] = '0644'
self.options['import'] = 'slapos.promise.plugin.check_site_available'
def makeRecipe(self):
return makeRecipe(
promise_plugin.Recipe,
options=self.options,
name='plugin')
def installRecipe(self):
self.makeRecipe().install()
def setConfig(self):
self.options['config-param1'] = "YY^@12"
self.options['config-param2'] = "23'91'"
self.options['config-param3'] = None
......@@ -28,88 +36,102 @@ class TestPromisePlugin(unittest.TestCase):
in multi line
123444
"""
recipe = makeRecipe(
promise_plugin.Recipe,
options=self.options,
name='plugin')
recipe.install()
def assertOutput(self, *expect):
self.assertTrue(os.path.exists(self.output))
with open(self.output, 'r') as f:
content = f.read()
self.assertIn("from slapos.promise.plugin.check_site_available import RunPromise", content)
self.assertEqual(stat.S_IMODE(os.stat(self.output).st_mode), int('644', 8))
expected_dict = dict(
param1=self.options['config-param1'],
param2=self.options['config-param2'],
param3=self.options['config-param3'],
param4=self.options['config-param4'],
)
self.assertIn('extra_config_dict = %s' % pformat(expected_dict, indent=2), content)
def test_no_module_set(self):
recipe = makeRecipe(
promise_plugin.Recipe,
options=self.options,
name='plugin')
with self.assertRaises(KeyError):
recipe.install()
def test_default(self):
self.options['import'] = 'slapos.promise.plugin.check_site_available'
recipe = makeRecipe(
promise_plugin.Recipe,
options=self.options,
name='plugin')
recipe.install()
self.assertTrue(os.path.exists(self.output))
for s in expect:
self.assertIn(s, content)
def assertConfig(self):
items = self.options.items()
expect = {k[7:] : v for k, v in items if k.startswith('config-')}
self.assertOutput("extra_config_dict = %s" % pformat(expect, indent=2))
def assertEmptyConfig(self):
self.assertOutput("extra_config_dict = %s" % ('{}' if six.PY3 else '{ }'))
def test_module(self):
self.options['module'] = 'slapos.promise.plugin.check_site_available'
self.installRecipe()
self.assertOutput("from %s import RunPromise" % self.options['module'])
self.assertEmptyConfig()
def test_file(self):
self.options['file'] = __file__
self.installRecipe()
self.assertOutput("exec(_(%r))" % self.options['file'])
self.assertEmptyConfig()
def test_module_and_parameters(self):
self.options['module'] = 'slapos.promise.plugin.check_site_available'
self.setConfig()
self.installRecipe()
self.assertOutput("from %s import RunPromise" % self.options['module'])
self.assertConfig()
def test_file_and_parameters(self):
self.options['file'] = __file__
self.setConfig()
self.installRecipe()
self.assertOutput("exec(_(%r))" % self.options['file'])
self.assertConfig()
def test_mode(self):
self.options['mode'] = '0644'
self.options['module'] = 'slapos.promise.plugin.check_site_available'
self.installRecipe()
self.assertEqual(stat.S_IMODE(os.stat(self.output).st_mode), int('644', 8))
with open(self.output) as f:
content = f.read()
self.assertIn("from slapos.promise.plugin.check_site_available import RunPromise", content)
self.assertIn('extra_config_dict = %s' % ('{}' if six.PY3 else '{ }'), content)
def test_module_and_class(self):
self.options['module'] = m = 'slapos.promise.plugin.check_site_available'
self.options['class'] = 'MyPromise'
self.installRecipe()
self.assertOutput("from %s import MyPromise as RunPromise" % m)
def test_file_and_class(self):
self.options['file'] = __file__
self.options['class'] = 'MyPromise'
self.installRecipe()
self.assertOutput("exec(_(%r))\n\nRunPromise = MyPromise" % __file__)
def test_no_module_or_file(self):
with self.assertRaises(UserError) as p:
self.makeRecipe()
msg = str(p.exception)
self.assertEqual(msg, "Either 'module' or 'file' is required but not both")
def test_module_and_file(self):
self.options['module'] = 'slapos.promise.plugin.check_site_available'
self.options['file'] = __file__
self.test_no_module_or_file()
def test_bad_parameters(self):
self.options['import'] = 'slapos.promise.plugin.check_site_available'
self.options['config-param1; print "toto"'] = """#xxxx"\nimport os; os.stat(f)"""
self.options['config-param2\n@domething'] = '"#$$*PPP\n\n p = 2*5; print "result is %s" % p'
recipe = makeRecipe(
promise_plugin.Recipe,
options=self.options,
name='plugin')
recipe.install()
self.assertTrue(os.path.exists(self.output))
with open(self.output) as f:
content = f.read()
expected_param1 = r"""'param1; print "toto"': '#xxxx"\nimport os; os.stat(f)',"""
expected_param2 = r"""'param2\n@domething': '"#$$*PPP\n\n p = 2*5; print "result is %s" % p'"""
self.assertIn(expected_param1, content)
self.assertIn(expected_param2, content)
def test_bad_module_path(self):
self.options['import'] = 'slapos.promise.plugin.check_site_available; print "toto"'
recipe = makeRecipe(
promise_plugin.Recipe,
options=self.options,
name='plugin')
with self.assertRaises(ValueError) as p:
recipe.install()
self.assertEqual(str(p.exception), "Import path %r is not a valid" % self.options['import'])
def test_bad_content(self):
self.options['content'] = 'from slapos.plugin.check_site_available import toto; print "toto"'
recipe = makeRecipe(
promise_plugin.Recipe,
options=self.options,
name='plugin')
with self.assertRaises(ValueError) as p:
recipe.install()
self.assertEqual(str(p.exception), "Promise content %r is not valid" % self.options['content'])
self.options['module'] = 'slapos.promise.plugin.check_site_available'
self.options.update((
('config-param1; print "toto"', '#xxxx"\nimport os; os.stat(f)'),
('config-param2\n@domething', '"#$$*PPP\np = 2*5; print "result=%s" % p')
))
self.installRecipe()
self.assertOutput(
r"""'param1; print "toto"': '#xxxx"\nimport os; os.stat(f)',""",
r"""'param2\n@domething': '"#$$*PPP\np = 2*5; print "result=%s" % p'"""
)
def test_bad_module(self):
self.options['module'] = 'slapos.promise.plugin.check_site_available; print "toto"'
with self.assertRaises(UserError) as p:
self.makeRecipe()
self.assertEqual(str(p.exception), "%r is not a valid module name" % self.options['module'])
def test_bad_file(self):
self.options['file'] = 'print "toto"'
self.installRecipe()
self.assertOutput(r"""exec(_('print "toto"'))""")
def test_bad_class(self):
self.options['class'] = 'MyPromise; print "toto"'
with self.assertRaises(UserError) as p:
self.makeRecipe()
self.assertEqual(str(p.exception), "%r is not a valid class name" % self.options['class'])
......@@ -20,19 +20,8 @@ parts +=
caucase-eggs
template
caddyprofiledeps
kedifa-develop
kedifa
[kedifa-repository]
recipe = slapos.recipe.build:gitclone
repository = https://lab.nexedi.com/nexedi/kedifa.git
git-executable = ${git:location}/bin/git
revision = 3fccc2ec945c59b644a12fa40225795abd61e0b0
[kedifa-develop]
recipe = zc.recipe.egg:develop
setup = ${kedifa-repository:location}
[kedifa]
recipe = zc.recipe.egg
eggs =
......@@ -222,6 +211,7 @@ mode = 0644
<=download-template
[versions]
kedifa = 0.0.6
# Modern KeDiFa requires zc.lockfile
zc.lockfile = 1.4
......
......@@ -24,6 +24,7 @@
# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
#
##############################################################################
from __future__ import absolute_import
from setuptools import setup, find_packages
version = '0.0.1.dev0'
......@@ -52,7 +53,6 @@ setup(name=name,
'cryptography',
'pexpect',
'pyOpenSSL',
'typing; python_version<"3"',
],
test_suite='test',
)
......@@ -25,6 +25,7 @@
#
##############################################################################
from __future__ import absolute_import
import json
import os
......
from __future__ import absolute_import
import glob
import hashlib
import json
......@@ -8,10 +9,8 @@ import shutil
import subprocess
import tempfile
import time
import urllib
import urlparse
from BaseHTTPServer import BaseHTTPRequestHandler
from typing import Dict
import six.moves.urllib.request, six.moves.urllib.parse, six.moves.urllib.error
from six.moves.BaseHTTPServer import BaseHTTPRequestHandler
import mock
import OpenSSL.SSL
......@@ -28,6 +27,7 @@ from slapos.testing.utils import (CrontabMixin, ManagedHTTPServer,
findFreeTCPPort)
from . import ERP5InstanceTestCase, setUpModule
from six.moves import range
setUpModule # pyflakes
......@@ -44,10 +44,10 @@ class EchoHTTPServer(ManagedHTTPServer):
response = json.dumps(
{
'Path': self.path,
'Incoming Headers': self.headers.dict
'Incoming Headers': dict(self.headers.items()),
},
indent=2,
)
).encode('utf-8')
self.end_headers()
self.wfile.write(response)
......@@ -67,11 +67,11 @@ class EchoHTTP11Server(ManagedHTTPServer):
response = json.dumps(
{
'Path': self.path,
'Incoming Headers': self.headers.dict
'Incoming Headers': dict(self.headers.items()),
},
indent=2,
)
self.send_header("Content-Length", len(response))
).encode('utf-8')
self.send_header("Content-Length", str(len(response)))
self.end_headers()
self.wfile.write(response)
......@@ -139,7 +139,7 @@ class BalancerTestCase(ERP5InstanceTestCase):
@classmethod
def _getInstanceParameterDict(cls):
# type: () -> Dict
# type: () -> dict
return {
'tcpv4-port': 8000,
'computer-memory-percent-threshold': 100,
......@@ -174,10 +174,11 @@ class BalancerTestCase(ERP5InstanceTestCase):
@classmethod
def getInstanceParameterDict(cls):
# type: () -> Dict
# type: () -> dict
return {'_': json.dumps(cls._getInstanceParameterDict())}
def setUp(self):
# type: () -> None
self.default_balancer_url = json.loads(
self.computer_partition.getConnectionParameterDict()['_'])['default']
......@@ -192,7 +193,7 @@ class SlowHTTPServer(ManagedHTTPServer):
self.send_header("Content-Type", "text/plain")
time.sleep(2)
self.end_headers()
self.wfile.write("OK\n")
self.wfile.write(b"OK\n")
log_message = logging.getLogger(__name__ + '.SlowHandler').info
......@@ -203,7 +204,7 @@ class TestLog(BalancerTestCase, CrontabMixin):
__partition_reference__ = 'l'
@classmethod
def _getInstanceParameterDict(cls):
# type: () -> Dict
# type: () -> dict
parameter_dict = super(TestLog, cls)._getInstanceParameterDict()
# use a slow server instead
parameter_dict['dummy_http_server'] = [[cls.getManagedResource("slow_web_server", SlowHTTPServer).netloc, 1, False]]
......@@ -212,7 +213,7 @@ class TestLog(BalancerTestCase, CrontabMixin):
def test_access_log_format(self):
# type: () -> None
requests.get(
urlparse.urljoin(self.default_balancer_url, '/url_path'),
six.moves.urllib.parse.urljoin(self.default_balancer_url, '/url_path'),
verify=False,
)
time.sleep(.5) # wait a bit more until access is logged
......@@ -285,6 +286,7 @@ class TestLog(BalancerTestCase, CrontabMixin):
self.assertFalse(os.path.exists(rotated_log_file))
def test_error_log(self):
# type: () -> None
# stop backend server
backend_server = self.getManagedResource("slow_web_server", SlowHTTPServer)
self.addCleanup(backend_server.open)
......@@ -326,7 +328,7 @@ class BalancerCookieHTTPServer(ManagedHTTPServer):
# The name of this cookie is SERVERID
assert self.headers['X-Balancer-Current-Cookie'] == 'SERVERID'
self.end_headers()
self.wfile.write(server._name)
self.wfile.write(server._name.encode('utf-8'))
log_message = logging.getLogger(__name__ + '.BalancerCookieHTTPServer').info
return RequestHandler
......@@ -338,7 +340,7 @@ class TestBalancer(BalancerTestCase):
__partition_reference__ = 'b'
@classmethod
def _getInstanceParameterDict(cls):
# type: () -> Dict
# type: () -> dict
parameter_dict = super(TestBalancer, cls)._getInstanceParameterDict()
# use two backend servers
......@@ -349,6 +351,7 @@ class TestBalancer(BalancerTestCase):
return parameter_dict
def test_balancer_round_robin(self):
# type: () -> None
# requests are by default balanced to both servers
self.assertEqual(
{requests.get(self.default_balancer_url, verify=False).text for _ in range(10)},
......@@ -356,6 +359,7 @@ class TestBalancer(BalancerTestCase):
)
def test_balancer_server_down(self):
# type: () -> None
# if one backend is down, it is excluded from balancer
self.getManagedResource("backend_web_server2", BalancerCookieHTTPServer).close()
self.addCleanup(self.getManagedResource("backend_web_server2", BalancerCookieHTTPServer).open)
......@@ -365,14 +369,16 @@ class TestBalancer(BalancerTestCase):
)
def test_balancer_set_cookie(self):
# type: () -> None
# if backend provides a "SERVERID" cookie, balancer will overwrite it with the
# backend selected by balancing algorithm
self.assertIn(
requests.get(urlparse.urljoin(self.default_balancer_url, '/set_cookie'), verify=False).cookies['SERVERID'],
requests.get(six.moves.urllib.parse.urljoin(self.default_balancer_url, '/set_cookie'), verify=False).cookies['SERVERID'],
('default-0', 'default-1'),
)
def test_balancer_respects_sticky_cookie(self):
# type: () -> None
# if request is made with the sticky cookie, the client stick on one balancer
cookies = dict(SERVERID='default-1')
self.assertEqual(
......@@ -388,6 +394,7 @@ class TestBalancer(BalancerTestCase):
'backend_web_server1')
def test_balancer_stats_socket(self):
# type: () -> None
# real time statistics can be obtained by using the stats socket and there
# is a wrapper which makes this a bit easier.
socat_process = subprocess.Popen(
......@@ -397,14 +404,14 @@ class TestBalancer(BalancerTestCase):
stderr=subprocess.STDOUT
)
try:
output, _ = socat_process.communicate("show stat\n")
output, _ = socat_process.communicate(b"show stat\n")
except:
socat_process.kill()
socat_process.wait()
raise
self.assertEqual(socat_process.poll(), 0)
# output is a csv
self.assertIn('family_default,FRONTEND,', output)
self.assertIn(b'family_default,FRONTEND,', output)
class TestTestRunnerEntryPoints(BalancerTestCase):
......@@ -413,7 +420,7 @@ class TestTestRunnerEntryPoints(BalancerTestCase):
__partition_reference__ = 't'
@classmethod
def _getInstanceParameterDict(cls):
# type: () -> Dict
# type: () -> dict
parameter_dict = super(
TestTestRunnerEntryPoints,
cls,
......@@ -436,23 +443,24 @@ class TestTestRunnerEntryPoints(BalancerTestCase):
return parameter_dict
def test_use_proper_backend(self):
# type: () -> None
# requests are directed to proper backend based on URL path
test_runner_url_list = self.getRootPartitionConnectionParameterDict(
)['default-test-runner-url-list']
url_0, url_1, url_2 = test_runner_url_list
self.assertEqual(
urlparse.urlparse(url_0).netloc,
urlparse.urlparse(url_1).netloc)
six.moves.urllib.parse.urlparse(url_0).netloc,
six.moves.urllib.parse.urlparse(url_1).netloc)
self.assertEqual(
urlparse.urlparse(url_0).netloc,
urlparse.urlparse(url_2).netloc)
six.moves.urllib.parse.urlparse(url_0).netloc,
six.moves.urllib.parse.urlparse(url_2).netloc)
path_0 = '/VirtualHostBase/https/{netloc}/VirtualHostRoot/_vh_unit_test_0/something'.format(
netloc=urlparse.urlparse(url_0).netloc)
netloc=six.moves.urllib.parse.urlparse(url_0).netloc)
path_1 = '/VirtualHostBase/https/{netloc}/VirtualHostRoot/_vh_unit_test_1/something'.format(
netloc=urlparse.urlparse(url_0).netloc)
netloc=six.moves.urllib.parse.urlparse(url_0).netloc)
path_2 = '/VirtualHostBase/https/{netloc}/VirtualHostRoot/_vh_unit_test_2/something'.format(
netloc=urlparse.urlparse(url_0).netloc)
netloc=six.moves.urllib.parse.urlparse(url_0).netloc)
self.assertEqual(
{
......@@ -489,7 +497,7 @@ class TestHTTP(BalancerTestCase):
"""
@classmethod
def _getInstanceParameterDict(cls):
# type: () -> Dict
# type: () -> dict
parameter_dict = super(TestHTTP, cls)._getInstanceParameterDict()
# use a HTTP/1.1 server instead
parameter_dict['dummy_http_server'] = [[cls.getManagedResource("HTTP/1.1 Server", EchoHTTP11Server).netloc, 1, False]]
......@@ -511,7 +519,7 @@ class TestHTTP(BalancerTestCase):
'%{http_version}',
self.default_balancer_url,
]),
'2',
b'2',
)
def test_keep_alive(self):
......@@ -530,7 +538,7 @@ class TestHTTP(BalancerTestCase):
session.get(self.default_balancer_url).raise_for_status()
new_conn.assert_not_called()
parsed_url = urlparse.urlparse(self.default_balancer_url)
parsed_url = six.moves.urllib.parse.urlparse(self.default_balancer_url)
# check that we have an open file for the ip connection
self.assertTrue([
c for c in psutil.Process(os.getpid()).connections()
......@@ -553,12 +561,12 @@ class ContentTypeHTTPServer(ManagedHTTPServer):
# type: () -> None
self.send_response(200)
if self.path == '/':
self.send_header("Content-Length", 0)
self.send_header("Content-Length", '0')
return self.end_headers()
content_type = self.path[1:]
body = "OK"
body = b"OK"
self.send_header("Content-Type", content_type)
self.send_header("Content-Length", len(body))
self.send_header("Content-Length", str(len(body)))
self.end_headers()
self.wfile.write(body)
......@@ -571,7 +579,7 @@ class TestContentEncoding(BalancerTestCase):
__partition_reference__ = 'ce'
@classmethod
def _getInstanceParameterDict(cls):
# type: () -> Dict
# type: () -> dict
parameter_dict = super(TestContentEncoding, cls)._getInstanceParameterDict()
parameter_dict['dummy_http_server'] = [
[cls.getManagedResource("content_type_server", ContentTypeHTTPServer).netloc, 1, False],
......@@ -599,7 +607,7 @@ class TestContentEncoding(BalancerTestCase):
'application/font-woff2',
'application/x-font-opentype',
'application/wasm',):
resp = requests.get(urlparse.urljoin(self.default_balancer_url, content_type), verify=False)
resp = requests.get(six.moves.urllib.parse.urljoin(self.default_balancer_url, content_type), verify=False)
self.assertEqual(resp.headers['Content-Type'], content_type)
self.assertEqual(
resp.headers.get('Content-Encoding'),
......@@ -609,7 +617,7 @@ class TestContentEncoding(BalancerTestCase):
def test_no_gzip_encoding(self):
# type: () -> None
resp = requests.get(urlparse.urljoin(self.default_balancer_url, '/image/png'), verify=False)
resp = requests.get(six.moves.urllib.parse.urljoin(self.default_balancer_url, '/image/png'), verify=False)
self.assertNotIn('Content-Encoding', resp.headers)
self.assertEqual(resp.text, 'OK')
......@@ -692,7 +700,7 @@ class CaucaseCertificate(ManagedResource):
cas_args + [
'--send-csr', self.csr_file,
],
).split()[0]
).split()[0].decode()
assert csr_id
for _ in range(30):
......@@ -708,11 +716,11 @@ class CaucaseCertificate(ManagedResource):
time.sleep(1)
else:
raise RuntimeError('getting service certificate failed.')
with open(self.cert_file) as f:
assert 'BEGIN CERTIFICATE' in f.read()
with open(self.cert_file) as cert_file:
assert 'BEGIN CERTIFICATE' in cert_file.read()
def revoke(self, caucase):
# type: (str, CaucaseService) -> None
# type: (CaucaseService) -> None
"""Revoke the client certificate on this caucase instance.
"""
subprocess.check_call([
......@@ -729,7 +737,7 @@ class TestFrontendXForwardedFor(BalancerTestCase):
@classmethod
def _getInstanceParameterDict(cls):
# type: () -> Dict
# type: () -> dict
frontend_caucase = cls.getManagedResource('frontend_caucase', CaucaseService)
certificate = cls.getManagedResource('client_certificate', CaucaseCertificate)
certificate.request(u'shared frontend', frontend_caucase)
......@@ -784,10 +792,10 @@ class TestServerTLSProvidedCertificate(BalancerTestCase):
@classmethod
def _getInstanceParameterDict(cls):
# type: () -> Dict
# type: () -> dict
server_caucase = cls.getManagedResource('server_caucase', CaucaseService)
server_certificate = cls.getManagedResource('server_certificate', CaucaseCertificate)
server_certificate.request(cls._ipv4_address.decode(), server_caucase)
server_certificate.request(six.ensure_text(cls._ipv4_address), server_caucase)
parameter_dict = super(TestServerTLSProvidedCertificate, cls)._getInstanceParameterDict()
with open(server_certificate.cert_file) as f:
parameter_dict['ssl']['cert'] = f.read()
......@@ -806,7 +814,7 @@ class TestClientTLS(BalancerTestCase):
@classmethod
def _getInstanceParameterDict(cls):
# type: () -> Dict
# type: () -> dict
frontend_caucase1 = cls.getManagedResource('frontend_caucase1', CaucaseService)
certificate1 = cls.getManagedResource('client_certificate1', CaucaseCertificate)
certificate1.request(u'client_certificate1', frontend_caucase1)
......@@ -827,7 +835,6 @@ class TestClientTLS(BalancerTestCase):
def test_refresh_crl(self):
# type: () -> None
logger = self.logger
class DebugLogFile:
......@@ -846,6 +853,7 @@ class TestClientTLS(BalancerTestCase):
# when client certificate can be authenticated, backend receive the CN of
# the client certificate in "remote-user" header
def _make_request():
# type: () -> dict
return requests.get(
self.default_balancer_url,
cert=(client_certificate.cert_file, client_certificate.key_file),
......@@ -897,6 +905,7 @@ class TestClientTLS(BalancerTestCase):
with self.assertRaisesRegexp(Exception, 'certificate revoked'):
_make_request()
class TestPathBasedRouting(BalancerTestCase):
"""Check path-based routing rewrites URLs as expected.
"""
......@@ -904,7 +913,7 @@ class TestPathBasedRouting(BalancerTestCase):
@classmethod
def _getInstanceParameterDict(cls):
# type: () -> Dict
# type: () -> dict
parameter_dict = super(
TestPathBasedRouting,
cls,
......@@ -936,14 +945,15 @@ class TestPathBasedRouting(BalancerTestCase):
published_dict = json.loads(self.computer_partition.getConnectionParameterDict()['_'])
scheme = 'scheme'
netloc = 'example.com:8080'
prefix = '/VirtualHostBase/' + scheme + '//' + urllib.quote(
prefix = '/VirtualHostBase/' + scheme + '//' + six.moves.urllib.parse.quote(
netloc,
safe='',
)
# For easier reading of test data, visualy separating the virtual host
# For easier reading of test data, visually separating the virtual host
# base from the virtual host root
vhr = '/VirtualHostRoot'
def assertRoutingEqual(family, path, expected_path):
# type: (str, str, str) -> None
# sanity check: unlike the rules, this test is sensitive to outermost
# slashes, and paths must be absolute-ish for code simplicity.
assert path.startswith('/')
......@@ -959,7 +969,7 @@ class TestPathBasedRouting(BalancerTestCase):
# test will need to be updated accordingly.
self.assertEqual(
requests.get(
urlparse.urljoin(published_dict[family], prefix + vhr + path),
six.moves.urllib.parse.urljoin(published_dict[family], prefix + vhr + path),
verify=False,
).json()['Path'],
expected_path,
......@@ -978,7 +988,7 @@ class TestPathBasedRouting(BalancerTestCase):
# Rule precedence: family rules applied before general rules.
assertRoutingEqual('default', '/next', prefix + '/erp5/web_site_module/another_next_website' + vhr + '/_vh_next')
# Fallback on general rules when no family-specific rule matches
# Note: the root is special in that there is aways a trailing slash in the
# Note: the root is special in that there is always a trailing slash in the
# produced URL.
assertRoutingEqual('default', '/', prefix + '/erp5/web_site_module/123' + vhr + '/')
# Rule-less family reach general rules.
......
......@@ -25,10 +25,11 @@
#
##############################################################################
from __future__ import absolute_import
import os
import json
import glob
import urlparse
import six.moves.urllib.parse
import socket
import time
......@@ -37,6 +38,9 @@ import requests
from . import ERP5InstanceTestCase
from . import setUpModule
import six
from six.moves import map
from six.moves import range
setUpModule # pyflakes
......@@ -48,7 +52,7 @@ class TestPublishedURLIsReachableMixin(object):
# We access ERP5 trough a "virtual host", which should make
# ERP5 produce URLs using https://virtual-host-name:1234/virtual_host_root
# as base.
virtual_host_url = urlparse.urljoin(
virtual_host_url = six.moves.urllib.parse.urljoin(
base_url,
'/VirtualHostBase/https/virtual-host-name:1234/{}/VirtualHostRoot/_vh_virtual_host_root/'
.format(site_id))
......@@ -76,7 +80,7 @@ class TestPublishedURLIsReachableMixin(object):
# login page can be rendered and contain the text "ERP5"
r = session.get(
urlparse.urljoin(base_url, '{}/login_form'.format(site_id)),
six.moves.urllib.parse.urljoin(base_url, '{}/login_form'.format(site_id)),
verify=verify,
allow_redirects=False,
)
......@@ -119,6 +123,7 @@ class TestMedusa(ERP5InstanceTestCase, TestPublishedURLIsReachableMixin):
def getInstanceParameterDict(cls):
return {'_': json.dumps({'wsgi': False})}
class TestJupyter(ERP5InstanceTestCase, TestPublishedURLIsReachableMixin):
"""Test ERP5 Jupyter notebook
"""
......@@ -143,6 +148,7 @@ class TestJupyter(ERP5InstanceTestCase, TestPublishedURLIsReachableMixin):
[result.status_code, result.is_redirect, result.headers['Location']]
)
class TestBalancerPorts(ERP5InstanceTestCase):
"""Instantiate with two zope families, this should create for each family:
- a balancer entry point with corresponding haproxy
......@@ -169,7 +175,7 @@ class TestBalancerPorts(ERP5InstanceTestCase):
}
def checkValidHTTPSURL(self, url):
parsed = urlparse.urlparse(url)
parsed = six.moves.urllib.parse.urlparse(url)
self.assertEqual(parsed.scheme, 'https')
self.assertTrue(parsed.hostname)
self.assertTrue(parsed.port)
......@@ -254,7 +260,7 @@ class TestSeleniumTestRunner(ERP5InstanceTestCase, TestPublishedURLIsReachableMi
with open(config_file.strip()) as f:
self.assertEqual(
f.read(),
json.dumps(json.loads(self.getInstanceParameterDict()['_'])['test-runner']))
json.dumps(json.loads(self.getInstanceParameterDict()['_'])['test-runner'], sort_keys=True))
class TestDisableTestRunner(ERP5InstanceTestCase, TestPublishedURLIsReachableMixin):
......@@ -270,8 +276,8 @@ class TestDisableTestRunner(ERP5InstanceTestCase, TestPublishedURLIsReachableMix
"""
# self.computer_partition_root_path is the path of root partition.
# we want to assert that no scripts exist in any partition.
bin_programs = map(os.path.basename,
glob.glob(self.computer_partition_root_path + "/../*/bin/*"))
bin_programs = list(map(os.path.basename,
glob.glob(self.computer_partition_root_path + "/../*/bin/*")))
self.assertTrue(bin_programs) # just to check the glob was correct.
self.assertNotIn('runUnitTest', bin_programs)
......@@ -352,7 +358,7 @@ class TestZopeNodeParameterOverride(ERP5InstanceTestCase, TestPublishedURLIsReac
storage["storage"] = "root"
storage["server"] = zeo_addr
with open('%s/etc/zope-%s.conf' % (partition, zope)) as f:
conf = map(str.strip, f.readlines())
conf = list(map(str.strip, f.readlines()))
i = conf.index("<zodb_db root>") + 1
conf = iter(conf[i:conf.index("</zodb_db>", i)])
for line in conf:
......@@ -361,23 +367,23 @@ class TestZopeNodeParameterOverride(ERP5InstanceTestCase, TestPublishedURLIsReac
if line == '</zeoclient>':
break
checkParameter(line, storage)
for k, v in storage.iteritems():
for k, v in six.iteritems(storage):
self.assertIsNone(v, k)
del storage
else:
checkParameter(line, zodb)
for k, v in zodb.iteritems():
for k, v in six.iteritems(zodb):
self.assertIsNone(v, k)
partition = self.getComputerPartitionPath('zope-a')
for zope in xrange(3):
for zope in range(3):
checkConf({
"cache-size-bytes": "20MB",
}, {
"cache-size": "50MB",
})
partition = self.getComputerPartitionPath('zope-bb')
for zope in xrange(5):
for zope in range(5):
checkConf({
"cache-size-bytes": "500MB" if zope else 1<<20,
}, {
......
......@@ -26,10 +26,11 @@
#
##############################################################################
from __future__ import absolute_import
import os
import json
import glob
import urlparse
import six.moves.urllib.parse
import socket
import sys
import time
......@@ -60,6 +61,7 @@ class MariaDBTestCase(ERP5InstanceTestCase):
@classmethod
def _getInstanceParameterDict(cls):
# type: () -> dict
return {
'tcpv4-port': 3306,
'max-connection-count': 5,
......@@ -75,12 +77,14 @@ class MariaDBTestCase(ERP5InstanceTestCase):
@classmethod
def getInstanceParameterDict(cls):
# type: () -> dict
return {'_': json.dumps(cls._getInstanceParameterDict())}
def getDatabaseConnection(self):
# type: () -> MySQLdb.connections.Connection
connection_parameter_dict = json.loads(
self.computer_partition.getConnectionParameterDict()['_'])
db_url = urlparse.urlparse(connection_parameter_dict['database-list'][0])
db_url = six.moves.urllib.parse.urlparse(connection_parameter_dict['database-list'][0])
self.assertEqual('mysql', db_url.scheme)
self.assertTrue(db_url.path.startswith('/'))
......@@ -91,12 +95,15 @@ class MariaDBTestCase(ERP5InstanceTestCase):
host=db_url.hostname,
port=db_url.port,
db=database_name,
use_unicode=True,
charset='utf8mb4'
)
class TestCrontabs(MariaDBTestCase, CrontabMixin):
def test_full_backup(self):
# type: () -> None
self._executeCrontabAtDate('mariadb-backup', '2050-01-01')
with gzip.open(
os.path.join(
......@@ -106,10 +113,11 @@ class TestCrontabs(MariaDBTestCase, CrontabMixin):
'mariadb-full',
'20500101000000.sql.gz',
),
'r') as dump:
'rt') as dump:
self.assertIn('CREATE TABLE', dump.read())
def test_logrotate_and_slow_query_digest(self):
# type: () -> None
# slow query digest needs to run after logrotate, since it operates on the rotated
# file, so this tests both logrotate and slow query digest.
......@@ -148,7 +156,7 @@ class TestCrontabs(MariaDBTestCase, CrontabMixin):
'slowquery_digest',
'slowquery_digest.txt-2050-01-01.xz',
)
with lzma.open(slow_query_report, 'r') as f:
with lzma.open(slow_query_report, 'rt') as f:
# this is the hash for our "select sleep(n)" slow query
self.assertIn("ID 0xF9A57DD5A41825CA", f.read())
......@@ -170,7 +178,7 @@ class TestCrontabs(MariaDBTestCase, CrontabMixin):
subprocess.check_output('faketime 2050-01-01 %s' % check_slow_query_promise_plugin['command'], shell=True)
self.assertEqual(
error_context.exception.output,
"""\
b"""\
Threshold is lower than expected:
Expected total queries : 1.0 and current is: 2
Expected slowest query : 0.1 and current is: 3
......@@ -179,6 +187,7 @@ Expected slowest query : 0.1 and current is: 3
class TestMariaDB(MariaDBTestCase):
def test_utf8_collation(self):
# type: () -> None
cnx = self.getDatabaseConnection()
with contextlib.closing(cnx):
cnx.query(
......@@ -199,11 +208,12 @@ class TestMariaDB(MariaDBTestCase):
"""
select * from test_utf8_collation where col1 = "a"
""")
self.assertEqual((('à',),), cnx.store_result().fetch_row(maxrows=2))
self.assertEqual(((u'à',),), cnx.store_result().fetch_row(maxrows=2))
class TestMroonga(MariaDBTestCase):
def test_mroonga_plugin_loaded(self):
# type: () -> None
cnx = self.getDatabaseConnection()
with contextlib.closing(cnx):
cnx.query("show plugins")
......@@ -213,6 +223,7 @@ class TestMroonga(MariaDBTestCase):
plugins)
def test_mroonga_normalize_udf(self):
# type: () -> None
# example from https://mroonga.org/docs/reference/udf/mroonga_normalize.html#usage
cnx = self.getDatabaseConnection()
with contextlib.closing(cnx):
......@@ -220,7 +231,8 @@ class TestMroonga(MariaDBTestCase):
"""
SELECT mroonga_normalize("ABCDあぃうぇ㍑")
""")
self.assertEqual((('abcdあぃうぇリットル',),),
# XXX this is returned as bytes by mroonga/mariadb (this might be a bug)
self.assertEqual(((u'abcdあぃうぇリットル'.encode('utf-8'),),),
cnx.store_result().fetch_row(maxrows=2))
if 0:
......@@ -233,10 +245,11 @@ class TestMroonga(MariaDBTestCase):
"""
SELECT mroonga_normalize("aBcDあぃウェ㍑", "NormalizerMySQLUnicodeCIExceptKanaCIKanaWithVoicedSoundMark")
""")
self.assertEqual((('ABCDあぃうぇ㍑',),),
self.assertEqual(((u'ABCDあぃうぇ㍑'.encode('utf-8'),),),
cnx.store_result().fetch_row(maxrows=2))
def test_mroonga_full_text_normalizer(self):
# type: () -> None
# example from https://mroonga.org//docs/tutorial/storage.html#how-to-specify-the-normalizer
cnx = self.getDatabaseConnection()
with contextlib.closing(cnx):
......@@ -269,11 +282,12 @@ class TestMroonga(MariaDBTestCase):
WHERE MATCH (content) AGAINST ("+ブラック" IN BOOLEAN MODE)
""")
self.assertEqual(
((datetime.date(2013, 4, 23), 'ブラックコーヒーを飲んだ。'),),
((datetime.date(2013, 4, 23), u'ブラックコーヒーを飲んだ。'),),
cnx.store_result().fetch_row(maxrows=2),
)
def test_mroonga_full_text_normalizer_TokenBigramSplitSymbolAlphaDigit(self):
# type: () -> None
# Similar to as ERP5's testI18NSearch with erp5_full_text_mroonga_catalog
cnx = self.getDatabaseConnection()
with contextlib.closing(cnx):
......@@ -317,11 +331,12 @@ class TestMroonga(MariaDBTestCase):
self.assertEqual(((1,),), cnx.store_result().fetch_row(maxrows=2))
def test_mroonga_full_text_stem(self):
# type: () -> None
# example from https://mroonga.org//docs/tutorial/storage.html#how-to-specify-the-token-filters
cnx = self.getDatabaseConnection()
with contextlib.closing(cnx):
cnx.query("SELECT mroonga_command('register token_filters/stem')")
self.assertEqual((('true',),), cnx.store_result().fetch_row(maxrows=2))
self.assertEqual(((b'true',),), cnx.store_result().fetch_row(maxrows=2))
cnx.query(
"""
CREATE TABLE memos (
......
......@@ -16,6 +16,7 @@
# See COPYING file for full licensing terms.
# See https://www.nexedi.com/licensing for rationale and options.
from __future__ import absolute_import
import json
import os.path
import unittest
......
......@@ -49,7 +49,7 @@ WebTest = 2.0.33
soupsieve = 1.8
#gitdb2 = 2.0.4
#smmap2 = 2.0.4
waitress = 1.2.1
waitress = 1.4.4
z3c.etestbrowser = 2.0.1
zope.testbrowser = 5.3.2
WSGIProxy2 = 0.4.6
......
......@@ -10,6 +10,9 @@ parts =
eggs
instance-cfg
[python]
part = python3
# eggs for instance.cfg
[eggs]
recipe = zc.recipe.egg
......@@ -22,7 +25,7 @@ eggs =
<= go-git-package
go.importpath = lab.nexedi.com/nexedi/galene
repository = https://lab.nexedi.com/nexedi/galene.git
revision = galene-0.3.3
revision = galene-0.4
[gowork]
install =
......
......@@ -15,7 +15,7 @@
[template-cfg]
filename = instance.cfg.in
md5sum = 2e4bed8c7b78c410b28f8becf81da596
md5sum = 6f5acc546a7e9ad502d5fe586a3c3072
[template_nginx_conf]
_update_hash_filename_ = templates/nginx_conf.in
......
......@@ -113,6 +113,7 @@ template =
. ${hugo:go-environment}
cd ${basedirectory:data}/${slap-configuration:configuration.site}
if [ -d "public" ]; then rm -Rf public; fi
export TMPDIR=${hugo:path-tmp}
hugo && hugo server --bind=${hugo:ip} --port=${hugo:hugo-port} --baseURL=${hugo-frontend:connection-secure_access} --appendPort=false
[hugo-server-service]
......
......@@ -43,12 +43,12 @@ KVM instance parameters:
- nbd2-port (default: 1024)
- nbd2-host
- ram-size (default: 1024)
- disk-size = (default: 10)
- ram-size (default: 4096)
- disk-size = (default: 40)
- disk-type (default: virtio)
Disk size and Disk type are used if no virtual hard drive is specified.
- cpu-count (default: 1)
- cpu-count (default: 2)
- cpu-options
cpu-option is a string: [cores=cores][,threads=threads][,sockets=sockets][,maxcpus=maxcpus]
- numa
......
......@@ -15,15 +15,15 @@
[template]
filename = instance.cfg.in
md5sum = b129c9b2a614563d3f7c3f9e906d59f2
md5sum = f2b0f1ed27148504f220e06eaceff935
[template-kvm]
filename = instance-kvm.cfg.jinja2
md5sum = 78c3b69a2b18106cd6e289732116881d
md5sum = 17c58f74d1ed4cb7dce11bf9af71dd33
[template-kvm-cluster]
filename = instance-kvm-cluster.cfg.jinja2.in
md5sum = 59b92e1300aad4e9b116c532caf7d042
md5sum = fcb35c32ef985e3d69a7914711675dcc
[template-kvm-resilient]
filename = instance-kvm-resilient.cfg.jinja2
......@@ -55,7 +55,7 @@ md5sum = b7e87479a289f472b634a046b44b5257
[template-kvm-run]
filename = template/template-kvm-run.in
md5sum = a502782244d1be536b732ebb40725f47
md5sum = f840e8b7af83982525f66d7ec12b7085
[template-kvm-controller]
filename = template/kvm-controller-run.in
......
......@@ -225,7 +225,7 @@
"title": "RAM size",
"description": "RAM size, in MB.",
"type": "integer",
"default": 1024,
"default": 4096,
"minimum": 1024,
"multipleOf": 512
},
......@@ -233,6 +233,7 @@
"title": "Maximum RAM size, in MB",
"description": "Define the maximum size of the memory. The size is in MB and should be a multiple of 512. Defaults to ram-size + 512",
"type": "integer",
"default": 4608,
"minimum": 1024,
"multipleOf": 512
},
......@@ -254,7 +255,7 @@
"title": "Disk size",
"description": "Disk size, in GB.",
"type": "integer",
"default": 10,
"default": 40,
"minimum": 1
},
"disk-format": {
......@@ -313,6 +314,7 @@
"title": "CPU count",
"description": "Number of CPU cores.",
"type": "integer",
"default": 2,
"minimum": 1
},
"cpu-max-count": {
......
......@@ -77,14 +77,14 @@ config-authorized-key = {{ dumps(slapparameter_dict.get('authorized-keys') | joi
{% endif -%}
config-nbd-port = {{ dumps(kvm_parameter_dict.get('nbd-port', 1024)) }}
config-nbd2-port = {{ dumps(kvm_parameter_dict.get('nbd-port2', 1024)) }}
config-ram-size = {{ dumps(kvm_parameter_dict.get('ram-size', 1024)) }}
config-ram-max-size = {{ dumps(kvm_parameter_dict.get('ram-max-size', int(kvm_parameter_dict.get('ram-size', 1024)) + 512)) }}
config-ram-size = {{ dumps(kvm_parameter_dict.get('ram-size', 4096)) }}
config-ram-max-size = {{ dumps(kvm_parameter_dict.get('ram-max-size', int(kvm_parameter_dict.get('ram-size', 4096)) + 512)) }}
config-enable-device-hotplug = {{ dumps(kvm_parameter_dict.get('enable-device-hotplug', False)) }}
config-ram-hotplug-slot-size = {{ dumps(kvm_parameter_dict.get('ram-hotplug-slot-size', 512)) }}
config-disk-size = {{ dumps(kvm_parameter_dict.get('disk-size', 10)) }}
config-disk-size = {{ dumps(kvm_parameter_dict.get('disk-size', 40)) }}
config-disk-type = {{ dumps(kvm_parameter_dict.get('disk-type', 'virtio')) }}
config-disk-format = {{ dumps(kvm_parameter_dict.get('disk-format', 'qcow2')) }}
config-cpu-count = {{ dumps(kvm_parameter_dict.get('cpu-count', 1)) }}
config-cpu-count = {{ dumps(kvm_parameter_dict.get('cpu-count', 2)) }}
config-cpu-max-count = {{ dumps(kvm_parameter_dict.get('cpu-max-count', int(kvm_parameter_dict.get('cpu-count', 1)) + 1)) }}
{{ setconfig('numa', kvm_parameter_dict.get('numa', '')) }}
{{ setconfig('machine-options', kvm_parameter_dict.get('machine-options', '')) }}
......
......@@ -13,7 +13,7 @@
"title": "RAM size",
"description": "RAM size, in MB.",
"type": "integer",
"default": 1024,
"default": 4096,
"minimum": 1024,
"multipleOf": 512
},
......@@ -21,6 +21,7 @@
"title": "Maximum RAM size, in MB",
"description": "Define the maximum size of the memory. The size is in MB and should be a multiple of 512. Defaults to ram-size + 512",
"type": "integer",
"default": 4608,
"minimum": 1024,
"multipleOf": 512
},
......@@ -42,7 +43,7 @@
"title": "Disk size",
"description": "Disk size, in GB.",
"type": "integer",
"default": 10,
"default": 40,
"minimum": 1
},
"disk-type": {
......@@ -101,7 +102,8 @@
"title": "CPU count",
"description": "Number of CPU cores.",
"type": "integer",
"minimum": 1
"minimum": 1,
"default": 2
},
"cpu-max-count": {
"title": "Maximum CPU amount",
......
......@@ -22,8 +22,8 @@
{% set boot_image_url_list_enabled = 'boot-image-url-list' in slapparameter_dict %}
{% set boot_image_url_select_enabled = 'boot-image-url-select' in slapparameter_dict %}
{% set bootstrap_script_url = slapparameter_dict.get('bootstrap-script-url') -%}
{% set cpu_max_count = dumps(slapparameter_dict.get('cpu-max-count', int(slapparameter_dict.get('cpu-count', 1)) + 1)) %}
{% set ram_max_size = dumps(slapparameter_dict.get('ram-max-size', int(slapparameter_dict.get('ram-size', 1024)) + 512)) %}
{% set cpu_max_count = dumps(slapparameter_dict.get('cpu-max-count', int(slapparameter_dict.get('cpu-count', 2)) + 1)) %}
{% set ram_max_size = dumps(slapparameter_dict.get('ram-max-size', int(slapparameter_dict.get('ram-size', 4096)) + 512)) %}
{% set extends_list = [] -%}
{% set part_list = [] -%}
......@@ -441,8 +441,8 @@ socket-path = ${kvm-controller-parameter-dict:socket-path}
smp-max-count = {{ cpu_max_count }}
ram-max-size = {{ ram_max_size }}
{%- if enable_device_hotplug %}
init-ram-size = 1024
init-smp-count = 1
init-ram-size = 4096
init-smp-count = 2
{%- else %}
init-ram-size = ${kvm-controller-parameter-dict:ram-size}
init-smp-count = ${kvm-controller-parameter-dict:cpu-count}
......@@ -1068,14 +1068,14 @@ nbd2-host =
boot-image-url-list =
enable-device-hotplug = False
ram-size = 1024
ram-size = 4096
ram-hotplug-slot-size = 512
disk-size = 10
disk-size = 40
disk-type = virtio
disk-format = qcow2
disk-device-path =
cpu-count = 1
cpu-count = 2
disk-cache =
disk-aio =
auto-ballooning = True
......
......@@ -82,7 +82,7 @@ extra-context =
raw dash_executable_location ${dash:location}/bin/dash
raw dnsresolver_executable ${buildout:bin-directory}/dnsresolver
raw dcron_executable_location ${dcron:location}/sbin/crond
raw debian_amd64_netinst_location ${debian-amd64-buster-netinst.iso:target}
raw debian_amd64_netinst_location ${debian-amd64-bullseye-netinst.iso:target}
raw whitelist_domains_default ${whitelist-domains-default:location}/${whitelist-domains-default:filename}
raw whitelist_firewall_download_controller ${whitelist-firewall-download-controller:target}
raw image_download_controller ${image-download-controller:target}
......
......@@ -277,6 +277,8 @@ kvm_argument_list = [qemu_path,
'-pidfile', pid_file_path, '-msg', 'timestamp=on',
'-D', logfile,
'-nodefaults',
# switch to tablet mode for the mouse to have it synced with a client, see https://wiki.gentoo.org/wiki/QEMU/Options#USB
'-usbdevice', 'tablet',
]
for disk_info in disk_info_list:
kvm_argument_list += (
......
......@@ -230,7 +230,7 @@ class TestMemoryManagement(InstanceTestCase, KvmMixin):
def test(self):
kvm_pid_1, info_list = self.getKvmProcessInfo(['-smp', '-m'])
self.assertEqual(
['1,maxcpus=2', '1024M,slots=128,maxmem=1536M'],
['2,maxcpus=3', '4096M,slots=128,maxmem=4608M'],
info_list
)
self.rerequestInstance({
......@@ -264,7 +264,7 @@ class TestMemoryManagement(InstanceTestCase, KvmMixin):
kvm_pid_1, info_list = self.getKvmProcessInfo(['-smp', '-m'])
self.assertEqual(
['1,maxcpus=2', '1024M,slots=128,maxmem=1536M'],
['2,maxcpus=3', '4096M,slots=128,maxmem=4608M'],
info_list
)
self.assertEqual(
......@@ -275,15 +275,15 @@ class TestMemoryManagement(InstanceTestCase, KvmMixin):
parameter_dict = {
'enable-device-hotplug': 'true',
# to avoid restarts the max RAM and CPU has to be static
'ram-max-size': '2048',
'cpu-max-count': '4',
'ram-max-size': '8192',
'cpu-max-count': '6',
}
self.rerequestInstance(parameter_dict)
self.slap.waitForInstance(max_retry=2)
kvm_pid_2, info_list = self.getKvmProcessInfo(['-smp', '-m'])
self.assertEqual(
['1,maxcpus=4', '1024M,slots=128,maxmem=2048M'],
['2,maxcpus=6', '4096M,slots=128,maxmem=8192M'],
info_list
)
self.assertEqual(
......@@ -292,21 +292,21 @@ class TestMemoryManagement(InstanceTestCase, KvmMixin):
)
self.assertNotEqual(kvm_pid_1, kvm_pid_2, "Unexpected: KVM not restarted")
parameter_dict.update(**{
'ram-size': '1536',
'cpu-count': '2'
'ram-size': '5120',
'cpu-count': '4'
})
self.rerequestInstance(parameter_dict)
self.slap.waitForInstance(max_retry=10)
kvm_pid_3, info_list = self.getKvmProcessInfo(['-smp', '-m'])
self.assertEqual(
['1,maxcpus=4', '1024M,slots=128,maxmem=2048M'],
['2,maxcpus=6', '4096M,slots=128,maxmem=8192M'],
info_list
)
self.assertEqual(kvm_pid_2, kvm_pid_3, "Unexpected: KVM restarted")
self.assertEqual(
getHotpluggedCpuRamValue(),
{'cpu_count': 1, 'ram_mb': 512}
{'cpu_count': 2, 'ram_mb': 1024}
)
......
......@@ -2,7 +2,7 @@
This software release is used to run integration test of slapos softwares.
The approach is to use setuptools' integrated test runner, `python setup.py test`, to run tests.
The approach is to use python unittest runner to run tests.
The `python` used in this command will be a `zc.recipe.egg` interpreter with
all eggs pre-installed by this software release.
......@@ -45,22 +45,25 @@ SLAPOS_TEST_DEBUG=1 python_for_test -m unittest discover -v
## Environment variables
The `environment-script` set all variabels except `SLAPOS_TEST_DEBUG` and `SLAPOS_TEST_VERBOSE` for you, but for reference, here is the list of variables which control the test runner:
The `environment-script` set all variables except `SLAPOS_TEST_DEBUG`, `SLAPOS_TEST_VERBOSE`, `SLAPOS_TEST_SKIP_SOFTWARE_CHECK` and `SLAPOS_TEST_SKIP_SOFTWARE_REBUILD` for you, but for reference, here is the list of variables which control the test runner:
| Variable | Description |
| --- | --- |
| `SLAPOS_TEST_VERBOSE` | If set to 1, debugging output will be printed on console. This also adjust the log level of python process running tests. When running on test nodes, this is not set, so keep this difference in mind if test rely on python logger |
| `SLAPOS_TEST_DEBUG` | If set to 1, `slapos node instance` and `slapos node software` will run with `--buildout-debug` flag, which will invoke python debugger on error. |
| `SLAPOS_TEST_SKIP_SOFTWARE_CHECK` | If set to 1, software checks will not be performed. This can be used to speed up running tests locally, as checking software is an expensive operation. |
| `SLAPOS_TEST_SKIP_SOFTWARE_REBUILD` | If set to 1, software will not be rebuilt before running the test. The default is to use `slapos node software --all`, which unconditionally rebuild all softwares and to use `slapos node software`, which installs the software only if it's not yet available when the environment variable is set to 1. |
| `SLAPOS_TEST_IPV6` | ipv6 used by this instance. Usually you want to use a global address here to be able to connect to this instance. |
| `SLAPOS_TEST_IPV4` | ipv4 used by this instance. |
| `SLAPOS_TEST_WORKING_DIR` | Path to use as a working directory to hold the standalone SlapOS. |
| `SLAPOS_TEST_SHARED_PART_LIST` | A `:` separated of paths to look for already installed shared parts. The SlapOS used in the test will not write in these, but will use a dedicated directory in `$SLAPOS_TEST_WORKING_DIR` |
| `SLAPOS_TEST_VERBOSE` | If set to 1, debugging output will be printed on console. This also adjust the log level of python process running tests. When running on test nodes, this is not set, so keep this difference in mind if test rely on python logger |
| `SLAPOS_TEST_DEBUG` | If set to 1, `slapos node instance` and `slapos node software` will run with `--buildout-debug` flag, which will invoke python debugger on error. |
## Frequently Asked Questions
### Where to find docs about the testing framework ?
Please refere to the docstrings from `slapos.testing` module, from `slapos.core` package.
Please refer to the docstrings from `slapos.testing` module, from `slapos.core` package.
This uses python unittest module from standard library, especially the setup hooks:
- `setUpModule` installs the software and perform some static checks
......@@ -70,7 +73,7 @@ This uses python unittest module from standard library, especially the setup hoo
### Can I run slapos commands to debug ?
The standalone slapos is created in `$SLAPOS_TEST_WORKING_DIR`. In this directory you will have a `bin/slapos` that you can run to start or stop services.
It's fine to use this command during development, but to programatically interract with the environment within the test, the recommended approach is to use supervisor XML-RPC API.
It's fine to use this command during development, but to programmatically interact with the environment within the test, the recommended approach is to use supervisor XML-RPC API.
### How to use a development version of `slapos.cookbook` ?
......@@ -83,3 +86,7 @@ At the end of the test, a snapshot of the slapos instances is created. Sometimes
Most of the time, problems are because on test nodes paths are very long. One advanced technique to reproduce the problem in your development environment is to set `SLAPOS_TEST_WORKING_DIR` environment variable to a path with the same length as the ones on test nodes.
One way to make instances uses a slightly shorter path is to define `__partition_reference__` class attribute, so that the instances uses this as prefix instead of the class name.
### Can I run only specific test ?
Yes, please refer to python unittest documentation. For example, you can use `python_for_test -m unittest -v test_module.TestClass.test_function` to run only test function.
......@@ -337,6 +337,7 @@ tests =
theia ${slapos.test.theia-setup:setup}
metabase ${slapos.test.metabase-setup:setup}
nginx-push-stream ${slapos.test.nginx-push-stream-setup:setup}
erp5 ${slapos.test.erp5-setup:setup}
###
${:extra}
......@@ -345,7 +346,6 @@ extra =
# You should not add more lines here.
backupserver ${slapos.test.backupserver-setup:setup}
caddy-frontend ${slapos.test.caddy-frontend-setup:setup}
erp5 ${slapos.test.erp5-setup:setup}
upgrade_erp5 ${slapos.test.upgrade_erp5-setup:setup}
htmlvalidatorserver ${slapos.test.htmlvalidatorserver-setup:setup}
slapos-master ${slapos.test.slapos-master-setup:setup}
......
......@@ -36,6 +36,6 @@ mode = 0644
depends = ${caucase-jinja2-library-eggs:eggs}
[versions]
caucase = 0.9.10
caucase = 0.9.12
pem = 21.1.0
PyJWT = 1.7.1
......@@ -94,7 +94,7 @@ md5sum = d6166515fda7b09df754672536b131be
[template-haproxy-cfg]
filename = haproxy.cfg.in
md5sum = 3f4f7e49c504cbf610fc5dc462713dfc
md5sum = 9d61e05c8578e0f17e349603ccaaf52c
[template-rsyslogd-cfg]
filename = rsyslogd.cfg.in
......
......@@ -172,7 +172,7 @@ listen family_{{ name }}
# logs
capture request header Referer len 512
capture request header User-Agent len 512
log-format "%{+Q}o %{-Q}ci - - [%trg] %r %ST %B %{+Q}[capture.req.hdr(0)] %{+Q}[capture.req.hdr(1)] %Tt"
log-format "%{+Q}o %{-Q}ci - - [%trg] %r %ST %B %{+Q}[capture.req.hdr(0)] %{+Q}[capture.req.hdr(1)] %Ta"
{% for outer_prefix, inner_prefix in family_path_routing_dict.get(name, []) + path_routing_list %}
{% set outer_prefix = outer_prefix.strip('/') -%}
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment