Skip to content

slapos
slapos
Commit 4371e39c authored by Jérome Perrin's avatar Jérome Perrin
Browse files
Options

Use nxdtest in SlapOS tests 

 * switch software/slapos-testing (`SlapOS.Eggs.UnitTest-*`) to use nxdtest and update README to describe how to run tests locally with nxdtest's runTestSuite. Switch tests to `python -m unittest` (or slight variations) instead of the deprecated `python setup.py`, except a few tests which only pass when invoked with `setup.py`.
 * switch software/slapos-sr-testing (`SlapOS.SoftwareReleases.IntegrationTest-*`) to use nxdtest and update README to describe how to run tests locally with nxdtest's runTestSuite. Switch all tests to `python -m unittest discover` intead of the deprecated `python setup.py` - which was especially problematic in these tests, because it sets `$PYTHONPATH` which affect subprocesses ran by the tests. In follow up commits we'll drop the workarounds for `$PYTHONPATH`.

`EggTestSuite` becomes unused and will be removed in follow up changes.

See merge request nexedi/slapos!862
parents d29781b2 1536ad2e
Hide whitespace changes
Inline Side-by-side
Showing with 249 additions and 122 deletions
component/pygolang/buildout.cfg
View file @ 4371e39c
......@@ -37,6 +37,14 @@ interpreter = python
# interpreter code that buildout generates cannot process `-m pytest --<pytest-option>`
# -> use pymain from gpython to workaround that.
initialization =
# minimal support for -O argument.
if '-O' in sys.argv:
import os
with open(sys.argv[0]) as f:
executable = f.readlines()[0][2:-1]
sys.argv.remove('-O')
os.execve(executable, sys.argv, dict(os.environ, PYTHONOPTIMIZE='1'))
# set sys.executable to self, so that subprocess and friends go through us
# and this way spawn children with correct sys.path where all eggs that
# parent have are present. TODO consider migrating this into pymain
......
software/slapos-sr-testing/README.md
View file @ 4371e39c
......@@ -31,15 +31,15 @@ slapos request --node=computer_guid=$COMP $INSTANCE_NAME $SR
# and load this script to set environment variables
source ( environment-script from step above )
# Clone a working copy somewhere
cd ~/srv/runner/project/
git clone https://lab.nexedi.com/nexedi/slapos.git slapos_work
# The source code is a git clone working copy on the instance
cd ~/srv/runner/instance/slappartXXX/parts/slapos/
# change directory to the directory containing test for this software
cd ~/srv/runner/project/slapos_work/software/helloworld/test/
cd ./software/helloworld/test/
# make changes to test code or profile
# run test (with debugging features activated)
SLAPOS_TEST_DEBUG= 1 python_for_test setup.py test
# run test for helloworld software release (with debugging features activated)
SLAPOS_TEST_DEBUG=1 python_for_test -m unittest discover -v
```
## Environment variables
......
software/slapos-sr-testing/buildout.hash.cfg
View file @ 4371e39c
......@@ -15,4 +15,4 @@
[template]
filename = instance.cfg
md5sum = 4246cde0a27138e057ba1635cc621edf
md5sum = 9639d0c0e161c094454808fb95fc9781
software/slapos-sr-testing/instance.cfg
View file @ 4371e39c
[buildout]
extends = {{ nxdtest_instance }}
parts =
slapos-test-runner
runTestSuite
publish
eggs-directory = {{ buildout['eggs-directory'] }}
......@@ -26,46 +27,53 @@ repository = {{ slapos_location }}
[directory]
recipe = slapos.cookbook:mkdirectory
bin = ${buildout:directory}/bin
working-dir = ${buildout:directory}/tmp
etc = ${buildout:directory}/etc
var = ${buildout:directory}/var
test-working-dir = ${buildout:directory}/tmp
nxdtest-working-dir = ${:var}/nxdtest
[slapos-test-runner]
recipe = slapos.cookbook:wrapper
wrapper-path = ${directory:bin}/runTestSuite
command-line =
{{ buildout['bin-directory'] }}/runTestSuite
--python_interpreter={{ buildout['bin-directory'] }}/{{ interpreter }}
--source_code_path_list={{ ','.join(tests.splitlines()) }}
[slapos-test-runner-nxdtest-environment.sh]
recipe = slapos.recipe.template:jinja2
rendered = ${directory:etc}/${:_buildout_section_name_}
template = inline:
export PATH={{ buildout['bin-directory'] }}:{{ curl_location }}/bin/:{{ faketime_location }}/bin/:{{ openssl_location }}/bin/:/usr/bin/:/bin
export SLAPOS_TEST_IPV4=${slap-configuration:ipv4-random}
export SLAPOS_TEST_IPV6=${slap-configuration:ipv6-random}
export SLAPOS_TEST_WORKING_DIR=${directory:test-working-dir}
environment =
PATH=${slapos-test-runner-environment:PATH}
SLAPOS_TEST_IPV4=${slapos-test-runner-environment:SLAPOS_TEST_IPV4}
SLAPOS_TEST_IPV6=${slapos-test-runner-environment:SLAPOS_TEST_IPV6}
SLAPOS_TEST_WORKING_DIR=${slapos-test-runner-environment:SLAPOS_TEST_WORKING_DIR}
[slapos-test-runner-dot-nxdtest]
recipe = slapos.recipe.template:jinja2
rendered = ${:workdir}/.nxdtest
workdir = ${directory:nxdtest-working-dir}
template = inline:
{% for test in tests.splitlines() %}
TestCase(
{{ repr(test.split()[0]) }},
[ {{ repr(interpreter) }}, '-m', 'unittest', 'discover', '-v'],
cwd={{ repr(test.split()[1]) }},
summaryf=UnitTest.summary,
)
{% endfor %}
[slapos-test-runner-environment]
PATH = {{ buildout['bin-directory'] }}:{{ curl_location }}/bin/:{{ faketime_location }}/bin/:{{ openssl_location }}/bin/:/usr/bin/:/bin
SLAPOS_TEST_IPV4 = ${slap-configuration:ipv4-random}
SLAPOS_TEST_IPV6 = ${slap-configuration:ipv6-random}
SLAPOS_TEST_WORKING_DIR = ${directory:working-dir}
[runTestSuite]
env.sh = ${slapos-test-runner-nxdtest-environment.sh:rendered}
workdir = ${slapos-test-runner-dot-nxdtest:workdir}
[slapos-local-development-environment.sh]
recipe = slapos.recipe.template:jinja2
rendered = ${directory:etc}/${:_buildout_section_name_}
template = inline:
export PATH=${slapos-test-runner-environment:PATH}:$PATH
export SLAPOS_TEST_IPV4=${slapos-test-runner-environment:SLAPOS_TEST_IPV4}
export SLAPOS_TEST_IPV6=${slapos-test-runner-environment:SLAPOS_TEST_IPV6}
export SLAPOS_TEST_WORKING_DIR=${slapos-test-runner-environment:SLAPOS_TEST_WORKING_DIR}
source ${slapos-test-runner-nxdtest-environment.sh:rendered}
{% set shared_part_list = [] %}
{% for shared_part in buildout['shared-part-list'].splitlines() -%}
{% do shared_part_list.append(shared_part) %}
{%- endfor %}
export SLAPOS_TEST_SHARED_PART_LIST={{ os.pathsep.join(shared_part_list) }}
echo "Environment loaded."
echo "The embedded SlapOS is in $SLAPOS_TEST_WORKING_DIR"
echo "To work on a test, execute:"
echo " SLAPOS_TEST_DEBUG=1 {{ interpreter }} setup.py test"
echo "from test folder"
echo " SLAPOS_TEST_DEBUG=1 {{ interpreter }} -m unittest discover -v"
echo "from a folder containing software release test."
echo
[publish]
......
software/slapos-sr-testing/software-py3.cfg
View file @ 4371e39c
......@@ -12,9 +12,9 @@ eggs +=
[template]
extra =
${slapos.test.helloworld-setup:setup}
${slapos.test.monitor-setup:setup}
${slapos.test.plantuml-setup:setup}
${slapos.test.powerdns-setup:setup}
${slapos.test.proftpd-setup:setup}
${slapos.test.repman-setup:setup}
helloworld ${slapos.test.helloworld-setup:setup}
monitor ${slapos.test.monitor-setup:setup}
plantuml ${slapos.test.plantuml-setup:setup}
powerdns ${slapos.test.powerdns-setup:setup}
proftpd ${slapos.test.proftpd-setup:setup}
repman ${slapos.test.repman-setup:setup}
software/slapos-sr-testing/software.cfg
View file @ 4371e39c
......@@ -13,11 +13,12 @@ extends =
../../component/python-backports-lzma/buildout.cfg
../../stack/slapos.cfg
../../stack/nxdtest.cfg
./buildout.hash.cfg
parts =
eggs
eggs/scripts
slapos-cookbook
template
......@@ -167,8 +168,8 @@ egg = slapos.core
setup = ${slapos.core-repository:location}
[eggs]
recipe = zc.recipe.egg
eggs =
<= python-interpreter
eggs +=
${lxml-python:egg}
${slapos.core-setup:egg}
${pillow-python:egg}
......@@ -202,21 +203,24 @@ eggs =
${slapos.test.dream-setup:egg}
${slapos.test.metabase-setup:egg}
${slapos.test.repman-setup:egg}
entry-points =
runTestSuite=erp5.util.testsuite:runTestSuite
scripts =
runTestSuite
slapos
supervisorctl
supervisord
interpreter=
python_for_test
# We don't name this interpreter `python`, so that when we run slapos node
# software, installation scripts running `python` use a python without any
# custom eggs pre-installed, not our special python interpreter.
interpreter = python_for_test
# patches for eggs
patch-binary = ${patch:location}/bin/patch
PyPDF2-patches = ${:_profile_base_location_}/../../component/egg-patch/PyPDF2/0001-Custom-implementation-of-warnings.formatwarning-remo.patch#d25bb0f5dde7f3337a0a50c2f986f5c8
PyPDF2-patch-options = -p1
[eggs/scripts]
recipe = zc.recipe.egg
eggs = ${eggs:eggs}
scripts =
slapos
supervisord
[git-clone-repository]
recipe = slapos.recipe.build:gitclone
git-executable = ${git:location}/bin/git
......@@ -236,6 +240,7 @@ mode = 640
context =
section buildout buildout
import os os
key nxdtest_instance nxdtest-instance.cfg:rendered
key git_location git:location
key slapos_location slapos-repository:location
key interpreter eggs:interpreter
......@@ -243,29 +248,31 @@ context =
key openssl_location openssl-output:bin
key faketime_location faketime:location
key tests :tests
tests =
${slapos.test.kvm-setup:setup}
${slapos.test.slaprunner-setup:setup}
${slapos.test.metabase-setup:setup}
json-schemas ${slapos.cookbook-setup:setup}
kvm ${slapos.test.kvm-setup:setup}
slaprunner ${slapos.test.slaprunner-setup:setup}
metabase ${slapos.test.metabase-setup:setup}
${:extra}
extra =
${slapos.cookbook-setup:setup}
${slapos.test.backupserver-setup:setup}
${slapos.test.caddy-frontend-setup:setup}
${slapos.test.erp5-setup:setup}
${slapos.test.htmlvalidatorserver-setup:setup}
${slapos.test.slapos-master-setup:setup}
${slapos.test.re6stnet-setup:setup}
${slapos.test.seleniumserver-setup:setup}
${slapos.test.jstestnode-setup:setup}
${slapos.test.jupyter-setup:setup}
${slapos.test.nextcloud-setup:setup}
${slapos.test.turnserver-setup:setup}
${slapos.test.theia-setup:setup}
${slapos.test.grafana-setup:setup}
${slapos.test.gitlab-setup:setup}
${slapos.test.cloudooo-setup:setup}
${slapos.test.dream-setup:setup}
backupserver ${slapos.test.backupserver-setup:setup}
caddy-frontend ${slapos.test.caddy-frontend-setup:setup}
erp5 ${slapos.test.erp5-setup:setup}
htmlvalidatorserver ${slapos.test.htmlvalidatorserver-setup:setup}
slapos-master ${slapos.test.slapos-master-setup:setup}
re6stnet ${slapos.test.re6stnet-setup:setup}
seleniumserver ${slapos.test.seleniumserver-setup:setup}
jstestnode ${slapos.test.jstestnode-setup:setup}
jupyter ${slapos.test.jupyter-setup:setup}
nextcloud ${slapos.test.nextcloud-setup:setup}
turnserver ${slapos.test.turnserver-setup:setup}
theia ${slapos.test.theia-setup:setup}
grafana ${slapos.test.grafana-setup:setup}
gitlab ${slapos.test.gitlab-setup:setup}
cloudooo ${slapos.test.cloudooo-setup:setup}
dream ${slapos.test.dream-setup:setup}
repman ${slapos.test.repman-setup:setup}
[versions]
# slapos.core is used from the clone always
......
software/slapos-testing/README.md
View file @ 4371e39c
......@@ -2,13 +2,11 @@
This software release is used to run unit test of slapos eggs.
The approach is to use setuptools' integrated test runner, `python setup.py test`, to run tests.
The approach is to use nxdtest test runner, which will run tests for each
projects, as described in `.nxdtest` file.
The `python` used in this command will be a `zc.recipe.egg` interpreter with
all eggs pre-installed by this software release.
The results of this test suite running on Nexedi ERP5 are published as `SlapOS.Eggs.UnitTest-Master.Python3`
and `SlapOS.Eggs.UnitTest-Master.Python2`.
The results of this test suite running on Nexedi ERP5 are published as
`SlapOS.Eggs.UnitTest-Master.Python3` and `SlapOS.Eggs.UnitTest-Master.Python2`.
Here's an example session of how a developer could use this software release in
......@@ -23,21 +21,27 @@ INSTANCE_NAME=$COMP
slapos supply $SR $COMP
slapos node software
slapos request --node=node=$COMP $INSTANCE_NAME $SR
slapos request --node=computer_guid=$COMP $INSTANCE_NAME $SR
slapos node instance
# The path of a an environment script was published by slapos parameters, as
# "environment-script"
slapos request --node=computer_guid=$COMP $INSTANCE_NAME $SR
# sourcing the script in the shell configure all environment variables and
# print a message explaining how to run tests
source ( environment script from step above )
# The source code is a git clone working copy on the instance
cd ~/srv/runner/instance/slappart0/parts/slapos.core/
cd ~/srv/runner/instance/slappartXXX/parts/slapos.core/
# make some changes to the code
vim slapos/tests/client.py
# run tests, using bundled python intepreter with pre-installed eggs dependencies
SLAPOS_TEST_IPV6=::1 \
SLAPOS_TEST_IPV4=127.0.0.1 \
SLAPOS_TEST_VERBOSE=1 \
SLAPOS_TEST_DEBUG=1 \
~/srv/runner/instance/slappart0/software_release/bin/python_for_test setup.py test
# run slapos.core tests
runTestSuite --run slapos.core
# ... or run all eggs tests
runTestSuite
# when satified, commit changes
git add -p && git commit
......
software/slapos-testing/buildout.hash.cfg
View file @ 4371e39c
......@@ -15,4 +15,4 @@
[template]
filename = instance.cfg
md5sum = 2df601dd3ccb3ba38b3aee7243b7f8e5
md5sum = 3b1b386f6ad4c9ac50ab1f1e1384e751
software/slapos-testing/instance.cfg
View file @ 4371e39c
[buildout]
extends = ${nxdtest-instance.cfg:rendered}
parts =
phantomjs-wrapper
slapos-test-runner
runTestSuite
publish
eggs-directory = ${buildout:eggs-directory}
develop-eggs-directory = ${buildout:develop-eggs-directory}
......@@ -21,6 +22,8 @@ bin = $${buildout:directory}/bin
etc = $${buildout:directory}/etc
services = $${:etc}/run
srv = $${buildout:directory}/srv
var = $${buildout:directory}/var
nxdtest-working-dir = $${:var}/nxdtest
[download-source]
recipe = slapos.recipe.build:gitclone
......@@ -75,30 +78,127 @@ repository = ${slapos.rebootstrap-repository:location}
repository = ${rubygemsrecipe-repository:location}
[slapos-test-runner]
recipe = slapos.cookbook:wrapper
wrapper-path = $${create-directory:bin}/runTestSuite
command-line =
${buildout:bin-directory}/runTestSuite
--python_interpreter=${buildout:bin-directory}/${eggs:interpreter}
--source_code_path_list=$${kedifa:location},$${caucase:location},$${erp5.util:location},$${slapos.cookbook:location},$${slapos.core:location},$${slapos.recipe.build:location},$${slapos.recipe.cmmi:location},$${slapos.recipe.template:location},$${slapos.toolbox:location},$${slapos.libnetworkcache:location},$${slapos.rebootstrap:location},$${rubygemsrecipe:location}
# Notes about environment:
# * slapos.cookbook:wrapper does not seem to allow "extending" PATH. Tests
# needs ping, which is a setuid binary that cannot be installed via slapos
# way of building software without root access, so we keep "standard"
# /usr/bin and /bin in $PATH
# * SLAPOS_TEST_environment is safe to be used by tests, but there is no
# guarantee about free ports on IPV4 and IPV6
# * LOCAL_IPV4 is backward compatible, to be migrated, SLAPOS_TEST_IPV4
environment =
PATH=${coreutils:location}/bin:${curl:location}/bin:${openssl:location}/bin:${git:location}/bin:${libxslt:location}/bin:${socat:location}/bin:${lmsensors:location}/bin:${rsync:location}/bin/:${buildout:bin-directory}:/usr/bin/:/bin/
LOCAL_IPV4=$${slap-configuration:ipv4-random}
SLAPOS_TEST_IPV4=$${slap-configuration:ipv4-random}
SLAPOS_TEST_IPV6=$${slap-configuration:ipv6-random}
[phantomjs-wrapper]
recipe = slapos.cookbook:wrapper
command-line = ${phantomjs:location}/phantomjs-slapos
wrapper-path = $${create-directory:bin}/phantomjs
[slapos-test-runner-nxdtest-environment.sh]
recipe = slapos.recipe.template:jinja2
rendered = $${create-directory:etc}/$${:_buildout_section_name_}
template = inline:
export PATH=${coreutils:location}/bin:${curl:location}/bin:${openssl:location}/bin:${git:location}/bin:${libxslt:location}/bin:${socat:location}/bin:${lmsensors:location}/bin:${rsync:location}/bin/:${buildout:bin-directory}:$PATH
export SLAPOS_TEST_IPV4=$${slap-configuration:ipv4-random}
export SLAPOS_TEST_IPV6=$${slap-configuration:ipv6-random}
[slapos-test-runner-dot-nxdtest]
recipe = slapos.recipe.template:jinja2
rendered = $${:workdir}/.nxdtest
workdir = $${create-directory:nxdtest-working-dir}
template = inline:
TestCase(
"kedifa",
['python', '-m', 'unittest', 'discover', '-v'],
cwd="""$${kedifa:location}""",
summaryf=UnitTest.summary,
)
TestCase(
"caucase",
# XXX caucase uses 2to3 dynamically in setup.py, so it only supports
# runnning tests with python setup.py test
['python', 'setup.py', 'test'],
cwd="""$${caucase:location}""",
summaryf=UnitTest.summary,
)
TestCase(
"erp5.util",
['python', '-m', 'unittest', 'discover', '-v', '--start', 'erp5/tests/'],
cwd="""$${erp5.util:location}""",
summaryf=UnitTest.summary,
)
TestCase(
"slapos.cookbook",
# slapos/test contain both tests for recipes and tests for
# json schemas, we only care about recipe tests here, json
# schemas are tested in software/slapos-sr-testing
['python', '-m', 'unittest', 'discover', '-v', '--start', 'slapos/test/recipe'],
cwd="""$${slapos.cookbook:location}""",
summaryf=UnitTest.summary,
)
TestCase(
"slapos.core",
# ['python', '-m', 'unittest', 'discover', '-v'],
# XXX some test fail when running with unittest (slapos*.cfg.example
# files cannot be found with pkg_resources.resource_string), so we keep
# using setup.py test for now.
['python', 'setup.py', 'test'],
cwd="""$${slapos.core:location}""",
summaryf=UnitTest.summary,
)
TestCase(
"slapos.recipe.build",
['python', '-m', 'unittest', 'discover', '-v'],
cwd="""$${slapos.recipe.build:location}""",
summaryf=UnitTest.summary,
)
TestCase(
"slapos.recipe.cmmi",
['python', '-m', 'unittest', 'discover', '-v'],
cwd="""$${slapos.recipe.cmmi:location}""",
summaryf=UnitTest.summary,
)
TestCase(
"slapos.recipe.template",
# ['python', '-m', 'unittest', 'slapos.recipe.template.tests.test_suite'],
# XXX some test fail when running with unittest, so we keep using setup.py test
['python', 'setup.py', 'test'],
cwd="""$${slapos.recipe.template:location}""",
summaryf=UnitTest.summary,
)
TestCase(
"slapos.toolbox",
# ['python', '-m', 'unittest', 'discover', '-v'],
# XXX We can not just run unittest discover here, since slapos/lamp
# imports MySQLDb and we currently don't have installed
# slapos.toolbox[lampconfigure] and on python3 discovery make a
# failing test for this import error.
# Currently slapos/lamp is not tested, but if it it is still used,
# the TODO seem to also install slapos.toolbox[lampconfigure] here.
['python', '-m', 'unittest', 'discover', '-v', '--start', 'slapos/test', '--top-level-directory', '.'],
cwd="""$${slapos.toolbox:location}""",
summaryf=UnitTest.summary,
)
TestCase(
"slapos.libnetworkcache",
['python', '-m', 'unittest', '-v', 'slapos.libnetworkcachetests'],
cwd="""$${slapos.libnetworkcache:location}""",
summaryf=UnitTest.summary,
)
TestCase(
"slapos.rebootstrap",
['python', '-m', 'unittest', '-v', 'slapos.rebootstrap.tests.test_suite'],
cwd="""$${slapos.rebootstrap:location}""",
summaryf=UnitTest.summary,
)
TestCase(
"rubygemsrecipe",
['python', '-m', 'unittest', 'discover', '-v'],
cwd="""$${rubygemsrecipe:location}""",
summaryf=UnitTest.summary,
)
[runTestSuite]
env.sh = $${slapos-test-runner-nxdtest-environment.sh:rendered}
workdir = $${slapos-test-runner-dot-nxdtest:workdir}
[slapos-local-development-environment.sh]
recipe = slapos.recipe.template:jinja2
rendered = $${create-directory:etc}/$${:_buildout_section_name_}
template = inline:
source $${slapos-test-runner-nxdtest-environment.sh:rendered}
echo "Environment loaded."
echo "To work on a test, execute:"
echo " $${runTestSuite:wrapper-path} -k test_name"
echo "replacing test_name by the name of the test."
echo
[publish]
recipe = slapos.cookbook:publish
environment-script = $${slapos-local-development-environment.sh:rendered}
software/slapos-testing/software.cfg
View file @ 4371e39c
......@@ -14,11 +14,12 @@ extends =
../../component/lmsensors/buildout.cfg
../../component/rsync/buildout.cfg
../../stack/slapos.cfg
../../stack/nxdtest.cfg
./buildout.hash.cfg
parts =
bootstrap-slapos.recipe.cmmi
eggs
eggs/scripts
phantomjs
template
......@@ -103,8 +104,8 @@ egg = rubygemsrecipe[test]
setup = ${rubygemsrecipe-repository:location}
[eggs]
recipe = zc.recipe.egg
eggs =
<= python-interpreter
eggs +=
${lxml-python:egg}
${python-cryptography:egg}
${backports.lzma:egg}
......@@ -126,14 +127,13 @@ eggs =
${rubygemsrecipe-setup:egg}
zope.testing
supervisor
entry-points =
runTestSuite=erp5.util.testsuite:runTestSuite
[eggs/scripts]
recipe = zc.recipe.egg
eggs = ${eggs:eggs}
scripts =
runTestSuite
slapos
supervisord
interpreter=
python_for_test
[git-clone-repository]
recipe = slapos.recipe.build:gitclone
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment
GitLab Nexedi Edition | About GitLab | About Nexedi | 沪ICP备2021021310号-2 | 沪ICP备2021021310号-7