software/slapos-testing: use nxdtest

We also try to switch the tests to use python -m unittest as a way of
invoking tests, instead of the deprecated python setup.py test

software/slapos-testing/README.md

@@ -2,13 +2,11 @@
 
 This software release is used to run unit test of slapos eggs.
 
-The approach is to use setuptools' integrated test runner, `python setup.py test`, to run tests.
+The approach is to use nxdtest test runner, which will run tests for each
+projects, as described in `.nxdtest` file.
 
-The `python` used in this command will be a `zc.recipe.egg` interpreter with
-all eggs pre-installed by this software release.
-
-The results of this test suite running on Nexedi ERP5 are published as `SlapOS.Eggs.UnitTest-Master.Python3`
-and `SlapOS.Eggs.UnitTest-Master.Python2`.
+The results of this test suite running on Nexedi ERP5 are published as 
+`SlapOS.Eggs.UnitTest-Master.Python3` and `SlapOS.Eggs.UnitTest-Master.Python2`.
 
 
 Here's an example session of how a developer could use this software release in
@@ -23,21 +21,27 @@ INSTANCE_NAME=$COMP
 
 slapos supply $SR $COMP
 slapos node software
-slapos request --node=node=$COMP $INSTANCE_NAME $SR
+slapos request --node=computer_guid=$COMP $INSTANCE_NAME $SR
 slapos node instance
 
+# The path of a an environment script was published by slapos parameters, as
+# "environment-script"
+slapos request --node=computer_guid=$COMP $INSTANCE_NAME $SR
+
+# sourcing the script in the shell configure all environment variables and
+# print a message explaining how to run tests
+source ( environment script from step above )
+
 # The source code is a git clone working copy on the instance
-cd ~/srv/runner/instance/slappart0/parts/slapos.core/
+cd ~/srv/runner/instance/slappartXXX/parts/slapos.core/
 
 # make some changes to the code
 vim slapos/tests/client.py
 
-# run tests, using bundled python intepreter with pre-installed eggs dependencies
-SLAPOS_TEST_IPV6=::1 \
-SLAPOS_TEST_IPV4=127.0.0.1 \
-SLAPOS_TEST_VERBOSE=1 \
-SLAPOS_TEST_DEBUG=1 \
-~/srv/runner/instance/slappart0/software_release/bin/python_for_test setup.py test
+# run slapos.core tests
+runTestSuite --run slapos.core
+# ... or run all eggs tests
+runTestSuite
 
 # when satified, commit changes
 git add -p && git commit

software/slapos-testing/buildout.hash.cfg

@@ -15,4 +15,4 @@
 
 [template]
 filename = instance.cfg
-md5sum = 2df601dd3ccb3ba38b3aee7243b7f8e5
+md5sum = 3b1b386f6ad4c9ac50ab1f1e1384e751

software/slapos-testing/instance.cfg

 [buildout]
+extends = ${nxdtest-instance.cfg:rendered}
 parts =
-  phantomjs-wrapper
-  slapos-test-runner
+  runTestSuite
+  publish
 
 eggs-directory = ${buildout:eggs-directory}
 develop-eggs-directory = ${buildout:develop-eggs-directory}
@@ -21,6 +22,8 @@ bin = $${buildout:directory}/bin
 etc = $${buildout:directory}/etc
 services = $${:etc}/run
 srv = $${buildout:directory}/srv
+var = $${buildout:directory}/var
+nxdtest-working-dir = $${:var}/nxdtest
 
 [download-source]
 recipe = slapos.recipe.build:gitclone
@@ -75,30 +78,127 @@ repository = ${slapos.rebootstrap-repository:location}
 repository = ${rubygemsrecipe-repository:location}
 
 
-[slapos-test-runner]
-recipe = slapos.cookbook:wrapper
-wrapper-path = $${create-directory:bin}/runTestSuite
-command-line =
-  ${buildout:bin-directory}/runTestSuite
-  --python_interpreter=${buildout:bin-directory}/${eggs:interpreter}
-  --source_code_path_list=$${kedifa:location},$${caucase:location},$${erp5.util:location},$${slapos.cookbook:location},$${slapos.core:location},$${slapos.recipe.build:location},$${slapos.recipe.cmmi:location},$${slapos.recipe.template:location},$${slapos.toolbox:location},$${slapos.libnetworkcache:location},$${slapos.rebootstrap:location},$${rubygemsrecipe:location}
-
-# Notes about environment:
-# * slapos.cookbook:wrapper does not seem to allow "extending" PATH. Tests
-#   needs ping, which is a setuid binary that cannot be installed via slapos
-#   way of building software without root access, so we keep "standard"
-#   /usr/bin and /bin in $PATH
-# * SLAPOS_TEST_environment is safe to be used by tests, but there is no
-#   guarantee about free ports on IPV4 and IPV6
-# * LOCAL_IPV4 is backward compatible, to be migrated, SLAPOS_TEST_IPV4
-environment =
-  PATH=${coreutils:location}/bin:${curl:location}/bin:${openssl:location}/bin:${git:location}/bin:${libxslt:location}/bin:${socat:location}/bin:${lmsensors:location}/bin:${rsync:location}/bin/:${buildout:bin-directory}:/usr/bin/:/bin/
-  LOCAL_IPV4=$${slap-configuration:ipv4-random}
-  SLAPOS_TEST_IPV4=$${slap-configuration:ipv4-random}
-  SLAPOS_TEST_IPV6=$${slap-configuration:ipv6-random}
-
-
-[phantomjs-wrapper]
-recipe = slapos.cookbook:wrapper
-command-line = ${phantomjs:location}/phantomjs-slapos
-wrapper-path = $${create-directory:bin}/phantomjs
+[slapos-test-runner-nxdtest-environment.sh]
+recipe = slapos.recipe.template:jinja2
+rendered = $${create-directory:etc}/$${:_buildout_section_name_}
+template = inline:
+  export PATH=${coreutils:location}/bin:${curl:location}/bin:${openssl:location}/bin:${git:location}/bin:${libxslt:location}/bin:${socat:location}/bin:${lmsensors:location}/bin:${rsync:location}/bin/:${buildout:bin-directory}:$PATH
+  export SLAPOS_TEST_IPV4=$${slap-configuration:ipv4-random}
+  export SLAPOS_TEST_IPV6=$${slap-configuration:ipv6-random}
+
+
+[slapos-test-runner-dot-nxdtest]
+recipe = slapos.recipe.template:jinja2
+rendered = $${:workdir}/.nxdtest
+workdir = $${create-directory:nxdtest-working-dir}
+
+template = inline:
+  TestCase(
+      "kedifa",
+      ['python', '-m', 'unittest', 'discover', '-v'],
+      cwd="""$${kedifa:location}""",
+      summaryf=UnitTest.summary,
+  )
+  TestCase(
+      "caucase",
+      # XXX caucase uses 2to3 dynamically in setup.py, so it only supports
+      # runnning tests with python setup.py test
+      ['python', 'setup.py', 'test'],
+      cwd="""$${caucase:location}""",
+      summaryf=UnitTest.summary,
+  )
+  TestCase(
+      "erp5.util",
+      ['python', '-m', 'unittest', 'discover', '-v', '--start', 'erp5/tests/'],
+      cwd="""$${erp5.util:location}""",
+      summaryf=UnitTest.summary,
+  )
+  TestCase(
+      "slapos.cookbook",
+      # slapos/test contain both tests for recipes and tests for
+      # json schemas, we only care about recipe tests here, json
+      # schemas are tested in software/slapos-sr-testing
+      ['python', '-m', 'unittest', 'discover', '-v', '--start', 'slapos/test/recipe'],
+      cwd="""$${slapos.cookbook:location}""",
+      summaryf=UnitTest.summary,
+  )
+  TestCase(
+      "slapos.core",
+      # ['python', '-m', 'unittest', 'discover', '-v'],
+      # XXX some test fail when running with unittest (slapos*.cfg.example
+      # files cannot be found with pkg_resources.resource_string), so we keep
+      # using setup.py test for now.
+      ['python', 'setup.py', 'test'],
+      cwd="""$${slapos.core:location}""",
+      summaryf=UnitTest.summary,
+  )
+  TestCase(
+      "slapos.recipe.build",
+      ['python', '-m', 'unittest', 'discover', '-v'],
+      cwd="""$${slapos.recipe.build:location}""",
+      summaryf=UnitTest.summary,
+  )
+  TestCase(
+      "slapos.recipe.cmmi",
+      ['python', '-m', 'unittest', 'discover', '-v'],
+      cwd="""$${slapos.recipe.cmmi:location}""",
+      summaryf=UnitTest.summary,
+  )
+  TestCase(
+      "slapos.recipe.template",
+      # ['python', '-m', 'unittest', 'slapos.recipe.template.tests.test_suite'],
+      # XXX some test fail when running with unittest, so we keep using setup.py test
+      ['python', 'setup.py', 'test'],
+      cwd="""$${slapos.recipe.template:location}""",
+      summaryf=UnitTest.summary,
+  )
+  TestCase(
+      "slapos.toolbox",
+      # ['python', '-m', 'unittest', 'discover', '-v'],
+      # XXX We can not just run unittest discover here, since slapos/lamp
+      # imports MySQLDb and we currently don't have installed
+      # slapos.toolbox[lampconfigure] and on python3 discovery make a
+      # failing test for this import error.
+      # Currently slapos/lamp is not tested, but if it it is still used,
+      # the TODO seem to also install slapos.toolbox[lampconfigure] here.
+      ['python', '-m', 'unittest', 'discover', '-v', '--start', 'slapos/test', '--top-level-directory', '.'],
+      cwd="""$${slapos.toolbox:location}""",
+      summaryf=UnitTest.summary,
+  )
+  TestCase(
+      "slapos.libnetworkcache",
+      ['python', '-m', 'unittest', '-v', 'slapos.libnetworkcachetests'],
+      cwd="""$${slapos.libnetworkcache:location}""",
+      summaryf=UnitTest.summary,
+  )
+  TestCase(
+      "slapos.rebootstrap",
+      ['python', '-m', 'unittest', '-v', 'slapos.rebootstrap.tests.test_suite'],
+      cwd="""$${slapos.rebootstrap:location}""",
+      summaryf=UnitTest.summary,
+  )
+  TestCase(
+      "rubygemsrecipe",
+      ['python', '-m', 'unittest', 'discover', '-v'],
+      cwd="""$${rubygemsrecipe:location}""",
+      summaryf=UnitTest.summary,
+  )
+
+[runTestSuite]
+env.sh = $${slapos-test-runner-nxdtest-environment.sh:rendered}
+workdir = $${slapos-test-runner-dot-nxdtest:workdir}
+
+[slapos-local-development-environment.sh]
+recipe = slapos.recipe.template:jinja2
+rendered = $${create-directory:etc}/$${:_buildout_section_name_}
+template = inline:
+  source $${slapos-test-runner-nxdtest-environment.sh:rendered}
+  echo "Environment loaded."
+  echo "To work on a test, execute:"
+  echo "   $${runTestSuite:wrapper-path} -k test_name"
+  echo "replacing test_name by the name of the test."
+  echo
+
+[publish]
+recipe = slapos.cookbook:publish
+environment-script = $${slapos-local-development-environment.sh:rendered}

software/slapos-testing/software.cfg

@@ -14,11 +14,12 @@ extends =
   ../../component/lmsensors/buildout.cfg
   ../../component/rsync/buildout.cfg
   ../../stack/slapos.cfg
+  ../../stack/nxdtest.cfg
   ./buildout.hash.cfg
 
 parts =
   bootstrap-slapos.recipe.cmmi
-  eggs
+  eggs/scripts
   phantomjs
   template
 
@@ -103,8 +104,8 @@ egg = rubygemsrecipe[test]
 setup = ${rubygemsrecipe-repository:location}
 
 [eggs]
-recipe = zc.recipe.egg
-eggs =
+<= python-interpreter
+eggs +=
   ${lxml-python:egg}
   ${python-cryptography:egg}
   ${backports.lzma:egg}
@@ -126,14 +127,13 @@ eggs =
   ${rubygemsrecipe-setup:egg}
   zope.testing
   supervisor
-entry-points =
-  runTestSuite=erp5.util.testsuite:runTestSuite
+
+[eggs/scripts]
+recipe = zc.recipe.egg
+eggs = ${eggs:eggs}
 scripts =
-  runTestSuite
   slapos
   supervisord
-interpreter=
-  python_for_test
 
 [git-clone-repository]
 recipe = slapos.recipe.build:gitclone