software/slapos-sr-testing: use nxdtest

Change to run all tests with python -m unittest, which will allow us to
remove workaround for the $PYTHONPATH set by setup.py test.

For this we use `[python-interpreter]` which uses pygolang.main to behave
like a "normal" python interpreter with eggs available, but without the
usual problems of interpreters generated by zc.recipe.eggs - it supports
all command line flags and have current directory in sys.path.

Tests now have an explicit name, that we set to the software name as before,
and the special test dynamically checking all profiles json schemas (part
of slapos.cookbook egg, at the root of slapos repository),is named
json-schemas. It is also changed to run as default tests in python2 and
python3, not only in extras for python2.

software/slapos-sr-testing/README.md

@@ -31,15 +31,15 @@ slapos request --node=computer_guid=$COMP $INSTANCE_NAME $SR
 # and load this script to set environment variables
 source ( environment-script from step above )
 
-# Clone a working copy somewhere
-cd ~/srv/runner/project/
-git clone https://lab.nexedi.com/nexedi/slapos.git slapos_work
+# The source code is a git clone working copy on the instance
+cd ~/srv/runner/instance/slappartXXX/parts/slapos/
 
 # change directory to the directory containing test for this software
-cd ~/srv/runner/project/slapos_work/software/helloworld/test/
+cd ./software/helloworld/test/
+# make changes to test code or profile
 
-# run test (with debugging features activated)
-SLAPOS_TEST_DEBUG= 1 python_for_test setup.py test
+# run test for helloworld software release (with debugging features activated)
+SLAPOS_TEST_DEBUG=1 python_for_test -m unittest discover -v
 ```
 
 ## Environment variables

software/slapos-sr-testing/buildout.hash.cfg

@@ -15,4 +15,4 @@
 
 [template]
 filename = instance.cfg
-md5sum = 4246cde0a27138e057ba1635cc621edf
+md5sum = 9639d0c0e161c094454808fb95fc9781

software/slapos-sr-testing/instance.cfg

 [buildout]
+extends = {{ nxdtest_instance }}
 parts =
-  slapos-test-runner
+  runTestSuite
   publish
 
 eggs-directory = {{ buildout['eggs-directory'] }}
@@ -26,46 +27,53 @@ repository = {{ slapos_location }}
 [directory]
 recipe = slapos.cookbook:mkdirectory
 bin = ${buildout:directory}/bin
-working-dir = ${buildout:directory}/tmp
 etc = ${buildout:directory}/etc
+var = ${buildout:directory}/var
+test-working-dir = ${buildout:directory}/tmp
+nxdtest-working-dir = ${:var}/nxdtest
 
-[slapos-test-runner]
-recipe = slapos.cookbook:wrapper
-wrapper-path = ${directory:bin}/runTestSuite
-command-line =
-  {{ buildout['bin-directory'] }}/runTestSuite
-  --python_interpreter={{ buildout['bin-directory'] }}/{{ interpreter }}
-  --source_code_path_list={{ ','.join(tests.splitlines()) }}
+[slapos-test-runner-nxdtest-environment.sh]
+recipe = slapos.recipe.template:jinja2
+rendered = ${directory:etc}/${:_buildout_section_name_}
+template = inline:
+  export PATH={{ buildout['bin-directory'] }}:{{ curl_location }}/bin/:{{ faketime_location }}/bin/:{{ openssl_location }}/bin/:/usr/bin/:/bin
+  export SLAPOS_TEST_IPV4=${slap-configuration:ipv4-random}
+  export SLAPOS_TEST_IPV6=${slap-configuration:ipv6-random}
+  export SLAPOS_TEST_WORKING_DIR=${directory:test-working-dir}
 
-environment =
-  PATH=${slapos-test-runner-environment:PATH}
-  SLAPOS_TEST_IPV4=${slapos-test-runner-environment:SLAPOS_TEST_IPV4}
-  SLAPOS_TEST_IPV6=${slapos-test-runner-environment:SLAPOS_TEST_IPV6}
-  SLAPOS_TEST_WORKING_DIR=${slapos-test-runner-environment:SLAPOS_TEST_WORKING_DIR}
+[slapos-test-runner-dot-nxdtest]
+recipe = slapos.recipe.template:jinja2
+rendered = ${:workdir}/.nxdtest
+workdir = ${directory:nxdtest-working-dir}
+template = inline:
+  {% for test in tests.splitlines() %}
+    TestCase(
+      {{ repr(test.split()[0]) }},
+      [ {{ repr(interpreter) }}, '-m', 'unittest', 'discover', '-v'],
+      cwd={{ repr(test.split()[1]) }},
+      summaryf=UnitTest.summary,
+    )
+  {% endfor %}
 
-[slapos-test-runner-environment]
-PATH = {{ buildout['bin-directory'] }}:{{ curl_location }}/bin/:{{ faketime_location }}/bin/:{{ openssl_location }}/bin/:/usr/bin/:/bin
-SLAPOS_TEST_IPV4 = ${slap-configuration:ipv4-random}
-SLAPOS_TEST_IPV6 = ${slap-configuration:ipv6-random}
-SLAPOS_TEST_WORKING_DIR = ${directory:working-dir}
+[runTestSuite]
+env.sh = ${slapos-test-runner-nxdtest-environment.sh:rendered}
+workdir = ${slapos-test-runner-dot-nxdtest:workdir}
 
 [slapos-local-development-environment.sh]
 recipe = slapos.recipe.template:jinja2
 rendered = ${directory:etc}/${:_buildout_section_name_}
 template = inline:
-  export PATH=${slapos-test-runner-environment:PATH}:$PATH
-  export SLAPOS_TEST_IPV4=${slapos-test-runner-environment:SLAPOS_TEST_IPV4}
-  export SLAPOS_TEST_IPV6=${slapos-test-runner-environment:SLAPOS_TEST_IPV6}
-  export SLAPOS_TEST_WORKING_DIR=${slapos-test-runner-environment:SLAPOS_TEST_WORKING_DIR}
+  source ${slapos-test-runner-nxdtest-environment.sh:rendered}
   {% set shared_part_list = [] %}
   {% for shared_part in buildout['shared-part-list'].splitlines() -%}
     {% do shared_part_list.append(shared_part) %}
   {%- endfor %}
   export SLAPOS_TEST_SHARED_PART_LIST={{ os.pathsep.join(shared_part_list) }}
   echo "Environment loaded."
+  echo "The embedded SlapOS is in $SLAPOS_TEST_WORKING_DIR"
   echo "To work on a test, execute:"
-  echo "   SLAPOS_TEST_DEBUG=1 {{ interpreter }} setup.py test"
-  echo "from test folder"
+  echo "   SLAPOS_TEST_DEBUG=1 {{ interpreter }} -m unittest discover -v"
+  echo "from a folder containing software release test."
   echo
 
 [publish]

software/slapos-sr-testing/software-py3.cfg

@@ -12,9 +12,9 @@ eggs +=
 
 [template]
 extra =
-  ${slapos.test.helloworld-setup:setup}
-  ${slapos.test.monitor-setup:setup}
-  ${slapos.test.plantuml-setup:setup}
-  ${slapos.test.powerdns-setup:setup}
-  ${slapos.test.proftpd-setup:setup}
-  ${slapos.test.repman-setup:setup}
+  helloworld ${slapos.test.helloworld-setup:setup}
+  monitor ${slapos.test.monitor-setup:setup}
+  plantuml ${slapos.test.plantuml-setup:setup}
+  powerdns ${slapos.test.powerdns-setup:setup}
+  proftpd ${slapos.test.proftpd-setup:setup}
+  repman ${slapos.test.repman-setup:setup}

software/slapos-sr-testing/software.cfg

@@ -13,11 +13,12 @@ extends =
   ../../component/python-backports-lzma/buildout.cfg
 
   ../../stack/slapos.cfg
+  ../../stack/nxdtest.cfg
 
   ./buildout.hash.cfg
 
 parts =
-  eggs
+  eggs/scripts
   slapos-cookbook
   template
 
@@ -167,8 +168,8 @@ egg = slapos.core
 setup = ${slapos.core-repository:location}
 
 [eggs]
-recipe = zc.recipe.egg
-eggs =
+<= python-interpreter
+eggs +=
   ${lxml-python:egg}
   ${slapos.core-setup:egg}
   ${pillow-python:egg}
@@ -202,21 +203,24 @@ eggs =
   ${slapos.test.dream-setup:egg}
   ${slapos.test.metabase-setup:egg}
   ${slapos.test.repman-setup:egg}
-entry-points =
-  runTestSuite=erp5.util.testsuite:runTestSuite
-scripts =
-  runTestSuite
-  slapos
-  supervisorctl
-  supervisord
-interpreter=
-  python_for_test
+
+# We don't name this interpreter `python`, so that when we run slapos node
+# software, installation scripts running `python` use a python without any
+# custom eggs pre-installed, not our special python interpreter.
+interpreter = python_for_test
 
 # patches for eggs
 patch-binary = ${patch:location}/bin/patch
 PyPDF2-patches = ${:_profile_base_location_}/../../component/egg-patch/PyPDF2/0001-Custom-implementation-of-warnings.formatwarning-remo.patch#d25bb0f5dde7f3337a0a50c2f986f5c8
 PyPDF2-patch-options = -p1
 
+[eggs/scripts]
+recipe = zc.recipe.egg
+eggs = ${eggs:eggs}
+scripts =
+  slapos
+  supervisord
+
 [git-clone-repository]
 recipe = slapos.recipe.build:gitclone
 git-executable = ${git:location}/bin/git
@@ -236,6 +240,7 @@ mode = 640
 context =
   section buildout buildout
   import os os
+  key nxdtest_instance nxdtest-instance.cfg:rendered
   key git_location git:location
   key slapos_location slapos-repository:location
   key interpreter eggs:interpreter
@@ -243,29 +248,31 @@ context =
   key openssl_location openssl-output:bin
   key faketime_location faketime:location
   key tests :tests
+
 tests =
-  ${slapos.test.kvm-setup:setup}
-  ${slapos.test.slaprunner-setup:setup}
-  ${slapos.test.metabase-setup:setup}
+  json-schemas ${slapos.cookbook-setup:setup}
+  kvm ${slapos.test.kvm-setup:setup}
+  slaprunner ${slapos.test.slaprunner-setup:setup}
+  metabase ${slapos.test.metabase-setup:setup}
   ${:extra}
 extra =
-  ${slapos.cookbook-setup:setup}
-  ${slapos.test.backupserver-setup:setup}
-  ${slapos.test.caddy-frontend-setup:setup}
-  ${slapos.test.erp5-setup:setup}
-  ${slapos.test.htmlvalidatorserver-setup:setup}
-  ${slapos.test.slapos-master-setup:setup}
-  ${slapos.test.re6stnet-setup:setup}
-  ${slapos.test.seleniumserver-setup:setup}
-  ${slapos.test.jstestnode-setup:setup}
-  ${slapos.test.jupyter-setup:setup}
-  ${slapos.test.nextcloud-setup:setup}
-  ${slapos.test.turnserver-setup:setup}
-  ${slapos.test.theia-setup:setup}
-  ${slapos.test.grafana-setup:setup}
-  ${slapos.test.gitlab-setup:setup}
-  ${slapos.test.cloudooo-setup:setup}
-  ${slapos.test.dream-setup:setup}
+  backupserver ${slapos.test.backupserver-setup:setup}
+  caddy-frontend ${slapos.test.caddy-frontend-setup:setup}
+  erp5 ${slapos.test.erp5-setup:setup}
+  htmlvalidatorserver ${slapos.test.htmlvalidatorserver-setup:setup}
+  slapos-master ${slapos.test.slapos-master-setup:setup}
+  re6stnet ${slapos.test.re6stnet-setup:setup}
+  seleniumserver ${slapos.test.seleniumserver-setup:setup}
+  jstestnode ${slapos.test.jstestnode-setup:setup}
+  jupyter ${slapos.test.jupyter-setup:setup}
+  nextcloud ${slapos.test.nextcloud-setup:setup}
+  turnserver ${slapos.test.turnserver-setup:setup}
+  theia ${slapos.test.theia-setup:setup}
+  grafana ${slapos.test.grafana-setup:setup}
+  gitlab ${slapos.test.gitlab-setup:setup}
+  cloudooo ${slapos.test.cloudooo-setup:setup}
+  dream ${slapos.test.dream-setup:setup}
+  repman ${slapos.test.repman-setup:setup}
 
 [versions]
 # slapos.core is used from the clone always