stack/erp5: support coverage when running tests 🚧

starting coverage in runUnitTest was too late, coverage has to be
started before anything is imported for correct reporting

software/erp5/instance-erp5-input-schema.json

@@ -525,6 +525,38 @@
           "default": true,
           "type": "boolean"
         },
+        "coverage": {
+          "type": "object",
+          "title": "Coverage",
+          "description": "Coverage configuration",
+          "additionalProperties": false,
+          "properties": {
+            "enabled": {
+              "description": "Collect python coverage data during test run.",
+              "default": false,
+              "type": "boolean"
+            },
+            "include": {
+              "description": "File name patterns to include in coverage data, relative to software buildout's directory. Default to all repositories defined in software by ${erp5_repository_list:repository_id_list}.",
+              "type": "array",
+              "items": {
+                "type": "string"
+              },
+              "examples": [
+                [
+                  "parts/erp5/*",
+                  "parts/custom-repository/*",
+                  "develop-eggs/custom-egg/*"
+                ]
+              ]
+            },
+            "branch": {
+              "description": "Enable branch coverage",
+              "type": "boolean",
+              "default": false
+            }
+          }
+        },
         "node-count": {
           "description": "Number of tests this instance can execute in parallel. This must be at least equal to the number of nodes configured on testnode running the test",
           "default": 3,

stack/erp5/buildout.cfg

@@ -325,9 +325,13 @@ command =
   echo '${local-bt5-repository:list}' |xargs ${buildout:executable} ${:genbt5list}
 update-command = ${:command}
 
+# BBB we keep this wrongly named section because it is often overridden in custom profiles
 [erp5_repository_list]
 repository_id_list = erp5 erp5-bin erp5-doc
 
+[erp5-repository-list]
+repository-id-list = ${erp5_repository_list:repository_id_list}
+
 # ERP5 defaults, which can be overridden in inheriting recipes (e.g. wendelin)
 [erp5-defaults]
 cloudooo-connection-url = https://cloudooo.erp5.net/
@@ -353,14 +357,52 @@ entry-points =
   runUnitTest=runUnitTest:main
 scripts = runUnitTest
 initialization =
-  import glob, os, sys
+  import glob, os, sys, json
+  buildout_directory = '''${buildout:directory}'''
+  parts_directory = '''${buildout:parts-directory}'''
+  repository_id_list = \
+    '''${erp5_repository_list:repository_id_list}'''.split()[::-1]
+
+  # read testrunner configuration from slapos instance parameters to
+  # configure coverage if enabled.
+  with open(os.environ['ERP5_TEST_RUNNER_CONFIGURATION']) as f:
+    test_runner_configuration = json.load(f)
+  test_runner_configuration.setdefault('coverage', {})
+  test_runner_configuration['coverage'].setdefault('enabled', False)
+
+  coverage_process = None
+  if test_runner_configuration['coverage']['enabled']:
+    test_runner_configuration['coverage'].setdefault(
+      'include', [os.path.join('parts', repo, '*') for repo in repository_id_list])
+
+    assets_directory = ''
+    if os.environ.get('SLAPOS_TEST_LOG_DIRECTORY'):
+      test_name = sys.argv[-1].replace(':', '_')
+      assets_directory = os.path.join(os.environ['SLAPOS_TEST_LOG_DIRECTORY'], test_name)
+      if not os.path.exists(assets_directory):
+        os.makedirs(assets_directory)
+    coverage_data_file = os.path.abspath(
+      os.path.join(assets_directory, 'coverage.sqlite3'))
+
+    curdir = os.path.abspath(os.curdir)
+    # change current directory when importing coverage so that it considers paths
+    # relative to the root of the software
+    os.chdir(buildout_directory)
+    import coverage
+    coverage_process = coverage.Coverage(
+      include=test_runner_configuration['coverage']['include'],
+      data_file=coverage_data_file,
+      branch=test_runner_configuration['coverage'].get('branch'),
+    )
+    coverage_process.set_option('run:relative_files', 'true')
+    coverage_process.set_option('run:plugins', ['erp5_coverage_plugin'])
+    coverage_process.start()
+    os.chdir(curdir)
+
   import Products
   Products.__path__[:0] = filter(None,
     os.getenv('INSERT_PRODUCTS_PATH', '').split(os.pathsep))
   os.environ['ZOPE_SCRIPTS'] = ''
-  parts_directory = '''${buildout:parts-directory}'''
-  repository_id_list = \
-    '''${erp5_repository_list:repository_id_list}'''.split()[::-1]
   os.environ['erp5_tests_bt5_path'] = ','.join(
     os.path.join(parts_directory, x, 'bt5') for x in repository_id_list)
   extra_path_list = '''${:extra-paths}'''.split()
@@ -373,6 +415,54 @@ initialization =
   sys.path[:0] = sum((
     glob.glob(os.path.join(x, 'Products', '*', 'tests'))
     for x in os.getenv('INSERT_PRODUCTS_PATH', '').split(os.pathsep)), [])
+  import runUnitTest
+  try:
+    sys.exit(runUnitTest.main())
+  finally:
+    if coverage_process:
+      coverage_process.stop()
+      coverage_process.save()
+      # Make html report for this test
+      os.chdir(buildout_directory)
+      coverage_process.html_report(
+        directory=os.path.join(assets_directory, 'coverage_report'),
+      )
+      # upload the coverage so that they can be combined from another machine
+      upload_url = test_runner_configuration['coverage'].get('upload-url')
+      if upload_url:
+        import requests
+        import time
+        import uritemplate
+        from six.moves.urllib.parse import urlparse
+
+        auth_list = (None, )
+        parsed_url = urlparse(upload_url)
+        if parsed_url.username:
+          # try Digest and Basic authentication and retry 5 times to tolerate transiant errors
+          auth_list = (
+            requests.auth.HTTPDigestAuth(parsed_url.username, parsed_url.password),
+            requests.auth.HTTPBasicAuth(parsed_url.username, parsed_url.password),
+          ) *  5
+
+        url = uritemplate.URITemplate(upload_url).expand(
+          test_name=test_name,
+          # Environment variables are set in parts/erp5/product/ERP5Type/tests/runTestSuite.py
+          test_result_id=os.environ.get('ERP5_TEST_RESULT_ID', 'unknown_test_result_id'),
+          test_result_revision=os.environ.get('ERP5_TEST_RESULT_REVISION', 'unknown_test_result_revision'),
+        )
+
+        for auth in auth_list:
+          with open(coverage_data_file, 'rb') as f:
+            resp = requests.put(url, data=f, auth=auth)
+            if resp.ok:
+              # print just the hostname, not to include the auth
+              print('Uploaded coverage data to {parsed_url.hostname}'.format(parsed_url=parsed_url))
+              break
+            print('Error {resp.status_code} uploading data to {parsed_url.hostname} with {auth.__class__.__name__}'.format(
+              resp=resp, parsed_url=parsed_url, auth=auth))
+            time.sleep(1)
+        else:
+          sys.stderr.write('Error uploading coverage data to {parsed_url}\n'.format(parsed_url=parsed_url))
 
 [test-suite-runner]
 # XXX: Workaround for fact ERP5Type is not an distribution and does not
@@ -479,7 +569,6 @@ eggs = ${neoppod:eggs}
   SOAPpy
   chardet
   collective.recipe.template
-  coverage
   erp5diff
   interval
   ipdb
@@ -638,6 +727,12 @@ Zope-patches =
   https://github.com/zopefoundation/Zope/commit/68f0c122cf938c3e6184185cd8519e26ff7d0b14.patch#5a2c9c31caaaba75677507ddb3a0becc
 Zope-patch-options = -p1
 
+
+# neoppod installs bin/coverage so we inject erp5 plugin here so that coverage script can use it in report
+[neoppod]
+eggs +=
+  erp5_coverage_plugin
+
 [eggs-all-scripts]
 recipe = zc.recipe.egg
 eggs =
@@ -801,7 +896,7 @@ strict-rfc3339 = 0.7
 webcolors = 1.10
 rfc3987 = 1.3.8
 jsonpointer = 2.2
-
+erp5-coverage-plugin = 0.0.1
 
 # WIP Zope 4 ⚠
 Products.CMFCore = 2.6.0
@@ -821,6 +916,7 @@ zope.error = 4.6
 zope.authentication = 4.5.0
 zope.session = 4.5
 zope.minmax = 2.3
+toml = 0.10.2
 
 # XXX do we need ?
 mechanize = 0.4.8