Merge branch 'tc-testing-next'

Lucas Bates says:

====================
tc-testing: implement command timeouts and better results tracking

Patch 1 adds a timeout feature for any command tdc launches in a subshell.
This prevents tdc from hanging indefinitely.

Patches 2-4 introduce a new method for tracking and generating test case
results, and implements it across the core script and all applicable
plugins.
====================
Signed-off-by: David S. Miller <davem@davemloft.net>

tools/testing/selftests/tc-testing/.gitignore

 __pycache__/
 *.pyc
+plugins/
+*.xml
+*.tap

tools/testing/selftests/tc-testing/TdcPlugin.py

@@ -18,11 +18,12 @@ class TdcPlugin:
         if self.args.verbose > 1:
             print(' -- {}.post_suite'.format(self.sub_class))
 
-    def pre_case(self, test_ordinal, testid):
+    def pre_case(self, test_ordinal, testid, test_name):
         '''run commands before test_runner does one test'''
         if self.args.verbose > 1:
             print(' -- {}.pre_case'.format(self.sub_class))
         self.args.testid = testid
+        self.args.test_name = test_name
         self.args.test_ordinal = test_ordinal
 
     def post_case(self):

tools/testing/selftests/tc-testing/TdcResults.py

+#!/usr/bin/env python3
+
+from enum import Enum
+
+class ResultState(Enum):
+    noresult = -1
+    skip = 0
+    success = 1
+    fail = 2
+
+class TestResult:
+    def __init__(self, test_id="", test_name=""):
+       self.test_id = test_id
+       self.test_name = test_name
+       self.result = ResultState.noresult
+       self.failmsg = ""
+       self.errormsg = ""
+       self.steps = []
+
+    def set_result(self, result):
+        if (isinstance(result, ResultState)):
+            self.result = result
+            return True
+        else:
+            raise TypeError('Unknown result type, must be type ResultState')
+
+    def get_result(self):
+        return self.result
+
+    def set_errormsg(self, errormsg):
+        self.errormsg = errormsg
+        return True
+
+    def append_errormsg(self, errormsg):
+        self.errormsg = '{}\n{}'.format(self.errormsg, errormsg)
+
+    def get_errormsg(self):
+        return self.errormsg
+
+    def set_failmsg(self, failmsg):
+        self.failmsg = failmsg
+        return True
+
+    def append_failmsg(self, failmsg):
+        self.failmsg = '{}\n{}'.format(self.failmsg, failmsg)
+
+    def get_failmsg(self):
+        return self.failmsg
+
+    def add_steps(self, newstep):
+        if type(newstep) == list:
+            self.steps.extend(newstep)
+        elif type(newstep) == str:
+            self.steps.append(step)
+        else:
+            raise TypeError('TdcResults.add_steps() requires a list or str')
+
+    def get_executed_steps(self):
+        return self.steps
+
+class TestSuiteReport():
+    _testsuite = []
+
+    def add_resultdata(self, result_data):
+        if isinstance(result_data, TestResult):
+            self._testsuite.append(result_data)
+            return True
+
+    def count_tests(self):
+        return len(self._testsuite)
+
+    def count_failures(self):
+        return sum(1 for t in self._testsuite if t.result == ResultState.fail)
+
+    def count_skips(self):
+        return sum(1 for t in self._testsuite if t.result == ResultState.skip)
+
+    def find_result(self, test_id):
+        return next((tr for tr in self._testsuite if tr.test_id == test_id), None)
+
+    def update_result(self, result_data):
+        orig = self.find_result(result_data.test_id)
+        if orig != None:
+            idx = self._testsuite.index(orig)
+            self._testsuite[idx] = result_data
+        else:
+            self.add_resultdata(result_data)
+
+    def format_tap(self):
+        ftap = ""
+        ftap += '1..{}\n'.format(self.count_tests())
+        index = 1
+        for t in self._testsuite:
+            if t.result == ResultState.fail:
+                ftap += 'not '
+            ftap += 'ok {} {} - {}'.format(str(index), t.test_id, t.test_name)
+            if t.result == ResultState.skip or t.result == ResultState.noresult:
+                ftap += ' # skipped - {}\n'.format(t.errormsg)
+            elif t.result == ResultState.fail:
+                if len(t.steps) > 0:
+                    ftap += '\tCommands executed in this test case:'
+                    for step in t.steps:
+                        ftap += '\n\t\t{}'.format(step)
+                ftap += '\n\t{}'.format(t.failmsg)
+            ftap += '\n'
+            index += 1
+        return ftap
+
+    def format_xunit(self):
+        from xml.sax.saxutils import escape
+        xunit = "<testsuites>\n"
+        xunit += '\t<testsuite tests=\"{}\" skips=\"{}\">\n'.format(self.count_tests(), self.count_skips())
+        for t in self._testsuite:
+            xunit += '\t\t<testcase classname=\"{}\" '.format(escape(t.test_id))
+            xunit += 'name=\"{}\">\n'.format(escape(t.test_name))
+            if t.failmsg:
+                xunit += '\t\t\t<failure>\n'
+                if len(t.steps) > 0:
+                    xunit += 'Commands executed in this test case:\n'
+                    for step in t.steps:
+                        xunit += '\t{}\n'.format(escape(step))
+                xunit += 'FAILURE: {}\n'.format(escape(t.failmsg))
+                xunit += '\t\t\t</failure>\n'
+            if t.errormsg:
+                xunit += '\t\t\t<error>\n{}\n'.format(escape(t.errormsg))
+                xunit += '\t\t\t</error>\n'
+            if t.result == ResultState.skip:
+                xunit += '\t\t\t<skipped/>\n'
+            xunit += '\t\t</testcase>\n'
+        xunit += '\t</testsuite>\n'
+        xunit += '</testsuites>\n'
+        return xunit

tools/testing/selftests/tc-testing/plugin-lib/valgrindPlugin.py

@@ -11,6 +11,7 @@ from string import Template
 import subprocess
 import time
 from TdcPlugin import TdcPlugin
+from TdcResults import *
 
 from tdc_config import *
 
@@ -21,6 +22,7 @@ class SubPlugin(TdcPlugin):
     def __init__(self):
         self.sub_class = 'valgrind/SubPlugin'
         self.tap = ''
+        self._tsr = TestSuiteReport()
         super().__init__()
 
     def pre_suite(self, testcount, testidlist):
@@ -34,10 +36,14 @@ class SubPlugin(TdcPlugin):
     def post_suite(self, index):
         '''run commands after test_runner goes into a test loop'''
         super().post_suite(index)
-        self._add_to_tap('\n|---\n')
         if self.args.verbose > 1:
             print('{}.post_suite'.format(self.sub_class))
-        print('{}'.format(self.tap))
+        #print('{}'.format(self.tap))
+        for xx in range(index - 1, self.testcount):
+            res = TestResult('{}-mem'.format(self.testidlist[xx]), 'Test skipped')
+            res.set_result(ResultState.skip)
+            res.set_errormsg('Skipped because of prior setup/teardown failure')
+            self._add_results(res)
         if self.args.verbose < 4:
             subprocess.check_output('rm -f vgnd-*.log', shell=True)
 
@@ -128,8 +134,17 @@ class SubPlugin(TdcPlugin):
                 nle_num = int(nle_mo.group(1))
 
         mem_results = ''
+        res = TestResult('{}-mem'.format(self.args.testid),
+              '{} memory leak check'.format(self.args.test_name))
         if (def_num > 0) or (ind_num > 0) or (pos_num > 0) or (nle_num > 0):
             mem_results += 'not '
+            res.set_result(ResultState.fail)
+            res.set_failmsg('Memory leak detected')
+            res.append_failmsg(content)
+        else:
+            res.set_result(ResultState.success)
+
+        self._add_results(res)
 
         mem_results += 'ok {} - {}-mem # {}\n'.format(
             self.args.test_ordinal, self.args.testid, 'memory leak check')
@@ -138,5 +153,8 @@ class SubPlugin(TdcPlugin):
             print('{}'.format(content))
             self._add_to_tap(content)
 
+    def _add_results(self, res):
+        self._tsr.add_resultdata(res)
+
     def _add_to_tap(self, more_tap_output):
         self.tap += more_tap_output

tools/testing/selftests/tc-testing/tdc_config.py

@@ -15,6 +15,8 @@ NAMES = {
           'DEV1': 'v0p1',
           'DEV2': '',
           'BATCH_FILE': './batch.txt',
+          # Length of time in seconds to wait before terminating a command
+          'TIMEOUT': 12,
           # Name of the namespace to use
           'NS': 'tcut',
           # Directory containing eBPF test programs