diff --git a/software/neoppod/instance-neo-storage-mysql.cfg.in b/software/neoppod/instance-neo-storage-mysql.cfg.in index a079892ead2161de50ccec0c0e6bf31b777538da..c33d4152775f0ac586b3297326ff212f4c0517bf 100644 --- a/software/neoppod/instance-neo-storage-mysql.cfg.in +++ b/software/neoppod/instance-neo-storage-mysql.cfg.in @@ -125,6 +125,7 @@ etc_run = ${:etc}/run var_run = ${:var}/run srv_mariadb = ${buildout:directory}/srv/mariadb log = ${buildout:directory}/var/log +tmp = ${buildout:directory}/tmp [logrotate-mysql] recipe = slapos.cookbook:logrotate.d @@ -134,6 +135,19 @@ name = mariadb log = ${my-cnf-parameters:error-log} ${my-cnf-parameters:slow-query-log} post = ${mysqld:mysql-base-directory}/bin/mysql --defaults-file="${my-cnf:rendered}" -e "FLUSH LOGS" +{% if runTestSuite_in is defined -%} +# bin/runTestSuite to run NEO tests +[{{ section('runTestSuite') }}] +recipe = slapos.recipe.template:jinja2 +rendered = ${directory:bin}/${:_buildout_section_name_} +template = {{ runTestSuite_in }} +mode = 0755 +context = + section directory directory + section my_cnf_parameters my-cnf-parameters + raw bin_directory {{ bin_directory }} +{% endif -%} + [buildout] extends = {{ logrotate_cfg }} diff --git a/software/neoppod/instance.cfg.in b/software/neoppod/instance.cfg.in index d0fa6d8762c8749d822cc5227d5ae5b765b05148..00a4cb92e1b79d539606d67245cb572db1473797 100644 --- a/software/neoppod/instance.cfg.in +++ b/software/neoppod/instance.cfg.in @@ -12,6 +12,10 @@ extra-context = import-list = rawfile root_common {{ root_common }} +[neo-storage-mysql] +extra-context += + raw runTestSuite_in {{ runTestSuite_in }} + [switch-softwaretype] recipe = slapos.cookbook:switch-softwaretype override = {{ dumps(override_switch_softwaretype |default) }} diff --git a/software/neoppod/runTestSuite.in b/software/neoppod/runTestSuite.in new file mode 100644 index 0000000000000000000000000000000000000000..01413ed0bd79a0a9309547f46c3f628257fab7ae --- /dev/null +++ b/software/neoppod/runTestSuite.in @@ -0,0 +1,134 @@ +#!{{ bin_directory }}/runTestSuite_py +""" + Script to run NEO test suite using Nexedi's test node framework. +""" +import argparse, os, re, shutil, subprocess, sys, traceback +from erp5.util import taskdistribution +from time import gmtime, strftime + +# pattern to get test counts from stdout +SUMMARY_RE = re.compile( + r'^(.*)Summary (.*) (?P<test_count>\d+) (.*) (?P<unexpected_count>\d+|\.)' + r' (.*) (?P<expected_count>\d+|\.) (.*) (?P<skip_count>\d+|\.)' + r' (.*) (?P<duration>\d+(\.\d*)?|\.\d+)s', re.MULTILINE) + +# NEO specific environment +TEMP_DIRECTORY = '{{directory.tmp}}/neo_tests' +NEO_DB_SOCKET = '{{my_cnf_parameters.socket}}' +RUN_NEO_TESTS_COMMAND = '{{ bin_directory }}/neotestrunner' + +def parseTestStdOut(data): + """ + Parse output of NEO testrunner script. + """ + test_count = 0 + unexpected_count = 0 + expected_count = 0 + skip_count = 0 + duration = 0 + search = SUMMARY_RE.search(data) + if search: + groupdict = search.groupdict() + test_count = int(groupdict['test_count']) + duration = float(groupdict['duration']) + try: + # it can match '.'! + skip_count = int(groupdict['skip_count']) + except ValueError: + pass + try: + # it can match '.'! + unexpected_count = int(groupdict['unexpected_count']) + except ValueError: + pass + try: + # it can match '.'! + expected_count = int(groupdict['expected_count']) + except ValueError: + pass + + return test_count, unexpected_count, expected_count, skip_count, duration + +def main(): + parser = argparse.ArgumentParser(description='Run a test suite.') + parser.add_argument('--test_suite', help='The test suite name') + parser.add_argument('--test_suite_title', help='The test suite title') + parser.add_argument('--test_node_title', help='The test node title') + parser.add_argument('--project_title', help='The project title') + parser.add_argument('--revision', help='The revision to test', + default='dummy_revision') + parser.add_argument('--node_quantity', help='ignored', type=int) + parser.add_argument('--master_url', + help='The Url of Master controling many suites') + + args = parser.parse_args() + + test_suite_title = args.test_suite_title or args.test_suite + revision = args.revision + + test_name_list = 'SQLite', 'MySQL' + + tool = taskdistribution.TaskDistributionTool(portal_url = args.master_url) + test_result = tool.createTestResult(revision = revision, + test_name_list = test_name_list, + node_title = args.test_node_title, + test_title = test_suite_title, + project_title = args.project_title) + if test_result is None: + return + # run NEO tests + while 1: + test_result_line = test_result.start() + if not test_result_line: + break + + if os.path.exists(TEMP_DIRECTORY): + shutil.rmtree(TEMP_DIRECTORY) + os.mkdir(TEMP_DIRECTORY) + + args = [RUN_NEO_TESTS_COMMAND, '-ufz'] + command = ' '.join(args) + env = {'TEMP': TEMP_DIRECTORY, + 'NEO_TESTS_ADAPTER': test_result_line.name, + 'NEO_TEST_ZODB_FUNCTIONAL': '1', + 'NEO_DB_USER': 'root', + 'NEO_DB_SOCKET': NEO_DB_SOCKET} + try: + with open(os.devnull) as stdin: + p = subprocess.Popen(args, stdin=stdin, stdout=subprocess.PIPE, + stderr=subprocess.PIPE, env=env) + except Exception: + # Catch any exception here, to warn user instead of being silent, + # by generating fake error result + result = dict(status_code=-1, + command=command, + stderr=traceback.format_exc(), + stdout='') + # XXX: inform test node master of error + raise EnvironmentError(result) + + # parse test stdout / stderr, hint to speed up use files first! + stdout, stderr = p.communicate() + date = strftime("%Y/%m/%d %H:%M:%S", gmtime()) + test_count, unexpected_count, expected_count, skip_count, duration = \ + parseTestStdOut(stdout) + + # print to stdout so we can see in testnode logs + sys.stdout.write(stdout) + sys.stderr.write(stderr) + + # report status back to Nexedi ERP5 + test_result_line.stop( + test_count = test_count, + error_count = unexpected_count, # XXX + failure_count = expected_count, # XXX + skip_count = skip_count, + duration = duration, + date = date, + command = command, + stdout= stdout, + stderr= stderr, + html_test_result='') + +if __name__ == "__main__": + main() diff --git a/software/neoppod/software-common.cfg b/software/neoppod/software-common.cfg index 8a19a88c7f8ad48f5388fe89e963f20001f989d3..0d42b020ab3e621f924135234b4cd3e92e387626 100644 --- a/software/neoppod/software-common.cfg +++ b/software/neoppod/software-common.cfg @@ -36,7 +36,7 @@ setup = ${neoppod-repository:location} [neoppod] recipe = zc.recipe.egg -eggs = neoppod[admin, ctl, master, storage-importer, storage-mysqldb] +eggs = neoppod[admin, ctl, master, storage-importer, storage-mysqldb, tests] ${python-mysqlclient:egg} ZODB3 ZODB3-patches = @@ -102,7 +102,7 @@ md5sum = 82f3f76f54ee9db355966a7ada61f56e [instance-neo-storage-mysql] <= download-base-neo -md5sum = 84b1150ce30ec827485f9c17debd6b44 +md5sum = 4572f8d3f92f1b1639600d0eb7119ab5 [template-neo-my-cnf] <= download-base-neo diff --git a/software/neoppod/software-zodb4.cfg b/software/neoppod/software-zodb4.cfg new file mode 100644 index 0000000000000000000000000000000000000000..5896d01e152e9e761b31a8929ee0f765c56a2313 --- /dev/null +++ b/software/neoppod/software-zodb4.cfg @@ -0,0 +1,17 @@ +[buildout] +extends = software.cfg + +[neoppod] +eggs = neoppod + ${python-mysqlclient:egg} + psutil + ZODB + zope.testing +ZODB-patches = + ${neoppod-repository:location}/ZODB.patch +ZODB-patch-options = -p1 + +[versions] +ZODB = 4.2.0+SlapOSPatched001 +transaction = +zdaemon = diff --git a/software/neoppod/software.cfg b/software/neoppod/software.cfg index ec557281f8825de01c96f44020728def33bb900c..1a874bce354454eabf8eb4b4731d3becc346ff2b 100644 --- a/software/neoppod/software.cfg +++ b/software/neoppod/software.cfg @@ -5,23 +5,41 @@ extends = parts += # NEO instanciation template + runTestSuite_py [template] recipe = slapos.recipe.template:jinja2 template = ${:_profile_base_location_}/instance.cfg.in -md5sum = 777c00a7dbcb145a75f980421a9b20b5 +md5sum = aaf5da66d45d4c08cadb0cd1c5342c54 # XXX: "template.cfg" is hardcoded in instanciation recipe rendered = ${buildout:directory}/template.cfg context = key cluster cluster:target key instance_common_cfg instance-common:rendered key root_common root-common:target + key runTestSuite_in runTestSuite.in:target [cluster] <= download-base-neo md5sum = ee8401a4e7d82bf488a57e3399f9ce48 +[runTestSuite.in] +recipe = slapos.recipe.build:download +url = ${:_profile_base_location_}/${:_buildout_section_name_} +md5sum = 1c8d903624310166629a173ecb8ad9f5 + +[runTestSuite_py] +recipe = zc.recipe.egg +eggs = erp5.util +interpreter = ${:_buildout_section_name_} + +[neoppod] +ZODB3-patches += + ${neoppod-repository:location}/ZODB3.patch + [versions] +ZODB3 = 3.10.5+SlapOSPatched002 +erp5.util = 0.4.44 # To match ERP5 transaction = 1.1.1 ZConfig = 2.9.1