From f552f88eea47c2bea49f1ecbe7287f02c27d8a4a Mon Sep 17 00:00:00 2001 From: Sebastien Robin <seb@nexedi.com> Date: Mon, 4 Feb 2013 15:24:10 +0100 Subject: [PATCH] initial import of distributors used for unit testing --- .../CloudPerformanceUnitTestDistributor.py | 84 +++++ .../ERP5ProjectUnitTestDistributor.py | 310 ++++++++++++++++++ 2 files changed, 394 insertions(+) create mode 100644 product/ERP5/Document/CloudPerformanceUnitTestDistributor.py create mode 100644 product/ERP5/Document/ERP5ProjectUnitTestDistributor.py diff --git a/product/ERP5/Document/CloudPerformanceUnitTestDistributor.py b/product/ERP5/Document/CloudPerformanceUnitTestDistributor.py new file mode 100644 index 0000000000..4f63dd72ee --- /dev/null +++ b/product/ERP5/Document/CloudPerformanceUnitTestDistributor.py @@ -0,0 +1,84 @@ +############################################################################## +# Copyright (c) 2013 Nexedi SA and Contributors. All Rights Reserved. +# Sebastien Robin <seb@nexedi.com> +# +# WARNING: This program as such is intended to be used by professional +# programmers who take the whole responsibility of assessing all potential +# consequences resulting from its eventual inadequacies and bugs +# End users who are looking for a ready-to-use solution with commercial +# guarantees and support are strongly advised to contract a Free Software +# Service Company +# +# This program is Free Software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License +# as published by the Free Software Foundation; either version 2 +# of the License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +# +############################################################################## +from ERP5ProjectUnitTestDistributor import ERP5ProjectUnitTestDistributor +import json +from zLOG import LOG,INFO,ERROR +from AccessControl import ClassSecurityInfo +from Products.ERP5Type import Permissions + +class CloudPerformanceUnitTestDistributor(ERP5ProjectUnitTestDistributor): + + security = ClassSecurityInfo() + security.declareObjectProtected(Permissions.AccessContentsInformation) + + security.declareProtected(Permissions.ManagePortal, + "cleanupInvalidatedTestNode") + def cleanupInvalidatedTestNode(self, test_node): + """ + When a test node is invalidated, we keep current configuration. Since + all test nodes runs all test suites, this is not a problem to keep + configuration + """ + pass + + security.declarePublic("optimizeConfiguration") + def optimizeConfiguration(self): + """ + We are going to add test suites to test nodes. + In the case of cloud performance, we associate all test suites to + every test node. Like this, every test suite will be executed by + every test node + """ + portal = self.getPortalObject() + test_node_module = self._getTestNodeModule() + test_suite_module = self._getTestSuiteModule() + test_node_list = [ + x.getObject() for x in test_node_module.searchFolder( + portal_type="Test Node", validation_state="validated", + specialise_uid=self.getUid())] + + test_suite_list = [x.getRelativeUrl() + for x in test_suite_module.searchFolder( + validation_state="validated", + specialise_uid=self.getUid())] + for test_node in test_node_list: + test_node.setAggregateList(test_suite_list) + + security.declarePublic("startTestSuite") + def startTestSuite(self,title): + """ + give the list of test suite to start. We will take all test suites + associated to the testnode. Then we add the test node title to the + test_suite_title to make sure that every test node will have it's + own test result without the possibility that another one participate + to the same test. + """ + config_list = super(CloudPerformanceUnitTestDistributor, + self).startTestSuite(title, batch_mode=1) + for config in config_list: + config["test_suite_title"] = config["test_suite_title"] + "-%s" % title + return json.dumps(config_list) \ No newline at end of file diff --git a/product/ERP5/Document/ERP5ProjectUnitTestDistributor.py b/product/ERP5/Document/ERP5ProjectUnitTestDistributor.py new file mode 100644 index 0000000000..b403f41820 --- /dev/null +++ b/product/ERP5/Document/ERP5ProjectUnitTestDistributor.py @@ -0,0 +1,310 @@ +############################################################################## +# Copyright (c) 2013 Nexedi SA and Contributors. All Rights Reserved. +# Sebastien Robin <seb@nexedi.com> +# +# WARNING: This program as such is intended to be used by professional +# programmers who take the whole responsibility of assessing all potential +# consequences resulting from its eventual inadequacies and bugs +# End users who are looking for a ready-to-use solution with commercial +# guarantees and support are strongly advised to contract a Free Software +# Service Company +# +# This program is Free Software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License +# as published by the Free Software Foundation; either version 2 +# of the License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +# +############################################################################## +from Products.ERP5Type.XMLObject import XMLObject +from Products.ERP5.Tool.TaskDistributionTool import TaskDistributionTool +from DateTime import DateTime +from datetime import datetime +import json +import sys +import itertools +from copy import deepcopy +import random +import string +from zLOG import LOG,INFO,ERROR +from AccessControl import ClassSecurityInfo +from Products.ERP5Type import Permissions +TEST_SUITE_MAX = 4 +# Depending on the test suite priority, we will affect +# more or less cores +PRIORITY_MAPPING = { + # int_index: (min cores, max cores) + 1: ( 3, 3), + 2: ( 3, 3), + 3: ( 3, 6), + 4: ( 3, 6), + 5: ( 3, 6), + 6: ( 6, 9), + 7: ( 6, 9), + 8: ( 6, 9), + 9: ( 9, 15), + } + +class ERP5ProjectUnitTestDistributor(XMLObject): + + security = ClassSecurityInfo() + security.declareObjectProtected(Permissions.AccessContentsInformation) + + security.declareProtected(Permissions.ManagePortal, + "cleanupInvalidatedTestNode") + def cleanupInvalidatedTestNode(self, test_node): + """ + When a test node is invalidated, the work will be distributed to + other test nodes, so we should clean association to test suites. + Like this, when this node will come back, we will not mess distribution + with stuff already distributed in other places + """ + if test_node.getAggregateList(): + test_node.setAggregateList([]) + + def _cleanupTestNodeList(self,test_node_list, test_suite_list_to_remove): + # Remove useless assigment of test suites. First remove from + # nodes with highest number of test suites + # test_suite_list_to_remove could be like ['foo','foo', 'bar'] + test_suite_list_to_remove.sort() + while len(test_suite_list_to_remove): + test_node_list.sort(key=lambda x: -len(x.getAggregateList())) + current_test_suite = test_suite_list_to_remove[0] + for test_node in test_node_list: + test_suite_list = test_node.getAggregateList() + if current_test_suite in test_suite_list: + test_suite_list.remove(current_test_suite) + test_node.setAggregateList(test_suite_list) + test_suite_list_to_remove.remove(current_test_suite) + if len(test_suite_list_to_remove): + if test_suite_list_to_remove[0] != current_test_suite: + break + else: + break + + def _checkCurrentConfiguration(self,test_node_list, test_suite_list_to_add): + """ + We look at what is already installed and then we remove from the list + of test suite list to add what is already installed. + We also build a list of installed test suites that should be removed + """ + test_suite_list_to_remove = [] + for test_node in test_node_list: + test_suite_list = test_node.getAggregateList() + for test_suite_title in test_suite_list: + try: + test_suite_list_to_add.remove(test_suite_title) + except ValueError: + test_suite_list_to_remove.append(test_suite_title) + return test_suite_list_to_remove + + security.declareProtected(Permissions.ManagePortal, "optimizeConfiguration") + def optimizeConfiguration(self): + """ + We are going to add test suites to test nodes. + First are completed test nodes with fewer test suites + """ + portal = self.getPortalObject() + test_node_module = self._getTestNodeModule() + test_node_list = [ + x.getObject() for x in test_node_module.searchFolder( + portal_type="Test Node", validation_state="validated", + specialise_uid=self.getUid(), sort_on=[('title','ascending')])] + + test_node_list_len = len(test_node_list) + def _optimizeConfiguration(test_suite_list_to_add, level=0): + if test_suite_list_to_add: + test_node_list_to_remove = [] + for test_node in test_node_list: + # We can no longer add more test suite on this test node + if TEST_SUITE_MAX < (level + 1): + test_node_list_to_remove.append(test_node) + continue + test_suite_list = test_node.getAggregateList() + if len(test_suite_list) == level: + for test_suite in test_suite_list_to_add: + if not(test_suite in test_suite_list): + test_node.setAggregateList([test_suite] + test_suite_list) + test_suite_list_to_add.remove(test_suite) + break + if len(test_suite_list_to_add) == 0: + break + for test_node in test_node_list_to_remove: + test_node_list.remove(test_node) + if test_suite_list_to_add and test_node_list: + _optimizeConfiguration(test_suite_list_to_add, level=level+1) + + test_suite_list_to_add = self._getSortedNodeTestSuiteList() + test_suite_list_to_remove = self._checkCurrentConfiguration(test_node_list, + test_suite_list_to_add) + self._cleanupTestNodeList(test_node_list, test_suite_list_to_remove) + _optimizeConfiguration(test_suite_list_to_add) + + def _getSortedNodeTestSuiteList(self): + """ + We build the list of test suite instances. If a test suite + can be installed on at most 2 test nodes, it will be twice + in the returned list. We give a score for every wished test suites. + The lower score, the better chance it has to be installed. + """ + test_suite_module = self._getTestSuiteModule() + portal = self.getPortalObject() + test_suite_list = test_suite_module.searchFolder(validation_state="validated", + specialise_uid=self.getUid()) + all_test_suite_list = [] + for test_suite in test_suite_list: + test_suite = test_suite.getObject() + test_suite_url = test_suite.getRelativeUrl() + title = test_suite.getTitle() + # suites required + int_index = test_suite.getIntIndex() + # we divide per 3 because we have 3 cores per node + node_quantity_min = PRIORITY_MAPPING[int_index][0]/3 + node_quantity_max = PRIORITY_MAPPING[int_index][1]/3 + for x in xrange(0, node_quantity_min): + all_test_suite_list.append((x/(x+1),test_suite_url, title)) + # additional suites, lower score + for x in xrange(0, node_quantity_max - + node_quantity_min ): + all_test_suite_list.append((1 + x/(x+1), test_suite_url, title)) + all_test_suite_list.sort(key=lambda x: (x[0], x[2])) + return [x[1] for x in all_test_suite_list] + + def _getTestNodeModule(self): + return self.getPortalObject().test_node_module + + def _getTestSuiteModule(self): + return self.getPortalObject().test_suite_module + + def getMemcachedDict(self): + portal = self.getPortalObject() + memcached_dict = portal.portal_memcached.getMemcachedDict( + "task_distribution", "default_memcached_plugin") + return memcached_dict + + security.declarePublic("startTestSuite") + def startTestSuite(self,title, batch_mode=0): + """ + startTestSuite doc + """ + test_node_module = self._getTestNodeModule() + test_suite_module = self._getTestSuiteModule() + portal = self.getPortalObject() + config_list = [] + tag = "%s_%s" % (self.getRelativeUrl(), title) + if portal.portal_activities.countMessageWithTag(tag) == 0: + test_node_list = test_node_module.searchFolder(portal_type="Test Node",title=title) + assert len(test_node_list) in (0, 1), "Unable to find testnode : %s" % title + test_node = None + if len(test_node_list) == 1: + test_node = test_node_list[0].getObject() + if test_node.getValidationState() != 'validated': + try: + test_node.validate() + except e: + LOG('Test Node Validate',ERROR,'%s' %e) + if test_node is None: + test_node = test_node_module.newContent(portal_type="Test Node", title=title, + specialise=self.getRelativeUrl(), + activate_kw={'tag': tag}) + self.activate(after_tag=tag).optimizeConfiguration() + test_node.setPingDate() + test_suite_list = test_node.getAggregateList() + # We sort the list according to timestamp + choice_list = [] + if len(test_suite_list): + choice_list = [x.getObject() for x in test_suite_module.searchFolder( + relative_url=test_suite_list, + sort_on=[('indexation_timestamp','ascending')], + )] + # XXX we should have first test suite with no test node working on + # them since a long time. However we do not have this information yet, + # so random sort is better for now. + choice_list.sort(key=lambda x: random.random()) + for test_suite in choice_list: + config = {} + config["project_title"] = test_suite.getSourceProjectTitle() + config["test_suite"] = test_suite.getTestSuite() + config["test_suite_title"] = test_suite.getTitle() + config["additional_bt5_repository_id"] = test_suite.getAdditionalBt5RepositoryId() + config["test_suite_reference"] = test_suite.getReference() + vcs_repository_list = [] + #In this case objectValues is faster than searchFolder + for repository in test_suite.objectValues(portal_type="Test Suite Repository"): + repository_dict = {} + for property_name in ["git_url", "profile_path", "buildout_section_id", "branch"]: + property_value = repository.getProperty(property_name) + # the property name url returns the object's url, so it is mandatory use another name. + if property_name == "git_url": + property_name="url" + if property_value is not None: + repository_dict[property_name] = property_value + vcs_repository_list.append(repository_dict) + config["vcs_repository_list"] = vcs_repository_list + to_delete_key_list = [x for x,y in config.items() if y==None] + [config.pop(x) for x in to_delete_key_list] + config_list.append(config) + LOG('ERP5ProjectUnitTestDistributor.startTestSuite, config_list',INFO,config_list) + if batch_mode: + return config_list + return json.dumps(config_list) + + security.declarePublic("createTestResult") + def createTestResult(self, name, revision, test_name_list, allow_restart, + test_title=None, node_title=None, project_title=None): + """ + Here this is only a proxy to the task distribution tool + """ + LOG('ERP5ProjectUnitTestDistributor.createTestResult', 0, (node_title, test_title)) + portal = self.getPortalObject() + test_node = self._getTestNodeFromTitle(node_title) + test_node.setPingDate() + test_suite = self._getTestSuiteFromTitle(name) + test_suite.setPingDate() + return portal.portal_task_distribution_tool.createTestResult(name, + revision, test_name_list, allow_restart, + test_title=title_title, node_title=node_title, + project_title=project_title) + + def _getTestNodeFromTitle(self, node_title): + test_node_list = self._getTestNodeModule().searchFolder( + portal_type='Test Node', title="=%s" % node_title) + assert len(test_node_list) == 1, "We found %i test nodes for %s" % ( + len(test_node_list), node_title) + test_node = test_node_list[0].getObject() + return test_node + + def _getTestSuiteFromTitle(self, suite_title): + test_suite_list = self._getTestSuiteModule().searchFolder( + portal_type='Test Suite', title="=%s" % suit_tile, validation_state="validated") + assert len(test_suite_list) == 1, "We found %i test suite for %s" % ( + len(test_suite_list), name) + test_suite = test_suite_list[0].getObject() + + security.declarePublic("startUnitTest") + def startUnitTest(self,test_result_path,exclude_list=()): + """ + Here this is only a proxy to the task distribution tool + """ + LOG('ERP5ProjectUnitTestDistributor.startUnitTest', 0, test_result_path) + portal = self.getPortalObject() + return portal.portal_task_distribution_tool.startUnitTest(test_result_path,exclude_list) + + security.declarePublic("stopUnitTest") + def stopUnitTest(self,test_path,status_dict): + """ + Here this is only a proxy to the task distribution tool + """ + LOG('ERP5ProjectUnitTestDistributor.stop_unit_test', 0, test_path) + portal = self.getPortalObject() + test_result = portal.unrestrictedTraverse(test_path) + test_suite_title = test_result.getTitle() + return portal.portal_task_distribution_tool.stopUnitTest(self,test_path,status_dict) \ No newline at end of file -- 2.30.9