erp5_test_result: ERP5ScalabilityDistributor refactoring

- code for generateConfigurationList moved to scalability distributor
- general code refactoring and cleanup

bt5/erp5_test_result/DocumentTemplateItem/portal_components/document.erp5.ERP5ScalabilityDistributor.py

@@ -32,6 +32,8 @@ from zLOG import LOG,ERROR
 from AccessControl import ClassSecurityInfo
 from Products.ERP5Type import Permissions
 import json
+import jinja2
+from Products.ERP5.Tool.TaskDistributionTool import TaskDistributionTool
 
 class ERP5ScalabilityDistributor(ERP5ProjectUnitTestDistributor):
   security = ClassSecurityInfo()
@@ -42,7 +44,6 @@ class ERP5ScalabilityDistributor(ERP5ProjectUnitTestDistributor):
     """
     Do master testnode selection and associate testsuites.
     """
-
     # Get modules
     test_node_module = self._getTestNodeModule()
     test_suite_module = self._getTestSuiteModule()
@@ -50,22 +51,25 @@ class ERP5ScalabilityDistributor(ERP5ProjectUnitTestDistributor):
     # Get lists of test node wich belong to the current distributor
     all_test_node_list = test_node_module.searchFolder(
         portal_type="Test Node", specialise_uid=self.getUid())
-    test_node_list = [ x.getObject() for x in all_test_node_list if x.getValidationState() == 'validated' ]
-    invalid_test_node_list = [ x.getObject() for x in all_test_node_list if x.getValidationState() != 'validated' ]
+    test_node_list = [ x.getObject() for x in all_test_node_list
+                        if x.getValidationState() == 'validated' ]
+    invalid_test_node_list = [ x.getObject() for x in all_test_node_list
+                                if x.getValidationState() != 'validated' ]
 
     # Set master property to False for all invalid testnode
     for node in invalid_test_node_list:
       node.setMaster(False)
 
     # Get all valid slave testnodes
-    slave_test_node_list = [ x.getObject() for x in test_node_list if x.getMaster()!=True ]
-    master_test_node_list = [ x.getObject() for x in test_node_list if x.getMaster()==True ]
+    slave_test_node_list = [ x.getObject() for x in test_node_list
+                              if x.getMaster()!=True ]
+    master_test_node_list = [ x.getObject() for x in test_node_list
+                               if x.getMaster()==True ]
 
     # Set to 'False' slaves
     for node in slave_test_node_list:
       node.setMaster(False)
 
-
     # if there is validated testnodes
     if len(test_node_list) > 0:
       # Only one testnode must be the master
@@ -80,8 +84,10 @@ class ERP5ScalabilityDistributor(ERP5ProjectUnitTestDistributor):
             node.setMaster(False)
 
     # Update testnode lists
-    slave_test_node_list = [ x.getObject() for x in test_node_list if x.getMaster()!=True ]
-    master_test_node_list = [ x.getObject() for x in test_node_list if x.getMaster()==True ]
+    slave_test_node_list = [ x.getObject() for x in test_node_list
+                              if x.getMaster()!=True ]
+    master_test_node_list = [ x.getObject() for x in test_node_list
+                               if x.getMaster()==True ]
 
     # Get test suites and aggregate them to all valid testnode
     test_suite_list = [ x.getRelativeUrl() for x in test_suite_module.searchFolder(
@@ -97,8 +103,9 @@ class ERP5ScalabilityDistributor(ERP5ProjectUnitTestDistributor):
     """
     test_node_module = self._getTestNodeModule()
     distributor_title = self.getTitle()
-    involved_nodes = []   
-    involved_nodes = involved_nodes + [ x.getObject() for x in test_node_module.searchFolder(validation_state='validated') if ( x.getSpecialiseTitle() == distributor_title ) ]
+    involved_nodes = [ x.getObject() for x in test_node_module.searchFolder(
+                       validation_state='validated')
+                       if ( x.getSpecialiseTitle() == distributor_title ) ]
     return involved_nodes
 
   security.declarePublic("getTestNode")
@@ -130,10 +137,12 @@ class ERP5ScalabilityDistributor(ERP5ProjectUnitTestDistributor):
   security.declarePublic("isMasterTestnode")
   def isMasterTestnode(self,title, batch_mode=0):
     """
-    isMasterTestnode : return True if the node given in parameter exists and is a validated master
+    isMasterTestnode : return True if the node given in
+    parameter exists and is a validated master
     """
     test_node_module = self._getTestNodeModule()
-    test_node_master = [ node for node in test_node_module.searchFolder(portal_type="Test Node",
+    test_node_master = [ node for node in test_node_module.searchFolder(
+                              portal_type="Test Node",
                               title=SimpleQuery(comparison_operator='=', title=title),
                               specialise_uid=self.getUid(),
                               validation_state="validated") if node.getMaster() == 1 ]
@@ -142,7 +151,6 @@ class ERP5ScalabilityDistributor(ERP5ProjectUnitTestDistributor):
     else:
       return False
 
-
   security.declarePublic("getTestType")
   def getTestType(self, batch_mode=0):
     """
@@ -150,21 +158,34 @@ class ERP5ScalabilityDistributor(ERP5ProjectUnitTestDistributor):
     """
     return 'ScalabilityTest'
 
+  security.declarePublic("createTestResult")
+  def createTestResult(self, name, revision, test_name_list, allow_restart,
+                       test_title=None, node_title=None, project_title=None):
+    """
+    Here this is only a proxy to the task distribution tool
+    """
+    LOG('[ERP5ScalabilityDistributor] createTestResult', 0, (node_title, test_title))
+    portal = self.getPortalObject()
+    return portal.portal_task_distribution.createTestResult(name,
+         revision, test_name_list, allow_restart,
+         test_title=test_title, node_title=node_title,
+         project_title=project_title)
 
   security.declarePublic("startTestSuite")
   def startTestSuite(self,title, computer_guid='unknown', batch_mode=0):
     """
     startTestSuite : subscribe node + return testsuite list to the master
     """
-    super(ERP5ScalabilityDistributor,
-                        self).subscribeNode(title=title, computer_guid=computer_guid, batch_mode=batch_mode)
+    super(ERP5ScalabilityDistributor, self).subscribeNode(
+      title=title, computer_guid=computer_guid, batch_mode=batch_mode)
     test_node_module =  self._getTestNodeModule()
 
     # If the testnode wich request testsuites is not the master
     # he does not have to receive any testsuites
     master_test_node_title_list = [x.getTitle() for x in test_node_module.searchFolder(
-                validation_state = 'validated') if (x.getMaster() == True) ]
-
+                                                            validation_state = 'validated',
+                                                            specialise_uid=self.getUid())
+                                   if (x.getMaster() == True) ]
     if len(master_test_node_title_list) == 1 and title == master_test_node_title_list[0]:
       return super(ERP5ScalabilityDistributor,
                    self).startTestSuite(title=title,
@@ -174,13 +195,13 @@ class ERP5ScalabilityDistributor(ERP5ProjectUnitTestDistributor):
         return []
       return json.dumps([])
 
-
   security.declarePublic("generateConfiguration")
   def generateConfiguration(self, test_suite_title, batch_mode=0):
     """
     generateConfiguration : this is just a proxy to an external method
     """
-    generated_configuration = self.ScalabilityTestSuiteUtils_generateConfigurationList(self, test_suite_title)
+    portal = self.getPortalObject()
+    generated_configuration = self._generateConfigurationList(test_suite_title, portal)
     if batch_mode:
       return generated_configuration
     return json.dumps(generated_configuration)
@@ -201,14 +222,13 @@ class ERP5ScalabilityDistributor(ERP5ProjectUnitTestDistributor):
       if state == "started":
         relative_path = test_result_line.getRelativeUrl()
         title = test_result_line.getTitle()
-        break;
+        break
     else:
       return None
 
     next_test = {"relative_path": relative_path,
                  "title": title, "count" : count,
                 }
-
     if batch_mode:
       return next_test
     return json.dumps(next_test)
@@ -221,3 +241,113 @@ class ERP5ScalabilityDistributor(ERP5ProjectUnitTestDistributor):
     portal = self.getPortalObject()
     test_result_line = portal.restrictedTraverse(test_result_line_path)
     return test_result_line.getSimulationState() == "started"
+
+  security.declarePublic("_unvalidateConfig")
+  def _unvalidateConfig(self, my_dict):
+    my_dict['launchable'] = False
+    if 'launcher_nodes_computer_guid' in my_dict:
+      my_dict['launcher_nodes_computer_guid'] = {}
+    if 'involved_nodes_computer_guid' in my_dict:
+      my_dict['involved_nodes_computer_guid'] = {}
+    if 'configuration_list' in my_dict:
+      my_dict['configuration_list'] = []
+
+  security.declarePublic("_getInvolvedNodes")
+  def _getInvolvedNodes(self, available_nodes, return_dict):
+    """
+    Get the list of all nodes involved in the test
+    """
+    def _isContained(my_value, my_container):
+      def _isInMyDictOrList(current_container):
+        if current_container == my_value :
+          return True
+        elif isinstance(current_container, dict):
+          for k,v in current_container.items():
+            if str(my_value) in str(k):
+              return True
+            if _isInMyDictOrList(current_container[k]) :
+              return True
+        elif isinstance(current_container, list):
+          for sub_container in current_container:
+            if _isInMyDictOrList(sub_container) :
+              return True
+        return False
+      return _isInMyDictOrList(my_container)
+
+    involved_nodes_computer_guid = []
+    for node in available_nodes:
+      if _isContained(node, return_dict):
+        involved_nodes_computer_guid.append(node)
+    return involved_nodes_computer_guid
+
+  security.declarePublic("_getAvailableNodes")
+  def _getAvailableNodes(self, test_suite_specialise_title, portal):
+    """
+    gets testnodes available for this distributor
+    """
+    distributor_uid = self.getUid()
+    test_node_module = portal.test_node_module
+    available_nodes = test_node_module.searchFolder(
+            portal_type="Test Node", validation_state="validated",
+            specialise_uid=distributor_uid)
+    available_nodes = [ node.getComputerGuid() for node in available_nodes ]
+    return available_nodes
+
+  security.declarePublic("_generateConfigurationList")
+  def _generateConfigurationList(self, test_suite_title, portal):
+    """
+    generateConfigurationList : generate a dict wich contains a list
+    of cluster configurations for testnodes, using a test_suite and
+    available testnodes.
+    If it is not possible to generate a configuration, the 'launchable'
+    entry is False, otherwise it is True.
+    """
+    return_dict = {}
+    # Get test_suite informations from his title given in parameter
+    try:
+      test_suite = portal.test_suite_module.searchFolder(title=test_suite_title)[0]
+      cluster_configuration = test_suite.getClusterConfiguration()
+      number_configuration_list = test_suite.getGraphCoordinate()
+      randomized_path = test_suite.getRandomizedPath()
+    except Exception as e:
+      return_dict['error_message'] = 'Error getting test suite information. \
+                                      Bad test suite title? ' + str(e)
+      self._unvalidateConfig(return_dict)
+      return return_dict
+
+    try:
+      available_nodes = self._getAvailableNodes(test_suite.getSpecialiseTitle(), portal)
+      remaining_nodes = list(available_nodes)
+      launcher_nodes = [ remaining_nodes.pop() ]
+      if len(remaining_nodes) == 0: # if there is only one computer, use launcher
+        remaining_nodes.append(launcher_nodes[0])
+    except Exception:
+      return_dict['error_message'] = 'Error getting available nodes. \
+                                      No test nodes for test suite?'
+      self._unvalidateConfig(return_dict)
+      return return_dict
+
+    configuration_list_json = []
+    return_dict['error_message'] = 'No error.'
+    return_dict['randomized_path'] = randomized_path
+
+    try:
+      template = jinja2.Template(cluster_configuration)
+      for index in xrange(0, len(number_configuration_list)):
+        template_vars = { "count" : number_configuration_list[index],
+                          "comp" : remaining_nodes }
+        configuration_list_json.append( json.loads(
+                                          template.render(template_vars)
+                                        )
+                                      )
+      return_dict['launchable'] = True
+      return_dict['configuration_list'] = configuration_list_json
+      return_dict['launcher_nodes_computer_guid'] = launcher_nodes
+    except Exception:
+      return_dict['error_message'] = 'Bad json cluster_configuration?'
+      self._unvalidateConfig(return_dict)
+      return return_dict
+
+    return_dict['involved_nodes_computer_guid'] = self._getInvolvedNodes(available_nodes,
+                                                                         return_dict)
+    return return_dict