task_distribution: better distribute the work depending on the priority of test suites

Before we were looking only create date of test results to give priority of work to do for a test nodes.
So we had cases where some test suites with medium priority had 5 testnodes, and test suites with
high priority having only 3 testnodes.

So clearly distribute the work depending on the maximum quantity of testnode we want.

bt5/erp5_test_result/TestTemplateItem/portal_components/test.erp5.testTaskDistribution.py

@@ -247,6 +247,33 @@ class TestTaskDistribution(ERP5TypeTestCase):
                     [('COMP32-Node1',set([u'B1'])), ('COMP32-Node2',set([u'B0']))]],
                     "%r" % ([config1, config2],))
 
+  def processTest(self, test_title, revision, start_count=2, stop_count=2,
+                  node_title='Node0'):
+    """start_count: number of test line to start
+       stop_count: number of test line to stop
+    """
+    status_dict = {}
+    test_result_path, revision = self._createTestResult(revision=revision,
+      test_list=['testFoo', 'testBar'], test_title=test_title, node_title=node_title)
+    if start_count:
+      line_url, test = self.distributor.startUnitTest(test_result_path,
+                                                      node_title=node_title)
+    if start_count == 2:
+      next_line_url, next_test = self.distributor.startUnitTest(test_result_path,
+                                                                node_title=node_title)
+      self.assertEqual(set(['testFoo', 'testBar']), set([test, next_test]))
+    if stop_count:
+      self.distributor.stopUnitTest(line_url, status_dict, node_title=node_title)
+    if stop_count == 2:
+      self.tool.stopUnitTest(next_line_url, status_dict, node_title=node_title)
+    test_result = self.portal.restrictedTraverse(test_result_path)
+    self.assertEqual(test_result.getSimulationState(), "started")
+    self.tic()
+    if stop_count == 2:
+      self.assertEquals(test_result.getSimulationState(), "stopped")
+    else:
+      self.assertEquals(test_result.getSimulationState(), "started")
+
   def test_04b_startTestSuiteOrder(self):
     """
     When we have many test suites associated to one test nodes, the method
@@ -266,75 +293,194 @@ class TestTaskDistribution(ERP5TypeTestCase):
     # By default we have random order between test suites
     self.assertEquals(set(["test suite 1", "test suite 2", "test suite 3"]),
                       set(getTestSuiteList()))
+
     # Check that if test suite 1 and test suite 2 are recently processed,
     # then next work must be test suite 3
-    def processTest(test_title, revision, start_count=2, stop_count=2):
-      """start_count: number of test line to start
-         stop_count: number of test line to stop
-      """
-      status_dict = {}
-      test_result_path, revision = self._createTestResult(revision=revision,
-        test_list=['testFoo', 'testBar'], test_title=test_title)
-      if start_count:
-        line_url, test = self.tool.startUnitTest(test_result_path)
-      if start_count == 2:
-        next_line_url, next_test = self.tool.startUnitTest(test_result_path)
-        self.assertEqual(set(['testFoo', 'testBar']), set([test, next_test]))
-      if stop_count:
-        self.tool.stopUnitTest(line_url, status_dict)
-      if stop_count == 2:
-        self.tool.stopUnitTest(next_line_url, status_dict)
-      test_result = self.portal.restrictedTraverse(test_result_path)
-      self.assertEqual(test_result.getSimulationState(), "started")
-      self.tic()
-      if stop_count == 2:
-        self.assertEquals(test_result.getSimulationState(), "stopped")
-      else:
-        self.assertEquals(test_result.getSimulationState(), "started")
-
-    processTest("test suite 1", "r0=a")
+    self.processTest("test suite 1", "r0=a")
     self.tic()
     sleep(1) # needed because creation date sql value does not record millesecond
-    processTest("test suite 2", "r0=b")
+    self.processTest("test suite 2", "r0=b")
     self.tic()
     sleep(1)
     self.assertEquals(getTestSuiteList()[0], "test suite 3")
-    processTest("test suite 3", "r0=b")
+    self.processTest("test suite 3", "r0=b")
     # after test suite 3, we now have to process test suite 1
     # since it is the oldest one
     self.tic()
     sleep(1)
     self.assertEquals(getTestSuiteList()[0], "test suite 1")
-    processTest("test suite 1", "r0=c")
+    self.processTest("test suite 1", "r0=c")
     # after test suite 1, we now have to process test suite 2
     # since it is the oldest one
     self.tic()
     sleep(1)
     self.assertEquals(getTestSuiteList()[0], "test suite 2")
-    processTest("test suite 2", "r0=d")
+    self.processTest("test suite 2", "r0=d")
     self.tic()
     sleep(1)
     # now let's say for any reason test suite 1 has been done
-    processTest("test suite 1", "r0=e")
+    self.processTest("test suite 1", "r0=e")
     self.tic()
     sleep(1)
     # we should then have by order 3, 2, 1
     self.assertEquals(["test suite 3", "test suite 2", "test suite 1"],
                       getTestSuiteList())
     # now launch all test of test 3, even if they are not finished yet
-    processTest("test suite 3", "r0=f", stop_count=1)
+    self.processTest("test suite 3", "r0=f", stop_count=1)
     self.tic()
     sleep(1)
     self.assertEquals(["test suite 2", "test suite 1", "test suite 3"],
                       getTestSuiteList())
     # now launch partially tests of suite 2, it must have priority over
     # test 3, even if test 3 is older because all tests of test 3 are ongoing
-    processTest("test suite 2", "r0=g", start_count=1, stop_count=0)
+    self.processTest("test suite 2", "r0=g", start_count=1, stop_count=0)
     self.tic()
     sleep(1)
     self.assertEquals(["test suite 1", "test suite 2", "test suite 3"],
                       getTestSuiteList())
 
+  def test_04c_startTestSuiteOrderWithManyTestNodes(self):
+    """
+    When we have many test suites associated to many test nodes, the method
+    startTestSuite should not only take into account the age of latest test result,
+    but it should also give more test nodes to test suites having higher priority.
+    This avoid affecting all test nodes to a test suite having medium priority
+    while a test suite with high priority is waiting for more nodes to speed up.
+    """
+    for x in  range(10):
+      config_list = json.loads(self.distributor.startTestSuite(
+                             title="COMP%s-Node1" % x))
+    test_suite_list = self._createTestSuite(quantity=3)
+    test_suite_list[0].setIntIndex(1) # test suite 1
+    test_suite_list[1].setIntIndex(5) # test suite 2
+    test_suite_list[2].setIntIndex(9) # test suite 3
+    self.tic()
+
+    self._callOptimizeAlarm()
+    def getTestSuiteList(title):
+      config_list = json.loads(self.distributor.startTestSuite(
+                             title=title))
+      return ["%s" % x["test_suite_title"] for x in config_list]
+    now = DateTime()
+    try:
+      # Check test suite distribution. This is not the purpose of this test
+      # initially, distribution might later change if algorithm is modified.
+      # Though next steps of the test depends on distribution, thus we have
+      # to make sure test does not fails only due to a change of distribution
+      self.assertEquals(set(getTestSuiteList("COMP0-Node1")), set(["test suite 1", "test suite 2", "test suite 3"]))
+      self.assertEquals(set(getTestSuiteList("COMP1-Node1")), set(["test suite 2", "test suite 3"]))
+      self.assertEquals(set(getTestSuiteList("COMP2-Node1")), set(["test suite 2", "test suite 3"]))
+      self.assertEquals(set(getTestSuiteList("COMP3-Node1")), set(["test suite 2", "test suite 3"]))
+      # COMP4 as less than COMP5 only due to distribution algorithm, though
+      # this is not an issue at all
+      self.assertEquals(set(getTestSuiteList("COMP4-Node1")), set(["test suite 3"]))
+      self.assertEquals(set(getTestSuiteList("COMP5-Node1")), set(["test suite 2", "test suite 3"]))
+      self.assertEquals(set(getTestSuiteList("COMP6-Node1")), set(["test suite 3"]))
+      self.assertEquals(set(getTestSuiteList("COMP7-Node1")), set(["test suite 3"]))
+      self.assertEquals(set(getTestSuiteList("COMP8-Node1")), set(["test suite 3"]))
+      self.assertEquals(set(getTestSuiteList("COMP9-Node1")), set(["test suite 3"]))
+      # process some test to have old test result in database
+      self.processTest("test suite 1", "r0=a", node_title="COMP0-Node1")
+      self.pinDateTime(now + 1.0/86400)
+      self.processTest("test suite 2", "r0=a", node_title="COMP1-Node1")
+      self.pinDateTime(now + 2.0/86400)
+      self.processTest("test suite 3", "r0=a", node_title="COMP1-Node1")
+      # so by default, when no test is running, older test suite is given first
+      self.assertEquals(["test suite 1", "test suite 2", "test suite 3"],
+            getTestSuiteList("COMP0-Node1"))
+      self.assertEquals(["test suite 2", "test suite 3"],
+            getTestSuiteList("COMP5-Node1"))
+      # Start some work
+      self.pinDateTime(now + 3.0/86400)
+      self.processTest("test suite 2", "r0=b", node_title="COMP1-Node1", start_count=0, stop_count=0)
+      # COMP1-Node1 : test suite 2
+      # suite 1 is now the newest, it is checked last, test suite 3 has priority on test suite 2
+      # since no test is running for it
+      self.assertEquals(["test suite 1", "test suite 3", "test suite 2"],
+            getTestSuiteList("COMP0-Node1"))
+      self.assertEquals(["test suite 3", "test suite 2"],
+            getTestSuiteList("COMP5-Node1"))
+      # Now let's say test suite 3 is starting too
+      self.pinDateTime(now + 4.0/86400)
+      self.processTest("test suite 3", "r0=b", node_title="COMP6-Node1", start_count=0, stop_count=0)
+      # COMP1-Node1 : test suite 2
+      # COMP6-Node1 : test suite 3
+      # test suite 3 still should have the priority other test suite 2, because it
+      # has higher priority
+      self.assertEquals(["test suite 3", "test suite 2"],
+            getTestSuiteList("COMP5-Node1"))
+      # another test node working on test suite 3
+      self.processTest("test suite 3", "r0=b", node_title="COMP7-Node1", start_count=0, stop_count=0)
+      # COMP1-Node1 : test suite 2
+      # COMP6-Node1 : test suite 3
+      # COMP7-Node1 : test suite 3
+      # Now we have 2 testnodes for test suite 3, and only 1 for test suite 2, time to give
+      # more priority to test suite 2
+      self.assertEquals(["test suite 2", "test suite 3"],
+            getTestSuiteList("COMP5-Node1"))
+      self.assertEquals(["test suite 1", "test suite 2", "test suite 3"],
+            getTestSuiteList("COMP0-Node1"))
+      # so now a testnode working on test suite 2
+      self.processTest("test suite 2", "r0=b", node_title="COMP2-Node1", start_count=0, stop_count=0)
+      # COMP1-Node1 : test suite 2
+      # COMP2-Node1 : test suite 2
+      # COMP6-Node1 : test suite 3
+      # COMP7-Node1 : test suite 3
+      # Now we have 2 testnodes for test suite 3, and 2 for test suite 2, time to give
+      # more priority to test suite 3
+      self.assertEquals(["test suite 3", "test suite 2"],
+            getTestSuiteList("COMP5-Node1"))
+      self.processTest("test suite 3", "r0=b", node_title="COMP8-Node1", start_count=0, stop_count=0)
+      # COMP1-Node1 : test suite 2
+      # COMP2-Node1 : test suite 2
+      # COMP6-Node1 : test suite 3
+      # COMP7-Node1 : test suite 3
+      # COMP8-Node1 : test suite 3
+      # Now we have 3 testnodes for test suite 3, and 2 for test suite 2, test suite 3 still
+      # need priority due to higher priority
+      self.assertEquals(["test suite 3", "test suite 2"],
+            getTestSuiteList("COMP5-Node1"))
+      self.processTest("test suite 3", "r0=b", node_title="COMP9-Node1", start_count=0, stop_count=0)
+      # COMP1-Node1 : test suite 2
+      # COMP2-Node1 : test suite 2
+      # COMP6-Node1 : test suite 3
+      # COMP7-Node1 : test suite 3
+      # COMP8-Node1 : test suite 3
+      # COMP9-Node1 : test suite 3
+      # Now we have 4 testnodes for test suite 3, and 2 for test suite 2, test suite 2 will
+      # have again priority
+      self.assertEquals(["test suite 2", "test suite 3"],
+            getTestSuiteList("COMP5-Node1"))
+      self.assertEquals(["test suite 1", "test suite 2", "test suite 3"],
+            getTestSuiteList("COMP0-Node1"))
+      self.processTest("test suite 2", "r0=b", node_title="COMP3-Node1", start_count=0, stop_count=0)
+      # COMP1-Node1 : test suite 2
+      # COMP2-Node1 : test suite 2
+      # COMP3-Node1 : test suite 2
+      # COMP6-Node1 : test suite 3
+      # COMP7-Node1 : test suite 3
+      # COMP8-Node1 : test suite 3
+      # COMP9-Node1 : test suite 3
+      # Now we have 4 testnodes for test suite 3, and 3 for test suite 2, test suite 3 will
+      # have again priority
+      self.assertEquals(["test suite 3", "test suite 2"],
+            getTestSuiteList("COMP5-Node1"))
+      self.processTest("test suite 3", "r0=b", node_title="COMP4-Node1", start_count=0, stop_count=0)
+      # COMP1-Node1 : test suite 2
+      # COMP2-Node1 : test suite 2
+      # COMP3-Node1 : test suite 2
+      # COMP4-Node1 : test suite 3
+      # COMP6-Node1 : test suite 3
+      # COMP7-Node1 : test suite 3
+      # COMP8-Node1 : test suite 3
+      # COMP9-Node1 : test suite 3
+      self.assertEquals(["test suite 3", "test suite 2"],
+            getTestSuiteList("COMP5-Node1"))
+      self.assertEquals(["test suite 1", "test suite 3", "test suite 2"],
+            getTestSuiteList("COMP0-Node1"))
+    finally:
+      self.unpinDateTime()
+
   def _cleanupTestResult(self):
     self.tic()
     cleanup_state_list = ['started', 'stopped']

product/ERP5/Document/ERP5ProjectUnitTestDistributor.py

@@ -303,6 +303,7 @@ class ERP5ProjectUnitTestDistributor(XMLObject):
     # sql server
     now = DateTime()
     from_date = now - 30
+    max_test_core_per_suite = max([x[1] for x in PRIORITY_MAPPING.values()])
     def getTestSuiteSortKey(test_suite):
       test_result_list = portal.portal_catalog(portal_type="Test Result",
                                           title=SimpleQuery(title=test_suite.getTitle()),
@@ -314,16 +315,27 @@ class ERP5ProjectUnitTestDistributor(XMLObject):
                                           limit=1)
       if len(test_result_list):
         test_result = test_result_list[0].getObject()
-        key = test_result.getModificationDate().timeTime()
+        modification_date = test_result.getModificationDate().timeTime()
+        key = (1, modification_date)
         # if a test result has all it's tests already ongoing, it is not a
         # priority at all to process it, therefore push it at the end of the list
         if test_result.getSimulationState() == "started":
           result_line_list = test_result.objectValues(portal_type="Test Result Line")
+          check_priority = True
           if len(result_line_list):
             if len([x for x in result_line_list if x.getSimulationState() == "draft"]) == 0:
-              key = now.timeTime()
+              key = (1000, now.timeTime())
+              check_priority = False
+          if check_priority:
+            # calculate key[0] in such a way that more the test suite has high
+            # priority and the more test result lack test node, the lower is key[0]
+            # This allows to affect more test nodes to test suites with higher priority
+            wanted_test_core_quantity = PRIORITY_MAPPING[test_suite.getIntIndex()][1]
+            factor = max_test_core_per_suite / wanted_test_core_quantity
+            missing_quantity = wanted_test_core_quantity/3 - len(test_result.objectValues(portal_type="Test Result Node"))
+            key = (max_test_core_per_suite - missing_quantity * 3 * factor, modification_date)
       else:
-        key = random.random()
+        key = (0, random.random())
       return key
     test_suite_list.sort(key=getTestSuiteSortKey)
     return test_suite_list