erp5testnode: Use a test suite dedicated log file instead of addWatcher()

erp5/tests/testERP5TestNode.py

@@ -15,6 +15,8 @@ import subprocess
 import tempfile
 import json
 import time
+import types
+import re
 
 class ERP5TestNode(TestCase):
 
@@ -497,7 +499,7 @@ branch = foo
     createFolder(folder, clean=True)
     self.assertEquals(False, os.path.exists(to_drop_path))
 
-  def test_15_log_directory(self):
+  def test_15_suite_log_directory(self):
     def doNothing(self, *args, **kw):
         pass
     test_self = self
@@ -508,16 +510,11 @@ branch = foo
     def patch_startTestSuite(self,test_node_title):
       global counter
       config_list = []
-      def _checkExistingTestSuite(reference_set):
-        test_self.assertEquals(set(reference_set),
-                    set(os.listdir(test_node.config["working_directory"])))
-        for x in reference_set:
-          test_self.assertTrue(os.path.exists(os.path.join(
-                               test_node.config["working_directory"],x)),True)
       if counter == 0:
-        config_list.append(test_self.getTestSuiteData(reference='foo')[0])
-      elif counter == 1:
-        _checkExistingTestSuite(set(['foo']))
+        config_list.append(test_self.getTestSuiteData(reference='aa')[0])
+      if counter == 1:
+        config_list.append(test_self.getTestSuiteData(reference='bb')[0])
+      elif counter == 2:
         raise StopIteration
       counter += 1
       return json.dumps(config_list)
@@ -527,6 +524,20 @@ branch = foo
       result = TestResultProxy(self._proxy, self._retry_time,
                self._logger, test_result_path, node_title, revision)
       return result
+    def checkTestSuite(test_node):
+      test_node.node_test_suite_dict
+      rand_part_set = set()
+      for ref, suite in test_node.node_test_suite_dict.items():
+        assert(suite.suite_log is not None)
+        assert(isinstance(suite.suite_log, types.MethodType))
+        assert('var/log/suite/%s' % suite.reference in suite.suite_log_path)
+        assert(suite.suite_log_path.endswith('suite.log'))
+        m = re.match('.*\/(.*)\/suite.log', suite.suite_log_path)
+        rand_part = m.groups()[0]
+        assert(len(rand_part) == 32)
+        assert(rand_part not in rand_part_set)
+        rand_part_set.add(rand_part)
+
     original_sleep = time.sleep
     time.sleep = doNothing
     self.generateTestRepositoryList()
@@ -543,6 +554,7 @@ branch = foo
     try:
       test_node.run()
     except Exception as e:
+      checkTestSuite(test_node)
       self.assertEqual(type(e),StopIteration)
     finally:
       time.sleep = original_sleep

erp5/util/testnode/testnode.py

@@ -35,6 +35,8 @@ import json
 import time
 import shutil
 import logging
+import string
+import random
 from ProcessManager import SubprocessError, ProcessManager, CancellationError
 from subprocess import CalledProcessError
 from Updater import Updater
@@ -109,11 +111,14 @@ class NodeTestSuite(SlapOSInstance):
 
   def createSuiteLog(self):
     # /srv/slapgrid/slappartXX/srv/var/log/suite/az/mlksjfmlk234Sljssdflkj23KSdfslj/suite.log
-    # /srv/slapgrid/slappartXX/srv/testnode is working directory
     if getattr(self, "log_directory", None) is not None:
       if getattr(self, "suite_log_path", None) is None:
+        alphabets = string.digits + string.letters
+        rand_part = ''.join(random.choice(alphabets) for i in xrange(32))
         suite_log_directory = os.path.join(self.log_directory,
-                                          'suite', self.reference)
+                                          'suite',
+                                           self.reference,
+                                           rand_part)
         SlapOSControler.createFolders(suite_log_directory)
         self.suite_log_path = os.path.join(suite_log_directory,
                                            'suite.log')
@@ -134,7 +139,7 @@ class NodeTestSuite(SlapOSInstance):
     logger_format = '%(asctime)s %(name)-13s: %(levelname)-8s %(message)s'
     formatter = logging.Formatter(logger_format)
     logging.basicConfig(level=logging.INFO, format=logger_format)
-    logger = logging.getLogger('erp5testsuite')
+    logger = logging.getLogger('testsuite')
     file_handler = logging.FileHandler(filename=self.suite_log_path)
     file_handler.setFormatter(formatter)
     logger.addHandler(file_handler)
@@ -241,15 +246,16 @@ branch = %(branch)s
     node_test_suite.revision = ','.join(full_revision_list)
     return full_revision_list
 
-  def addWatcher(self,test_result, node_test_suite):
+  def registerSuiteLog(self, test_result, node_test_suite):
+    """
+      Create a log dedicated for the test suite,
+      and register the url to master node.
+    """
     log_file_name = node_test_suite.createSuiteLog()
     if log_file_name is None and config.get('log_file'):
       log_file_name = config['log_file']
-    if log_file_name is not None:
-      log_file = open(log_file_name)
-      log_file.seek(0, 2)
-      log_file.seek(-min(5000, log_file.tell()), 2)
-      test_result.addWatch(log_file_name,log_file,max_history_bytes=10000)
+    # TODO make the path into url
+    test_result.reportStatus('registerSuiteLog', log_file_name, '')
     return log_file_name
 
   def checkRevision(self, test_result, node_test_suite):
@@ -284,7 +290,6 @@ branch = %(branch)s
     reset_software = slapos_instance.retry_software_count > 10
     log('testnode, retry_software_count : %r' % \
              slapos_instance.retry_software_count)
-    # XXX:TATUYA TO BE FIXED 
     self.slapos_controler = SlapOSControler.SlapOSControler(
       working_directory, self.config, log)
     self.slapos_controler.initializeSlapOSControler(slapproxy_log=slapproxy_log,
@@ -379,13 +384,6 @@ branch = %(branch)s
     log = self.log
     log('Testnode.cleanUp')
     self.process_manager.killPreviousRun()
-    if test_result is not None:
-      try:
-        for node_test_suite in self.node_test_suite_dict.values():
-          log_file = node_test_suite.getSuiteLogPath()
-          test_result.removeWatch(log_file)
-      except KeyError:
-        log("KeyError, Watcher already deleted or not added correctly")
 
   def run(self):
     log = self.log
@@ -434,7 +432,7 @@ branch = %(branch)s
             remote_test_result_needs_cleanup = True
             log("testnode, test_result : %r" % (test_result, ))
             if test_result is not None:
-              log_file_name = self.addWatcher(test_result, node_test_suite)
+              self.registerSuiteLog(test_result, node_test_suite)
               self.checkRevision(test_result,node_test_suite)
               # Now prepare the installation of SlapOS and create instance
               status_dict = self.prepareSlapOSForTestSuite(node_test_suite)
@@ -443,7 +441,6 @@ branch = %(branch)s
               # a reliable way to check if they are up or not ...
               time.sleep(20)
               self.runTestSuite(node_test_suite,portal_url)
-              test_result.removeWatch(log_file_name)
               # break the loop to get latest priorities from master
               break
             self.cleanUp(test_result)