Testnode: Use shared parts when building softwares

/reviewed-on nexedi/erp5!940

erp5/tests/testERP5TestNode.py

@@ -25,6 +25,7 @@ import tempfile
 import json
 import time
 import re
+from six.moves.configparser import ConfigParser
 try:
   from unittest import mock
 except ImportError:
@@ -100,6 +101,7 @@ class ERP5TestNode(TestCase):
     config["ipv6_address"] = "::1"
     config["slapos_binary"] = "/opt/slapgrid/HASH/bin/slapos"
     config["srv_directory"] = "srv_directory"
+    config["shared_part_list"] = "/not/exists\n /not/exists_either"
 
     testnode = TestNode(config)
     # By default, keep suite logs to stdout for easier debugging
@@ -611,6 +613,7 @@ shared = true
         ('--frontend_url', 'http://frontend/'),
         ('--node_quantity', '3'),
         ('--xvfb_bin', part('xserver/bin/Xvfb')),
+        ('--shared_part_list', "/not/exists:/not/exists_either:%s/shared" % node_test_suite.working_directory),
       ):
       parser.add_argument(option[0])
       expected_parameter_list += option
@@ -621,47 +624,72 @@ shared = true
     test_node_slapos = SlapOSInstance(self.slapos_directory)
     runner = test_type_registry[my_test_type](test_node)
     node_test_suite = test_node.getNodeTestSuite('foo')
-    status_dict = {"status_code" : 0}
-    global call_list
-    call_list = []
-    class Patch:
-      def __init__(self, method_name, status_code=0):
-        self.method_name = method_name
-        self.status_code = status_code
-      def __call__(self, *args, **kw):
-        global call_list
-        call_list.append({"method_name": self.method_name,
-                         "args": [x for x in args],
-                          "kw": kw})
-        return {"status_code": self.status_code}
-
-    original_SlapOSControler_initializeSlapOSControler = SlapOSControler.initializeSlapOSControler
-    original_SlapOSControler_runSoftwareRelease = SlapOSControler.runSoftwareRelease
-    original_SlapOSControler_runComputerPartition = SlapOSControler.runComputerPartition
-    try:
-      SlapOSControler.initializeSlapOSControler = Patch("initializeSlapOSControler")
-      SlapOSControler.runSoftwareRelease = Patch("runSoftwareRelease")
-      SlapOSControler.runComputerPartition = Patch("runComputerPartition")
-      method_list_for_prepareSlapOSForTestNode = ["initializeSlapOSControler",
-                                                    "runSoftwareRelease"]
-      method_list_for_prepareSlapOSForTestSuite = ["initializeSlapOSControler",
-                                  "runSoftwareRelease", "runComputerPartition"]
+
+    with mock.patch(
+          'erp5.util.testnode.SlapOSControler.SlapOSControler.runSoftwareRelease',
+          return_value={"status_code": 0}
+        ) as runSoftwareRelease,\
+        mock.patch(
+          'erp5.util.testnode.SlapOSControler.SlapOSControler.runComputerPartition',
+          return_value={"status_code": 0}
+        )  as runComputerPartition,\
+        mock.patch('erp5.util.testnode.SlapOSControler.slapos.slap'),\
+        mock.patch('subprocess.Popen'):
+
       runner.prepareSlapOSForTestNode(test_node_slapos)
-      self.assertEqual(method_list_for_prepareSlapOSForTestNode,
-                        [x["method_name"] for x in call_list])
-      call_list = []
+      self.assertEqual(1, runSoftwareRelease.call_count)
+      self.assertEqual(0, runComputerPartition.call_count)
+
+    with mock.patch(
+          'erp5.util.testnode.SlapOSControler.SlapOSControler.runSoftwareRelease',
+          return_value={"status_code": 0}
+        ) as runSoftwareRelease,\
+        mock.patch(
+          'erp5.util.testnode.SlapOSControler.SlapOSControler.runComputerPartition',
+          return_value={"status_code": 0}
+        )  as runComputerPartition,\
+        mock.patch('erp5.util.testnode.SlapOSControler.slapos.slap'),\
+        mock.patch('subprocess.Popen'):
+
       runner.prepareSlapOSForTestSuite(node_test_suite)
-      self.assertEqual(method_list_for_prepareSlapOSForTestSuite,
-                        [x["method_name"] for x in call_list])
-      call_list = []
-      SlapOSControler.runSoftwareRelease = Patch("runSoftwareRelease", status_code=1)
-      # TODO : write a test for scalability case
-      self.assertRaises(SubprocessError, runner.prepareSlapOSForTestSuite,
-                      node_test_suite)
-    finally:
-      SlapOSControler.initializeSlapOSControler = original_SlapOSControler_initializeSlapOSControler
-      SlapOSControler.runSoftwareRelease = original_SlapOSControler_runSoftwareRelease
-      SlapOSControler.runComputerPartition = original_SlapOSControler_runComputerPartition
+      self.assertEqual(1, runSoftwareRelease.call_count)
+      self.assertEqual(1, runComputerPartition.call_count)
+
+    # test node slapos slapos uses the shared parts defined in config
+    cfg_parser = ConfigParser()
+    with open(os.path.join(test_node_slapos.working_directory, 'slapos.cfg')) as f:
+      cfg_parser.readfp(f)
+    self.assertEqual(
+        '/not/exists\n/not/exists_either',
+        cfg_parser.get('slapos', 'shared_part_list'))
+
+    # test suite slapos uses the shared parts from the config, plus
+    # a "local" folder for used as shared when installing tested
+    # softwares.
+    cfg_parser = ConfigParser()
+    with open(os.path.join(node_test_suite.working_directory, 'slapos.cfg')) as f:
+      cfg_parser.readfp(f)
+    self.assertEqual(
+        '/not/exists\n/not/exists_either\n%s/shared' % node_test_suite.working_directory,
+        cfg_parser.get('slapos', 'shared_part_list'))
+
+    # If running software has status_code 1 we have an error
+    with mock.patch(
+          'erp5.util.testnode.SlapOSControler.SlapOSControler.runSoftwareRelease',
+          return_value={"status_code": 1}
+        ) as runSoftwareRelease,\
+        mock.patch(
+          'erp5.util.testnode.SlapOSControler.SlapOSControler.runComputerPartition',
+          return_value={"status_code": 0}
+        )  as runComputerPartition,\
+        mock.patch('erp5.util.testnode.SlapOSControler.slapos.slap'),\
+        mock.patch('subprocess.Popen'):
+
+      self.assertRaises(
+          SubprocessError,
+          runner.prepareSlapOSForTestSuite,
+          node_test_suite)
+
 
   def test_11_run(self, my_test_type='UnitTest', grade='master'):
     def doNothing(self, *args, **kw):

erp5/util/testnode/SlapOSControler.py

@@ -42,9 +42,17 @@ MAX_SR_RETRIES = 3
 
 class SlapOSControler(object):
 
-  def __init__(self, working_directory, config):
+  def __init__(self, working_directory, config, use_local_shared_part=False):
     self.config = config
     self.software_root = os.path.join(working_directory, 'soft')
+    self.shared_part_list = [
+        path.strip() for path in config['shared_part_list'].splitlines()
+    ]
+    if use_local_shared_part:
+      shared = os.path.join(working_directory, 'shared')
+      createFolder(shared)
+      self.shared_part_list = self.shared_part_list + [shared]
+
     self.instance_root = os.path.join(working_directory, 'inst')
     self.slapos_config = os.path.join(working_directory, 'slapos.cfg')
     self.proxy_database = os.path.join(working_directory, 'proxy.db')
@@ -217,7 +225,9 @@ class SlapOSControler(object):
     slapos_config_dict = config.copy()
     slapos_config_dict.update(software_root=self.software_root,
                               instance_root=self.instance_root,
-                              proxy_database=self.proxy_database)
+                              proxy_database=self.proxy_database,
+                              shared_part_list='\n  '.join(self.shared_part_list))
+
     with open(self.slapos_config, 'w') as f:
       f.write(pkg_resources.resource_string(
          'erp5.util.testnode', 'template/slapos.cfg.in').decode() %

erp5/util/testnode/UnitTestRunner.py

@@ -44,16 +44,17 @@ class UnitTestRunner(object):
   def __init__(self, testnode):
     self.testnode = testnode
 
-  def _getSlapOSControler(self, working_directory):
+  def _getSlapOSControler(self, working_directory, use_local_shared_part):
     """
     Create a SlapOSControler
     """
     return SlapOSControler(
                working_directory,
-               self.testnode.config)
- 
+               self.testnode.config,
+               use_local_shared_part=use_local_shared_part)
+
   def _prepareSlapOS(self, working_directory, slapos_instance,
-          create_partition=1, software_path_list=None,**kw):
+          create_partition=1, software_path_list=None, use_local_shared_part=False, **kw):
     """
     Launch slapos to build software and partitions
     """
@@ -67,8 +68,10 @@ class UnitTestRunner(object):
              slapos_instance.retry_software_count)
 
     # XXX Create a new controler because working_directory can be
-    #     Diferent depending of the preparation
-    slapos_controler = self._getSlapOSControler(working_directory)
+    #     Different depending of the preparation
+    slapos_controler = self._getSlapOSControler(
+        working_directory,
+        use_local_shared_part)
 
     slapos_controler.initializeSlapOSControler(slapproxy_log=slapproxy_log,
        process_manager=self.testnode.process_manager, reset_software=reset_software,
@@ -113,16 +116,17 @@ class UnitTestRunner(object):
 
   def prepareSlapOSForTestSuite(self, node_test_suite):
     """
-    Build softwares needed by testsuites
+    Build softwares needed by testsuites.
     """
     return self._prepareSlapOS(node_test_suite.working_directory,
               node_test_suite,
               software_path_list=[node_test_suite.custom_profile_path],
-              cluster_configuration={'_': json.dumps(node_test_suite.cluster_configuration)})
+              cluster_configuration={'_': json.dumps(node_test_suite.cluster_configuration)},
+              use_local_shared_part=True)
 
   def getInstanceRoot(self, node_test_suite):
     return self._getSlapOSControler(
-      node_test_suite.working_directory).instance_root
+      node_test_suite.working_directory, True).instance_root
 
   def runTestSuite(self, node_test_suite, portal_url):
     config = self.testnode.config
@@ -156,6 +160,11 @@ class UnitTestRunner(object):
         ('--node_quantity', lambda: config['node_quantity']),
         ('--xvfb_bin', lambda: path('xvfb', 'xserver/bin/Xvfb')),
         ('--project_title', lambda: node_test_suite.project_title),
+        ('--shared_part_list', lambda: os.pathsep.join(
+            self._getSlapOSControler(
+                node_test_suite.working_directory,
+                True
+            ).shared_part_list)),
         ):
       if option in supported_parameter_set:
         invocation_list += option, value()

erp5/util/testnode/__init__.py

@@ -74,7 +74,7 @@ def main(*args):
               'proxy_port', 'git_binary','zip_binary','node_quantity',
               'test_node_title', 'ipv4_address','ipv6_address','test_suite_master_url',
               'slapos_binary', 'httpd_ip', 'httpd_port', 'httpd_software_access_port',
-              'computer_id', 'server_url'):
+              'computer_id', 'server_url', 'shared_part_list'):
     CONFIG[key] = config.get('testnode',key)
 
   for key in ('slapos_directory', 'working_directory', 'test_suite_directory',

erp5/util/testnode/template/slapos.cfg.in

 [slapos]
 software_root = %(software_root)s
 instance_root = %(instance_root)s
+shared_part_list = %(shared_part_list)s
 master_url = %(master_url)s
 computer_id = %(computer_id)s
 root_check = False

erp5/util/testsuite/__init__.py

@@ -245,8 +245,10 @@ class EggTestSuite(TestSuite):
   def run(self, test):
     print(test)
     try:
-      status_dict = self.spawn(self.python_interpreter, 'setup.py', 'test',
-                               cwd=self.egg_test_path_dict[test])
+      status_dict = self.spawn(
+          self.python_interpreter, 'setup.py', 'test',
+          cwd=self.egg_test_path_dict[test],
+          SLAPOS_TEST_SHARED_PART_LIST=self.shared_part_list)
     except SubprocessError as e:
       status_dict = e.status_dict
     test_log = status_dict['stderr']
@@ -302,6 +304,9 @@ def runTestSuite():
   parser.add_argument('--source_code_path_list',
                       help='Coma separated list of Eggs folders to test',
                       default='.')
+  parser.add_argument('--shared_part_list',
+                      help='Shared parts for recursive slapos',
+                      default='')
 
   args = parser.parse_args()
   master = taskdistribution.TaskDistributor(args.master_url)
@@ -324,6 +329,7 @@ def runTestSuite():
                     revision=revision,
                     python_interpreter=args.python_interpreter,
                     egg_test_path_dict=egg_test_path_dict,
+                    shared_part_list=args.shared_part_list
                     )
 
   test_result = master.createTestResult(revision, suite.getTestList(),