slapos: Implement partition timeout parameter

It's very useful in some deployments to control maximum time of
partition processing. This parmater can end up in slapos.cfg configuration
file or command line.

slapos/cli/slapgrid.py

@@ -85,6 +85,9 @@ class SlapgridCommand(ConfigCommand):
                         type=int,
                         help='Promise timeout in seconds'
                              ' (default: %(default)s)')
+        ap.add_argument('--partition-timeout',
+                        type=int,
+                        help='Partition timeout in seconds (optional)')
         ap.add_argument('--now',
                         action='store_true',
                         help='Launch slapgrid without delay. Default behavior.')

slapos/grid/SlapObject.py

@@ -423,6 +423,7 @@ class Partition(object):
                instance_storage_home='',
                ipv4_global_network='',
                buildout_debug=False,
+               partition_timeout=None,
                ):
     """Initialisation of class parameters"""
     self.buildout = buildout
@@ -443,6 +444,7 @@ class Partition(object):
     self.software_release_url = software_release_url
     self.instance_storage_home = instance_storage_home
     self.ipv4_global_network = ipv4_global_network
+    self.partition_timeout = partition_timeout
 
     self.key_file = ''
     self.cert_file = ''
@@ -701,7 +703,8 @@ class Partition(object):
     utils.launchBuildout(path=self.instance_path,
                          buildout_binary=buildout_binary,
                          logger=self.logger,
-                         debug=self.buildout_debug)
+                         debug=self.buildout_debug,
+                         timeout=self.partition_timeout)
     self.generateSupervisorConfigurationFile()
     self.createRetentionLockDelay()
     self.instance_python = getPythonExecutableFromSoftwarePath(self.software_path)

slapos/grid/slapgrid.py

@@ -260,6 +260,12 @@ def create_slapgrid_object(options, logger):
   software_min_free_space = human2bytes(op.get('software_min_free_space', '1000M'))
   instance_min_free_space = human2bytes(op.get('instance_min_free_space', '1000M'))
 
+  # Nicely check and convert partition_timeout, in order to support slapos.cfg
+  # provided, which is string and command line, which is int, and in the same
+  # time resort to proper None as default - no timeout
+  partition_timeout = op.get('partition_timeout', op.get('partition-timeout'))
+  if partition_timeout is not None:
+    partition_timeout = int(partition_timeout)
   return Slapgrid(software_root=op['software_root'],
                   instance_root=op['instance_root'],
                   shared_part_list=op.get('shared_part_list', ''),
@@ -307,7 +313,8 @@ def create_slapgrid_object(options, logger):
                   ipv4_global_network=op.get('ipv4_global_network'),
                   firewall_conf=op.get('firewall'),
                   config=options,
-                  force_stop=op.get('force_stop', False))
+                  force_stop=op.get('force_stop', False),
+                  partition_timeout=partition_timeout)
 
 
 def check_required_only_partitions(existing, required):
@@ -370,6 +377,7 @@ class Slapgrid(object):
                buildout_debug=False,
                shared_part_list='',
                force_stop=False,
+               partition_timeout=None,
                ):
     """Makes easy initialisation of class parameters"""
     # Parses arguments
@@ -413,6 +421,7 @@ class Slapgrid(object):
     self.buildout = buildout
     self.buildout_debug = buildout_debug
     self.promise_timeout = promise_timeout
+    self.partition_timeout = partition_timeout
     self.develop = develop
     if software_release_filter_list is not None:
       self.software_release_filter_list = \
@@ -1180,6 +1189,7 @@ stderr_logfile_backups=1
       instance_min_free_space=self.instance_min_free_space,
       instance_storage_home=self.instance_storage_home,
       ipv4_global_network=self.ipv4_global_network,
+      partition_timeout=self.partition_timeout
     )
 
     # let managers modify current partition

slapos/grid/utils.py

@@ -402,7 +402,7 @@ def bootstrapBuildout(path, logger, buildout=None,
 
 def launchBuildout(path, buildout_binary, logger,
                    additional_buildout_parameter_list=None,
-                   debug=False):
+                   debug=False, timeout=None):
   """ Launches buildout."""
   if additional_buildout_parameter_list is None:
     additional_buildout_parameter_list = []
@@ -431,13 +431,16 @@ def launchBuildout(path, buildout_binary, logger,
     logger.debug('Set umask from %03o to %03o' % (umask, SAFE_UMASK))
     logger.debug('Invoking: %r in directory %r' % (' '.join(invocation_list),
       path))
+    if timeout is not None:
+      logger.debug('Launching buildout with %ss timeout', timeout)
     process_handler = SlapPopen(invocation_list,
                                 preexec_fn=lambda: dropPrivileges(uid, gid, logger=logger),
                                 cwd=path,
                                 env=getCleanEnvironment(logger=logger,
                                                         home_path=path),
                                 debug=debug,
-                                logger=logger)
+                                logger=logger,
+                                timeout=timeout)
     if process_handler.returncode is None or process_handler.returncode != 0:
       message = 'Failed to run buildout profile in directory %r' % path
       logger.error(message)

slapos/tests/test_object.py

@@ -50,9 +50,13 @@ class FakeCallAndStore(object):
   """
   def __init__(self):
     self.called = False
+    self.args = []
+    self.kwargs = {}
 
   def __call__(self, *args, **kwargs):
     self.called = True
+    self.args = args
+    self.kwargs = kwargs
 
 class FakeCallAndNoop(object):
   """
@@ -131,6 +135,7 @@ class MasterMixin(BasicMixin, unittest.TestCase):
       partition_id=None,
       slap_computer_partition=None,
       retention_delay=None,
+      partition_timeout=None
   ):
     """
     Create a partition, and return a Partition object created
@@ -172,6 +177,7 @@ class MasterMixin(BasicMixin, unittest.TestCase):
       software_release_url=software_release_url,
       buildout=self.buildout,
       logger=logging.getLogger(),
+      partition_timeout=partition_timeout,
     )
 
     partition.updateSupervisor = FakeCallAndNoop
@@ -377,6 +383,24 @@ class TestPartitionSlapObject(MasterMixin, unittest.TestCase):
     MasterMixin.tearDown(self)
     Partition.generateSupervisorConfigurationFile = originalPartitionGenerateSupervisorConfigurationFile
 
+  def test_partition_timeout_default(self):
+    software = self.createSoftware()
+
+    partition = self.createPartition(software.url)
+    partition.install()
+
+    self.assertTrue(utils.launchBuildout.called)
+    self.assertEqual(utils.launchBuildout.kwargs['timeout'], None)
+
+  def test_partition_timeout_passed(self):
+    software = self.createSoftware()
+
+    partition = self.createPartition(software.url, partition_timeout=123)
+    partition.install()
+
+    self.assertTrue(utils.launchBuildout.called)
+    self.assertEqual(utils.launchBuildout.kwargs['timeout'], 123)
+
   def test_instance_is_deploying_if_software_release_exists(self):
     """
     Test that slapgrid deploys an instance if its Software Release exists and

slapos/tests/test_slapgrid.py

@@ -4511,3 +4511,24 @@ For help, use -c -h"""):
         supervisord_new_socket_path,
         supervisord_legacy_socket_path,
     )
+
+
+class TestSlapgridPartitionTimeoutWithMaster(MasterMixin, unittest.TestCase):
+  def _test(self, partition_timeout, delay, result):
+    computer = self.getTestComputerClass()(self.software_root, self.instance_root)
+    with httmock.HTTMock(computer.request_handler):
+      instance = computer.instance_list[0]
+      instance.requested_state = 'started'
+      instance.software.setBuildout('#!/bin/sh\nsleep %s' % (delay,))
+      self.setSlapgrid(develop=False, force_stop=False)
+      self.grid.partition_timeout = partition_timeout
+      self.assertEqual(result, self.grid.processComputerPartitionList())
+
+  def test_timeouted(self):
+    self._test(1, 2, 1)
+
+  def test_finished(self):
+    self._test(2, 1, 0)
+
+  def test_default(self):
+    self._test(None, 2, 0)