slapos: Create supervisor configuration when running CLI.

Also move and simplify management of configuration files.

slapos/cli/configure_local/__init__.py

@@ -245,17 +245,6 @@ def do_configure(args, fetch_config_func, logger):
     configp = fetch_config_func(args)
     conf = FormatConfig(logger=logger)
     conf.mergeConfig(args, configp)
-    supervisord_socket_path = os.path.join(conf.instance_root,
-       'supervisord.socket')
-    supervisord_conf_path = os.path.join(conf.instance_root,
-       'etc', 'supervisord.conf')
-    conf_property_list = (
-      ('supervisord_socket', supervisord_socket_path),
-      ('supervisord_configuration_path', supervisord_conf_path),
-    )
-    for key, value in conf_property_list:
-        if not getattr(conf, key, None):
-            setattr(conf, key, value)
     slapgrid = create_slapgrid_object(conf.__dict__, logger)
     createPrivateDirectory(os.path.join(conf.slapos_buildout_directory, 'log'))
     _runFormat(conf.slapos_buildout_directory)
@@ -268,7 +257,6 @@ def do_configure(args, fetch_config_func, logger):
     slapos_client_cfg_path = '%s/.slapos/slapos-client.cfg' % home_folder_path
     if not os.path.exists(slapos_client_cfg_path):
         os.symlink(slapos_node_config_path, slapos_client_cfg_path)
-    launchSupervisord(socket=supervisord_socket_path,
-      configuration_file=supervisord_conf_path, logger=logger)
+    launchSupervisord(instance_root=conf.instance_root, logger=logger)
     _runFormat(conf.slapos_buildout_directory)
     return 0

slapos/cli/supervisorctl.py

@@ -28,13 +28,10 @@
 ##############################################################################
 
 import argparse
-import os
 
 from slapos.cli.command import check_root_user
 from slapos.cli.config import ConfigCommand
-from slapos.grid.svcbackend import launchSupervisord
-
-from slapos.util import string_to_boolean
+from slapos.grid.svcbackend import (launchSupervisord, _getSupervisordConfigurationFilePath)
 
 import supervisor.supervisorctl
 
@@ -66,11 +63,10 @@ class SupervisorctlCommand(ConfigCommand):
             check_root_user(self)
 
         instance_root = configp.get('slapos', 'instance_root')
-        configuration_file = os.path.join(instance_root, 'etc', 'supervisord.conf')
-        launchSupervisord(socket=os.path.join(instance_root, 'supervisord.socket'),
-                          configuration_file=configuration_file,
-                          logger=self.app.log)
-        supervisor.supervisorctl.main(args=['-c', configuration_file] + args.supervisor_args)
+        launchSupervisord(instance_root=instance_root, logger=self.app.log)
+        supervisor.supervisorctl.main(
+            args=['-c', _getSupervisordConfigurationFilePath(instance_root)] + args.supervisor_args
+        )
 
 
 class SupervisorctlAliasCommand(SupervisorctlCommand):

slapos/cli/supervisord.py

@@ -27,11 +27,8 @@
 #
 ##############################################################################
 
-import os
-
 from slapos.cli.config import ConfigCommand
-from slapos.grid.svcbackend import launchSupervisord
-
+from slapos.grid.svcbackend import (launchSupervisord, createSupervisordConfiguration)
 
 class SupervisordCommand(ConfigCommand):
     """
@@ -54,7 +51,8 @@ class SupervisordCommand(ConfigCommand):
           supervisord_additional_argument_list = ['--nodaemon']
         else:
           supervisord_additional_argument_list = []
-        launchSupervisord(socket=os.path.join(instance_root, 'supervisord.socket'),
-                          configuration_file=os.path.join(instance_root, 'etc', 'supervisord.conf'),
-                          logger=self.app.log,
-                          supervisord_additional_argument_list=supervisord_additional_argument_list)
+        createSupervisordConfiguration(instance_root)
+        launchSupervisord(
+            instance_root=instance_root, logger=self.app.log,
+            supervisord_additional_argument_list=supervisord_additional_argument_list
+        )

slapos/grid/slapgrid.py

@@ -37,7 +37,6 @@ import subprocess
 import sys
 import tempfile
 import time
-import stat
 import traceback
 import warnings
 import logging
@@ -53,9 +52,11 @@ from slapos.slap.slap import ServerError
 from slapos.util import mkdir_p, chownDirectory
 from slapos.grid.exception import BuildoutFailedError
 from slapos.grid.SlapObject import Software, Partition
-from slapos.grid.svcbackend import launchSupervisord
-from slapos.grid.utils import (md5digest, createPrivateDirectory, dropPrivileges,
-                               SlapPopen, updateFile)
+from slapos.grid.svcbackend import (launchSupervisord,
+                                    createSupervisordConfiguration,
+                                    _getSupervisordConfigurationDirectory,
+                                    _getSupervisordSocketPath)
+from slapos.grid.utils import (md5digest, dropPrivileges, SlapPopen)
 from slapos.human import human2bytes
 import slapos.slap
 
@@ -71,12 +72,12 @@ SLAPGRID_FAIL = 1
 SLAPGRID_PROMISE_FAIL = 2
 PROMISE_TIMEOUT = 3
 
-# XXX hardcoded watchdog_path
-WATCHDOG_PATH = '/opt/slapos/bin/slapos-watchdog'
-
 COMPUTER_PARTITION_TIMESTAMP_FILENAME = '.timestamp'
 COMPUTER_PARTITION_LATEST_BANG_TIMESTAMP_FILENAME = '.slapos_latest_bang_timestamp'
 
+# XXX hardcoded watchdog_path
+WATCHDOG_PATH = '/opt/slapos/bin/slapos-watchdog'
+
 
 class _formatXMLError(Exception):
   pass
@@ -149,15 +150,6 @@ def merged_options(args, configp):
   if options.get('all'):
     options['develop'] = True
 
-  # Supervisord configuration location
-  if not options.get('supervisord_configuration_path'):
-    options['supervisord_configuration_path'] = \
-      os.path.join(options['instance_root'], 'etc', 'supervisord.conf')
-  # Supervisord socket
-  if not options.get('supervisord_socket'):
-    options['supervisord_socket'] = \
-      os.path.join(options['instance_root'], 'supervisord.socket')
-
   # Parse cache / binary cache options
   # Backward compatibility about "binary-cache-url-blacklist" deprecated option
   if (options.get("binary-cache-url-blacklist") and not
@@ -209,8 +201,6 @@ def create_slapgrid_object(options, logger):
                   instance_root=op['instance_root'],
                   master_url=op['master_url'],
                   computer_id=op['computer_id'],
-                  supervisord_socket=op['supervisord_socket'],
-                  supervisord_configuration_path=op['supervisord_configuration_path'],
                   buildout=op.get('buildout'),
                   logger=logger,
                   maximum_periodicity = op.get('maximum_periodicity', 86400),
@@ -267,8 +257,6 @@ class Slapgrid(object):
                instance_root,
                master_url,
                computer_id,
-               supervisord_socket,
-               supervisord_configuration_path,
                buildout,
                logger,
                maximum_periodicity=86400,
@@ -303,8 +291,7 @@ class Slapgrid(object):
     self.instance_root = os.path.abspath(instance_root)
     self.master_url = master_url
     self.computer_id = computer_id
-    self.supervisord_socket = supervisord_socket
-    self.supervisord_configuration_path = supervisord_configuration_path
+    self.supervisord_socket = _getSupervisordSocketPath(instance_root)
     self.key_file = key_file
     self.cert_file = cert_file
     self.master_ca_file = master_ca_file
@@ -332,8 +319,6 @@ class Slapgrid(object):
         cert_file=self.cert_file, master_ca_file=self.master_ca_file)
     self.computer = self.slap.registerComputer(self.computer_id)
     # Defines all needed paths
-    self.supervisord_configuration_directory = \
-        os.path.join(self.instance_root, 'etc', 'supervisord.conf.d')
     self.buildout = buildout
     self.promise_timeout = promise_timeout
     self.develop = develop
@@ -350,7 +335,7 @@ class Slapgrid(object):
     self.software_min_free_space = software_min_free_space
     self.instance_min_free_space = instance_min_free_space
 
-  def getWatchdogLine(self):
+  def _getWatchdogLine(self):
     invocation_list = [WATCHDOG_PATH]
     invocation_list.append("--master-url '%s' " % self.master_url)
     if self.certificate_repository_path:
@@ -367,42 +352,11 @@ class Slapgrid(object):
     # Checks for software_root and instance_root existence
     if not os.path.isdir(self.software_root):
       raise OSError('%s does not exist.' % self.software_root)
-    if not os.path.isdir(self.instance_root):
-      raise OSError('%s does not exist.' % self.instance_root)
-    # Creates everything needed
-
-    # Create directory accessible for the instances.
-    var_directory = os.path.join(self.instance_root, 'var')
-    if not os.path.isdir(var_directory):
-      os.mkdir(var_directory)
-
-    os.chmod(var_directory, stat.S_IRWXU | stat.S_IROTH | stat.S_IXOTH | \
-                            stat.S_IRGRP | stat.S_IXGRP )
-
-    mkdir_p(os.path.join(self.instance_root, 'var'), 0o755)
-
-    # Creates instance_root structure
-    createPrivateDirectory(os.path.join(self.instance_root, 'var', 'log'))
-    createPrivateDirectory(os.path.join(self.instance_root, 'var', 'run'))
-
-    createPrivateDirectory(os.path.join(self.instance_root, 'etc'))
-    createPrivateDirectory(self.supervisord_configuration_directory)
-
-    # Creates supervisord configuration
-    updateFile(self.supervisord_configuration_path,
-      pkg_resources.resource_stream(__name__,
-        'templates/supervisord.conf.in').read() % {
-            'supervisord_configuration_directory': self.supervisord_configuration_directory,
-            'supervisord_socket': os.path.abspath(self.supervisord_socket),
-            'supervisord_loglevel': 'info',
-            'supervisord_logfile': os.path.abspath(os.path.join(self.instance_root, 'var', 'log', 'supervisord.log')),
-            'supervisord_logfile_maxbytes': '50MB',
-            'supervisord_nodaemon': 'false',
-            'supervisord_pidfile': os.path.abspath(os.path.join(self.instance_root, 'var', 'run', 'supervisord.pid')),
-            'supervisord_logfile_backups': '10',
-            'watchdog_command': self.getWatchdogLine(),
-        }
-    )
+
+    createSupervisordConfiguration(self.instance_root, self._getWatchdogLine())
+
+  def _launchSupervisord(self):
+    launchSupervisord(instance_root=self.instance_root, logger=self.logger)
 
   def getComputerPartitionList(self):
     try:
@@ -501,11 +455,6 @@ class Slapgrid(object):
       return SLAPGRID_FAIL
     return SLAPGRID_SUCCESS
 
-  def _launchSupervisord(self):
-    launchSupervisord(self.supervisord_socket,
-                      self.supervisord_configuration_path,
-                      logger=self.logger)
-
   def _checkPromises(self, computer_partition):
     self.logger.info("Checking promises...")
     instance_path = os.path.join(self.instance_root, computer_partition.getId())
@@ -678,7 +627,7 @@ class Slapgrid(object):
         software_path=software_path,
         instance_path=instance_path,
         supervisord_partition_configuration_path=os.path.join(
-          self.supervisord_configuration_directory, '%s.conf' %
+          _getSupervisordConfigurationDirectory(self.instance_root), '%s.conf' %
           computer_partition_id),
         supervisord_socket=self.supervisord_socket,
         computer_partition=computer_partition,
@@ -1136,7 +1085,7 @@ class Slapgrid(object):
             instance_path=os.path.join(self.instance_root,
                 computer_partition.getId()),
             supervisord_partition_configuration_path=os.path.join(
-              self.supervisord_configuration_directory, '%s.conf' %
+              _getSupervisordConfigurationDirectory(self.instance_root), '%s.conf' %
               computer_partition_id),
             supervisord_socket=self.supervisord_socket,
             computer_partition=computer_partition,

slapos/grid/svcbackend.py

@@ -29,14 +29,17 @@
 ##############################################################################
 
 import os
+import pkg_resources
+import socket as socketlib
+import subprocess
+import stat
 import sys
 import time
 import xmlrpclib
-import socket as socketlib
-import subprocess
+
+from slapos.grid.utils import (createPrivateDirectory, SlapPopen, updateFile)
 
 from supervisor import xmlrpc
-from slapos.grid.utils import SlapPopen
 
 
 def getSupervisorRPC(socket):
@@ -47,7 +50,65 @@ def getSupervisorRPC(socket):
   return getattr(server_proxy, 'supervisor')
 
 
-def launchSupervisord(socket, configuration_file, logger, supervisord_additional_argument_list=None):
+def _getSupervisordSocketPath(instance_root):
+  return os.path.join(instance_root, 'supervisord.socket')
+
+def _getSupervisordConfigurationFilePath(instance_root):
+  return os.path.join(instance_root, 'etc', 'supervisord.conf')
+
+def _getSupervisordConfigurationDirectory(instance_root):
+  return os.path.join(instance_root, 'etc', 'supervisord.conf.d')
+
+def createSupervisordConfiguration(instance_root, watchdog_command='sleep 10'):
+  """
+  Create supervisord related files and directories.
+  """
+  if not os.path.isdir(instance_root):
+    raise OSError('%s does not exist.' % instance_root)
+
+  supervisord_configuration_file_path = _getSupervisordConfigurationFilePath(instance_root)
+  supervisord_configuration_directory = _getSupervisordConfigurationDirectory(instance_root)
+  supervisord_socket = _getSupervisordSocketPath(instance_root)
+
+  # Create directory accessible for the instances.
+  var_directory = os.path.join(instance_root, 'var')
+  if not os.path.isdir(var_directory):
+    os.mkdir(var_directory)
+  os.chmod(var_directory, stat.S_IRWXU | stat.S_IROTH | stat.S_IXOTH | \
+                          stat.S_IRGRP | stat.S_IXGRP )
+  etc_directory = os.path.join(instance_root, 'etc')
+  if not os.path.isdir(etc_directory):
+    os.mkdir(etc_directory)
+
+  # Creates instance_root structure
+  createPrivateDirectory(os.path.join(instance_root, 'var', 'log'))
+  createPrivateDirectory(os.path.join(instance_root, 'var', 'run'))
+
+  createPrivateDirectory(os.path.join(instance_root, 'etc'))
+  createPrivateDirectory(supervisord_configuration_directory)
+
+  # Creates supervisord configuration
+  updateFile(supervisord_configuration_file_path,
+    pkg_resources.resource_stream(__name__,
+      'templates/supervisord.conf.in').read() % {
+          'supervisord_configuration_directory': supervisord_configuration_directory,
+          'supervisord_socket': os.path.abspath(supervisord_socket),
+          'supervisord_loglevel': 'info',
+          'supervisord_logfile': os.path.abspath(
+              os.path.join(instance_root, 'var', 'log', 'supervisord.log')),
+          'supervisord_logfile_maxbytes': '50MB',
+          'supervisord_nodaemon': 'false',
+          'supervisord_pidfile': os.path.abspath(
+              os.path.join(instance_root, 'var', 'run', 'supervisord.pid')),
+          'supervisord_logfile_backups': '10',
+          'watchdog_command': watchdog_command,
+      }
+  )
+
+def launchSupervisord(instance_root, logger,
+                      supervisord_additional_argument_list=None):
+  configuration_file = _getSupervisordConfigurationFilePath(instance_root)
+  socket = _getSupervisordSocketPath(instance_root)
   if os.path.exists(socket):
     trynum = 1
     while trynum < 6:
@@ -66,6 +127,18 @@ def launchSupervisord(socket, configuration_file, logger, supervisord_additional
       else:
         if status['statename'] == 'RUNNING' and status['statecode'] == 1:
           logger.debug('Supervisord already running.')
+
+          # Update watchdog
+          supervisor = getSupervisorRPC(socket)
+          try:
+            # XXX workaround for https://github.com/Supervisor/supervisor/issues/339
+            # In theory, only reloadConfig is needed.
+            supervisor.stopProcess('watchdog')
+            supervisor.removeProcessGroup('watchdog')
+          except:
+            pass
+          supervisor.reloadConfig()
+          supervisor.addProcessGroup('watchdog')
           return
         elif status['statename'] == 'SHUTDOWN_STATE' and status['statecode'] == 6:
           logger.info('Supervisor in shutdown procedure, will check again later.')
@@ -123,3 +196,4 @@ def launchSupervisord(socket, configuration_file, logger, supervisord_additional
     logger.warning('Issue while checking supervisord.')
   finally:
     socketlib.setdefaulttimeout(default_timeout)
+

slapos/grid/templates/supervisord.conf.in

@@ -22,3 +22,4 @@ chmod=0700
 [eventlistener:watchdog]
 command=%(watchdog_command)s
 events=PROCESS_STATE_EXITED, PROCESS_STATE_FATAL
+autorestart=true

slapos/tests/configure_local.py

@@ -38,8 +38,7 @@ from ConfigParser import ConfigParser
 
 # Disable any command to launch slapformat and supervisor
 slapos.cli.configure_local._runFormat = lambda x: "Do nothing"
-slapos.cli.configure_local.launchSupervisord = lambda socket, \
-  configuration_file, logger: "Do nothing"
+slapos.cli.configure_local.launchSupervisord = lambda instance_root, logger: "Do nothing"
 
 class TestConfigureLocal(unittest.TestCase):
 

slapos/tests/slapgrid.py

@@ -122,8 +122,6 @@ class BasicMixin(object):
                                   self.instance_root,
                                   self.master_url,
                                   self.computer_id,
-                                  self.supervisord_socket,
-                                  self.supervisord_configuration_path,
                                   self.buildout,
                                   develop=develop,
                                   logger=logging.getLogger())
@@ -185,6 +183,12 @@ class BasicMixin(object):
         self.fail('%s should not be created' % path)
       time.sleep(0.1)
 
+  def assertInstanceDirectoryListEqual(self, instance_list):
+    instance_list.append('etc')
+    instance_list.append('var')
+    instance_list.append('supervisord.socket')
+    self.assertItemsEqual(os.listdir(self.instance_root), instance_list)
+
   def tearDown(self):
     # XXX: Hardcoded pid, as it is not configurable in slapos
     svc = os.path.join(self.instance_root, 'var', 'run', 'supervisord.pid')
@@ -520,7 +524,7 @@ class TestSlapgridCPWithMaster(MasterMixin, unittest.TestCase):
     computer = ComputerForTest(self.software_root, self.instance_root, 0, 0)
     with httmock.HTTMock(computer.request_handler):
       self.assertEqual(self.grid.processComputerPartitionList(), slapgrid.SLAPGRID_SUCCESS)
-      self.assertItemsEqual(os.listdir(self.instance_root), ['etc', 'var'])
+      self.assertInstanceDirectoryListEqual([])
       self.assertItemsEqual(os.listdir(self.software_root), [])
       st = os.stat(os.path.join(self.instance_root, 'var'))
       self.assertEquals(stat.S_IMODE(st.st_mode), 0o755)
@@ -530,7 +534,7 @@ class TestSlapgridCPWithMaster(MasterMixin, unittest.TestCase):
     with httmock.HTTMock(computer.request_handler):
       instance = computer.instance_list[0]
       self.assertEqual(self.grid.processComputerPartitionList(), slapgrid.SLAPGRID_SUCCESS)
-      self.assertItemsEqual(os.listdir(self.instance_root), ['0', 'etc', 'var'])
+      self.assertInstanceDirectoryListEqual(['0'])
       partition = os.path.join(self.instance_root, '0')
       self.assertItemsEqual(os.listdir(partition), ['.slapgrid', 'buildout.cfg',
                                                     'software_release', 'worked', '.slapos-retention-lock-delay'])
@@ -548,7 +552,7 @@ class TestSlapgridCPWithMaster(MasterMixin, unittest.TestCase):
     with httmock.HTTMock(computer.request_handler):
       instance = computer.instance_list[0]
       self.assertEqual(self.grid.processComputerPartitionList(), slapgrid.SLAPGRID_SUCCESS)
-      self.assertItemsEqual(os.listdir(self.instance_root), ['0', 'etc', 'var'])
+      self.assertInstanceDirectoryListEqual(['0'])
       partition = os.path.join(self.instance_root, '0')
       self.assertItemsEqual(os.listdir(partition), ['.slapgrid', 'buildout.cfg',
                                                     'software_release', 'worked', '.slapos-retention-lock-delay'])
@@ -568,7 +572,7 @@ class TestSlapgridCPWithMaster(MasterMixin, unittest.TestCase):
       partition = computer.instance_list[0]
       partition.requested_state = 'destroyed'
       self.assertEqual(self.grid.processComputerPartitionList(), slapgrid.SLAPGRID_SUCCESS)
-      self.assertItemsEqual(os.listdir(self.instance_root), ['0', 'etc', 'var'])
+      self.assertInstanceDirectoryListEqual(['0'])
       self.assertItemsEqual(os.listdir(partition.partition_path), [])
       self.assertItemsEqual(os.listdir(self.software_root), [])
       self.assertEqual(partition.sequence, [])
@@ -580,7 +584,7 @@ class TestSlapgridCPWithMaster(MasterMixin, unittest.TestCase):
       partition.requested_state = 'started'
       partition.software.setBuildout(WRAPPER_CONTENT)
       self.assertEqual(self.grid.processComputerPartitionList(), slapgrid.SLAPGRID_SUCCESS)
-      self.assertItemsEqual(os.listdir(self.instance_root), ['0', 'etc', 'var'])
+      self.assertInstanceDirectoryListEqual(['0'])
       self.assertItemsEqual(os.listdir(partition.partition_path),
                             ['.slapgrid', '.0_wrapper.log', 'buildout.cfg',
                              'etc', 'software_release', 'worked', '.slapos-retention-lock-delay'])
@@ -617,7 +621,7 @@ HEREDOC
 chmod 755 etc/run/wrapper
 """ % {'python': sys.executable})
       self.assertEqual(self.grid.processComputerPartitionList(), slapgrid.SLAPGRID_SUCCESS)
-      self.assertItemsEqual(os.listdir(self.instance_root), ['0', 'etc', 'var'])
+      self.assertInstanceDirectoryListEqual(['0'])
       self.assertItemsEqual(os.listdir(instance.partition_path),
                             ['.slapgrid', '.0_wrapper.log', 'buildout.cfg',
                              'etc', 'software_release', 'worked', '.slapos-retention-lock-delay'])
@@ -632,7 +636,7 @@ chmod 755 etc/run/wrapper
       computer.sequence = []
       instance.requested_state = 'stopped'
       self.assertEqual(self.launchSlapgrid(), slapgrid.SLAPGRID_SUCCESS)
-      self.assertItemsEqual(os.listdir(self.instance_root), ['0', 'etc', 'var'])
+      self.assertInstanceDirectoryListEqual(['0'])
       self.assertItemsEqual(os.listdir(instance.partition_path),
                             ['.slapgrid', '.0_wrapper.log', 'buildout.cfg',
                              'etc', 'software_release', 'worked', '.slapos-retention-lock-delay'])
@@ -672,7 +676,7 @@ HEREDOC
 chmod 755 etc/run/wrapper
 """ % {'python': sys.executable})
       self.assertEqual(self.grid.processComputerPartitionList(), slapgrid.SLAPGRID_SUCCESS)
-      self.assertItemsEqual(os.listdir(self.instance_root), ['0', 'etc', 'var'])
+      self.assertInstanceDirectoryListEqual(['0'])
       self.assertItemsEqual(os.listdir(instance.partition_path),
                             ['.slapgrid', '.0_wrapper.log', 'buildout.cfg',
                              'etc', 'software_release', 'worked', '.slapos-retention-lock-delay'])
@@ -691,8 +695,7 @@ chmod 755 etc/run/wrapper
 exit 1
 """)
       self.assertEqual(self.launchSlapgrid(), slapgrid.SLAPGRID_FAIL)
-      self.assertItemsEqual(os.listdir(self.instance_root),
-                            ['0', 'etc', 'var'])
+      self.assertInstanceDirectoryListEqual(['0'])
       self.assertItemsEqual(os.listdir(instance.partition_path),
                             ['.slapgrid', '.0_wrapper.log', 'buildout.cfg',
                              'etc', 'software_release', 'worked', '.slapos-retention-lock-delay'])
@@ -710,7 +713,7 @@ exit 1
       instance.software.setBuildout(WRAPPER_CONTENT)
       self.assertEqual(self.grid.processComputerPartitionList(), slapgrid.SLAPGRID_SUCCESS)
 
-      self.assertItemsEqual(os.listdir(self.instance_root), ['0', 'etc', 'var'])
+      self.assertInstanceDirectoryListEqual(['0'])
       partition = os.path.join(self.instance_root, '0')
       self.assertItemsEqual(os.listdir(partition),
                             ['.slapgrid', 'buildout.cfg', 'etc', 'software_release', 'worked', '.slapos-retention-lock-delay'])
@@ -724,7 +727,7 @@ exit 1
       instance.requested_state = 'started'
       computer.sequence = []
       self.assertEqual(self.launchSlapgrid(), slapgrid.SLAPGRID_SUCCESS)
-      self.assertItemsEqual(os.listdir(self.instance_root), ['0', 'etc', 'var'])
+      self.assertInstanceDirectoryListEqual(['0'])
       partition = os.path.join(self.instance_root, '0')
       self.assertItemsEqual(os.listdir(partition),
                             ['.slapgrid', '.0_wrapper.log', 'etc',
@@ -754,7 +757,7 @@ exit 1
 
       self.assertEqual(self.grid.processComputerPartitionList(), slapgrid.SLAPGRID_SUCCESS)
 
-      self.assertItemsEqual(os.listdir(self.instance_root), ['0', 'etc', 'var'])
+      self.assertInstanceDirectoryListEqual(['0'])
       partition = os.path.join(self.instance_root, '0')
       self.assertItemsEqual(os.listdir(partition), ['.slapgrid', dummy_file_name])
       self.assertItemsEqual(os.listdir(self.software_root), [instance.software.software_hash])
@@ -799,7 +802,7 @@ class TestSlapgridCPWithMasterWatchdog(MasterMixin, unittest.TestCase):
       partition.software.setBuildout(DAEMON_CONTENT)
 
       self.assertEqual(self.grid.processComputerPartitionList(), slapgrid.SLAPGRID_SUCCESS)
-      self.assertItemsEqual(os.listdir(self.instance_root), ['0', 'etc', 'var'])
+      self.assertInstanceDirectoryListEqual(['0'])
       self.assertItemsEqual(os.listdir(partition.partition_path),
                             ['.slapgrid', '.0_daemon.log', 'buildout.cfg',
                              'etc', 'software_release', 'worked', '.slapos-retention-lock-delay'])
@@ -844,8 +847,7 @@ class TestSlapgridCPWithMasterWatchdog(MasterMixin, unittest.TestCase):
       partition.software.setBuildout(BUILDOUT_RUN_CONTENT)
 
       self.assertEqual(self.grid.processComputerPartitionList(), slapgrid.SLAPGRID_SUCCESS)
-      self.assertItemsEqual(os.listdir(self.instance_root),
-                            ['0', 'etc', 'var'])
+      self.assertInstanceDirectoryListEqual(['0'])
       self.assertItemsEqual(os.listdir(partition.partition_path),
                             ['.slapgrid', '.0_daemon.log', 'buildout.cfg',
                              'etc', 'software_release', 'worked', '.slapos-retention-lock-delay'])
@@ -1126,7 +1128,7 @@ class TestSlapgridCPPartitionProcessing(MasterMixin, unittest.TestCase):
       instance.timestamp = timestamp
 
       self.assertEqual(self.grid.processComputerPartitionList(), slapgrid.SLAPGRID_SUCCESS)
-      self.assertItemsEqual(os.listdir(self.instance_root), ['0', 'etc', 'var'])
+      self.assertInstanceDirectoryListEqual(['0'])
       partition = os.path.join(self.instance_root, '0')
       self.assertItemsEqual(os.listdir(partition),
                             ['.slapgrid', '.timestamp', 'buildout.cfg', 'software_release', 'worked', '.slapos-retention-lock-delay'])
@@ -1146,7 +1148,7 @@ class TestSlapgridCPPartitionProcessing(MasterMixin, unittest.TestCase):
       instance.timestamp = timestamp
 
       self.assertEqual(self.grid.processComputerPartitionList(), slapgrid.SLAPGRID_SUCCESS)
-      self.assertItemsEqual(os.listdir(self.instance_root), ['0', 'etc', 'var'])
+      self.assertInstanceDirectoryListEqual(['0'])
       partition = os.path.join(self.instance_root, '0')
       self.assertItemsEqual(os.listdir(partition),
                             ['.slapgrid', '.timestamp', 'buildout.cfg',
@@ -1169,7 +1171,7 @@ class TestSlapgridCPPartitionProcessing(MasterMixin, unittest.TestCase):
       instance.timestamp = timestamp
 
       self.assertEqual(self.grid.processComputerPartitionList(), slapgrid.SLAPGRID_SUCCESS)
-      self.assertItemsEqual(os.listdir(self.instance_root), ['0', 'etc', 'var'])
+      self.assertInstanceDirectoryListEqual(['0'])
       partition = os.path.join(self.instance_root, '0')
       self.assertItemsEqual(os.listdir(partition),
                             ['.slapgrid', '.timestamp', 'buildout.cfg', 'software_release', 'worked', '.slapos-retention-lock-delay'])
@@ -1187,7 +1189,7 @@ class TestSlapgridCPPartitionProcessing(MasterMixin, unittest.TestCase):
       instance.timestamp = timestamp
 
       self.assertEqual(self.launchSlapgrid(), slapgrid.SLAPGRID_SUCCESS)
-      self.assertItemsEqual(os.listdir(self.instance_root), ['0', 'etc', 'var'])
+      self.assertInstanceDirectoryListEqual(['0'])
       partition = os.path.join(self.instance_root, '0')
       self.assertItemsEqual(os.listdir(partition),
                             ['.slapgrid', '.timestamp', 'buildout.cfg', 'software_release', 'worked', '.slapos-retention-lock-delay'])
@@ -1212,7 +1214,7 @@ class TestSlapgridCPPartitionProcessing(MasterMixin, unittest.TestCase):
       instance.timestamp = timestamp
 
       self.launchSlapgrid()
-      self.assertItemsEqual(os.listdir(self.instance_root), ['0', 'etc', 'var'])
+      self.assertInstanceDirectoryListEqual(['0'])
       partition = os.path.join(self.instance_root, '0')
       self.assertItemsEqual(os.listdir(partition),
                             ['.slapgrid', '.timestamp', 'buildout.cfg', 'software_release', 'worked', '.slapos-retention-lock-delay'])
@@ -1536,7 +1538,7 @@ class TestSlapgridUsageReport(MasterMixin, unittest.TestCase):
       instance.requested_state = 'started'
       instance.software.setBuildout(WRAPPER_CONTENT)
       self.assertEqual(self.grid.processComputerPartitionList(), slapgrid.SLAPGRID_SUCCESS)
-      self.assertItemsEqual(os.listdir(self.instance_root), ['0', 'etc', 'var'])
+      self.assertInstanceDirectoryListEqual(['0'])
       self.assertItemsEqual(os.listdir(instance.partition_path),
                             ['.slapgrid', '.0_wrapper.log', 'buildout.cfg',
                              'etc', 'software_release', 'worked', '.slapos-retention-lock-delay'])
@@ -1554,7 +1556,7 @@ class TestSlapgridUsageReport(MasterMixin, unittest.TestCase):
       instance.requested_state = 'destroyed'
       self.assertEqual(self.grid.agregateAndSendUsage(), slapgrid.SLAPGRID_SUCCESS)
       # Assert partition directory is empty
-      self.assertItemsEqual(os.listdir(self.instance_root), ['0', 'etc', 'var'])
+      self.assertInstanceDirectoryListEqual(['0'])
       self.assertItemsEqual(os.listdir(instance.partition_path), [])
       self.assertItemsEqual(os.listdir(self.software_root),
                             [instance.software.software_hash])
@@ -1585,7 +1587,7 @@ class TestSlapgridUsageReport(MasterMixin, unittest.TestCase):
       instance.requested_state = 'destroyed'
       self.assertEqual(self.grid.agregateAndSendUsage(), slapgrid.SLAPGRID_SUCCESS)
       # Assert partition directory is empty
-      self.assertItemsEqual(os.listdir(self.instance_root), ['0', 'etc', 'var'])
+      self.assertInstanceDirectoryListEqual(['0'])
       self.assertItemsEqual(os.listdir(instance.partition_path), [])
       self.assertItemsEqual(os.listdir(self.software_root),
                             [instance.software.software_hash])
@@ -1607,7 +1609,7 @@ class TestSlapgridUsageReport(MasterMixin, unittest.TestCase):
       instance.requested_state = 'started'
       instance.software.setBuildout(WRAPPER_CONTENT)
       self.assertEqual(self.grid.processComputerPartitionList(), slapgrid.SLAPGRID_SUCCESS)
-      self.assertItemsEqual(os.listdir(self.instance_root), ['0', 'etc', 'var'])
+      self.assertInstanceDirectoryListEqual(['0'])
       self.assertItemsEqual(os.listdir(instance.partition_path),
                             ['.slapgrid', '.0_wrapper.log', 'buildout.cfg',
                              'etc', 'software_release', 'worked', '.slapos-retention-lock-delay'])
@@ -1623,13 +1625,13 @@ class TestSlapgridUsageReport(MasterMixin, unittest.TestCase):
       # Then run usage report and see if it is still working
       computer.sequence = []
       self.assertEqual(self.grid.agregateAndSendUsage(), slapgrid.SLAPGRID_SUCCESS)
-      self.assertItemsEqual(os.listdir(self.instance_root), ['0', 'etc', 'var'])
+      self.assertInstanceDirectoryListEqual(['0'])
       self.assertItemsEqual(os.listdir(instance.partition_path),
                             ['.slapgrid', '.0_wrapper.log', 'buildout.cfg',
                              'etc', 'software_release', 'worked', '.slapos-retention-lock-delay'])
       wrapper_log = os.path.join(instance.partition_path, '.0_wrapper.log')
       self.assertLogContent(wrapper_log, 'Working')
-      self.assertItemsEqual(os.listdir(self.instance_root), ['0', 'etc', 'var'])
+      self.assertInstanceDirectoryListEqual(['0'])
       self.assertItemsEqual(os.listdir(instance.partition_path),
                             ['.slapgrid', '.0_wrapper.log', 'buildout.cfg',
                              'etc', 'software_release', 'worked', '.slapos-retention-lock-delay'])
@@ -1653,7 +1655,7 @@ class TestSlapgridUsageReport(MasterMixin, unittest.TestCase):
       instance.requested_state = 'destroyed'
       self.assertEqual(self.grid.processComputerPartitionList(), slapgrid.SLAPGRID_SUCCESS)
       # Assert partition directory is empty
-      self.assertItemsEqual(os.listdir(self.instance_root), ['0', 'etc', 'var'])
+      self.assertInstanceDirectoryListEqual(['0'])
       self.assertItemsEqual(os.listdir(instance.partition_path), [])
       self.assertItemsEqual(os.listdir(self.software_root), [instance.software.software_hash])
       self.assertEqual(computer.sequence, ['/getFullComputerInformation'])
@@ -1672,7 +1674,7 @@ class TestSlapgridUsageReport(MasterMixin, unittest.TestCase):
       instance.requested_state = 'destroyed'
       self.assertEqual(self.grid.agregateAndSendUsage(), slapgrid.SLAPGRID_SUCCESS)
       # Assert partition directory is empty
-      self.assertItemsEqual(os.listdir(self.instance_root), ['0', 'etc', 'var'])
+      self.assertInstanceDirectoryListEqual(['0'])
       self.assertItemsEqual(os.listdir(instance.partition_path), [])
       self.assertItemsEqual(os.listdir(self.software_root), [instance.software.software_hash])
       self.assertEqual(computer.sequence, ['/getFullComputerInformation'])