allow to format and configure external data storage folder

slapos/format.py

@@ -218,6 +218,7 @@ class Computer(object):
   "Object representing the computer"
   instance_root = None
   software_root = None
+  instance_storage_home = None
 
   def __init__(self, reference, interface=None, addr=None, netmask=None,
                ipv6_interface=None, software_user='slapsoft',
@@ -386,6 +387,7 @@ class Computer(object):
         tap = Tap(partition_dict['reference'])
 
       address_list = partition_dict['address_list']
+      external_storage_list = partition_dict.get('external_storage_list', [])
 
       partition = Partition(
           reference=partition_dict['reference'],
@@ -393,6 +395,7 @@ class Computer(object):
           user=user,
           address_list=address_list,
           tap=tap,
+          external_storage_list=external_storage_list,
       )
 
       computer.partition_list.append(partition)
@@ -464,6 +467,15 @@ class Computer(object):
       os.chown(slapsoft.path, slapsoft_pw.pw_uid, slapsoft_pw.pw_gid)
     os.chmod(self.software_root, 0o755)
 
+    # get list of instance external storage if exist
+    instance_external_list = []
+    if self.instance_storage_home:
+      # XXX - Hard limit for storage number to 4
+      for i in range(1, 5):
+        storage_path = os.path.join(self.instance_storage_home, 'data%s' % i)
+        if os.path.exists(storage_path):
+          instance_external_list.append(storage_path)
+
     tap_address_list = []
     if alter_network and self.tap_gateway_interface and create_tap:
       gateway_addr_dict = getIfaceAddressIPv4(self.tap_gateway_interface)
@@ -481,6 +493,8 @@ class Computer(object):
         partition.path = os.path.join(self.instance_root, partition.reference)
         partition.user.setPath(partition.path)
         partition.user.additional_group_list = [slapsoft.name]
+        partition.external_storage_list = ['%s/%s' % (path, partition.reference)
+                                            for path in instance_external_list]
         if alter_user:
           partition.user.create()
 
@@ -510,6 +524,7 @@ class Computer(object):
 
         # Reconstructing partition's directory
         partition.createPath(alter_user)
+        partition.createExternalPath(alter_user)
 
         # Reconstructing partition's address
         # There should be two addresses on each Computer Partition:
@@ -555,7 +570,7 @@ class Computer(object):
 class Partition(object):
   "Represent a computer partition"
 
-  def __init__(self, reference, path, user, address_list, tap):
+  def __init__(self, reference, path, user, address_list, tap, external_storage_list=[]):
     """
     Attributes:
       reference: String, the name of the partition.
@@ -563,6 +578,7 @@ class Partition(object):
       user: User, the user linked to this partition.
       address_list: List of associated IP addresses.
       tap: Tap, the tap interface linked to this partition.
+      external_storage_list: Base path list of folder to format for data storage
     """
 
     self.reference = str(reference)
@@ -570,6 +586,7 @@ class Partition(object):
     self.user = user
     self.address_list = address_list or []
     self.tap = tap
+    self.external_storage_list = []
 
   def __getinitargs__(self):
     return (self.reference, self.path, self.user, self.address_list, self.tap)
@@ -589,6 +606,21 @@ class Partition(object):
       os.chown(self.path, owner_pw.pw_uid, owner_pw.pw_gid)
     os.chmod(self.path, 0o750)
 
+  def createExternalPath(self, alter_user=True):
+    """
+    Create and external directory of the partition, assign to the partition user
+    and give it the 750 permission. In case if path exists just modifies it.
+    """
+
+    for path in self.external_storage_list:
+      storage_path = os.path.abspath(path)
+      owner = self.user if self.user else User('root')
+      if not os.path.exists(storage_path):
+        os.mkdir(storage_path, 0o750)
+      if alter_user:
+        owner_pw = pwd.getpwnam(owner.name)
+        os.chown(storage_path, owner_pw.pw_uid, owner_pw.pw_gid)
+      os.chmod(storage_path, 0o750)
 
 class User(object):
   """User: represent and manipulate a user on the system."""
@@ -1167,6 +1199,7 @@ def do_format(conf):
 
   computer.instance_root = conf.instance_root
   computer.software_root = conf.software_root
+  computer.instance_storage_home = conf.instance_storage_home
   conf.logger.info('Updating computer')
   address = computer.getAddress(conf.create_tap)
   computer.address = address['addr']
@@ -1208,6 +1241,7 @@ class FormatConfig(object):
   software_user = None
   tap_gateway_interface = None
   use_unique_local_address_block = None
+  instance_storage_home = None
 
   def __init__(self, logger):
     self.logger = logger

slapos/grid/SlapObject.py

@@ -57,6 +57,8 @@ WATCHDOG_MARK = '-on-watch'
 
 REQUIRED_COMPUTER_PARTITION_PERMISSION = 0o750
 
+CP_STORAGE_FOLDER_NAME = 'DATA'
+
 # XXX not very clean. this is changed when testing
 PROGRAM_PARTITION_TEMPLATE = pkg_resources.resource_stream(__name__,
             'templates/program_partition_supervisord.conf.in').read()
@@ -341,7 +343,8 @@ class Partition(object):
                logger,
                certificate_repository_path=None,
                retention_delay='0',
-               instance_min_free_space=None
+               instance_min_free_space=None,
+               instance_storage_home=''
                ):
     """Initialisation of class parameters"""
     self.buildout = buildout
@@ -358,6 +361,7 @@ class Partition(object):
     self.partition_id = partition_id
     self.server_url = server_url
     self.software_release_url = software_release_url
+    self.instance_storage_home = instance_storage_home
 
     self.key_file = ''
     self.cert_file = ''
@@ -512,6 +516,7 @@ class Partition(object):
             'software_release_url': self.software_release_url,
             'key_file': self.key_file,
             'cert_file': self.cert_file,
+            'storage_home': self.instance_storage_home,
         }
     open(config_location, 'w').write(buildout_text)
     os.chmod(config_location, 0o640)
@@ -684,12 +689,22 @@ class Partition(object):
       sr_symlink = os.path.join(self.instance_path, 'software_release')
       if os.path.islink(sr_symlink):
         os.unlink(sr_symlink)
+      data_base_link = os.path.join(self.instance_path, CP_STORAGE_FOLDER_NAME)
+      if self.instance_storage_home and os.path.exists(data_base_link) and \
+                                os.path.isdir(data_base_link):
+        for filename in os.listdir(data_base_link):
+          data_symlink = os.path.join(data_base_link, filename)
+          partition_data_path = os.path.join(self.instance_storage_home,
+                                                    filename, self.partition_id)
+          if os.path.lexists(data_symlink):
+            os.unlink(data_symlink)
+          if os.path.exists(partition_data_path):
+            self.cleanupFolder(partition_data_path)
+
+      self.cleanupFolder(self.instance_path)
+      
+      # Cleanup all Data storage location of this partition
       
-      for root, dirs, file_list in os.walk(self.instance_path):
-        for directory in dirs:
-          shutil.rmtree(os.path.join(self.instance_path, directory))
-        for file in file_list:
-          os.remove(os.path.join(self.instance_path, file))
 
       if os.path.exists(self.supervisord_partition_configuration_path):
         os.remove(self.supervisord_partition_configuration_path)
@@ -699,6 +714,15 @@ class Partition(object):
 
     return True
 
+  def cleanupFolder(self, folder_path):
+    """Delete all files and folders in a specified directory
+    """
+    for root, dirs, file_list in os.walk(folder_path):
+      for directory in dirs:
+        shutil.rmtree(os.path.join(folder_path, directory))
+      for file in file_list:
+        os.remove(os.path.join(folder_path, file))
+
   def fetchInformations(self):
     """Fetch usage informations with buildout, returns it.
     """

slapos/grid/slapgrid.py

@@ -232,7 +232,8 @@ def create_slapgrid_object(options, logger):
                   # Try to fetch from deprecated argument
                   computer_partition_filter_list=op.get('only-cp', op.get('only_cp')),
                   software_min_free_space=software_min_free_space,
-                  instance_min_free_space=instance_min_free_space)
+                  instance_min_free_space=instance_min_free_space,
+                  instance_storage_home=op.get('instance_storage_home'))
 
 
 def check_required_only_partitions(existing, required):
@@ -286,6 +287,7 @@ class Slapgrid(object):
                computer_partition_filter_list=None,
                software_min_free_space=None,
                instance_min_free_space=None,
+               instance_storage_home=None,
                ):
     """Makes easy initialisation of class parameters"""
     # Parses arguments
@@ -337,6 +339,10 @@ class Slapgrid(object):
     self.maximum_periodicity = maximum_periodicity
     self.software_min_free_space = software_min_free_space
     self.instance_min_free_space = instance_min_free_space
+    if instance_storage_home:
+      self.instance_storage_home = os.path.abspath(instance_storage_home)
+    else:
+      self.instance_storage_home = ""
 
   def _getWatchdogLine(self):
     invocation_list = [WATCHDOG_PATH]
@@ -644,6 +650,7 @@ class Slapgrid(object):
         logger=self.logger,
         retention_delay=retention_delay,
         instance_min_free_space=self.instance_min_free_space,
+        instance_storage_home=self.instance_storage_home,
       )
       computer_partition_state = computer_partition.getState()
 
@@ -1102,6 +1109,7 @@ class Slapgrid(object):
             certificate_repository_path=self.certificate_repository_path,
             buildout=self.buildout,
             logger=self.logger,
+            instance_storage_home=self.instance_storage_home,
           )
           local_partition.stop()
           try:

slapos/grid/templates/buildout-tail.cfg.in

@@ -22,4 +22,7 @@ server_url = %(server_url)s
 software_release_url = %(software_release_url)s
 key_file = %(key_file)s
 cert_file = %(cert_file)s
+
+[storage-configuration]
+storage-home = %(storage_home)s
 # This is end of zc.builodout profile's tail added by slapgrid