Adding local cache to slap lib, and use it in slapgrid

slapos/grid/slapgrid.py

@@ -38,6 +38,7 @@ from random import random
 import socket
 import subprocess
 import StringIO
+import shutil
 import sys
 import tempfile
 import time
@@ -125,7 +126,6 @@ def parseArgumentTupleAndReturnSlapgridObject(*argument_tuple):
            "(ie.:slappartX, slappartY),"
            "this option will make all others computer partitions be ignored.")
 
-
   # Parses arguments
   if argument_tuple == ():
     # No arguments given to entry point : we parse sys.argv.
@@ -225,11 +225,24 @@ def parseArgumentTupleAndReturnSlapgridObject(*argument_tuple):
   else:
     signature_certificate_list = None
 
-  # Parse cache / binary options
+  # Parse shacache / binary cache options
   option_dict["binary-cache-url-blacklist"] = [
       url.strip() for url in option_dict.get("binary-cache-url-blacklist", ""
           ).split('\n') if url]
 
+  # Set local cache option, use default one if not defined
+  # XXX-Cedric: put one cache per cp/sr/ur
+  # XXX-Cedric: clean after run
+  local_cache = option_dict.get('local-cache',
+      # We put by default on instance_root, because we know this path
+      os.path.join(option_dict['instance_root'], 'cache'))
+      # XXX-Cedric should be shutil.mkdtemp())
+  # Clean old rubbish cache
+  if os.path.exists(local_cache):
+    shutil.rmtree(local_cache)
+  # Create new one
+  os.mkdir(local_cache)
+
   # Sleep for a random time to avoid SlapOS Master being DDOSed by an army of
   # SlapOS Nodes configured with cron.
   if option_dict["now"]:
@@ -278,6 +291,7 @@ def parseArgumentTupleAndReturnSlapgridObject(*argument_tuple):
             develop=option_dict.get('develop', False),
             software_release_filter_list=option_dict.get('only_sr', None),
             computer_partition_filter_list=option_dict.get('only_cp', None),
+            local_cache=local_cache,
             ),
           option_dict])
 
@@ -367,7 +381,8 @@ class Slapgrid(object):
                shadir_key_file=None,
                develop=False,
                software_release_filter_list=None,
-               computer_partition_filter_list=None):
+               computer_partition_filter_list=None,
+               local_cache=None):
     """Makes easy initialisation of class parameters"""
     # Parses arguments
     self.software_root = os.path.abspath(software_root)
@@ -396,7 +411,7 @@ class Slapgrid(object):
     # Configures logger
     self.logger = logging.getLogger('Slapgrid')
     # Creates objects from slap module
-    self.slap = slap.slap()
+    self.slap = slap.slap(local_cache) # XXX-Cedric give it as well to childs
     self.slap.initializeConnection(self.master_url, key_file=self.key_file,
         cert_file=self.cert_file, master_ca_file=self.master_ca_file)
     self.computer = self.slap.registerComputer(self.computer_id)
@@ -898,10 +913,8 @@ class Slapgrid(object):
             # Warn the SlapOS Master that a partition generates corrupted xml
             # report
           else:
-            computer_partition_usage = self.slap.registerComputerPartition(
-                    self.computer_id, computer_partition_id)
-            computer_partition_usage.setUsage(usage)
-            computer_partition_usage_list.append(computer_partition_usage)
+            computer_partition.setUsage(usage)
+            computer_partition_usage_list.append(computer_partition)
             filename_delete_list.append(filename)
         else:
           logger.debug("Usage report %r not found, ignored" % file_path)

slapos/slap/slap.py

@@ -562,7 +562,8 @@ class ConnectionHelper:
   def getFullComputerInformation(self, computer_id):
     # Check if cache (file) is enabled and exists: return directly from cache
     if self.cache_location:
-      computer_cache_location = os.path.join(cache_location, 'computer.xml')
+      computer_cache_location = os.path.join(self.cache_location,
+          'computer.xml')
     if self.cache_location and os.path.exists(computer_cache_location):
       return xml_marshaller.loads(open(computer_cache_location, "r").read())
     # Fetch up-to-date information if no valid cache or cache is not enabled
@@ -570,7 +571,7 @@ class ConnectionHelper:
     # If cache enabled: set it
     if self.cache_location and not os.path.exists(computer_cache_location):
       cache = self.response.read()
-      open(computer_cache_location, "w+").write(cache)
+      open(computer_cache_location, "w").write(cache)
       return xml_marshaller.loads(cache)
     return xml_marshaller.loads(self.response.read())
 
@@ -674,7 +675,7 @@ class slap:
           'rotocol' % (slapgrid_uri, scheme))
     self._connection_helper = ConnectionHelper(connection_wrapper,
           netloc, path, key_file, cert_file, master_ca_file, timeout,
-          cache_location)
+          self.cache_location)
 
   def registerSoftwareRelease(self, software_release):
     """
@@ -697,25 +698,29 @@ class slap:
     Registers connected representation of computer partition and
     returns Computer Partition class object
     """
+    # If local cache directory location is defined: we'll use a file inside it
     if self.cache_location:
-      partition_cache_location = os.path.join(cache_location,
+      partition_cache_location = os.path.join(self.cache_location,
           'partition%s%s.xml' % (computer_guid, partition_id))
+    # If cache for this partition exists: use it
     if self.cache_location and os.path.exists(partition_cache_location):
-      cache = open(partition_cache_location, "r").read()
+      xml_result = open(partition_cache_location, "r").read()
+    # Otherwise: go to server
     else:
       self._connection_helper.GET('/registerComputerPartition?' \
           'computer_reference=%s&computer_partition_reference=%s' % (
             computer_guid, partition_id))
+      xml_result = self._connection_helper.response.read()
+      # If cache is defined, let's write it.
       if self.cache_location and not os.path.exists(partition_cache_location):
-        cache = self._connection_helper.response.read()
-        open(partition_cache_location, "w+").write(cache)
-        
-    result = xml_marshaller.loads(cache)
+        open(partition_cache_location, "w").write(xml_result)
+    result = xml_marshaller.loads(xml_result)
     # XXX: dirty hack to make computer partition usable. xml_marshaller is too
     # low-level for our needs here.
     result._connection_helper = self._connection_helper
     return result
 
+
   def registerOpenOrder(self):
     return OpenOrder(connection_helper=self._connection_helper)