Merge branch 'binary_cache'

slapos/grid/SlapObject.py

@@ -38,6 +38,8 @@ import utils
 from svcbackend import getSupervisorRPC
 from exception import BuildoutFailedError, WrongPermissionError, \
     PathDoesNotExistError
+from networkcache import download_network_cached, upload_network_cached
+import tarfile
 
 REQUIRED_COMPUTER_PARTITION_PERMISSION = '0750'
 
@@ -45,27 +47,75 @@ REQUIRED_COMPUTER_PARTITION_PERMISSION = '0750'
 class Software(object):
   """This class is responsible of installing a software release"""
   def __init__(self, url, software_root, console, buildout,
-      signature_private_key_file=None, upload_cache_url=None,
-      upload_dir_url=None, shacache_cert_file=None, shacache_key_file=None,
-      shadir_cert_file=None, shadir_key_file=None):
+      signature_private_key_file=None, signature_certificate_list=None,
+      upload_cache_url=None, upload_dir_url=None, shacache_cert_file=None,
+      shacache_key_file=None, shadir_cert_file=None, shadir_key_file=None,
+      download_binary_cache_url=None, upload_binary_cache_url=None,
+      download_binary_dir_url=None, upload_binary_dir_url=None):
     """Initialisation of class parameters
     """
     self.url = url
     self.software_root = software_root
+    self.software_url_hash = utils.getSoftwareUrlHash(self.url)
     self.software_path = os.path.join(self.software_root,
-                                      utils.getSoftwareUrlHash(self.url))
+                                      self.software_url_hash)
     self.buildout = buildout
     self.logger = logging.getLogger('BuildoutManager')
     self.console = console
     self.signature_private_key_file = signature_private_key_file
+    self.signature_certificate_list = signature_certificate_list
     self.upload_cache_url = upload_cache_url
     self.upload_dir_url = upload_dir_url
     self.shacache_cert_file = shacache_cert_file
     self.shacache_key_file = shacache_key_file
     self.shadir_cert_file = shadir_cert_file
     self.shadir_key_file = shadir_key_file
+    self.download_binary_cache_url = download_binary_cache_url
+    self.upload_binary_cache_url = upload_binary_cache_url
+    self.download_binary_dir_url = download_binary_dir_url
+    self.upload_binary_dir_url = upload_binary_dir_url
 
   def install(self):
+    """ Fetches binary cache if possible.
+    Installs from buildout otherwise.
+    """
+    tarname = self.software_url_hash
+    cache_dir = tempfile.mkdtemp()
+    tarpath = os.path.join(cache_dir, tarname)
+    if (not os.path.exists(self.software_path)) \
+      and download_network_cached(
+          self.download_binary_cache_url,
+          self.download_binary_dir_url,
+          self.url, self.software_root,
+          self.software_url_hash,
+          tarpath, self.logger,
+          self.signature_certificate_list):
+        tar = tarfile.open(tarpath)
+        try:
+          tar.extractall(path=self.software_root)
+        finally:
+          tar.close()
+    else:
+        self._install_from_buildout()
+        tar = tarfile.open(tarpath, "w:gz")
+        try:
+          tar.add(self.software_path, arcname=self.software_url_hash)
+        finally:
+          tar.close()
+        upload_network_cached(
+          self.software_root,
+          self.url, self.software_url_hash,
+          self.upload_binary_cache_url,
+          self.upload_binary_dir_url,
+          tarpath, self.logger,
+          self.signature_private_key_file,
+          self.shacache_cert_file,
+          self.shacache_key_file,
+          self.shadir_cert_file,
+          self.shadir_key_file)
+    shutil.rmtree(cache_dir)
+      
+  def _install_from_buildout(self):
     """ Fetches buildout configuration from the server, run buildout with
     it. If it fails, we notify the server.
     """

slapos/grid/networkcache.py

+##############################################################################
+#
+# Copyright (c) 2010 ViFiB SARL and Contributors.
+# All Rights Reserved.
+#
+# This software is subject to the provisions of the Zope Public License,
+# Version 2.1 (ZPL).  A copy of the ZPL should accompany this distribution.
+# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
+# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
+# FOR A PARTICULAR PURPOSE.
+#
+##############################################################################
+
+
+import hashlib
+import os
+import posixpath
+import re
+import shutil
+import urlparse
+import traceback
+import utils
+import json
+import platform
+
+try:
+    try:
+        from slapos.libnetworkcache import NetworkcacheClient, UploadError, \
+          DirectoryNotFound
+    except ImportError:
+        LIBNETWORKCACHE_ENABLED = False
+    else:
+        LIBNETWORKCACHE_ENABLED = True
+except:
+    print 'There was problem while trying to import slapos.libnetworkcache:'\
+        '\n%s' % traceback.format_exc()
+    LIBNETWORKCACHE_ENABLED = False
+    print 'Networkcache forced to be disabled.'
+
+def fallback_call(function):
+    """Decorator which disallow to have any problem while calling method"""
+    def wrapper(self, *args, **kwd):
+        """
+        Log the call, and the result of the call
+        """
+        try:
+            return function(self, *args, **kwd)
+        except: # indeed, *any* exception is swallowed
+            print 'There was problem while calling method %r:\n%s' % (
+                function.__name__, traceback.format_exc())
+            return False
+    wrapper.__doc__ = function.__doc__
+    return wrapper
+
+
+@fallback_call
+def download_network_cached(cache_url, dir_url, software_url, software_root,
+                            key, path, logger, signature_certificate_list):
+    """Downloads from a network cache provider
+
+    return True if download succeeded.
+    """
+    if not LIBNETWORKCACHE_ENABLED:
+        return False
+
+    if not(cache_url and dir_url and software_url and software_root):
+        return False
+
+    # In order to call nc nicely.
+    if len(signature_certificate_list) == 0:
+        signature_certificate_list = None
+    try:
+        nc = NetworkcacheClient(cache_url, dir_url,
+            signature_certificate_list=signature_certificate_list)
+    except TypeError:
+      logger.warning('Incompatible version of networkcache, not using it.')
+      return False
+
+    logger.info('Downloading %s binary from network cache.' % software_url)
+    try:
+        file_descriptor = None
+        json_entry_list = nc.select_generic(key)
+        for entry in json_entry_list:
+            json_information, _ = entry
+            try:
+                tags = json.loads(json_information)
+                if tags.get('machine') != platform.machine():
+                    continue
+                if tags.get('os') != str(platform.linux_distribution()):
+                    continue
+                if tags.get('software_url') != software_url:
+                    continue
+                if tags.get('software_root') != software_root:
+                    continue
+                sha512 = tags.get('sha512')
+                file_descriptor = nc.download(sha512)
+                break
+            except Exception:
+                continue
+        if file_descriptor is not None:
+            f = open(path, 'w+b')
+            try:
+                shutil.copyfileobj(file_descriptor, f)
+            finally:
+                f.close()
+                file_descriptor.close()
+            return True
+    except (IOError, DirectoryNotFound), e:
+        logger.info('Failed to download from network cache %s: %s' % \
+                                                       (software_url, str(e)))
+    return False
+
+
+@fallback_call
+def upload_network_cached(software_root, software_url, cached_key,
+    cache_url, dir_url, path, logger, signature_private_key_file,
+    shacache_cert_file, shacache_key_file, shadir_cert_file, shadir_key_file):
+    """Upload file to a network cache server"""
+    if not LIBNETWORKCACHE_ENABLED:
+        return False
+
+    if not (software_root and software_url and cached_key \
+                          and cache_url and dir_url):
+        return False
+
+    logger.info('Uploading %s binary into network cache.' % software_url)
+
+    # YXU: "file" and "urlmd5" should be removed when server side is ready
+    kw = dict(
+      file="file",
+      urlmd5="urlmd5",
+      software_url=software_url,
+      software_root=software_root,
+      machine=platform.machine(),
+      os=str(platform.linux_distribution())
+    )
+
+    f = open(path, 'r')
+    # convert '' into None in order to call nc nicely
+    if not signature_private_key_file:
+        signature_private_key_file = None
+    if not shacache_cert_file:
+        shacache_cert_file = None
+    if not shacache_key_file:
+        shacache_key_file = None
+    if not shadir_cert_file:
+        shadir_cert_file = None
+    if not shadir_key_file:
+        shadir_key_file = None
+    try:
+        nc = NetworkcacheClient(cache_url, dir_url,
+            signature_private_key_file=signature_private_key_file,
+            shacache_cert_file=shacache_cert_file,
+            shacache_key_file=shacache_key_file,
+            shadir_cert_file=shadir_cert_file,
+            shadir_key_file=shadir_key_file)
+    except TypeError:
+        logger.warning('Incompatible version of networkcache, not using it.')
+        return False
+
+    try:
+        return nc.upload_generic(f, cached_key, **kw)
+    except (IOError, UploadError), e:
+        logger.info('Fail to upload file. %s' % (str(e)))
+        return False
+    finally:
+      f.close()
+
+    return True

slapos/grid/slapgrid.py

@@ -204,6 +204,17 @@ def parseArgumentTupleAndReturnSlapgridObject(*argument_tuple):
   if not option_dict.get('supervisord_socket'):
     option_dict['supervisord_socket'] = \
       os.path.join(option_dict['instance_root'], 'supervisord.socket')
+
+  signature_certificate_list_string = \
+    option_dict.get('signature-certificate-list', None)
+  if signature_certificate_list_string is not None:
+    cert_marker = "-----BEGIN CERTIFICATE-----"
+    signature_certificate_list = [cert_marker + '\n' + q.strip() \
+      for q in signature_certificate_list_string.split(cert_marker) \
+        if q.strip()]
+  else:
+    signature_certificate_list = None
+
   # Returning new Slapgrid instance and options
   return ([Slapgrid(software_root=option_dict['software_root'],
             instance_root=option_dict['instance_root'],
@@ -218,7 +229,16 @@ def parseArgumentTupleAndReturnSlapgridObject(*argument_tuple):
             master_ca_file=master_ca_file,
             certificate_repository_path=certificate_repository_path,
             signature_private_key_file=signature_private_key_file,
+            signature_certificate_list=signature_certificate_list,
+            download_binary_cache_url=\
+              option_dict.get('download-binary-cache-url', None),
+            upload_binary_cache_url=\
+              option_dict.get('upload-binary-cache-url', None),
             upload_cache_url=option_dict.get('upload-cache-url', None),
+            download_binary_dir_url=\
+              option_dict.get('download-binary-dir-url', None),
+            upload_binary_dir_url=\
+              option_dict.get('upload-binary-dir-url', None),
             upload_dir_url=option_dict.get('upload-dir-url', None),
             console=option_dict['console'],
             buildout=option_dict.get('buildout'),
@@ -299,7 +319,12 @@ class Slapgrid(object):
                key_file=None,
                cert_file=None,
                signature_private_key_file=None,
+               signature_certificate_list=None,
+               download_binary_cache_url=None,
+               upload_binary_cache_url=None,
                upload_cache_url=None,
+               download_binary_dir_url=None,
+               upload_binary_dir_url=None,
                upload_dir_url=None,
                master_ca_file=None,
                certificate_repository_path=None,
@@ -323,7 +348,12 @@ class Slapgrid(object):
     self.master_ca_file = master_ca_file
     self.certificate_repository_path = certificate_repository_path
     self.signature_private_key_file = signature_private_key_file
+    self.signature_certificate_list = signature_certificate_list
+    self.download_binary_cache_url = download_binary_cache_url
+    self.upload_binary_cache_url = upload_binary_cache_url
     self.upload_cache_url = upload_cache_url
+    self.download_binary_dir_url = download_binary_dir_url
+    self.upload_binary_dir_url = upload_binary_dir_url
     self.upload_dir_url = upload_dir_url
     self.shacache_cert_file = shacache_cert_file
     self.shacache_key_file = shacache_key_file
@@ -403,7 +433,12 @@ class Slapgrid(object):
         Software(url=software_release_uri, software_root=self.software_root,
             console=self.console, buildout=self.buildout,
             signature_private_key_file=self.signature_private_key_file,
+            signature_certificate_list=self.signature_certificate_list,
+            download_binary_cache_url=self.download_binary_cache_url,
+            upload_binary_cache_url=self.upload_binary_cache_url,
             upload_cache_url=self.upload_cache_url,
+            download_binary_dir_url=self.download_binary_dir_url,
+            upload_binary_dir_url=self.upload_binary_dir_url,
             upload_dir_url=self.upload_dir_url,
             shacache_cert_file=self.shacache_cert_file,
             shacache_key_file=self.shacache_key_file,