First version of network cache integration with zc.buildout.

src/zc/buildout/buildout.py

@@ -138,6 +138,7 @@ _buildout_default_options = _annotate_section({
     'socket-timeout': '',
     'unzip': 'false',
     'use-dependency-links': 'true',
+    'network-cache': 'http://127.0.0.1:5001/',
     }, 'DEFAULT_VALUE')
 
 
@@ -362,6 +363,7 @@ class Buildout(UserDict.DictMixin):
                 newest=self.newest,
                 allow_hosts=self._allow_hosts,
                 prefer_final=not self.accept_buildout_test_releases,
+                network_cache=options['network-cache'],
                 )
 
         # Now copy buildout and setuptools eggs, and record destination eggs:
@@ -860,6 +862,7 @@ class Buildout(UserDict.DictMixin):
             path = [options['develop-eggs-directory']],
             allow_hosts = self._allow_hosts,
             prefer_final=not self.accept_buildout_test_releases,
+            network_cache=options.get('network-cache'),
             )
 
         upgraded = []
@@ -1087,7 +1090,8 @@ def _install_and_load(spec, group, entry, buildout):
                 working_set=pkg_resources.working_set,
                 newest=buildout.newest,
                 allow_hosts=buildout._allow_hosts,
-                prefer_final=not buildout.accept_buildout_test_releases)
+                prefer_final=not buildout.accept_buildout_test_releases,
+                network_cache=buildout_options.get('network-cache'))
 
         __doing__ = 'Loading %s recipe entry %s:%s.', group, spec, entry
         return pkg_resources.load_entry_point(

src/zc/buildout/download.py

@@ -40,6 +40,11 @@ class ChecksumError(zc.buildout.UserError):
 url_opener = URLOpener()
 
 
+from zc.buildout.networkcache import get_sha256sum_from_networkcached, \
+                                     download_network_cached, \
+                                     upload_network_cached
+
+
 class Download(object):
     """Configurable download utility.
 
@@ -72,6 +77,7 @@ class Download(object):
         self.fallback = fallback
         self.hash_name = hash_name
         self.logger = logger or logging.getLogger('zc.buildout')
+        self.network_cache = options.get('network-cache')
 
     @property
     def download_cache(self):
@@ -174,20 +180,28 @@ class Download(object):
         self.logger.info('Downloading %s' % url)
         urllib._urlopener = url_opener
         handle, tmp_path = tempfile.mkstemp(prefix='buildout-')
+
+        sha256sum = get_sha256sum_from_networkcached(self.network_cache,
+                                                          url,
+                                                          self.logger)
         try:
+            try:
+                if not download_network_cached(self.network_cache,
+                                            tmp_path, sha256sum, self.logger):
+
+                    # Download from original url
                     tmp_path, headers = urllib.urlretrieve(url, tmp_path)
                     if not check_md5sum(tmp_path, md5sum):
                         raise ChecksumError(
                             'MD5 checksum mismatch downloading %r' % url)
-        except IOError, e:
-            os.remove(tmp_path)
-            raise zc.buildout.UserError("Error downloading extends for URL "
-                              "%s: %r" % (url, e[1:3]))
-        except Exception, e:
-            os.remove(tmp_path)
-            raise
+                    # Upload the file to networkcached.
+                    upload_network_cached(self.network_cache, url,
+                                                         tmp_path, self.logger)
             finally:
                 os.close(handle)
+        except:
+            os.remove(tmp_path)
+            raise
 
         if path:
             shutil.move(tmp_path, path)

src/zc/buildout/easy_install.py

@@ -18,6 +18,8 @@ It doesn't install scripts.  It uses setuptools and requires it to be
 installed.
 """
 
+from zc.buildout.networkcache import fetch_from_network_cache, \
+                                     upload_network_cached
 import distutils.errors
 import fnmatch
 import glob
@@ -335,6 +337,7 @@ class Installer:
                  include_site_packages=None,
                  allowed_eggs_from_site_packages=None,
                  prefer_final=None,
+                 network_cache=None,
                  ):
         self._dest = dest
         self._allow_hosts = allow_hosts
@@ -403,6 +406,8 @@ class Installer:
         if versions is not None:
             self._versions = versions
 
+        self._network_cache = network_cache
+
     _allowed_eggs_from_site_packages_regex = None
     def allow_site_package_egg(self, name):
         if (not self._include_site_packages or
@@ -701,8 +706,13 @@ class Installer:
             and (realpath(os.path.dirname(dist.location)) == download_cache)
             ):
             return dist
-
+        new_location = fetch_from_network_cache(self._network_cache,
+                                                dist.location, tmp, logger)
+        if new_location is None:
             new_location = self._index.download(dist.location, tmp)
+            upload_network_cached(self._network_cache,
+                                  dist.location, new_location, logger)
+
         if (download_cache
             and (realpath(new_location) == realpath(dist.location))
             and os.path.isfile(new_location)
@@ -1080,13 +1090,13 @@ def install(specs, dest,
             path=None, working_set=None, newest=True, versions=None,
             use_dependency_links=None, allow_hosts=('*',),
             include_site_packages=None, allowed_eggs_from_site_packages=None,
-            prefer_final=None):
+            prefer_final=None, network_cache=None):
     installer = Installer(
         dest, links, index, executable, always_unzip, path, newest,
         versions, use_dependency_links, allow_hosts=allow_hosts,
         include_site_packages=include_site_packages,
         allowed_eggs_from_site_packages=allowed_eggs_from_site_packages,
-        prefer_final=prefer_final)
+        prefer_final=prefer_final, network_cache=network_cache)
     return installer.install(specs, working_set)
 
 

src/zc/buildout/networkcache.py

+##############################################################################
+#
+# Copyright (c) 2010 ViFiB SARL and Contributors.
+# All Rights Reserved.
+#
+# This software is subject to the provisions of the Zope Public License,
+# Version 2.1 (ZPL).  A copy of the ZPL should accompany this distribution.
+# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
+# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
+# FOR A PARTICULAR PURPOSE.
+#
+##############################################################################
+
+
+import os
+import urllib
+import re
+import posixpath
+import base64
+import hashlib
+import json
+
+_md5_re = re.compile(r'md5=([a-f0-9]+)')
+
+def _get_md5_from_url(url):
+  match = _md5_re.search(url)
+  if match:
+    return match.group(1)
+  return None
+
+def _get_hash_from_file(path, hash_object):
+    """
+      Calculate the hash from file.
+    """
+    f = open(path, 'rb')
+    try:
+        chunk = f.read(2**16)
+        while chunk:
+            hash_object.update(chunk)
+            chunk = f.read(2**16)
+        return hash_object.hexdigest()
+    finally:
+        f.close()
+
+
+def _get_sha256_from_file(path):
+    """
+      Return the sha256sum from file.
+    """
+    if not os.path.isfile(path):
+        raise ValueError, 'Could not extract sha256sum.' \
+                                         ' The path is not a file: %s ' % path
+    return _get_hash_from_file(path, hashlib.sha256())
+
+def check_sha256sum(path, sha256sum):
+    """Tell whether the SHA256sum checksum of the file at path matches
+    """
+    return sha256sum == _get_hash_from_file(path, hashlib.sha256())
+
+def _update_network_cached_map(network_cache, sha256sum, url, logger):
+    """
+      If the url is not into the map file, it must be updated.
+    """
+    # Keep the log
+    print os.environ.get('PWD')
+    map_path = os.path.join(os.environ.get('PWD'), '.networkcached.log')
+    mode = 'r+'
+    if not os.path.exists(map_path):
+        mode = 'w+'
+    f = open(map_path, mode)
+    try:
+        flag = False
+        for line in f:
+            if url in line:
+                flag = True
+                break
+        if not flag:
+          f.seek(0, 2)
+          f.write('%s %s\n' % (sha256sum, url))
+          f.truncate(f.tell())
+    finally:
+        f.close()
+
+    # Update the map at networkcached server.
+    network_cached_url = os.path.join(network_cache, 'map/add')
+    try:
+        result = urllib.urlopen(network_cached_url, \
+                     urllib.urlencode({'sha256sum': sha256sum, 'url': url}))
+        if int(result.code) != 200:
+            logger.info('Fail to update the network cache map: %s %s' % \
+                                                         (sha256sum, url))
+    except IOError, e:
+         logger.info('An error occurred to update the map in networkcached.' \
+                                                                ' %s' % str(e))
+
+
+def get_sha256sum_from_networkcached(network_cache, url, logger):
+    """
+      Return the sha256sum if the url exists in networkcached server.
+    """
+    network_cached_url = os.path.join(network_cache,
+                                          'map/get', base64.encodestring(url))
+    try:
+        result = urllib.urlopen(network_cached_url)
+        if int(result.code) == 200:
+          return json.loads(result.read().strip()).get('sha256sum', None)
+    except IOError, e:
+        logger.info('An error occurred to get sha256sum from networkcached:' \
+                                                               ' %s' % str(e))
+
+
+def download_network_cached(network_cache, path, sha256sum, logger):
+    """Download from a network cache provider
+
+    If something fail (providor be offline, or hash_string fail), we ignore
+    network cached files.
+
+    return True if download succeeded.
+    """
+    if sha256sum is None:
+        # Not able to use network cache
+        return False
+
+    url = os.path.join(network_cache, sha256sum)
+    logger.info('Downloading from network cache %s' % url)
+    try:
+        path, headers = urllib.urlretrieve(url, path)
+        if not check_sha256sum(path, sha256sum):
+            logger.info('MD5/SHA256 checksum mismatch downloading %r' % url)
+            return False
+    except IOError, e:
+        logger.info('Fail to download from network cache %s' % url)
+        return False
+
+    return True
+
+def upload_network_cached(network_cache, external_url, path, logger):
+    """Upload file to a network cache server"""
+    if network_cache in [None, '']:
+        logger.debug(
+         'Upload cache ignored, network-cache was not provided')
+        return False
+
+    sha256sum = _get_sha256_from_file(path)
+    try:
+        f = open(path, 'r')
+        data = f.read()
+        url = os.path.join(network_cache, sha256sum)
+        try:
+            result = urllib.urlopen(url, urllib.urlencode({
+                                         "data" : base64.encodestring(data)}))
+            if result.code == 200 and \
+                        json.loads(result.read()).get('sha') == sha256sum:
+                _update_network_cached_map(network_cache, sha256sum,
+                                           external_url, logger)
+
+        except (IOError, EOFError), e:
+            logger.info('Fail to upload cache on %s. %s' % (external_url, str(e)))
+    finally:
+        f.close()
+    return True
+
+
+def _get_filename_from_url(url):
+    """Inspired how pip get filename from url.
+    """
+    url = url.split('#', 1)[0]
+    url = url.split('?', 1)[0]
+    url = url.rstrip('/')
+    name = posixpath.basename(url)
+    assert name, (
+           'URL %r produced no filename' % url)
+    return name
+
+def fetch_from_network_cache(network_cache, location, tmp, logger=None):
+    """ Try to download from a network cache and preserve
+        original filename.
+    """
+    # Make a id from URL
+    md5sum = _get_md5_from_url(location)
+
+    filename = _get_filename_from_url(location)
+    path = os.path.join(tmp, filename)
+
+    sha256sum = get_sha256sum_from_networkcached(network_cache,
+                                                 location, logger)
+    if sha256sum is None:
+      return None
+
+    is_downloaded = download_network_cached(network_cache, path, sha256sum, logger)
+    if is_downloaded:
+        return path
+
+    return None

zc.recipe.egg_/src/zc/recipe/egg/egg.py

@@ -88,6 +88,9 @@ class Eggs(object):
             kw = {}
             if 'unzip' in options:
                 kw['always_unzip'] = options.query_bool('unzip', None)
+            if 'network-cache' in b_options:
+                kw['network_cache'] = b_options.get('network-cache')
+
             ws = zc.buildout.easy_install.install(
                 distributions, options['eggs-directory'],
                 links=self.links,