Cache pypi index, extensions and recipes using buildout internals.

src/zc/buildout/buildout.py

@@ -1193,7 +1193,17 @@ class Buildout(UserDict.DictMixin):
                 links = self['buildout'].get('find-links', '').split(),
                 index = self['buildout'].get('index'),
                 newest=self.newest, allow_hosts=self._allow_hosts,
-                prefer_final=not self.accept_buildout_test_releases)
+                prefer_final=not self.accept_buildout_test_releases,
+                download_cache_url=self.download_cache_url,
+                download_dir_url=self.download_dir_url,
+                upload_cache_url=self.upload_cache_url,
+                upload_dir_url=self.upload_dir_url,
+                signature_private_key_file=self.signature_private_key_file,
+                signature_certificate_list=self.signature_certificate_list,
+                shacache_cert_file=self.shacache_cert_file,
+                shacache_key_file=self.shacache_key_file,
+                shadir_cert_file=self.shadir_cert_file,
+                shadir_key_file=self.shadir_key_file,)
 
             # Clear cache because extensions might now let us read pages we
             # couldn't read before.

src/zc/buildout/networkcache.py

@@ -12,12 +12,12 @@
 #
 ##############################################################################
 
+#XXX factor with slapos/grid/networkcache.py and use libnetworkcache helpers
 
 import hashlib
 import os
 import posixpath
 import re
-import shutil
 import urllib2
 import urlparse
 import traceback
@@ -25,7 +25,11 @@ import traceback
 try:
     try:
         from slapos.libnetworkcache import NetworkcacheClient, UploadError, \
-          DirectoryNotFound
+           DirectoryNotFound
+        from slapos.networkcachehelper import \
+           helper_download_network_cached, \
+           helper_upload_network_cached_from_file, \
+           helper_download_network_cached_to_file
     except ImportError:
         LIBNETWORKCACHE_ENABLED = False
     else:
@@ -68,12 +72,19 @@ def get_directory_key(url):
     Basically check if the url belongs to pypi:
       - if yes, the directory key will be pypi-buildout-urlmd5
       - if not, the directory key will be slapos-buildout-urlmd5
+    # XXX why is that?
     """
     urlmd5 = hashlib.md5(url).hexdigest()
     if 'pypi' in url:
       return 'pypi-buildout-%s' % urlmd5
     return 'slapos-buildout-%s' % urlmd5
 
+@fallback_call
+def get_index_directory_key(url, requirement):
+    """Returns directory hash based on egg requirement.
+    """
+    return 'pypi-index-%s-%s' % (hashlib.md5(url).hexdigest(), requirement)
+
 
 @fallback_call
 def download_network_cached(dir_url, cache_url, path, url, logger,
@@ -88,56 +99,74 @@ def download_network_cached(dir_url, cache_url, path, url, logger,
     if not LIBNETWORKCACHE_ENABLED:
         return False
 
-    if not(dir_url and cache_url):
-        return False
     if md5sum is None:
         md5sum = _get_md5_from_url(url)
 
-
     directory_key = get_directory_key(url)
     url = os.path.basename(url)
 
-    if len(signature_certificate_list) == 0:
-        # convert [] into None in order to call nc nicely
-        signature_certificate_list = None
-    try:
-        nc = NetworkcacheClient(cache_url, dir_url,
-            signature_certificate_list=signature_certificate_list)
-    except TypeError:
-      logger.warning('Incompatible version of networkcache, not using it.')
-      return False
-
     logger.debug('Trying to download %s from network cache...' % url)
-    try:
-        file_descriptor = nc.select(directory_key)
 
-        f = open(path, 'w+b')
-        try:
-            shutil.copyfileobj(file_descriptor, f)
-        finally:
-            f.close()
-            file_descriptor.close()
-            logger.info('Downloaded %s from network cache.' % url)
+    if helper_download_network_cached_to_file(
+       dir_url=dir_url,
+       cache_url=cache_url,
+       signature_certificate_list=signature_certificate_list,
+       directory_key=directory_key,
+       path=path):
+        logger.info('Downloaded %s from network cache.' % url)
 
         if not check_md5sum(path, md5sum):
             logger.info('MD5 checksum mismatch downloading %s' % url)
             return False
+        return True
+    return False
 
-    except (IOError, DirectoryNotFound), e:
-        if isinstance(e, urllib2.HTTPError) and e.code == 404:
-          logger.debug('%s does not exist in network cache.' % url)
-        else:
-          logger.debug('Failed to download from network cache %s: %s' % \
-                                                         (url, str(e)))
+@fallback_call
+def download_index_network_cached(dir_url, cache_url, url, requirement, logger,
+                            signature_certificate_list):
+    """
+    XXX description
+    Downloads pypi index from a network cache provider
+
+    If something fail (providor be offline, or hash_string fail), we ignore
+    network cached index.
+
+    return index if succeeded, False otherwise.
+    """
+    if not LIBNETWORKCACHE_ENABLED:
         return False
-    return True
 
+    directory_key = get_index_directory_key(url, requirement)
+
+    wanted_metadata_dict = {
+       'urlmd5':hashlib.md5(url).hexdigest(),
+       'requirement':requirement
+    }
+    required_key_list = ['base']
+
+    result = helper_download_network_cached(dir_url, cache_url,
+       signature_certificate_list,
+       directory_key, wanted_metadata_dict, required_key_list)
+    if result:
+        file_descriptor, metadata = result
+        try:
+            content = file_descriptor.read()
+            logger.info('Downloaded %s from network cache.' % url)
+            return content, metadata['base']
+        except (IOError, DirectoryNotFound), e:
+            if isinstance(e, urllib2.HTTPError) and e.code == 404:
+              logger.debug('%s does not exist in network cache.' % url)
+            else:
+              logger.debug('Failed to download from network cache %s: %s' % \
+                                                             (url, str(e)))
+    return False
 
 @fallback_call
 def upload_network_cached(dir_url, cache_url, external_url, path, logger,
    signature_private_key_file, shacache_cert_file, shacache_key_file,
    shadir_cert_file, shadir_key_file):
     """Upload file to a network cache server"""
+    # XXX use helper and FACTOR code
     if not LIBNETWORKCACHE_ENABLED:
         return False
 
@@ -187,6 +216,64 @@ def upload_network_cached(dir_url, cache_url, external_url, path, logger,
 
     return True
 
+@fallback_call
+def upload_index_network_cached(dir_url, cache_url, external_url, base, requirement, content, logger,
+   signature_private_key_file, shacache_cert_file, shacache_key_file,
+   shadir_cert_file, shadir_key_file):
+    # XXX use helper and FACTOR code
+    """Upload content of a web page to a network cache server"""
+    if not LIBNETWORKCACHE_ENABLED:
+        return False
+
+    if not (dir_url and cache_url):
+        return False
+
+    logger.info('Uploading %s content into network cache.' % external_url)
+
+    directory_key = get_index_directory_key(external_url, requirement)
+    kw = dict(file="file",
+              base=base,
+              urlmd5=hashlib.md5(external_url).hexdigest(),
+              requirement=requirement)
+
+    import tempfile
+    f = tempfile.TemporaryFile()
+    f.write(content)
+
+    # convert '' into None in order to call nc nicely
+    if not signature_private_key_file:
+        signature_private_key_file = None
+    if not shacache_cert_file:
+        shacache_cert_file = None
+    if not shacache_key_file:
+        shacache_key_file = None
+    if not shadir_cert_file:
+        shadir_cert_file = None
+    if not shadir_key_file:
+        shadir_key_file = None
+    try:
+        nc = NetworkcacheClient(cache_url, dir_url,
+            signature_private_key_file=signature_private_key_file,
+            shacache_cert_file=shacache_cert_file,
+            shacache_key_file=shacache_key_file,
+            shadir_cert_file=shadir_cert_file,
+            shadir_key_file=shadir_key_file)
+    except TypeError:
+        logger.warning('Incompatible version of networkcache, not using it.')
+        return False
+
+    try:
+        return nc.upload_generic(f, directory_key, **kw)
+    except (IOError, UploadError), e:
+        logger.info('Fail to upload file. %s' % \
+                                                  (str(e)))
+        return False
+
+    finally:
+      f.close()
+
+    return True
+
 
 @fallback_call
 def get_filename_from_url(url):