download: add support for slapos.libnetworkcache

This is a rewrite of commit 6e4b8efe
("Support network cache in Download.download()").

src/zc/buildout/buildout.py

@@ -245,7 +245,7 @@ _buildout_default_options = _annotate_section({
     'use-dependency-links': 'true',
     }, 'DEFAULT_VALUE')
 
-network_cache_parameter_dict = {}
+networkcache_client = None
 
 class Buildout(DictMixin):
 
@@ -516,27 +516,16 @@ class Buildout(DictMixin):
         networkcache_section_name = options.get('networkcache-section')
         if networkcache_section_name:
             networkcache_section = self[networkcache_section_name]
-            for k in (
-                'download-cache-url',
-                'download-dir-url',
-                'upload-cache-url',
-                'upload-dir-url',
-                'signature-certificate-list',
-                'signature-private-key-file',
-                'shacache-ca-file',
-                'shacache-cert-file',
-                'shacache-key-file',
-                'shadir-ca-file',
-                'shadir-cert-file',
-                'shadir-key-file',
-            ):
-                network_cache_parameter_dict[k] = networkcache_section.get(k, '')
-                # parse signature list
-                cert_marker = '-----BEGIN CERTIFICATE-----'
-            network_cache_parameter_dict['signature-certificate-list'] = \
-                [cert_marker + '\n' + q.strip() \
-                    for q in network_cache_parameter_dict['signature-certificate-list'].split(cert_marker) \
-                    if q.strip()]
+            try:
+                from slapos.libnetworkcache import NetworkcacheClient
+                global networkcache_client
+                networkcache_client = NetworkcacheClient(networkcache_section)
+            except ImportError:
+                pass
+            except Exception:
+                self._logger.exception("There was problem while trying to"
+                    " import slapos.libnetworkcache."
+                    " Networkcache forced to be disabled.")
 
     def _buildout_path(self, name):
         if '${' in name:

src/zc/buildout/download.py

@@ -20,13 +20,14 @@ except ImportError:
 
 try:
     # Python 3
+    from urllib.error import HTTPError
     from urllib.request import Request, splitport, splituser, urlopen
     from urllib.parse import urlparse, urlunparse
 except ImportError:
     # Python 2
     from urlparse import urlparse
     from urlparse import urlunparse
-    from urllib2 import Request, splitport, splituser, urlopen
+    from urllib2 import HTTPError, Request, splitport, splituser, urlopen
 
 from zc.buildout.easy_install import realpath
 from base64 import b64encode
@@ -216,46 +217,31 @@ class Download(object):
             if not path:
                 handle, tmp_path = tempfile.mkstemp(prefix='buildout-')
                 os.close(handle)
-            from .buildout import network_cache_parameter_dict as nc
-            if not download_network_cached(
-                nc.get('download-dir-url'),
-                nc.get('download-cache-url'),
-                tmp_path, url, self.logger,
-                nc.get('signature-certificate-list'), md5sum):
-                # Download from original url if not cached or md5sum doesn't match.
+            self._download(url, tmp_path, md5sum, alternate_url)
+            cleanup = False
+        finally:
+            if cleanup and tmp_path:
+                remove(tmp_path)
+
+        return tmp_path, not path
+
+    def _download(self, url, path, md5sum=None, alternate_url=None):
+        download_url = url
+        try:
             try:
-                    tmp_path, headers = self.urlretrieve(url, tmp_path)
+                self.urlretrieve(url, path)
             except HTTPError:
                 if not alternate_url:
                     raise
                 self.logger.info('using alternate URL: %s', alternate_url)
                 download_url = alternate_url
-                    tmp_path, headers = self.urlretrieve(
-                        alternate_url, tmp_path)
-                if not check_md5sum(tmp_path, md5sum):
-                    raise ChecksumError(
-                        'MD5 checksum mismatch downloading %r' % download_url)
-                # Upload the file to network cache.
-                if nc.get('upload-cache-url') and nc.get('upload-dir-url'):
-                    upload_network_cached(
-                        nc.get('upload-dir-url'),
-                        nc.get('upload-cache-url'), url, tmp_path, self.logger,
-                        nc.get('signature-private-key-file'),
-                        nc.get('shacache-ca-file'),
-                        nc.get('shacache-cert-file'),
-                        nc.get('shacache-key-file'),
-                        nc.get('shadir-ca-file'),
-                        nc.get('shadir-cert-file'),
-                        nc.get('shadir-key-file'))
-            cleanup = False
+                self.urlretrieve(alternate_url, path)
+            if not check_md5sum(path, md5sum):
+                raise ChecksumError('MD5 checksum mismatch downloading %r'
+                                    % download_url)
         except IOError as e:
             raise zc.buildout.UserError("Error downloading %s: %s"
                                         % (download_url, e))
-        finally:
-            if cleanup and tmp_path:
-                remove(tmp_path)
-
-        return tmp_path, not path
 
     def filename(self, url):
         """Determine a file name from a URL according to the configuration.
@@ -306,6 +292,30 @@ class Download(object):
             return tmp_path, src.info()
 
 
+class Download(Download):
+
+    def _download(self, url, path, md5sum=None, alternate_url=None):
+        from .buildout import networkcache_client as nc
+        key = 'file-urlmd5:' + md5(url.encode()).hexdigest()
+        if nc and nc.tryDownload(key):
+            with nc:
+                entry = next(nc.select(key, {'url': url}), None)
+                if entry is None:
+                    err = 'no matching entry'
+                else:
+                    with closing(nc.download(entry['sha512'])) as src, \
+                         open(path, 'wb') as dst:
+                        shutil.copyfileobj(src, dst)
+                    if check_md5sum(path, md5sum):
+                        return
+                    err = 'MD5 checksum mismatch'
+                self.logger.info('Cannot download from network cache: %s', err)
+        super(Download, self)._download(url, path, md5sum, alternate_url)
+        if nc and nc.tryUpload(key):
+            with nc, open(path, 'rb') as f:
+                nc.upload(f, key, url=url)
+
+
 def check_md5sum(path, md5sum):
     """Tell whether the MD5 checksum of the file at path matches.
 
@@ -331,8 +341,6 @@ def remove(path):
     if os.path.exists(path):
         os.remove(path)
 
-from zc.buildout.networkcache import \
-  download_network_cached, upload_network_cached, HTTPError
 
 def locate_at(source, dest):
     if dest is None or realpath(dest) == realpath(source):

src/zc/buildout/networkcache.py

-##############################################################################
-#
-# Copyright (c) 2010 ViFiB SARL and Contributors.
-# All Rights Reserved.
-#
-# This software is subject to the provisions of the Zope Public License,
-# Version 2.1 (ZPL).  A copy of the ZPL should accompany this distribution.
-# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
-# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
-# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
-# FOR A PARTICULAR PURPOSE.
-#
-##############################################################################
-
-#XXX factor with slapos/grid/networkcache.py and use libnetworkcache helpers
-
-from __future__ import absolute_import, print_function, division
-import hashlib
-import posixpath
-import re
-import traceback
-try:
-    # Python 3
-    from urllib.error import HTTPError
-    from urllib.parse import urlparse
-    strify = bytes.decode
-except ImportError:
-    # Python 2
-    from urllib2 import HTTPError
-    from urlparse import urlparse
-    strify = str
-
-try:
-    try:
-        from slapos.libnetworkcache import NetworkcacheClient, UploadError, \
-           DirectoryNotFound
-        from slapos.networkcachehelper import \
-           helper_download_network_cached, \
-           helper_download_network_cached_to_file
-    except ImportError:
-        LIBNETWORKCACHE_ENABLED = False
-    else:
-        LIBNETWORKCACHE_ENABLED = True
-except:
-    print('There was problem while trying to import slapos.libnetworkcache:\n'
-          + traceback.format_exc())
-    LIBNETWORKCACHE_ENABLED = False
-    print('Networkcache forced to be disabled.')
-
-_md5_re = re.compile(r'md5=([a-f0-9]+)')
-
-
-def _get_md5_from_url(url):
-  match = _md5_re.search(url)
-  if match:
-    return match.group(1)
-
-def fallback_call(function):
-    """Decorator which disallow to have any problem while calling method"""
-    def wrapper(self, *args, **kwd):
-        """
-        Log the call, and the result of the call
-        """
-        try:
-            return function(self, *args, **kwd)
-        except: # indeed, *any* exception is swallowed
-            print('There was problem while calling method %r:\n%s' % (
-                function.__name__, traceback.format_exc()))
-            return False
-    wrapper.__doc__ = function.__doc__
-    return wrapper
-
-
-@fallback_call
-def get_directory_key(url):
-    """Returns directory hash based on url.
-
-    Basically check if the url belongs to pypi:
-      - if yes, the directory key will be pypi-buildout-urlmd5
-      - if not, the directory key will be slapos-buildout-urlmd5
-    # XXX why is that?
-    """
-    urlmd5 = hashlib.md5(url.encode()).hexdigest()
-    if 'pypi' in url:
-      return 'pypi-buildout-%s' % urlmd5
-    return 'slapos-buildout-%s' % urlmd5
-
-@fallback_call
-def get_index_directory_key(url, requirement):
-    """Returns directory hash based on egg requirement.
-    """
-    urlmd5 = hashlib.md5(url.encode()).hexdigest()
-    return 'pypi-index-%s-%s' % (urlmd5, requirement)
-
-
-@fallback_call
-def download_network_cached(dir_url, cache_url, path, url, logger,
-                            signature_certificate_list, md5sum=None):
-    """Downloads from a network cache provider
-
-    If something fail (providor be offline, or hash_string fail), we ignore
-    network cached files.
-
-    return True if download succeeded.
-    """
-    if not LIBNETWORKCACHE_ENABLED:
-        return False
-
-    if md5sum is None:
-        md5sum = _get_md5_from_url(url)
-
-    directory_key = get_directory_key(url)
-
-    logger.debug('Trying to download %s from network cache...', url)
-
-    if helper_download_network_cached_to_file(
-       dir_url=dir_url,
-       cache_url=cache_url,
-       signature_certificate_list=signature_certificate_list,
-       directory_key=directory_key,
-       path=path):
-        logger.info('Downloaded %s from network cache.', url)
-
-        if check_md5sum(path, md5sum):
-            return True
-        logger.info('MD5 checksum mismatch downloading %s', url)
-    else:
-        logger.info('Cannot download %s from network cache.', url)
-    return False
-
-@fallback_call
-def download_index_network_cached(dir_url, cache_url, url, requirement, logger,
-                            signature_certificate_list):
-    """
-    XXX description
-    Downloads pypi index from a network cache provider
-
-    If something fail (providor be offline, or hash_string fail), we ignore
-    network cached index.
-
-    return index if succeeded, False otherwise.
-    """
-    if not LIBNETWORKCACHE_ENABLED:
-        return False
-
-    directory_key = get_index_directory_key(url, requirement)
-
-    wanted_metadata_dict = {
-       'urlmd5': hashlib.md5(url.encode()).hexdigest(),
-       'requirement': requirement,
-    }
-    required_key_list = ['base']
-
-    result = helper_download_network_cached(dir_url, cache_url,
-       signature_certificate_list,
-       directory_key, wanted_metadata_dict, required_key_list)
-    if result:
-        file_descriptor, metadata = result
-        try:
-            content = strify(file_descriptor.read())
-            logger.info('Downloaded %s from network cache.', url)
-            return content, metadata['base']
-        except (IOError, DirectoryNotFound) as e:
-            if isinstance(e, HTTPError) and e.code == 404:
-              logger.debug('%s does not exist in network cache.', url)
-            else:
-              logger.debug('Failed to download from network cache %s: %s',
-                  url, e)
-    return False
-
-@fallback_call
-def upload_network_cached(dir_url, cache_url, external_url, path, logger,
-   signature_private_key_file, shacache_ca_file, shacache_cert_file,
-   shacache_key_file, shadir_ca_file, shadir_cert_file, shadir_key_file):
-    """Upload file to a network cache server"""
-    # XXX use helper and FACTOR code
-    if not LIBNETWORKCACHE_ENABLED:
-        return False
-
-    if not (dir_url and cache_url):
-        return False
-
-    logger.info('Uploading %s into network cache.', external_url)
-
-    file_name = get_filename_from_url(external_url)
-
-    directory_key = get_directory_key(external_url)
-    kw = dict(file_name=file_name,
-              urlmd5=hashlib.md5(external_url.encode()).hexdigest())
-
-    # convert '' into None in order to call nc nicely
-    if not signature_private_key_file:
-        signature_private_key_file = None
-    if not shacache_ca_file:
-        shacache_ca_file = None
-    if not shacache_cert_file:
-        shacache_cert_file = None
-    if not shacache_key_file:
-        shacache_key_file = None
-    if not shadir_ca_file:
-        shadir_ca_file = None
-    if not shadir_cert_file:
-        shadir_cert_file = None
-    if not shadir_key_file:
-        shadir_key_file = None
-    try:
-        nc = NetworkcacheClient(cache_url, dir_url,
-            signature_private_key_file=signature_private_key_file,
-            shacache_ca_file=shacache_ca_file,
-            shacache_cert_file=shacache_cert_file,
-            shacache_key_file=shacache_key_file,
-            shadir_ca_file=shadir_ca_file,
-            shadir_cert_file=shadir_cert_file,
-            shadir_key_file=shadir_key_file)
-    except TypeError:
-        logger.warning('Incompatible version of networkcache, not using it.')
-        return False
-
-    try:
-        with open(path, 'rb') as f:
-            return nc.upload(f, directory_key, **kw)
-    except (IOError, UploadError) as e:
-        logger.info('Fail to upload file. %s', e)
-        return False
-
-@fallback_call
-def upload_index_network_cached(dir_url, cache_url, external_url, base, requirement, content, logger,
-   signature_private_key_file, shacache_ca_file, shacache_cert_file,
-   shacache_key_file, shadir_ca_file, shadir_cert_file, shadir_key_file):
-    # XXX use helper and FACTOR code
-    """Upload content of a web page to a network cache server"""
-    if not LIBNETWORKCACHE_ENABLED:
-        return False
-
-    if not (dir_url and cache_url):
-        return False
-
-    logger.info('Uploading %s content into network cache.', external_url)
-
-    directory_key = get_index_directory_key(external_url, requirement)
-    kw = dict(file="file",
-              base=base,
-              urlmd5=hashlib.md5(external_url.encode()).hexdigest(),
-              requirement=requirement)
-
-    import tempfile
-    f = tempfile.TemporaryFile()
-    f.write(content.encode())
-
-    # convert '' into None in order to call nc nicely
-    if not signature_private_key_file:
-        signature_private_key_file = None
-    if not shacache_ca_file:
-        shacache_ca_file = None
-    if not shacache_cert_file:
-        shacache_cert_file = None
-    if not shacache_key_file:
-        shacache_key_file = None
-    if not shadir_ca_file:
-        shadir_ca_file = None
-    if not shadir_cert_file:
-        shadir_cert_file = None
-    if not shadir_key_file:
-        shadir_key_file = None
-    try:
-        nc = NetworkcacheClient(cache_url, dir_url,
-            signature_private_key_file=signature_private_key_file,
-            shacache_ca_file=shacache_ca_file,
-            shacache_cert_file=shacache_cert_file,
-            shacache_key_file=shacache_key_file,
-            shadir_ca_file=shadir_ca_file,
-            shadir_cert_file=shadir_cert_file,
-            shadir_key_file=shadir_key_file)
-    except TypeError:
-        logger.warning('Incompatible version of networkcache, not using it.')
-        return False
-
-    try:
-        return nc.upload_generic(f, directory_key, **kw)
-    except (IOError, UploadError) as e:
-        logger.info('Fail to upload file. %s', e)
-        return False
-
-    finally:
-      f.close()
-
-    return True
-
-
-@fallback_call
-def get_filename_from_url(url):
-    """Inspired how pip get filename from url.
-    """
-    parsed_url = urlparse(url)
-    if parsed_url.query and parsed_url.path.endswith('/'):
-      name = parsed_url.query.split('?', 1)[0]
-    elif parsed_url.path.endswith('/') and not parsed_url.query:
-      name = parsed_url.path.split('/')[-2]
-    else:
-      name = posixpath.basename(parsed_url.path)
-
-    name = name.split('#', 1)[0]
-    assert name, (
-           'URL %r produced no filename' % url)
-    return name
-
-
-from .download import check_md5sum