Revert "Merge branch 'bug_778748'"

This reverts commit 37b515f8, reversing
changes made to ffeeb901.

Please note that re-merging shall be done by reverting this revert and merging
back rest of bug_778748, as described in:
http://www.kernel.org/pub/software/scm/git/docs/howto/revert-a-faulty-merge.txt

buildout.cfg

@@ -27,3 +27,4 @@ defaults =
   '-t',
   '!(bootstrap|selectingpython|selecting-python)',
   ]
+

setup.py

@@ -82,7 +82,7 @@ setup(
     packages = ['zc', 'zc.buildout'],
     package_dir = {'': 'src'},
     namespace_packages = ['zc'],
-    install_requires = ['setuptools'],
+    install_requires = 'setuptools',
     include_package_data = True,
     entry_points = entry_points,
     extras_require = dict(test=['zope.testing']),

src/zc/buildout/buildout.py

@@ -362,8 +362,6 @@ class Buildout(UserDict.DictMixin):
                 newest=self.newest,
                 allow_hosts=self._allow_hosts,
                 prefer_final=not self.accept_buildout_test_releases,
-                sha_cache=options.get('sha-cache'),
-                sha_dir=options.get('sha-dir'),
                 )
 
         # Now copy buildout and setuptools eggs, and record destination eggs:
@@ -862,8 +860,6 @@ class Buildout(UserDict.DictMixin):
             path = [options['develop-eggs-directory']],
             allow_hosts = self._allow_hosts,
             prefer_final=not self.accept_buildout_test_releases,
-            sha_cache=options.get('sha-cache'),
-            sha_dir=options.get('sha-cache'),
             )
 
         upgraded = []
@@ -1091,9 +1087,7 @@ def _install_and_load(spec, group, entry, buildout):
                 working_set=pkg_resources.working_set,
                 newest=buildout.newest,
                 allow_hosts=buildout._allow_hosts,
-                prefer_final=not buildout.accept_buildout_test_releases,
-                sha_dir=buildout_options.get('sha-dir'),
-                sha_cache=buildout_options.get('sha-cache'))
+                prefer_final=not buildout.accept_buildout_test_releases)
 
         __doing__ = 'Loading %s recipe entry %s:%s.', group, spec, entry
         return pkg_resources.load_entry_point(

src/zc/buildout/download.py

@@ -17,6 +17,7 @@ try:
     from hashlib import md5
 except ImportError:
     from md5 import new as md5
+from zc.buildout.easy_install import realpath
 import logging
 import os
 import os.path
@@ -71,8 +72,6 @@ class Download(object):
         self.fallback = fallback
         self.hash_name = hash_name
         self.logger = logger or logging.getLogger('zc.buildout')
-        self.sha_dir = options.get('sha-dir')
-        self.sha_cache = options.get('sha-cache')
 
     @property
     def download_cache(self):
@@ -180,25 +179,20 @@ class Download(object):
         self.logger.info('Downloading %s' % url)
         urllib._urlopener = url_opener
         handle, tmp_path = tempfile.mkstemp(prefix='buildout-')
-
         try:
-            try:
-                if not download_network_cached(self.sha_dir, self.sha_cache,
-                                          tmp_path, url, self.logger, md5sum):
-
-                    # Download from original url
-                    tmp_path, headers = urllib.urlretrieve(url, tmp_path)
-                    if not check_md5sum(tmp_path, md5sum):
-                        raise ChecksumError(
-                            'MD5 checksum mismatch downloading %r' % url)
-                    # Upload the file to networkcached.
-                    upload_network_cached(self.sha_cache, self.sha_dir, url,
-                                                        tmp_path, self.logger)
-            finally:
-                os.close(handle)
-        except:
+            tmp_path, headers = urllib.urlretrieve(url, tmp_path)
+            if not check_md5sum(tmp_path, md5sum):
+                raise ChecksumError(
+                    'MD5 checksum mismatch downloading %r' % url)
+        except IOError, e:
+            os.remove(tmp_path)
+            raise zc.buildout.UserError("Error downloading extends for URL "
+                              "%s: %r" % (url, e[1:3]))
+        except Exception, e:
             os.remove(tmp_path)
             raise
+        finally:
+            os.close(handle)
 
         if path:
             shutil.move(tmp_path, path)
@@ -259,9 +253,6 @@ def remove(path):
     if os.path.exists(path):
         os.remove(path)
 
-from zc.buildout.networkcache import download_network_cached, \
-                                     upload_network_cached
-from zc.buildout.easy_install import realpath
 
 def locate_at(source, dest):
     if dest is None or realpath(dest) == realpath(source):

src/zc/buildout/easy_install.py

@@ -41,12 +41,6 @@ _oprp = getattr(os.path, 'realpath', lambda path: path)
 def realpath(path):
     return os.path.normcase(os.path.abspath(_oprp(path)))
 
-
-from zc.buildout.networkcache import get_filename_from_url, \
-                                     upload_network_cached, \
-                                     download_network_cached
-
-
 default_index_url = os.environ.get(
     'buildout-testing-index-url',
     'http://pypi.python.org/simple',
@@ -341,8 +335,6 @@ class Installer:
                  include_site_packages=None,
                  allowed_eggs_from_site_packages=None,
                  prefer_final=None,
-                 sha_dir=None,
-                 sha_cache=None
                  ):
         self._dest = dest
         self._allow_hosts = allow_hosts
@@ -411,9 +403,6 @@ class Installer:
         if versions is not None:
             self._versions = versions
 
-        self._sha_dir = sha_dir
-        self._sha_cache = sha_cache
-
     _allowed_eggs_from_site_packages_regex = None
     def allow_site_package_egg(self, name):
         if (not self._include_site_packages or
@@ -713,14 +702,7 @@ class Installer:
             ):
             return dist
 
-        filename = get_filename_from_url(dist.location)
-        new_location = os.path.join(tmp, filename)
-        if not download_network_cached(self._sha_dir, self._sha_cache,
-                                         new_location, dist.location, logger):
-            new_location = self._index.download(dist.location, tmp)
-            upload_network_cached(self._sha_cache, self._sha_dir,
-                                      dist.location, new_location, logger)
-
+        new_location = self._index.download(dist.location, tmp)
         if (download_cache
             and (realpath(new_location) == realpath(dist.location))
             and os.path.isfile(new_location)
@@ -1098,13 +1080,13 @@ def install(specs, dest,
             path=None, working_set=None, newest=True, versions=None,
             use_dependency_links=None, allow_hosts=('*',),
             include_site_packages=None, allowed_eggs_from_site_packages=None,
-            prefer_final=None, sha_dir=None, sha_cache=None):
+            prefer_final=None):
     installer = Installer(
         dest, links, index, executable, always_unzip, path, newest,
         versions, use_dependency_links, allow_hosts=allow_hosts,
         include_site_packages=include_site_packages,
         allowed_eggs_from_site_packages=allowed_eggs_from_site_packages,
-        prefer_final=prefer_final, sha_dir=sha_dir, sha_cache=sha_cache)
+        prefer_final=prefer_final)
     return installer.install(specs, working_set)
 
 

src/zc/buildout/networkcache.py

-##############################################################################
-#
-# Copyright (c) 2010 ViFiB SARL and Contributors.
-# All Rights Reserved.
-#
-# This software is subject to the provisions of the Zope Public License,
-# Version 2.1 (ZPL).  A copy of the ZPL should accompany this distribution.
-# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
-# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
-# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
-# FOR A PARTICULAR PURPOSE.
-#
-##############################################################################
-
-
-import hashlib
-import os
-import posixpath
-import re
-import urllib
-import urlparse
-
-
-_md5_re = re.compile(r'md5=([a-f0-9]+)')
-
-
-def _get_md5_from_url(url):
-  match = _md5_re.search(url)
-  if match:
-    return match.group(1)
-  return None
-
-
-def get_directory_key(url):
-    """It must retorn the directory key based on the URL.
-
-    Basically check if the url belongs to pypi:
-      - if yes, the directory key will be pypi-buildout-urlmd5
-      - if not, the directory key will be slapos-buildout-urlmd5
-    """
-    urlmd5 = hashlib.md5(url).hexdigest()
-    if 'pypi' in url:
-      return 'pypi-buildout-%s' % urlmd5
-    return 'slapos-buildout-%s' % urlmd5
-
-
-def download_network_cached(sha_dir, sha_cache, path, url, logger, md5sum=None):
-    """Download from a network cache provider
-
-    If something fail (providor be offline, or hash_string fail), we ignore
-    network cached files.
-
-    return True if download succeeded.
-    """
-    try:
-      from slapos.libnetworkcache import NetworkcacheClient, UploadError, \
-                                   DirectoryNotFound
-    except:
-      return False
-
-
-    if sha_cache in (None, '',):
-        # Not able to use network cache
-        return False
-    if md5sum is None:
-        md5sum = _get_md5_from_url(url)
-
-
-    directory_key = get_directory_key(url)
-    file_name = os.path.basename(url)
-
-    logger.info('Downloading %s from network cache.' % file_name)
-    try:
-        nc = NetworkcacheClient(shacache=sha_cache, shadir=sha_dir)
-        file_content = nc.select(directory_key)
-
-        f = open(path, 'w+b')
-        try:
-          f.seek(0)
-          f.write(file_content)
-        finally:
-          f.close()
-
-        if not check_md5sum(path, md5sum):
-            logger.info('MD5 checksum mismatch downloading %s' % file_name)
-            return False
-
-    except (IOError, DirectoryNotFound), e:
-        logger.info('Fail to download from network cache %s: %s' % \
-                                                       (file_name, str(e)))
-        return False
-    return True
-
-
-def upload_network_cached(sha_cache, sha_dir, external_url, path, logger):
-    """Upload file to a shacache server"""
-    try:
-      from slapos.libnetworkcache import NetworkcacheClient, UploadError, \
-                                   DirectoryNotFound
-    except:
-      return False
-
-    if sha_cache in [None, ''] or sha_dir in [None, '']:
-        logger.debug(
-             'Upload cache ignored, shacache or shadir was not provided!')
-        return False
-
-    file_name = external_url.split('/')[-1].split('#')[0].split('?')[0]
-    directory_key = get_directory_key(external_url)
-    kw = dict(file=file_name,
-              urlmd5=hashlib.md5(external_url).hexdigest())
-
-    f = open(path, 'r')
-    try:
-        nc = NetworkcacheClient(shacache=sha_cache,
-                                shadir=sha_dir)
-        data = nc.upload(f, directory_key, **kw)
-    except (IOError, UploadError), e:
-        logger.info('Fail to upload file. %s' % \
-                                                  (str(e)))
-        return False
-
-    finally:
-      f.close()
-
-    return True
-
-
-def get_filename_from_url(url):
-    """Inspired how pip get filename from url.
-    """
-    url = url.split('#', 1)[0]
-    url = url.split('?', 1)[0]
-    url = url.rstrip('/')
-    name = posixpath.basename(url)
-    assert name, (
-           'URL %r produced no filename' % url)
-    return name
-
-
-from download import check_md5sum

src/zc/buildout/networkcache.txt

-Using buildout with network-cache
-=================================
-
-NOTE: You need to install the networkcached into your system to run this
-      test, otherwise it will not be possible to get proper results.
-
-The buildout itself must be able to cache everything which is downloadble
-into an external and universal cache server.
-
-Such feature is not enabled by default, but you can enable defining the
-option called ```network-cache```.
-
-   >>> write(sample_buildout, 'buildout.cfg',
-   ... '''
-   ... [buildout]
-   ... network-cache = http://127.0.0.1:5001/
-   ... find-links = %(link_server)s
-   ... parts = eggs
-   ...
-   ... [eggs]
-   ... recipe = zc.recipe.egg:eggs
-   ... eggs = demo ==0.2
-   ... ''' % globals())
-
-   >>> print system(buildout)
-   Installing eggs.
-   Getting distribution for 'demo==0.2'.
-   The url is not cached yet: http://localhost/demo-0.2-py2.6.egg
-   Got demo 0.2.
-   Getting distribution for 'demoneeded'.
-   The url is not cached yet: http://localhost/demoneeded-1.2c1.zip
-   Got demoneeded 1.2c1.
-   ...
-
-   >>> from glob import glob
-   >>> from os.path import join
-   >>> for egg in glob(join(sample_buildout, 'eggs', 'demoneeded*.egg')):
-   ...     remove(sample_buildout, 'eggs', egg)
-
-   >>> print system(buildout)
-   Updating eggs.
-   Getting distribution for 'demoneeded'.
-   Downloading from network cache http://127.0.0.1:5001/105e4a9ba3f9c7f45b46cc8c47ac2d2ca6440ae564538090c6d390a8959dd03e
-   Got demoneeded 1.2c1.
-   ...

src/zc/buildout/tests.py

@@ -4053,7 +4053,7 @@ def test_suite():
             ),
 
         doctest.DocFileSuite(
-            'download.txt', 'extends-cache.txt', 'networkcache.txt',
+            'download.txt', 'extends-cache.txt',
             setUp=easy_install_SetUp,
             tearDown=zc.buildout.testing.buildoutTearDown,
             optionflags=doctest.NORMALIZE_WHITESPACE | doctest.ELLIPSIS,

zc.recipe.egg_/src/zc/recipe/egg/egg.py

@@ -88,11 +88,6 @@ class Eggs(object):
             kw = {}
             if 'unzip' in options:
                 kw['always_unzip'] = options.query_bool('unzip', None)
-            if 'sha-cache' in b_options:
-                kw['sha_cache'] = b_options.get('sha-cache')
-            if 'sha-dir' in b_options:
-                kw['sha_dir'] = b_options.get('sha-dir')
-
             ws = zc.buildout.easy_install.install(
                 distributions, options['eggs-directory'],
                 links=self.links,