Added basic low-level support for a download cache.

This led to a pretty major refactoring.


git-svn-id: http://svn.zope.org/repos/main/zc.buildout/trunk@73327 62d5b8a3-27da-0310-9561-8e5933582275

src/zc/buildout/easy_install.py

@@ -104,9 +104,11 @@ _easy_install_cmd = _safe_arg(
     'from setuptools.command.easy_install import main; main()'
     )
 
+
 class Installer:
 
     _versions = {}
+    _download_cache = None
 
     def __init__(self,
                  dest=None,
@@ -119,7 +121,10 @@ class Installer:
                  versions=None,
                  ):
         self._dest = dest
-        self._links = list(links)
+        self._links = links = list(links)
+        if self._download_cache and (self._download_cache not in links):
+            links.insert(0, self._download_cache)
+
         self._index_url = index
         self._executable = executable
         self._always_unzip = always_unzip
@@ -127,6 +132,8 @@ class Installer:
         if dest is not None and dest not in path:
             path.insert(0, dest)
         self._path = path
+        if self._dest is None:
+            newest = False
         self._newest = newest
         self._env = pkg_resources.Environment(path,
                                               python=_get_version(executable))
@@ -135,12 +142,12 @@ class Installer:
         if versions is not None:
             self._versions = versions
 
-    def _satisfied(self, req):
+    def _satisfied(self, req, source=None):
         dists = [dist for dist in self._env[req.project_name] if dist in req]
         if not dists:
             logger.debug('We have no distributions for %s that satisfies %s.',
                          req.project_name, req)
-            return None
+            return None, self._obtain(req, source)
 
         # Note that dists are sorted from best to worst, as promised by
         # env.__getitem__
@@ -148,13 +155,13 @@ class Installer:
         for dist in dists:
             if (dist.precedence == pkg_resources.DEVELOP_DIST):
                 logger.debug('We have a develop egg for %s', req)
-                return dist
+                return dist, None
 
         if not self._newest:
             # We don't need the newest, so we'll use the newest one we
             # find, which is the first returned by
             # Environment.__getitem__.
-            return dists[0]
+            return dists[0], None
 
         # Find an upper limit in the specs, if there is one:
         specs = [(pkg_resources.parse_version(v), op) for (op, v) in req.specs]
@@ -189,7 +196,7 @@ class Installer:
         if maxv is not None and best_we_have.version == maxv:
             logger.debug('We have the best distribution that satisfies\n%s',
                          req)
-            return best_we_have
+            return best_we_have, None
 
         # We have some installed distros.  There might, theoretically, be
         # newer ones.  Let's find out which ones are available and see if
@@ -198,7 +205,7 @@ class Installer:
 
         
         if self._dest is not None:
-            best_available = self._index.obtain(req)
+            best_available = self._obtain(req, source)
         else:
             best_available = None
 
@@ -208,7 +215,7 @@ class Installer:
             logger.debug(
                 'There are no distros available that meet %s. Using our best.',
                 req)
-            return best_we_have
+            return best_we_have, None
         else:
             # Let's find out if we already have the best available:
             if best_we_have.parsed_version >= best_available.parsed_version:
@@ -216,16 +223,27 @@ class Installer:
                 logger.debug(
                     'We have the best distribution that satisfies\n%s',
                     req)
-                return best_we_have
+                return best_we_have, None
 
-        return None
+        return None, best_available
 
-    def _call_easy_install(self, spec, ws, dest):
+    def _load_dist(self, dist):
+        dists = pkg_resources.Environment(
+            dist.location,
+            python=_get_version(self._executable),
+            )[dist.project_name]
+        assert len(dists) == 1
+        return dists[0]
+
+    def _call_easy_install(self, spec, ws, dest, dist):
 
-        path = self._get_dist(pkg_resources.Requirement.parse('setuptools'),
-                              ws, False).location
+        tmp = tempfile.mkdtemp(dir=dest)
+        try:
+            path = self._get_dist(
+                pkg_resources.Requirement.parse('setuptools'), ws, False,
+                )[0].location
 
-        args = ('-c', _easy_install_cmd, '-mUNxd', _safe_arg(dest))
+            args = ('-c', _easy_install_cmd, '-mUNxd', _safe_arg(tmp))
             if self._always_unzip:
                 args += ('-Z', )
             level = logger.getEffectiveLevel()
@@ -242,10 +260,105 @@ class Installer:
 
             args += (dict(os.environ, PYTHONPATH=path), )
             sys.stdout.flush() # We want any pending output first
-        exit_code = os.spawnle(os.P_WAIT, self._executable, self._executable,
+            exit_code = os.spawnle(
+                os.P_WAIT, self._executable, self._executable,
                 *args)
-        assert exit_code == 0
 
+            dists = []
+            env = pkg_resources.Environment(
+                [tmp],
+                python=_get_version(self._executable),
+                )
+            for project in env:
+                dists.extend(env[project])
+                
+            if exit_code:
+                logger.error(
+                    "An error occured when trying to install %s."
+                    "Look above this message for any errors that"
+                    "were output by easy_install.",
+                    dist)
+
+            if not dists:
+                raise zc.buildout.UserError("Couldn't install: %s" % dist)
+
+            if len(dists) > 1:
+                logger.warn("Installing %s\n"
+                            "caused multiple distributions to be installed:\n"
+                            "%s\n",
+                            dist, '\n'.join(map(str, dists)))
+            else:
+                d = dists[0]
+                if d.project_name != dist.project_name:
+                    logger.warn("Installing %s\n"
+                                "Caused installation of a distribution:\n"
+                                "%s\n"
+                                "with a different project name.",
+                                dist, d)
+                if d.version != dist.version:
+                    logger.warn("Installing %s\n"
+                                "Caused installation of a distribution:\n"
+                                "%s\n"
+                                "with a different version.",
+                                dist, d)
+
+            result = []
+            for d in dists:
+                newloc = os.path.join(dest, os.path.basename(d.location))
+                if os.path.exists(newloc):
+                    if os.path.is_dir(newloc):
+                        shutil.rmtree(newloc)
+                    else:
+                        os.remove(newloc)
+                os.rename(d.location, newloc)
+
+                [d] = pkg_resources.Environment(
+                    [newloc],
+                    python=_get_version(self._executable),
+                    )[d.project_name]
+                    
+                result.append(d)
+
+            return result
+
+        finally:
+            shutil.rmtree(tmp)
+            
+    def _obtain(self, requirement, source=None):
+        index = self._index
+        if index.obtain(requirement) is None:
+            return None
+        
+        best = []
+        bestv = ()
+        for dist in index[requirement.project_name]:
+            if dist not in requirement:
+                continue
+            if source and dist.precedence != pkg_resources.SOURCE_DIST:
+                continue
+            distv = dist.parsed_version
+            if distv > bestv:
+                best = [dist]
+                bestv = distv
+            elif distv == bestv:
+                best.append(dist)
+
+        if not best:
+            return None
+
+        if len(best) == 1:
+            return best[0]
+        
+        if self._download_cache:
+            for dist in best:
+                if os.path.dirname(dist.location) == self._download_cache:
+                    return dist
+
+        best.sort()
+        return best[-1]
+
+    def _fetch(self, dist, tmp):
+        return dist.clone(location=self._index.download(dist.location, tmp))
 
     def _get_dist(self, requirement, ws, always_unzip):
 
@@ -253,40 +366,46 @@ class Installer:
 
         # Maybe an existing dist is already the best dist that satisfies the
         # requirement
-        dist = self._satisfied(requirement)
+        dist, avail = self._satisfied(requirement)
 
         if dist is None:
             if self._dest is not None:
                 logger.info("Getting new distribution for %s", requirement)
 
-                # Retrieve the dist:grokonepage
-                index = self._index
-                dist = index.obtain(requirement)
-                if dist is None:
+            # Retrieve the dist:
+            if avail is None:
                 raise zc.buildout.UserError(
                     "Couldn't find a distribution for %s."
                     % requirement)
 
-                fname = dist.location
-                if url_match(fname):
-                    fname = urlparse.urlparse(fname)[2]
+            # We may overwrite distributions, so clear importer
+            # cache.
+            sys.path_importer_cache.clear()
 
-                if fname.endswith('.egg'):
-                    # It's already an egg, just fetch it into the dest
-                    tmp = tempfile.mkdtemp('get_dist')
+            tmp = self._download_cache
             try:
-                        dist = index.fetch_distribution(requirement, tmp)
+                if tmp:
+                    if os.path.dirname(avail.location) == tmp:
+                        dist = avail
+                    else:
+                        dist = self._fetch(avail, tmp)
+                else:
+                    tmp = tempfile.mkdtemp('get_dist')
+                    dist = self._fetch(avail, tmp)
+
                 if dist is None:
                     raise zc.buildout.UserError(
-                                "Couln't download a distribution for %s."
-                                % requirement)
+                        "Couln't download distribution %s." % avail)
+
+                if dist.precedence == pkg_resources.EGG_DIST:
+                    # It's already an egg, just fetch it into the dest
 
                     newloc = os.path.join(
                         self._dest, os.path.basename(dist.location))
 
                     if os.path.isdir(dist.location):
                         # we got a directory. It must have been
-                            # obtained locally.  Jut copy it.
+                        # obtained locally.  Just copy it.
                         shutil.copytree(dist.location, newloc)
                     else:
 
@@ -298,7 +417,8 @@ class Installer:
                                 )
                             should_unzip = (
                                 metadata.has_metadata('not-zip-safe')
-                                    or not metadata.has_metadata('zip-safe')
+                                or
+                                not metadata.has_metadata('zip-safe')
                                 )
 
                         if should_unzip:
@@ -307,57 +427,53 @@ class Installer:
                         else:
                             shutil.copyfile(dist.location, newloc)
 
-                    finally:
-                        shutil.rmtree(tmp)
-
+                    # Getting the dist from the environment causes the
+                    # distribution meta data to be read.  Cloning isn't
+                    # good enough.
+                    dists = pkg_resources.Environment(
+                        [newloc],
+                        python=_get_version(self._executable),
+                        )[dist.project_name]
                 else:
-                    # It's some other kind of dist.  We'll download it to
-                    # a temporary directory and let easy_install have it's
-                    # way with it:
-                    tmp = tempfile.mkdtemp('get_dist')
-                    try:
-                        dist = index.fetch_distribution(requirement, tmp)
+                    # It's some other kind of dist.  We'll let easy_install
+                    # deal with it:
+                    dists = self._call_easy_install(
+                        dist.location, ws, self._dest, dist)
 
-                        # May need a new one.  Call easy_install
-                        self._call_easy_install(dist.location, ws, self._dest)
             finally:
+                if tmp != self._download_cache:
                     shutil.rmtree(tmp)
 
-
-                # Because we have added a new egg, we need to rescan
-                # the destination directory.
-
-                # We may overwrite distributions, so clear importer
-                # cache.
-                sys.path_importer_cache.clear()
-
             self._env.scan([self._dest])
             dist = self._env.best_match(requirement, ws)
             logger.info("Got %s", dist)            
-            else:
-                dist = self._env.best_match(requirement, ws)
 
-        if dist is None:
-            raise ValueError("Couldn't find", requirement)
+        else:
+            dists = [dist]
 
         # XXX Need test for this
+        for dist in dists:
             if dist.has_metadata('dependency_links.txt'):
                 for link in dist.get_metadata_lines('dependency_links.txt'):
                     link = link.strip()
                     if link not in self._links:
+                        logger.debug('Adding find link %r from %s', link, dist)
                         self._links.append(link)
                         self._index = _get_index(self._executable,
                                                  self._index_url, self._links)
 
+        for dist in dists:
             # Check whether we picked a version and, if we did, report it:
             if not (
                 dist.precedence == pkg_resources.DEVELOP_DIST
                 or
-            (len(requirement.specs) == 1 and requirement.specs[0][0] == '==')
+                (len(requirement.specs) == 1
+                 and
+                 requirement.specs[0][0] == '==')
                 ):
                 picked.debug('%s = %s', dist.project_name, dist.version)
 
-        return dist
+        return dists
 
     def _maybe_add_setuptools(self, ws, dist):
         if dist.has_metadata('namespace_packages.txt'):
@@ -376,7 +492,7 @@ class Installer:
                     pkg_resources.Requirement.parse('setuptools')
                     )
                 if ws.find(requirement) is None:
-                    dist = self._get_dist(requirement, ws, False)
+                    for dist in self._get_dist(requirement, ws, False):
                         ws.add(dist)
 
 
@@ -413,7 +529,7 @@ class Installer:
             ws = working_set
 
         for requirement in requirements:
-            dist = self._get_dist(requirement, ws, self._always_unzip)
+            for dist in self._get_dist(requirement, ws, self._always_unzip):
                 ws.add(dist)
                 self._maybe_add_setuptools(ws, dist)
 
@@ -433,7 +549,8 @@ class Installer:
                 requirement = self._constrain(requirement)
                 if dest:
                     logger.debug('Getting required %s', requirement)
-                dist = self._get_dist(requirement, ws, self._always_unzip)
+                for dist in self._get_dist(requirement, ws, self._always_unzip
+                                           ):
                     ws.add(dist)
                     self._maybe_add_setuptools(ws, dist)
             else:
@@ -442,33 +559,41 @@ class Installer:
         return ws
 
     def build(self, spec, build_ext):
-        logger.debug('Building %r', spec)
 
         requirement = self._constrain(pkg_resources.Requirement.parse(spec))
 
-        dist = self._satisfied(requirement)
+        dist, avail = self._satisfied(requirement, 1)
         if dist is not None:
-            return dist.location
-
-        undo = []
-        try:
-            tmp = tempfile.mkdtemp('build')
-            undo.append(lambda : shutil.rmtree(tmp)) 
-            tmp2 = tempfile.mkdtemp('build')
-            undo.append(lambda : shutil.rmtree(tmp2))
+            return [dist.location]
 
-            dist = self._index.fetch_distribution(
-                requirement, tmp2, False, True)
-            if dist is None:
+        # Retrieve the dist:
+        if avail is None:
             raise zc.buildout.UserError(
                 "Couldn't find a source distribution for %s."
                 % requirement)
-            setuptools.archive_util.unpack_archive(dist.location, tmp)
 
-            if os.path.exists(os.path.join(tmp, 'setup.py')):
-                base = tmp
+        logger.debug('Building %r', spec)
+
+        tmp = self._download_cache
+        try:
+            if tmp:
+                if os.path.dirname(avail.location) == tmp:
+                    dist = avail
+                else:
+                    dist = self._fetch(avail, tmp)
             else:
-                setups = glob.glob(os.path.join(tmp, '*', 'setup.py'))
+                tmp = tempfile.mkdtemp('get_dist')
+                dist = self._fetch(avail, tmp)
+
+            build_tmp = tempfile.mkdtemp('build')
+            try:
+                setuptools.archive_util.unpack_archive(dist.location,
+                                                       build_tmp)
+                if os.path.exists(os.path.join(build_tmp, 'setup.py')):
+                    base = build_tmp
+                else:
+                    setups = glob.glob(
+                        os.path.join(build_tmp, '*', 'setup.py'))
                     if not setups:
                         raise distutils.errors.DistutilsError(
                             "Couldn't find a setup script in %s"
@@ -481,7 +606,6 @@ class Installer:
                             )
                     base = os.path.dirname(setups[0])
             
-
                 setup_cfg = os.path.join(base, 'setup.cfg')
                 if not os.path.exists(setup_cfg):
                     f = open(setup_cfg, 'w')
@@ -489,16 +613,18 @@ class Installer:
                 setuptools.command.setopt.edit_config(
                     setup_cfg, dict(build_ext=build_ext))
 
-            tmp3 = tempfile.mkdtemp('build', dir=self._dest)
-            undo.append(lambda : shutil.rmtree(tmp3)) 
-
-            self._call_easy_install(base, pkg_resources.WorkingSet(), tmp3)
+                dists = self._call_easy_install(
+                    base, pkg_resources.WorkingSet(),
+                    self._dest, dist)
 
-            return _copyeggs(tmp3, self._dest, '.egg', undo)
+                return [dist.location for dist in dists]
+            finally:
+                shutil.rmtree(build_tmp)
 
         finally:
-            undo.reverse()
-            [f() for f in undo]
+            if tmp != self._download_cache:
+                shutil.rmtree(tmp)
+
 
 def default_versions(versions=None):
     old = Installer._versions
@@ -506,6 +632,12 @@ def default_versions(versions=None):
         Installer._versions = versions
     return old
 
+def download_cache(path=-1):
+    old = Installer._download_cache
+    if path != -1:
+        Installer._download_cache = path
+    return old
+
 def install(specs, dest,
             links=(), index=None,
             executable=sys.executable, always_unzip=False,

src/zc/buildout/easy_install.txt

@@ -645,7 +645,7 @@ extension, extdemo.c::
   #endif
   }
 
-The extension depends on a system-dependnt include file, extdemo.h,
+The extension depends on a system-dependent include file, extdemo.h,
 that defines a constant, EXTDEMO, that is exposed by the extension.
 
 We'll add an include directory to our sample buildout and add the
@@ -664,7 +664,7 @@ distribution:
     ...   'extdemo', dest, 
     ...   {'include-dirs': os.path.join(sample_buildout, 'include')},
     ...   links=[link_server], index=link_server+'index/')
-    '/sample-install/extdemo-1.4-py2.4-unix-i686.egg'
+    ['/sample-install/extdemo-1.4-py2.4-unix-i686.egg']
 
 The function returns the list of eggs 
 
@@ -706,7 +706,7 @@ If we run build with newest set to False, we won't get an update:
     ...   {'include-dirs': os.path.join(sample_buildout, 'include')},
     ...   links=[link_server], index=link_server+'index/',
     ...   newest=False)
-    '/sample-install/extdemo-1.4-py2.4-linux-i686.egg'
+    ['/sample-install/extdemo-1.4-py2.4-linux-i686.egg']
 
     >>> ls(dest)
     -  demo-0.2-py2.4.egg
@@ -722,7 +722,7 @@ get an updated egg:
     ...   'extdemo', dest, 
     ...   {'include-dirs': os.path.join(sample_buildout, 'include')},
     ...   links=[link_server], index=link_server+'index/')
-    '/sample-install/extdemo-1.5-py2.4-unix-i686.egg'
+    ['/sample-install/extdemo-1.5-py2.4-unix-i686.egg']
 
     >>> ls(dest)
     -  demo-0.2-py2.4.egg
@@ -746,7 +746,7 @@ first:
     ...   {'include-dirs': os.path.join(sample_buildout, 'include')},
     ...   links=[link_server], index=link_server+'index/',
     ...   versions=dict(extdemo='1.4'))
-    '/sample-install/extdemo-1.4-py2.4-unix-i686.egg'
+    ['/sample-install/extdemo-1.4-py2.4-unix-i686.egg']
 
     >>> ls(dest)
     d  extdemo-1.4-py2.4-unix-i686.egg
@@ -814,3 +814,112 @@ And that the source directory contains the compiled extension:
     d  extdemo.egg-info
     -  extdemo.so
     -  setup.py
+
+Download cache
+--------------
+
+Normally, when distributions are installed, if any processing is
+needed, they are downloaded from the internet to a temporary directory
+and then installed from there.  A download cache can be used to avoid
+the download step.  This can be useful to reduce network access and to
+create source distributions of an entire buildout.
+
+A download cache is specified by calling the download_cache
+function.  The function always returns the previous setting. If no
+argument is passed, then the setting is unchanged.  Is an argument is
+passed, the download cache is set to the given path, which must point
+to an existing directory.  Passing None clears the cache setting.
+
+To see this work, we'll create a directory and set it as the cache
+directory:
+
+    >>> cache = tmpdir('cache')
+    >>> zc.buildout.easy_install.download_cache(cache)
+
+We'll recreate out destination directory:
+
+    >>> remove(dest)
+    >>> dest = tmpdir('sample-install')
+
+We'd like to see what is being fetched from the server, so we'll
+enable server logging:
+   
+    >>> get(link_server+'enable_server_logging')
+    GET 200 /enable_server_logging
+    ''
+
+Now, if we install demo, and extdemo:
+
+    >>> ws = zc.buildout.easy_install.install(
+    ...     ['demo==0.2'], dest,
+    ...     links=[link_server], index=link_server+'index/', 
+    ...     always_unzip=True)
+    GET 200 /
+    GET 404 /index/demo/
+    GET 200 /index/
+    GET 200 /demo-0.2-py2.4.egg
+    GET 404 /index/demoneeded/
+    GET 200 /demoneeded-1.1.zip
+    GET 404 /index/setuptools/
+
+    >>> zc.buildout.easy_install.build(
+    ...   'extdemo', dest, 
+    ...   {'include-dirs': os.path.join(sample_buildout, 'include')},
+    ...   links=[link_server], index=link_server+'index/')
+    GET 404 /index/extdemo/
+    GET 200 /extdemo-1.5.zip
+    ['/sample-install/extdemo-1.5-py2.4-linux-i686.egg']
+
+Not only will we get eggs in our destination directory:
+
+    >>> ls(dest)
+    d  demo-0.2-py2.4.egg
+    d  demoneeded-1.1-py2.4.egg
+    d  extdemo-1.5-py2.4-linux-i686.egg
+
+But we'll get distributions in the cache directory:
+
+    >>> ls(cache)
+    -  demo-0.2-py2.4.egg
+    -  demoneeded-1.1.zip
+    -  extdemo-1.5.zip
+
+The cache directory contains uninstalled distributions, such as zipped
+eggs or source distributions.  
+
+Let's recreate our destination directory and clear the index cache:
+
+    >>> remove(dest)
+    >>> dest = tmpdir('sample-install')
+    >>> zc.buildout.easy_install.clear_index_cache()
+
+Now when we install the distributions:
+
+    >>> ws = zc.buildout.easy_install.install(
+    ...     ['demo==0.2'], dest,
+    ...     links=[link_server], index=link_server+'index/', 
+    ...     always_unzip=True)
+    GET 200 /
+    GET 404 /index/demo/
+    GET 200 /index/
+    GET 404 /index/demoneeded/
+    GET 404 /index/setuptools/
+
+    >>> zc.buildout.easy_install.build(
+    ...   'extdemo', dest, 
+    ...   {'include-dirs': os.path.join(sample_buildout, 'include')},
+    ...   links=[link_server], index=link_server+'index/')
+    GET 404 /index/extdemo/
+    ['/sample-install/extdemo-1.5-py2.4-linux-i686.egg']
+
+    >>> ls(dest)
+    d  demo-0.2-py2.4.egg
+    d  demoneeded-1.1-py2.4.egg
+    d  extdemo-1.5-py2.4-linux-i686.egg
+
+Note that we didn't download the distributions from the link server.
+
+.. Disable the download cache:
+
+    >>> zc.buildout.easy_install.download_cache(None)
+    '/cache'