Added newest keyword parameter to the install and build functions to

allow for getting less than the newest but still getting what's
necessary.


git-svn-id: http://svn.zope.org/repos/main/zc.buildout/trunk@72383 62d5b8a3-27da-0310-9561-8e5933582275

src/zc/buildout/easy_install.py

-##############################################################################
+#############################################################################
 #
 # Copyright (c) 2005 Zope Corporation and Contributors.
 # All Rights Reserved.
@@ -85,8 +85,47 @@ def _get_index(executable, index_url, find_links):
     _indexes[key] = index
     return index
 
-def _satisfied(req, env, dest, executable, index, links):
-    dists = [dist for dist in env[req.project_name] if dist in req]
+clear_index_cache = _indexes.clear
+
+if sys.platform == 'win32':
+    # work around spawn lamosity on windows
+    # XXX need safe quoting (see the subproces.list2cmdline) and test
+    def _safe_arg(arg):
+        return '"%s"' % arg
+else:
+    _safe_arg = str
+
+_easy_install_cmd = _safe_arg(
+    'from setuptools.command.easy_install import main; main()'
+    )
+
+class Installer:
+
+    def __init__(self,
+                 dest=None,
+                 links=(),
+                 index=None,
+                 executable=sys.executable,
+                 always_unzip=False,
+                 path=None,
+                 newest=True,
+                 ):
+        self._dest = dest
+        self._links = list(links)
+        self._index_url = index
+        self._executable = executable
+        self._always_unzip = always_unzip
+        path = (path and path[:] or []) + buildout_and_setuptools_path
+        if dest is not None and dest not in path:
+            path.insert(0, dest)
+        self._path = path
+        self._newest = newest
+        self._env = pkg_resources.Environment(path,
+                                              python=_get_version(executable))
+        self._index = _get_index(executable, index, links)
+
+    def _satisfied(self, req):
+        dists = [dist for dist in self._env[req.project_name] if dist in req]
         if not dists:
             logger.debug('We have no distributions for %s', req.project_name)
             return None
@@ -99,6 +138,12 @@ def _satisfied(req, env, dest, executable, index, links):
                 logger.debug('We have a develop egg for %s', req)
                 return dist
 
+        if not self._newest:
+            # We don't need the newest, so we'll use the newest one we
+            # find, which is the first returned by
+            # Environment.__getitem__.
+            return dists[0]
+
         # Find an upprt limit in the specs, if there is one:
         specs = [(pkg_resources.parse_version(v), op) for (op, v) in req.specs]
         specs.sort()
@@ -139,8 +184,9 @@ def _satisfied(req, env, dest, executable, index, links):
         # any are newer.  We only do this if we're willing to install
         # something, which is only true if dest is not None:
 
-    if dest is not None:
-        best_available = _get_index(executable, index, links).obtain(req)
+        
+        if self._dest is not None:
+            best_available = self._index.obtain(req)
         else:
             best_available = None
 
@@ -148,38 +194,27 @@ def _satisfied(req, env, dest, executable, index, links):
             # That's a bit odd.  There aren't any distros available.
             # We should use the best one we have that meets the requirement.
             logger.debug(
-            'There are no distros available that meet %s. Using our best.', req)
+                'There are no distros available that meet %s. Using our best.',
+                req)
             return best_we_have
         else:
             # Let's find out if we already have the best available:
             if best_we_have.parsed_version >= best_available.parsed_version:
                 # Yup. Use it.
-            logger.debug('We have the best distribution that satisfies\n%s', req)
+                logger.debug(
+                    'We have the best distribution that satisfies\n%s',
+                    req)
                 return best_we_have
 
         return None
 
+    def _call_easy_install(self, spec, ws, dest):
 
-if sys.platform == 'win32':
-    # work around spawn lamosity on windows
-    # XXX need safe quoting (see the subproces.list2cmdline) and test
-    def _safe_arg(arg):
-        return '"%s"' % arg
-else:
-    _safe_arg = str
-
-_easy_install_cmd = _safe_arg(
-    'from setuptools.command.easy_install import main; main()'
-    )
-
-def _call_easy_install(spec, env, ws, dest, links, index,
-                       executable, always_unzip):
-
-    path = _get_dist(pkg_resources.Requirement.parse('setuptools'),
-                     env, ws, dest, links, index, executable, False).location
+        path = self._get_dist(pkg_resources.Requirement.parse('setuptools'),
+                              ws, False).location
 
         args = ('-c', _easy_install_cmd, '-mUNxd', _safe_arg(dest))
-    if always_unzip:
+        if self._always_unzip:
             args += ('-Z', )
         level = logger.getEffectiveLevel()
         if level > logging.DEBUG:
@@ -195,23 +230,23 @@ def _call_easy_install(spec, env, ws, dest, links, index,
 
         args += (dict(os.environ, PYTHONPATH=path), )
         sys.stdout.flush() # We want any pending output first
-    exit_code = os.spawnle(os.P_WAIT, executable, executable, *args)
+        exit_code = os.spawnle(os.P_WAIT, self._executable, self._executable,
+                               *args)
         assert exit_code == 0
 
 
-def _get_dist(requirement, env, ws,
-              dest, links, index_url, executable, always_unzip):
+    def _get_dist(self, requirement, ws, always_unzip):
 
         # Maybe an existing dist is already the best dist that satisfies the
         # requirement
-    dist = _satisfied(requirement, env, dest, executable, index_url, links)
+        dist = self._satisfied(requirement)
 
         if dist is None:
-        if dest is not None:
+            if self._dest is not None:
                 logger.info("Getting new distribution for %s", requirement)
 
                 # Retrieve the dist:
-            index = _get_index(executable, index_url, links)
+                index = self._index
                 dist = index.obtain(requirement)
                 if dist is None:
                     raise zc.buildout.UserError(
@@ -233,7 +268,7 @@ def _get_dist(requirement, env, ws,
                                 % requirement)
 
                         newloc = os.path.join(
-                        dest, os.path.basename(dist.location))
+                            self._dest, os.path.basename(dist.location))
 
                         if os.path.isdir(dist.location):
                             # we got a directory. It must have been
@@ -241,7 +276,7 @@ def _get_dist(requirement, env, ws,
                             shutil.copytree(dist.location, newloc)
                         else:
 
-                        if always_unzip:
+                            if self._always_unzip:
                                 should_unzip = True
                             else:
                                 metadata = pkg_resources.EggMetadata(
@@ -270,9 +305,7 @@ def _get_dist(requirement, env, ws,
                         dist = index.fetch_distribution(requirement, tmp)
 
                         # May need a new one.  Call easy_install
-                    _call_easy_install(
-                        dist.location, env, ws, dest, links, index_url,
-                        executable, always_unzip)
+                        self._call_easy_install(dist.location, ws, self._dest)
                     finally:
                         shutil.rmtree(tmp)
 
@@ -284,11 +317,11 @@ def _get_dist(requirement, env, ws,
                 # cache.
                 sys.path_importer_cache.clear()
 
-            env.scan([dest])
-            dist = env.best_match(requirement, ws)
+                self._env.scan([self._dest])
+                dist = self._env.best_match(requirement, ws)
                 logger.info("Got %s", dist)            
             else:
-            dist = env.best_match(requirement, ws)
+                dist = self._env.best_match(requirement, ws)
 
         if dist is None:
             raise ValueError("Couldn't find", requirement)
@@ -297,12 +330,14 @@ def _get_dist(requirement, env, ws,
         if dist.has_metadata('dependency_links.txt'):
             for link in dist.get_metadata_lines('dependency_links.txt'):
                 link = link.strip()
-            if link not in links:
-                links.append(link)
+                if link not in self._links:
+                    self._links.append(link)
+                    self._index = _get_index(self._executable,
+                                             self._index_url, self._links)
 
         return dist
 
-def _maybe_add_setuptools(ws, dist, env, dest, links, index, executable):
+    def _maybe_add_setuptools(self, ws, dist):
         if dist.has_metadata('namespace_packages.txt'):
             for r in dist.requires():
                 if r.project_name == 'setuptools':
@@ -317,33 +352,21 @@ def _maybe_add_setuptools(ws, dist, env, dest, links, index, executable):
                         dist)
                 requirement = pkg_resources.Requirement.parse('setuptools')
                 if ws.find(requirement) is None:
-                dist = _get_dist(requirement, env, ws,
-                                 dest, links, index, executable,
-                                 False)
+                    dist = self._get_dist(requirement, ws, False)
                     ws.add(dist)
 
 
-def install(specs, dest,
-            links=(), index=None,
-            executable=sys.executable, always_unzip=False,
-            path=None, working_set=None):
+    def install(self, specs, working_set=None):
 
         logger.debug('Installing %r', specs)
 
-    path = path and path[:] or []
+        path = self._path
+        dest = self._dest
         if dest is not None and dest not in path:
             path.insert(0, dest)
 
-    path += buildout_and_setuptools_path
-
-    links = list(links) # make copy, because we may need to mutate
-    
-
-    # For each spec, see if it is already installed.  We create a working
-    # set to keep track of what we've collected and to make sue than the
-    # distributions assembled are consistent.
-    env = pkg_resources.Environment(path, python=_get_version(executable))
-    requirements = [pkg_resources.Requirement.parse(spec) for spec in specs]
+        requirements = [pkg_resources.Requirement.parse(spec)
+                        for spec in specs]
 
         if working_set is None:
             ws = pkg_resources.WorkingSet([])
@@ -351,11 +374,9 @@ def install(specs, dest,
             ws = working_set
 
         for requirement in requirements:
-        dist = _get_dist(requirement, env, ws,
-                         dest, links, index, executable, always_unzip)
+            dist = self._get_dist(requirement, ws, self._always_unzip)
             ws.add(dist)
-        _maybe_add_setuptools(ws, dist,
-                              env, dest, links, index, executable)
+            self._maybe_add_setuptools(ws, dist)
 
         # OK, we have the requested distributions and they're in the working
         # set, but they may have unmet requirements.  We'll simply keep
@@ -372,42 +393,22 @@ def install(specs, dest,
                 [requirement] = err
                 if dest:
                     logger.debug('Getting required %s', requirement)
-            dist = _get_dist(requirement, env, ws,
-                             dest, links, index, executable, always_unzip)
+                dist = self._get_dist(requirement, ws, self._always_unzip)
                 ws.add(dist)
-            _maybe_add_setuptools(ws, dist,
-                                  env, dest, links, index, executable)
+                self._maybe_add_setuptools(ws, dist)
             else:
                 break
 
         return ws
 
-def build(spec, dest, build_ext,
-          links=(), index=None,
-          executable=sys.executable,
-          path=None):
-
-    index_url = index
-
+    def build(self, spec, build_ext):
         logger.debug('Building %r', spec)
 
-    path = path and path[:] or []
-    if dest is not None:
-        path.insert(0, dest)
-
-    path += buildout_and_setuptools_path
-
-    links = list(links) # make copy, because we may need to mutate
-    
-    # For each spec, see if it is already installed.  We create a working
-    # set to keep track of what we've collected and to make sue than the
-    # distributions assembled are consistent.
-    env = pkg_resources.Environment(path, python=_get_version(executable))
         requirement = pkg_resources.Requirement.parse(spec)
 
-    dist = _satisfied(requirement, env, dest, executable, index_url, links)
+        dist = self._satisfied(requirement)
         if dist is not None:
-        return [dist.location]
+            return dist.location
 
         undo = []
         try:
@@ -416,8 +417,8 @@ def build(spec, dest, build_ext,
             tmp2 = tempfile.mkdtemp('build')
             undo.append(lambda : shutil.rmtree(tmp2))
 
-        index = _get_index(executable, index_url, links)
-        dist = index.fetch_distribution(requirement, tmp2, False, True)
+            dist = self._index.fetch_distribution(
+                requirement, tmp2, False, True)
             if dist is None:
                 raise zc.buildout.UserError(
                     "Couldn't find a source distribution for %s."
@@ -448,19 +449,36 @@ def build(spec, dest, build_ext,
             setuptools.command.setopt.edit_config(
                 setup_cfg, dict(build_ext=build_ext))
 
-        tmp3 = tempfile.mkdtemp('build', dir=dest)
+            tmp3 = tempfile.mkdtemp('build', dir=self._dest)
             undo.append(lambda : shutil.rmtree(tmp3)) 
 
-        _call_easy_install(base, env, pkg_resources.WorkingSet(),
-                           tmp3, links, index_url, executable, True)
+            self._call_easy_install(base, pkg_resources.WorkingSet(), tmp3)
 
-        return _copyeggs(tmp3, dest, '.egg', undo)
+            return _copyeggs(tmp3, self._dest, '.egg', undo)
 
         finally:
             undo.reverse()
             [f() for f in undo]
 
 
+def install(specs, dest,
+            links=(), index=None,
+            executable=sys.executable, always_unzip=False,
+            path=None, working_set=None, newest=True):
+    installer = Installer(dest, links, index, executable, always_unzip, path,
+                          newest)
+    return installer.install(specs, working_set)
+
+
+def build(spec, dest, build_ext,
+          links=(), index=None,
+          executable=sys.executable,
+          path=None, newest=True):
+    installer = Installer(dest, links, index, executable, True, path, newest)
+    return installer.build(spec, build_ext)
+
+        
+
 def _rm(*paths):
     for path in paths:
         if os.path.isdir(path):
@@ -478,7 +496,7 @@ def _copyeggs(src, dest, suffix, undo):
             os.rename(os.path.join(src, name), new)
             result.append(new)
         
-    assert len(result) == 1
+    assert len(result) == 1, str(result)
     undo.pop()
     
     return result[0]

src/zc/buildout/easy_install.txt

@@ -21,7 +21,10 @@ level that is similar to easy_install, with a few exceptions:
 
 - Distutils options for building extensions can be passed.
 
-The easy_install module provides a method, install, for installing one
+Distribution installation
+-------------------------
+
+The easy_install module provides a function, install, for installing one
 or more packages and their dependencies.  The install function takes 2
 positional arguments:
 
@@ -68,6 +71,13 @@ working_set
    you to call install multiple times, if necessary, to gather
    multiple sets of requirements.
 
+newest
+   A boolian value indicating whether to search for new distributions
+   when already-installed distributions meet the requirement.  When
+   this is true, the default, and when the destination directory is
+   not None, then the install function will search for the newest
+   distributions that satisfy the requirements.
+
 The install method returns a working set containing the distributions
 needed to meet the given requirements.
 
@@ -111,8 +121,17 @@ And the actual eggs were added to the eggs directory.
     -  demo-0.2-py2.4.egg
     -  demoneeded-1.1-py2.4.egg
 
-If we ask for the demo distribution without a version restriction,
-we'll get the newer version:
+If we remove the version restriction on demo, but specify a false
+value for newest, no new didstributions will be installed:
+
+    >>> ws = zc.buildout.easy_install.install(
+    ...     ['demo'], dest, links=[link_server], index=link_server+'index/',
+    ...     newest=False)
+    >>> ls(dest)
+    -  demo-0.2-py2.4.egg
+    -  demoneeded-1.1-py2.4.egg
+
+If we leave off the newst option, we'll get an update for demo:
 
     >>> ws = zc.buildout.easy_install.install(
     ...     ['demo'], dest, links=[link_server], index=link_server+'index/')
@@ -480,9 +499,13 @@ path
    A list of additional directories to search for locally-installed
    distributions.
 
-always_unzip
-   A flag indicating that newly-downloaded distributions should be
-   directories even if they could be installed as zip files.
+newest
+   A boolian value indicating whether to search for new distributions
+   when already-installed distributions meet the requirement.  When
+   this is true, the default, and when the destination directory is
+   not None, then the install function will search for the newest
+   distributions that satisfy the requirements.
+
 
 Our link server included a source distribution that includes a simple
 extension, extdemo.c::
@@ -534,6 +557,57 @@ Now if we look in our destination directory, we see we have an extdemo egg:
     d  demoneeded-1.1-py2.4.egg
     d  extdemo-1.4-py2.4-unix-i686.egg
 
+Let's update our link server with a new version of extdemo:
+
+    >>> update_extdemo()
+    >>> print get(link_server),
+    <html><body>
+    <a href="demo-0.1-py2.4.egg">demo-0.1-py2.4.egg</a><br>
+    <a href="demo-0.2-py2.4.egg">demo-0.2-py2.4.egg</a><br>
+    <a href="demo-0.3-py2.4.egg">demo-0.3-py2.4.egg</a><br>
+    <a href="demoneeded-1.0.zip">demoneeded-1.0.zip</a><br>
+    <a href="demoneeded-1.1.zip">demoneeded-1.1.zip</a><br>
+    <a href="extdemo-1.4.zip">extdemo-1.4.zip</a><br>
+    <a href="extdemo-1.5.zip">extdemo-1.5.zip</a><br>
+    <a href="index/">index/</a><br>
+    <a href="other-1.0-py2.4.egg">other-1.0-py2.4.egg</a><br>
+    </body></html>
+
+The easy_install caches information about servers to reduce network
+access. To see the update, we have to call the clear_index_cache
+function to clear the index cache:
+
+    >>> zc.buildout.easy_install.clear_index_cache()
+
+If we run build with newest set to False, we won't get an update:
+
+    >>> zc.buildout.easy_install.build(
+    ...   'extdemo', dest, 
+    ...   {'include-dirs': os.path.join(sample_buildout, 'include')},
+    ...   links=[link_server], index=link_server+'index/',
+    ...   newest=False)
+    '/sample-install/extdemo-1.4-py2.4-linux-i686.egg'
+
+    >>> ls(dest)
+    d  demo-0.3-py2.4.egg
+    d  demoneeded-1.1-py2.4.egg
+    d  extdemo-1.4-py2.4-unix-i686.egg
+
+But if we run it with the default True setting for newest, then we'll
+get an updated egg:
+
+    >>> zc.buildout.easy_install.build(
+    ...   'extdemo', dest, 
+    ...   {'include-dirs': os.path.join(sample_buildout, 'include')},
+    ...   links=[link_server], index=link_server+'index/')
+    '/sample-install/extdemo-1.5-py2.4-unix-i686.egg'
+
+    d  demo-0.3-py2.4.egg
+    d  demoneeded-1.1-py2.4.egg
+    d  extdemo-1.4-py2.4-unix-i686.egg
+    d  extdemo-1.5-py2.4-unix-i686.egg
+
+
 Handling custom build options for extensions in develop eggs
 ------------------------------------------------------------
 
@@ -586,6 +660,7 @@ egg link:
     d  demo-0.3-py2.4.egg
     d  demoneeded-1.1-py2.4.egg
     d  extdemo-1.4-py2.4-linux-i686.egg
+    d  extdemo-1.5-py2.4-linux-i686.egg
     -  extdemo.egg-link
 
 And that the source directory contains the compiled extension:

src/zc/buildout/tests.py

@@ -1047,19 +1047,22 @@ initextdemo(void)
 extdemo_setup_py = """
 from distutils.core import setup, Extension
 
-setup(name = "extdemo", version = "1.4", url="http://www.zope.org",
+setup(name = "extdemo", version = "%s", url="http://www.zope.org",
       author="Demo", author_email="demo@demo.com",
       ext_modules = [Extension('extdemo', ['extdemo.c'])],
       )
 """
 
-def add_source_dist(test):
+def add_source_dist(test, version=1.4):
 
-    tmp = test.globs['extdemo'] = test.globs['tmpdir']('extdemo')
+    if 'extdemo' not in test.globs:
+        test.globs['extdemo'] = test.globs['tmpdir']('extdemo')
+
+    tmp = test.globs['extdemo']
     write = test.globs['write']
     try:
         write(tmp, 'extdemo.c', extdemo_c);
-        write(tmp, 'setup.py', extdemo_setup_py);
+        write(tmp, 'setup.py', extdemo_setup_py % version);
         write(tmp, 'README', "");
         write(tmp, 'MANIFEST.in', "include *.c\n");
         test.globs['sdist'](tmp, test.globs['sample_eggs'])
@@ -1075,6 +1078,8 @@ def easy_install_SetUp(test):
     add_source_dist(test)
     test.globs['link_server'] = test.globs['start_server'](
         test.globs['sample_eggs'])
+    test.globs['update_extdemo'] = lambda : add_source_dist(test, 1.5)
+
         
 egg_parse = re.compile('([0-9a-zA-Z_.]+)-([0-9a-zA-Z_.]+)-py(\d[.]\d).egg$'
                        ).match