diff --git a/bootstrap.py b/bootstrap.py index 5883a0cf893893dc05daee9463de4ff6c323d2ee..6a620fe1c252981d7c492f51481fd28a125fe0eb 100644 --- a/bootstrap.py +++ b/bootstrap.py @@ -25,42 +25,17 @@ for d in 'eggs', 'develop-eggs', 'bin': ez = {} exec urllib2.urlopen('http://peak.telecommunity.com/dist/ez_setup.py' ).read() in ez - ez['use_setuptools'](to_dir='eggs', download_delay=0) -import setuptools.command.easy_install import pkg_resources -import setuptools.package_index -import distutils.dist os.spawnle(os.P_WAIT, sys.executable, sys.executable, 'setup.py', '-q', 'develop', '-m', '-x', '-d', 'develop-eggs', {'PYTHONPATH': os.path.dirname(pkg_resources.__file__)}, ) +pkg_resources.working_set.add_entry('src') -## easy = setuptools.command.easy_install.easy_install( -## distutils.dist.Distribution(), -## multi_version=True, -## exclude_scripts=True, -## sitepy_installed=True, -## install_dir='eggs', -## outputs=[], -## quiet=True, -## zip_ok=True, -## args=['zc.buildout'], -## ) -## easy.finalize_options() -## easy.easy_install('zc.buildout') - -env = pkg_resources.Environment(['develop-eggs', 'eggs']) - -ws = pkg_resources.WorkingSet() -sys.path[0:0] = [ - d.location - for d in ws.resolve([pkg_resources.Requirement.parse('zc.buildout')], env) - ] - -import zc.buildout.egglinker -zc.buildout.egglinker.scripts(['zc.buildout'], 'bin', ['eggs']) - +import zc.buildout.easy_install +zc.buildout.easy_install.scripts( + ['zc.buildout'], pkg_resources.working_set , sys.executable, 'bin') sys.exit(os.spawnl(os.P_WAIT, 'bin/buildout', 'bin/buildout')) diff --git a/buildout.cfg b/buildout.cfg index 935546240795fc206c72e1d96284e526f420a9b5..af2e04e5dcd63fc742108cdbb820f249d421c3c7 100644 --- a/buildout.cfg +++ b/buildout.cfg @@ -2,10 +2,12 @@ develop = eggrecipe testrunnerrecipe parts = test +# prevent slow access to cheeseshop: +index = http://download.zope.org + [test] recipe = zc.recipe.testrunner distributions = zc.buildout zc.recipe.egg zc.recipe.testrunner - diff --git a/eggrecipe/setup.py b/eggrecipe/setup.py index 3e00da9e175a40c84c391dc8d1aa5323b7bd1b7c..46761f294a81a061c3ec8392f4c84366a65f3d19 100644 --- a/eggrecipe/setup.py +++ b/eggrecipe/setup.py @@ -7,7 +7,7 @@ setup( include_package_data = True, package_dir = {'':'src'}, namespace_packages = ['zc', 'zc.recipe'], - install_requires = ['zc.buildout'], + install_requires = ['zc.buildout', 'setuptools'], tests_require = ['zope.testing'], test_suite = 'zc.recipe.eggs.tests.test_suite', author = "Jim Fulton", diff --git a/eggrecipe/src/zc/recipe/egg/README.txt b/eggrecipe/src/zc/recipe/egg/README.txt index dc52f7b80a97a8d98434dafef085ef90004c4df3..e2125af99e86fde93f3e7bc6592f81e10351cfc2 100644 --- a/eggrecipe/src/zc/recipe/egg/README.txt +++ b/eggrecipe/src/zc/recipe/egg/README.txt @@ -11,9 +11,23 @@ distribution If not specified, the distribution defaults to the part name. + Multiple requirements can be given, separated by newlines. Each + requirement has to be on a separate line. + find-links A list of URLs, files, or directories to search for distributions. +index + The URL of an index server, or almost any other valid URL. :) + + If not specified, the Python Package Index, + http://cheeseshop.python.org/pypi, is used. You can specify an + alternate index with this option. If you use the links option and + if the links point to the needed distributions, then the index can + be anything and will be largely ignored. In the examples, here, + we'll just point to an empty directory on our link server. This + will make our examples run a little bit faster. + python The name of a section to get the Python executable from. If not specified, then the buildout python option is used. The @@ -26,13 +40,20 @@ unzip only effective when an egg is installed. If a zipped egg already exists in the eggs directory, it will not be unzipped. -To illustrate this, we've created a directory with some sample eggs: - >>> ls(sample_eggs) - - demo-0.1-py2.3.egg - - demo-0.2-py2.3.egg - - demo-0.3-py2.3.egg - - demoneeded-1.0-py2.3.egg +We have a link server that has a number of eggs: + + >>> print get(link_server), + <html><body> + <a href="demo-0.1-py2.3.egg">demo-0.1-py2.3.egg</a><br> + <a href="demo-0.2-py2.3.egg">demo-0.2-py2.3.egg</a><br> + <a href="demo-0.3-py2.3.egg">demo-0.3-py2.3.egg</a><br> + <a href="demoneeded-1.0-py2.3.egg">demoneeded-1.0-py2.3.egg</a><br> + <a href="demoneeded-1.1-py2.3.egg">demoneeded-1.1-py2.3.egg</a><br> + <a href="index/">index/</a><br> + <a href="other-1.0-py2.3.egg">other-1.0-py2.3.egg</a><br> + </body></html> + We have a sample buildout. Let's update it's configuration file to install the demo package. @@ -44,9 +65,10 @@ install the demo package. ... ... [demo] ... recipe = zc.recipe.egg - ... distribution = demo <0.3 - ... find-links = %s - ... """ % sample_eggs) + ... distribution = demo<0.3 + ... find-links = %(server)s + ... index = %(server)s/index + ... """ % dict(server=link_server)) In this example, we limited ourself to revisions before 0.3. We also specified where to find distributions using the find-links option. @@ -55,14 +77,14 @@ Let's run the buildout: >>> import os >>> os.chdir(sample_buildout) - >>> runscript = os.path.join(sample_buildout, 'bin', 'buildout') - >>> print system(runscript), + >>> buildout = os.path.join(sample_buildout, 'bin', 'buildout') + >>> print system(buildout), Now, if we look at the buildout eggs directory: >>> ls(sample_buildout, 'eggs') - demo-0.2-py2.3.egg - - demoneeded-1.0-py2.3.egg + - demoneeded-1.1-py2.3.egg We see that we got an egg for demo that met the requirement, as well as the egg for demoneeded, wich demo requires. (We also see an egg @@ -114,21 +136,22 @@ specification. For example, We remove the restriction on demo: ... ... [demo] ... recipe = zc.recipe.egg - ... find-links = %s + ... find-links = %(server)s + ... index = %(server)s/index ... unzip = true - ... """ % sample_eggs) + ... """ % dict(server=link_server)) We also used the unzip uption to request a directory, rather than a zip file. - >>> print system(runscript), + >>> print system(buildout), Then we'll get a new demo egg: >>> ls(sample_buildout, 'eggs') - demo-0.2-py2.3.egg d demo-0.3-py2.3.egg - - demoneeded-1.0-py2.3.egg + d demoneeded-1.0-py2.3.egg Note that we removed the distribution option, and the distribution defaulted to the part name. @@ -150,12 +173,13 @@ arguments: ... ... [demo] ... recipe = zc.recipe.egg - ... find-links = %s + ... find-links = %(server)s + ... index = %(server)s/index ... scripts = - ... """ % sample_eggs) + ... """ % dict(server=link_server)) - >>> print system(runscript), + >>> print system(buildout), >>> ls(sample_buildout, 'bin') - buildout @@ -169,11 +193,12 @@ You can also control the name used for scripts: ... ... [demo] ... recipe = zc.recipe.egg - ... find-links = %s + ... find-links = %(server)s + ... index = %(server)s/index ... scripts = demo=foo - ... """ % sample_eggs) + ... """ % dict(server=link_server)) - >>> print system(runscript), + >>> print system(buildout), >>> ls(sample_buildout, 'bin') - buildout diff --git a/eggrecipe/src/zc/recipe/egg/egg.py b/eggrecipe/src/zc/recipe/egg/egg.py index 8e313c723c50fa509a9bde08db61e88ba0d2bd57..28a53c7795df5ae30c633eea9d1a17a5bf96a0b9 100644 --- a/eggrecipe/src/zc/recipe/egg/egg.py +++ b/eggrecipe/src/zc/recipe/egg/egg.py @@ -16,8 +16,7 @@ $Id$ """ -import os, zipfile -import zc.buildout.egglinker +import os, re, zipfile import zc.buildout.easy_install class Egg: @@ -29,14 +28,17 @@ class Egg: links = options.get('find-links', buildout['buildout'].get('find-links')) if links: - buildout_directory = buildout['buildout']['directory'] - links = [os.path.join(buildout_directory, link) - for link in links.split()] + links = links.split() options['find-links'] = '\n'.join(links) else: links = () self.links = links + index = options.get('index', buildout['buildout'].get('index')) + if index is not None: + options['index'] = index + self.index = index + options['_b'] = buildout['buildout']['bin-directory'] options['_e'] = buildout['buildout']['eggs-directory'] options['_d'] = buildout['buildout']['develop-eggs-directory'] @@ -48,13 +50,20 @@ class Egg: def install(self): options = self.options - distribution = options.get('distribution', self.name) + distributions = [ + r.strip() + for r in options.get('distribution', self.name).split('\n') + if r.strip()] - zc.buildout.easy_install.install( - distribution, options['_e'], self.links, options['executable'], - always_unzip=options.get('unzip') == 'true') + ws = zc.buildout.easy_install.install( + distributions, options['_e'], + links = self.links, + index = self.index, + executable = options['executable'], + always_unzip=options.get('unzip') == 'true', + path=[options['_d']] + ) - eggss = [options['_d'], options['_e']] scripts = options.get('scripts') if scripts or scripts is None: if scripts is not None: @@ -63,7 +72,7 @@ class Egg: ('=' in s) and s.split('=', 1) or (s, s) for s in scripts ]) - return zc.buildout.egglinker.scripts( - [distribution], options['_b'], eggss, - scripts=scripts, executable=options['executable']) + return zc.buildout.easy_install.scripts( + distributions, ws, options['executable'], + options['_b'], scripts=scripts) diff --git a/eggrecipe/src/zc/recipe/egg/selecting-python.txt b/eggrecipe/src/zc/recipe/egg/selecting-python.txt index 8921745a2d0cdc3eb5db3d967347ec015db3af3e..a5a2c47071171523cdb4f5510798fd0936828e91 100644 --- a/eggrecipe/src/zc/recipe/egg/selecting-python.txt +++ b/eggrecipe/src/zc/recipe/egg/selecting-python.txt @@ -9,17 +9,24 @@ We can specify the python to use by specifying the name of a section to read the Python executable from. The default is the section defined by the python buildout option. -We have a directory with some sample eggs: - - >>> ls(sample_eggs) - - demo-0.1-py2.3.egg - - demo-0.1-py2.4.egg - - demo-0.2-py2.3.egg - - demo-0.2-py2.4.egg - - demo-0.3-py2.3.egg - - demo-0.3-py2.4.egg - - demoneeded-1.0-py2.3.egg - - demoneeded-1.0-py2.4.egg +We have a link server: + + >>> print get(link_server), + <html><body> + <a href="demo-0.1-py2.3.egg">demo-0.1-py2.3.egg</a><br> + <a href="demo-0.1-py2.4.egg">demo-0.1-py2.4.egg</a><br> + <a href="demo-0.2-py2.3.egg">demo-0.2-py2.3.egg</a><br> + <a href="demo-0.2-py2.4.egg">demo-0.2-py2.4.egg</a><br> + <a href="demo-0.3-py2.3.egg">demo-0.3-py2.3.egg</a><br> + <a href="demo-0.3-py2.4.egg">demo-0.3-py2.4.egg</a><br> + <a href="demoneeded-1.0-py2.3.egg">demoneeded-1.0-py2.3.egg</a><br> + <a href="demoneeded-1.0-py2.4.egg">demoneeded-1.0-py2.4.egg</a><br> + <a href="demoneeded-1.1-py2.3.egg">demoneeded-1.1-py2.3.egg</a><br> + <a href="demoneeded-1.1-py2.4.egg">demoneeded-1.1-py2.4.egg</a><br> + <a href="index/">index/</a><br> + <a href="other-1.0-py2.3.egg">other-1.0-py2.3.egg</a><br> + <a href="other-1.0-py2.4.egg">other-1.0-py2.4.egg</a><br> + </body></html> We have a sample buildout. Let's update it's configuration file to install the demo package using Python 2.3. @@ -33,9 +40,10 @@ install the demo package using Python 2.3. ... [demo] ... recipe = zc.recipe.egg ... distribution = demo <0.3 - ... find-links = %s + ... find-links = %(server)s + ... index = %(server)s/index ... python = python2.3 - ... """ % sample_eggs) + ... """ % dict(server=link_server)) In our default.cfg file in the .buildout subdirectiry of our directory, we have something like:: @@ -59,7 +67,7 @@ we'll get the Python 2.3 eggs for demo and demoneeded: >>> ls(sample_buildout, 'eggs') - demo-0.2-py2.3.egg - - demoneeded-1.0-py2.3.egg + - demoneeded-1.1-py2.3.egg And the generated scripts invoke Python 2.3: @@ -71,7 +79,7 @@ And the generated scripts invoke Python 2.3: import sys sys.path[0:0] = [ '/private/tmp/tmpOEtRO8sample-buildout/eggs/demo-0.2-py2.3.egg', - '/private/tmp/tmpOEtRO8sample-buildout/eggs/demoneeded-1.0-py2.3.egg' + '/private/tmp/tmpOEtRO8sample-buildout/eggs/demoneeded-1.1-py2.3.egg' ] <BLANKLINE> import eggrecipedemo @@ -87,7 +95,7 @@ And the generated scripts invoke Python 2.3: import sys sys.path[0:0] = [ '/tmp/tmpOBTxDMsample-buildout/eggs/demo-0.2-py2.3.egg', - '/tmp/tmpOBTxDMsample-buildout/eggs/demoneeded-1.0-py2.3.egg' + '/tmp/tmpOBTxDMsample-buildout/eggs/demoneeded-1.1-py2.3.egg' ] If we change the Python version to 2.4, we'll use Python 2.4 eggs: @@ -101,17 +109,18 @@ If we change the Python version to 2.4, we'll use Python 2.4 eggs: ... [demo] ... recipe = zc.recipe.egg ... distribution = demo <0.3 - ... find-links = %s + ... find-links = %(server)s + ... index = %(server)s/index ... python = python2.4 - ... """ % sample_eggs) + ... """ % dict(server=link_server)) >>> print system(buildout), >>> ls(sample_buildout, 'eggs') - demo-0.2-py2.3.egg - demo-0.2-py2.4.egg - - demoneeded-1.0-py2.3.egg - - demoneeded-1.0-py2.4.egg + - demoneeded-1.1-py2.3.egg + - demoneeded-1.1-py2.4.egg >>> f = open(os.path.join(sample_buildout, 'bin', 'demo')) >>> f.readline().strip() == '#!' + python2_4_executable @@ -121,7 +130,7 @@ If we change the Python version to 2.4, we'll use Python 2.4 eggs: import sys sys.path[0:0] = [ '/private/tmp/tmpOEtRO8sample-buildout/eggs/demo-0.2-py2.4.egg', - '/private/tmp/tmpOEtRO8sample-buildout/eggs/demoneeded-1.0-py2.4.egg' + '/private/tmp/tmpOEtRO8sample-buildout/eggs/demoneeded-1.1-py2.4.egg' ] <BLANKLINE> import eggrecipedemo @@ -137,7 +146,7 @@ If we change the Python version to 2.4, we'll use Python 2.4 eggs: import sys sys.path[0:0] = [ '/tmp/tmpOBTxDMsample-buildout/eggs/demo-0.2-py2.4.egg', - '/tmp/tmpOBTxDMsample-buildout/eggs/demoneeded-1.0-py2.4.egg' + '/tmp/tmpOBTxDMsample-buildout/eggs/demoneeded-1.1-py2.4.egg' ] diff --git a/eggrecipe/src/zc/recipe/egg/tests.py b/eggrecipe/src/zc/recipe/egg/tests.py index 0613ceca9df77391e32a7fe0c5d80735506dac3b..1c6dbdc6cd9ad25c86a79d6c0da823950a10cfa6 100644 --- a/eggrecipe/src/zc/recipe/egg/tests.py +++ b/eggrecipe/src/zc/recipe/egg/tests.py @@ -29,10 +29,16 @@ def setUp(test): 'develop-eggs', 'zc.recipe.egg.egg-link'), 'w').write(dirname(__file__, 4)) zc.buildout.testing.create_sample_eggs(test) + test.globs['link_server'] = ( + 'http://localhost:%s/' + % zc.buildout.testing.start_server(zc.buildout.testing.make_tree(test)) + ) + def tearDown(test): shutil.rmtree(test.globs['_sample_eggs_container']) zc.buildout.testing.buildoutTearDown(test) + zc.buildout.testing.stop_server(test.globs['link_server']) def setUpPython(test): zc.buildout.testing.buildoutSetUp(test, clear_home=False) @@ -42,6 +48,10 @@ def setUpPython(test): 'w').write(dirname(__file__, 4)) zc.buildout.testing.multi_python(test) + test.globs['link_server'] = ( + 'http://localhost:%s/' + % zc.buildout.testing.start_server(zc.buildout.testing.make_tree(test)) + ) def test_suite(): return unittest.TestSuite(( @@ -54,7 +64,8 @@ def test_suite(): '(\\w+-)[^ \t\n%(sep)s/]+.egg' % dict(sep=os.path.sep) ), - '\\2-VVV-egg') + '\\2-VVV-egg'), + (re.compile('-py\d[.]\d.egg'), '-py2.4.egg'), ]) ), doctest.DocFileSuite( diff --git a/src/zc/buildout/buildout.py b/src/zc/buildout/buildout.py index ccda40654c947f5b37787997f021526e0dbd6ca4..0d33d0c007abb64bc9a43820a16aa75b9f92e3cc 100644 --- a/src/zc/buildout/buildout.py +++ b/src/zc/buildout/buildout.py @@ -28,7 +28,6 @@ import ConfigParser import zc.buildout.easy_install import pkg_resources import zc.buildout.easy_install -import zc.buildout.egglinker class MissingOption(KeyError): """A required option was missing @@ -262,32 +261,50 @@ class Buildout(dict): os.chdir(os.path.dirname(setup)) os.spawnle( os.P_WAIT, sys.executable, sys.executable, - setup, '-q', 'develop', '-m', '-x', + setup, '-q', 'develop', '-m', '-x', '-N', '-f', ' '.join(self._links), '-d', self['buildout']['develop-eggs-directory'], {'PYTHONPATH': os.path.dirname(pkg_resources.__file__)}, ) finally: - os.chdir(os.path.dirname(here)) + os.chdir(here) def _load_recipes(self, parts): recipes = {} + if not parts: + return recipes + recipes_requirements = [] pkg_resources.working_set.add_entry( self['buildout']['develop-eggs-directory']) pkg_resources.working_set.add_entry(self['buildout']['eggs-directory']) - # Install the recipe distros + # Gather requirements for part in parts: options = self.get(part) if options is None: options = self[part] = {} recipe, entry = self._recipe(part, options) - zc.buildout.easy_install.install( - recipe, self['buildout']['eggs-directory'], self._links) recipes_requirements.append(recipe) + # Install the recipe distros + offline = self['buildout'].get('offline', 'false') + if offline not in ('true', 'false'): + self._error('Invalif value for offline option: %s', offline) + if offline == 'true': + ws = zc.buildout.easy_install.working_set( + recipes_requirements, sys.executable, + [self['buildout']['eggs-directory'], + self['buildout']['develop-eggs-directory'], + ], + ) + else: + ws = zc.buildout.easy_install.install( + recipes_requirements, self['buildout']['eggs-directory'], + links=self._links, index=self['buildout'].get('index'), + path=[self['buildout']['develop-eggs-directory']]) + # Add the distros to the working set pkg_resources.require(recipes_requirements) @@ -503,6 +520,10 @@ def main(args=None): else: verbosity -= 10 op = op[1:] + if op == 'd': + op = op[1:] + import pdb; pdb.set_trace() + if op[:1] == 'c': op = op[1:] if op: diff --git a/src/zc/buildout/easy_install.py b/src/zc/buildout/easy_install.py index 364b6aba407a0883642cfa5bcb220e98d805b067..497bf7dffec33071dc826f11239ec898a41e2fd7 100644 --- a/src/zc/buildout/easy_install.py +++ b/src/zc/buildout/easy_install.py @@ -20,18 +20,299 @@ installed. $Id$ """ -import os, sys +import logging, os, re, sys +import pkg_resources +import zc.buildout -def install(spec, dest, links, executable=sys.executable, always_unzip=False): +logger = logging.getLogger('zc.buildout.easy_install') + +# Include buildout and setuptools eggs in paths +buildout_and_setuptools_path = [ + (('.egg' in m.__file__) + and m.__file__[:m.__file__.rfind('.egg')+4] + or os.path.dirname(m.__file__) + ) + for m in (pkg_resources,) + ] +buildout_and_setuptools_path += [ + (('.egg' in m.__file__) + and m.__file__[:m.__file__.rfind('.egg')+4] + or os.path.dirname(os.path.dirname(os.path.dirname(m.__file__))) + ) + for m in (zc.buildout,) + ] + +_versions = {sys.executable: '%d.%d' % sys.version_info[:2]} +def _get_version(executable): + try: + return _versions[executable] + except KeyError: + i, o = os.popen4(executable + ' -V') + i.close() + version = o.read().strip() + o.close() + pystring, version = version.split() + assert pystring == 'Python' + version = re.match('(\d[.]\d)[.]\d$', version).group(1) + _versions[executable] = version + return version + +def _satisfied(req, env): + dists = env[req.project_name] + + best = None + for dist in dists: + if (dist.precedence == pkg_resources.DEVELOP_DIST) and (dist in req): + if best is not None and best.location != dist.location: + raise ValueError('Multiple devel eggs for', req) + best = dist + + if best is not None: + return best + + specs = [(pkg_resources.parse_version(v), op) for (op, v) in req.specs] + specs.sort() + maxv = None + greater = False + lastv = None + for v, op in specs: + if op == '==' and not greater: + maxv = v + elif op in ('>', '>=', '!='): + maxv = None + greater == True + elif op == '<': + maxv = None + greater == False + elif op == '<=': + maxv = v + greater == False + + if v == lastv: + # Repeated versions values are undefined, so + # all bets are off + maxv = None + greater = True + else: + lastv = v + + if maxv is not None: + for dist in dists: + if dist.parsed_version == maxv: + return dist + + return None + +def _call_easy_install(spec, dest, links=(), + index = None, + executable=sys.executable, + always_unzip=False, + ): prefix = sys.exec_prefix + os.path.sep path = os.pathsep.join([p for p in sys.path if not p.startswith(prefix)]) args = ( '-c', 'from setuptools.command.easy_install import main; main()', - '-mqxd', dest) + '-mUNxd', dest) if links: args += ('-f', ' '.join(links)) + if index: + args += ('-i', index) if always_unzip: args += ('-Z', ) - args += (spec, dict(PYTHONPATH=path)) + level = logger.getEffectiveLevel() + if level > logging.DEBUG: + args += ('-q', ) + elif level < logging.DEBUG: + args += ('-v', ) + + args += (spec, ) + + if level <= logging.DEBUG: + logger.debug('Running easy_install:\n%s "%s"\npath=%s\n', + executable, '" "'.join(args), path) - os.spawnle(os.P_WAIT, executable, executable, *args) + args += (dict(PYTHONPATH=path), ) + sys.stdout.flush() # We want any pending output first + exit_code = os.spawnle(os.P_WAIT, executable, executable, *args) + + # We may overwrite distributions, so clear importer + # cache. + sys.path_importer_cache.clear() + + assert exit_code == 0 + + +def _get_dist(requirement, env, ws, + dest, links, index, executable, always_unzip): + # Maybe an existing dist is already the best dist that satisfies the + # requirement + dist = _satisfied(requirement, env) + + # XXX Special case setuptools because: + # 1. Almost everything depends on it and + # 2. It is expensive to checl for. + # Need to think of a cleaner way to handle this. + # If we already have a satisfactory version, use it. + if dist is None and requirement.project_name == 'setuptools': + dist = env.best_match(requirement, ws) + + if dist is None: + if dest is not None: + # May need a new one. Call easy_install + _call_easy_install(str(requirement), dest, links, index, + executable, always_unzip) + + # Because we may have added new eggs, we need to rescan + # the destination directory. A possible optimization + # is to get easy_install to recod the files installed + # and either firgure out the distribution added, or + # only rescan if any files have been added. + env.scan([dest]) + + dist = env.best_match(requirement, ws) + + # XXX Need test for this + if dist.has_metadata('dependency_links.txt'): + for link in dist.get_metadata_lines('dependency_links.txt'): + link = link.strip() + if link not in links: + links.append(link) + + return dist + +def install(specs, dest, + links=(), index=None, + executable=sys.executable, always_unzip=False, + path=None): + + logger.debug('Installing %r', specs) + + path = path and path[:] or [] + if dest is not None: + path.insert(0, dest) + + path += buildout_and_setuptools_path + + links = list(links) # make copy, because we may need to mutate + + + # For each spec, see if it is already installed. We create a working + # set to keep track of what we've collected and to make sue than the + # distributions assembled are consistent. + env = pkg_resources.Environment(path, python=_get_version(executable)) + requirements = [pkg_resources.Requirement.parse(spec) for spec in specs] + + ws = pkg_resources.WorkingSet([]) + + for requirement in requirements: + ws.add(_get_dist(requirement, env, ws, + dest, links, index, executable, always_unzip) + ) + + # OK, we have the requested distributions and they're in the working + # set, but they may have unmet requirements. We'll simply keep + # trying to resolve requirements, adding missing requirements as they + # are reported. + # + # Note that we don't pass in the environment, because we + # want to look for new eggs unless what we have is the best that matches + # the requirement. + while 1: + try: + ws.resolve(requirements) + except pkg_resources.DistributionNotFound, err: + [requirement] = err + if dest: + logger.debug('Getting required %s', requirement) + ws.add(_get_dist(requirement, env, ws, + dest, links, index, executable, always_unzip) + ) + else: + break + + return ws + +def working_set(specs, executable, path): + return install(specs, None, executable=executable, path=path) + +def scripts(reqs, working_set, executable, dest, scripts=None): + reqs = [pkg_resources.Requirement.parse(r) for r in reqs] + projects = [r.project_name for r in reqs] + path = "',\n '".join([dist.location for dist in working_set]) + generated = [] + + for dist in working_set: + if dist.project_name in projects: + for name in pkg_resources.get_entry_map(dist, 'console_scripts'): + if scripts is not None: + sname = scripts.get(name) + if sname is None: + continue + else: + sname = name + + sname = os.path.join(dest, sname) + generated.append(sname) + _script(dist, 'console_scripts', name, path, sname, executable) + + name = 'py_'+dist.project_name + if scripts is not None: + sname = scripts.get(name) + else: + sname = name + + if sname is not None: + sname = os.path.join(dest, sname) + generated.append(sname) + _pyscript(path, sname, executable) + + return generated + +def _script(dist, group, name, path, dest, executable): + entry_point = dist.get_entry_info(group, name) + open(dest, 'w').write(script_template % dict( + python = executable, + path = path, + project = dist.project_name, + name = name, + module_name = entry_point.module_name, + attrs = '.'.join(entry_point.attrs), + )) + try: + os.chmod(dest, 0755) + except (AttributeError, os.error): + pass + +script_template = '''\ +#!%(python)s + +import sys +sys.path[0:0] = [ + '%(path)s' + ] + +import %(module_name)s + +if __name__ == '__main__': + %(module_name)s.%(attrs)s() +''' + + +def _pyscript(path, dest, executable): + open(dest, 'w').write(py_script_template % dict( + python = executable, + path = path, + )) + try: + os.chmod(dest,0755) + except (AttributeError, os.error): + pass + +py_script_template = '''\ +#!%(python)s -i + +import sys +sys.path[0:0] = [ + '%(path)s' + ] +''' diff --git a/src/zc/buildout/easy_install.txt b/src/zc/buildout/easy_install.txt index 9bdecb205df18321b6fe302e9ac77883ffb42704..7a101b7b60b931a78c21e87c6858333716e52e6e 100644 --- a/src/zc/buildout/easy_install.txt +++ b/src/zc/buildout/easy_install.txt @@ -2,64 +2,263 @@ Minimal Python interface to easy_install ======================================== The easy_install module provides a minimal interface to the setuptools -easy_install command. This API is likely to grow, although I hope -that it will ultimately be replaced by a setuptools-provided API. +easy_install command that provides some additional semantics: + +- By default, we look for new packages *and* the packages that + they depend on. This is somewhat like (and uses) the --upgrade + option of easy_install, except that we also upgrade required + packages. + +- If the highest-revision package satisfying a specification is + already present, then we don't try to get another one. This saves a + lot of search time in the common case that packages are pegged to + specific versions. + +- If there is a develop egg that satisfies a requirement, we don't + look for additional distributions. We always give preference to + develop eggs. The easy_install module provides a single method, install. The -install function takes 3 arguments: +install function takes 2 positional arguments: -- A setuptools requirement specification for a distribution to be - installed, +- An iterable of setuptools requirement strings for the distributions + to be installed, and -- A destination egg directory to install to and to satisfy - requirements from, and +- A destination directory to install to and to satisfy + requirements from. -- a sequence of lications to look for distributions. +It supports a number of optional keyword arguments: -For example, given the sample eggs: +links + a sequence of URLs, file names, or directories to look for + links to distributions, - >>> ls(sample_eggs) - - demo-0.1-py2.3.egg - - demo-0.1-py2.4.egg - - demo-0.2-py2.3.egg - - demo-0.2-py2.4.egg - - demo-0.3-py2.3.egg - - demo-0.3-py2.4.egg - - demoneeded-1.0-py2.3.egg - - demoneeded-1.0-py2.4.egg +index + The URL of an index server, or almost any other valid URL. :) + + If not specified, the Python Package Index, + http://cheeseshop.python.org/pypi, is used. You can specify an + alternate index with this option. If you use the links option and + if the links point to the needed distributions, then the index can + be anything and will be largely ignored. In the examples, here, + we'll just point to an empty directory on our link server. This + will make our examples run a little bit faster. -let's make directory and install the demo egg to it: +executable + A path to a Python executable. Distributions will ne installed + using this executable and will be for the matching Python version. + +path + A list of additional directories to search for locally-installed + distributions. + +always_unzip + A flag indicating that newly-downloaded distributions should be + directories even if they could be installed as zip files. + +The install method returns a working set containing the distributions +needed to meet the given requirements. + +We have a link server that has a number of eggs: + + >>> print get(link_server), + <html><body> + <a href="demo-0.1-py2.3.egg">demo-0.1-py2.3.egg</a><br> + <a href="demo-0.1-py2.4.egg">demo-0.1-py2.4.egg</a><br> + <a href="demo-0.2-py2.3.egg">demo-0.2-py2.3.egg</a><br> + <a href="demo-0.2-py2.4.egg">demo-0.2-py2.4.egg</a><br> + <a href="demo-0.3-py2.3.egg">demo-0.3-py2.3.egg</a><br> + <a href="demo-0.3-py2.4.egg">demo-0.3-py2.4.egg</a><br> + <a href="demoneeded-1.0-py2.3.egg">demoneeded-1.0-py2.3.egg</a><br> + <a href="demoneeded-1.0-py2.4.egg">demoneeded-1.0-py2.4.egg</a><br> + <a href="demoneeded-1.1-py2.3.egg">demoneeded-1.1-py2.3.egg</a><br> + <a href="demoneeded-1.1-py2.4.egg">demoneeded-1.1-py2.4.egg</a><br> + <a href="index/">index/</a><br> + <a href="other-1.0-py2.3.egg">other-1.0-py2.3.egg</a><br> + <a href="other-1.0-py2.4.egg">other-1.0-py2.4.egg</a><br> + </body></html> + +let's make directory and install the demo egg to it, using the demo: >>> import tempfile - >>> dest = tempfile.mkdtemp() + >>> dest = tempfile.mkdtemp('sample-install') >>> import zc.buildout.easy_install - >>> zc.buildout.easy_install.install('demo', dest, [sample_eggs]) + >>> ws = zc.buildout.easy_install.install( + ... ['demo==0.2'], dest, + ... links=[link_server], index=link_server+'index/') + +We requested version 0.2 of the demo distribution to be installed into +the destination server. We specified that we should search for links +on the link server and that we should use the (empty) link server +index directory as a package index. + +The working set contains the distributions we retrieved. + + >>> for dist in ws: + ... print dist + demo 0.2 + demoneeded 1.1 + +And the actual eggs were added to the eggs directory. + + >>> ls(dest) + - demo-0.2-py2.3.egg + - demoneeded-1.1-py2.3.egg + +If we ask for the demo distribution without a version restriction, +we'll get the newer version: + + >>> ws = zc.buildout.easy_install.install( + ... ['demo'], dest, links=[link_server], index=link_server+'index/') >>> ls(dest) + - demo-0.2-py2.3.egg + - demo-0.3-py2.3.egg + - demoneeded-1.1-py2.3.egg + +We can supply additional distributions. We can also supply +specifications for distributions that would normally be found via +dependencies. We might do this to specify a sprcific version. + + >>> ws = zc.buildout.easy_install.install( + ... ['demo', 'other', 'demoneeded==1.0'], dest, + ... links=[link_server], index=link_server+'index/') + + >>> for dist in ws: + ... print dist + demo 0.3 + other 1.0 + demoneeded 1.0 + + >>> ls(dest) + - demo-0.2-py2.3.egg - demo-0.3-py2.3.egg - demoneeded-1.0-py2.3.egg + - demoneeded-1.1-py2.3.egg + - other-1.0-py2.3.egg We can specify an alternate Python executable, and we can specify that, when we retrieve (or create) an egg, it should be unzipped. >>> import shutil >>> shutil.rmtree(dest) - >>> dest = tempfile.mkdtemp() - >>> zc.buildout.easy_install.install( - ... 'demo', dest, [sample_eggs], + >>> dest = tempfile.mkdtemp('sample-install') + >>> ws = zc.buildout.easy_install.install( + ... ['demo'], dest, links=[link_server], index=link_server+'index/', ... always_unzip=True, executable= python2_3_executable) >>> ls(dest) d demo-0.3-py2.3.egg - d demoneeded-1.0-py2.3.egg + d demoneeded-1.1-py2.3.egg >>> shutil.rmtree(dest) - >>> dest = tempfile.mkdtemp() - >>> zc.buildout.easy_install.install( - ... 'demo', dest, [sample_eggs], - ... always_unzip=True, executable= python2_4_executable) + >>> dest = tempfile.mkdtemp('sample-install') + >>> ws = zc.buildout.easy_install.install( + ... ['demo'], dest, links=[link_server], index=link_server+'index/', + ... always_unzip=True, executable=python2_4_executable) >>> ls(dest) d demo-0.3-py2.4.egg - d demoneeded-1.0-py2.4.egg + d demoneeded-1.1-py2.4.egg + +Script generation +----------------- + +The easy_install module provides support for creating scripts from +eggs. It provides a function similar to setuptools except that it +provides facilities for baking a script's path into the script. This +has two advantages: + +- The eggs to be used by a script are not chosen at run time, making + startup faster and, more importantly, deterministic. + +- The script doesn't have to import pkg_resources because the logic + that pkg_resources would execute at run time is executed at + script-creation time. + +The scripts method can be used to generate scripts. Let's create a +destination directory for it to place them in: + + >>> import tempfile + >>> bin = tempfile.mkdtemp() + +Now, we'll use the scripts method to generate scripts in this directory +from the demo egg: + + >>> scripts = zc.buildout.easy_install.scripts( + ... ['demo==0.1'], ws, python2_4_executable, bin) + +the four arguments we passed were: + +1. A sequence of distribution requirements. These are of the same + form as setuptools requirements. Here we passed a single + requirement, for the version 0.1 demo distribution. + +2. A working set, + +3. The Python executable to use, and +3. The destination directory. + +The bin directory now contains 2 generated scripts: + + >>> ls(bin) + - demo + - py_demo + +The return value is a list of the scripts generated: + + >>> import os + >>> scripts == [os.path.join(bin, 'demo'), os.path.join(bin, 'py_demo')] + True + +The demo script run the entry point defined in the demo egg: + + >>> cat(bin, 'demo') + #!/usr/local/bin/python2.3 + <BLANKLINE> + import sys + sys.path[0:0] = [ + '/tmp/xyzsample-install/demo-0.3-py2.3.egg', + '/tmp/xyzsample-install/demoneeded-1.1-py2.3.egg' + ] + <BLANKLINE> + import eggrecipedemo + <BLANKLINE> + if __name__ == '__main__': + eggrecipedemo.main() + +Some things to note: + +- The demo and demoneeded eggs are added to the beginning of sys.path. + +- The module for the script entry point is imported and the entry + point, in this case, 'main', is run. + +The py_demo script simply run the Python interactive interpreter with +the path set: + + >>> cat(bin, 'py_demo') + #!/usr/local/bin/python2.3 -i + <BLANKLINE> + import sys + sys.path[0:0] = [ + '/tmp/xyzsample-install/demo-0.3-py2.3.egg', + '/tmp/xyzsample-install/demoneeded-1.1-py2.3.egg' + ] + +An additional argumnet can be passed to define which scripts to install +and to provie script names. The argument is a dictionary mapping +original script names to new script names. + + >>> import shutil + >>> shutil.rmtree(bin) + >>> bin = tempfile.mkdtemp() + >>> scripts = zc.buildout.easy_install.scripts( + ... ['demo==0.1'], ws, python2_4_executable, bin, dict(demo='run')) + >>> scripts == [os.path.join(bin, 'run')] + True + >>> ls(bin) + - run + >>> print system(os.path.join(bin, 'run')), + 3 1 diff --git a/src/zc/buildout/egglinker.py b/src/zc/buildout/egglinker.py deleted file mode 100644 index e16f0e54081e8a7d2514e753ea42523fe68c832b..0000000000000000000000000000000000000000 --- a/src/zc/buildout/egglinker.py +++ /dev/null @@ -1,149 +0,0 @@ -############################################################################## -# -# Copyright (c) 2005 Zope Corporation and Contributors. -# All Rights Reserved. -# -# This software is subject to the provisions of the Zope Public License, -# Version 2.1 (ZPL). A copy of the ZPL should accompany this distribution. -# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED -# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED -# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS -# FOR A PARTICULAR PURPOSE. -# -############################################################################## -"""Egg linker -- Link eggs together to build applications - -Egg linker is a script that generates startup scripts for eggs that -include an egg's working script in the generated script. - -The egg linker module also exports helper functions of varous kinds to -assist in custom script generation. - -$Id$ -""" - -# XXX need to deal with extras - -import os -import re -import sys - -import pkg_resources - -_versions = {sys.executable: '%d.%d' % sys.version_info[:2]} -def _get_version(executable): - try: - return _versions[executable] - except KeyError: - i, o = os.popen4(executable + ' -V') - i.close() - version = o.read().strip() - o.close() - pystring, version = version.split() - assert pystring == 'Python' - version = re.match('(\d[.]\d)[.]\d$', version).group(1) - _versions[executable] = version - return version - -def distributions(reqs, eggss, executable=sys.executable): - env = pkg_resources.Environment(eggss, python=_get_version(executable)) - ws = pkg_resources.WorkingSet() - reqs = [pkg_resources.Requirement.parse(r) for r in reqs] - return ws.resolve(reqs, env=env) - -def path(reqs, eggss, executable=sys.executable): - dists = distributions(reqs, eggss, executable) - return [dist.location for dist in dists] - -def location(spec, eggss, executable=sys.executable): - env = pkg_resources.Environment(eggss, python=_get_version(executable)) - req = pkg_resources.Requirement.parse(spec) - dist = env.best_match(req, pkg_resources.WorkingSet()) - return dist.location - -def scripts(reqs, dest, eggss, scripts=None, executable=sys.executable): - dists = distributions(reqs, eggss, executable) - reqs = [pkg_resources.Requirement.parse(r) for r in reqs] - projects = [r.project_name for r in reqs] - path = "',\n '".join([dist.location for dist in dists]) - generated = [] - - for dist in dists: - if dist.project_name in projects: - for name in pkg_resources.get_entry_map(dist, 'console_scripts'): - if scripts is not None: - sname = scripts.get(name) - if sname is None: - continue - else: - sname = name - - sname = os.path.join(dest, sname) - generated.append(sname) - _script(dist, 'console_scripts', name, path, sname, executable) - - name = 'py_'+dist.project_name - if scripts is not None: - sname = scripts.get(name) - else: - sname = name - - if sname is not None: - sname = os.path.join(dest, sname) - generated.append(sname) - _pyscript(path, sname, executable) - - return generated - -def _script(dist, group, name, path, dest, executable): - entry_point = dist.get_entry_info(group, name) - open(dest, 'w').write(script_template % dict( - python = executable, - path = path, - project = dist.project_name, - name = name, - module_name = entry_point.module_name, - attrs = '.'.join(entry_point.attrs), - )) - try: - os.chmod(dest, 0755) - except (AttributeError, os.error): - pass - -script_template = '''\ -#!%(python)s - -import sys -sys.path[0:0] = [ - '%(path)s' - ] - -import %(module_name)s - -if __name__ == '__main__': - %(module_name)s.%(attrs)s() -''' - - -def _pyscript(path, dest, executable): - open(dest, 'w').write(py_script_template % dict( - python = executable, - path = path, - )) - try: - os.chmod(dest,0755) - except (AttributeError, os.error): - pass - -py_script_template = '''\ -#!%(python)s -i - -import sys -sys.path[0:0] = [ - '%(path)s' - ] -''' - -def main(): - import pdb; pdb.set_trace() - diff --git a/src/zc/buildout/egglinker.txt b/src/zc/buildout/egglinker.txt deleted file mode 100644 index 4a6f9f02078967f296c7bcc99ffed3769093c179..0000000000000000000000000000000000000000 --- a/src/zc/buildout/egglinker.txt +++ /dev/null @@ -1,215 +0,0 @@ -Custom script support -===================== - -The egg-linker module provides support for creating scripts from -eggs. It provides a function similar to setup tools except that it -provides facilities for baking a script's path into the script. This -has two advantages: - -- The eggs to be used by a script are not chosen at run time, making - startup faster and, more importantly, deterministic. - -- The script doesn't have to import pkg_resources because the logic - that pkg_resources would execute at run time is executed at - script-creation time. - -We have a directory with some sample eggs in it: - - >>> ls(sample_eggs) - - demo-0.1-py2.3.egg - - demo-0.1-py2.4.egg - - demo-0.2-py2.3.egg - - demo-0.2-py2.4.egg - - demo-0.3-py2.3.egg - - demo-0.3-py2.4.egg - - demoneeded-1.0-py2.3.egg - - demoneeded-1.0-py2.4.egg - -The demo package depends on the demoneeded package. - -The egglinker module can be used to generate scripts. Let's create a -desitnation directory for it to place them in: - - >>> import tempfile - >>> bin = tempfile.mkdtemp() - -Now, we'll use the egg linker to generate scripts in this directory -from the demo egg: - - >>> import zc.buildout.egglinker - >>> scripts = zc.buildout.egglinker.scripts(['demo==0.1'], bin, - ... [sample_eggs]) - -the three arguments we passed were: - -1. A sequence of distribution requirements. These are of the same - form as setuptools requirements. Here we passed a single - requirement, for the version 0.1 demo distribution. - -2. The destination directory. - -3, A sequence of egg directories, which are searched for suitable - distributions. - -The bin directory now contains 2 generated scripts: - - >>> ls(bin) - - demo - - py_demo - -The return value is a list of the scripts generated: - - >>> import os - >>> scripts == [os.path.join(bin, 'demo'), os.path.join(bin, 'py_demo')] - True - -The demo script run the entry point defined in the demo egg: - - >>> cat(bin, 'demo') - #!/usr/local/bin/python2.3 - <BLANKLINE> - import sys - sys.path[0:0] = [ - '/tmp/xyzsample-eggs/demo-0.1-py2.3.egg', - '/tmp/xyzsample-eggs/demoneeded-1.0-py2.3.egg' - ] - <BLANKLINE> - import eggrecipedemo - <BLANKLINE> - if __name__ == '__main__': - eggrecipedemo.main() - -Some things to note: - -- The demo and demoneeded eggs are added to the beginning of sys.path. - -- The module for the script entry point is imported and the entry - point, in this case, 'main', is run. - -The py_demo script simply run the Python interactive interpreter with -the path set: - - >>> cat(bin, 'py_demo') - #!/usr/local/bin/python2.3 -i - <BLANKLINE> - import sys - sys.path[0:0] = [ - '/tmp/xyzsample-eggs/demo-0.1-py2.3.egg', - '/tmp/xyzsample-eggs/demoneeded-1.0-py2.3.egg' - ] - -An additional argumnet can be passed to define which scripts to install -and to provie script names. The argument is a dictionary mapping -original script names to new script names. - - >>> import shutil - >>> shutil.rmtree(bin) - >>> bin = tempfile.mkdtemp() - >>> scripts = zc.buildout.egglinker.scripts( - ... ['demo==0.1'], bin, [sample_eggs], - ... dict(demo='run')) - >>> scripts == [os.path.join(bin, 'run')] - True - >>> ls(bin) - - run - - >>> print system(os.path.join(bin, 'run')), - 1 1 - -Sometimes we need more control over script generation. Some -lower-level APIs are available to help us generate scripts ourselves. -These apis are a little bit higher level than those provided by -the pkg_resources from the setuptools distribution. - -The path method returns a path to a set of eggs satisftying a sequence -of requirements from a sequence of egg directories: - - >>> zc.buildout.egglinker.path(['demo==0.1'], [sample_eggs]) - ... # doctest: +NORMALIZE_WHITESPACE - ['/tmp/xyzsample-eggs/demo-0.1-py2.3.egg', - '/tmp/xyzsample-eggs/demoneeded-1.0-py2.3.egg'] - - -The location method returns the distribution location for an egg that -satisfies a requirement: - - >>> zc.buildout.egglinker.location('demo==0.1', [sample_eggs]) - '/tmp/xyzsample-eggs/demo-0.1-py2.3.egg' - -The distributions function can retrieve a list of distributions found -ineg directories that match a sequence of requirements: - - >>> [(d.project_name, d.version) for d in - ... zc.buildout.egglinker.distributions(['demo==0.1'], [sample_eggs])] - [('demo', '0.1'), ('demoneeded', '1.0')] - -Using a custom Python interpreter ---------------------------------- - -You can pass an executable argument to egglinker methods: - - >>> scripts = zc.buildout.egglinker.scripts( - ... ['demo==0.1'], bin, [sample_eggs], - ... executable=python2_3_executable) - - >>> f = open(os.path.join(bin, 'demo')) - >>> f.readline().strip() == '#!' + python2_3_executable - True - >>> print f.read(), - <BLANKLINE> - import sys - sys.path[0:0] = [ - '/tmp/sample-eggs/dist/demo-0.1-py2.3.egg', - '/tmp/sample-eggs/dist/demoneeded-1.0-py2.3.egg' - ] - <BLANKLINE> - import eggrecipedemo - <BLANKLINE> - if __name__ == '__main__': - eggrecipedemo.main() - - >>> zc.buildout.egglinker.path(['demo==0.1'], [sample_eggs], - ... python2_3_executable) - ... # doctest: +NORMALIZE_WHITESPACE - ['/tmp/sample-eggs/dist/demo-0.1-py2.3.egg', - '/tmp/sample-eggs/dist/demoneeded-1.0-py2.3.egg'] - - - >>> zc.buildout.egglinker.location('demo==0.1', [sample_eggs], - ... python2_3_executable) - '/tmp/sample-eggs/demo-0.1-py2.3.egg' - - >>> [(d.project_name, d.version) for d in - ... zc.buildout.egglinker.distributions( - ... ['demo==0.1'], [sample_eggs], python2_3_executable)] - [('demo', '0.1'), ('demoneeded', '1.0')] - - - >>> scripts = zc.buildout.egglinker.scripts( - ... ['demo==0.1'], bin, [sample_eggs], - ... executable=python2_4_executable) - - >>> f = open(os.path.join(bin, 'demo')) - >>> f.readline().strip() == '#!' + python2_4_executable - True - >>> print f.read(), - <BLANKLINE> - import sys - sys.path[0:0] = [ - '/tmp/sample-eggs/dist/demo-0.1-py2.4.egg', - '/tmp/sample-eggs/dist/demoneeded-1.0-py2.4.egg' - ] - <BLANKLINE> - import eggrecipedemo - <BLANKLINE> - if __name__ == '__main__': - eggrecipedemo.main() - - >>> zc.buildout.egglinker.path(['demo==0.1'], [sample_eggs], - ... python2_4_executable) - ... # doctest: +NORMALIZE_WHITESPACE - ['/tmp/sample-eggs/dist/demo-0.1-py2.4.egg', - '/tmp/sample-eggs/dist/demoneeded-1.0-py2.4.egg'] - - >>> shutil.rmtree(bin) - diff --git a/src/zc/buildout/testing.py b/src/zc/buildout/testing.py index f58180790f2e0b5c54540150a251a3bc89bf45ad..8e1ff493f84d27d08dcec5c6277b46ccc6b75657 100644 --- a/src/zc/buildout/testing.py +++ b/src/zc/buildout/testing.py @@ -16,11 +16,13 @@ $Id$ """ -import ConfigParser, os, re, shutil, sys, tempfile, unittest + +import BaseHTTPServer, ConfigParser, os, random, re, shutil, socket, sys +import tempfile, threading, time, urllib2, unittest + from zope.testing import doctest, renormalizing import pkg_resources - def cat(dir, *names): path = os.path.join(dir, *names) print open(path).read(), @@ -52,6 +54,9 @@ def system(command, input=''): i.close() return o.read() +def get(url): + return urllib2.urlopen(url).read() + def buildoutSetUp(test, clear_home=True): if clear_home: # we both need to make sure that HOME isn't set and be prepared @@ -91,6 +96,7 @@ def buildoutSetUp(test, clear_home=True): mkdir = mkdir, write = write, system = system, + get = get, __original_wd__ = os.getcwd(), )) @@ -133,14 +139,25 @@ def create_sample_eggs(test, executable=sys.executable): test.globs['sample_eggs'] = os.path.join(sample, 'dist') write(sample, 'README.txt', '') - write(sample, 'eggrecipedemobeeded.py', 'y=1\n') + for i in (0, 1): + write(sample, 'eggrecipedemobeeded.py', 'y=%s\n' % i) + write( + sample, 'setup.py', + "from setuptools import setup\n" + "setup(name='demoneeded', py_modules=['eggrecipedemobeeded']," + " zip_safe=True, version='1.%s')\n" + % i + ) + runsetup(sample, executable) + write( sample, 'setup.py', "from setuptools import setup\n" - "setup(name='demoneeded', py_modules=['eggrecipedemobeeded']," - " zip_safe=True, version='1.0')\n" - ) + "setup(name='other', zip_safe=True, version='1.0', " + "py_modules=['eggrecipedemobeeded'])\n" + ) runsetup(sample, executable) + os.remove(os.path.join(sample, 'eggrecipedemobeeded.py')) for i in (1, 2, 3): @@ -170,5 +187,131 @@ def multi_python(test): create_sample_eggs(test, executable=p24) test.globs['python2_3_executable'] = p23 test.globs['python2_4_executable'] = p24 + + +def make_tree(test): + sample_eggs = test.globs['sample_eggs'] + tree = dict( + [(n, open(os.path.join(sample_eggs, n), 'rb').read()) + for n in os.listdir(sample_eggs) + ]) + tree['index'] = {} + return tree - +class Server(BaseHTTPServer.HTTPServer): + + def __init__(self, tree, *args): + BaseHTTPServer.HTTPServer.__init__(self, *args) + self.tree = tree + + __run = True + def serve_forever(self): + while self.__run: + self.handle_request() + + def handle_error(self, *_): + self.__run = False + +class Handler(BaseHTTPServer.BaseHTTPRequestHandler): + + def __init__(self, request, address, server): + self.tree = server.tree + BaseHTTPServer.BaseHTTPRequestHandler.__init__( + self, request, address, server) + + def do_GET(self): + if '__stop__' in self.path: + raise SystemExit + + tree = self.tree + for name in self.path.split('/'): + if not name: + continue + tree = tree.get(name) + if tree is None: + self.send_response(404, 'Not Found') + out = '<html><body>Not Found</body></html>' + self.send_header('Content-Length', str(len(out))) + self.send_header('Content-Type', 'text/html') + self.end_headers() + self.wfile.write(out) + return + + self.send_response(200) + if isinstance(tree, dict): + out = ['<html><body>\n'] + items = tree.items() + items.sort() + for name, v in items: + if isinstance(v, dict): + name += '/' + out.append('<a href="%s">%s</a><br>\n' % (name, name)) + out.append('</body></html>\n') + out = ''.join(out) + self.send_header('Content-Length', str(len(out))) + self.send_header('Content-Type', 'text/html') + else: + out = tree + self.send_header('Content-Length', len(out)) + if name.endswith('.egg'): + self.send_header('Content-Type', 'application/zip') + else: + self.send_header('Content-Type', 'text/html') + self.end_headers() + + self.wfile.write(out) + + def log_request(*s): + pass + +def _run(tree, port): + server_address = ('localhost', port) + httpd = Server(tree, server_address, Handler) + httpd.serve_forever() + +def get_port(): + for i in range(10): + port = random.randrange(20000, 30000) + s = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + try: + try: + s.connect(('localhost', port)) + except socket.error: + return port + finally: + s.close() + raise RuntimeError, "Can't find port" + +def start_server(tree): + port = get_port() + threading.Thread(target=_run, args=(tree, port)).start() + wait(port, up=True) + return port + +def stop_server(url): + try: + urllib2.urlopen(url+'__stop__') + except Exception: + pass + +def wait(port, up): + addr = 'localhost', port + for i in range(120): + time.sleep(0.25) + try: + s = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + s.connect(addr) + s.close() + if up: + break + except socket.error, e: + if e[0] not in (errno.ECONNREFUSED, errno.ECONNRESET): + raise + s.close() + if not up: + break + else: + if up: + raise + else: + raise SystemError("Couln't stop server") diff --git a/src/zc/buildout/tests.py b/src/zc/buildout/tests.py index 5a63f1f15d51acbc59a6ba1d1bcbf850192b351c..419cc27aab24eebd1fff1e9ebc083c6218ee245e 100644 --- a/src/zc/buildout/tests.py +++ b/src/zc/buildout/tests.py @@ -66,21 +66,26 @@ It is an error to create a variable-reference cycle: def linkerSetUp(test): zc.buildout.testing.buildoutSetUp(test, clear_home=False) zc.buildout.testing.multi_python(test) + test.globs['link_server'] = ( + 'http://localhost:%s/' + % zc.buildout.testing.start_server(zc.buildout.testing.make_tree(test)) + ) def linkerTearDown(test): shutil.rmtree(test.globs['_sample_eggs_container']) zc.buildout.testing.buildoutTearDown(test) + zc.buildout.testing.stop_server(test.globs['link_server']) + def buildoutTearDown(test): shutil.rmtree(test.globs['extensions']) shutil.rmtree(test.globs['home']) zc.buildout.testing.buildoutTearDown(test) - class PythonNormalizing(renormalizing.RENormalizing): def _transform(self, want, got): - if '/xyzsample-eggs/' in want: + if '/xyzsample-install/' in want: got = got.replace('-py2.4.egg', '-py2.3.egg') firstg = got.split('\n')[0] firstw = want.split('\n')[0] @@ -149,14 +154,15 @@ def test_suite(): ), doctest.DocFileSuite( - 'egglinker.txt', 'easy_install.txt', + 'easy_install.txt', setUp=linkerSetUp, tearDown=linkerTearDown, checker=PythonNormalizing([ - (re.compile("'%(sep)s\S+sample-eggs%(sep)s(dist%(sep)s)?" + (re.compile("'%(sep)s\S+sample-install%(sep)s(dist%(sep)s)?" % dict(sep=os.path.sep)), '/sample-eggs/'), - (re.compile("(- demo(needed)?-\d[.]\d-py)\d[.]\d[.]egg"), + (re.compile("(- (demo(needed)?|other)" + "-\d[.]\d-py)\d[.]\d[.]egg"), '\\1V.V.egg'), ]), ), diff --git a/testrunnerrecipe/setup.py b/testrunnerrecipe/setup.py index 8238d258cf88acf633f54f29458a71d8a988eef4..71089eb323472e10221651d047f526a4a45e611d 100644 --- a/testrunnerrecipe/setup.py +++ b/testrunnerrecipe/setup.py @@ -7,7 +7,7 @@ setup( include_package_data = True, package_dir = {'':'src'}, namespace_packages = ['zc', 'zc.recipe'], - install_requires = ['zc.buildout', 'zope.testing'], + install_requires = ['zc.buildout', 'zope.testing', 'setuptools'], dependency_links = ['http://download.zope.org/distribution/'], test_suite = 'zc.recipe.testrunner.tests.test_suite', author = "Jim Fulton", diff --git a/testrunnerrecipe/src/zc/recipe/testrunner/README.txt b/testrunnerrecipe/src/zc/recipe/testrunner/README.txt index 9dfc16150ab36842586389e8434107f7633bad70..49017cccd558c2eb97125723212dd8a7730f3b14 100644 --- a/testrunnerrecipe/src/zc/recipe/testrunner/README.txt +++ b/testrunnerrecipe/src/zc/recipe/testrunner/README.txt @@ -75,6 +75,7 @@ develop egg and to create the test script: ... [buildout] ... develop = demo demo2 ... parts = testdemo + ... offline = true ... ... [testdemo] ... recipe = zc.recipe.testrunner @@ -87,6 +88,8 @@ develop egg and to create the test script: Note that we specified both demo and demo2 in the distributions section and that we put them on separate lines. +We also specified the offline option to run the buildout in offline mode. + Now when we run the buildout: >>> import os @@ -113,6 +116,7 @@ script will get it's name from the part: ... [buildout] ... develop = demo ... parts = testdemo + ... offline = true ... ... [testdemo] ... recipe = zc.recipe.testrunner diff --git a/testrunnerrecipe/src/zc/recipe/testrunner/__init__.py b/testrunnerrecipe/src/zc/recipe/testrunner/__init__.py index 6446b41140ab03a3044e66da8b59c64dc30e43e9..af77c01ef6cee704c959e11a62d7f72b301fc4cf 100644 --- a/testrunnerrecipe/src/zc/recipe/testrunner/__init__.py +++ b/testrunnerrecipe/src/zc/recipe/testrunner/__init__.py @@ -17,7 +17,8 @@ $Id$ """ import os, sys -import zc.buildout.egglinker +import pkg_resources +import zc.buildout.easy_install class TestRunner: @@ -30,25 +31,28 @@ class TestRunner: ) options['_e'] = buildout['buildout']['eggs-directory'] options['_d'] = buildout['buildout']['develop-eggs-directory'] + python = options.get('python', buildout['buildout']['python']) + options['executable'] = buildout[python]['executable'] def install(self): - distributions = [ - req.strip() - for req in self.options['distributions'].split('\n') - if req.split() - ] - path = zc.buildout.egglinker.path( - distributions+['zope.testing'], - [self.options['_d'], self.options['_e']], + options = self.options + requirements = [r.strip() + for r in options['distributions'].split('\n') + if r.strip()] + + ws = zc.buildout.easy_install.working_set( + requirements+['zope.testing'], + executable = options['executable'], + path=[options['_d'], options['_e']] ) - locations = [zc.buildout.egglinker.location( - distribution, - [self.options['_d'], self.options['_e']]) - for distribution in distributions] - script = self.options['script'] + path = [dist.location for dist in ws] + locations = [dist.location for dist in ws + if dist.project_name != 'zope.testing'] + + script = options['script'] open(script, 'w').write(tests_template % dict( - PYTHON=sys.executable, + PYTHON=options['executable'], PATH="',\n '".join(path), TESTPATH="',\n '--test-path', '".join(locations), )) diff --git a/testrunnerrecipe/src/zc/recipe/testrunner/tests.py b/testrunnerrecipe/src/zc/recipe/testrunner/tests.py index a828e7cbb426ee903dbaaa928bcfb1f69b4bb0df..1638dc281b34d71a8f0d521e65fa5459f0830f70 100644 --- a/testrunnerrecipe/src/zc/recipe/testrunner/tests.py +++ b/testrunnerrecipe/src/zc/recipe/testrunner/tests.py @@ -17,7 +17,8 @@ import pkg_resources import zc.buildout.testing import unittest -from zope.testing import doctest, renormalizing +import zope.testing +from zope.testing import doctest def dirname(d, level=1): if level == 0: @@ -29,6 +30,11 @@ def setUp(test): open(os.path.join(test.globs['sample_buildout'], 'eggs', 'zc.recipe.testrunner.egg-link'), 'w').write(dirname(__file__, 4)) + + # XXX assumes that zope.testing egg is a directory + open(os.path.join(test.globs['sample_buildout'], + 'eggs', 'zope.testing.egg-link'), + 'w').write(dirname(zope.testing.__file__, 3)) def tearDown(test): zc.buildout.testing.buildoutTearDown(test) diff --git a/todo.txt b/todo.txt index 9852310fcafca30346c968289be8a39d870a1ea8..7ebb3c2bcc1cb7b1cd1fe5f5d9387ffa00977459 100644 --- a/todo.txt +++ b/todo.txt @@ -1,7 +1,17 @@ +- tests + + - distribution dependency links + + - offline mode (there is an indirect test in the testrunner tests) + - Windows support - Load from urls +- control python for develop (probbaly a new recipe) + +- proper handling of extras + - Common recipes - configure-make-make-install @@ -14,10 +24,6 @@ - Python -- Need to better understand the way upgrading works in setuptools. - -- Offline mode - - Some way to freeze versions so we can have reproducable buildouts. Maybe simple approach: @@ -27,6 +33,8 @@ - Egg recipe has option to specify dependencies. When used, don't automatically fetch newer data. +- Option to search python path for distros + - Part dependencies - custom uninstall @@ -34,14 +42,21 @@ - Fix develop so thet ordinary eggs fetched as dependencies end up in eggs directory. + "fixed" except that fix is ineffective due to setuptools bug. :( + + - spelling :) - document recipe initialization order + + Issues -- Want to be able to control whether eggs get unzipped when they are - installed. This requires looking at a distribution after it's - installed and unzipping it if it's zipped. - +- Should we include setuptools and buildout eggs for buildout process + in environment when searching for requirements? + +- We don't want to look for new versions of setuptools all the time. + For now, we always use a local dist if there is one. Needs more + thought.