Commit fdb75a6e authored by jim's avatar jim

Added a script-generation option to generate relative paths for eggs

in scripts when both the script and the eggs have a common base directory.


git-svn-id: http://svn.zope.org/repos/main/zc.buildout/trunk@98175 62d5b8a3-27da-0310-9561-8e5933582275
parent fd34d441
......@@ -108,7 +108,7 @@ class AllowHostsPackageIndex(setuptools.package_index.PackageIndex):
if FILE_SCHEME(url):
return True
return setuptools.package_index.PackageIndex.url_ok(self, url, False)
_indexes = {}
def _get_index(executable, index_url, find_links, allow_hosts=('*',)):
......@@ -122,7 +122,7 @@ def _get_index(executable, index_url, find_links, allow_hosts=('*',)):
index = AllowHostsPackageIndex(
index_url, hosts=allow_hosts, python=_get_version(executable)
)
if find_links:
index.add_find_links(find_links)
......@@ -152,7 +152,7 @@ class Installer:
_use_dependency_links = True
_allow_picked_versions = True
_always_unzip = False
def __init__(self,
dest=None,
links=(),
......@@ -204,7 +204,7 @@ class Installer:
if not dists:
logger.debug('We have no distributions for %s that satisfies %r.',
req.project_name, str(req))
return None, self._obtain(req, source)
# Note that dists are sorted from best to worst, as promised by
......@@ -242,7 +242,7 @@ class Installer:
# newer ones. Let's find out which ones are available and see if
# any are newer. We only do this if we're willing to install
# something, which is only true if dest is not None:
if self._dest is not None:
best_available = self._obtain(req, source)
else:
......@@ -282,7 +282,7 @@ class Installer:
best_available.parsed_version
):
return None, best_available
logger.debug(
'We have the best distribution that satisfies %r.',
str(req))
......@@ -326,10 +326,10 @@ class Installer:
args += (dict(os.environ, PYTHONPATH=path), )
sys.stdout.flush() # We want any pending output first
if is_jython:
exit_code = subprocess.Popen(
[_safe_arg(self._executable)] + list(args),
[_safe_arg(self._executable)] + list(args),
env=extra_env).wait()
else:
exit_code = os.spawnle(
......@@ -343,7 +343,7 @@ class Installer:
)
for project in env:
dists.extend(env[project])
if exit_code:
logger.error(
"An error occured when trying to install %s."
......@@ -388,18 +388,18 @@ class Installer:
[newloc],
python=_get_version(self._executable),
)[d.project_name]
result.append(d)
return result
finally:
shutil.rmtree(tmp)
def _obtain(self, requirement, source=None):
# initialize out index for this project:
index = self._index
if index.obtain(requirement) is None:
# Nothing is available.
return None
......@@ -440,7 +440,7 @@ class Installer:
if len(best) == 1:
return best[0]
if self._download_cache:
for dist in best:
if (realpath(os.path.dirname(dist.location))
......@@ -467,7 +467,7 @@ class Installer:
# to the download cache
shutil.copy2(new_location, tmp)
new_location = os.path.join(tmp, os.path.basename(new_location))
return dist.clone(location=new_location)
def _get_dist(self, requirement, ws, always_unzip):
......@@ -477,7 +477,7 @@ class Installer:
# Maybe an existing dist is already the best dist that satisfies the
# requirement
dist, avail = self._satisfied(requirement)
if dist is None:
if self._dest is not None:
logger.info(*__doing__)
......@@ -554,7 +554,7 @@ class Installer:
self._env.scan([self._dest])
dist = self._env.best_match(requirement, ws)
logger.info("Got %s.", dist)
logger.info("Got %s.", dist)
else:
dists = [dist]
......@@ -570,7 +570,7 @@ class Installer:
logger.debug('Adding find link %r from %s', link, dist)
self._links.append(link)
self._index = _get_index(self._executable,
self._index_url, self._links,
self._index_url, self._links,
self._allow_hosts)
for dist in dists:
......@@ -619,7 +619,7 @@ class Installer:
logger.error("The version, %s, is not consistent with the "
"requirement, %r.", version, str(requirement))
raise IncompatibleVersionError("Bad version", version)
requirement = pkg_resources.Requirement.parse(
"%s ==%s" % (requirement.project_name, version))
......@@ -637,7 +637,7 @@ class Installer:
requirements = [self._constrain(pkg_resources.Requirement.parse(spec))
for spec in specs]
if working_set is None:
ws = pkg_resources.WorkingSet([])
......@@ -668,10 +668,10 @@ class Installer:
else:
logger.debug('Adding required %r', str(requirement))
_log_requirement(ws, requirement)
for dist in self._get_dist(requirement, ws, self._always_unzip
):
ws.add(dist)
self._maybe_add_setuptools(ws, dist)
except pkg_resources.VersionConflict, err:
......@@ -724,7 +724,7 @@ class Installer:
% os.path.basename(dist.location)
)
base = os.path.dirname(setups[0])
setup_cfg = os.path.join(base, 'setup.cfg')
if not os.path.exists(setup_cfg):
f = open(setup_cfg, 'w')
......@@ -797,7 +797,7 @@ def install(specs, dest,
path=None, working_set=None, newest=True, versions=None,
use_dependency_links=None, allow_hosts=('*',)):
installer = Installer(dest, links, index, executable, always_unzip, path,
newest, versions, use_dependency_links,
newest, versions, use_dependency_links,
allow_hosts=allow_hosts)
return installer.install(specs, working_set)
......@@ -810,7 +810,7 @@ def build(spec, dest, build_ext,
versions, allow_hosts=allow_hosts)
return installer.build(spec, build_ext)
def _rm(*paths):
for path in paths:
......@@ -828,10 +828,10 @@ def _copyeggs(src, dest, suffix, undo):
_rm(new)
os.rename(os.path.join(src, name), new)
result.append(new)
assert len(result) == 1, str(result)
undo.pop()
return result[0]
def develop(setup, dest,
......@@ -843,7 +843,7 @@ def develop(setup, dest,
setup = os.path.join(directory, 'setup.py')
else:
directory = os.path.dirname(setup)
undo = []
try:
if build_ext:
......@@ -873,7 +873,7 @@ def develop(setup, dest,
))
tmp3 = tempfile.mkdtemp('build', dir=dest)
undo.append(lambda : shutil.rmtree(tmp3))
undo.append(lambda : shutil.rmtree(tmp3))
args = [
zc.buildout.easy_install._safe_arg(tsetup),
......@@ -893,15 +893,16 @@ def develop(setup, dest,
if is_jython:
assert subprocess.Popen([_safe_arg(executable)] + args).wait() == 0
else:
assert os.spawnl(os.P_WAIT, executable, _safe_arg (executable), *args) == 0
assert os.spawnl(os.P_WAIT, executable, _safe_arg(executable),
*args) == 0
return _copyeggs(tmp3, dest, '.egg-link', undo)
finally:
undo.reverse()
[f() for f in undo]
def working_set(specs, executable, path):
return install(specs, None, executable=executable, path=path)
......@@ -911,11 +912,13 @@ def scripts(reqs, working_set, executable, dest,
arguments='',
interpreter=None,
initialization='',
relative_paths=False,
):
path = [dist.location for dist in working_set]
path.extend(extra_paths)
path = repr(path)[1:-1].replace(', ', ',\n ')
path = map(realpath, path)
generated = []
if isinstance(reqs, str):
......@@ -938,7 +941,7 @@ def scripts(reqs, working_set, executable, dest,
)
else:
entry_points.append(req)
for name, module_name, attrs in entry_points:
if scripts is not None:
sname = scripts.get(name)
......@@ -948,19 +951,87 @@ def scripts(reqs, working_set, executable, dest,
sname = name
sname = os.path.join(dest, sname)
spath, rpsetup = _relative_path_and_setup(sname, path, relative_paths)
generated.extend(
_script(module_name, attrs, path, sname, executable, arguments,
initialization)
_script(module_name, attrs, spath, sname, executable, arguments,
initialization, rpsetup)
)
if interpreter:
sname = os.path.join(dest, interpreter)
generated.extend(_pyscript(path, sname, executable))
spath, rpsetup = _relative_path_and_setup(sname, path, relative_paths)
generated.extend(_pyscript(spath, sname, executable, rpsetup))
return generated
def _relative_path_and_setup(sname, path, relative_paths):
if relative_paths:
sname = os.path.abspath(sname)
spath = ',\n '.join(
[_relativitize(path_item, sname, relative_paths)
for path_item in path]
)
rpsetup = relative_paths_setup
else:
spath = repr(path)[1:-1].replace(', ', ',\n ')
rpsetup = ''
return spath, rpsetup
def _relative_depth(common, path):
n = 0
while 1:
dirname = os.path.dirname(path)
if dirname == path:
raise AssertionError("dirname of %s is the same" % dirname)
if dirname == common:
break
n += 1
path = dirname
return n
def _relative_path(common, path):
r = []
while 1:
dirname, basename = os.path.split(path)
r.append(basename)
if dirname == common:
break
if dirname == path:
raise AssertionError("dirname of %s is the same" % dirname)
path = dirname
r.reverse()
return os.path.join(*r)
def _relativitize(path, script, relative_paths):
if path == script:
raise AssertionError("path == script")
common = os.path.dirname(os.path.commonprefix([path, script]))
if (common == relative_paths or
common.startswith(os.path.join(relative_paths, ''))
):
return "join(dirname(%s, __file__), %r)" % (
_relative_depth(common, script), _relative_path(common, path)
)
else:
return repr(path)
relative_paths_setup = """
import os
def dirname(n, path):
while n >= 0:
n -= 1
path = os.path.dirname(path)
return path
join = os.path.join
"""
def _script(module_name, attrs, path, dest, executable, arguments,
initialization):
initialization, rsetup):
generated = []
script = dest
if is_win32:
......@@ -973,6 +1044,7 @@ def _script(module_name, attrs, path, dest, executable, arguments,
attrs = attrs,
arguments = arguments,
initialization = initialization,
relative_paths_setup = rsetup,
)
changed = not (os.path.exists(dest) and open(dest).read() == contents)
......@@ -984,7 +1056,7 @@ def _script(module_name, attrs, path, dest, executable, arguments,
# Only write it if it's different.
open(exe, 'wb').write(new_data)
generated.append(exe)
if changed:
open(dest, 'w').write(contents)
logger.info("Generated script %r.", script)
......@@ -993,7 +1065,7 @@ def _script(module_name, attrs, path, dest, executable, arguments,
os.chmod(dest, 0755)
except (AttributeError, os.error):
pass
generated.append(dest)
return generated
......@@ -1002,9 +1074,10 @@ if is_jython and jython_os_name == 'linux':
else:
script_header = '#!%(python)s'
script_template = script_header + '''\
script_template = script_header + '''\
%(relative_paths_setup)s
import sys
sys.path[0:0] = [
%(path)s,
......@@ -1017,7 +1090,7 @@ if __name__ == '__main__':
'''
def _pyscript(path, dest, executable):
def _pyscript(path, dest, executable, rsetup):
generated = []
script = dest
if is_win32:
......@@ -1026,6 +1099,7 @@ def _pyscript(path, dest, executable):
contents = py_script_template % dict(
python = _safe_arg(executable),
path = path,
relative_paths_setup = rsetup,
)
changed = not (os.path.exists(dest) and open(dest).read() == contents)
......@@ -1051,8 +1125,9 @@ def _pyscript(path, dest, executable):
py_script_template = script_header + '''\
%(relative_paths_setup)s
import sys
sys.path[0:0] = [
%(path)s,
]
......@@ -1067,7 +1142,7 @@ if len(sys.argv) > 1:
_interactive = True
elif _opt == '-c':
exec _val
if _args:
sys.argv[:] = _args
execfile(sys.argv[0])
......@@ -1076,7 +1151,7 @@ if _interactive:
import code
code.interact(banner="", local=globals())
'''
runsetup_template = """
import sys
sys.path.insert(0, %(setupdir)r)
......@@ -1124,7 +1199,7 @@ def _log_requirement(ws, req):
for dist in ws:
if req in dist.requires():
logger.debug(" required by %s." % dist)
def _fix_file_links(links):
for link in links:
if link.startswith('file://') and link[-1] != '/':
......@@ -1150,7 +1225,7 @@ def redo_pyc(egg):
filepath = os.path.join(dirpath, filename)
if not (os.path.exists(filepath+'c')
or os.path.exists(filepath+'o')):
# If it wasn't compiled, it may not be compilable
# If it wasn't compiled, it may not be compilable
continue
# OK, it looks like we should try to compile.
......@@ -1171,9 +1246,9 @@ def redo_pyc(egg):
if __debug__:
args.append('-O')
args.extend(['-m', 'py_compile', _safe_arg(filepath)])
if is_jython:
subprocess.call([sys.executable, args])
else:
os.spawnv(os.P_WAIT, sys.executable, args)
......@@ -8,7 +8,7 @@ level that is similar to easy_install, with a few exceptions:
- By default, we look for new packages *and* the packages that
they depend on. This is somewhat like (and uses) the --upgrade
option of easy_install, except that we also upgrade required
packages.
packages.
- If the highest-revision package satisfying a specification is
already present, then we don't try to get another one. This saves a
......@@ -50,7 +50,7 @@ index
alternate index with this option. If you use the links option and
if the links point to the needed distributions, then the index can
be anything and will be largely ignored. In the examples, here,
we'll just point to an empty directory on our link server. This
we'll just point to an empty directory on our link server. This
will make our examples run a little bit faster.
executable
......@@ -89,6 +89,11 @@ use_dependency_links
for using dependency_links in preference to other
locations. Defaults to true.
relative_paths
Adjust egg paths so they are relative to the script path. This
allows scripts to work when scripts and eggs are moved, as long as
they are both moved in the same way.
The install method returns a working set containing the distributions
needed to meet the given requirements.
......@@ -116,10 +121,10 @@ Let's make a directory and install the demo egg to it, using the demo:
>>> ws = zc.buildout.easy_install.install(
... ['demo==0.2'], dest,
... links=[link_server], index=link_server+'index/')
We requested version 0.2 of the demo distribution to be installed into
the destination server. We specified that we should search for links
on the link server and that we should use the (empty) link server
on the link server and that we should use the (empty) link server
index directory as a package index.
The working set contains the distributions we retrieved.
......@@ -220,7 +225,7 @@ can be useful when debugging.
>>> ls(dest)
d demo-0.3-py2.4.egg
d demoneeded-1.1-py2.4.egg
>>> rmdir(dest)
>>> dest = tmpdir('sample-install')
>>> ws = zc.buildout.easy_install.install(
......@@ -314,7 +319,7 @@ we'll get an error:
>>> handler.clear()
If no versions are specified, a debugging message will be output
If no versions are specified, a debugging message will be output
reporting that a version was picked automatically:
>>> ws = zc.buildout.easy_install.install(
......@@ -402,7 +407,7 @@ dependencies. This option is called dependency_links. Buildout has its
own notion of where to look for dependencies, but it also uses the
setup tools dependency_links information if it's available.
Let's demo this by creating an egg that specifies dependency_links.
Let's demo this by creating an egg that specifies dependency_links.
To begin, let's create a new egg repository. This repository hold a
newer version of the 'demoneeded' egg than the sample repository does.
......@@ -414,7 +419,7 @@ newer version of the 'demoneeded' egg than the sample repository does.
Turn on logging on this server so that we can see when eggs are pulled
from it.
>>> get(link_server2 + 'enable_server_logging')
GET 200 /enable_server_logging
''
......@@ -423,7 +428,7 @@ Now we can create an egg that specifies that its dependencies are
found on this server.
>>> repoloc = tmpdir('repo2')
>>> create_egg('hasdeps', '1.0', repoloc,
>>> create_egg('hasdeps', '1.0', repoloc,
... install_requires = "'demoneeded'",
... dependency_links = [link_server2])
......@@ -549,7 +554,7 @@ the four arguments we passed were:
2. A working set,
3. The Python executable to use, and
3. The Python executable to use, and
3. The destination directory.
......@@ -559,10 +564,10 @@ The bin directory now contains a generated script:
- demo
The return value is a list of the scripts generated:
>>> import os, sys
>>> if sys.platform == 'win32':
... scripts == [os.path.join(bin, 'demo.exe'),
... scripts == [os.path.join(bin, 'demo.exe'),
... os.path.join(bin, 'demo-script.py')]
... else:
... scripts == [os.path.join(bin, 'demo')]
......@@ -762,7 +767,7 @@ You can also pass script initialization code:
>>> scripts = zc.buildout.easy_install.scripts(
... ['demo'], ws, sys.executable, bin, dict(demo='run'),
... arguments='1, 2',
... arguments='1, 2',
... initialization='import os\nos.chdir("foo")')
>>> cat(bin, 'run') # doctest: +NORMALIZE_WHITESPACE
......@@ -781,6 +786,107 @@ You can also pass script initialization code:
if __name__ == '__main__':
eggrecipedemo.main(1, 2)
Relative paths
--------------
Sometimes, you want to be able to move a buildout directory around and
have scripts still work without having to rebuild them. We can
control this using the relative_paths option to install. You need
to pass a common base directory of the scripts and eggs:
>>> bo = tmpdir('bo')
>>> mkdir(bo, 'eggs')
>>> mkdir(bo, 'bin')
>>> mkdir(bo, 'other')
>>> ws = zc.buildout.easy_install.install(
... ['demo'], join(bo, 'eggs'), links=[link_server],
... index=link_server+'index/')
>>> scripts = zc.buildout.easy_install.scripts(
... ['demo'], ws, sys.executable, join(bo, 'bin'), dict(demo='run'),
... extra_paths=[os.path.sep+'foo', join(bo, 'bar')],
... interpreter='py',
... relative_paths=bo)
>>> cat(bo, 'bin', 'run')
#!/usr/local/bin/python2.4
<BLANKLINE>
import os
<BLANKLINE>
def dirname(n, path):
while n >= 0:
n -= 1
path = os.path.dirname(path)
return path
<BLANKLINE>
join = os.path.join
<BLANKLINE>
import sys
sys.path[0:0] = [
join(dirname(1, __file__), 'eggs/demo-0.3-py2.4.egg'),
join(dirname(1, __file__), 'eggs/demoneeded-1.1-py2.4.egg'),
'/foo',
join(dirname(1, __file__), 'bar'),
]
<BLANKLINE>
import eggrecipedemo
<BLANKLINE>
if __name__ == '__main__':
eggrecipedemo.main()
Note that the extra path we specified that was outside the directory
passed as relative_paths wasn't converted to a relative path.
Of course, running the script works:
>>> print system(join(bo, 'bin', 'run')),
3 1
We specified an interpreter and its paths are adjusted too:
>>> cat(bo, 'bin', 'py')
#!/usr/local/bin/python2.4
<BLANKLINE>
<BLANKLINE>
import os
<BLANKLINE>
def dirname(n, path):
while n >= 0:
n -= 1
path = os.path.dirname(path)
return path
<BLANKLINE>
join = os.path.join
<BLANKLINE>
import sys
<BLANKLINE>
sys.path[0:0] = [
join(dirname(1, __file__), 'eggs/demo-0.3-py2.4.egg'),
join(dirname(1, __file__), 'eggs/demoneeded-1.1-py2.4.egg'),
'/foo',
join(dirname(1, __file__), 'bar'),
]
<BLANKLINE>
_interactive = True
if len(sys.argv) > 1:
import getopt
_options, _args = getopt.getopt(sys.argv[1:], 'ic:')
_interactive = False
for (_opt, _val) in _options:
if _opt == '-i':
_interactive = True
elif _opt == '-c':
exec _val
<BLANKLINE>
if _args:
sys.argv[:] = _args
execfile(sys.argv[0])
<BLANKLINE>
if _interactive:
import code
code.interact(banner="", local=globals())
Handling custom build options for extensions provided in source distributions
-----------------------------------------------------------------------------
......@@ -816,7 +922,7 @@ index
alternate index with this option. If you use the links option and
if the links point to the needed distributions, then the index can
be anything and will be largely ignored. In the examples, here,
we'll just point to an empty directory on our link server. This
we'll just point to an empty directory on our link server. This
will make our examples run a little bit faster.
executable
......@@ -876,12 +982,12 @@ Now, we can use the build function to create an egg from the source
distribution:
>>> zc.buildout.easy_install.build(
... 'extdemo', dest,
... 'extdemo', dest,
... {'include-dirs': os.path.join(sample_buildout, 'include')},
... links=[link_server], index=link_server+'index/')
['/sample-install/extdemo-1.4-py2.4-unix-i686.egg']
The function returns the list of eggs
The function returns the list of eggs
Now if we look in our destination directory, we see we have an extdemo egg:
......@@ -920,7 +1026,7 @@ function to clear the index cache:
If we run build with newest set to False, we won't get an update:
>>> zc.buildout.easy_install.build(
... 'extdemo', dest,
... 'extdemo', dest,
... {'include-dirs': os.path.join(sample_buildout, 'include')},
... links=[link_server], index=link_server+'index/',
... newest=False)
......@@ -937,7 +1043,7 @@ But if we run it with the default True setting for newest, then we'll
get an updated egg:
>>> zc.buildout.easy_install.build(
... 'extdemo', dest,
... 'extdemo', dest,
... {'include-dirs': os.path.join(sample_buildout, 'include')},
... links=[link_server], index=link_server+'index/')
['/sample-install/extdemo-1.5-py2.4-unix-i686.egg']
......@@ -960,7 +1066,7 @@ first:
... remove(dest, name)
>>> zc.buildout.easy_install.build(
... 'extdemo', dest,
... 'extdemo', dest,
... {'include-dirs': os.path.join(sample_buildout, 'include')},
... links=[link_server], index=link_server+'index/',
... versions=dict(extdemo='1.4'))
......@@ -974,7 +1080,7 @@ Handling custom build options for extensions in develop eggs
The develop function is similar to the build function, except that,
rather than building an egg from a source directory containing a
setup.py script.
setup.py script.
The develop function takes 2 positional arguments:
......@@ -1008,7 +1114,7 @@ Now, we can use the develop function to create a develop egg from the source
distribution:
>>> zc.buildout.easy_install.develop(
... extdemo, dest,
... extdemo, dest,
... {'include-dirs': os.path.join(sample_buildout, 'include')})
'/sample-install/extdemo.egg-link'
......@@ -1061,7 +1167,7 @@ We'll recreate our destination directory:
We'd like to see what is being fetched from the server, so we'll
enable server logging:
>>> get(link_server+'enable_server_logging')
GET 200 /enable_server_logging
''
......@@ -1070,7 +1176,7 @@ Now, if we install demo, and extdemo:
>>> ws = zc.buildout.easy_install.install(
... ['demo==0.2'], dest,
... links=[link_server], index=link_server+'index/',
... links=[link_server], index=link_server+'index/',
... always_unzip=True)
GET 200 /
GET 404 /index/demo/
......@@ -1081,7 +1187,7 @@ Now, if we install demo, and extdemo:
GET 404 /index/setuptools/
>>> zc.buildout.easy_install.build(
... 'extdemo', dest,
... 'extdemo', dest,
... {'include-dirs': os.path.join(sample_buildout, 'include')},
... links=[link_server], index=link_server+'index/')
GET 404 /index/extdemo/
......@@ -1103,7 +1209,7 @@ But we'll get distributions in the cache directory:
- extdemo-1.5.zip
The cache directory contains uninstalled distributions, such as zipped
eggs or source distributions.
eggs or source distributions.
Let's recreate our destination directory and clear the index cache:
......@@ -1115,7 +1221,7 @@ Now when we install the distributions:
>>> ws = zc.buildout.easy_install.install(
... ['demo==0.2'], dest,
... links=[link_server], index=link_server+'index/',
... links=[link_server], index=link_server+'index/',
... always_unzip=True)
GET 200 /
GET 404 /index/demo/
......@@ -1124,7 +1230,7 @@ Now when we install the distributions:
GET 404 /index/setuptools/
>>> zc.buildout.easy_install.build(
... 'extdemo', dest,
... 'extdemo', dest,
... {'include-dirs': os.path.join(sample_buildout, 'include')},
... links=[link_server], index=link_server+'index/')
GET 404 /index/extdemo/
......@@ -1142,7 +1248,7 @@ from the link server:
>>> ws = zc.buildout.easy_install.install(
... ['demo'], dest,
... links=[link_server], index=link_server+'index/',
... links=[link_server], index=link_server+'index/',
... always_unzip=True)
GET 200 /demo-0.3-py2.4.egg
......@@ -1184,7 +1290,7 @@ recreate the destination directory, and reinstall demo:
>>> ws = zc.buildout.easy_install.install(
... ['demo'], dest,
... links=[link_server], index=link_server+'index/',
... links=[link_server], index=link_server+'index/',
... always_unzip=True)
>>> ls(dest)
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment