Skip to content
Projects
Groups
Snippets
Help
Loading...
Help
Support
Keyboard shortcuts
?
Submit feedback
Contribute to GitLab
Sign in / Register
Toggle navigation
S
slapos.buildout
Project overview
Project overview
Details
Activity
Releases
Repository
Repository
Files
Commits
Branches
Tags
Contributors
Graph
Compare
Issues
0
Issues
0
List
Boards
Labels
Milestones
Merge Requests
0
Merge Requests
0
Analytics
Analytics
Repository
Value Stream
Wiki
Wiki
Snippets
Snippets
Members
Members
Collapse sidebar
Close sidebar
Activity
Graph
Create a new issue
Commits
Issue Boards
Open sidebar
Rafael Monnerat
slapos.buildout
Commits
db50caab
Commit
db50caab
authored
Aug 09, 2006
by
Jim Fulton
Browse files
Options
Browse Files
Download
Email Patches
Plain Diff
Added support controlling how eggs with extensions are built.
parent
93a6b845
Changes
10
Hide whitespace changes
Inline
Side-by-side
Showing
10 changed files
with
460 additions
and
11 deletions
+460
-11
src/zc/buildout/easy_install.py
src/zc/buildout/easy_install.py
+80
-4
src/zc/buildout/easy_install.txt
src/zc/buildout/easy_install.txt
+96
-1
src/zc/buildout/testing.py
src/zc/buildout/testing.py
+45
-0
src/zc/buildout/tests.py
src/zc/buildout/tests.py
+10
-3
zc.recipe.egg_/README.txt
zc.recipe.egg_/README.txt
+6
-0
zc.recipe.egg_/setup.py
zc.recipe.egg_/setup.py
+4
-1
zc.recipe.egg_/src/zc/recipe/egg/__init__.py
zc.recipe.egg_/src/zc/recipe/egg/__init__.py
+2
-1
zc.recipe.egg_/src/zc/recipe/egg/custom.py
zc.recipe.egg_/src/zc/recipe/egg/custom.py
+82
-0
zc.recipe.egg_/src/zc/recipe/egg/custom.txt
zc.recipe.egg_/src/zc/recipe/egg/custom.txt
+114
-0
zc.recipe.egg_/src/zc/recipe/egg/tests.py
zc.recipe.egg_/src/zc/recipe/egg/tests.py
+21
-1
No files found.
src/zc/buildout/easy_install.py
View file @
db50caab
...
...
@@ -20,10 +20,15 @@ installed.
$Id$
"""
import
logging
,
os
,
re
,
sys
import
pkg_resources
import
logging
,
os
,
re
,
tempfile
,
sys
import
pkg_resources
,
setuptools
.
command
.
setopt
import
zc.buildout
# XXX we could potentially speed this up quite a bit by keeping our
# own PackageIndex to analyse whether there are newer dists. A hitch
# is that the package index seems to go out of its way to only handle
# one Python version at a time. :(
logger
=
logging
.
getLogger
(
'zc.buildout.easy_install'
)
# Include buildout and setuptools eggs in paths
...
...
@@ -131,15 +136,15 @@ def _call_easy_install(spec, dest, links=(),
logger.debug('
Running
easy_install
:
\
n
%
s
"%s"
\
npath
=%
s
\
n
',
executable, '" "'.join(args), path)
args += (dict(PYTHONPATH=path), )
args += (dict(
os.environ,
PYTHONPATH=path), )
sys.stdout.flush() # We want any pending output first
exit_code = os.spawnle(os.P_WAIT, executable, executable, *args)
assert exit_code == 0
# We may overwrite distributions, so clear importer
# cache.
sys.path_importer_cache.clear()
assert exit_code == 0
def _get_dist(requirement, env, ws,
...
...
@@ -235,6 +240,77 @@ def install(specs, dest,
return ws
def _editable(spec, dest, links=(), index = None, executable=sys.executable):
prefix = sys.exec_prefix + os.path.sep
path = os.pathsep.join([p for p in sys.path if not p.startswith(prefix)])
args = (
'-c', 'from setuptools.command.easy_install import main; main()',
'-eb', dest)
if links:
args += ('-f', ' '.join(links))
if index:
args += ('-i', index)
level = logger.getEffectiveLevel()
if level > logging.DEBUG:
args += ('-q', )
elif level < logging.DEBUG:
args += ('-v', )
args += (spec, )
if level <= logging.DEBUG:
logger.debug('Running easy_install:
\
n
%s "
%
s
"
\
n
path=%s
\
n
',
executable, '"
"'.join(args), path)
args += (dict(os.environ, PYTHONPATH=path), )
sys.stdout.flush() # We want any pending output first
exit_code = os.spawnle(os.P_WAIT, executable, executable, *args)
assert exit_code == 0
def build(spec, dest, build_ext,
links=(), index=None,
executable=sys.executable,
path=None):
# XXX we're going to download and build the egg every stinking time.
# We need to not do that.
logger.debug('Building %r', spec)
path = path and path[:] or []
if dest is not None:
path.insert(0, dest)
path += buildout_and_setuptools_path
links = list(links) # make copy, because we may need to mutate
# For each spec, see if it is already installed. We create a working
# set to keep track of what we've collected and to make sue than the
# distributions assembled are consistent.
env = pkg_resources.Environment(path, python=_get_version(executable))
requirement = pkg_resources.Requirement.parse(spec)
dist = _satisfied(requirement, env)
if dist is not None:
return dist
# Get an editable version of the package to a temporary directory:
tmp = tempfile.mkdtemp('editable')
_editable(spec, tmp, links, index, executable)
setup_cfg = os.path.join(tmp, requirement.key, 'setup.cfg')
if not os.path.exists(setup_cfg):
f = open(setup_cfg, 'w')
f.close()
setuptools.command.setopt.edit_config(setup_cfg, dict(build_ext=build_ext))
# Now run easy_install for real:
_call_easy_install(
os.path.join(tmp, requirement.key),
dest, links, index, executable, True)
def working_set(specs, executable, path):
return install(specs, None, executable=executable, path=path)
...
...
src/zc/buildout/easy_install.txt
View file @
db50caab
...
...
@@ -18,7 +18,10 @@ easy_install command that provides some additional semantics:
look for additional distributions. We always give preference to
develop eggs.
The easy_install module provides a single method, install. The
- Distutils options for building extensions can be passed.
The easy_install module provides a method, install, for installing one
or more packages and their dependencies. The
install function takes 2 positional arguments:
- An iterable of setuptools requirement strings for the distributions
...
...
@@ -73,6 +76,7 @@ We have a link server that has a number of eggs:
<a href="demoneeded-1.0-py2.4.egg">demoneeded-1.0-py2.4.egg</a><br>
<a href="demoneeded-1.1-py2.3.egg">demoneeded-1.1-py2.3.egg</a><br>
<a href="demoneeded-1.1-py2.4.egg">demoneeded-1.1-py2.4.egg</a><br>
<a href="extdemo-1.4.tar.gz">extdemo-1.4.tar.gz</a><br>
<a href="index/">index/</a><br>
<a href="other-1.0-py2.3.egg">other-1.0-py2.3.egg</a><br>
<a href="other-1.0-py2.4.egg">other-1.0-py2.4.egg</a><br>
...
...
@@ -267,3 +271,94 @@ original script names to new script names.
>>> print system(os.path.join(bin, 'run')),
3 1
Handling custom build options for extensions
--------------------------------------------
Sometimes, we need to control how extension modules are built. The
build method provides this level of control. It takes a single
package specification, downloads a source distribution, and builds it
with specified custom build options.
The build method takes 3 positional arguments:
spec
A package specification
dest
A destination directory
build_ext
A dictionary of options to be passed to the distutils build_ext
command when building extensions.
It supports a number of optional keyword arguments:
links
a sequence of URLs, file names, or directories to look for
links to distributions,
index
The URL of an index server, or almost any other valid URL. :)
If not specified, the Python Package Index,
http://cheeseshop.python.org/pypi, is used. You can specify an
alternate index with this option. If you use the links option and
if the links point to the needed distributions, then the index can
be anything and will be largely ignored. In the examples, here,
we'll just point to an empty directory on our link server. This
will make our examples run a little bit faster.
executable
A path to a Python executable. Distributions will ne installed
using this executable and will be for the matching Python version.
path
A list of additional directories to search for locally-installed
distributions.
always_unzip
A flag indicating that newly-downloaded distributions should be
directories even if they could be installed as zip files.
Our link server included a source distribution that includes a simple
extension, extdemo.c::
#include <Python.h>
#include <extdemo.h>
static PyMethodDef methods[] = {};
PyMODINIT_FUNC
initextdemo(void)
{
PyObject *d;
d = Py_InitModule3("extdemo", methods, "");
PyDict_SetItemString(d, "val", PyInt_FromLong(EXTDEMO));
}
The extension depends on a system-dependnt include file, extdemo.h,
that defines a constant, EXTDEMO, that is exposed by the extension.
We'll add an include directory to our sample buildout and add the
needed include file to it:
>>> mkdir(sample_buildout, 'include')
>>> open(os.path.join(sample_buildout, 'include', 'extdemo.h'), 'w').write(
... "#define EXTDEMO 42\n")
Now, we can use the build function to create an egg from the source
distribution:
>>> zc.buildout.easy_install.build(
... 'extdemo', dest,
... {'include-dirs': os.path.join(sample_buildout, 'include')},
... links=[link_server], index=link_server+'index/')
Now if we look in our destination directory, we see we have an extdemo egg:
>>> ls(dest)
d demo-0.3-py2.4.egg
d demoneeded-1.1-py2.4.egg
d extdemo-1.4-py2.3-unix-i686.egg
src/zc/buildout/testing.py
View file @
db50caab
...
...
@@ -200,6 +200,49 @@ def multi_python(test):
test
.
globs
[
'python2_4_executable'
]
=
p24
extdemo_c
=
"""
#include <Python.h>
#include <extdemo.h>
static PyMethodDef methods[] = {};
PyMODINIT_FUNC
initextdemo(void)
{
PyObject *d;
d = Py_InitModule3("extdemo", methods, "");
PyDict_SetItemString(d, "val", PyInt_FromLong(EXTDEMO));
}
"""
extdemo_setup_py
=
"""
from distutils.core import setup, Extension
setup(name = "extdemo", version = "1.4", url="http://www.zope.org",
author="Demo", author_email="demo@demo.com",
ext_modules = [Extension('extdemo', ['extdemo.c'])],
)
"""
def
add_source_dist
(
test
):
import
tarfile
tmp
=
tempfile
.
mkdtemp
(
'test-sdist'
)
open
(
os
.
path
.
join
(
tmp
,
'extdemo.c'
),
'w'
).
write
(
extdemo_c
);
open
(
os
.
path
.
join
(
tmp
,
'setup.py'
),
'w'
).
write
(
extdemo_setup_py
);
open
(
os
.
path
.
join
(
tmp
,
'README'
),
'w'
).
write
(
""
);
open
(
os
.
path
.
join
(
tmp
,
'MANIFEST.in'
),
'w'
).
write
(
"include *.c
\
n
"
);
here
=
os
.
getcwd
()
os
.
chdir
(
tmp
)
status
=
os
.
spawnl
(
os
.
P_WAIT
,
sys
.
executable
,
sys
.
executable
,
os
.
path
.
join
(
tmp
,
'setup.py'
),
'-q'
,
'sdist'
)
os
.
chdir
(
here
)
assert
status
==
0
shutil
.
move
(
os
.
path
.
join
(
tmp
,
'dist'
,
'extdemo-1.4.tar.gz'
),
os
.
path
.
join
(
test
.
globs
[
'sample_eggs'
],
'extdemo-1.4.tar.gz'
),
)
def
make_tree
(
test
):
sample_eggs
=
test
.
globs
[
'sample_eggs'
]
tree
=
dict
(
...
...
@@ -266,6 +309,8 @@ class Handler(BaseHTTPServer.BaseHTTPRequestHandler):
self
.
send_header
(
'Content-Length'
,
len
(
out
))
if
name
.
endswith
(
'.egg'
):
self
.
send_header
(
'Content-Type'
,
'application/zip'
)
elif
name
.
endswith
(
'.gz'
):
self
.
send_header
(
'Content-Type'
,
'application/x-gzip'
)
else
:
self
.
send_header
(
'Content-Type'
,
'text/html'
)
self
.
end_headers
()
...
...
src/zc/buildout/tests.py
View file @
db50caab
...
...
@@ -141,6 +141,12 @@ def linkerSetUp(test):
zc
.
buildout
.
testing
.
multi_python
(
test
)
zc
.
buildout
.
testing
.
setUpServer
(
test
,
zc
.
buildout
.
testing
.
make_tree
(
test
))
def
easy_install_SetUp
(
test
):
zc
.
buildout
.
testing
.
buildoutSetUp
(
test
,
clear_home
=
False
)
zc
.
buildout
.
testing
.
multi_python
(
test
)
zc
.
buildout
.
testing
.
add_source_dist
(
test
)
zc
.
buildout
.
testing
.
setUpServer
(
test
,
zc
.
buildout
.
testing
.
make_tree
(
test
))
class
PythonNormalizing
(
renormalizing
.
RENormalizing
):
def
_transform
(
self
,
want
,
got
):
...
...
@@ -216,14 +222,15 @@ def test_suite():
doctest.DocFileSuite(
'
easy_install
.
txt
',
setUp=linkerSetUp, tearDown=zc.buildout.testing.buildoutTearDown,
setUp=easy_install_SetUp,
tearDown=zc.buildout.testing.buildoutTearDown,
checker=PythonNormalizing([
(re.compile("'
%
(
sep
)
s
\
S
+
sample
-
install
%
(
sep
)
s
(
dist
%
(
sep
)
s
)
?
"
% dict(sep=os.path.sep)),
'/sample-eggs/'),
(re.compile("
(
-
(
demo
(
needed
)
?
|
other
)
"
"
-
\
d
[.]
\
d
-
py
)
\
d
[.]
\
d
[.]
egg
"),
(re.compile("
(
[
d
-
]
((
ext
)
?
demo
(
needed
)
?
|
other
)
"
"
-
\
d
[.]
\
d
-
py
)
\
d
[.]
\
d
(
-
[
^
.
\
t
\
n
]
+
)
?
[.]
egg
"),
'
\
\
1V.V.egg'),
]),
),
...
...
zc.recipe.egg_/README.txt
View file @
db50caab
...
...
@@ -35,6 +35,12 @@ scripts
disabled. If the option isn't given at all, then all scripts
defined by the named eggs will be generated.
Custom eggs
-----------
The zc.recipe.egg:custom recipe supports building custom eggs,
currently with specialized options for building extensions.
To do
-----
...
...
zc.recipe.egg_/setup.py
View file @
db50caab
...
...
@@ -19,6 +19,9 @@ setup(
install_requires
=
[
'zc.buildout'
,
'setuptools'
],
tests_require
=
[
'zope.testing'
],
test_suite
=
name
+
'.tests.test_suite'
,
entry_points
=
{
'zc.buildout'
:
[
'default = %s:Egg'
%
name
]},
entry_points
=
{
'zc.buildout'
:
[
'default = %s:Egg'
%
name
,
'custom = %s:Custom'
%
name
,
]
},
dependency_links
=
[
'http://download.zope.org/distribution/'
],
)
zc.recipe.egg_/src/zc/recipe/egg/__init__.py
View file @
db50caab
from
zc.recipe.egg.egg
import
Egg
from
zc.recipe.egg.egg
import
Egg
from
zc.recipe.egg.custom
import
Custom
zc.recipe.egg_/src/zc/recipe/egg/custom.py
0 → 100644
View file @
db50caab
##############################################################################
#
# Copyright (c) 2006 Zope Corporation and Contributors.
# All Rights Reserved.
#
# This software is subject to the provisions of the Zope Public License,
# Version 2.1 (ZPL). A copy of the ZPL should accompany this distribution.
# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
# FOR A PARTICULAR PURPOSE.
#
##############################################################################
"""Install packages as eggs
$Id$
"""
import
os
,
re
,
zipfile
import
zc.buildout.easy_install
class
Custom
:
def
__init__
(
self
,
buildout
,
name
,
options
):
self
.
buildout
=
buildout
self
.
name
=
name
self
.
options
=
options
links
=
options
.
get
(
'find-links'
,
buildout
[
'buildout'
].
get
(
'find-links'
))
if
links
:
links
=
links
.
split
()
options
[
'find-links'
]
=
'
\
n
'
.
join
(
links
)
else
:
links
=
()
self
.
links
=
links
index
=
options
.
get
(
'index'
,
buildout
[
'buildout'
].
get
(
'index'
))
if
index
is
not
None
:
options
[
'index'
]
=
index
self
.
index
=
index
options
[
'_b'
]
=
buildout
[
'buildout'
][
'bin-directory'
]
options
[
'_e'
]
=
buildout
[
'buildout'
][
'eggs-directory'
]
options
[
'_d'
]
=
buildout
[
'buildout'
][
'develop-eggs-directory'
]
assert
options
.
get
(
'unzip'
)
in
(
'true'
,
'false'
,
None
)
python
=
options
.
get
(
'python'
,
buildout
[
'buildout'
][
'python'
])
options
[
'executable'
]
=
buildout
[
python
][
'executable'
]
build_ext
=
{}
for
be_option
in
(
'include-dirs'
,
'library-dirs'
,
'rpath'
):
value
=
options
.
get
(
be_option
)
if
value
is
None
:
continue
value
=
[
os
.
path
.
join
(
buildout
[
'buildout'
][
'directory'
],
v
.
strip
()
)
for
v
in
value
.
strip
().
split
(
'
\
n
'
)
if
v
.
strip
()
]
build_ext
[
be_option
]
=
':'
.
join
(
value
)
options
[
be_option
]
=
':'
.
join
(
value
)
self
.
build_ext
=
build_ext
def
install
(
self
):
if
self
.
buildout
[
'buildout'
].
get
(
'offline'
)
==
'true'
:
return
options
=
self
.
options
distribution
=
options
.
get
(
'eggs'
,
self
.
name
).
strip
()
build_ext
=
dict
([
(
k
,
options
[
k
])
for
k
in
(
'include-dirs'
,
'library-dirs'
,
'rpath'
)
if
k
in
options
])
zc
.
buildout
.
easy_install
.
build
(
distribution
,
options
[
'_d'
],
self
.
build_ext
,
self
.
links
,
self
.
index
,
options
[
'executable'
],
[
options
[
'_e'
]],
)
zc.recipe.egg_/src/zc/recipe/egg/custom.txt
0 → 100644
View file @
db50caab
Custon eggs
===========
Sometimes, It's necessary to provide extra control over how an egg is
created. This is commonly true for eggs with extension modules that
need to access libraries or include files.
The zc.recipe.egg:custom recipe can be used to define an egg with
custom build parameters. The currently defined parameters are:
include-dirs
A new-line separated list of directories to search for include
files.
library-dirs
A new-line separated list of directories to search for libraries
to link with.
rpath
A new-line separated list of directories to search for dynamic libraries
at run time.
In addition, the following options can be used to specify the egg:
egg
An eggs to install given as a setuptools requirement string.
This defaults to the part name.
find-links
A list of URLs, files, or directories to search for distributions.
index
The URL of an index server, or almost any other valid URL. :)
If not specified, the Python Package Index,
http://cheeseshop.python.org/pypi, is used. You can specify an
alternate index with this option. If you use the links option and
if the links point to the needed distributions, then the index can
be anything and will be largely ignored. In the examples, here,
we'll just point to an empty directory on our link server. This
will make our examples run a little bit faster.
python
The name of a section to get the Python executable from.
If not specified, then the buildout python option is used. The
Python executable is found in the executable option of the named
section.
To illustrate this, we'll define a buildout that builds an egg for a
package that has a simple extension module::
#include <Python.h>
#include <extdemo.h>
static PyMethodDef methods[] = {};
PyMODINIT_FUNC
initextdemo(void)
{
PyObject *d;
d = Py_InitModule3("extdemo", methods, "");
PyDict_SetItemString(d, "val", PyInt_FromLong(EXTDEMO));
}
The extension depends on a system-dependnt include file, extdemo.h,
that defines a constant, EXTDEMO, that is exposed by the extension.
The extension module is available as a source distribution,
extdemo-1.4.tar.gz, on a distribution server.
We have a sample buildout that we'll add an include directory to with
the necessary include file:
>>> mkdir(sample_buildout, 'include')
>>> import os
>>> open(os.path.join(sample_buildout, 'include', 'extdemo.h'), 'w').write(
... "#define EXTDEMO 42\n")
We'll also update the buildout configuration file to define a part for
the egg:
>>> write(sample_buildout, 'buildout.cfg',
... """
... [buildout]
... parts = extdemo
...
... [extdemo]
... recipe = zc.recipe.egg:custom
... find-links = %(server)s
... index = %(server)s/index
... include-dirs = include
... """ % dict(server=link_server))
>>> os.chdir(sample_buildout)
>>> buildout = os.path.join(sample_buildout, 'bin', 'buildout')
>>> print system(buildout),
zip_safe flag not set; analyzing archive contents...
We got the zip_safe warning because the source distribution we used
wasn't setuptools based and thus didn't set the option.
The egg is created in the develop-eggs directory *not* the eggs
directory because it depends on buildout-specific parameters and the
eggs directory can be shared across multiple buildouts.
>>> ls(sample_buildout, 'develop-eggs')
d extdemo-1.4-py2.4-unix-i686.egg
- zc.recipe.egg.egg-link
Note that no scripts or dependencies are installed. To install
dependencies or scripts for a custom egg, define another part and use
the zc.recipe.egg recipe, listing the custom egg as one of the eggs to
be installed. The zc.recipe.egg recipe will use the installed egg.
zc.recipe.egg_/src/zc/recipe/egg/tests.py
View file @
db50caab
...
...
@@ -41,6 +41,16 @@ def setUpPython(test):
zc
.
buildout
.
testing
.
multi_python
(
test
)
zc
.
buildout
.
testing
.
setUpServer
(
test
,
zc
.
buildout
.
testing
.
make_tree
(
test
))
def
setUpCustom
(
test
):
zc
.
buildout
.
testing
.
buildoutSetUp
(
test
)
open
(
os
.
path
.
join
(
test
.
globs
[
'sample_buildout'
],
'develop-eggs'
,
'zc.recipe.egg.egg-link'
),
'w'
).
write
(
dirname
(
__file__
,
4
))
zc
.
buildout
.
testing
.
create_sample_eggs
(
test
)
zc
.
buildout
.
testing
.
add_source_dist
(
test
)
zc
.
buildout
.
testing
.
setUpServer
(
test
,
zc
.
buildout
.
testing
.
make_tree
(
test
))
def
test_suite
():
return
unittest
.
TestSuite
((
...
...
@@ -87,7 +97,17 @@ def test_suite():
r'
/
sample
-
\
1
/
\
2
'),
(re.compile('
\
S
+
sample
-
(
\
w
+
)
'), r'
/
sample
-
\
1
'),
]),
),
),
doctest.DocFileSuite(
'
custom
.
txt
',
setUp=setUpCustom, tearDown=zc.buildout.testing.buildoutTearDown,
checker=renormalizing.RENormalizing([
(re.compile("(d ((ext)?demo(needed)?|other)"
"-
\
d[.]
\
d-py)
\
d[.]
\
d(-[^.
\
t
\
n
]+)?[.]egg"),
'
\\
1
V
.
V
.
egg
'),
]),
),
))
if __name__ == '
__main__
':
...
...
Write
Preview
Markdown
is supported
0%
Try again
or
attach a new file
Attach a file
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment