Commit 8a9e3766 authored by Julien Muchembled's avatar Julien Muchembled

Fix shared=true, other bugs, and inconsistencies between recipes; much cleanup

parent 2044d9e2
......@@ -4,9 +4,6 @@
.. contents::
Default
-------
The default recipe can be used to execute ad-hoc Python code at
init/install/update phases. `install` must create the path pointed to by
`location` (default is ${buildout:parts-directory}/${:_buildout_section_name_})
......@@ -48,103 +45,454 @@ Using the init option::
[section-two]
bar = ${section-one:foo}
In case of error, a proper traceback is displayed and nothing is installed::
A simplified example::
>>> write(sample_buildout, 'buildout.cfg',
... """
>>> write(sample_buildout, 'buildout.cfg', """
... [buildout]
... parts = section-two
...
... [section-one]
... recipe = slapos.recipe.build
... init =
... options['foo'] = 'foo from section-one'
... parts = script
...
... [section-two]
... [script]
... recipe = slapos.recipe.build
... bar = ${section-one:foo}
... install =
... import os
... os.mkdir(options['location'])
... print('Installed section-two with option %s.' % options['bar'])
... os.mkdir(location)
... print(1 / 0.) # this is an error !
... """)
>>> print(system(buildout))
Installing section-one.
Installing section-two.
Installed section-two with option foo from section-one.
Installing script.
While:
Installing script.
<BLANKLINE>
An internal error occurred due to a bug in either zc.buildout or in a
recipe being used:
Traceback (most recent call last):
...
File "script", line 3, in <module>
print(1 / 0.) # this is an error !
ZeroDivisionError: float division by zero
>>> ls(sample_buildout, 'parts')
d section-two
<BLANKLINE>
In case of error, a proper traceback is displayed and nothing is installed
option: environment
-------------------
>>> write(sample_buildout, 'buildout.cfg',
Customizing environment variables can be easier with the this option.
Values are expanded with Python %-dict formatting, using ``os.environ``. The
resulting environ dict is computed on first access of ``self.environ``.
Environment variables can be either inlined::
>>> base = """
... [buildout]
... parts = script
...
... [script]
... recipe = slapos.recipe.build
... update =
... import os
... os.environ["FOO"] = "1"
... print("%(FOO)s %(BAR)s" % self.environ)
... os.environ["FOO"] = "2"
... print("%(FOO)s %(BAR)s" % self.environ)
... """
>>> write(sample_buildout, 'buildout.cfg', base + """
... environment =
... BAR=%(FOO)s:%%
... """)
>>> print(system(buildout))
Installing script.
script: [ENV] BAR = 1:%
1 1:%
1 1:%
or put inside a separate section::
>>> write(sample_buildout, 'buildout.cfg', base + """
... environment = env
... [env]
... BAR=%(FOO)s:%%
... """)
>>> print(system(buildout))
Uninstalling script.
Installing script.
script: [ENV] BAR = 1:%
1 1:%
1 1:%
This option works the same way in other recipes that support it, in which case
the resulting environ dict is computed at install/update.
option: shared
--------------
Boolean (``false`` by default, or ``true``), this option specifies that the
part can be installed in a shared mode. This is enabled if paths are listed in
the ``shared-part-list`` option of the ``[buildout]`` section: the location of
the part is ``<one of shared-part-list>/<part name>/<hash of options>`` and
it contains a signature file ``.buildout-shared.json``.
`install` option is required::
>>> del MD5SUM[:]
>>> base = """
... [buildout]
... parts = section-two
... parts = script
... shared-part-list =
... ${:directory}/shared1
... ${:directory}/shared2
...
... [section-two]
... [script]
... recipe = slapos.recipe.build
... shared = true
... """
>>> write(sample_buildout, 'buildout.cfg', base + """
... init = pass
... """)
>>> print(system(buildout))
script: shared at .../shared2/script/<MD5SUM:0>
While:
Installing.
Getting section script.
Initializing section script.
Error: When shared=true, option 'install' must be set
`update` option is incompatible::
>>> base += """
... install =
... import os
... os.mkdir(options['location'])
... print(1 / 0.) # this is an error !
... os.makedirs(os.path.join(location, 'foo'))
... print("directory created")
... """
>>> write(sample_buildout, 'buildout.cfg', base)
>>> print(system(buildout + ' script:update=pass'))
script: shared at .../shared2/script/<MD5SUM:1>
While:
Installing.
Getting section script.
Initializing section script.
Error: When shared=true, option 'update' can't be set
A shared part is installed in the last folder that is listed by
``shared-part-list``::
>>> print(system(buildout))
script: shared at .../shared2/script/<MD5SUM:2>
Uninstalling script.
Installing script.
directory created
>>> shared = 'shared2/script/' + MD5SUM[2]
>>> ls(shared)
- .buildout-shared.json
l .buildout-shared.signature
d foo
``.buildout-shared.signature`` is only there for backward compatibility.
Uninstalling the part leaves the shared part available::
>>> print(system(buildout + ' buildout:parts='))
Uninstalling script.
Unused options for buildout: 'shared-part-list'.
>>> ls(shared)
- .buildout-shared.json
l .buildout-shared.signature
d foo
And reinstalling is instantaneous::
>>> print(system(buildout))
script: shared at .../shared2/script/<MD5SUM:2>
Installing script.
script: shared part is already installed
Setting `location` option is incompatible::
>>> write(sample_buildout, 'buildout.cfg', base + """
... init =
... import os
... options['location'] = os.path.join(
... self.buildout['buildout']['parts-directory'], 'foo')
... """)
>>> print(system(buildout))
script: shared at .../shared2/script/<MD5SUM:3>
While:
Installing.
Getting section script.
Initializing section script.
Error: When shared=true, option 'location' can't be set
=============================
slapos.recipe.build:download
=============================
Simplest usage is to only specify a URL::
>>> base = """
... [buildout]
... parts = download
...
... [download]
... recipe = slapos.recipe.build:download
... url = https://lab.nexedi.com/nexedi/slapos.recipe.build/raw/master/MANIFEST.in
... """
>>> write(sample_buildout, 'buildout.cfg', base)
>>> print(system(buildout))
Uninstalling script.
Installing download.
Downloading ...
>>> ls('parts/download')
- download
The file is downloaded to ``parts/<section_name>/<section_name>``.
option: filename
----------------
In the part folder, the filename can be customized::
>>> write(sample_buildout, 'buildout.cfg', base + """
... filename = somefile
... """)
>>> print(system(buildout))
Uninstalling download.
Installing download.
Downloading ...
>>> ls('parts/download')
- somefile
When an MD5 checksum is not given, updating the part downloads the file again::
>>> remove('parts/download/somefile')
>>> print(system(buildout))
Updating download.
Downloading ...
>>> ls('parts/download')
- somefile
option: destination
-------------------
Rather than having a file inside a part folder, a full path can be given::
>>> write(sample_buildout, 'buildout.cfg', base + """
... destination = ${buildout:parts-directory}/somepath
... """)
>>> print(system(buildout))
Uninstalling section-two.
Uninstalling section-one.
Installing section-two.
Uninstalling download.
Installing download.
Downloading ...
>>> ls('parts')
- somepath
option: target
--------------
In any case, path to download file is exposed by the ``target`` option::
>>> cat('.installed.cfg')
[buildout]
...
[download]
__buildout_installed__ = .../parts/somepath
__buildout_signature__ = ...
destination = .../parts/somepath
recipe = slapos.recipe.build:download
target = .../parts/somepath
url = ...
option: md5sum
--------------
An MD5 checksum can be specified to check the contents::
>>> base += """
... md5sum = b90c12a875df544907bc84d9c7930653
... """
>>> write(sample_buildout, 'buildout.cfg', base)
>>> print(system(buildout))
Uninstalling download.
Installing download.
Downloading ...
>>> ls('parts/download')
- download
In such case, updating the part does nothing::
>>> remove('parts/download/download')
>>> print(system(buildout))
Updating download.
>>> ls('parts/download')
In case of checksum mismatch::
>>> print(system(buildout
... + ' download:md5sum=00000000000000000000000000000000'
... ))
Uninstalling download.
Installing download.
Downloading ...
While:
Installing section-two.
<BLANKLINE>
An internal error occurred due to a bug in either zc.buildout or in a
recipe being used:
Traceback (most recent call last):
Installing download.
Error: MD5 checksum mismatch downloading '...'
>>> ls('parts')
option: mode
------------
Octal (e.g. 644 for rw-r--r--), this option
allows to set mode of the downloaded file.
option: shared
--------------
Works like the default recipe. Constraints on options are:
- ``md5sum`` option is required
- ``destination`` option is incompatible
Example::
>>> del MD5SUM[4:] # drop added values since previous shared test
>>> write(sample_buildout, 'buildout.cfg', base + """
... shared = true
...
File "section-two", line 3, in <module>
print(1 / 0.) # this is an error !
ZeroDivisionError: float division by zero
... [buildout]
... shared-part-list =
... ${:directory}/shared
... """)
>>> print(system(buildout))
download: shared at .../shared/download/<MD5SUM:4>
Installing download.
Downloading ...
>>> shared = 'shared/download/' + MD5SUM[4]
>>> ls(shared)
- .buildout-shared.json
l .buildout-shared.signature
- download
>>> ls(sample_buildout, 'parts')
<BLANKLINE>
=======================================
slapos.recipe.build:download-unpacked
=======================================
Pure download
~~~~~~~~~~~~~
Downloads and extracts an archive. In addition to format that setuptools is
able to extract, XZ & lzip compression are also supported if ``xzcat`` &
``lunzip`` executables are available.
::
By default, the archive is extracted to ``parts/<section_name>`` and a single
directory at the root of the archive is stripped::
[buildout]
parts =
download
>>> URL = "https://lab.nexedi.com/nexedi/slapos.recipe.build/-/archive/master/slapos.recipe.build-master.tar.gz?path=slapos/recipe/build"
>>> base = """
... [buildout]
... download-cache = download-cache
... parts = download
...
... [download]
... recipe = slapos.recipe.build:download-unpacked
... url = %s
... """ % URL
>>> write(sample_buildout, 'buildout.cfg', base)
>>> print(system(buildout))
Creating directory '.../download-cache'.
Uninstalling download.
Installing download.
Downloading ...
>>> ls('parts/download')
d slapos
The download cache will avoid to download the same tarball several times.
option: destination
-------------------
Similar to ``download`` recipe::
>>> write(sample_buildout, 'buildout.cfg', base + """
... destination = ${buildout:parts-directory}/somepath
... """)
>>> print(system(buildout))
Uninstalling download.
Installing download.
>>> ls('parts/somepath')
d slapos
option: target
--------------
Like for ``download`` recipe, the installation path of the part is exposed by
the ``target`` option::
>>> cat('.installed.cfg')
[buildout]
...
[download]
recipe = slapos.recipe.build:download
url = https://some.url/file
__buildout_installed__ = .../parts/somepath
__buildout_signature__ = ...
destination = .../parts/somepath
recipe = slapos.recipe.build:download-unpacked
target = .../parts/somepath
url = ...
option: strip-top-level-dir
---------------------------
Stripping can be enforced::
>>> print(system(buildout + ' download:strip-top-level-dir=true'))
Uninstalling download.
Installing download.
>>> ls('parts/somepath')
d slapos
Or disabled::
>>> print(system(buildout + ' download:strip-top-level-dir=false'))
Uninstalling download.
Installing download.
>>> ls('parts/somepath')
d slapos.recipe.build-master-slapos-recipe-build
option: md5sum
--------------
Such profile will download https://some.url/file and put it in
buildout:parts-directory/download/download
An MD5 checksum can be specified to check the downloaded file, like for the
``download`` recipe. However, if unset, updating the part does nothing.
filename parameter can be used to change destination named filename.
option: environment
-------------------
destination parameter allows to put explicit destination.
Like for the default recipe, environment variables can be customized, here
for ``xzcat`` & ``lunzip`` subprocesses (e.g. PATH).
md5sum parameter allows pass md5sum.
option: shared
--------------
mode (octal, so for rw-r--r-- use 0644) allows to set mode
Works like the default recipe. The only constraint on options is that
the ``destination`` option is incompatible.
Exposes target attribute which is path to downloaded file.
Example::
Notes
-----
>>> del MD5SUM[5:] # drop added values since previous shared test
>>> write(sample_buildout, 'buildout.cfg', """
... [buildout]
... download-cache = download-cache
... parts = download
... shared-part-list = ${:directory}/shared
...
... [download]
... recipe = slapos.recipe.build:download-unpacked
... url = %s
... shared = true
... """ % URL)
>>> print(system(buildout))
download: shared at .../shared/download/<MD5SUM:5>
Uninstalling download.
Installing download.
This recipe suffers from buildout download utility issue, which will do not
try to redownload resource with wrong md5sum.
==============================
slapos.recipe.build:gitclone
......@@ -180,6 +528,7 @@ This will clone the git repository in `parts/git-clone` directory.
Then let's run the buildout::
>>> print(system(buildout))
Uninstalling download.
Installing git-clone.
Cloning into '/sample-buildout/parts/git-clone'...
......@@ -284,7 +633,6 @@ When updating, it shouldn't do anything as revision is mentioned::
>>> cd(sample_buildout)
>>> print(system(buildout))
Updating git-clone.
...
Empty revision/branch
~~~~~~~~~~~~~~~~~~~~~
......@@ -393,9 +741,6 @@ repository::
>>> cd(sample_buildout)
>>> print(system(buildout))
Updating git-clone.
...
<BLANKLINE>
>>> cd(sample_buildout, 'parts', 'git-clone')
>>> print(system('cat setup.py'))
......@@ -412,7 +757,6 @@ Then, when update occurs, nothing is done::
>>> cd(sample_buildout)
>>> print(system(buildout))
Updating git-clone.
...
>>> cd(sample_buildout, 'parts', 'git-clone')
>>> print(system('cat local_change'))
......@@ -569,9 +913,7 @@ location
Default: ${buildout:parts-directory}/<section_name>
environment
Extra environment for the spawn executables. It can either be the name of a
section or a list of variables (1 per line, in the form ``key=value``).
Values are expanded with current environment using Python %-dict formatting.
Extra environment to spawn executables. See the default recipe.
mem
Python expression evaluating to an integer that specifies the
......
......@@ -5,59 +5,177 @@ except ImportError:
from pkgutil import extend_path
__path__ = extend_path(__path__, __name__)
import errno, logging, os, shutil
import zc.buildout
logger = logging.getLogger(__name__)
import errno, json, logging, os, shutil, stat
from hashlib import md5
from zc.buildout import UserError
from zc.buildout.rmtree import rmtree as buildout_rmtree
def generatePassword(length=8):
from random import SystemRandom
from string import ascii_lowercase
return ''.join(SystemRandom().sample(ascii_lowercase, length))
def is_true(value, default=False):
return default if value is None else ('false', 'true').index(value)
def make_read_only(path):
if not os.path.islink(path):
os.chmod(path, os.stat(path).st_mode & 0o555)
def make_read_only_recursively(path):
make_read_only(path)
for root, dir_list, file_list in os.walk(path):
for dir_ in dir_list:
make_read_only(os.path.join(root, dir_))
for file_ in file_list:
make_read_only(os.path.join(root, file_))
def rmtree(path):
try:
os.remove(path)
buildout_rmtree(path)
except OSError as e:
if e.errno != errno.EISDIR:
if e.errno == errno.ENOENT:
return
if e.errno != errno.ENOTDIR:
raise
shutil.rmtree(path)
os.remove(path)
class EnvironMixin:
def __init__(self, allow_none=True):
environment = self.options.get('environment', '').strip()
class EnvironMixin(object):
def __init__(self, allow_none=True, compat=False):
environment = self.options.get('environment')
if environment:
from os import environ
if '=' in environment:
self._environ = env = {}
if compat: # for slapos.recipe.cmmi
environment_section = self.options.get('environment-section')
if environment_section:
env.update(self.buildout[environment_section])
compat = set(env)
else:
compat = ()
for line in environment.splitlines():
line = line.strip()
if line:
try:
k, v = line.split('=', 1)
except ValueError:
raise zc.buildout.UserError('Line %r in environment is incorrect' %
line)
k = k.strip()
raise UserError('Line %r in environment is incorrect' % line)
k = k.rstrip()
if k in env:
raise zc.buildout.UserError('Key %r is repeated' % k)
env[k] = v.strip() % environ
if k in compat:
compat.remove(k)
else:
self._environ = dict((k, v.strip() % environ)
for k, v in self.buildout[environment].items())
raise UserError('Key %r is repeated' % k)
env[k] = v.lstrip()
else:
self._environ = self.buildout[environment]
else:
self._environ = None if allow_none else {}
@property
def environ(self):
if self._environ is not None:
def __getattr__(self, attr):
if attr == 'logger':
value = logging.getLogger(self.name)
elif attr == 'environ':
env = self._environ
del self._environ
if env is None:
value = None
else:
from os import environ
env = self._environ.copy()
for k, v in env.items():
logger.info(
'Environment %r set to %r' if k in environ else
'Environment %r added with %r', k, v)
for kw in environ.items():
env.setdefault(*kw)
return env
value = environ.copy()
for k in sorted(env):
value[k] = v = env[k] % environ
self.logger.info('[ENV] %s = %s', k, v)
else:
return self.__getattribute__(attr)
setattr(self, attr, value)
return value
class Shared(object):
keep_on_error = False
mkdir_location = True
signature = None
def __init__(self, buildout, name, options):
self.maybe_shared = shared = is_true(options.get('shared'))
if shared:
# Trigger computation of part signature for shared signature.
# From now on, we should not pull new dependencies.
# Ignore if buildout is too old.
options.get('__buildout_signature__')
shared = buildout['buildout'].get('shared-part-list')
if shared:
profile_base_location = options.get('_profile_base_location_')
signature = json.dumps({
k: (v.replace(profile_base_location, '${:_profile_base_location_}')
if profile_base_location else v)
for k, v in options.items()
if k != '_profile_base_location_'
}, indent=0, sort_keys=True)
if not isinstance(signature, bytes): # BBB: Python 3
signature = signature.encode()
digest = md5(signature).hexdigest()
location = None
for shared in shared.splitlines():
shared = shared.strip().rstrip('/')
if shared:
location = os.path.join(os.path.join(shared, name), digest)
if os.path.exists(location):
break
if location:
self.logger = logging.getLogger(name)
self.logger.info('shared at %s', location)
self.location = location
self.signature = signature
return
self.location = os.path.join(buildout['buildout']['parts-directory'], name)
def assertNotShared(self, reason):
if self.maybe_shared:
raise UserError("When shared=true, " + reason)
def install(self, install):
signature = self.signature
location = self.location
if signature is not None:
path = os.path.join(location, '.buildout-shared.json')
  • After I released 0.48, I realized that JSON does not support all types our buildout supports (slapos.buildout@4e13dcb9). Currently, we don't combine the 2 features (shared part with non-str value), and we have no plan to do so, but it may be wiser and cleaner to use a format without this limitation.

    I chose to change to JSON because it's so simple and any external non-Python code could easily parse this file: again, no use case for this, now or planned.

    I am considering to switch to a .buildout-shared.signature file that use the same serialization as .installed.cfg.

    Edited by Julien Muchembled
  • Given how slapos node prune is implemented, non-ascii chars must be written as is (utf-8 encoding) rather than being escaped by serialization format.

    • repr (previous format) could not do that with Python 2
    • I can fix .buildout-shared.json by passing ensure_ascii=False to json.dumps
    • .installed.cfg has this issue for non-str types with Python 2 (repr again)
    Edited by Julien Muchembled
  • Do we still rely on that buildout patch (slapos.buildout@4e13dcb9) ? I see we are using more and more explicit json serialization instead. We probably still rely on it for instance buildout, so we are not ready to drop this patch yet, but I think we should not make ourselves more depending on this patch

    Edited by Julien Muchembled
  • (I edited my comment and yours to fix the hash of the commit I referred to)

    I see we are using more and more explicit json serialization instead.

    Where ? We may see more JSON (like _ slapos parameter or slapos@7918c2b2) but internally it uses !py! buildout serialization.

    • .installed.cfg has this issue for non-str types with Python 2 (repr again)

    Note that I don't see this as an issue. I don't see any use of non-str types in near future and by then, we can hope to have switched to Python 3.

    A bigger issue (still theoretical, with the goal to choose good design) is that .installed.cfg is subject to escape other things: we changed buildout to escape ${ into $${.

    I looks like I'm going to keep JSON and update slapos node prune to recognize the new signature file name (so later I can drop the symlink).

    Edited by Julien Muchembled
  • Ah yes, sorry, I think it was about the jsonkey for slapos parameter and maybe something else where my memory failed me.

    In any case, if we rename the signature files, it's better to keep a symlink like it's done here for a transition period, to prevent problems when old slapos.core is used to install new softwares.

Please register or sign in to reply
if os.path.exists(path):
self.logger.info('shared part is already installed')
return ()
rmtree(location)
try:
if self.mkdir_location:
os.makedirs(location)
else:
parent = os.path.dirname(location)
if not os.path.isdir(parent):
os.makedirs(parent)
install()
try:
s = os.stat(location)
except OSError as e:
if e.errno != errno.ENOENT:
raise
raise UserError('%r was not created' % location)
if self.maybe_shared and not stat.S_ISDIR(s.st_mode):
raise UserError('%r is not a directory' % location)
if signature is None:
return [location]
tmp = path + '.tmp'
with open(tmp, 'wb') as f:
f.write(signature)
# XXX: The following symlink is for backward compatibility with old
# 'slapos node prune' (slapos.core).
os.symlink('.buildout-shared.json', os.path.join(location,
'.buildout-shared.signature'))
os.rename(tmp, path)
except:
if not self.keep_on_error:
rmtree(location)
raise
make_read_only_recursively(location)
return ()
......@@ -36,7 +36,7 @@ import subprocess
import sys
import tempfile
import zc.buildout
from slapos.recipe import rmtree, EnvironMixin
from .. import is_true, rmtree, EnvironMixin, Shared
ARCH_MAP = {
'i386': 'x86',
......@@ -90,9 +90,7 @@ def guessPlatform():
return ARCH_MAP[uname()[-2]]
GLOBALS = (lambda *x: {x.__name__: x for x in x})(
call, guessPlatform, guessworkdir)
TRUE_LIST = ('y', 'on', 'yes', 'true', '1')
call, guessPlatform, guessworkdir, is_true)
class Script(EnvironMixin):
"""Free script building system"""
......@@ -154,10 +152,9 @@ class Script(EnvironMixin):
raise zc.buildout.UserError('Promise not met, found issues:\n %s\n' %
'\n '.join(promise_problem_list))
def download(self, url, md5sum=None):
download = zc.buildout.download.Download(self.buildout['buildout'],
hash_name=True, cache=self.buildout['buildout'].get('download-cache'))
path, is_temp = download(url, md5sum=md5sum)
def download(self, *args, **kw):
path, is_temp = zc.buildout.download.Download(self.buildout['buildout'],
hash_name=True)(*args, **kw)
if is_temp:
self.cleanup_list.append(path)
return path
......@@ -227,7 +224,6 @@ class Script(EnvironMixin):
self.options = options
self.buildout = buildout
self.name = name
self.logger = logging.getLogger('SlapOS build of %s' % self.name)
missing = True
keys = 'init', 'install', 'update'
for option in keys:
......@@ -238,17 +234,29 @@ class Script(EnvironMixin):
if missing:
raise zc.buildout.UserError(
'at least one of the following option is required: ' + ', '.join(keys))
if self.options.get('keep-on-error', '').strip().lower() in TRUE_LIST:
if is_true(self.options.get('keep-on-error')):
self.logger.debug('Keeping directories in case of errors')
self.keep_on_error = True
else:
self.keep_on_error = False
if self._install and 'location' not in options:
options['location'] = os.path.join(
buildout['buildout']['parts-directory'], self.name)
EnvironMixin.__init__(self, False)
if self._init:
self._exec(self._init)
shared = Shared(buildout, name, options)
if self._update:
shared.assertNotShared("option 'update' can't be set")
if self._install:
location = options.get('location')
if location:
shared.assertNotShared("option 'location' can't be set")
shared.location = location
else:
options['location'] = shared.location
shared.keep_on_error = True
shared.mkdir_location = False
self._shared = shared
else:
shared.assertNotShared("option 'install' must be set")
def _exec(self, script):
options = self.options
......@@ -268,13 +276,13 @@ class Script(EnvironMixin):
exec(code, g)
def install(self):
if not self._install:
if self._install:
return self._shared.install(self.__install)
self.update()
return ""
return ()
def __install(self):
location = self.options['location']
if os.path.lexists(location):
self.logger.warning('Removing already existing path %r', location)
rmtree(location)
self.cleanup_list = []
try:
self._exec(self._install)
......@@ -290,9 +298,6 @@ class Script(EnvironMixin):
else:
self.logger.debug('Removing %r', path)
rmtree(path)
if not os.path.exists(location):
raise zc.buildout.UserError('%r was not created' % location)
return location
def update(self):
if self._update:
......
......@@ -11,9 +11,8 @@ from zc.buildout.testing import buildoutTearDown
from contextlib import contextmanager
from functools import wraps
from subprocess import check_call, check_output, CalledProcessError, STDOUT
from slapos.recipe.gitclone import GIT_CLONE_ERROR_MESSAGE, \
GIT_CLONE_CACHE_ERROR_MESSAGE
from slapos.recipe.downloadunpacked import make_read_only_recursively
from ..gitclone import GIT_CLONE_ERROR_MESSAGE, GIT_CLONE_CACHE_ERROR_MESSAGE
from .. import make_read_only_recursively
optionflags = (doctest.ELLIPSIS | doctest.NORMALIZE_WHITESPACE)
......@@ -563,6 +562,26 @@ class MakeReadOnlyTests(unittest.TestCase):
make_read_only_recursively(self.tmp_dir)
self.assertRaises(IOError, open, os.path.join(self.tmp_dir, 'folder', 'symlink'), 'w')
MD5SUM = []
def md5sum(m):
x = m.group(0)
try:
i = MD5SUM.index(x)
except ValueError:
i = len(MD5SUM)
MD5SUM.append(x)
return '<MD5SUM:%s>' % i
renormalizing_patters = [
zc.buildout.testing.normalize_path,
zc.buildout.testing.not_found,
(re.compile(
'.*CryptographyDeprecationWarning: Python 2 is no longer supported by the Python core team. '
'Support for it is now deprecated in cryptography, and will be removed in the next release.\n.*'
), ''),
(re.compile('[0-9a-f]{32}'), md5sum),
]
def test_suite():
suite = unittest.TestSuite((
......@@ -573,12 +592,9 @@ def test_suite():
tearDown=zc.buildout.testing.buildoutTearDown,
optionflags=optionflags,
checker=renormalizing.RENormalizing([
zc.buildout.testing.normalize_path,
(re.compile(r'http://localhost:\d+'), 'http://test.server'),
# Clean up the variable hashed filenames to avoid spurious
# test failures
(re.compile(r'[a-f0-9]{32}'), ''),
]),
] + renormalizing_patters),
globs={'MD5SUM': MD5SUM},
),
unittest.makeSuite(GitCloneNonInformativeTests),
unittest.makeSuite(MakeReadOnlyTests),
......
......@@ -26,91 +26,57 @@
##############################################################################
import errno
import os
import shutil
import zc.buildout
import logging
from hashlib import md5
from .downloadunpacked import make_read_only_recursively, Signature
from zc.buildout import download
from . import Shared
class Recipe(object):
_parts = None
_shared = None
def __init__(self, buildout, name, options):
buildout_section = buildout['buildout']
self._downloader = zc.buildout.download.Download(buildout_section,
hash_name=True)
self._buildout = buildout['buildout']
self._url = options['url']
self._md5sum = options.get('md5sum')
self._md5sum = options.get('md5sum') or None
self._name = name
mode = options.get('mode')
log = logging.getLogger(name)
self._shared = shared = ((options.get('shared', '').lower() == 'true') and
buildout['buildout'].get('shared-parts', None))
if mode is not None:
mode = int(mode, 8)
self._mode = mode
if 'filename' in options and 'destination' in options:
raise zc.buildout.UserError('Parameters filename and destination are '
'exclusive.')
destination = options.get('destination', None)
if destination is None:
if shared:
shared_part = buildout['buildout'].get('shared-parts', None)
shared = os.path.join(shared_part.strip().rstrip('/'), name)
if not os.path.exists(shared):
os.makedirs(shared)
self._signature = Signature('.slapos.recipe.build.signature')
profile_base_location = options.get('_profile_base_location_', '')
for k, v in sorted(options.items()):
if profile_base_location:
v = v.replace(profile_base_location, '${:_profile_base_location_}')
self._signature.update(k, v)
shared = os.path.join(shared, self._signature.hexdigest())
self._parts = parts = shared
log.info('shared directory %s set for %s', shared, name)
else:
self._parts = parts = os.path.join(buildout_section['parts-directory'],
name)
shared = Shared(buildout, name, options)
if not self._md5sum:
shared.assertNotShared("option 'md5sum' must be set")
destination = os.path.join(parts, options.get('filename', name))
destination = options.get('destination')
if destination:
shared.assertNotShared("option 'destination' can't be set")
else:
self._shared = shared
destination = os.path.join(shared.location,
options.get('filename') or name)
# Compatibility with other recipes: expose location
options['location'] = parts
options['location'] = shared.location
options['target'] = self._destination = destination
def install(self):
shared = self._shared
if shared:
return shared.install(self._download)
destination = self._destination
result = [destination]
parts = self._parts
log = logging.getLogger(self._name)
if self._shared:
log.info('Checking whether package is installed at shared path: %s', destination)
if self._signature.test(self._parts):
log.info('This shared package has been installed by other package')
return []
if parts is not None and not os.path.isdir(parts):
os.mkdir(parts)
result.append(parts)
path, is_temp = self._downloader(self._url, md5sum=self._md5sum)
with open(path, 'rb') as fsrc:
if is_temp:
os.remove(path)
try:
os.remove(destination)
except OSError as e:
if e.errno != errno.ENOENT:
raise
with open(destination, 'wb') as fdst:
if self._mode is not None:
os.fchmod(fdst.fileno(), self._mode)
shutil.copyfileobj(fsrc, fdst)
self._download()
return [destination]
if self._shared:
self._signature.save(parts)
make_read_only_recursively(self._parts)
return result
def _download(self):
download.Download(self._buildout, hash_name=True)(
self._url, self._md5sum, self._destination)
if self._mode is not None:
os.chmod(self._destination, self._mode)
def update(self):
if not self._md5sum:
self.install()
self._download()
......@@ -24,152 +24,90 @@
# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
#
##############################################################################
import contextlib
import os
import logging
import shutil
import subprocess
import tarfile
import zc.buildout
import tempfile
import setuptools.archive_util
from hashlib import md5
from setuptools import archive_util
from zc.buildout import download
from . import is_true, EnvironMixin, Shared
from . import make_read_only_recursively # for slapos.recipe.cmmi
is_true = ('false', 'true').index
class Recipe:
class Recipe(EnvironMixin):
def __init__(self, buildout, name, options):
self.buildout = buildout
self.name = name
self.options = options
self.logger = logging.getLogger(self.name)
if 'filename' in self.options and 'destination' in self.options:
raise zc.buildout.UserError('Parameters filename and destination are '
'exclusive.')
self.parts = None
self.destination = self.options.get('destination', None)
self.shared = shared = (is_true(options.get('shared', 'false').lower()) and
buildout['buildout'].get('shared-parts', None))
if self.destination is None:
if shared:
shared_part = buildout['buildout'].get('shared-parts', None)
top_location = options.get('top_location', '')
shared = os.path.join(shared_part.strip().rstrip('/'), top_location, name)
if not os.path.exists(shared):
os.makedirs(shared)
self._signature = Signature('.slapos.recipe.build.signature')
profile_base_location = options.get('_profile_base_location_', '')
for k, v in sorted(options.items()):
if profile_base_location:
v = v.replace(profile_base_location, '${:_profile_base_location_}')
self._signature.update(k, v)
shared = os.path.join(shared, self._signature.hexdigest())
self.parts = shared
self.logger.info('shared directory %s set for %s', shared, name)
else:
self.parts = os.path.join(self.buildout['buildout']['parts-directory'],
self.name)
self.destination = self.parts
self._strip = is_true(options.get('strip-top-level-dir'), None)
self._url = options['url']
shared = Shared(buildout, name, options)
destination = options.get('destination')
if destination:
shared.assertNotShared("option 'destination' can't be set")
shared.location = destination
self._shared = shared
# backward compatibility with other recipes -- expose location
options['location'] = self.parts
options['target'] = self.destination
options.setdefault('extract-directory', '')
options['location'] = \
options['target'] = shared.location
self.environ = {}
self.original_environment = os.environ.copy()
environment_section = self.options.get('environment-section', '').strip()
if environment_section and environment_section in buildout:
# Use environment variables from the designated config section.
self.environ.update(buildout[environment_section])
for variable in self.options.get('environment', '').splitlines():
if variable.strip():
try:
key, value = variable.split('=', 1)
self.environ[key.strip()] = value
except ValueError:
raise zc.buildout.UserError('Invalid environment variable definition: %s', variable)
# Extrapolate the environment variables using values from the current
# environment.
for key in self.environ:
self.environ[key] = self.environ[key] % os.environ
EnvironMixin.__init__(self, True, True)
def install(self):
if self.shared:
self.logger.info('Checking whether package is installed at shared path : %s', self.destination)
if self._signature.test(self.destination):
self.logger.info('This shared package has been installed by other package')
return []
if self.parts is not None:
if not os.path.isdir(self.parts):
os.mkdir(self.parts)
return self._shared.install(self._install)
download = zc.buildout.download.Download(self.buildout['buildout'],
hash_name=True, cache=self.buildout['buildout'].get('download-cache'))
extract_dir = tempfile.mkdtemp(self.name)
try:
self.logger.debug('Created working directory %r', extract_dir)
path, is_temp = download(self.options['url'],
md5sum=self.options.get('md5sum'))
def _install(self):
location = self._shared.location
path, is_temp = download.Download(self.buildout['buildout'],
hash_name=True)(self._url, self.options.get('md5sum') or None)
try:
patch_archive_util()
archive_util.extraction_drivers = patched_extraction_drivers
# ad-hoc support for .xz and .lz archive
hdr = open(path, 'rb').read(6)
with open(path, 'rb') as f:
hdr = f.read(6)
for magic, cmd in ((b'\xfd7zXZ\x00', ('xzcat',)),
(b'LZIP', ('lunzip', '-c'))):
if hdr.startswith(magic):
new_path = os.path.join(extract_dir, os.path.basename(path))
with open(new_path, 'wb') as stdout:
with tempfile.NamedTemporaryFile() as uncompressed_archive:
subprocess.check_call(cmd + (path,),
stdout=stdout, env=self.environ)
setuptools.archive_util.unpack_archive(new_path, extract_dir)
os.unlink(new_path)
stdout=uncompressed_archive, env=self.environ)
archive_util.unpack_archive(
uncompressed_archive.name, location)
break
else:
setuptools.archive_util.unpack_archive(path, extract_dir)
archive_util.unpack_archive(path, location)
finally:
unpatch_archive_util()
if is_temp:
os.unlink(path)
if os.path.exists(self.destination):
shutil.rmtree(self.destination)
os.makedirs(self.destination)
strip = self.options.get('strip-top-level-dir')
if strip:
if is_true(strip.lower()):
base_dir, = os.listdir(extract_dir)
base_dir = os.path.join(extract_dir, base_dir)
else:
base_dir = extract_dir
archive_util.extraction_drivers = extraction_drivers
strip = self._strip
if strip is None:
a = os.listdir(location)
if len(a) == 1:
a = os.path.join(location, *a)
if not os.path.isdir(a):
return
elif strip:
a, = os.listdir(location)
a = os.path.join(location, a)
else:
directories = os.listdir(extract_dir)
if len(directories) == 1:
base_dir = os.path.join(extract_dir, directories[0])
if not os.path.isdir(base_dir):
base_dir = extract_dir
base_dir = os.path.join(base_dir, self.options['extract-directory'])
for filename in os.listdir(base_dir):
shutil.move(os.path.join(base_dir, filename), self.destination)
finally:
shutil.rmtree(extract_dir)
self.logger.debug('Downloaded %r and saved to %r.',
self.options['url'], self.destination)
if self.shared:
self._signature.save(self.parts)
make_read_only_recursively(self.parts)
return []
if self.parts is not None:
return [self.parts]
else:
return []
return
b = os.path.join(location, os.path.basename(tempfile.mktemp(dir=a)))
os.rename(a, b)
for a in os.listdir(b):
os.rename(os.path.join(b, a), os.path.join(location, a))
os.rmdir(b)
def update(self):
pass
# Monkey patch to keep symlinks in tarfile
def unpack_tarfile_patched(filename, extract_dir, progress_filter=setuptools.archive_util.default_filter):
def unpack_tarfile_patched(filename, extract_dir,
progress_filter=archive_util.default_filter):
"""Unpack tar/tar.gz/tar.bz2 `filename` to `extract_dir`
Raises ``UnrecognizedFormat`` if `filename` is not a tarfile (as determined
......@@ -179,10 +117,10 @@ def unpack_tarfile_patched(filename, extract_dir, progress_filter=setuptools.arc
try:
tarobj = tarfile.open(filename)
except tarfile.TarError:
raise setuptools.archive_util.UnrecognizedFormat(
raise archive_util.UnrecognizedFormat(
"%s is not a compressed or uncompressed tar file" % (filename,)
)
with setuptools.archive_util.contextlib.closing(tarobj):
with contextlib.closing(tarobj):
# don't do any chowning!
tarobj.chown = lambda *args: None
for member in tarobj:
......@@ -207,33 +145,12 @@ def unpack_tarfile_patched(filename, extract_dir, progress_filter=setuptools.arc
pass
return True
def patch_archive_util():
setuptools.archive_util.extraction_drivers = (
setuptools.archive_util.unpack_directory,
setuptools.archive_util.unpack_zipfile,
extraction_drivers = archive_util.extraction_drivers
patched_extraction_drivers = extraction_drivers[:2] + (
unpack_tarfile_patched,
)
def unpatch_archive_util():
setuptools.archive_util.extraction_drivers = (
setuptools.archive_util.unpack_directory,
setuptools.archive_util.unpack_zipfile,
setuptools.archive_util.unpack_tarfile,
)
def make_read_only(path):
if not os.path.islink(path):
os.chmod(path, os.stat(path).st_mode & 0o555)
def make_read_only_recursively(path):
make_read_only(path)
for root, dir_list, file_list in os.walk(path):
for dir_ in dir_list:
make_read_only(os.path.join(root, dir_))
for file_ in file_list:
make_read_only(os.path.join(root, file_))
class Signature:
class Signature: # for slapos.recipe.cmmi
def __init__(self, filename):
self.filename = filename
......
......@@ -31,15 +31,13 @@ from io import BytesIO
from collections import defaultdict
from contextlib import contextmanager
from os.path import join
from slapos.recipe import EnvironMixin, generatePassword, logger, rmtree
from zc.buildout import UserError
from . import EnvironMixin, generatePassword, is_true, rmtree
ARCH = os.uname()[4]
@contextmanager
def building_directory(directory):
if os.path.lexists(directory):
logger.warning('Removing already existing path %r', directory)
rmtree(directory)
os.makedirs(directory)
try:
......@@ -48,8 +46,6 @@ def building_directory(directory):
shutil.rmtree(directory)
raise
is_true = ('false', 'true').index
class Popen(subprocess.Popen):
def stop(self):
......@@ -99,6 +95,7 @@ class BaseRecipe(EnvironMixin):
def __init__(self, buildout, name, options, allow_none=True):
self.buildout = buildout
self.name = name
self.options = options
try:
options['location'] = options['location'].strip()
......@@ -255,7 +252,7 @@ class InstallDebianRecipe(BaseRecipe):
raise NotImplementedError
p[k] = v.strip()
vm_run = is_true(options.get('vm.run', 'true'))
vm_run = is_true(options.get('vm.run'), True)
packages = ['ssh', 'sudo'] if vm_run else []
packages += options.get('packages', '').split()
if packages:
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment