Skip to content

S
slapos.buildout
Commit 517e6352 authored by Jim Fulton's avatar Jim Fulton
Browse files
Options

- Changed the way the installed database (.installed.cfg) is handled 

  to avoid database corruption when a user breaks out of a buildout
  with control-c.

- Don't save an installed database if there are no installed parts or
  develop egg links.
parent 4f330ba0
Hide whitespace changes
Inline Side-by-side
Showing with 415 additions and 129 deletions
CHANGES.txt
View file @ 517e6352
......@@ -17,18 +17,25 @@ Change History
Feature Changes
---------------
Improved error reporting and debugging support:
- Improved error reporting and debugging support:
- Added "logical tracebacks" that show functionally what the buildout
was doing when an error occurs. Don't show a Python traceback
unless the -D option is used.
- Added "logical tracebacks" that show functionally what the buildout
was doing when an error occurs. Don't show a Python traceback
unless the -D option is used.
- Added a -D option that causes the buildout to print a traceback and
start the pdb post-mortem debugger when an error occurs.
- Added a -D option that causes the buildout to print a traceback and
start the pdb post-mortem debugger when an error occurs.
- Warnings are printed for unused options in the buildout section and
installed-part sections. This should make it easier to catch option
misspellings.
- Warnings are printed for unused options in the buildout section and
installed-part sections. This should make it easier to catch option
misspellings.
- Changed the way the installed database (.installed.cfg) is handled
to avoid database corruption when a user breaks out of a buildout
with control-c.
- Don't save an installed database if there are no installed parts or
develop egg links.
1.0.0b21 (2007-03-06)
=====================
......
src/zc/buildout/buildout.py
View file @ 517e6352
......@@ -202,7 +202,8 @@ class Buildout(UserDict.DictMixin):
self._maybe_upgrade()
# load installed data
installed_part_options = self._read_installed_part_options()
(installed_part_options, installed_exists
)= self._read_installed_part_options()
# Remove old develop eggs
self._uninstall(
......@@ -212,6 +213,12 @@ class Buildout(UserDict.DictMixin):
# Build develop eggs
installed_develop_eggs = self._develop()
installed_part_options['buildout']['installed_develop_eggs'
] = installed_develop_eggs
if installed_exists:
self._update_installed(
installed_develop_eggs=installed_develop_eggs)
# get configured and installed part lists
conf_parts = self['buildout']['parts']
......@@ -244,118 +251,147 @@ class Buildout(UserDict.DictMixin):
# compute new part recipe signatures
self._compute_part_signatures(install_parts)
try:
# uninstall parts that are no-longer used or who's configs
# have changed
for part in reversed(installed_parts):
if part in install_parts:
old_options = installed_part_options[part].copy()
installed_files = old_options.pop('__buildout_installed__')
new_options = self.get(part)
if old_options == new_options:
# The options are the same, but are all of the
# installed files still there? If not, we should
# reinstall.
if not installed_files:
continue
for f in installed_files.split('\n'):
if not os.path.exists(self._buildout_path(f)):
break
else:
continue
# output debugging info
for k in old_options:
if k not in new_options:
self._logger.debug("Part: %s, dropped option %s",
part, k)
elif old_options[k] != new_options[k]:
self._logger.debug(
"Part: %s, option %s, %r != %r",
part, k, new_options[k], old_options[k],
)
for k in new_options:
if k not in old_options:
self._logger.debug("Part: %s, new option %s",
part, k)
elif not uninstall_missing:
continue
self._uninstall_part(part, installed_part_options)
installed_parts = [p for p in installed_parts if p != part]
# Check for unused buildout options:
_check_for_unused_options_in_section(self, 'buildout')
# install new parts
for part in install_parts:
signature = self[part].pop('__buildout_signature__')
saved_options = self[part].copy()
recipe = self[part].recipe
if part in installed_parts:
__doing__ = 'Updating %s', part
self._logger.info(*__doing__)
old_options = installed_part_options[part]
old_installed_files = old_options['__buildout_installed__']
try:
update = recipe.update
except AttributeError:
update = recipe.install
self._logger.warning(
"The recipe for %s doesn't define an update "
"method. Using its install method.",
part)
try:
installed_files = update()
except:
installed_parts.remove(part)
self._uninstall(old_installed_files)
raise
if installed_files is None:
installed_files = old_installed_files.split('\n')
# uninstall parts that are no-longer used or who's configs
# have changed
for part in reversed(installed_parts):
if part in install_parts:
old_options = installed_part_options[part].copy()
installed_files = old_options.pop('__buildout_installed__')
new_options = self.get(part)
if old_options == new_options:
# The options are the same, but are all of the
# installed files still there? If not, we should
# reinstall.
if not installed_files:
continue
for f in installed_files.split('\n'):
if not os.path.exists(self._buildout_path(f)):
break
else:
if isinstance(installed_files, str):
installed_files = [installed_files]
else:
installed_files = list(installed_files)
installed_files += [
p for p in old_installed_files.split('\n')
if p and p not in installed_files]
continue
# output debugging info
for k in old_options:
if k not in new_options:
self._logger.debug("Part: %s, dropped option %s",
part, k)
elif old_options[k] != new_options[k]:
self._logger.debug(
"Part: %s, option %s, %r != %r",
part, k, new_options[k], old_options[k],
)
for k in new_options:
if k not in old_options:
self._logger.debug("Part: %s, new option %s",
part, k)
elif not uninstall_missing:
continue
self._uninstall_part(part, installed_part_options)
installed_parts = [p for p in installed_parts if p != part]
if installed_exists:
self._update_installed(parts=' '.join(installed_parts))
# Check for unused buildout options:
_check_for_unused_options_in_section(self, 'buildout')
# install new parts
for part in install_parts:
signature = self[part].pop('__buildout_signature__')
saved_options = self[part].copy()
recipe = self[part].recipe
if part in installed_parts: # update
need_to_save_installed = False
__doing__ = 'Updating %s', part
self._logger.info(*__doing__)
old_options = installed_part_options[part]
old_installed_files = old_options['__buildout_installed__']
try:
update = recipe.update
except AttributeError:
update = recipe.install
self._logger.warning(
"The recipe for %s doesn't define an update "
"method. Using its install method.",
part)
try:
installed_files = update()
except:
installed_parts.remove(part)
self._uninstall(old_installed_files)
if installed_exists:
self._update_installed(
parts=' '.join(installed_parts))
raise
old_installed_files = old_installed_files.split('\n')
if installed_files is None:
installed_files = old_installed_files
else:
__doing__ = 'Installing %s', part
self._logger.info(*__doing__)
installed_files = recipe.install()
if installed_files is None:
self._logger.warning(
"The %s install returned None. A path or "
"iterable os paths should be returned.",
part)
installed_files = ()
if isinstance(installed_files, str):
if isinstance(installed_files, str):
installed_files = [installed_files]
else:
installed_files = list(installed_files)
need_to_save_installed = [
p for p in installed_files
if p not in old_installed_files]
if need_to_save_installed:
installed_files = (old_installed_files
+ need_to_save_installed)
else: # install
need_to_save_installed = True
__doing__ = 'Installing %s', part
self._logger.info(*__doing__)
installed_files = recipe.install()
if installed_files is None:
self._logger.warning(
"The %s install returned None. A path or "
"iterable os paths should be returned.",
part)
installed_files = ()
elif isinstance(installed_files, str):
installed_files = [installed_files]
else:
installed_files = list(installed_files)
installed_part_options[part] = saved_options
saved_options['__buildout_installed__'
] = '\n'.join(installed_files)
saved_options['__buildout_signature__'] = signature
installed_parts = [p for p in installed_parts if p != part]
installed_parts.append(part)
_check_for_unused_options_in_section(self, part)
if need_to_save_installed:
installed_part_options['buildout']['parts'] = (
' '.join(installed_parts))
self._save_installed_options(installed_part_options)
installed_exists = True
else:
assert installed_exists
self._update_installed(parts=' '.join(installed_parts))
installed_part_options[part] = saved_options
saved_options['__buildout_installed__'
] = '\n'.join(installed_files)
saved_options['__buildout_signature__'] = signature
installed_parts = [p for p in installed_parts if p != part]
installed_parts.append(part)
_check_for_unused_options_in_section(self, part)
if installed_develop_eggs:
if not installed_exists:
self._save_installed_options(installed_part_options)
elif (not installed_parts) and installed_exists:
os.remove(self['buildout']['installed'])
finally:
installed_part_options['buildout']['parts'] = (
' '.join(installed_parts))
installed_part_options['buildout']['installed_develop_eggs'
] = installed_develop_eggs
self._save_installed_options(installed_part_options)
def _update_installed(self, **buildout_options):
installed = self['buildout']['installed']
f = open(installed, 'a')
f.write('\n[buildout]\n')
for option, value in buildout_options.items():
_save_option(option, value, f)
f.close()
def _uninstall_part(self, part, installed_part_options):
# ununstall part
......@@ -466,9 +502,11 @@ class Buildout(UserDict.DictMixin):
options[option] = value
result[section] = Options(self, section, options)
return result
return result, True
else:
return {'buildout': Options(self, 'buildout', {'parts': ''})}
return ({'buildout': Options(self, 'buildout', {'parts': ''})},
False,
)
def _uninstall(self, installed):
for f in installed.split('\n'):
......@@ -891,17 +929,20 @@ def _quote_spacey_nl(match):
)
return result
def _save_option(option, value, f):
value = _spacey_nl.sub(_quote_spacey_nl, value)
if value.startswith('\n\t'):
value = '%(__buildout_space_n__)s' + value[2:]
if value.endswith('\n\t'):
value = value[:-2] + '%(__buildout_space_n__)s'
print >>f, option, '=', value
def _save_options(section, options, f):
print >>f, '[%s]' % section
items = options.items()
items.sort()
for option, value in items:
value = _spacey_nl.sub(_quote_spacey_nl, value)
if value.startswith('\n\t'):
value = '%(__buildout_space_n__)s' + value[2:]
if value.endswith('\n\t'):
value = value[:-2] + '%(__buildout_space_n__)s'
print >>f, option, '=', value
_save_option(option, value, f)
def _open(base, filename, seen):
"""Open a configuration file and return the result as a dictionary,
......
src/zc/buildout/buildout.txt
View file @ 517e6352
......@@ -1756,8 +1756,20 @@ information on installed parts. This option is initialized to
".installed.cfg", but it can be overridded in the configuration file
or on the command line:
>>> os.remove('.installed.cfg')
>>> write('buildout.cfg',
... """
... [buildout]
... develop = recipes
... parts = debug
...
... [debug]
... recipe = recipes:debug
... """)
>>> print system(buildout+' buildout:installed=inst.cfg'),
buildout: Develop: /sample-buildout/recipes
buildout: Installing debug
recipe recipes:debug
>>> ls(sample_buildout)
- b1.cfg
......@@ -1771,12 +1783,14 @@ or on the command line:
d parts
d recipes
The installation database can be disabled by supplying an empty
buildout installed opttion:
>>> os.remove('inst.cfg')
>>> print system(buildout+' buildout:installed='),
buildout: Develop: /sample-buildout/recipes
buildout: Installing debug
recipe recipes:debug
>>> ls(sample_buildout)
- b1.cfg
......@@ -1790,6 +1804,28 @@ buildout installed opttion:
d recipes
Note that there will be no installation database if there are no
parts:
>>> write('buildout.cfg',
... """
... [buildout]
... parts =
... """)
>>> print system(buildout+' buildout:installed=inst.cfg'),
>>> ls(sample_buildout)
- b1.cfg
- b2.cfg
- base.cfg
d bin
- buildout.cfg
d develop-eggs
d eggs
d parts
d recipes
Extensions
----------
......
src/zc/buildout/tests.py
View file @ 517e6352
......@@ -1240,11 +1240,11 @@ uninstall
[buildout]
...
[foo]
__buildout_installed__ = c
__buildout_installed__ = a
b
c
d
e
a
b
__buildout_signature__ = ...
"""
......@@ -1386,6 +1386,208 @@ def whine_about_unused_options():
buildout: Unused options for foo: 'z'
'''
def abnormal_exit():
"""
People sometimes hit control-c while running a builout. We need to make
sure that the installed database Isn't corrupted. To test this, we'll create
some evil recipes that exit uncleanly:
>>> mkdir('recipes')
>>> write('recipes', 'recipes.py',
... '''
... import os
...
... class Clean:
... def __init__(*_): pass
... def install(_): return ()
... def update(_): pass
...
... class EvilInstall(Clean):
... def install(_): os._exit(1)
...
... class EvilUpdate(Clean):
... def update(_): os._exit(1)
... ''')
>>> write('recipes', 'setup.py',
... '''
... import setuptools
... setuptools.setup(name='recipes',
... entry_points = {
... 'zc.buildout': [
... 'clean = recipes:Clean',
... 'evil_install = recipes:EvilInstall',
... 'evil_update = recipes:EvilUpdate',
... 'evil_uninstall = recipes:Clean',
... ],
... },
... )
... ''')
Now let's look at 3 cases:
1. We exit during installation after installing some other parts:
>>> write('buildout.cfg',
... '''
... [buildout]
... develop = recipes
... parts = p1 p2 p3 p4
...
... [p1]
... recipe = recipes:clean
...
... [p2]
... recipe = recipes:clean
...
... [p3]
... recipe = recipes:evil_install
...
... [p4]
... recipe = recipes:clean
... ''')
>>> print system(buildout),
buildout: Develop: /sample-buildout/recipes
buildout: Installing p1
buildout: Installing p2
buildout: Installing p3
>>> print system(buildout),
buildout: Develop: /sample-buildout/recipes
buildout: Updating p1
buildout: Updating p2
buildout: Installing p3
>>> print system(buildout+' buildout:parts='),
buildout: Develop: /sample-buildout/recipes
buildout: Uninstalling p2
buildout: Uninstalling p1
2. We exit while updating:
>>> write('buildout.cfg',
... '''
... [buildout]
... develop = recipes
... parts = p1 p2 p3 p4
...
... [p1]
... recipe = recipes:clean
...
... [p2]
... recipe = recipes:clean
...
... [p3]
... recipe = recipes:evil_update
...
... [p4]
... recipe = recipes:clean
... ''')
>>> print system(buildout),
buildout: Develop: /sample-buildout/recipes
buildout: Installing p1
buildout: Installing p2
buildout: Installing p3
buildout: Installing p4
>>> print system(buildout),
buildout: Develop: /sample-buildout/recipes
buildout: Updating p1
buildout: Updating p2
buildout: Updating p3
>>> print system(buildout+' buildout:parts='),
buildout: Develop: /sample-buildout/recipes
buildout: Uninstalling p2
buildout: Uninstalling p1
buildout: Uninstalling p4
buildout: Uninstalling p3
3. We exit while installing or updating after uninstalling:
>>> write('buildout.cfg',
... '''
... [buildout]
... develop = recipes
... parts = p1 p2 p3 p4
...
... [p1]
... recipe = recipes:evil_update
...
... [p2]
... recipe = recipes:clean
...
... [p3]
... recipe = recipes:clean
...
... [p4]
... recipe = recipes:clean
... ''')
>>> print system(buildout),
buildout: Develop: /sample-buildout/recipes
buildout: Installing p1
buildout: Installing p2
buildout: Installing p3
buildout: Installing p4
>>> write('buildout.cfg',
... '''
... [buildout]
... develop = recipes
... parts = p1 p2 p3 p4
...
... [p1]
... recipe = recipes:evil_update
...
... [p2]
... recipe = recipes:clean
...
... [p3]
... recipe = recipes:clean
...
... [p4]
... recipe = recipes:clean
... x = 1
... ''')
>>> print system(buildout),
buildout: Develop: /sample-buildout/recipes
buildout: Uninstalling p4
buildout: Updating p1
>>> write('buildout.cfg',
... '''
... [buildout]
... develop = recipes
... parts = p1 p2 p3 p4
...
... [p1]
... recipe = recipes:clean
...
... [p2]
... recipe = recipes:clean
...
... [p3]
... recipe = recipes:clean
...
... [p4]
... recipe = recipes:clean
... ''')
>>> print system(buildout),
buildout: Develop: /sample-buildout/recipes
buildout: Uninstalling p1
buildout: Installing p1
buildout: Updating p2
buildout: Updating p3
buildout: Installing p4
"""
######################################################################
def create_sample_eggs(test, executable=sys.executable):
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment
GitLab Nexedi Edition | About GitLab | About Nexedi | 沪ICP备2021021310号-2 | 沪ICP备2021021310号-7