Commit eb4d16a6 authored by Kazuhiko Shiozaki's avatar Kazuhiko Shiozaki

Write .installed.cfg only once, in safe way and only if there's any change.

Also, updating a part does not put it anymore at the end of the list of
installed parts, that was making .installed.cfg too big.
parent c74466ed
...@@ -32,6 +32,7 @@ from cStringIO import StringIO ...@@ -32,6 +32,7 @@ from cStringIO import StringIO
import copy import copy
import datetime import datetime
import distutils.errors import distutils.errors
import errno
import glob import glob
import itertools import itertools
import logging import logging
...@@ -173,6 +174,12 @@ def _print_annotate(data): ...@@ -173,6 +174,12 @@ def _print_annotate(data):
line = ' ' line = ' '
print_() print_()
def _remove_ignore_missing(path):
try:
os.remove(path)
except OSError, e:
if e.errno != errno.ENOENT:
raise
def _unannotate_section(section): def _unannotate_section(section):
for key in section: for key in section:
...@@ -615,6 +622,15 @@ class Buildout(DictMixin): ...@@ -615,6 +622,15 @@ class Buildout(DictMixin):
self.install(()) self.install(())
def install(self, install_args): def install(self, install_args):
try:
self._install_parts(install_args)
finally:
self._save_installed_options()
if self.show_picked_versions or self.update_versions_file:
self._print_picked_versions()
self._unload_extensions()
def _install_parts(self, install_args):
__doing__ = 'Installing.' __doing__ = 'Installing.'
self._load_extensions() self._load_extensions()
...@@ -628,28 +644,23 @@ class Buildout(DictMixin): ...@@ -628,28 +644,23 @@ class Buildout(DictMixin):
self._maybe_upgrade() self._maybe_upgrade()
# load installed data # load installed data
(installed_part_options, installed_exists self.installed_part_options = self._read_installed_part_options()
)= self._read_installed_part_options()
# Remove old develop eggs # Remove old develop eggs
self._uninstall( self._uninstall(
installed_part_options['buildout'].get( self.installed_part_options['buildout'].get(
'installed_develop_eggs', '') 'installed_develop_eggs', '')
) )
# Build develop eggs # Build develop eggs
installed_develop_eggs = self._develop() installed_develop_eggs = self._develop()
installed_part_options['buildout']['installed_develop_eggs' self.installed_part_options['buildout']['installed_develop_eggs'
] = installed_develop_eggs ] = installed_develop_eggs
if installed_exists:
self._update_installed(
installed_develop_eggs=installed_develop_eggs)
# get configured and installed part lists # get configured and installed part lists
conf_parts = self['buildout']['parts'] conf_parts = self['buildout']['parts']
conf_parts = conf_parts and conf_parts.split() or [] conf_parts = conf_parts and conf_parts.split() or []
installed_parts = installed_part_options['buildout']['parts'] installed_parts = self.installed_part_options['buildout']['parts']
installed_parts = installed_parts and installed_parts.split() or [] installed_parts = installed_parts and installed_parts.split() or []
if install_args: if install_args:
...@@ -681,7 +692,7 @@ class Buildout(DictMixin): ...@@ -681,7 +692,7 @@ class Buildout(DictMixin):
# have changed # have changed
for part in reversed(installed_parts): for part in reversed(installed_parts):
if part in install_parts: if part in install_parts:
old_options = installed_part_options[part].copy() old_options = self.installed_part_options[part].copy()
installed_files = old_options.pop('__buildout_installed__') installed_files = old_options.pop('__buildout_installed__')
new_options = self.get(part) new_options = self.get(part)
if old_options == new_options: if old_options == new_options:
...@@ -715,12 +726,9 @@ class Buildout(DictMixin): ...@@ -715,12 +726,9 @@ class Buildout(DictMixin):
elif not uninstall_missing: elif not uninstall_missing:
continue continue
self._uninstall_part(part, installed_part_options) self._uninstall_part(part, self.installed_part_options)
installed_parts = [p for p in installed_parts if p != part] installed_parts = [p for p in installed_parts if p != part]
if installed_exists:
self._update_installed(parts=' '.join(installed_parts))
# Check for unused buildout options: # Check for unused buildout options:
_check_for_unused_options_in_section(self, 'buildout') _check_for_unused_options_in_section(self, 'buildout')
...@@ -730,10 +738,9 @@ class Buildout(DictMixin): ...@@ -730,10 +738,9 @@ class Buildout(DictMixin):
saved_options = self[part].copy() saved_options = self[part].copy()
recipe = self[part].recipe recipe = self[part].recipe
if part in installed_parts: # update if part in installed_parts: # update
need_to_save_installed = False
__doing__ = 'Updating %s.', part __doing__ = 'Updating %s.', part
self._logger.info(*__doing__) self._logger.info(*__doing__)
old_options = installed_part_options[part] old_options = self.installed_part_options[part]
old_installed_files = old_options['__buildout_installed__'] old_installed_files = old_options['__buildout_installed__']
try: try:
...@@ -750,9 +757,8 @@ class Buildout(DictMixin): ...@@ -750,9 +757,8 @@ class Buildout(DictMixin):
except: except:
installed_parts.remove(part) installed_parts.remove(part)
self._uninstall(old_installed_files) self._uninstall(old_installed_files)
if installed_exists: self.installed_part_options['buildout']['parts'] = (
self._update_installed( ' '.join(installed_parts))
parts=' '.join(installed_parts))
raise raise
old_installed_files = old_installed_files.split('\n') old_installed_files = old_installed_files.split('\n')
...@@ -764,19 +770,15 @@ class Buildout(DictMixin): ...@@ -764,19 +770,15 @@ class Buildout(DictMixin):
else: else:
installed_files = list(installed_files) installed_files = list(installed_files)
need_to_save_installed = [
p for p in installed_files
if p not in old_installed_files]
if need_to_save_installed:
installed_files = (old_installed_files
+ need_to_save_installed)
else: # install else: # install
need_to_save_installed = True
__doing__ = 'Installing %s.', part __doing__ = 'Installing %s.', part
self._logger.info(*__doing__) self._logger.info(*__doing__)
try:
installed_files = self[part]._call(recipe.install) installed_files = self[part]._call(recipe.install)
except:
self.installed_part_options['buildout']['parts'] = (
' '.join(installed_parts))
raise
if installed_files is None: if installed_files is None:
self._logger.warning( self._logger.warning(
"The %s install returned None. A path or " "The %s install returned None. A path or "
...@@ -788,7 +790,7 @@ class Buildout(DictMixin): ...@@ -788,7 +790,7 @@ class Buildout(DictMixin):
else: else:
installed_files = list(installed_files) installed_files = list(installed_files)
installed_part_options[part] = saved_options self.installed_part_options[part] = saved_options
saved_options['__buildout_installed__' saved_options['__buildout_installed__'
] = '\n'.join(installed_files) ] = '\n'.join(installed_files)
saved_options['__buildout_signature__'] = signature saved_options['__buildout_signature__'] = signature
...@@ -797,32 +799,11 @@ class Buildout(DictMixin): ...@@ -797,32 +799,11 @@ class Buildout(DictMixin):
installed_parts.append(part) installed_parts.append(part)
_check_for_unused_options_in_section(self, part) _check_for_unused_options_in_section(self, part)
if need_to_save_installed: self.installed_part_options['buildout']['parts'] = (
installed_part_options['buildout']['parts'] = (
' '.join(installed_parts)) ' '.join(installed_parts))
self._save_installed_options(installed_part_options)
installed_exists = True
else:
assert installed_exists
self._update_installed(parts=' '.join(installed_parts))
if installed_develop_eggs:
if not installed_exists:
self._save_installed_options(installed_part_options)
elif (not installed_parts) and installed_exists:
os.remove(self['buildout']['installed'])
if self.show_picked_versions or self.update_versions_file:
self._print_picked_versions()
self._unload_extensions()
def _update_installed(self, **buildout_options): self.installed_part_options['buildout']['parts'] = (
installed = self['buildout']['installed'] ' '.join(installed_parts))
f = open(installed, 'a')
f.write('\n[buildout]\n')
for option, value in list(buildout_options.items()):
_save_option(option, value, f)
f.close()
def _uninstall_part(self, part, installed_part_options): def _uninstall_part(self, part, installed_part_options):
# uninstall part # uninstall part
...@@ -941,11 +922,9 @@ class Buildout(DictMixin): ...@@ -941,11 +922,9 @@ class Buildout(DictMixin):
options[option] = value options[option] = value
result[section] = self.Options(self, section, options) result[section] = self.Options(self, section, options)
return result, True return result
else: else:
return ({'buildout': self.Options(self, 'buildout', {'parts': ''})}, return {'buildout': self.Options(self, 'buildout', {'parts': ''})}
False,
)
def _uninstall(self, installed): def _uninstall(self, installed):
for f in installed.split('\n'): for f in installed.split('\n'):
...@@ -986,16 +965,40 @@ class Buildout(DictMixin): ...@@ -986,16 +965,40 @@ class Buildout(DictMixin):
return ' '.join(installed) return ' '.join(installed)
def _save_installed_options(self, installed_options): def _save_installed_options(self):
installed_options = getattr(self, 'installed_part_options', None)
installed = self['buildout']['installed'] installed = self['buildout']['installed']
if not installed: if not installed_options or not installed:
return return
f = open(installed, 'w') buildout = installed_options['buildout']
_save_options('buildout', installed_options['buildout'], f) installed_parts = buildout['parts'].split()
for part in installed_options['buildout']['parts'].split(): if installed_parts or buildout['installed_develop_eggs']:
print_(file=f) new = StringIO()
_save_options(part, installed_options[part], f) buildout['parts'] = ' '.join(installed_parts)
f.close() _save_options('buildout', buildout, new)
for part in installed_parts:
print >>new
_save_options(part, installed_options[part], new)
new = new.getvalue()
try:
with open(installed) as f:
save = f.read(1+len(new)) != new
except IOError, e:
if e.errno != errno.ENOENT:
raise
save = True
if save:
installed_tmp = installed + ".tmp"
try:
with open(installed_tmp, "w") as f:
f.write(new)
f.flush()
os.fsync(f.fileno())
os.rename(installed_tmp, installed)
finally:
_remove_ignore_missing(installed_tmp)
else:
_remove_ignore_missing(installed)
def _error(self, message, *args): def _error(self, message, *args):
raise zc.buildout.UserError(message % args) raise zc.buildout.UserError(message % args)
...@@ -1395,11 +1398,15 @@ class Options(DictMixin): ...@@ -1395,11 +1398,15 @@ class Options(DictMixin):
def _dosub(self, option, v): def _dosub(self, option, v):
__doing__ = 'Getting option %s:%s.', self.name, option __doing__ = 'Getting option %s:%s.', self.name, option
seen = [(self.name, option)] seen = [(self.name, option)]
v = '$$'.join([self._sub(s, seen) for s in v.split('$$')]) v = '$$'.join([self._sub(s, seen, last=False)
for s in v.split('$$')])
self._cooked[option] = v self._cooked[option] = v
def get(self, option, default=None, seen=None): def get(self, option, default=None, seen=None, last=True):
try: try:
if last:
return self._data[option].replace('$${', '${')
else:
return self._data[option] return self._data[option]
except KeyError: except KeyError:
pass pass
...@@ -1422,16 +1429,20 @@ class Options(DictMixin): ...@@ -1422,16 +1429,20 @@ class Options(DictMixin):
) )
else: else:
seen.append(key) seen.append(key)
v = '$$'.join([self._sub(s, seen) for s in v.split('$$')]) v = '$$'.join([self._sub(s, seen, last=False)
for s in v.split('$$')])
seen.pop() seen.pop()
self._data[option] = v self._data[option] = v
if last:
return v.replace('$${', '${')
else:
return v return v
_template_split = re.compile('([$]{[^}]*})').split _template_split = re.compile('([$]{[^}]*})').split
_simple = re.compile('[-a-zA-Z0-9 ._]+$').match _simple = re.compile('[-a-zA-Z0-9 ._]+$').match
_valid = re.compile('\${[-a-zA-Z0-9 ._]*:[-a-zA-Z0-9 ._]+}$').match _valid = re.compile('\${[-a-zA-Z0-9 ._]*:[-a-zA-Z0-9 ._]+}$').match
def _sub(self, template, seen): def _sub(self, template, seen, last=True):
value = self._template_split(template) value = self._template_split(template)
subs = [] subs = []
for ref in value[1::2]: for ref in value[1::2]:
...@@ -1461,7 +1472,7 @@ class Options(DictMixin): ...@@ -1461,7 +1472,7 @@ class Options(DictMixin):
section = self.name section = self.name
elif section != 'buildout': elif section != 'buildout':
self._dependency.add(section) self._dependency.add(section)
v = self.buildout[section].get(option, None, seen) v = self.buildout[section].get(option, None, seen, last=last)
if v is None: if v is None:
if option == '_buildout_section_name_': if option == '_buildout_section_name_':
v = self.name v = self.name
...@@ -1474,14 +1485,6 @@ class Options(DictMixin): ...@@ -1474,14 +1485,6 @@ class Options(DictMixin):
return ''.join([''.join(v) for v in zip(value[::2], subs)]) return ''.join([''.join(v) for v in zip(value[::2], subs)])
def __getitem__(self, key): def __getitem__(self, key):
try:
v = self._data[key]
if v.startswith(SERIALISED_VALUE_MAGIC):
v = loads(v)
return v
except KeyError:
pass
v = self.get(key) v = self.get(key)
if v is None: if v is None:
raise MissingOption("Missing option: %s:%s" % (self.name, key)) raise MissingOption("Missing option: %s:%s" % (self.name, key))
......
...@@ -85,6 +85,8 @@ supply some input: ...@@ -85,6 +85,8 @@ supply some input:
File "/zc/buildout/buildout.py", line 1352, in main File "/zc/buildout/buildout.py", line 1352, in main
getattr(buildout, command)(args) getattr(buildout, command)(args)
File "/zc/buildout/buildout.py", line 383, in install File "/zc/buildout/buildout.py", line 383, in install
self._install_parts(install_args)
File buildout.py", line 791, in _install_parts
installed_files = self[part]._call(recipe.install) installed_files = self[part]._call(recipe.install)
File "/zc/buildout/buildout.py", line 961, in _call File "/zc/buildout/buildout.py", line 961, in _call
return f() return f()
......
...@@ -1492,7 +1492,7 @@ some evil recipes that exit uncleanly: ...@@ -1492,7 +1492,7 @@ some evil recipes that exit uncleanly:
>>> mkdir('recipes') >>> mkdir('recipes')
>>> write('recipes', 'recipes.py', >>> write('recipes', 'recipes.py',
... ''' ... '''
... import os ... import sys
... ...
... class Clean: ... class Clean:
... def __init__(*_): pass ... def __init__(*_): pass
...@@ -1500,10 +1500,10 @@ some evil recipes that exit uncleanly: ...@@ -1500,10 +1500,10 @@ some evil recipes that exit uncleanly:
... def update(_): pass ... def update(_): pass
... ...
... class EvilInstall(Clean): ... class EvilInstall(Clean):
... def install(_): os._exit(1) ... def install(_): sys.exit(1)
... ...
... class EvilUpdate(Clean): ... class EvilUpdate(Clean):
... def update(_): os._exit(1) ... def update(_): sys.exit(1)
... ''') ... ''')
>>> write('recipes', 'setup.py', >>> write('recipes', 'setup.py',
...@@ -1600,7 +1600,6 @@ Now let's look at 3 cases: ...@@ -1600,7 +1600,6 @@ Now let's look at 3 cases:
Uninstalling p2. Uninstalling p2.
Uninstalling p1. Uninstalling p1.
Uninstalling p4. Uninstalling p4.
Uninstalling p3.
3. We exit while installing or updating after uninstalling: 3. We exit while installing or updating after uninstalling:
...@@ -1676,7 +1675,6 @@ Now let's look at 3 cases: ...@@ -1676,7 +1675,6 @@ Now let's look at 3 cases:
>>> print_(system(buildout), end='') >>> print_(system(buildout), end='')
Develop: '/sample-buildout/recipes' Develop: '/sample-buildout/recipes'
Uninstalling p1.
Installing p1. Installing p1.
Updating p2. Updating p2.
Updating p3. Updating p3.
...@@ -2736,6 +2734,73 @@ def increment_on_command_line(): ...@@ -2736,6 +2734,73 @@ def increment_on_command_line():
recipe='zc.buildout:debug' recipe='zc.buildout:debug'
""" """
def bug_664539_simple_buildout():
r"""
>>> write('buildout.cfg', '''
... [buildout]
... parts = escape
...
... [escape]
... recipe = zc.buildout:debug
... foo = $${nonexistent:option}
... ''')
>>> print system(buildout),
Installing escape.
foo='${nonexistent:option}'
recipe='zc.buildout:debug'
"""
def bug_664539_reference():
r"""
>>> write('buildout.cfg', '''
... [buildout]
... parts = escape
...
... [escape]
... recipe = zc.buildout:debug
... foo = ${:bar}
... bar = $${nonexistent:option}
... ''')
>>> print system(buildout),
Installing escape.
bar='${nonexistent:option}'
foo='${nonexistent:option}'
recipe='zc.buildout:debug'
"""
def bug_664539_complex_buildout():
r"""
>>> write('buildout.cfg', '''
... [buildout]
... parts = escape
...
... [escape]
... recipe = zc.buildout:debug
... foo = ${level1:foo}
...
... [level1]
... recipe = zc.buildout:debug
... foo = ${level2:foo}
...
... [level2]
... recipe = zc.buildout:debug
... foo = $${nonexistent:option}
... ''')
>>> print system(buildout),
Installing level2.
foo='${nonexistent:option}'
recipe='zc.buildout:debug'
Installing level1.
foo='${nonexistent:option}'
recipe='zc.buildout:debug'
Installing escape.
foo='${nonexistent:option}'
recipe='zc.buildout:debug'
"""
def test_constrained_requirement(): def test_constrained_requirement():
""" """
zc.buildout.easy_install._constrained_requirement(constraint, requirement) zc.buildout.easy_install._constrained_requirement(constraint, requirement)
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment