Commit 2731aa6e authored by Jérome Perrin's avatar Jérome Perrin

Introduce an InstallMonitor

To monitor which files are created by buildout when installing parts

This version is based on git because it was easy to implement, but with
git tools, it's easy to inspect what exactly installing a part had
changed and also to revert to a previous state by using git commands
with the same `--git-dir` and `--work-dir`.
parent c970f5cf
...@@ -51,6 +51,8 @@ import pprint ...@@ -51,6 +51,8 @@ import pprint
import zc.buildout import zc.buildout
import zc.buildout.download import zc.buildout.download
from zc.buildout.install_monitor import InstallMonitor
PY3 = sys.version_info[0] == 3 PY3 = sys.version_info[0] == 3
if PY3: if PY3:
text_type = str text_type = str
...@@ -257,6 +259,8 @@ class Buildout(DictMixin): ...@@ -257,6 +259,8 @@ class Buildout(DictMixin):
data = dict(buildout=_buildout_default_options.copy()) data = dict(buildout=_buildout_default_options.copy())
self._buildout_dir = os.getcwd() self._buildout_dir = os.getcwd()
self._install_monitor = InstallMonitor(self)
if config_file and not _isurl(config_file): if config_file and not _isurl(config_file):
config_file = os.path.abspath(config_file) config_file = os.path.abspath(config_file)
base = os.path.dirname(config_file) base = os.path.dirname(config_file)
...@@ -634,6 +638,7 @@ class Buildout(DictMixin): ...@@ -634,6 +638,7 @@ class Buildout(DictMixin):
self.install(()) self.install(())
def install(self, install_args): def install(self, install_args):
self._install_monitor.begin_install(install_args)
try: try:
self._install_parts(install_args) self._install_parts(install_args)
finally: finally:
...@@ -644,6 +649,7 @@ class Buildout(DictMixin): ...@@ -644,6 +649,7 @@ class Buildout(DictMixin):
del self.installed_part_options del self.installed_part_options
if self.show_picked_versions or self.update_versions_file: if self.show_picked_versions or self.update_versions_file:
self._print_picked_versions() self._print_picked_versions()
self._install_monitor.end_install()
self._unload_extensions() self._unload_extensions()
def _install_parts(self, install_args): def _install_parts(self, install_args):
...@@ -754,7 +760,10 @@ class Buildout(DictMixin): ...@@ -754,7 +760,10 @@ class Buildout(DictMixin):
elif not uninstall_missing: elif not uninstall_missing:
continue continue
self._install_monitor.begin_uninstall_part(part, installed_part_options)
self._uninstall_part(part, installed_part_options) self._uninstall_part(part, installed_part_options)
self._install_monitor.end_uninstall_part(part, installed_part_options)
installed_parts = [p for p in installed_parts if p != part] installed_parts = [p for p in installed_parts if p != part]
installed_part_options['buildout']['parts'] = ( installed_part_options['buildout']['parts'] = (
' '.join(installed_parts)) ' '.join(installed_parts))
...@@ -785,7 +794,9 @@ class Buildout(DictMixin): ...@@ -785,7 +794,9 @@ class Buildout(DictMixin):
part) part)
try: try:
self._install_monitor.begin_install_part(part, update=True)
updated_files = self[part]._call(update) updated_files = self[part]._call(update)
self._install_monitor.end_install_part(part, updated_files, update=True)
except Exception: except Exception:
installed_parts.remove(part) installed_parts.remove(part)
self._uninstall(installed_files) self._uninstall(installed_files)
...@@ -804,7 +815,9 @@ class Buildout(DictMixin): ...@@ -804,7 +815,9 @@ class Buildout(DictMixin):
self._logger.info(*__doing__) self._logger.info(*__doing__)
if self.dry_run: if self.dry_run:
continue continue
self._install_monitor.begin_install_part(part)
installed_files = self[part]._call(recipe.install) installed_files = self[part]._call(recipe.install)
self._install_monitor.end_install_part(part, installed_files)
if installed_files is None: if installed_files is None:
self._logger.warning( self._logger.warning(
"The %s install returned None. A path or " "The %s install returned None. A path or "
......
import os
import logging
import subprocess
import time
import textwrap
import pprint
"""
Monitor what files are installed by buildout
# Notes
## Packaging
Instead of direct patches to buildout, this could be a buildout extension
* parts installation: ? monkey patchs ?
* `end_install` -> `zc.buildout.unloadextension`
## random notes about monitor
I had in mind we could have a ptrace monitor that knows were part is
supposed to install and abort the process before it writes in a place
where it is not supposed to write.
Hard part is to defined what's an "allowed directory".
For "custom" recipes, a section can install in:
* `${buildout:parts-directory}/${:_buildout_section_name}`
* `${buildout:bin-directory}` ( but is this really needed ? if it is, it must
be with restrictions, to prevent a part from overriding scripts from another part)
* the corresponding cache subdirectory for recipes using cache
A `LD_PRELOAD` or [PRoot](https://github.com/proot-me/PRoot) based solution seems also
possible, but the dynamic aspects of the cache seems to make this hard.
For eggs, installing egg `X` should only be able to write to `eggs/X-{version_spec}`.
Setuptools have some [sandboxing](https://github.com/pypa/setuptools/blob/c2262d9fe4eaac507ff128ae60b6682e8d132e4d/setuptools/sandbox.py) support (that I have not studied at this point), but it only monitor the
python part of the installation process.
Develop eggs seems to be similar - only write to `develop-eggs/X-{version_spec}` but this also
probably need to include some hash in version spec.
( because `eggs` and `develop-eggs` can be shared too )
Related bugs that could be solved by such a "don't write here" approach:
https://nexedi.erp5.net/bug_module/20150413-15EC498
https://nexedi.erp5.net/bug_module/20110718-8E43A9
"""
class GitInstallMonitor(object):
"""Stupid monitor making a (big) git commit to hold everything installed by each part.
Assumes git command is in path
"""
def __init__(self, buildout):
self._buildout = buildout
self._logger = logging.getLogger('zc.buildout.installmonitor')
self._start_install_time = {}
def _record_changes(self, message):
self._logger.debug("Adding %s to git, this may take some time", self._work_tree)
if self._git("status", "--porcelain"):
self._git("add", self._work_tree)
self._git("commit", "-m", message)
self._logger.debug("Added %s to git", self._work_tree)
else:
self._logger.info("No changes to record")
def _git(self, *args):
return subprocess.check_output(
(
'git',
'--git-dir', self._git_dir,
'--work-tree', self._work_tree,
) + args,
env={
'GIT_COMMITTER_NAME': 'SlapOS Install Monitor',
'GIT_COMMITTER_EMAIL': 'noemail@example.com',
'GIT_AUTHOR_NAME': 'SlapOS Install Monitor',
'GIT_AUTHOR_EMAIL': 'noemail@example.com',
})
def begin_install(self, install_args):
"""Called at the beginning of buildout installation.
"""
self._start_install_time[self] = time.time()
self._work_tree = os.path.dirname(
os.path.commonprefix((
self._buildout['buildout']['parts-directory'],
self._buildout['buildout']['directory'],
self._buildout['buildout']['slapos-recipe-cmmi-shared-path'],
)))
self._git_dir = os.path.join(
self._buildout['buildout']['directory'],
'slapos.gitinstallmonitor.git'
)
self._git("init")
self._logger.info(
"Initialised git dir in %s with work tree %s",
self._git_dir,
self._work_tree
)
# ignore git dir ( as a path relative to work tree )
with open(os.path.join(self._git_dir, 'info', 'exclude'), 'w') as gitignore:
gitignore.write(self._git_dir[len(self._work_tree):])
self._record_changes("Begin install with args {install_args}".format(**locals()))
def end_install(self):
took = time.time() - self._start_install_time[self]
self._record_changes("Finished installation in {took:.2f} seconds".format(**locals()))
def begin_install_part(self, name, update=False):
self._start_install_time[name] = time.time()
def end_install_part(self, name, installed_files, update=False):
took = time.time() - self._start_install_time[name]
recipe = self._buildout[name]['recipe']
installed_files = '\n '.join(installed_files or [])
options = '\n '.join(pprint.pformat(dict(self._buildout[name])).splitlines())
did = "Updated" if update else "Installed"
self._record_changes(
textwrap.dedent("""\
{did} part {name} in {took:.2f} seconds
Recipe: {recipe}
Options:
{options}
Installed files:
{installed_files}
""").format(**locals()))
def begin_uninstall_part(self, name, installed_part_options):
self._start_install_time[name] = time.time()
def end_uninstall_part(self, name, installed_part_options):
took = time.time() - self._start_install_time[name]
self._record_changes(
textwrap.dedent("""\
Uninstalled part {name} in {took:.2f} seconds
Installed part options:
{installed_part_options}
""").format(**locals()))
InstallMonitor = GitInstallMonitor
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment