Commit 4270184f authored by Jérome Perrin's avatar Jérome Perrin Committed by Rafael Monnerat

cli: new "node prune" command

Used to remove shared parts that are no longer referenced.

With shared part installed for example in `/opt/slapgrd/shared/name/HASH`
the heuristic is to assume that if `HASH` is present in a software's
`.installed.cfg`, then this software is using this shared part.
Similar, if `HASH` is present in
`/opt/slapgrid/shared/another/ANOTHER_HASH/.slapos*signature` then share
part another for hash `ANOTHER_HASH` is using the shared part.

This also have some support to recursively look in instances and if they
contain a `slapos.cfg` file, look for software and shared parts to see if
they use the shared part.

In other cases, this shared part can be deleted.

/reviewed-on !139
parent 22c3d895
...@@ -105,6 +105,7 @@ setup(name=name, ...@@ -105,6 +105,7 @@ setup(name=name,
'node promise = slapos.cli.slapgrid:PromiseCommand', 'node promise = slapos.cli.slapgrid:PromiseCommand',
'node boot = slapos.cli.boot:BootCommand', 'node boot = slapos.cli.boot:BootCommand',
'node collect = slapos.cli.collect:CollectCommand', 'node collect = slapos.cli.collect:CollectCommand',
'node prune = slapos.cli.prune:PruneCommand',
# SlapOS client commands # SlapOS client commands
'console = slapos.cli.console:ConsoleCommand', 'console = slapos.cli.console:ConsoleCommand',
'configure local = slapos.cli.configure_local:ConfigureLocalCommand', 'configure local = slapos.cli.configure_local:ConfigureLocalCommand',
......
# -*- coding: utf-8 -*-
##############################################################################
#
# Copyright (c) 2019 Vifib SARL and Contributors.
# All Rights Reserved.
#
# WARNING: This program as such is intended to be used by professional
# programmers who take the whole responsibility of assessing all potential
# consequences resulting from its eventual inadequacies and bugs
# End users who are looking for a ready-to-use solution with commercial
# guarantees and support are strongly adviced to contract a Free Software
# Service Company
#
# This program is Free Software; you can redistribute it and/or
# modify it under the terms of the GNU Lesser General Public License
# as published by the Free Software Foundation; either version 2.1
# of the License, or (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU Lesser General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
#
##############################################################################
import sys
import glob
import os
import six.moves.configparser as configparser
from slapos.cli.config import ConfigCommand
from slapos.grid.slapgrid import merged_options
class PruneCommand(ConfigCommand):
"""Clean up unused shared slapos.recipe.cmmi parts.
This simple script does not detect inter-dependencies and needs to run
multiple times. For example, if A depend on B, when B is not used, B will
be removed on first run and A will then become unused.
"""
command_group = 'node'
def get_parser(self, prog_name):
ap = super(PruneCommand, self).get_parser(prog_name)
ap.add_argument(
'--dry-run', help="Don't delete, just log", action='store_true')
return ap
def take_action(self, args):
configp = self.fetch_config(args)
options = merged_options(args, configp)
if not options.get('shared_part_list'):
self.app.log.error('No shared_part_list options in slapos config')
sys.exit(-1)
pidfile_software = options.get('pidfile_software')
if not args.dry_run and pidfile_software and os.path.exists(
pidfile_software):
self.app.log.error('Cannot prune while software is running')
sys.exit(-1)
sys.exit(do_prune(self.app.log, options, args.dry_run))
def do_prune(logger, options, dry_run):
shared_root = options['shared_part_list'].splitlines()[-1].strip()
logger.warning("Pruning shared directories at %s", shared_root)
signatures = getUsageSignatureFromSoftwareAndSharedPart(
logger, options['software_root'], shared_root)
# recursively look in instance
signatures.update(getUsageSignaturesFromSubInstance(logger, options['instance_root']))
for shared_part in glob.glob(os.path.join(shared_root, '*', '*')):
logger.debug("checking shared part %s", shared_part)
h = os.path.basename(shared_part)
for soft, installed_cfg in signatures.items():
if h in installed_cfg:
logger.debug("It is used in %s", soft)
break
else:
if not dry_run:
rmtree(shared_part)
logger.warning(
'Unusued shared parts at %s%s', shared_part,
'' if dry_run else ' ... removed')
def getUsageSignaturesFromSubInstance(logger, instance_root):
"""Look at instances in instance_root to find used shared parts,
if instances are recursive slapos.
The heuristic is that if an instance contain a file named slapos.cfg,
this is a recursive slapos.
"""
signatures = {}
for slapos_cfg in getInstanceSlaposCfgList(logger, instance_root):
cfg = readSlaposCfg(logger, slapos_cfg)
shared_root = None
if cfg['shared_part_list']:
shared_root = cfg['shared_part_list'][-1]
signatures.update(
getUsageSignatureFromSoftwareAndSharedPart(
logger, cfg['software_root'], shared_root))
signatures.update(
getUsageSignaturesFromSubInstance(logger, cfg['instance_root'])
)
return signatures
def getInstanceSlaposCfgList(logger, instance_root):
"""Find all slapos.cfg from instance directory, as instance
can contain recursive slapos (that refer parts from outer slapos).
"""
for root, _, filenames in os.walk(instance_root):
if 'slapos.cfg' in filenames:
yield os.path.join(root, 'slapos.cfg')
def readSlaposCfg(logger, path):
"""Read a slapos.cfg found in an instance directory.
"""
logger.debug('Reading config at %s', path)
parser = configparser.ConfigParser({'shared_part_list': ''})
parser.read([path])
cfg = {
'software_root': parser.get('slapos', 'software_root'),
'instance_root': parser.get('slapos', 'instance_root'),
'shared_part_list': parser.get('slapos', 'shared_part_list').splitlines()
}
logger.debug('Read config: %s', cfg)
return cfg
def getUsageSignatureFromSoftwareAndSharedPart(
logger, software_root, shared_root):
"""Look in all softwares and shared parts to collect the signatures
that are used.
"""
signatures = {}
for installed_cfg in glob.glob(os.path.join(software_root, '*',
'.installed.cfg')):
with open(installed_cfg) as f:
signatures[installed_cfg] = f.read()
if shared_root:
for shared_signature in glob.glob(os.path.join(shared_root, '*', '*',
'.*signature')):
with open(shared_signature) as f:
signatures[shared_signature] = f.read()
return signatures
# XXX copied from https://lab.nexedi.com/nexedi/erp5/blob/31804f683fd36322fb38aeb9654bee70cebe4fdb/erp5/util/testnode/Utils.py
# TODO: move to shared place or ... isn't there already such an utility function in slapos.core ?
import shutil
import errno
import six
from six.moves import map
try:
PermissionError
except NameError: # make pylint happy on python2...
PermissionError = Exception
def rmtree(path):
"""Delete a path recursively.
Like shutil.rmtree, but supporting the case that some files or folder
might have been marked read only. """
def chmod_retry(func, failed_path, exc_info):
"""Make sure the directories are executable and writable.
"""
# Depending on the Python version, the following items differ.
if six.PY3:
expected_error_type = PermissionError
expected_func = os.lstat
else:
expected_error_type = OSError
expected_func = os.listdir
e = exc_info[1]
if isinstance(e, expected_error_type):
if e.errno == errno.ENOENT:
# because we are calling again rmtree on listdir errors, this path might
# have been already deleted by the recursive call to rmtree.
return
if e.errno == errno.EACCES:
if func is expected_func:
os.chmod(failed_path, 0o700)
# corner case to handle errors in listing directories.
# https://bugs.python.org/issue8523
return shutil.rmtree(failed_path, onerror=chmod_retry)
# If parent directory is not writable, we still cannot delete the file.
# But make sure not to change the parent of the folder we are deleting.
if failed_path != path:
os.chmod(os.path.dirname(failed_path), 0o700)
return func(failed_path)
raise e # XXX make pylint happy
shutil.rmtree(path, onerror=chmod_retry)
# / erp5/util/testnode/Utils.py code
##############################################################################
#
# Copyright (c) 2010 Vifib SARL and Contributors. All Rights Reserved.
#
# WARNING: This program as such is intended to be used by professional
# programmers who take the whole responsibility of assessing all potential
# consequences resulting from its eventual inadequacies and bugs
# End users who are looking for a ready-to-use solution with commercial
# guarantees and support are strongly adviced to contract a Free Software
# Service Company
#
# This program is Free Software; you can redistribute it and/or
# modify it under the terms of the GNU General Public License
# as published by the Free Software Foundation; either version 3
# of the License, or (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
#
##############################################################################
import tempfile
import logging
import os
import shutil
import unittest
import slapos.client
try:
import mock
except ImportError:
from unittest import mock
from slapos.cli.prune import do_prune
class TestPrune(unittest.TestCase):
def setUp(self):
self.logger = mock.create_autospec(logging.getLogger())
self.base_directory = tempfile.mkdtemp()
self.shared_part_root = os.path.join(self.base_directory, 'shared')
self.software_root = os.path.join(self.base_directory, 'software')
self.instance_root = os.path.join(self.base_directory, 'instance')
for d in (self.shared_part_root, self.software_root, self.instance_root):
os.mkdir(d)
self.addCleanup(shutil.rmtree, self.base_directory)
self.config = {
'shared_part_list': self.shared_part_root,
'software_root': self.software_root,
'instance_root': self.instance_root,
}
def _createFakeSoftware(self, name, using='', software_root=None):
software_path = os.path.join(software_root or self.software_root, name)
os.mkdir(software_path)
with open(os.path.join(software_path, '.installed.cfg'), 'w') as f:
f.write("""[buildout]
using = {}
""".format(using))
return software_path
def _createSharedPart(self, name, using='', shared_part_root=None):
shared_part_name = os.path.join(shared_part_root or self.shared_part_root, name)
shared_part_with_version = os.path.join(shared_part_name, name)
os.makedirs(shared_part_with_version)
with open(os.path.join(shared_part_with_version,
'.slapos.recipe.cmmi.signature'), 'w') as f:
f.write("""signature
using = {}
""".format(using))
return shared_part_with_version
def test_simple_not_used_share_part(self):
not_used = self._createSharedPart('not_used')
used = self._createSharedPart('used_part')
self._createFakeSoftware(self.id(), using=used)
do_prune(self.logger, self.config, False)
self.assertTrue(os.path.exists(used))
self.assertFalse(os.path.exists(not_used))
self.logger.warning.assert_called_with(
'Unusued shared parts at %s%s', not_used, ' ... removed')
def test_dry_run(self):
not_used = self._createSharedPart('not_used')
used = self._createSharedPart('used_part')
self._createFakeSoftware(self.id(), using=used)
do_prune(self.logger, self.config, True)
self.assertTrue(os.path.exists(used))
self.assertTrue(os.path.exists(not_used))
self.logger.warning.assert_called_with(
'Unusued shared parts at %s%s', not_used, '')
def test_shared_part_used_in_another_shared_part(self):
not_used = self._createSharedPart('not_used')
indirectly_used_part = self._createSharedPart('part_used_indirectly')
directly_used_part = self._createSharedPart('directly_used_part', using=indirectly_used_part)
self._createFakeSoftware(self.id(), using=directly_used_part)
do_prune(self.logger, self.config, False)
self.assertTrue(os.path.exists(indirectly_used_part))
self.assertTrue(os.path.exists(directly_used_part))
self.assertFalse(os.path.exists(not_used))
self.logger.warning.assert_called_with(
'Unusued shared parts at %s%s', not_used, ' ... removed')
def test_shared_part_used_in_recursive_instance(self):
used_in_software_from_instance = self._createSharedPart('used_in_software_from_instance')
used_in_shared_part_from_instance = self._createSharedPart('used_in_shared_part_from_instance')
not_used = self._createSharedPart('not_used')
# create instance
instance = os.path.join(self.instance_root, 'slappart0')
instance_etc = os.path.join(instance, 'etc')
instance_software = os.path.join(instance, 'software')
instance_instance = os.path.join(instance, 'instance')
instance_shared_part_root = os.path.join(instance, 'shared')
for p in instance_etc, instance_software, instance_instance, instance_shared_part_root:
os.makedirs(p)
instance_slapos_cfg = os.path.join(instance_etc, 'slapos.cfg')
with open(instance_slapos_cfg, 'w') as f:
f.write('''
[slapos]
software_root = {instance_software}
instance_root = {instance_instance}
shared_part_list =
{self.shared_part_root}
{instance_shared_part_root}
'''.format(**locals()))
# install software and shared part in instance
software_in_instance = self._createFakeSoftware(
'soft_in_instance',
using=used_in_software_from_instance,
software_root=instance_software
)
shared_part_in_instance = self._createSharedPart(
'shared_part_in_instance',
using=used_in_shared_part_from_instance,
shared_part_root=instance_shared_part_root
)
unused_shared_part_in_instance = self._createSharedPart(
'unused_shared_part_in_instance',
shared_part_root=instance_shared_part_root
) # could be pruned, but prune is not recursive.
do_prune(self.logger, self.config, False)
self.logger.debug.assert_any_call(
'Reading config at %s', instance_slapos_cfg)
self.assertTrue(os.path.exists(used_in_software_from_instance))
self.assertTrue(os.path.exists(used_in_software_from_instance))
self.assertTrue(os.path.exists(software_in_instance))
self.assertTrue(os.path.exists(shared_part_in_instance))
self.assertFalse(os.path.exists(not_used))
self.logger.warning.assert_called_with(
'Unusued shared parts at %s%s', not_used, ' ... removed')
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment