prune: prune dependencies recursively

prune was not recursive when checking dependencies, for example when we
have for example a shared part for openssl that is used only by a shared
part for curl that is not used, we had to run prune twice, the first
iteration would remove curl and only the second iteration would see that
openssl is not used.

This prune in a loop until nothing is pruned by the last iteration. To
support dry-run, we added a ignored_shared_parts parameter to ignore the
parts that are supposed to be deleted.

See merge request !205

slapos/cli/prune.py

@@ -68,29 +68,64 @@ class PruneCommand(ConfigCommand):
     sys.exit(do_prune(self.app.log, options, args.dry_run))
 
 
-def do_prune(logger, options, dry_run):
-  shared_root = options['shared_part_list'].splitlines()[-1].strip()
-  logger.warning("Pruning shared directories at %s", shared_root)
-
+def _prune(
+    logger,
+    shared_root,
+    software_root,
+    instance_root,
+    ignored_shared_parts,
+    dry_run,
+):
   signatures = getUsageSignatureFromSoftwareAndSharedPart(
-      logger, options['software_root'], shared_root)
+      logger, software_root, shared_root, ignored_shared_parts)
 
   # recursively look in instance
-  signatures.update(getUsageSignaturesFromSubInstance(logger, options['instance_root']))
+  signatures.update(getUsageSignaturesFromSubInstance(logger, instance_root))
 
   for shared_part in glob.glob(os.path.join(shared_root, '*', '*')):
-    logger.debug("checking shared part %s", shared_part)
-    h = os.path.basename(shared_part)
-    for soft, installed_cfg in signatures.items():
-      if h in installed_cfg:
-        logger.debug("It is used in %s", soft)
-        break
-    else:
-      if not dry_run:
-        rmtree(shared_part)
-      logger.warning(
-          'Unusued shared parts at %s%s', shared_part,
-          '' if dry_run else ' ... removed')
+    if shared_part not in ignored_shared_parts:
+      logger.debug("checking shared part %s", shared_part)
+      h = os.path.basename(shared_part)
+      for soft, installed_cfg in signatures.items():
+        if h in installed_cfg:
+          logger.debug("It is used in %s", soft)
+          break
+      else:
+        if not dry_run:
+          rmtree(shared_part)
+        logger.warning(
+            'Unusued shared parts at %s%s', shared_part,
+            '' if dry_run else ' ... removed')
+        yield shared_part
+
+
+def _prune_loop(logger, shared_root, software_root, instance_root, dry_run):
+  ignored_shared_parts = set([])
+  while True:
+    pruned = list(
+        _prune(
+            logger,
+            shared_root,
+            software_root,
+            instance_root,
+            ignored_shared_parts,
+            dry_run,
+        ))
+    ignored_shared_parts.update(pruned)
+    if not pruned:
+      break
+
+
+def do_prune(logger, options, dry_run):
+  shared_root = options['shared_part_list'].splitlines()[-1].strip()
+  logger.warning("Pruning shared directories at %s", shared_root)
+  _prune_loop(
+      logger,
+      shared_root,
+      options['software_root'],
+      options['instance_root'],
+      dry_run,
+  )
 
 
 def getUsageSignaturesFromSubInstance(logger, instance_root):
@@ -110,8 +145,7 @@ def getUsageSignaturesFromSubInstance(logger, instance_root):
         getUsageSignatureFromSoftwareAndSharedPart(
             logger, cfg['software_root'], shared_root))
     signatures.update(
-        getUsageSignaturesFromSubInstance(logger, cfg['instance_root'])
-    )
+        getUsageSignaturesFromSubInstance(logger, cfg['instance_root']))
   return signatures
 
 
@@ -140,10 +174,14 @@ def readSlaposCfg(logger, path):
 
 
 def getUsageSignatureFromSoftwareAndSharedPart(
-    logger, software_root, shared_root):
+    logger, software_root, shared_root, ignored_shared_parts=None):
   """Look in all softwares and shared parts to collect the signatures
   that are used.
+  `ignored_shared_parts` is useful during dry-run, we want to ignore
+  already the parts that we are about to delete.
   """
+  if ignored_shared_parts is None:
+    ignored_shared_parts = set([])
   signatures = {}
   for installed_cfg in glob.glob(os.path.join(software_root, '*',
                                               '.installed.cfg')):
@@ -154,13 +192,14 @@ def getUsageSignatureFromSoftwareAndSharedPart(
       signatures[script] = f.read()
   if shared_root:
     for shared_signature in glob.glob(os.path.join(shared_root, '*', '*',
-                                                  '.*signature')):
-      with open(shared_signature) as f:
-        signatures[shared_signature] = f.read()
+                                                   '.*signature')):
+      if not any(shared_signature.startswith(ignored_shared_part)
+                 for ignored_shared_part in ignored_shared_parts):
+        with open(shared_signature) as f:
+          signatures[shared_signature] = f.read()
   return signatures
 
 
-
 # XXX copied from https://lab.nexedi.com/nexedi/erp5/blob/31804f683fd36322fb38aeb9654bee70cebe4fdb/erp5/util/testnode/Utils.py
 # TODO: move to shared place or ... isn't there already such an utility function in slapos.core ?
 import shutil

slapos/tests/test_prune.py

@@ -110,6 +110,27 @@ class TestPrune(unittest.TestCase):
     self.logger.warning.assert_called_with(
         'Unusued shared parts at %s%s', not_used, ' ... removed')
 
+  def test_shared_part_not_used_recursive_dependencies(self):
+    used_only_by_orphean_part = self._createSharedPart(
+        'used_only_by_orphean_part')
+    not_used = self._createSharedPart(
+        'not_used', using=used_only_by_orphean_part)
+    used_directly = self._createSharedPart('used_directly')
+
+    self._createFakeSoftware(self.id(), using=used_directly)
+    do_prune(self.logger, self.config, True)
+    self.assertIn(
+        mock.call('Unusued shared parts at %s%s', not_used, ''),
+        self.logger.warning.mock_calls)
+    self.assertIn(
+        mock.call(
+            'Unusued shared parts at %s%s', used_only_by_orphean_part, ''),
+        self.logger.warning.mock_calls)
+    do_prune(self.logger, self.config, False)
+    self.assertTrue(os.path.exists(used_directly))
+    self.assertFalse(os.path.exists(not_used))
+    self.assertFalse(os.path.exists(used_only_by_orphean_part))
+
   def test_shared_part_used_in_buildout_script(self):
     not_used = self._createSharedPart('not_used')
     used_in_script = self._createSharedPart('used_in_script')