testing: assorted fixes for software upgrade tests

See also slapos!875

See merge request !271

slapos/testing/check_software.py

+# -*- coding: utf-8 -*-
+##############################################################################
+#
+# Copyright (c) 2018 Vifib SARL and Contributors. All Rights Reserved.
+#
+# WARNING: This program as such is intended to be used by professional
+# programmers who take the whole responsibility of assessing all potential
+# consequences resulting from its eventual inadequacies and bugs
+# End users who are looking for a ready-to-use solution with commercial
+# guarantees and support are strongly adviced to contract a Free Software
+# Service Company
+#
+# This program is Free Software; you can redistribute it and/or
+# modify it under the terms of the GNU General Public License
+# as published by the Free Software Foundation; either version 3
+# of the License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA  02111-1307, USA.
+#
+##############################################################################
+
+import fnmatch
+import glob
+import os
+import re
+import warnings
+
+import pkg_resources
+import requests
+
+try:
+  import subprocess32 as subprocess
+except ImportError:
+  import subprocess  # type: ignore
+  subprocess  # pyflakes
+
+try:
+  from typing import  Dict, Iterable, List
+except ImportError:
+  pass
+
+from ..slap.standalone import StandaloneSlapOS
+from ..grid.utils import md5digest
+
+
+def checkSoftware(slap, software_url):
+  # type: (StandaloneSlapOS, str) -> None
+  """Check software installation.
+
+  This perform a few basic static checks for common problems
+  with software installations.
+  """
+  # Check that all components set rpath correctly and we don't have miss linking any libraries.
+  # Also check that they are not linked against system libraries, except a white list of core
+  # system libraries.
+  system_lib_white_list = set((
+      'libc',
+      'libcrypt',
+      'libdl',
+      'libgcc_s',
+      'libgomp',
+      'libm',
+      'libnsl',
+      'libpthread',
+      'libresolv',
+      'librt',
+      'libstdc++',
+      'libutil',
+  ))
+
+  # we also ignore a few patterns for part that are known to be binary distributions,
+  # for which we generate LD_LIBRARY_PATH wrappers or we don't use directly.
+  ignored_file_patterns = set((
+      '*/parts/java-re*/*',
+      '*/parts/firefox*/*',
+      '*/parts/chromium-*/*',
+      '*/parts/chromedriver*/*',
+      '*/parts/libreoffice-bin/*',
+      '*/parts/wkhtmltopdf/*',
+      # R uses wrappers to relocate symbols
+      '*/r-language/lib*/R/*',
+      # pulp is a git checkout with some executables
+      '*/pulp-repository.git/src/pulp/solverdir/cbc*',
+      # nss is not a binary distribution, but for some reason it has invalid rpath, but it does
+      # not seem to be a problem in our use cases.
+      '*/parts/nss/*',
+      # npm packages containing binaries
+      '*/node_modules/phantomjs*/*',
+      '*/grafana/tools/phantomjs/*',
+      '*/node_modules/puppeteer/*',
+      # left over of compilation failures
+      '*/*__compile__/*',
+  ))
+
+  software_hash = md5digest(software_url)
+  error_list = []
+  warning_list = []
+
+  ldd_so_resolved_re = re.compile(
+      r'\t(?P<library_name>.*) => (?P<library_path>.*) \(0x')
+  ldd_already_loaded_re = re.compile(r'\t(?P<library_name>.*) \(0x')
+  ldd_not_found_re = re.compile(r'.*not found.*')
+
+  class DynamicLibraryNotFound(Exception):
+    """Exception raised when ldd cannot resolve a library.
+    """
+  def getLddOutput(path):
+    # type: (str) -> Dict[str, str]
+    """Parse ldd output on shared object/executable as `path` and returns a mapping
+    of library paths or None when library is not found, keyed by library so name.
+
+    Raises a `DynamicLibraryNotFound` if any dynamic library is not found.
+
+    Special entries, like VDSO ( linux-vdso.so.1 ) or ELF interpreter
+    ( /lib64/ld-linux-x86-64.so.2 ) are ignored.
+    """
+    libraries = {}  # type: Dict[str, str]
+    try:
+      ldd_output = subprocess.check_output(
+          ('ldd', path),
+          stderr=subprocess.STDOUT,
+          universal_newlines=True,
+      )
+    except subprocess.CalledProcessError as e:
+      if e.output not in ('\tnot a dynamic executable\n',):
+        raise
+      return libraries
+    if ldd_output == '\tstatically linked\n':
+      return libraries
+
+    not_found = []
+    for line in ldd_output.splitlines():
+      resolved_so_match = ldd_so_resolved_re.match(line)
+      ldd_already_loaded_match = ldd_already_loaded_re.match(line)
+      not_found_match = ldd_not_found_re.match(line)
+      if resolved_so_match:
+        libraries[resolved_so_match.group(
+            'library_name')] = resolved_so_match.group('library_path')
+      elif ldd_already_loaded_match:
+        # VDSO or ELF, ignore . See https://stackoverflow.com/a/35805410/7294664 for more about this
+        pass
+      elif not_found_match:
+        not_found.append(line)
+      else:
+        raise RuntimeError('Unknown ldd line %s for %s.' % (line, path))
+    if not_found:
+      not_found_text = '\n'.join(not_found)
+      raise DynamicLibraryNotFound(
+          '{path} has some not found libraries:\n{not_found_text}'.format(
+              **locals()))
+    return libraries
+
+  def checkExecutableLink(paths_to_check, valid_paths_for_libs):
+    # type: (Iterable[str], Iterable[str]) -> List[str]
+    """Check shared libraries linked with executables in `paths_to_check`.
+    Only libraries from `valid_paths_for_libs` are accepted.
+    Returns a list of error messages.
+    """
+    valid_paths_for_libs = [os.path.realpath(x) for x in valid_paths_for_libs]
+    executable_link_error_list = []
+    for path in paths_to_check:
+      for root, dirs, files in os.walk(path):
+        for f in files:
+          f = os.path.join(root, f)
+          if any(fnmatch.fnmatch(f, ignored_pattern)
+                 for ignored_pattern in ignored_file_patterns):
+            continue
+          if os.access(f, os.X_OK) or fnmatch.fnmatch('*.so', f):
+            try:
+              libs = getLddOutput(f)
+            except DynamicLibraryNotFound as e:
+              executable_link_error_list.append(str(e))
+            else:
+              for lib, lib_path in libs.items():
+                if lib.split('.')[0] in system_lib_white_list:
+                  continue
+                lib_path = os.path.realpath(lib_path)
+                # dynamically linked programs can only be linked with libraries
+                # present in software or in shared parts repository.
+                if any(lib_path.startswith(valid_path)
+                       for valid_path in valid_paths_for_libs):
+                  continue
+                executable_link_error_list.append(
+                    '{f} uses system library {lib_path} for {lib}'.format(
+                        **locals()))
+    return executable_link_error_list
+
+  paths_to_check = (
+      os.path.join(slap.software_directory, software_hash),
+      slap.shared_directory,
+  )
+  error_list.extend(
+      checkExecutableLink(
+          paths_to_check,
+          paths_to_check + tuple(slap._shared_part_list),
+      ))
+
+  # check this software is not referenced in any shared parts.
+  for signature_file in glob.glob(os.path.join(slap.shared_directory, '*', '*',
+                                               '.*slapos.*.signature')):
+    with open(signature_file) as f:
+      signature_content = f.read()
+    if software_hash in signature_content:
+      error_list.append(
+          "Software hash present in signature {}\n{}\n".format(
+              signature_file, signature_content))
+
+  def checkEggsVersionsKnownVulnerabilities(
+      egg_directories,
+      safety_db=requests.get(
+          'https://raw.githubusercontent.com/pyupio/safety-db/master/data/insecure_full.json'
+      ).json()):
+    # type: (List[str], Dict) -> Iterable[str]
+    """Check eggs against known vulnerabilities database from https://github.com/pyupio/safety-db
+    """
+    env = pkg_resources.Environment(egg_directories)
+    for egg in env:
+      known_vulnerabilities = safety_db.get(egg)
+      if known_vulnerabilities:
+        for distribution in env[egg]:
+          for known_vulnerability in known_vulnerabilities:
+            for vulnerable_spec in known_vulnerability['specs']:
+              for req in pkg_resources.parse_requirements(egg +
+                                                          vulnerable_spec):
+                vulnerability_description = "\n".join(
+                    u"{}: {}".format(*item)
+                    for item in known_vulnerability.items())
+                if distribution in req:
+                  yield (
+                      u"{egg} use vulnerable version {distribution.version} because {vulnerable_spec}.\n"
+                      "{vulnerability_description}\n".format(**locals()))
+
+  warning_list.extend(
+      checkEggsVersionsKnownVulnerabilities(
+          glob.glob(
+              os.path.join(
+                  slap.software_directory,
+                  software_hash,
+                  'eggs',
+                  '*',
+              )) + glob.glob(
+                  os.path.join(
+                      slap.software_directory,
+                      software_hash,
+                      'develop-eggs',
+                      '*',
+                  ))))
+
+  if warning_list:
+    warnings.warn('\n'.join(warning_list))
+  if error_list:
+    raise RuntimeError('\n'.join(error_list))

slapos/testing/utils.py

@@ -218,10 +218,16 @@ class CrontabMixin(object):
     be a relative time.
     """
     crontab_command =  self._getCrontabCommand(crontab_name)
+    try:
       crontab_output = subprocess.check_output(
           "faketime {date} bash -o pipefail -e -c '{crontab_command}'".format(**locals()),
           shell=True,
+          stderr=subprocess.STDOUT,
       )
+    except subprocess.CalledProcessError as e:
+      self.logger.debug('error executing crontab %s output: %s', crontab_command, e.output)
+      raise
+    else:
       self.logger.debug("crontab %s output: %s", crontab_command, crontab_output)