resilient: port the export script of the webrunner to python

Rewrite the [runner-exporter bash script](https://lab.nexedi.com/nexedi/slapos/blob/2a01586bfe17f46e2ef26ec6e45bef692435ed28/software/slaprunner/template/runner-export.sh.jinja2) in python.

Also parametarize the signature function to not use only sha256sum which can create issues on some systems. This should be a separated commit, but that shouldn't be a blocker.

Write unit tests for the main functions of this exporter script. As every trick in the original bash script were not obvious / understood / found by me, I may have missed some.

/reviewed-on nexedi/slapos.toolbox!44

setup.py

@@ -55,6 +55,7 @@ setup(name=name,
         'dnspython',
         'requests',
         'jsonschema',
+        'zc.buildout',
       ] + additional_install_requires,
       extras_require = {
         'lampconfigure':  ["mysqlclient"], #needed for MySQL Database access
@@ -110,6 +111,7 @@ setup(name=name,
           'pubsubserver = slapos.pubsub:main',
           'qemu-qmp-client = slapos.qemuqmpclient:main',
           'rdiffbackup.genstatrss = slapos.resilient.rdiffBackupStat2RSS:main',
+          'runner-exporter = slapos.resilient.runner_exporter:runExport',
           'securedelete = slapos.securedelete:main',
           'slapos-kill = slapos.systool:kill',
           'slaprunnertest = slapos.runner.runnertest:main',

slapos/resilient/runner_exporter.py

+from __future__ import print_function
+
+import argparse
+import errno
+import glob
+import itertools
+import os
+import re
+import shutil
+import subprocess
+import sys
+import time
+
+from contextlib import contextmanager
+from datetime import datetime
+from hashlib import sha256
+from zc.buildout.configparser import parse
+
+
+os.environ['LC_ALL'] = 'C'
+os.umask(0o77)
+
+
+def read_file_by_chunk(path, chunk_size=1024 * 1024):
+  with open(path, 'rb') as f:
+    chunk = f.read(chunk_size)
+    while chunk:
+      yield chunk
+      chunk = f.read(chunk_size)
+
+
+@contextmanager
+def CwdContextManager(path):
+  """
+  Context Manager for executing code in a given directory.
+  There is no need to provide fallback or basic checks
+  in this code, as these checkes should exist in the code
+  invoking this Context Manager.
+  If someone needs to add checks here, I'm pretty sure
+  it means that they are trying to hide legitimate errors.
+  See tests to see examples of invokation
+  """
+  old_path = os.getcwd()
+  try:
+    os.chdir(path)
+    yield
+  finally:
+    os.chdir(old_path)
+
+
+def parseArgumentList():
+  parser = argparse.ArgumentParser()
+  parser.add_argument('--srv-path', required=True)
+  parser.add_argument('--backup-path', required=True)
+  parser.add_argument('--etc-path', required=True)
+  parser.add_argument('--rsync-binary', default='rsync')
+  parser.add_argument('--backup-wait-time', type=int, required=True)
+  parser.add_argument('-n', action='store_true', dest='dry', default=False)
+
+  return parser.parse_args()
+
+
+def rsync(rsync_binary, source, destination, extra_args=None, dry=False):
+  arg_list = [
+    rsync_binary,
+    '-rlptgov',
+    '--stats',
+    '--safe-links',
+    '--ignore-missing-args',
+    '--delete',
+    '--delete-excluded'
+  ]
+  if isinstance(extra_args, list):
+    arg_list.extend(extra_args)
+  if isinstance(source, list):
+    arg_list.extend(source)
+  else:
+    arg_list.append(source)
+  arg_list.append(destination)
+
+  if dry:
+    print('DEBUG:', arg_list)
+    return
+
+  try:
+    print(subprocess.check_output(arg_list))
+  except subprocess.CalledProcessError as e:
+    # All rsync errors are not to be considered as errors
+    allowed_error_message_list = \
+      '^(file has vanished: |rsync warning: some files vanished before they could be transferred)'
+    if e.returncode != 24 or \
+        re.search(allowed_error_message_regex, e.output, re.M) is None:
+      raise
+
+
+def getExcludePathList(path):
+  excluded_path_list = [
+    "*.sock",
+    "*.socket",
+    "*.pid",
+    ".installed*.cfg",
+  ]
+
+  def append_relative(path_list):
+    for p in path_list:
+      p = p.strip()
+      if p:
+        excluded_path_list.append(os.path.relpath(p, path))
+
+  for partition in glob.glob(os.path.join(path, "instance", "slappart*")):
+    if not os.path.isdir(partition):
+      continue
+
+    with CwdContextManager(partition):
+      try:
+        with open("srv/exporter.exclude") as f:
+          exclude = f.readlines()
+      except IOError as e:
+        if e.errno != errno.ENOENT:
+          raise
+      else:
+        append_relative(exclude)
+      for installed in glob.glob(".installed*.cfg"):
+        try:
+          with open(installed) as f:
+            installed = parse(f, installed)
+        except IOError as e:
+          if e.errno != errno.ENOENT:
+            raise
+        else:
+          for section in installed.itervalues():
+            append_relative(section.get(
+              '__buildout_installed__', '').splitlines())
+
+  return excluded_path_list
+
+
+def getSha256Sum(file_path_list):
+  result_list = []
+
+  for file_path in file_path_list:
+    hash_sum = sha256()
+    for chunk in read_file_by_chunk(file_path):
+      hash_sum.update(chunk)
+    result_list.append("%s  %s" % (hash_sum.hexdigest(), file_path))
+
+  return result_list
+
+
+def synchroniseRunnerConfigurationDirectory(config, backup_path):
+  if not os.path.exists(backup_path):
+    os.makedirs(backup_path)
+
+  file_list = ['config.json']
+  for hidden_file in os.listdir('.'):
+    if hidden_file[0] == '.':
+      file_list.append(hidden_file)
+  rsync(config.rsync_binary, file_list, backup_path, dry=config.dry)
+
+
+def synchroniseRunnerWorkingDirectory(config, backup_path):
+  file_list = []
+  exclude_list = []
+
+  if os.path.isdir('instance'):
+    file_list.append('instance')
+    exclude_list = getExcludePathList(os.getcwd())
+
+  # XXX: proxy.db should be properly dumped to leverage its
+  # atomic properties
+  for file in ('project', 'public', 'proxy.db'):
+    if os.path.exists(file):
+      file_list.append(file)
+
+  if file_list:
+    rsync(
+      config.rsync_binary, file_list, backup_path,
+      ["--exclude={}".format(x) for x in exclude_list],
+      dry=config.dry
+    )
+
+
+def getSlappartSignatureMethodDict():
+  slappart_signature_method_dict = {}
+  for partition in glob.glob("./instance/slappart*"):
+    if os.path.isdir(partition):
+      script_path = os.path.join(partition, 'srv', '.backup_identity_script')
+      if os.path.exists(script_path):
+        slappart_signature_method_dict[partition] = script_path
+  return slappart_signature_method_dict
+
+
+def writeSignatureFile(slappart_signature_method_dict, runner_working_path, signature_file_path='backup.signature'):
+  special_slappart_list = slappart_signature_method_dict.keys()
+  signature_list = []
+
+  for dirpath, dirname_list, filename_list in os.walk('.'):
+    if dirpath == '.' or not filename_list:
+      continue
+
+    # Find if special signature function should be applied
+    for special_slappart in special_slappart_list:
+      backup_identity_script_path = os.path.join(
+        runner_working_path,
+        slappart_signature_method_dict[special_slappart]
+      )
+
+      if dirpath.startswith(special_slappart):
+        signature_process = subprocess.Popen(
+          backup_identity_script_path,
+          stdin=subprocess.PIPE,
+          stdout=subprocess.PIPE,
+        )
+
+        (output, error_output) = signature_process.communicate(
+          '\0'.join([os.path.join(dirpath, filename) for filename in filename_list])
+        )
+
+        if signature_process.returncode != 0:
+          print(
+            "An issue occured when calculating the custom signature"
+            " with %s :\n%s" % (
+              backup_identity_script_path, error_output
+            )
+          )
+          sys.exit(1)
+
+        # We have to rstrip as most programs return an empty line
+        # at the end of their output
+        signature_list.extend(output.strip('\n').split('\n'))
+      else:
+        signature_list.extend(
+          getSha256Sum([
+            os.path.join(dirpath, filename)
+            for filename in filename_list
+          ])
+        )
+
+    # Write the signatures in file
+    with open(signature_file_path, 'w+') as signature_file:
+      signature_file.write("\n".join(sorted(signature_list)))
+
+
+def backupFilesWereModifiedDuringExport(export_start_date):
+  export_time = time.time() - export_start_date
+  return bool(
+    subprocess.check_output((
+      'find', '-cmin',  str(export_time / 60.), '-type', 'f', '-path', '*/srv/backup/*'
+    ))
+  )
+
+
+def runExport():
+  export_start_date = int(time.time())
+  print(datetime.fromtimestamp(export_start_date).isoformat())
+
+  args = parseArgumentList()
+
+  def _rsync(*params):
+    return rsync(args.rsync_binary, *params, dry=args.dry)
+
+  runner_working_path = os.path.join(args.srv_path, 'runner')
+  backup_runner_path = os.path.join(args.backup_path, 'runner')
+
+  # Synchronise runner's etc directory
+  with CwdContextManager(args.etc_path):
+    with open('.resilient-timestamp', 'w') as f:
+      f.write(str(export_start_date))
+
+    # "+ '/'" is mandatory otherwise rsyncing the etc directory
+    # will create in the backup_etc_path only a file called etc
+    backup_etc_path = os.path.join(args.backup_path, 'etc') + '/'
+    synchroniseRunnerConfigurationDirectory(args, backup_etc_path)
+
+  # Synchronise runner's working directory
+  # and aggregate signature functions as we are here
+  with CwdContextManager(runner_working_path):
+    synchroniseRunnerWorkingDirectory(args, backup_runner_path)
+    slappart_signature_method_dict = getSlappartSignatureMethodDict()
+
+  # Calculate signature of synchronised files
+  with CwdContextManager(args.backup_path):
+    writeSignatureFile(slappart_signature_method_dict, runner_working_path)
+
+  # BBB: clean software folder if it was synchronized
+  # in an old instance
+  backup_software_path = os.path.join(backup_runner_path, 'software')
+  if os.path.isdir(backup_software_path):
+    shutil.rmtree(backup_software_path)
+
+
+  # Wait a little to increase the probability to detect an ongoing backup.
+  time.sleep(10)
+
+  # Check that export didn't happen during backup of instances
+  with CwdContextManager(backup_runner_path):
+    if backupFilesWereModifiedDuringExport(export_start_date):
+      print("ERROR: Some backups are not consistent, exporter should be re-run."
+            " Let's sleep %s minutes, to let the backup end..." % backup_wait_time)
+      time.sleep(backup_wait_time * 60)
+      system.exit(1)
\ No newline at end of file

slapos/test/test_runner_exporter.py

+import mock
+import os
+import shutil
+import time
+import unittest
+
+from slapos.resilient import runner_exporter
+from StringIO import StringIO
+
+tested_instance_cfg = """[buildout]
+installed_develop_eggs = 
+parts = folders hello-nicolas hello-rafael exclude
+
+[folders]
+__buildout_installed__ = 
+__buildout_signature__ = wcwidth-0.1.7-py2.7.egg contextlib2-0.5.5-py2.7.egg ...
+etc = /some/prefix/slappart18/test/etc
+home = /srv/slapgrid/slappart18/test
+recipe = slapos.cookbook:mkdirectory
+srv = /some/prefix/slappart18/test/srv
+
+[hello-nicolas]
+__buildout_installed__ = {cwd}/instance/slappart0/etc/nicolas.txt
+__buildout_signature__ = MarkupSafe-1.0-py2.7-linux-x86_64.egg Jinja2-2.10-py2.7.egg zc.buildout-2.12.2-py2.7.egg slapos.recipe.template-4.3-py2.7.egg setuptools-40.4.3-py2.7.egg
+mode = 0644
+name = Nicolas
+output = /some/prefix/slappart18/test/etc/nicolas.txt
+recipe = slapos.recipe.template
+
+[hello-rafael]
+__buildout_installed__ = {cwd}/instance/slappart0/etc//rafael.txt
+__buildout_signature__ = MarkupSafe-1.0-py2.7-linux-x86_64.egg Jinja2-2.10-py2.7.egg zc.buildout-2.12.2-py2.7.egg slapos.recipe.template-4.3-py2.7.egg setuptools-40.4.3-py2.7.egg
+name = Rafael
+output = /some/prefix/slappart18/test/etc/rafael.txt
+recipe = slapos.recipe.template
+
+[exclude]
+__buildout_installed__ = {cwd}/instance/slappart0/srv/exporter.exclude
+__buildout_signature__ = MarkupSafe-1.0-py2.7-linux-x86_64.egg Jinja2-2.10-py2.7.egg zc.buildout-2.12.2-py2.7.egg slapos.recipe.template-4.3-py2.7.egg setuptools-40.4.3-py2.7.egg
+recipe = slapos.recipe.template:jinja2
+rendered = /some/prefix/slappart18/test/srv/exporter.exclude
+template = inline:
+        srv/backup/**"""
+
+
+class Config():
+  pass
+
+
+class TestRunnerExporter(unittest.TestCase):
+  def setUp(self):
+    if not os.path.exists('test_folder'):
+      os.mkdir('test_folder')
+      os.chdir('test_folder')
+
+
+  def tearDown(self):
+    if os.path.basename(os.getcwd()) == 'test_folder':
+      os.chdir('..')
+      shutil.rmtree('test_folder')
+    elif 'test_folder' in os.listdir('.'):
+      shutil.rmtree('test_folder')
+
+  def _createFile(self, path, content=''):
+    with open(path, 'w') as f:
+      f.write(content)
+
+  def _createExecutableFile(self, path, content=''):
+    self._createFile(path, content)
+    os.chmod(path, 0700)
+
+  def _setUpFakeInstanceFolder(self):
+    self._createFile('proxy.db')
+    os.makedirs('project')
+    os.makedirs('public')
+
+    """Create data mirroring tested_instance_cfg"""
+    os.makedirs('instance/slappart0/etc')
+    os.makedirs('instance/slappart0/srv/backup')
+
+    os.makedirs('instance/slappart1/etc')
+    os.makedirs('instance/slappart1/srv/backup')
+
+    self._createFile('instance/slappart0/.installed.cfg',
+                     tested_instance_cfg.format(cwd=os.getcwd()))
+
+    self._createFile('instance/slappart0/srv/backup/data.dat',
+                     'all my fortune lays on this secret !')
+    self._createFile('instance/slappart0/srv/exporter.exclude',
+                     'srv/backup/**')
+
+    self._createFile('instance/slappart0/etc/config.json')
+    self._createFile('instance/slappart0/etc/.parameters.xml')
+    self._createFile('instance/slappart0/etc/.project',
+                     'workspace/slapos-dev/software/erp5')
+
+    self._createExecutableFile(
+      'instance/slappart1/srv/.backup_identity_script',
+      "#!/bin/sh\nexec xargs -0 md5sum"
+    )
+
+
+  def test_CwdContextManager(self):
+    os.makedirs('a/b')
+    with runner_exporter.CwdContextManager('a'):
+      self.assertEqual(os.listdir('.'), ['b'])
+      os.mkdir('c')
+    self.assertEqual(os.listdir('.'), ['a'])
+    self.assertEqual(sorted(os.listdir('a')), ['b', 'c'])
+
+
+  def test_getExcludePathList(self):
+    self._setUpFakeInstanceFolder()
+    self.assertEqual(
+      sorted(runner_exporter.getExcludePathList(os.getcwd())),
+      [
+        '*.pid',
+        '*.sock',
+        '*.socket',
+        '.installed*.cfg',
+        'instance/slappart0/etc/nicolas.txt',
+        'instance/slappart0/etc/rafael.txt',
+        'instance/slappart0/srv/backup/**',
+        'instance/slappart0/srv/exporter.exclude',
+      ]
+    )
+
+
+  @mock.patch('subprocess.check_output')
+  def test_synchroniseRunnerConfigurationDirectory(self, check_output_mock):
+    self._setUpFakeInstanceFolder()
+    config = Config()
+    config.rsync_binary = 'rsync'
+    config.dry = False
+    with runner_exporter.CwdContextManager('instance/slappart0/etc'):
+      runner_exporter.synchroniseRunnerConfigurationDirectory(
+        config, 'backup/runner/etc/'
+      )
+    self.assertEqual(check_output_mock.call_count, 1)
+    check_output_mock.assert_any_call(
+      ['rsync', '-rlptgov', '--stats', '--safe-links', '--ignore-missing-args', '--delete', '--delete-excluded', 'config.json', '.parameters.xml', '.project', 'backup/runner/etc/']
+    )
+
+
+  @mock.patch('subprocess.check_output')
+  def test_synchroniseRunnerWorkingDirectory(self, check_output_mock):
+    self._setUpFakeInstanceFolder()
+    config = Config()
+    config.rsync_binary = 'rsync'
+    config.dry = False
+    with runner_exporter.CwdContextManager(os.getcwd()):
+      runner_exporter.synchroniseRunnerWorkingDirectory(
+        config, 'backup/runner/runner'
+      )
+
+    self.assertEqual(check_output_mock.call_count, 1)
+    check_output_mock.assert_any_call(
+      ['rsync', '-rlptgov', '--stats', '--safe-links', '--ignore-missing-args', '--delete', '--delete-excluded', '--exclude=*.sock', '--exclude=*.socket', '--exclude=*.pid', '--exclude=.installed*.cfg', '--exclude=instance/slappart0/srv/backup/**', '--exclude=instance/slappart0/etc/nicolas.txt', '--exclude=instance/slappart0/etc/rafael.txt', '--exclude=instance/slappart0/srv/exporter.exclude', 'instance', 'project', 'public', 'proxy.db', 'backup/runner/runner']
+    )
+
+  def test_getSlappartSignatureMethodDict(self):
+    self._setUpFakeInstanceFolder()
+    slappart_signature_method_dict = runner_exporter.getSlappartSignatureMethodDict()
+    self.assertEqual(
+      slappart_signature_method_dict,
+      {
+        './instance/slappart1': './instance/slappart1/srv/.backup_identity_script',
+      }
+    )
+    
+
+  def test_writeSignatureFile(self):
+    self._setUpFakeInstanceFolder()
+
+    os.makedirs('backup/instance/etc')
+    os.makedirs('backup/instance/slappart0')
+    os.makedirs('backup/instance/slappart1')
+
+    self._createFile('backup/instance/etc/.project', 'workspace/slapos-dev/software/erp5')
+    self._createFile('backup/instance/slappart0/data', 'hello')
+    self._createFile('backup/instance/slappart1/data', 'world')
+
+    slappart_signature_method_dict = {
+      './instance/slappart1': './instance/slappart1/srv/.backup_identity_script',
+    }
+
+    with runner_exporter.CwdContextManager('backup'):
+      runner_exporter.writeSignatureFile(slappart_signature_method_dict, '..', signature_file_path='backup.signature')
+
+      with open('backup.signature', 'r') as f:
+        signature_file_content = f.read()
+
+    # Slappart1 is using md5sum as signature, others are using sha256sum (default)
+    self.assertEqual(signature_file_content, """2cf24dba5fb0a30e26e83b2ac5b9e29e1b161e5c1fa7425e73043362938b9824  ./instance/slappart0/data
+49b74873d57ff0307b7c9364e2fe2a3876d8722fbe7ce3a6f1438d47647a86f4  ./instance/etc/.project
+7d793037a0760186574b0282f2f435e7  ./instance/slappart1/data""")
+
+  def test_backupFilesWereModifiedDuringExport(self):
+    self._setUpFakeInstanceFolder()
+    with runner_exporter.CwdContextManager('instance'):
+      self.assertTrue(runner_exporter.backupFilesWereModifiedDuringExport(time.time() - 5))
+      time.sleep(2)
+      self.assertFalse(runner_exporter.backupFilesWereModifiedDuringExport(time.time() - 1))
+      self._createFile('slappart1/srv/backup/bakckup.data', 'my backup')
+      self.assertTrue(runner_exporter.backupFilesWereModifiedDuringExport(time.time() - 1))