software/theia: Include ~/.netrc in resiliency

software/theia/buildout.hash.cfg

@@ -23,11 +23,11 @@ md5sum = 4f752dd5444a6f9e7c617ec7ccfe62d6
 
 [instance-import]
 _update_hash_filename_ = instance-import.cfg.jinja.in
-md5sum = 647d99aa6f96b2515ac28013145fe81f
+md5sum = 854c9234d719ddf855330f9cb5c6a00b
 
 [instance-export]
 _update_hash_filename_ = instance-export.cfg.jinja.in
-md5sum = b982e83fa42103b7391d97eb36591174
+md5sum = a723e9ba1d27c4c988e3193eb54e428f
 
 [instance-resilient]
 _update_hash_filename_ = instance-resilient.cfg.jinja
@@ -43,11 +43,11 @@ md5sum = 6a25c6a7f1beb27232a3c9acd8a76500
 
 [theia-export]
 _update_hash_filename_ = theia_export.py
-md5sum = e2f6c483cce09f87ab1e63ae8be0daf4
+md5sum = 8cfa554ef1398b808217268656d2441f
 
 [theia-import]
 _update_hash_filename_ = theia_import.py
-md5sum = 45e757f216374d22f0a92d5334dc00f0
+md5sum = a846598bd4a301b4a35a8ca8b843dd3f
 
 [slapos.css.in]
 _update_hash_filename_ = slapos.css.in

software/theia/instance-export.cfg.jinja.in

@@ -47,6 +47,7 @@ context =
   raw project_path $${directory:project}
   raw public_path $${directory:frontend-static-public}
   raw statefiles_path $${directory:statefiles}
+  raw netrc_path_if_exists $${buildout:directory}/.netrc
   key exitfile :exitcode-file
   key errorfile :error-file
 {%- raw %}
@@ -61,6 +62,7 @@ inline =
   --dirs {{ project_path }} \
   --dirs {{ public_path }} \
   --dirs {{ statefiles_path }} \
+  --files {{ netrc_path_if_exists }} \
   --exitfile {{ exitfile }} \
   --errorfile {{ errorfile }}
 {%- endraw %}

software/theia/instance-import.cfg.jinja.in

@@ -111,6 +111,7 @@ context =
   raw project_path $${directory:project}
   raw public_path $${directory:frontend-static-public}
   raw statefiles_path $${directory:statefiles}
+  raw netrc_path_if_exists $${buildout:directory}/.netrc
   key exitfile :exitcode-file
   key errorfile :error-file
 {%- raw %}
@@ -132,6 +133,7 @@ inline =
   --dirs {{ project_path }} \
   --dirs {{ public_path }} \
   --dirs {{ statefiles_path }} \
+  --files {{ netrc_path_if_exists }} \
   --exitfile {{ exitfile }} \
   --errorfile {{ errorfile }}
 {%- endraw %}

software/theia/test/test_resiliency.py

@@ -247,7 +247,7 @@ class ExportAndImportMixin(object):
     self.assertPromiseSucess()
 
 
-class TestTheiaExportAndImportFailures(ExportAndImportMixin, ResilientTheiaTestCase):
+class TestTheiaExportAndImport(ExportAndImportMixin, ResilientTheiaTestCase):
   script_relpath = os.path.join(
     'srv', 'runner', 'instance', 'slappart0',
     'srv', '.backup_identity_script')
@@ -327,15 +327,15 @@ class TestTheiaExportAndImportFailures(ExportAndImportMixin, ResilientTheiaTestC
     except OSError:
       pass
 
-  def test_export_promise(self):
+  def test_export_promise_error(self):
     self.writeFile(self.getExportExitfile(), '1')
     self.assertPromiseFailure('ERROR export script failed')
 
-  def test_import_promise(self):
+  def test_import_promise_error(self):
     self.writeFile(self.getImportExitfile(), '1')
     self.assertPromiseFailure('ERROR import script failed')
 
-  def test_custom_hash_script(self):
+  def test_custom_hash_script_error(self):
     errmsg = 'Bye bye'
     self.customSignatureScript(content='>&2 echo "%s"\nexit 1' % errmsg)
     custom_script = self.getPartitionPath('export', self.script_relpath)
@@ -354,8 +354,42 @@ class TestTheiaExportAndImportFailures(ExportAndImportMixin, ResilientTheiaTestC
     restore_script = self.customRestoreScript('exit 1')
     self.assertImportFailure('Run custom restore script %s\n ... ERROR !' % restore_script)
 
+  def assertFileState(self, filepath, expect_content):
+    if expect_content is None:
+      self.assertFalse(os.path.exists(filepath))
+    else:
+      self.assertTrue(os.path.exists(filepath))
+      with open(filepath) as f:
+        self.assertEqual(f.read(), expect_content)
+
+  def test_export_import_netrc(self):
+    netrc_content = 'machine example.com login someone password secret'
+    backup_relpath = os.path.join('srv', 'backup', 'theia', '.netrc')
+    # Propagate .netrc to import theia and check each step
+    self.writeFile(self.getPartitionPath('export', '.netrc'), netrc_content)
+    self._doExport()
+    self.assertFileState(
+      self.getPartitionPath('export', backup_relpath), netrc_content)
+    self._doTransfer()
+    self.assertFileState(
+      self.getPartitionPath('import', backup_relpath), netrc_content)
+    self._doImport()
+    self.assertFileState(
+      self.getPartitionPath('import', '.netrc'), netrc_content)
+    # Propagate deletion of .netrc to import theia and check each step
+    os.remove(self.getPartitionPath('export', '.netrc'))
+    self._doExport()
+    self.assertFileState(
+      self.getPartitionPath('export', backup_relpath), None)
+    self._doTransfer()
+    self.assertFileState(
+      self.getPartitionPath('import', backup_relpath), None)
+    self._doImport()
+    self.assertFileState(
+      self.getPartitionPath('import', '.netrc'), None)
+
 
-class TestTheiaExportAndImport(ResilienceMixin, ExportAndImportMixin, ResilientTheiaTestCase):
+class TestTheiaResilienceImportAndExport(ResilienceMixin, ExportAndImportMixin, ResilientTheiaTestCase):
   def test_twice(self):
     # Run two synchronisations on the same instances
     # to make sure everything still works the second time

software/theia/theia_export.py

@@ -28,6 +28,7 @@ def main():
   parser.add_argument('--backup', required=True)
   parser.add_argument('--cfg', required=True)
   parser.add_argument('--dirs', action='append')
+  parser.add_argument('--files', action='append')
   parser.add_argument('--exitfile', required=True)
   parser.add_argument('--errorfile', required=True)
   args = parser.parse_args()
@@ -44,6 +45,7 @@ class TheiaExport(object):
     self.backup_dir = args.backup
     self.slapos_cfg = cfg = args.cfg
     self.dirs = args.dirs
+    self.files = args.files
     self.exit_file = args.exitfile
     self.error_file = args.errorfile
     configp = configparser.SafeConfigParser()
@@ -62,8 +64,15 @@ class TheiaExport(object):
   def backup_tree(self, src):
     return copytree(self.rsync_bin, src, self.mirror_path(src))
 
-  def backup_file(self, src):
-    return copyfile(src, self.mirror_path(src))
+  def backup_file(self, src, fail_if_missing=False):
+    if os.path.exists(src):
+      self.log('Backup file ' + src)
+      copyfile(src, self.mirror_path(src))
+    elif fail_if_missing:
+      raise Exception('File %s is missing' % src)
+    else:
+      self.log('Delete file from backup ' + src)
+      remove(self.mirror_path(src))
 
   def backup_db(self):
     copydb(self.sqlite3_bin, self.proxy_db, self.mirror_path(self.proxy_db))
@@ -153,13 +162,15 @@ class TheiaExport(object):
 
     self.remove_signatures()
 
-    self.log('Backup resilient timestamp ' + timestamp)
-    self.backup_file(timestamp)
+    self.backup_file(timestamp, fail_if_missing=True)
 
     for d in self.dirs:
       self.log('Backup directory ' + d)
       self.backup_tree(d)
 
+    for f in self.files:
+      self.backup_file(f)
+
     self.log('Backup slapproxy database')
     self.backup_db()
 

software/theia/theia_import.py

@@ -34,6 +34,7 @@ def main():
   parser.add_argument('--backup', required=True)
   parser.add_argument('--cfg', required=True)
   parser.add_argument('--dirs', action='append')
+  parser.add_argument('--files', action='append')
   parser.add_argument('--exitfile', required=True)
   parser.add_argument('--errorfile', required=True)
   args = parser.parse_args()
@@ -55,6 +56,7 @@ class TheiaImport(object):
     self.backup_dir = args.backup
     self.slapos_cfg = cfg = args.cfg
     self.dirs = args.dirs
+    self.files = args.files
     self.exit_file = args.exitfile
     self.error_file = args.errorfile
     configp = configparser.SafeConfigParser()
@@ -79,9 +81,16 @@ class TheiaImport(object):
     src = self.mirror_path(dst)
     return copytree(self.rsync_bin, src, dst, exclude, extrargs, verbosity)
 
-  def restore_file(self, dst):
+  def restore_file(self, dst, fail_if_missing=False):
     src = self.mirror_path(dst)
-    return copyfile(src, dst)
+    if os.path.exists(src):
+      self.log('Restore file ' + dst)
+      copyfile(src, dst)
+    elif fail_if_missing:
+      raise Exception('File %s is missing from backup' % dst)
+    else:
+      self.log('Remove deleted file ' + dst)
+      remove(os.path.abspath(dst))
 
   def restore_db(self):
     copydb(self.sqlite3_bin, self.mirror_path(self.proxy_db), self.proxy_db)
@@ -224,12 +233,14 @@ class TheiaImport(object):
       self.log('Restore directory ' + d)
       self.restore_tree(d)
 
+    for f in self.files:
+      self.restore_file(f)
+
     self.log('Restore slapproxy database')
     self.restore_db()
 
     timestamp = os.path.join(self.root_dir, 'etc', '.resilient_timestamp')
-    self.log('Restore resilient timestamp ' + timestamp)
-    self.restore_file(timestamp)
+    self.restore_file(timestamp, fail_if_missing=True)
 
     custom_script = os.path.join(self.root_dir, 'srv', 'runner-import-restore')
     if os.path.exists(custom_script):