Update Release Candidate

setup.py

@@ -28,7 +28,7 @@ from setuptools import setup, find_packages
 import glob
 import os
 
-version = '1.0.373'
+version = '1.0.386'
 name = 'slapos.cookbook'
 long_description = open("README.rst").read()
 
@@ -150,6 +150,7 @@ setup(name=name,
           'simplehttpserver = slapos.recipe.simplehttpserver:Recipe',
           'slapconfiguration = slapos.recipe.slapconfiguration:Recipe',
           'slapconfiguration.serialised = slapos.recipe.slapconfiguration:Serialised',
+          'slapconfiguration.jsonschema = slapos.recipe.slapconfiguration:JsonSchema',
           'slapconfiguration.jsondump = slapos.recipe.slapconfiguration:JsonDump',
           'squid = slapos.recipe.squid:Recipe',
           'sshkeys_authority = slapos.recipe.sshkeys_authority:Recipe',

software/theia/buildout.hash.cfg

@@ -23,11 +23,11 @@ md5sum = 4f752dd5444a6f9e7c617ec7ccfe62d6
 
 [instance-import]
 _update_hash_filename_ = instance-import.cfg.jinja.in
-md5sum = 647d99aa6f96b2515ac28013145fe81f
+md5sum = 854c9234d719ddf855330f9cb5c6a00b
 
 [instance-export]
 _update_hash_filename_ = instance-export.cfg.jinja.in
-md5sum = b982e83fa42103b7391d97eb36591174
+md5sum = a723e9ba1d27c4c988e3193eb54e428f
 
 [instance-resilient]
 _update_hash_filename_ = instance-resilient.cfg.jinja
@@ -43,11 +43,11 @@ md5sum = 6a25c6a7f1beb27232a3c9acd8a76500
 
 [theia-export]
 _update_hash_filename_ = theia_export.py
-md5sum = e2f6c483cce09f87ab1e63ae8be0daf4
+md5sum = bc58b7191243bce2bbc95696abce9ef8
 
 [theia-import]
 _update_hash_filename_ = theia_import.py
-md5sum = 45e757f216374d22f0a92d5334dc00f0
+md5sum = 43720e0bf8487ae6bd7e7c462ea5c9f6
 
 [slapos.css.in]
 _update_hash_filename_ = slapos.css.in

software/theia/instance-export.cfg.jinja.in

@@ -47,6 +47,7 @@ context =
   raw project_path $${directory:project}
   raw public_path $${directory:frontend-static-public}
   raw statefiles_path $${directory:statefiles}
+  raw netrc_path_if_exists $${buildout:directory}/.netrc
   key exitfile :exitcode-file
   key errorfile :error-file
 {%- raw %}
@@ -61,6 +62,7 @@ inline =
   --dirs {{ project_path }} \
   --dirs {{ public_path }} \
   --dirs {{ statefiles_path }} \
+  --files {{ netrc_path_if_exists }} \
   --exitfile {{ exitfile }} \
   --errorfile {{ errorfile }}
 {%- endraw %}

software/theia/instance-import.cfg.jinja.in

@@ -111,6 +111,7 @@ context =
   raw project_path $${directory:project}
   raw public_path $${directory:frontend-static-public}
   raw statefiles_path $${directory:statefiles}
+  raw netrc_path_if_exists $${buildout:directory}/.netrc
   key exitfile :exitcode-file
   key errorfile :error-file
 {%- raw %}
@@ -132,6 +133,7 @@ inline =
   --dirs {{ project_path }} \
   --dirs {{ public_path }} \
   --dirs {{ statefiles_path }} \
+  --files {{ netrc_path_if_exists }} \
   --exitfile {{ exitfile }} \
   --errorfile {{ errorfile }}
 {%- endraw %}

software/theia/test/test_resiliency.py

@@ -247,7 +247,7 @@ class ExportAndImportMixin(object):
     self.assertPromiseSucess()
 
 
-class TestTheiaExportAndImportFailures(ExportAndImportMixin, ResilientTheiaTestCase):
+class TestTheiaExportAndImport(ExportAndImportMixin, ResilientTheiaTestCase):
   script_relpath = os.path.join(
     'srv', 'runner', 'instance', 'slappart0',
     'srv', '.backup_identity_script')
@@ -327,15 +327,15 @@ class TestTheiaExportAndImportFailures(ExportAndImportMixin, ResilientTheiaTestC
     except OSError:
       pass
 
-  def test_export_promise(self):
+  def test_export_promise_error(self):
     self.writeFile(self.getExportExitfile(), '1')
     self.assertPromiseFailure('ERROR export script failed')
 
-  def test_import_promise(self):
+  def test_import_promise_error(self):
     self.writeFile(self.getImportExitfile(), '1')
     self.assertPromiseFailure('ERROR import script failed')
 
-  def test_custom_hash_script(self):
+  def test_custom_hash_script_error(self):
     errmsg = 'Bye bye'
     self.customSignatureScript(content='>&2 echo "%s"\nexit 1' % errmsg)
     custom_script = self.getPartitionPath('export', self.script_relpath)
@@ -354,8 +354,42 @@ class TestTheiaExportAndImportFailures(ExportAndImportMixin, ResilientTheiaTestC
     restore_script = self.customRestoreScript('exit 1')
     self.assertImportFailure('Run custom restore script %s\n ... ERROR !' % restore_script)
 
+  def assertFileState(self, filepath, expect_content):
+    if expect_content is None:
+      self.assertFalse(os.path.exists(filepath))
+    else:
+      self.assertTrue(os.path.exists(filepath))
+      with open(filepath) as f:
+        self.assertEqual(f.read(), expect_content)
+
+  def test_export_import_netrc(self):
+    netrc_content = 'machine example.com login someone password secret'
+    backup_relpath = os.path.join('srv', 'backup', 'theia', '.netrc')
+    # Propagate .netrc to import theia and check each step
+    self.writeFile(self.getPartitionPath('export', '.netrc'), netrc_content)
+    self._doExport()
+    self.assertFileState(
+      self.getPartitionPath('export', backup_relpath), netrc_content)
+    self._doTransfer()
+    self.assertFileState(
+      self.getPartitionPath('import', backup_relpath), netrc_content)
+    self._doImport()
+    self.assertFileState(
+      self.getPartitionPath('import', '.netrc'), netrc_content)
+    # Propagate deletion of .netrc to import theia and check each step
+    os.remove(self.getPartitionPath('export', '.netrc'))
+    self._doExport()
+    self.assertFileState(
+      self.getPartitionPath('export', backup_relpath), None)
+    self._doTransfer()
+    self.assertFileState(
+      self.getPartitionPath('import', backup_relpath), None)
+    self._doImport()
+    self.assertFileState(
+      self.getPartitionPath('import', '.netrc'), None)
+
 
-class TestTheiaExportAndImport(ResilienceMixin, ExportAndImportMixin, ResilientTheiaTestCase):
+class TestTheiaResilienceImportAndExport(ResilienceMixin, ExportAndImportMixin, ResilientTheiaTestCase):
   def test_twice(self):
     # Run two synchronisations on the same instances
     # to make sure everything still works the second time

software/theia/theia_export.py

@@ -28,6 +28,7 @@ def main():
   parser.add_argument('--backup', required=True)
   parser.add_argument('--cfg', required=True)
   parser.add_argument('--dirs', action='append')
+  parser.add_argument('--files', action='append')
   parser.add_argument('--exitfile', required=True)
   parser.add_argument('--errorfile', required=True)
   args = parser.parse_args()
@@ -44,6 +45,7 @@ class TheiaExport(object):
     self.backup_dir = args.backup
     self.slapos_cfg = cfg = args.cfg
     self.dirs = args.dirs
+    self.files = args.files
     self.exit_file = args.exitfile
     self.error_file = args.errorfile
     configp = configparser.SafeConfigParser()
@@ -62,8 +64,15 @@ class TheiaExport(object):
   def backup_tree(self, src):
     return copytree(self.rsync_bin, src, self.mirror_path(src))
 
-  def backup_file(self, src):
-    return copyfile(src, self.mirror_path(src))
+  def backup_file(self, src, fail_if_missing=False):
+    if os.path.exists(src):
+      self.log('Backup file ' + src)
+      copyfile(src, self.mirror_path(src))
+    elif fail_if_missing:
+      raise Exception('File %s is missing' % src)
+    else:
+      self.log('Delete file from backup ' + src)
+      remove(self.mirror_path(src))
 
   def backup_db(self):
     copydb(self.sqlite3_bin, self.proxy_db, self.mirror_path(self.proxy_db))
@@ -153,13 +162,15 @@ class TheiaExport(object):
 
     self.remove_signatures()
 
-    self.log('Backup resilient timestamp ' + timestamp)
-    self.backup_file(timestamp)
+    self.backup_file(timestamp, fail_if_missing=True)
 
     for d in self.dirs:
       self.log('Backup directory ' + d)
       self.backup_tree(d)
 
+    for f in self.files:
+      self.backup_file(f)
+
     self.log('Backup slapproxy database')
     self.backup_db()
 

software/theia/theia_import.py

@@ -34,6 +34,7 @@ def main():
   parser.add_argument('--backup', required=True)
   parser.add_argument('--cfg', required=True)
   parser.add_argument('--dirs', action='append')
+  parser.add_argument('--files', action='append')
   parser.add_argument('--exitfile', required=True)
   parser.add_argument('--errorfile', required=True)
   args = parser.parse_args()
@@ -55,6 +56,7 @@ class TheiaImport(object):
     self.backup_dir = args.backup
     self.slapos_cfg = cfg = args.cfg
     self.dirs = args.dirs
+    self.files = args.files
     self.exit_file = args.exitfile
     self.error_file = args.errorfile
     configp = configparser.SafeConfigParser()
@@ -79,9 +81,16 @@ class TheiaImport(object):
     src = self.mirror_path(dst)
     return copytree(self.rsync_bin, src, dst, exclude, extrargs, verbosity)
 
-  def restore_file(self, dst):
+  def restore_file(self, dst, fail_if_missing=False):
     src = self.mirror_path(dst)
-    return copyfile(src, dst)
+    if os.path.exists(src):
+      self.log('Restore file ' + dst)
+      copyfile(src, dst)
+    elif fail_if_missing:
+      raise Exception('File %s is missing from backup' % dst)
+    else:
+      self.log('Remove deleted file ' + dst)
+      remove(os.path.abspath(dst))
 
   def restore_db(self):
     copydb(self.sqlite3_bin, self.mirror_path(self.proxy_db), self.proxy_db)
@@ -224,12 +233,14 @@ class TheiaImport(object):
       self.log('Restore directory ' + d)
       self.restore_tree(d)
 
+    for f in self.files:
+      self.restore_file(f)
+
     self.log('Restore slapproxy database')
     self.restore_db()
 
     timestamp = os.path.join(self.root_dir, 'etc', '.resilient_timestamp')
-    self.log('Restore resilient timestamp ' + timestamp)
-    self.restore_file(timestamp)
+    self.restore_file(timestamp, fail_if_missing=True)
 
     custom_script = os.path.join(self.root_dir, 'srv', 'runner-import-restore')
     if os.path.exists(custom_script):

stack/slapos.cfg

@@ -138,10 +138,10 @@ eggs =
 # The last version of setuptools compatible with Python 3.7
 setuptools = 67.8.0
 # Use SlapOS patched zc.buildout
-zc.buildout = 3.0.1+slapos008
+zc.buildout = 3.0.1+slapos009
 pip = 23.2.1
 # Use SlapOS patched zc.recipe.egg (zc.recipe.egg 2.x is for Buildout 2)
-zc.recipe.egg = 2.0.8.dev0+slapos008
+zc.recipe.egg = 2.0.8.dev0+slapos009
 
 aiofiles = 23.1.0:whl
 aiohttp = 3.8.5:whl
@@ -359,7 +359,7 @@ simplegeneric = 0.8.1
 singledispatch = 3.4.0.3
 six = 1.16.0
 slapos.cookbook = 1.0.373
-slapos.core = 1.14.0
+slapos.core = 1.14.1
 slapos.extension.shared = 1.0
 slapos.libnetworkcache = 0.25
 slapos.rebootstrap = 4.7