software/theia: Fix export and import script

The import script was overwriting the contents of ~/etc
which destroyed all the services running in the clone,
including the ones that allow the PBS to push new backup files.

software/theia/buildout.hash.cfg

@@ -35,15 +35,15 @@ md5sum = d78a9f885bdebf6720197209e0c21aa0
 
 [theia-common]
 _update_hash_filename_ = theia_common.py
-md5sum = e57396473b4b6a17d26a747f0030293c
+md5sum = 38eba0fb605953677b5a2a2e686a66a2
 
 [theia-export]
 _update_hash_filename_ = theia_export.py
-md5sum = b5f5ac1924b27d3f2be2e5ea291c119e
+md5sum = d338b2d3ba1dcd7a919c0baf51dd11da
 
 [theia-import]
 _update_hash_filename_ = theia_import.py
-md5sum = 9e8c17a4b2d802695caf0c2c052f0d11
+md5sum = 3580f1eec1099bebd550ba1eacd9c116
 
 [yarn.lock]
 _update_hash_filename_ = yarn.lock

software/theia/theia_common.py

@@ -4,6 +4,7 @@ import glob
 import hashlib
 import os
 import re
+import shutil
 import subprocess as sp
 import sqlite3
 
@@ -21,13 +22,19 @@ EXCLUDE_FLAGS = ['--exclude={}'.format(x) for x in sorted(EXCLUDE_PATTERNS)]
 
 def makedirs(path):
   try:
-    os.makedirs(path if os.path.isdir(path) else os.path.dirname(path))
+    os.makedirs(path)
   except OSError as e:
     if e.errno != errno.EEXIST:
       raise
 
 
-def copytree(rsyncbin, src, dst, exclude=[], extrargs=[], verbosity='-v'):
+def copyfile(src, dst):
+  dst = os.path.abspath(dst)
+  makedirs(os.path.dirname(dst))
+  shutil.copy2(src, dst)
+
+
+def copytree(rsyncbin, src, dst, exclude=(), extrargs=(), verbosity='-v'):
   # Ensure there is a trailing slash in the source directory
   # to avoid creating an additional directory level at the destination
   src = os.path.join(src, '')
@@ -60,15 +67,15 @@ def copytree(rsyncbin, src, dst, exclude=[], extrargs=[], verbosity='-v'):
 
 
 def copydb(sqlite3bin, src_db, dst_db):
-  makedirs(dst_db)
+  makedirs(os.path.dirname(dst_db))
   sp.check_output((sqlite3bin, src_db, '.backup ' + dst_db))
 
 
 def remove(path):
   try:
     os.remove(path)
-  except OSError:
-    if os.path.exists(path):
+  except OSError as e:
+    if e.errno != errno.ENOENT:
       raise
 
 

software/theia/theia_export.py

@@ -9,8 +9,8 @@ import traceback
 import six
 from six.moves import configparser
 
-sys.path.append(os.path.dirname(__file__))
-from theia_common import copytree, copydb, hashwalk, parse_installed, remove
+sys.path.insert(0, os.path.dirname(__file__))
+from theia_common import *
 
 
 os.environ['LC_ALL'] = 'C'
@@ -59,10 +59,14 @@ class TheiaExport(object):
     return os.path.abspath(os.path.join(
       self.backup_dir, os.path.relpath(src, start=self.root_dir)))
 
-  def backuptree(self, src, exclude=[], extrargs=[], verbosity='-v'):
+  def backuptree(self, src, exclude=(), extrargs=(), verbosity='-v'):
     dst = self.mirrorpath(src)
     return copytree(self.rsync_bin, src, dst, exclude, extrargs, verbosity)
 
+  def backupfile(self, src):
+    dst = self.mirrorpath(src)
+    return copyfile(src, dst)
+
   def backupdb(self):
     copydb(self.sqlite3_bin, self.proxy_db, self.mirrorpath(self.proxy_db))
 
@@ -118,12 +122,12 @@ class TheiaExport(object):
   def export(self):
     export_start_date = int(time.time())
 
-    etc_dir = os.path.join(self.root_dir, 'etc')
-    with open(os.path.join(etc_dir, '.resilient_timestamp'), 'w') as f:
+    timestamp = os.path.join(self.root_dir, 'etc', '.resilient_timestamp')
+    with open(timestamp, 'w') as f:
       f.write(str(export_start_date))
 
-    self.loginfo('Backup directory ' + etc_dir)
-    self.backuptree(etc_dir, extrargs=('--filter=- */', '--filter=-! .*'))
+    self.loginfo('Backup resilient timestamp ' + timestamp)
+    self.backupfile(timestamp)
 
     for d in self.dirs:
       self.loginfo('Backup directory ' + d)

software/theia/theia_import.py

@@ -10,7 +10,7 @@ import six
 from six.moves import configparser
 
 sys.path.append(os.path.dirname(__file__))
-from theia_common import copytree, copydb, hashwalk, parse_installed, remove
+from theia_common import *
 
 
 os.environ['LC_ALL'] = 'C'
@@ -70,10 +70,14 @@ class TheiaImport(object):
     return os.path.abspath(os.path.join(
       self.root_dir, os.path.relpath(src, start=self.backup_dir)))
 
-  def restoretree(self, dst, exclude=[], extrargs=[], verbosity='-v'):
+  def restoretree(self, dst, exclude=(), extrargs=(), verbosity='-v'):
     src = self.mirrorpath(dst)
     return copytree(self.rsync_bin, src, dst, exclude, extrargs, verbosity)
 
+  def restorefile(self, dst):
+    src = self.mirrorpath(dst)
+    return copyfile(src, dst)
+
   def restoredb(self):
     copydb(self.sqlite3_bin, self.mirrorpath(self.proxy_db), self.proxy_db)
 
@@ -157,9 +161,9 @@ class TheiaImport(object):
     self.loginfo('Restore slapproxy database')
     self.restoredb()
 
-    etc_dir = os.path.join(self.root_dir, 'etc')
-    self.loginfo('Restore directory ' + etc_dir)
-    self.restoretree(etc_dir, extrargs=('--filter=- */', '--filter=-! .*'))
+    timestamp = os.path.join(self.root_dir, 'etc', '.resilient_timestamp')
+    self.loginfo('Restore resilient timestamp ' + timestamp)
+    self.restorefile(timestamp)
 
     custom_script = os.path.join(self.root_dir, 'srv', 'runner-import-restore')
     if os.path.exists(custom_script):