Switch rdiff backup recipe to slave instance based recipe.

slapos/recipe/dropbear.py

@@ -28,6 +28,32 @@ import os
 import itertools
 from slapos.recipe.librecipe import GenericBaseRecipe
 
+class KnownHostsFile(dict):
+
+  def __init__(self, filename):
+    self._filename = filename
+
+  def _load(self):
+    if os.path.exists(self._filename):
+      with open(self._filename, 'r') as keyfile:
+        for line in keyfile:
+          host, key = [column.strip() for column in line.split(' ', 1)]
+          self[host] = key
+
+  def _dump(self):
+    with open(self._filename, 'w') as keyfile:
+      for key, value in self.items():
+        keyfile.write('%(host)s %(key)s\n' % {'host': key,
+                                              'key': value})
+
+  def __enter__(self):
+    self._load()
+
+  def __exit__(self, exc_type, exc_value, traceback):
+    self._dump()
+
+
+
 class AuthorizedKeysFile(object):
 
   def __init__(self, filename):

slapos/recipe/rdiffbackup.py

@@ -24,11 +24,15 @@
 # Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA  02111-1307, USA.
 #
 ##############################################################################
+from json import loads as unjson
+from hashlib import sha512
+from urlparse import urlparse
 import os
 import subprocess
 import sys
 
 from slapos.recipe.librecipe import GenericBaseRecipe
+from slapos.recipe.dropbear import KnownHostsFile
 from slapos import slap as slapmodule
 
 
@@ -41,9 +45,7 @@ def promise(args):
 
   # Rdiff Backup protocol quit command
   quitcommand = 'q' + chr(255) + chr(0) * 7
-  ssh_cmdline = [args['ssh_client'], '-T',
-                 '-i', args['key'],
-                 '-y', '%(user)s@%(host)s/%(port)s' % args]
+  ssh_cmdline = [args['ssh_client'], '%(user)s@%(host)s/%(port)s' % args]
 
   ssh = subprocess.Popen(ssh_cmdline, stdin=subprocess.PIPE,
                          stdout=open(os.devnull), stderr=open(os.devnull))
@@ -58,55 +60,106 @@ def promise(args):
 
   return ssh.returncode
 
+
+
 class Recipe(GenericBaseRecipe):
 
-  def install(self):
+  def add_slave(self, entry, known_hosts_file):
+    path_list = []
+
+    url = entry.get('url')
+    if url is None:
+      url = ''
+
+    # We assume that thanks to sha512 there's no collisions
+    url_hash = sha512(url).hexdigest()
+    name_hash = sha512(entry['name']).hexdigest()
+
+    promise_path = os.path.join(self.options['promises-directory'],
+                                url_hash)
+    parsed_url = urlparse(url)
+    promise_dict = self.promise_base_dict.copy()
+    promise_dict.update(user=parsed_url.username,
+                        host=parsed_url.hostname,
+                        port=parsed_url.port)
+    promise = self.createPythonScript(promise_path,
+                                      __name__ + '.promise',
+                                      promise_dict)
+    path_list.append(promise)
+
+
+    host = parsed_url.hostname
+    known_hosts_file[host] = entry['server-key']
+
+    remote_schema = '%(ssh)s -p %%s %(user)s@%(host)s' % \
+      {
+        'ssh': self.options['sshclient-binary'],
+        'user': parsed_url.username,
+        'host': parsed_url.hostname,
+      }
+
     command = [self.options['rdiffbackup-binary']]
+    command.extend(['--remote-schema', remote_schema])
+
+    remote_directory = '%(port)s::%(path)s' % {'port': parsed_url.port,
+                                               'path': parsed_url.path}
+
+    local_directory = self.createDirectory(self.options['directory'],
+                                           name_hash)
+
+    if entry['type'] == 'push':
+      command.extend(['--restore-as-of', 'now'])
+      command.extend([local_directory, remote_directory])
+    else:
+      command.extend([remote_directory, local_directory])
+
+    wrapper = self.createPythonScript(
+      os.path.join(self.options['wrappers-directory'], url_hash),
+      'slapos.recipe.librecipe.execute.execute',
+      command
+    )
+    path_list.append(wrapper)
+
+    cron_entry = os.path.join(self.options['cron-entries'], url_hash)
+    with open(cron_entry, 'w') as cron_entry_file:
+      cron_entry_file.write('%s %s' % (entry['frequency'], wrapper))
+    path_list.append(cron_entry)
+
+    return path_list
+
+  def install(self):
+    path_list = []
+
 
     if self.optionIsTrue('client', True):
       self.logger.info("Client mode")
 
-      # XXX-Antoine: Automaticaly accept unknown key with -y
-      # we should generate a known_host file.
-      remote_schema = '%(ssh)s -i %%s -T -y %(user)s@%(host)s/%(port)s' % \
-        {
-          'ssh': self.options['sshclient-binary'],
-          'user': self.options['user'],
-          'host': self.options['host'],
-          'port': self.options['port'],
-        }
-
-      command.extend(['--remote-schema', remote_schema])
-      command.append('%(key)s::%(path)s' % {'key': self.options['key'],
-                                            'path': self.options['path']})
-      command.append(self.options['localpath'])
-
-      if 'promise' in self.options:
-        slap_connection = self.buildout['slap-connection']
-        self.createPythonScript(self.options['promise'],
-          __name__ + '.promise',
-          dict(
-            server_url=slap_connection['server-url'],
-            computer_id=slap_connection['computer-id'],
-            cert_file=slap_connection.get('cert-file'),
-            key_file=slap_connection.get('key-file'),
-            partition_id=slap_connection['partition-id'],
-            ssh_client=self.options['sshclient-binary'],
-            user=self.options['user'],
-            host=self.options['host'],
-            port=self.options['port'],
-            key=self.options['key'],
-          ),
-        )
+      slap_connection = self.buildout['slap-connection']
+      self.promise_base_dict = dict(
+        server_url=slap_connection['server-url'],
+        computer_id=slap_connection['computer-id'],
+        cert_file=slap_connection.get('cert-file'),
+        key_file=slap_connection.get('key-file'),
+        partition_id=slap_connection['partition-id'],
+        ssh_client=self.options['sshclient-binary'],
+      )
+
+      slaves = unjson(self.buildout['slap-parameter']['slave_instance_list'])
+      known_hosts = KnownHostsFile(self.options['known-hosts'])
+      with known_hosts:
+        for slave in slaves:
+          path_list.extend(self.add_slave(slave, known_hosts))
 
     else:
+      command = [self.options['rdiffbackup-binary']]
       self.logger.info("Server mode")
       command.extend(['--restrict', self.options['path']])
       command.append('--server')
 
-    wrapper = self.createPythonScript(
-      self.options['wrapper'],
-      'slapos.recipe.librecipe.execute.execute',
-      command)
+      wrapper = self.createPythonScript(
+        self.options['wrapper'],
+        'slapos.recipe.librecipe.execute.execute',
+        command)
+      path_list.append(wrapper)
 
-    return [wrapper]
+    return path_list