Merge branch 'erp5' into erp5-z2.13

component/apache/buildout.cfg

@@ -120,8 +120,9 @@ make-targets =
 [apache-2.2]
 # inspired on http://old.aclark.net/team/aclark/blog/a-lamp-buildout-for-wordpress-and-other-php-apps/
 recipe = slapos.recipe.cmmi
-url = http://mir2.ovh.net/ftp.apache.org/dist//httpd/httpd-2.2.24.tar.gz
-md5sum = 64a3392018ad60583209a16d728180d3
+version = 2.2.25
+url = http://mir2.ovh.net/ftp.apache.org/dist/httpd/httpd-${:version}.tar.bz2
+md5sum = 9ebe3070c0bb4311f21a0cd0e34f0045
 patch-options = -p1
 configure-options = --disable-static
                     --enable-authn-alias
@@ -182,6 +183,8 @@ environment =
 recipe = slapos.recipe.cmmi
 url = http://sourceforge.net/projects/mod-antiloris/files/mod_antiloris-0.4.tar.bz2/download
 md5sum = 66862bf10e9be3a023e475604a28a0b4
+depends =
+  ${apache-2.2:version}
 configure-command = ${apache-2.2:location}/bin/apxs
 configure-options = -c mod_antiloris.c
 make-binary = ${:configure-command}

component/nodejs/buildout.cfg

@@ -14,8 +14,8 @@ parts =
 [nodejs-0.8]
 # Server-side Javascript.
 recipe = slapos.recipe.cmmi
-url = http://nodejs.org/dist/v0.8.14/node-v0.8.14.tar.gz
-md5sum = 284fd2c7578064c339d9cf6a3a475ac7
+url = http://nodejs.org/dist/v0.8.23/node-v0.8.23.tar.gz
+md5sum = 22fe54ac365f52d3d80ecf748e7323d5
 configure-options =
   --openssl-includes=${openssl:location}/include
   --openssl-libpath=${openssl:location}/lib

setup.py

@@ -188,6 +188,8 @@ setup(name=name,
           'slapreport = slapos.recipe.slapreport:Recipe',
           'slaprunner = slapos.recipe.slaprunner:Recipe',
           'slaprunner.test = slapos.recipe.slaprunner:Test',
+          'slaprunner.export = slapos.recipe.slaprunner.backup:ExportRecipe',
+          'slaprunner.import = slapos.recipe.slaprunner.backup:ImportRecipe',
           'softwaretype = slapos.recipe.softwaretype:Recipe',
           'sphinx= slapos.recipe.sphinx:Recipe',
           'sshkeys_authority = slapos.recipe.sshkeys_authority:Recipe',

slapos/recipe/neoppod.py

@@ -50,8 +50,6 @@ class NeoBaseRecipe(GenericBaseRecipe):
       #'-n', options['name'],
       '-c', options['cluster'],
     ]
-    if options['verbose']:
-      option_list.append('-v')
     option_list.extend(self._getOptionList())
     return [self.createPythonScript(
       options['wrapper'],

slapos/recipe/slaprunner/backup.py

+##############################################################################
+#
+# Copyright (c) 2013 Vifib SARL and Contributors. All Rights Reserved.
+#
+# WARNING: This program as such is intended to be used by professional
+# programmers who take the whole responsibility of assessing all potential
+# consequences resulting from its eventual inadequacies and bugs
+# End users who are looking for a ready-to-use solution with commercial
+# guarantees and support are strongly adviced to contract a Free Software
+# Service Company
+#
+# This program is Free Software; you can redistribute it and/or
+# modify it under the terms of the GNU General Public License
+# as published by the Free Software Foundation; either version 3
+# of the License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA  02111-1307, USA.
+#
+##############################################################################
+
+import textwrap
+
+from slapos.recipe.librecipe import GenericBaseRecipe
+
+
+
+class ExportRecipe(GenericBaseRecipe):
+    """\
+    This recipe creates an exporter script for using with the resilient stack.
+
+    Required options:
+        backup-directory
+            folder that will contain the dump file.
+        srv-directory
+            folder that contain the runner directory.
+        wrapper
+            full path of the exporter script to create.
+    """
+
+    def install(self):
+        wrapper = self.options['wrapper']
+        self.createBackupScript(wrapper)
+        return [wrapper]
+
+
+    def createBackupScript(self, wrapper):
+        """\
+        Create a script to backup the database in 'custom' format.
+        """
+        content = textwrap.dedent("""\
+                #!%(shell-binary)s
+                umask 077
+                sync_element () {
+                  path=$1
+                  backup_path=$2
+                  shift 2
+                  element_list=$*
+                  for element in $element_list
+                  do
+                    cd $path;
+                    if [ -f $element ] || [ -d $element ]; then
+                       %(rsync-binary)s -avz --safe-links --delete $element  $backup_path;
+                    fi
+                  done
+                }
+                sync_element %(srv-directory)s/runner  %(backup-directory)s/runner/ instance project  proxy.db softwareLink
+                sync_element %(etc-directory)s  %(backup-directory)s/etc/ .rcode .project .users ssh
+                if [ -d %(backup-directory)s/runner/software ]; then
+                  rm %(backup-directory)s/runner/software/*
+                fi
+                """ % self.options)
+        self.createExecutable(wrapper, content=content)
+
+
+
+class ImportRecipe(GenericBaseRecipe):
+    """\
+    This recipe creates an importer script for using with the resilient stack.
+
+    Required options:
+        backup-directory
+            folder that will contain the dump file.
+        srv-directory
+            folder that contain the runner directory.
+        wrapper
+            full path of the exporter script to create.
+    """
+
+    def install(self):
+        wrapper = self.options['wrapper']
+        self.createRestoreScript(wrapper)
+        return [wrapper]
+
+
+    def createRestoreScript(self, wrapper):
+        """\
+        Create a script to restore the database from 'custom' format.
+        """
+        content = textwrap.dedent("""\
+                #!%(shell-binary)s
+                umask 077
+                restore_element () {
+                  backup_path=$1
+                  restore_path=$2
+                  shift 2
+                  element_list=$*
+                  for element in $element_list
+                  do
+                    cd $backup_path;
+                    if [ -f $element ] || [ -d $element ]; then
+                       %(rsync-binary)s -avz --delete $backup_path/$element $restore_path;
+                    fi
+                  done
+                }
+                restore_element %(backup-directory)s/runner/ %(srv-directory)s/runner  instance project  proxy.db softwareLink
+                restore_element  %(backup-directory)s/etc/ %(etc-directory)s .rcode .project .users ssh
+                ifs=$IFS IFS=';'
+                read user pass remaining < %(etc-directory)s/.users
+                IFS=$ifs
+                %(curl-binary)s -vg6L -F clogin="$user" -F cpwd="$pass" --dump-header login_cookie  %(backend-url)s/doLogin;
+                %(curl-binary)s -vg6LX POST --cookie login_cookie --max-time 5  %(backend-url)s/runSoftwareProfile;
+                rm -f login_cookie
+                """ % self.options)
+        self.createExecutable(wrapper, content=content)
+
+

software/neoppod/README.txt

@@ -37,11 +37,6 @@ Defautls to 12.
 Number of replicates.
 Defaults to 0 (no resilience).
 
-'verbose' (int, optional)
-~~~~~~~~~~~~~~~~~~~~~~~~~
-Non-zero to increase verbosity.
-Defaults to 0.
-
 'mysql-storage-count' (int, optional)
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 Number of MySQL-based storage nodes to deploy. One master node is deployed

software/neoppod/cluster.cfg.in

@@ -19,10 +19,9 @@ partition-id = ${slap-connection:partition-id}
 config-cluster = {{ slapparameter_dict['cluster'] }}
 config-partitions = {{ slapparameter_dict.get('partitions', 12) }}
 config-replicas = {{ slapparameter_dict.get('replicas', 0) }}
-config-verbose = {{ dumps(slapparameter_dict.get('verbose', 0)) }}
 
 [request-config]
-base = cluster verbose
+base = cluster
 master = partitions replicas
 node = masters
 

software/neoppod/instance-neo-admin.cfg.in

@@ -16,7 +16,6 @@ ip = ${publish:ip}
 port = ${publish:port}
 cluster = ${slap-parameter:cluster}
 masters = ${slap-parameter:masters}
-verbose = ${slap-parameter:verbose}
 
 [publish]
 recipe = slapos.cookbook:publish
@@ -33,6 +32,7 @@ link-binary = ${neo-admin:wrapper}
 
 [directory]
 recipe = slapos.cookbook:mkdirectory
+promises = ${buildout:directory}/etc/promises
 run = ${buildout:directory}/etc/run
 log = ${buildout:directory}/var/log
 bin = ${buildout:directory}/bin

software/neoppod/instance-neo-master.cfg.in

@@ -17,7 +17,6 @@ cluster = ${slap-parameter:cluster}
 partitions = ${slap-parameter:partitions}
 replicas = ${slap-parameter:replicas}
 masters = ${slap-parameter:masters}
-verbose = ${slap-parameter:verbose}
 
 [publish]
 recipe = slapos.cookbook:publish
@@ -34,6 +33,7 @@ link-binary = ${neo-master:wrapper}
 
 [directory]
 recipe = slapos.cookbook:mkdirectory
+promises = ${buildout:directory}/etc/promises
 run = ${buildout:directory}/etc/run
 log = ${buildout:directory}/var/log
 bin = ${buildout:directory}/bin

software/neoppod/instance-neo-storage-mysql.cfg.in

@@ -10,6 +10,7 @@ database = neo
 conf-file = ${directory:etc}/mariadb.cnf
 # No networking (ip & port options)
 data-directory = ${directory:srv_mariadb}
+bin-directory = ${directory:bin}
 pid-file = ${directory:var_run}/mariadb.pid
 socket = ${directory:var_run}/mariadb.sock
 error-log = ${directory:log}/mariadb_error.log
@@ -33,7 +34,6 @@ logfile = ${directory:log}/neostorage.log
 ip = [${slap-network-information:global-ipv6}]
 cluster = ${slap-parameter:cluster}
 masters = ${slap-parameter:masters}
-verbose = ${slap-parameter:verbose}
 database-adapter = MySQL
 database-parameters = ${mariadb-instance:user}:${mariadb-instance:password}@${mariadb-instance:database}${mariadb-instance:socket}
 wait-database = 60

software/neoppod/software.cfg

-# Note on COOKBOOK/END COOKBOOK: the areas delimited by those tags should be
-# dropped when a cookbook version containing neoppod recipe is released as an
-# egg on pypi.
 # Note on LXML/END LXML: they delimit areas where lxml magic is needed. lxml is
 # a slapos.cookbook dependency, so it should be fetched automatically. But when
 # automatically fetched, it gets built against system headers/libs, which is
@@ -10,9 +7,6 @@
 [buildout]
 extends =
     ../../stack/slapos.cfg
-#COOKBOOK
-    ../../component/git/buildout.cfg
-#END COOKBOOK
 #LXML
     ../../component/lxml-python/buildout.cfg
 #END LXML
@@ -20,19 +14,9 @@ extends =
     ../../component/mariadb/buildout.cfg
     ../../component/mysql-python/buildout.cfg
 
-develop =
-#COOKBOOK
-# Note: as we develop a path we fetch, first buildout execution will fail as
-# develop couldn't find the clone. Second (and later) runs will succeed.
-    ${:parts-directory}/slapos.cookbook-repository
-#END COOKBOOK
-
 parts =
-#COOKBOOK
-    slapos.cookbook-repository
-    check-recipe
-#END COOKBOOK
     slapos-deps-eggs
+    slapos-cookbook
 # NEO & dependencies
     python2.7
     mariadb
@@ -47,29 +31,12 @@ eggs =
 #LXML
     ${lxml-python:egg}
 #END LXML
-#COOKBOOK
-    slapos.cookbook
-#END COOKBOOK
 
 #LXML
 [lxml-python]
 python = python2.7
 #END LXML
 
-#COOKBOOK
-[slapos.cookbook-repository]
-recipe = slapos.recipe.build:gitclone
-repository = http://git.erp5.org/repos/slapos.git
-branch = neoppod
-git-executable = ${git:location}/bin/git
-
-[check-recipe]
-recipe = plone.recipe.command
-stop-on-error = true
-update-command = ${:command}
-command = test -e ${buildout:develop-eggs-directory}/slapos.cookbook.egg-link
-#END COOKBOOK
-
 [base-template]
 recipe = slapos.recipe.template:jinja2
 template = ${:_profile_base_location_}/${:filename}.in
@@ -85,12 +52,12 @@ context =
 [template-neo-master]
 < = base-template
 filename = instance-neo-master.cfg
-md5sum = b48e9446fb8fdce24d954d7d4f4c7460
+md5sum = 8947620b15535b95163c11a0efab546e
 
 [template-neo-storage-mysql]
 < = base-template
 filename = instance-neo-storage-mysql.cfg
-md5sum = e767c66e7de58232de30bdf26188efe1
+md5sum = 518c8c41ce73f0b608fbb1c3b889ca1b
 extra-context =
     key mariadb_location mariadb:location
     key master_cfg template-neo-master:rendered
@@ -98,12 +65,12 @@ extra-context =
 [template-neo-admin]
 < = base-template
 filename = instance-neo-admin.cfg
-md5sum = fcf1a5c937a609a934c8ccd7c1326726
+md5sum = 987f69333c563898cab21455509f8024
 
 [template-cluster]
 recipe = slapos.recipe.build:download
 url = ${:_profile_base_location_}/cluster.cfg.in
-md5sum = fe37b2ff36fee65b44945393f63e51c3
+md5sum = 7177d129e171b20044bf435ed549af43
 
 [template-instance]
 < = base-template

software/slaprunner/TODO.txt

+- Use https to connect to the runner. !!! Critical
+- resilient sr: Cloned instances should not launch slapgrid-sr if it was not launched on export instance

software/slaprunner/common.cfg

 [buildout]
 extends =
+  ../../component/bash/buildout.cfg
   ../../component/cloud9/buildout.cfg
+  ../../component/curl/buildout.cfg
   ../../component/dash/buildout.cfg
   ../../component/dropbear/buildout.cfg
   ../../component/git/buildout.cfg
   ../../component/lxml-python/buildout.cfg
+  ../../component/rsync/buildout.cfg
   ../../stack/flask.cfg
   ../../stack/shacache-client.cfg
+  ../../stack/resilient/buildout.cfg
   ../../stack/slapos.cfg
 
 parts =
+  slapos.cookbook-repository
+  rdiff-backup
   template
-  slapos-cookbook
   eggs
 
+  instance-runner-import
+  instance-runner-export
+
+#  slapos-cookbook
+
 [template]
 recipe = slapos.recipe.template
 url = ${:_profile_base_location_}/instance.cfg
 output = ${buildout:directory}/template.cfg
 mode = 0644
-md5sum = 5307e4200f044ae57b504ad68444491c
+#md5sum = 5307e4200f044ae57b504ad68444491c
 
 [template-runner]
 recipe = slapos.recipe.template
 url = ${:_profile_base_location_}/instance-runner.cfg
 output = ${buildout:directory}/template-runner.cfg
-md5sum = 91d6550c43b7a43a999724af4650ae40
+#md5sum = 91d6550c43b7a43a999724af4650ae40
+mode = 0644
+
+[instance-resilient]
+recipe = slapos.recipe.template:jinja2
+template = ${:_profile_base_location_}/instance-resilient.cfg.jinja2
+rendered = ${buildout:directory}/instance-resilient.cfg
+
+context = key buildout buildout:bin-directory
+          key develop_eggs_directory buildout:develop-eggs-directory
+          key eggs_directory buildout:eggs-directory
+
+import-list = file parts template-parts:destination
+              file replicated template-replicated:destination
+
+mode = 0644
+
+[instance-runner-import]
+recipe = slapos.recipe.template
+url = ${:_profile_base_location_}/instance-runner-import.cfg.in
+output = ${buildout:directory}/instance-runner-import.cfg
+mode = 0644
+
+[instance-runner-export]
+recipe = slapos.recipe.template
+url = ${:_profile_base_location_}/instance-runner-export.cfg.in
+output = ${buildout:directory}/instance-runner-export.cfg
 mode = 0644
 
 [eggs]

software/slaprunner/development.cfg

@@ -10,39 +10,43 @@ extends = common.cfg
 
 parts +=
   slapos.cookbook-repository
-  slapos.core-repository
-  slapos.toolbox-repository
-  check-recipe
+
+
+#  slapos.toolbox-repository
+
+#  slapos.core-repository
+#  check-recipe
 
 develop =
   ${:parts-directory}/slapos.cookbook-repository
-  ${:parts-directory}/slapos.core-repository
-  ${:parts-directory}/slapos.toolbox-repository
+#  ${:parts-directory}/slapos.toolbox-repository
+#  ${:parts-directory}/slapos.core-repository
 
 
-[slapos.toolbox-repository]
-recipe = slapos.recipe.build:gitclone
-repository = http://git.erp5.org/repos/slapos.toolbox.git
-branch = slaprunner
-git-executable = ${git:location}/bin/git
+#[slapos.toolbox-repository]
+#recipe = slapos.recipe.build:gitclone
+#repository = http://git.erp5.org/repos/slapos.toolbox.git
+#branch = slaprunner-resiliency
+#git-executable = ${git:location}/bin/git
 
 [slapos.cookbook-repository]
 recipe = slapos.recipe.build:gitclone
 repository = http://git.erp5.org/repos/slapos.git
-branch = slaprunner
-git-executable = ${git:location}/bin/git
-
-[slapos.core-repository]
-recipe = slapos.recipe.build:gitclone
-repository = http://git.erp5.org/repos/slapos.core.git
-branch = master
+branch = slaprunner-resiliency
 git-executable = ${git:location}/bin/git
 
+#[slapos.core-repository]
+#recipe = slapos.recipe.build:gitclone
+#repository = http://git.erp5.org/repos/slapos.core.git
+#branch = master
+#git-executable = ${git:location}/bin/git
+#
 [check-recipe]
 recipe = plone.recipe.command
 stop-on-error = true
 update-command = ${:command}
 command =
   grep parts ${buildout:develop-eggs-directory}/slapos.cookbook.egg-link &&
-  grep parts ${buildout:develop-eggs-directory}/slapos.core.egg-link &&
   grep parts ${buildout:develop-eggs-directory}/slapos.toolbox.egg-link
+
+#  grep parts ${buildout:develop-eggs-directory}/slapos.core.egg-link &&

software/slaprunner/instance-resilient.cfg.jinja2

+# vim: set ft=cfg:
+
+{% import 'parts' as parts %}
+{% import 'replicated' as replicated %}
+
+[buildout]
+eggs-directory = {{ eggs_directory }}
+develop-eggs-directory = {{ develop_eggs_directory }}
+offline = true
+
+# += because we need to take up parts (like instance-custom, slapmonitor etc) from the profile we extended
+parts +=
+  {{ parts.replicate("runner", "3") }}
+  publish-connection-informations
+
+{{ replicated.replicate("runner", "3", "runner-export", "runner-import") }}
+
+# Bubble up the parameters
+[request-runner]
+return = url ssh-public-key ssh-url notification-id ip backend_url url cloud9_url ssh_command password_recovery_code
+config = instance-amount debug domain number authorized-key notify ip-list namebase runner1-computer-guid pbs-runner1-computer-guid runner2-computer-guid pbs-runner2-computer-guid runner3-computer-guid pbs-runner3-computer-guid
+# XXX Cedric LN Ugly hack, resilient stack and slaprunner stack sharing too much ssh sections
+config-authorized-key = ${request-pbs-runner-1:connection-ssh-key} ${request-pbs-runner-2:connection-ssh-key} ${slap-parameter:authorized-key}
+config-instance-amount = ${slap-parameter:instance-amount}
+config-debug = ${slap-parameter:debug}
+config-runner1-computer-guid = ${slap-parameter:runner1-computer-guid}
+config-pbs-runner1-computer-guid = ${slap-parameter:pbs-runner1-computer-guid}
+config-runner2-computer-guid = ${slap-parameter:runner2-computer-guid}
+config-pbs-runner2-computer-guid = ${slap-parameter:pbs-runner2-computer-guid}
+config-runner3-computer-guid = ${slap-parameter:runner3-computer-guid}
+config-pbs-runner3-computer-guid = ${slap-parameter:pbs-runner3-computer-guid}
+config-domain = ${slap-parameter:domain}
+
+
+[publish-connection-informations]
+recipe = slapos.cookbook:publish
+backend_url = ${request-runner:connection-backend_url}
+url = ${request-runner:connection-url}
+cloud9_url = ${request-runner:connection-cloud9_url}
+ssh_command = ${request-runner:connection-ssh_command}
+password_recovery_code = ${request-runner:connection-password_recovery_code}
+
+[slap-parameter]
+# Default parameters for distributed deployment
+# I.e state "backup1 of maria should go there, ..."
+# XXX-Cedric: Hardcoded number of backups. Should be dynamically generated.
+runner1-computer-guid =
+pbs-runner1-computer-guid =
+runner2-computer-guid =
+pbs-runner2-computer-guid =
+runner3-computer-guid =
+pbs-runner3-computer-guid =
+# XXX-Cedric: Hardcoded parameters. Should be dynamically generated.
+domain =
+authorized-key =
+instance-amount = 10
+debug = false

software/slaprunner/instance-runner-export.cfg.in

+[buildout]
+extends = ${template-runner:output}
+          ${pbsready-export:output}
+
+parts +=
+    urls
+    slaprunner
+    cron-entry-backup
+
+[exporter]
+recipe = slapos.cookbook:slaprunner.export
+wrapper = $${directory:bin}/$${slap-parameter:namebase}-exporter
+srv-directory = $${directory:srv}
+etc-directory = $${directory:etc}
+backup-directory = $${directory:backup}
+shell-binary = ${dash:location}/bin/dash
+rsync-binary = ${rsync:location}/bin/rsync
+
+
+# Extends publish section with resilient parameters
+[urls]
+<= resilient-publish-connection-parameter
+backend_url = $${slaprunner:access-url}
+url =  $${request-frontend:connection-site_url}
+cloud9_url = $${cloud9:access-url}
+ssh_command = ssh $${dropbear-runner-server:host} -p $${dropbear-runner-server:port}
+password_recovery_code = $${recovery-code:passwd}

software/slaprunner/instance-runner-import.cfg.in

+[buildout]
+extends = ${template-runner:output}
+          ${pbsready-import:output}
+
+
+parts +=
+    slaprunner
+# have to repeat the next one, as it's not inherited from pbsready-import
+    import-on-notification
+
+[importer]
+recipe = slapos.cookbook:slaprunner.import
+wrapper = $${directory:bin}/$${slap-parameter:namebase}-importer
+srv-directory = $${directory:srv}
+etc-directory = $${directory:etc}
+backup-directory = $${directory:backup}
+shell-binary = ${dash:location}/bin/dash
+rsync-binary = ${rsync:location}/bin/rsync
+curl-binary = ${curl:location}/bin/curl
+backend-url = $${slaprunner:access-url}

software/slaprunner/instance-runner.cfg

@@ -3,7 +3,7 @@ parts =
   cloud9
   slaprunner
   test-runner
-  sshkeys-dropbear
+  sshkeys-dropbear-runner
   dropbear-server-add-authorized-key
   sshkeys-authority
   publish-connection-informations
@@ -13,7 +13,6 @@ parts =
   dropbear-promise
   symlinks
 
-
 eggs-directory = ${buildout:eggs-directory}
 develop-eggs-directory = ${buildout:develop-eggs-directory}
 offline = true
@@ -82,9 +81,9 @@ instance-directory = $${runnerdirectory:instance-root}
 etc_dir = $${directory:etc}
 log_dir =  $${directory:log}
 run_dir = $${directory:run}
-ssh_client = $${sshkeys-dropbear:wrapper}
-public_key = $${sshkeys-dropbear:public-key}
-private_key = $${sshkeys-dropbear:private-key}
+ssh_client = $${sshkeys-dropbear-runner:wrapper}
+public_key = $${sshkeys-dropbear-runner:public-key}
+private_key = $${sshkeys-dropbear-runner:private-key}
 ipv4 = $${slap-network-information:local-ipv4}
 ipv6 = $${slap-network-information:global-ipv6}
 proxy_port = 50000
@@ -121,28 +120,28 @@ keys-directory = $${sshkeys-directory:keys}
 wrapper = $${directory:services}/sshkeys_authority
 keygen-binary = ${dropbear:location}/bin/dropbearkey
 
-[dropbear-server]
+[dropbear-runner-server]
 recipe = slapos.cookbook:dropbear
 host = $${slap-network-information:global-ipv6}
-port = 2222
+port = 22222
 home = $${directory:ssh}
-wrapper = $${directory:bin}/raw_sshd
-shell = /bin/bash
+wrapper = $${directory:bin}/runner_sshd
+shell = ${bash:location}/bin/bash
 rsa-keyfile = $${directory:ssh}/server_key.rsa
 dropbear-binary = ${dropbear:location}/sbin/dropbear
 
-[sshkeys-dropbear]
+[sshkeys-dropbear-runner]
 <= sshkeys-authority
 recipe = slapos.cookbook:sshkeys_authority.request
 name = dropbear
 type = rsa
-executable = $${dropbear-server:wrapper}
-public-key = $${dropbear-server:rsa-keyfile}.pub
-private-key = $${dropbear-server:rsa-keyfile}
-wrapper = $${directory:services}/sshd
+executable = $${dropbear-runner-server:wrapper}
+public-key = $${dropbear-runner-server:rsa-keyfile}.pub
+private-key = $${dropbear-runner-server:rsa-keyfile}
+wrapper = $${directory:services}/runner_sshd
 
 [dropbear-server-add-authorized-key]
-<= dropbear-server
+<= dropbear-runner-server
 recipe = slapos.cookbook:dropbear.add_authorized_key
 key = $${slap-parameter:authorized-key}
 
@@ -166,7 +165,7 @@ recipe = slapos.cookbook:publish
 backend_url = $${slaprunner:access-url}
 url =  $${request-frontend:connection-site_url}
 cloud9_url = $${cloud9:access-url}
-ssh_command = ssh $${dropbear-server:host} -p $${dropbear-server:port}
+ssh_command = ssh $${dropbear-runner-server:host} -p $${dropbear-runner-server:port}
 password_recovery_code = $${recovery-code:passwd}
 
 
@@ -194,8 +193,8 @@ curl_path = ${curl:location}/bin/curl
 [dropbear-promise]
 recipe = slapos.cookbook:check_port_listening
 path = $${directory:promises}/dropbear
-hostname = $${dropbear-server:host}
-port = $${dropbear-server:port}
+hostname = $${dropbear-runner-server:host}
+port = $${dropbear-runner-server:port}
 
 [symlinks]
 recipe = cns.recipe.symlink

software/slaprunner/instance.cfg

@@ -8,4 +8,11 @@ offline = true
 
 [switch_softwaretype]
 recipe = slapos.cookbook:softwaretype
-default = ${template-runner:output}
\ No newline at end of file
+default = ${template-runner:output}
+resilient = ${instance-resilient:rendered}
+runner = ${template-runner:output}
+runner-import = ${instance-runner-import:output}
+runner-export = ${instance-runner-export:output}
+
+frozen = ${instance-frozen:output}
+pull-backup = ${template-pull-backup:output}

software/slaprunner/software.cfg

@@ -5,7 +5,7 @@
 # 2/ Define list of trusted certificates for the cache.
 
 [buildout]
-extends = common.cfg
+extends = development.cfg
 
 [networkcache]
 # signature certificates of the following uploaders.
@@ -56,12 +56,12 @@ netaddr = 0.7.10
 plone.recipe.command = 1.1
 pycrypto = 2.6
 pytz = 2012j
-slapos.cookbook = 0.71.1
-slapos.core = 0.34
-slapos.libnetworkcache = 0.13.3
+#slapos.cookbook = 0.71.1
+slapos.core = 0.35.1
+slapos.libnetworkcache = 0.13.4
 slapos.recipe.build = 0.11.5
 slapos.recipe.template = 2.4.2
-slapos.toolbox = 0.34.0
+#slapos.toolbox = 0.34.0
 smmap = 0.8.2
 xml-marshaller = 0.9.7
 z3c.recipe.scripts = 1.0.1

stack/resilient/README.txt

@@ -127,6 +127,27 @@ wrapper. I suggest you only add options and specify your export/import recipe.
 
 
 
+
+Checking that it works
+----------------------
+
+To check that your software instance is resilient you can proceed this way:
+Once all instances are successfully deployed, go to your export instance, connect as the instance user and run:
+$ ~/bin/exporter
+It is the script responsible for triggering the resiliency stack on your instance. After doing a backup of your data, it will notify the pull-backup instances of a new backup, triggering the transfer of this data to the import instances.
+
+Once this script is run successfully, go to your import instance, connect as its instance user and check ~/srv/backup/"your sofwtare"/, the location of the data you wanted to receive. The last part of the resiliency is up to your import script.
+
+DEBUGGING:
+Here is a partial list of things you can check to understand what is causing the problem:
+
+- Check that your import script does not fail and successfully places your data in ~/srv/backup/"your software" (as the import instance user) by runnig:
+$ ~/bin/"your software"-exporter
+- Check the export instance script is run successfully as this instance user by running:
+$ ~/bin/exporter
+- Check the pull-instance system did its job by going to one of your pull-backup instance, connect as its user and check the log : ~/var/log/equeue.log
+
+
 -----------------------------------------------------------------------------------------
 
 Finally, instance-mysoftware-import.cfg.in and

stack/resilient/buildout.cfg

 [buildout]
+extends =
+  ../../component/gzip/buildout.cfg
+  ../../component/rdiff-backup/buildout.cfg
 
 parts =
+  rdiff-backup
   pbsready
   pbsready-import
   pbsready-export
@@ -9,6 +13,9 @@ parts =
   instance-frozen
   template-resilient
 
+  # needed tools for resiliency
+  gzip
+
 
 #----------------
 #--