Merge branch 'master' into openstack

CHANGES.txt

 Changes
 =======
 
+0.84.2 (2013-10-04)
+-------------------
+
+ * sshkeys_authority: don't allow to return None as parameter. [9e340a0]
+
+0.84.1 (2013-10-03)
+-------------------
+
+ * Resiliency: PBS: promise should NOT bang. [64886cd]
+
 0.84 (2013-09-30)
 -----------------
 

component/groonga/buildout.cfg

@@ -11,9 +11,9 @@ extends =
 
 [groonga]
 recipe = slapos.recipe.cmmi
-version = 3.0.5
+version = 3.0.9
 url = http://packages.groonga.org/source/groonga/groonga-${:version}.tar.gz
-md5sum = 2894bbdd2275cb3c62aea14446dc2561
+md5sum = 2e9f7090f40876233c1f077fd25c26ee
 configure-options =
   --disable-static
   --disable-glibtest

component/libpng/buildout.cfg

@@ -28,6 +28,6 @@ md5sum = fd85af68f84cbdf549147811006488c1
 
 [libpng]
 <= libpng-common
-url = http://download.sourceforge.net/libpng/libpng-1.6.2.tar.xz
-md5sum = 9d838f6fca9948a9f360a0cc1b516d5f
+url = http://download.sourceforge.net/libpng/libpng-1.6.6.tar.xz
+md5sum = 3a41dcd58bcac7cc191c2ec80c7fb2ac
 so_version = 16

component/mariadb/buildout.cfg

@@ -62,8 +62,8 @@ environment =
 # mroonga - a storage engine for MySQL. It provides fast fulltext search feature to all MySQL users.
 # http://mroonga.github.com/
 recipe = slapos.recipe.cmmi
-url = http://packages.groonga.org/source/mroonga/mroonga-3.05.tar.gz
-md5sum = ba4cbd79274d832b9343a0b2fe7d0787
+url = http://packages.groonga.org/source/mroonga/mroonga-3.09.tar.gz
+md5sum = bdcd1d86185a64520881d33e12b7d7a7
 configure-options =
   --with-mysql-source=${mariadb:location}__compile__/mariadb-${mariadb:version}
   --with-mysql-config=${mariadb:location}/bin/mysql_config

component/nano/buildout.cfg

+[buildout]
+parts =
+  nano
+  
+extends = 
+  ../ncurses/buildout.cfg
+
+[nano]
+recipe = slapos.recipe.cmmi
+version = 2.2.6
+url = http://www.nano-editor.org/dist/v2.2/nano-2.2.6.tar.gz
+md5sum = 03233ae480689a008eb98feb1b599807
+environment=
+    CFLAGS=-I${ncurses:location}/include
+    LDFLAGS=-L${ncurses:location}/lib/ -Wl,-rpath=${ncurses:location}/lib/
\ No newline at end of file

component/percona-toolkit/buildout.cfg

@@ -10,8 +10,8 @@ parts =
 recipe = slapos.recipe.cmmi
 depends =
   ${perl:version}
-version = 2.1.9
+version = 2.2.5
 url = http://www.percona.com/redir/downloads/percona-toolkit/${:version}/percona-toolkit-${:version}.tar.gz
-md5sum = 94545d0fe6a4893dcad8a3411531107d
+md5sum = 66165271fc3ddf8311e5ff3b1a954595
 configure-command =
   ${perl:location}/bin/perl Makefile.PL

component/python-2.7/buildout.cfg

@@ -10,6 +10,7 @@ extends =
   ../sqlite3/buildout.cfg
   ../zlib/buildout.cfg
   ../file/buildout.cfg
+  ../xz-utils/buildout.cfg
 
 parts =
     python2.7
@@ -25,9 +26,9 @@ python = python2.7
 
 [python2.7]
 recipe = slapos.recipe.cmmi
-package_version = 2.7.5
-package_version_suffix =
-md5sum = 6334b666b7ff2038c761d7b27ba699c1
+package_version = 2.7.6
+package_version_suffix = rc1
+md5sum = 74e1f6abe529350ac0d239387ee98616
 
 depends =
   ${gdbm:version}
@@ -38,7 +39,7 @@ version = 2.7
 executable = ${:prefix}/bin/python${:version}
 
 url =
-  http://python.org/ftp/python/${:package_version}/Python-${:package_version}${:package_version_suffix}.tar.bz2
+  http://python.org/ftp/python/${:package_version}/Python-${:package_version}${:package_version_suffix}.tar.xz
 configure-options =
   --enable-ipv6
   --enable-unicode=ucs4
@@ -51,5 +52,6 @@ make-targets = make profile-opt && make install
 # the entry "-Wl,-rpath=${file:location}/lib" below is needed by python-magic,
 # which would otherwise load the system libmagic.so with ctypes
 environment =
+  PATH=${xz-utils:location}/bin:%(PATH)s
   CPPFLAGS=-I${zlib:location}/include -I${readline:location}/include -I${libexpat:location}/include -I${ncurses:location}/include -I${ncurses:location}/include/ncursesw -I${bzip2:location}/include  -I${gdbm:location}/include -I${openssl:location}/include -I${sqlite3:location}/include -I${gettext:location}/include
   LDFLAGS=-L${zlib:location}/lib -L${readline:location}/lib -L${libexpat:location}/lib -L${ncurses:location}/lib -L${bzip2:location}/lib -L${gdbm:location}/lib -L${openssl:location}/lib -L${sqlite3:location}/lib -Wl,-rpath=${zlib:location}/lib -Wl,-rpath=${readline:location}/lib -Wl,-rpath=${libexpat:location}/lib -Wl,-rpath=${ncurses:location}/lib -Wl,-rpath=${bzip2:location}/lib -Wl,-rpath=${gdbm:location}/lib -Wl,-rpath=${openssl:location}/lib -Wl,-rpath=${sqlite3:location}/lib -L${gettext:location}/lib -Wl,-rpath=${gettext:location}/lib -Wl,-rpath=${file:location}/lib

component/qemu-kvm/buildout.cfg

@@ -15,8 +15,8 @@ extends =
 [kvm]
 recipe = slapos.recipe.cmmi
 # qemu-kvm and qemu are now the same since 1.3.
-url = http://wiki.qemu-project.org/download/qemu-1.5.1.tar.bz2
-md5sum = b56e73bdcfdb214d5c68e13111aca96f
+url = http://wiki.qemu-project.org/download/qemu-1.6.1.tar.bz2
+md5sum = 3a897d722457c5a895cd6ac79a28fda0
 depends =
   ${libpng:so_version}
 configure-options =
@@ -57,9 +57,9 @@ configure-options =
 [debian-amd64-netinst.iso]
 # Download the installer of Debian 7 (Wheezy)
 recipe = hexagonit.recipe.download
-url = http://cdimage.debian.org/debian-cd/7.1.0/amd64/iso-cd/debian-7.1.0-amd64-netinst.iso
+url = http://cdimage.debian.org/debian-cd/7.2.0/amd64/iso-cd/debian-7.2.0-amd64-netinst.iso 
 filename = ${:_buildout_section_name_}
-md5sum = 80f498a1f9daa76bc911ae13692e4495
+md5sum = b86774fe4de88be6378ba3d71b8029bd 
 download-only = true
 mode = 0644
 location = ${buildout:parts-directory}/${:_buildout_section_name_}

component/slapos/buildout.cfg

@@ -17,6 +17,7 @@ extends =
   ../pkgconfig/buildout.cfg
   ../popt/buildout.cfg
   ../python-2.7/buildout.cfg
+  ../python-openssl/buildout.cfg
   ../readline/buildout.cfg
   ../sqlite3/buildout.cfg
   ../swig/buildout.cfg
@@ -90,10 +91,14 @@ output = ${buildout:directory}/environment.sh
 [lxml-python]
 python = python2.7
 
+[python-openssl]
+python = python2.7
+
 [slapos]
 recipe = z3c.recipe.scripts
 python = python2.7
 eggs =
+  ${python-openssl:egg}
   slapos.libnetworkcache
   zc.buildout
   ${lxml-python:egg}
@@ -131,46 +136,64 @@ scripts = py
 
 [versions]
 # Use our own buildout version
-zc.buildout = 1.6.0-dev-SlapOS-010
+zc.buildout = 1.6.0-dev-SlapOS-012
 
 # Force to use zc.recipe.egg 1.x
 zc.recipe.egg = 1.3.2
 
 # Use own version of h.r.download to be able to open archives not supported by python2.x: .xz
-hexagonit.recipe.download = 1.6nxd002
+hexagonit.recipe.download = 1.7nxd002
 
-Jinja2 = 2.7
+Jinja2 = 2.7.1
 MarkupSafe = 0.18
-Werkzeug = 0.8.3
+Pygments = 1.6
+Werkzeug = 0.9.4
 buildout-versions = 1.7
+cmd2 = 0.6.7
 collective.recipe.template = 1.10
-lxml = 3.1.2
+itsdangerous = 0.23
+lxml = 3.2.3
 meld3 = 0.6.10
 netaddr = 0.7.10
+prettytable = 0.7.2
+pyOpenSSL = 0.13.1
+pyparsing = 2.0.1
+setuptools = 1.1.6
+slapos.core = 1.0.0rc6
 slapos.libnetworkcache = 0.13.4
+slapos.recipe.cmmi = 0.1.1
 xml-marshaller = 0.9.7
 z3c.recipe.scripts = 1.0.1
 
 # Required by:
-# slapos.core==0.35.2-dev
-Flask = 0.9
+# slapos.core==1.0.0rc6
+Flask = 0.10.1
 
 # Required by:
-# slapos.core==0.35.2-dev
+# slapos.core==1.0.0rc6
+bpython = 0.12
+
+# Required by:
+# slapos.core==1.0.0rc6
+cliff = 1.4.5
+
+# Required by:
+# slapos.core==1.0.0rc6
+ipython = 1.1.0
+
+# Required by:
+# slapos.core==1.0.0rc6
 netifaces = 0.8
 
 # Required by:
-# slapos.core==0.35.2-dev
-# slapos.libnetworkcache==0.13.3
-# supervisor==3.0b1
-# zc.buildout==1.6.0-dev-SlapOS-010
-# zope.interface==4.0.5
-setuptools = 0.6c12dev-r88846
+# slapos.core==1.0.0rc6
+requests = 2.0.0
 
 # Required by:
-# slapos.core==0.35.2-dev
-supervisor = 3.0b2
+# slapos.core==1.0.0rc6
+supervisor = 3.0
 
 # Required by:
-# slapos.core==0.35.2-dev
+# slapos.core==1.0.0rc6
 zope.interface = 4.0.5
+

component/slapos/testing.cfg

+# This file is used to install testing, not-stable-yet, version of SlapOS Node
+[buildout]
+extends =
+  buildout.cfg
+
+# Add hosting location of testing version of slapos.core
+find-links +=
+  http://www.nexedi.org/static/packages/source/slapos.core-testing/
+
+[versions]
+slapos.core =

component/sqlite3/buildout.cfg

@@ -5,8 +5,8 @@ parts =
 
 [sqlite3]
 recipe = slapos.recipe.cmmi
-url = http://www.sqlite.org/2013/sqlite-autoconf-3071700.tar.gz
-md5sum = 18c285053e9562b848209cb0ee16d4ab
+url = http://www.sqlite.org/2013/sqlite-autoconf-3080100.tar.gz
+md5sum = 8b5a0a02dfcb0c7daf90856a5cfd485a
 configure-options =
   --disable-static
   --enable-readline

setup.py

@@ -28,7 +28,7 @@ from setuptools import setup, find_packages
 import glob
 import os
 
-version = '0.84'
+version = '0.84.2'
 name = 'slapos.cookbook'
 long_description = open("README.txt").read() + "\n" + \
     open("CHANGES.txt").read() + "\n"

slapos/recipe/erp5_bootstrap/template/erp5_bootstrap.in

@@ -12,7 +12,8 @@ erp5_catalog_storage = 'erp5_mysql_innodb_catalog'
 mysql_url = "%(sql_connection_string)s"
 
 header_dict = {'Authorization': 'Basic %%s' %% \
-  base64.encodestring('%%s:%%s' %% (user, password)).strip()}
+  base64.encodestring('%%s:%%s' %% (user, password)).strip(),
+               'Referer':'http://%%s/manage_addProduct/ERP5/addERP5Site' %% host}
 zope_connection = httplib.HTTPConnection(host)
 
 # Check if an ERP5 site is already created, as ERP5 does support having

slapos/recipe/erp5_update/erp5.py

@@ -50,6 +50,7 @@ class ERP5Updater(object):
     base64string = base64.encodestring(authentication_string).strip()
 
     self.header_dict['Authorization'] = 'Basic %s' % base64string
+    self.header_dict['Referer'] = 'http://%s/manage_addProduct/ERP5/addERP5Site' % host
 
     self.host = host
     self.site_id = site_id

slapos/recipe/kvm/template/kvm_controller_run.in

@@ -6,6 +6,8 @@
 import socket
 import time
 
+# XXX: to be factored with slapos.toolbox qemu qmp wrapper.
+
 socket_path = '%(socket-path)s'
 vnc_password = '%(vnc-passwd)s'
 

slapos/recipe/pbs.py

@@ -27,7 +27,6 @@
 
 import json
 import os
-import signal
 import subprocess
 import sys
 import textwrap
@@ -37,31 +36,13 @@ from slapos.recipe.librecipe import GenericSlapRecipe
 from slapos.recipe.dropbear import KnownHostsFile
 from slapos.recipe.notifier import Notify
 from slapos.recipe.notifier import Callback
-from slapos import slap as slapmodule
 
 
 def promise(args):
-
-  def failed_ssh():
-    sys.stderr.write("SSH Connection failed\n")
-    partition = slap.registerComputerPartition(args['computer_id'],
-                                               args['partition_id'])
-    partition.bang("SSH Connection failed. rdiff-backup is unusable.")
-
-  def sigterm_handler(signum, frame):
-    failed_ssh()
-
-  signal.signal(signal.SIGTERM, sigterm_handler)
-
-  slap = slapmodule.slap()
-  slap.initializeConnection(args['server_url'],
-                            key_file=args.get('key_file'),
-                            cert_file=args.get('cert_file'))
-
-  ssh = subprocess.Popen([args['ssh_client'], '%(user)s@%(host)s/%(port)s' % args],
-                         stdin=subprocess.PIPE,
-                         stdout=open(os.devnull, 'w'),
-                         stderr=open(os.devnull, 'w'))
+  ssh = subprocess.Popen(
+      [args['ssh_client'], '%(user)s@%(host)s/%(port)s' % args],
+      stdin=subprocess.PIPE, stdout=None, stderr=None
+  )
 
   # Rdiff Backup protocol quit command
   quitcommand = 'q' + chr(255) + chr(0) * 7
@@ -74,7 +55,7 @@ def promise(args):
   if ssh.poll() is None:
     return 1
   if ssh.returncode != 0:
-    failed_ssh()
+    sys.stderr.write("SSH Connection failed\n")
   return ssh.returncode
 
 
@@ -98,8 +79,8 @@ class Recipe(GenericSlapRecipe, Notify, Callback):
     print 'Processing PBS slave %s with type %s' % (slave_id, slave_type)
 
     promise_path = os.path.join(self.options['promises-directory'], slave_id)
-    promise_dict = self.promise_base_dict.copy()
-    promise_dict.update(user=parsed_url.username,
+    promise_dict = dict(ssh_client=self.options['sshclient-binary'],
+                        user=parsed_url.username,
                         host=parsed_url.hostname,
                         port=parsed_url.port)
     promise = self.createPythonScript(promise_path,
@@ -221,16 +202,6 @@ class Recipe(GenericSlapRecipe, Notify, Callback):
     if self.optionIsTrue('client', True):
       self.logger.info("Client mode")
 
-      slap_connection = self.buildout['slap-connection']
-      self.promise_base_dict = {
-              'server_url': slap_connection['server-url'],
-              'computer_id': slap_connection['computer-id'],
-              'cert_file': slap_connection.get('cert-file'),
-              'key_file': slap_connection.get('key-file'),
-              'partition_id': slap_connection['partition-id'],
-              'ssh_client': self.options['sshclient-binary'],
-        }
-
       slaves = json.loads(self.options['slave-instance-list'])
       known_hosts = KnownHostsFile(self.options['known-hosts'])
       with known_hosts:

slapos/recipe/sshkeys_authority.py

@@ -113,11 +113,11 @@ class Request(GenericBaseRecipe):
       hashlib.sha256(options['name']).hexdigest())
     self.public_key = self.private_key + '.pub'
 
-    if os.path.exists(self.public_key):
-      with open(self.public_key) as key:
-        options['public-key-value'] = key.read()
-    else:
     options['public-key-value'] = ''
+    if os.path.exists(self.public_key):
+      key_content = open(self.public_key).read()
+      if key_content:
+        options['public-key-value'] = key_content
 
   def install(self):
     requests_directory = self.options['request-directory']

software/kvm/README.txt

@@ -16,7 +16,7 @@ The following examples list how to request different possible instances of KVM
 Software Release from slap console or command line.
 
 KVM instance (1GB of RAM, 10GB of SSD, one core)
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
 Note that the KVM instance will try to request a frontend slave instance in order
 to be accessible from IPv4.
@@ -32,22 +32,51 @@ to be accessible from IPv4.
 
 See the instance-kvm-input-schema.json file for more instance parameters (cpu-count, ram-size, disk-size, etc).
 
-NBD instance
+
+KVM instance (1GB of RAM, 10GB of SSD, one core)
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
-This type of instance will allow to host a disk image that will be used by
-any KVM instance.
+Note that the KVM instance will try to request a frontend slave instance in order
+to be accessible from IPv4.
 
 ::
-  mynbd = request(
+  myawesomekvm = request(
       software_release=kvm,
-      partition_reference="mynbd",
-      software_type="nbd",
+      partition_reference="My awesome KVM",
+      partition_parameter_kw={
+          "nbd-host":"ubuntu-1204.nbd.vifib.net",
+      }
   )
 
+See the instance-kvm-input-schema.json file for more instance parameters (cpu-count, ram-size, disk-size, etc).
+
+Resilient KVM instance
+~~~~~~~~~~~~~~~~~~~~~
+
+Like KVM instance, but backed-up (with history) in two places.
+
+::
+  kvm = 'http://git.erp5.org/gitweb/slapos.git/blob_plain/refs/tags/slapos-0.188:/software/kvm/software.cfg'
+  myresilientkvm = request(
+      software_release=kvm,
+      partition_reference="My resilient KVM",
+      software_type="kvm-resilient",
+      partition_parameter_kw={
+          "-sla-0-computer_guid": "COMP-1000", # Location of the main instance (KVM)
+          "-sla-1-computer_guid": "COMP-1001", # Location of the first clone
+          "-sla-2-computer_guid": "COMP-1002", # Location of the second clone
+      }
+  )
+
+See the instance-kvm-input-schema.json AND instance-kvm-resilient-input-schema.json AND /stack/resilient/README.txt
+files for more instance parameters (cpu-count, ram-size, disk-size, specific location of clones, etc).
+
+Then, if you want one of the two clones to takeover, you need to login into
+the hosting machine, go to the partition of the clone, and invoke bin/takeover.
+
 
 KVM Frontend Master Instance (will host all frontend Slave Instances)
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
 This type of instance will allow to host any frontend slave instance requested
 by KVM instances. Slave instances (and thus KVM instance) will be accessible

software/kvm/common.cfg

@@ -21,6 +21,7 @@ parts =
 
 # XXX: we have to manually add this for resilience
   rdiff-backup
+  collective.recipe.template-egg
 
 #XXX-Cedric : Currently, one can only access to KVM using noVNC.
 #             Ideally one should be able to access KVM by using either NoVNC or VNC.
@@ -79,29 +80,30 @@ command =
 [template]
 recipe = slapos.recipe.template
 url = ${:_profile_base_location_}/instance.cfg.in
-#md5sum = bdd0495ef729e7272ec9c97aca919c09
+md5sum = bc5a986c7208d02d3284a897ea90b39d
 output = ${buildout:directory}/template.cfg
 mode = 0644
 
 [template-kvm]
-recipe = slapos.recipe.template
-url = ${:_profile_base_location_}/instance-kvm.cfg.in
-#md5sum = c3c888c78bbff334135be9e8ad5885a9
-output = ${buildout:directory}/template-kvm.cfg
-mode = 0644
+recipe = hexagonit.recipe.download
+url = ${:_profile_base_location_}/instance-kvm.cfg.jinja2
+mode = 644
+md5sum = e16c15f72fdeb92ce1854bc25daf5ad7
+download-only = true
+on-update = true
 
 [template-kvm-resilient]
 recipe = hexagonit.recipe.download
 url = ${:_profile_base_location_}/instance-kvm-resilient.cfg.jinja2
 mode = 644
-#md5sum = 6753004b582c0470bd028253ce1964ad
+md5sum = a07c96b53fe9145278cd64a3b27a459a
 download-only = true
 on-update = true
 
 [template-kvm-resilient-test]
 recipe = hexagonit.recipe.download
 url = ${:_profile_base_location_}/instance-kvm-resilient-test.cfg.jinja2
-#md5sum = 027d68d9decbc6aec59365fa723975d7
+md5sum = b4894680283d3912df4e9740f3e7848b
 mode = 0644
 download-only = true
 on-update = true
@@ -122,17 +124,18 @@ download-only = true
 mode = 0755
 
 [template-kvm-export]
-recipe = slapos.recipe.template
-url = ${:_profile_base_location_}/instance-kvm-export.cfg.in
-md5sum = 64a1a505aff9fde52afac46240811047
-output = ${buildout:directory}/template-kvm-export.cfg
-mode = 0644
+recipe = hexagonit.recipe.download
+url = ${:_profile_base_location_}/instance-kvm-export.cfg.jinja2
+mode = 644
+md5sum = 900f416956903fa4858e67e93b5169a1
+download-only = true
+on-update = true
 
 [template-kvm-export-script]
 recipe = hexagonit.recipe.download
 url = ${:_profile_base_location_}/template/kvm-export.sh.in
 filename = kvm-export.sh.in
-md5sum = 3e878b3343c76f0d6950986fffcb6a8c
+md5sum = 95fde96f35cbf90d677c44d18b60fafb
 download-only = true
 mode = 0755
 
@@ -149,4 +152,3 @@ url = ${:_profile_base_location_}/instance-frontend.cfg.in
 md5sum = cdb690495e9eb007d2b7d2f8e12f5c59
 output = ${buildout:directory}/template-frontend.cfg
 mode = 0644
-

software/kvm/instance-kvm-export.cfg.in → software/kvm/instance-kvm-export.cfg.jinja2

 [buildout]
-extends = ${template-kvm:output}
-          ${pbsready-export:output}
+extends =
+  {{ kvm_template }}
+  {{ pbsready_export_template }}
 
 parts +=
   cron-entry-backup
 
   certificate-authority
   publish-connection-information
-  kvm-promise
+  kvm-vnc-promise
+  kvm-disk-image-corruption-promise
   websockify-sighandler
   novnc-promise
   cron
@@ -16,12 +18,12 @@ parts +=
 # Create the exporter executable, which is a simple shell script
 [exporter]
 recipe = slapos.recipe.template
-url = ${template-kvm-export-script:location}/${template-kvm-export-script:filename}
-output = $${directory:bin}/$${slap-parameter:namebase}-exporter
+url = {{ template_kvm_export }}
+output = ${directory:bin}/${slap-parameter:namebase}-exporter
 mode = 0755
-backup-disk-path = $${directory:backup}/virtual.qcow2
+backup-disk-path = ${directory:backup}/virtual.qcow2
 # Resilient stack wants a "wrapper" parameter
-wrapper = $${:output}
+wrapper = ${:output}
 
 # Extends publish section with resilient parameters
 [publish-connection-information]

software/kvm/instance-kvm-input-schema.json

@@ -78,7 +78,6 @@
       "description": "MD5 checksum of virtual hard drive, used if virtual-hard-drive-url is specified.",
       "type": "string",
     },
-virtual-hard-drive-md5sum
 
     "use-tap": {
       "title": "Use QEMU TAP network interface",

software/kvm/instance-kvm-resilient-input-schema.json

+{
+  "type": "object",
+  "$schema": "http://json-schema.org/draft-04/schema",
+  "items": {
+      "allOf": [
+          {
+              "$ref": "instance-kvm-input-schema.json"
+          }
+      ],
+  "title": "Input Parameters",
+  "properties": {
+    "-sla-0-computer_guid": {
+      "title": "Target computer for main instance",
+      "description": "Target computer GUID for main instance.",
+      "type": "string"
+    },
+    "-sla-1-computer_guid": {
+      "title": "Target computer for first clone",
+      "description": "Target computer for first clone and PBS.",
+      "type": "string"
+    },
+    "-sla-2-computer_guid": {
+      "title": "Target computer for second clone",
+      "description": "Target computer for second clone and PBS.",
+      "type": "string"
+    },
+    "resiliency-backup-periodicity": {
+      "title": "Periodicity of backup",
+      "description": "Periodicity of backup, in cron format.",
+      "type": "string"
+    },
+    "remove-backup-older-than": {
+      "title": "Remove backups older than...",
+      "description": "Remove all the backups in PBS that are older than specified value. It should be rdiff-backup-compatible.",
+      "type": "string",
+      "default": "3B"
+    }
+  }
+}

software/kvm/instance-kvm-resilient-test.cfg.jinja2

@@ -49,13 +49,11 @@ config-{{ key }} = {{ dumps(value) }}
 {% endfor -%}
 config-virtual-hard-drive-url = ${slap-parameter:virtual-hard-drive-url}
 config-virtual-hard-drive-md5sum = ${slap-parameter:virtual-hard-drive-md5sum}
-config-resiliency-backup-periodicity = */5
-# We don't use url parameter, but we want it to be there to make sure root instance is ready.
-return = url
+config-resiliency-backup-periodicity = */5 * * * *
 # XXX What to do?
 sla = computer_guid
 sla-computer_guid = ${slap-connection:computer-id}
 
 [slap-parameter]
-virtual-hard-drive-url = https://softinst43236.host.vifib.net/data/public/8e2138.php?dl=true
-virtual-hard-drive-md5sum = de0f10c7c6538e9928879332afd9be7a
+virtual-hard-drive-url = https://softinst43236.host.vifib.net/data/public/fbd4ad.php?dl=true
+virtual-hard-drive-md5sum = 465e1024447997e7b86ee2e5151e031b

software/kvm/instance-kvm-resilient.cfg.jinja2

@@ -12,9 +12,16 @@ offline = true
 parts +=
   {{ parts.replicate("kvm", "3") }}
   publish-connection-informations
+  kvm-frontend-url-promise
+  kvm-backend-url-promise
 
 {{ replicated.replicate("kvm", "3", "kvm-export", "kvm-import", slapparameter_dict=slapparameter_dict) }}
 
+[directory]
+recipe = slapos.cookbook:mkdirectory
+etc = ${buildout:directory}/etc
+promises = ${:etc}/promise
+
 # Bubble down the parameters of the requested instance to the user
 [request-kvm]
 # Note: += doesn't work.
@@ -22,10 +29,29 @@ return =
 # Resilient related parameters
   url ssh-public-key ssh-url notification-id ip
 # KVM related parameters
-  backend-url url ipv6
+# XXX: return ALL parameters (like nat rules), through jinja
+  backend-url url
 
 [publish-connection-informations]
 recipe = slapos.cookbook:publish
 backend-url = ${request-kvm:connection-backend-url}
 url = ${request-kvm:connection-url}
-ipv6 = ${request-kvm:connection-ipv6}
+
+[kvm-frontend-url-promise]
+# Check that url parameter is complete
+recipe = collective.recipe.template
+input = inline:#!/bin/sh
+  URL="${request-kvm:connection-url}"
+  if [[ ! "$URL" == https://* ]]; then
+    exit 1
+  fi
+output = ${resilient-directory:promise}/kvm-frontend-url
+mode = 700
+
+[kvm-backend-url-promise]
+# Check that backend url is reachable
+recipe = slapos.cookbook:check_url_available
+path = ${directory:promises}/frontend_promise
+url = ${publish-connection-informations:url}
+dash_path = /bin/sh
+curl_path = {{ curl_executable_location }}

software/kvm/instance-kvm.cfg.in

-#############################
-#
-# Instanciate kvm
-#
-#############################
-[buildout]
-parts =
-  certificate-authority
-  publish-connection-information
-  kvm-promise
-  websockify-sighandler
-  novnc-promise
-#  kvm-monitor
-  cron
-#  cron-entry-monitor
-  frontend-promise
-
-eggs-directory = ${buildout:eggs-directory}
-develop-eggs-directory = ${buildout:develop-eggs-directory}
-offline = true
-
-[directory]
-recipe = slapos.cookbook:mkdirectory
-etc = $${buildout:directory}/etc
-bin = $${buildout:directory}/bin
-srv = $${buildout:directory}/srv
-var = $${buildout:directory}/var
-log = $${:var}/log
-scripts = $${:etc}/run
-services = $${:etc}/service
-promises = $${:etc}/promise
-novnc-conf = $${:etc}/novnc
-run = $${:var}/run
-ca-dir = $${:srv}/ssl
-cron-entries = $${:etc}/cron.d
-crontabs = $${:etc}/crontabs
-cronstamps = $${:etc}/cronstamps
-
-[create-mac]
-recipe = slapos.cookbook:generate.mac
-storage-path = $${directory:srv}/mac
-
-[gen-passwd]
-recipe = slapos.cookbook:generate.password
-storage-path = $${directory:srv}/passwd
-bytes = 8
-
-
-[kvm-instance]
-# XXX-Cedric: change "KVM" recipe to simple "create wrappers". No need for this
-# Specific code. It needs Jinja.
-recipe = slapos.cookbook:kvm
-
-vnc-passwd = $${gen-passwd:passwd}
-
-ipv4 = $${slap-network-information:local-ipv4}
-ipv6 = $${slap-network-information:global-ipv6}
-vnc-ip = $${:ipv4}
-
-vnc-port = 5901
-
-# XXX-Cedric: should be named "default-cdrom-iso"
-default-disk-image = ${debian-amd64-netinst.iso:location}/${debian-amd64-netinst.iso:filename}
-nbd-host = $${slap-parameter:nbd-host}
-nbd-port = $${slap-parameter:nbd-port}
-nbd2-host = $${slap-parameter:nbd2-host}
-nbd2-port = $${slap-parameter:nbd2-port}
-
-tap-interface = $${slap-network-information:network-interface}
-
-disk-path = $${directory:srv}/virtual.qcow2
-disk-size = $${slap-parameter:disk-size}
-disk-type = $${slap-parameter:disk-type}
-
-socket-path = $${directory:var}/qmp_socket
-pid-file-path = $${directory:run}/pid_file
-
-smp-count = $${slap-parameter:cpu-count}
-ram-size = $${slap-parameter:ram-size}
-mac-address = $${create-mac:mac-address}
-
-# XXX-Cedric: should be named runner-wrapper-path and controller-wrapper-path
-runner-path = $${directory:services}/kvm
-controller-path = $${directory:scripts}/kvm_controller
-
-use-tap = $${slap-parameter:use-tap}
-nat-rules = $${slap-parameter:nat-rules}
-6tunnel-wrapper-path = $${directory:services}/6tunnel
-
-virtual-hard-drive-url = $${slap-parameter:virtual-hard-drive-url}
-virtual-hard-drive-md5sum = $${slap-parameter:virtual-hard-drive-md5sum}
-
-shell-path = ${dash:location}/bin/dash
-qemu-path = ${kvm:location}/bin/qemu-system-x86_64
-qemu-img-path = ${kvm:location}/bin/qemu-img
-6tunnel-path = ${6tunnel:location}/bin/6tunnel
-
-
-[kvm-promise]
-recipe = slapos.cookbook:check_port_listening
-path = $${directory:promises}/vnc_promise
-hostname = $${kvm-instance:vnc-ip}
-port = $${kvm-instance:vnc-port}
-
-
-[novnc-instance]
-recipe = slapos.cookbook:novnc
-path = $${ca-novnc:executable}
-ip = $${slap-network-information:global-ipv6}
-port = 6080
-vnc-ip = $${kvm-instance:vnc-ip}
-vnc-port = $${kvm-instance:vnc-port}
-novnc-location = ${noVNC:location}
-websockify-path = ${buildout:directory}/bin/websockify
-ssl-key-path = $${ca-novnc:key-file}
-ssl-cert-path = $${ca-novnc:cert-file}
-
-[websockify-sighandler]
-recipe = slapos.cookbook:signalwrapper
-wrapper-path = $${directory:services}/websockify
-wrapped-path = $${novnc-instance:path}
-
-[certificate-authority]
-recipe = slapos.cookbook:certificate_authority
-openssl-binary = ${openssl:location}/bin/openssl
-ca-dir = $${directory:ca-dir}
-requests-directory = $${cadirectory:requests}
-wrapper = $${directory:services}/certificate_authority
-ca-private = $${cadirectory:private}
-ca-certs = $${cadirectory:certs}
-ca-newcerts = $${cadirectory:newcerts}
-ca-crl = $${cadirectory:crl}
-
-[cadirectory]
-recipe = slapos.cookbook:mkdirectory
-requests = $${directory:ca-dir}/requests/
-private = $${directory:ca-dir}/private/
-certs = $${directory:ca-dir}/certs/
-newcerts = $${directory:ca-dir}/newcerts/
-crl = $${directory:ca-dir}/crl/
-
-[ca-novnc]
-<= certificate-authority
-recipe = slapos.cookbook:certificate_authority.request
-key-file = $${directory:novnc-conf}/novnc.key
-cert-file = $${directory:novnc-conf}/novnc.crt
-executable = $${directory:bin}/novnc
-wrapper = $${directory:bin}/websockify
-
-[novnc-promise]
-recipe = slapos.cookbook:check_port_listening
-path = $${directory:promises}/novnc_promise
-hostname = $${novnc-instance:ip}
-port = $${novnc-instance:port}
-
-
-#[kvm-monitor]
-#recipe = slapos.cookbook:wrapper
-#wrapper-path = $${directory:services}/kvm_monitor
-#command-line = ${buildout:bin-directory}/kvm.monitor.test
-#             $${buildout:directory}/buildout-switch-softwaretype.cfg
-#             $${buildout:directory}/report.xml
-#             -s slap-parameter
-#             -opts disk-size ram-size cpu-count
-
-
-
-#----------------
-#--
-#-- Deploy cron.
-
-[cron]
-recipe = slapos.cookbook:cron
-dcrond-binary = ${dcron:location}/sbin/crond
-cron-entries = $${directory:cron-entries}
-crontabs = $${directory:crontabs}
-cronstamps = $${directory:cronstamps}
-catcher = $${cron-simplelogger:wrapper}
-binary = $${directory:services}/crond
-
-[cron-simplelogger]
-recipe = slapos.cookbook:simplelogger
-wrapper = $${directory:bin}/cron_simplelogger
-log = $${directory:log}/crond.log
-
-
-#[cron-entry-monitor]
-#<= cron
-#recipe = slapos.cookbook:cron.d
-#name = kvm_monitor
-#frequency = 0 0 * * *
-#command = $${kvm-monitor:wrapper-path}
-
-[request-slave-frontend]
-recipe = slapos.cookbook:requestoptional
-software-url = $${slap-parameter:frontend-software-url}
-server-url = $${slap-connection:server-url}
-key-file = $${slap-connection:key-file}
-cert-file = $${slap-connection:cert-file}
-computer-id = $${slap-connection:computer-id}
-partition-id = $${slap-connection:partition-id}
-name = VNC Frontend
-software-type = $${slap-parameter:frontend-software-type}
-slave = true
-config = host port
-config-host = $${novnc-instance:ip}
-config-port = $${novnc-instance:port}
-return = url resource port domainname
-sla = instance_guid
-sla-instance_guid = $${slap-parameter:frontend-instance-guid}
-
-
-[publish-connection-information]
-recipe = slapos.cookbook:publish
-backend-url = https://[$${novnc-instance:ip}]:$${novnc-instance:port}/vnc_auto.html?host=[$${novnc-instance:ip}]&port=$${novnc-instance:port}&encrypt=1&password=$${kvm-instance:vnc-passwd}
-url = $${request-slave-frontend:connection-url}/vnc_auto.html?host=$${request-slave-frontend:connection-domainname}&port=$${request-slave-frontend:connection-port}&encrypt=1&path=$${request-slave-frontend:connection-resource}&password=$${kvm-instance:vnc-passwd}
-ipv6 = $${slap-network-information:global-ipv6}
-
-[frontend-promise]
-recipe = slapos.cookbook:check_url_available
-path = $${directory:promises}/frontend_promise
-url = $${publish-connection-information:url}
-dash_path = ${dash:location}/bin/dash
-curl_path = ${curl:location}/bin/curl
-
-[slap-parameter]
-# Default values if not specified
-frontend-software-type = frontend
-frontend-software-url = http://git.erp5.org/gitweb/slapos.git/blob_plain/refs/tags/slapos-0.92:/software/kvm/software.cfg
-frontend-instance-guid =
-nbd-port = 1024
-nbd-host =
-nbd2-port = 1024
-nbd2-host =
-
-ram-size = 1024
-disk-size = 10
-disk-type = virtio
-
-cpu-count = 1
-
-nat-rules = 22 80 443
-use-tap = False
-
-virtual-hard-drive-url =
-virtual-hard-drive-md5sum =

software/kvm/instance-kvm.cfg.jinja2

+#############################
+#
+# Instanciate kvm
+#
+#############################
+[buildout]
+parts =
+  certificate-authority
+  publish-connection-information
+  kvm-vnc-promise
+  kvm-disk-image-corruption-promise
+  websockify-sighandler
+  novnc-promise
+#  kvm-monitor
+  cron
+#  cron-entry-monitor
+  frontend-promise
+
+eggs-directory = {{ eggs_directory }}
+develop-eggs-directory = {{ develop_eggs_directory }}
+offline = true
+
+[directory]
+recipe = slapos.cookbook:mkdirectory
+etc = ${buildout:directory}/etc
+bin = ${buildout:directory}/bin
+srv = ${buildout:directory}/srv
+var = ${buildout:directory}/var
+log = ${:var}/log
+scripts = ${:etc}/run
+services = ${:etc}/service
+promises = ${:etc}/promise
+novnc-conf = ${:etc}/novnc
+run = ${:var}/run
+ca-dir = ${:srv}/ssl
+cron-entries = ${:etc}/cron.d
+crontabs = ${:etc}/crontabs
+cronstamps = ${:etc}/cronstamps
+
+[create-mac]
+recipe = slapos.cookbook:generate.mac
+storage-path = ${directory:srv}/mac
+
+[gen-passwd]
+recipe = slapos.cookbook:generate.password
+storage-path = ${directory:srv}/passwd
+bytes = 8
+
+
+[kvm-instance]
+# XXX-Cedric: change "KVM" recipe to simple "create wrappers". No need for this
+# Specific code. It needs Jinja.
+recipe = slapos.cookbook:kvm
+
+vnc-passwd = ${gen-passwd:passwd}
+
+ipv4 = ${slap-network-information:local-ipv4}
+ipv6 = ${slap-network-information:global-ipv6}
+vnc-ip = ${:ipv4}
+
+vnc-port = 5901
+
+# XXX-Cedric: should be named "default-cdrom-iso"
+default-disk-image = {{ debian_amd64_netinst_location }}
+nbd-host = ${slap-parameter:nbd-host}
+nbd-port = ${slap-parameter:nbd-port}
+nbd2-host = ${slap-parameter:nbd2-host}
+nbd2-port = ${slap-parameter:nbd2-port}
+
+tap-interface = ${slap-network-information:network-interface}
+
+disk-path = ${directory:srv}/virtual.qcow2
+disk-size = ${slap-parameter:disk-size}
+disk-type = ${slap-parameter:disk-type}
+
+socket-path = ${directory:var}/qmp_socket
+pid-file-path = ${directory:run}/pid_file
+
+smp-count = ${slap-parameter:cpu-count}
+ram-size = ${slap-parameter:ram-size}
+mac-address = ${create-mac:mac-address}
+
+# XXX-Cedric: should be named runner-wrapper-path and controller-wrapper-path
+runner-path = ${directory:services}/kvm
+controller-path = ${directory:scripts}/kvm_controller
+
+use-tap = ${slap-parameter:use-tap}
+nat-rules = ${slap-parameter:nat-rules}
+6tunnel-wrapper-path = ${directory:services}/6tunnel
+
+virtual-hard-drive-url = ${slap-parameter:virtual-hard-drive-url}
+virtual-hard-drive-md5sum = ${slap-parameter:virtual-hard-drive-md5sum}
+
+shell-path = {{ dash_executable_location }}
+qemu-path =  {{ qemu_executable_location }}
+qemu-img-path = {{ qemu_img_executable_location }}
+6tunnel-path = {{ sixtunnel_executable_location }}
+
+
+[kvm-vnc-promise]
+recipe = slapos.cookbook:check_port_listening
+path = ${directory:promises}/vnc_promise
+hostname = ${kvm-instance:vnc-ip}
+port = ${kvm-instance:vnc-port}
+
+[kvm-disk-image-corruption-promise]
+# Check that disk image is not corrupted
+recipe = collective.recipe.template
+input = inline:#!/bin/sh
+  ${kvm-instance:qemu-img-path} check ${kvm-instance:disk-path}
+output = ${directory:promises}/kvm-disk-image-corruption
+mode = 700
+
+
+[novnc-instance]
+recipe = slapos.cookbook:novnc
+path = ${ca-novnc:executable}
+ip = ${slap-network-information:global-ipv6}
+port = 6080
+vnc-ip = ${kvm-instance:vnc-ip}
+vnc-port = ${kvm-instance:vnc-port}
+novnc-location = {{ novnc_location }}
+websockify-path = {{ websockify_executable_location }}
+ssl-key-path = ${ca-novnc:key-file}
+ssl-cert-path = ${ca-novnc:cert-file}
+
+[websockify-sighandler]
+recipe = slapos.cookbook:signalwrapper
+wrapper-path = ${directory:services}/websockify
+wrapped-path = ${novnc-instance:path}
+
+[certificate-authority]
+recipe = slapos.cookbook:certificate_authority
+openssl-binary = {{ openssl_executable_location }}
+ca-dir = ${directory:ca-dir}
+requests-directory = ${cadirectory:requests}
+wrapper = ${directory:services}/certificate_authority
+ca-private = ${cadirectory:private}
+ca-certs = ${cadirectory:certs}
+ca-newcerts = ${cadirectory:newcerts}
+ca-crl = ${cadirectory:crl}
+
+[cadirectory]
+recipe = slapos.cookbook:mkdirectory
+requests = ${directory:ca-dir}/requests/
+private = ${directory:ca-dir}/private/
+certs = ${directory:ca-dir}/certs/
+newcerts = ${directory:ca-dir}/newcerts/
+crl = ${directory:ca-dir}/crl/
+
+[ca-novnc]
+<= certificate-authority
+recipe = slapos.cookbook:certificate_authority.request
+key-file = ${directory:novnc-conf}/novnc.key
+cert-file = ${directory:novnc-conf}/novnc.crt
+executable = ${directory:bin}/novnc
+wrapper = ${directory:bin}/websockify
+
+[novnc-promise]
+recipe = slapos.cookbook:check_port_listening
+path = ${directory:promises}/novnc_promise
+hostname = ${novnc-instance:ip}
+port = ${novnc-instance:port}
+
+
+#----------------
+#--
+#-- Deploy cron.
+
+[cron]
+recipe = slapos.cookbook:cron
+dcrond-binary = {{ dcron_executable_location }}
+cron-entries = ${directory:cron-entries}
+crontabs = ${directory:crontabs}
+cronstamps = ${directory:cronstamps}
+catcher = ${cron-simplelogger:wrapper}
+binary = ${directory:services}/crond
+
+[cron-simplelogger]
+recipe = slapos.cookbook:simplelogger
+wrapper = ${directory:bin}/cron_simplelogger
+log = ${directory:log}/crond.log
+
+#----------------
+#--
+#-- Deploy frontend.
+
+[request-slave-frontend]
+recipe = slapos.cookbook:requestoptional
+software-url = ${slap-parameter:frontend-software-url}
+server-url = ${slap-connection:server-url}
+key-file = ${slap-connection:key-file}
+cert-file = ${slap-connection:cert-file}
+computer-id = ${slap-connection:computer-id}
+partition-id = ${slap-connection:partition-id}
+name = VNC Frontend
+software-type = ${slap-parameter:frontend-software-type}
+slave = true
+config = host port
+config-host = ${novnc-instance:ip}
+config-port = ${novnc-instance:port}
+return = url resource port domainname
+sla = instance_guid
+sla-instance_guid = ${slap-parameter:frontend-instance-guid}
+
+[frontend-promise]
+recipe = slapos.cookbook:check_url_available
+path = ${directory:promises}/frontend_promise
+url = ${publish-connection-information:url}
+dash_path = {{ dash_executable_location }}
+curl_path = {{ curl_executable_location }}
+
+
+[publish-connection-information]
+recipe = slapos.cookbook:publish
+backend-url = https://[${novnc-instance:ip}]:${novnc-instance:port}/vnc_auto.html?host=[${novnc-instance:ip}]&port=${novnc-instance:port}&encrypt=1&password=${kvm-instance:vnc-passwd}
+url = ${request-slave-frontend:connection-url}/vnc_auto.html?host=${request-slave-frontend:connection-domainname}&port=${request-slave-frontend:connection-port}&encrypt=1&path=${request-slave-frontend:connection-resource}&password=${kvm-instance:vnc-passwd}
+# Publish NAT port mapping status
+# XXX: hardcoded value from [slap-parameter]
+{% set nat_rule_list = slapparameter_dict.get('nat-rules', '22 80 443') %}
+{% for port in nat_rule_list.split(' ') -%}
+  {% set external_port = 10000 + port|int() -%}
+  nat-rule-port-{{port}} = ${slap-network-information:global-ipv6} : {{external_port}}
+{% endfor -%}
+
+
+[slap-parameter]
+# Default values if not specified
+frontend-software-type = frontend
+frontend-software-url = http://git.erp5.org/gitweb/slapos.git/blob_plain/refs/tags/slapos-0.92:/software/kvm/software.cfg
+frontend-instance-guid =
+nbd-port = 1024
+nbd-host =
+nbd2-port = 1024
+nbd2-host =
+
+ram-size = 1024
+disk-size = 10
+disk-type = virtio
+
+cpu-count = 1
+
+nat-rules = 22 80 443
+use-tap = False
+
+virtual-hard-drive-url =
+virtual-hard-drive-md5sum =

software/kvm/instance.cfg.in

@@ -7,14 +7,14 @@ develop-eggs-directory = ${buildout:develop-eggs-directory}
 
 [switch-softwaretype]
 recipe = slapos.cookbook:softwaretype
-default = ${template-kvm:output}
-kvm = ${template-kvm:output}
+default = $${:kvm}
+kvm = $${dynamic-template-kvm:rendered}
 nbd = ${template-nbd:output}
 frontend = ${template-frontend:output}
 
 kvm-resilient = $${dynamic-template-kvm-resilient:rendered}
 kvm-import = ${template-kvm-import:output}
-kvm-export = ${template-kvm-export:output}
+kvm-export = $${dynamic-template-kvm-export:rendered}
 
 # Used for the test of resiliency. The system wants a "test" software_type.
 test = $${dynamic-template-kvm-resilient-test:rendered}
@@ -30,6 +30,31 @@ url = $${slap-connection:server-url}
 key = $${slap-connection:key-file}
 cert = $${slap-connection:cert-file}
 
+[dynamic-template-kvm]
+recipe = slapos.recipe.template:jinja2
+template = ${template-kvm:location}/instance-kvm.cfg.jinja2
+rendered = $${buildout:directory}/template-kvm.cfg
+extensions = jinja2.ext.do
+context =
+    key develop_eggs_directory buildout:develop-eggs-directory
+    key eggs_directory buildout:eggs-directory
+    key slapparameter_dict slap-configuration:configuration
+    raw curl_executable_location ${curl:location}/bin/curl
+    raw dash_executable_location ${dash:location}/bin/dash
+    raw dcron_executable_location ${dcron:location}/sbin/crond
+    raw debian_amd64_netinst_location ${debian-amd64-netinst.iso:location}/${debian-amd64-netinst.iso:filename}
+    raw novnc_location ${noVNC:location}
+    raw openssl_executable_location ${openssl:location}/bin/openssl
+    raw qemu_executable_location ${kvm:location}/bin/qemu-system-x86_64
+    raw qemu_img_executable_location ${kvm:location}/bin/qemu-img
+    raw sixtunnel_executable_location ${6tunnel:location}/bin/6tunnel
+    raw websockify_executable_location ${buildout:directory}/bin/websockify
+template-parts-destination = ${template-parts:destination}
+template-replicated-destination = ${template-replicated:destination}
+import-list = file parts :template-parts-destination
+              file replicated :template-replicated-destination
+mode = 0644
+
 [dynamic-template-kvm-resilient]
 recipe = slapos.recipe.template:jinja2
 template = ${template-kvm-resilient:location}/instance-kvm-resilient.cfg.jinja2
@@ -39,20 +64,33 @@ context =
     key develop_eggs_directory buildout:develop-eggs-directory
     key eggs_directory buildout:eggs-directory
     key slapparameter_dict slap-configuration:configuration
+    raw curl_executable_location ${curl:location}/bin/curl
 template-parts-destination = ${template-parts:destination}
 template-replicated-destination = ${template-replicated:destination}
 import-list = file parts :template-parts-destination
               file replicated :template-replicated-destination
 mode = 0644
 
+[dynamic-template-kvm-export]
+recipe = slapos.recipe.template:jinja2
+template = ${template-kvm-export:location}/instance-kvm-export.cfg.jinja2
+rendered = $${buildout:directory}/template-kvm-export.cfg
+extensions = jinja2.ext.do
+context =
+    key develop_eggs_directory buildout:develop-eggs-directory
+    key eggs_directory buildout:eggs-directory
+    raw kvm_template $${dynamic-template-kvm:rendered}
+    raw template_kvm_export ${template-kvm-export-script:location}/${template-kvm-export-script:filename}
+    raw pbsready_export_template ${pbsready-export:output}
+mode = 0644
+
 [dynamic-template-kvm-resilient-test]
 recipe = slapos.recipe.template:jinja2
 template = ${template-kvm-resilient-test:location}/instance-kvm-resilient-test.cfg.jinja2
 rendered = $${buildout:directory}/template-kvm-resilient-test.cfg
-bin-directory = ${buildout:bin-directory}
 context =
-    key bin_directory dynamic-template-kvm-resilient-test:bin-directory
     key develop_eggs_directory buildout:develop-eggs-directory
     key eggs_directory buildout:eggs-directory
     key slapparameter_dict slap-configuration:configuration
+    raw bin_directory ${buildout:bin-directory}
 mode = 0644

software/kvm/software.cfg

@@ -4,8 +4,13 @@ extends = common.cfg
 [networkcache]
 # signature certificates of the following uploaders.
 #   Romain Courteaud
+#   Sebastien Robin
+#   Kazuhiko Shiozaki
 #   Cedric de Saint Martin
-#   Test Agent
+#   Yingjie Xu
+#   Gabriel Monnerat
+#   Test Agent (Automatic update from tests)
+#   Aurélien Calonne
 signature-certificate-list =
   -----BEGIN CERTIFICATE-----
   MIIB4DCCAUkCADANBgkqhkiG9w0BAQsFADA5MQswCQYDVQQGEwJGUjEZMBcGA1UE
@@ -21,6 +26,32 @@ signature-certificate-list =
   QUUGLQ==
   -----END CERTIFICATE-----
   -----BEGIN CERTIFICATE-----
+  MIIB8jCCAVugAwIBAgIJAPu2zchZ2BxoMA0GCSqGSIb3DQEBBQUAMBIxEDAOBgNV
+  BAMMB3RzeGRldjMwHhcNMTExMDE0MTIxNjIzWhcNMTIxMDEzMTIxNjIzWjASMRAw
+  DgYDVQQDDAd0c3hkZXYzMIGfMA0GCSqGSIb3DQEBAQUAA4GNADCBiQKBgQCrPbh+
+  YGmo6mWmhVb1vTqX0BbeU0jCTB8TK3i6ep3tzSw2rkUGSx3niXn9LNTFNcIn3MZN
+  XHqbb4AS2Zxyk/2tr3939qqOrS4YRCtXBwTCuFY6r+a7pZsjiTNddPsEhuj4lEnR
+  L8Ax5mmzoi9nE+hiPSwqjRwWRU1+182rzXmN4QIDAQABo1AwTjAdBgNVHQ4EFgQU
+  /4XXREzqBbBNJvX5gU8tLWxZaeQwHwYDVR0jBBgwFoAU/4XXREzqBbBNJvX5gU8t
+  LWxZaeQwDAYDVR0TBAUwAwEB/zANBgkqhkiG9w0BAQUFAAOBgQA07q/rKoE7fAda
+  FED57/SR00OvY9wLlFEF2QJ5OLu+O33YUXDDbGpfUSF9R8l0g9dix1JbWK9nQ6Yd
+  R/KCo6D0sw0ZgeQv1aUXbl/xJ9k4jlTxmWbPeiiPZEqU1W9wN5lkGuLxV4CEGTKU
+  hJA/yXa1wbwIPGvX3tVKdOEWPRXZLg==
+  -----END CERTIFICATE-----
+  -----BEGIN CERTIFICATE-----
+  MIIB7jCCAVegAwIBAgIJAJWA0jQ4o9DGMA0GCSqGSIb3DQEBBQUAMA8xDTALBgNV
+  BAMMBHg2MXMwIBcNMTExMTI0MTAyNDQzWhgPMjExMTEwMzExMDI0NDNaMA8xDTAL
+  BgNVBAMMBHg2MXMwgZ8wDQYJKoZIhvcNAQEBBQADgY0AMIGJAoGBANdJNiFsRlkH
+  vq2kHP2zdxEyzPAWZH3CQ3Myb3F8hERXTIFSUqntPXDKXDb7Y/laqjMXdj+vptKk
+  3Q36J+8VnJbSwjGwmEG6tym9qMSGIPPNw1JXY1R29eF3o4aj21o7DHAkhuNc5Tso
+  67fUSKgvyVnyH4G6ShQUAtghPaAwS0KvAgMBAAGjUDBOMB0GA1UdDgQWBBSjxFUE
+  RfnTvABRLAa34Ytkhz5vPzAfBgNVHSMEGDAWgBSjxFUERfnTvABRLAa34Ytkhz5v
+  PzAMBgNVHRMEBTADAQH/MA0GCSqGSIb3DQEBBQUAA4GBAFLDS7zNhlrQYSQO5KIj
+  z2RJe3fj4rLPklo3TmP5KLvendG+LErE2cbKPqnhQ2oVoj6u9tWVwo/g03PMrrnL
+  KrDm39slYD/1KoE5kB4l/p6KVOdeJ4I6xcgu9rnkqqHzDwI4v7e8/D3WZbpiFUsY
+  vaZhjNYKWQf79l6zXfOvphzJ
+  -----END CERTIFICATE-----
+  -----BEGIN CERTIFICATE-----
   MIIB9jCCAV+gAwIBAgIJAO4V/jiMoICoMA0GCSqGSIb3DQEBBQUAMBMxETAPBgNV
   BAMMCENPTVAtMjMyMCAXDTEyMDIxNjExMTAyM1oYDzIxMTIwMTIzMTExMDIzWjAT
   MREwDwYDVQQDDAhDT01QLTIzMjCBnzANBgkqhkiG9w0BAQEFAAOBjQAwgYkCgYEA
@@ -34,6 +65,32 @@ signature-certificate-list =
   If1a2ZoqHRxoNo2yTmm7TSYRORWVS+vvfjY=
   -----END CERTIFICATE-----
   -----BEGIN CERTIFICATE-----
+  MIIB9jCCAV+gAwIBAgIJAIlBksrZVkK8MA0GCSqGSIb3DQEBBQUAMBMxETAPBgNV
+  BAMMCENPTVAtMzU3MCAXDTEyMDEyNjEwNTUyOFoYDzIxMTIwMTAyMTA1NTI4WjAT
+  MREwDwYDVQQDDAhDT01QLTM1NzCBnzANBgkqhkiG9w0BAQEFAAOBjQAwgYkCgYEA
+  ts+iGUwi44vtIfwXR8DCnLtHV4ydl0YTK2joJflj0/Ws7mz5BYkxIU4fea/6+VF3
+  i11nwBgYgxQyjNztgc9u9O71k1W5tU95yO7U7bFdYd5uxYA9/22fjObaTQoC4Nc9
+  mTu6r/VHyJ1yRsunBZXvnk/XaKp7gGE9vNEyJvPn2bkCAwEAAaNQME4wHQYDVR0O
+  BBYEFKuGIYu8+6aEkTVg62BRYaD11PILMB8GA1UdIwQYMBaAFKuGIYu8+6aEkTVg
+  62BRYaD11PILMAwGA1UdEwQFMAMBAf8wDQYJKoZIhvcNAQEFBQADgYEAMoTRpBxK
+  YLEZJbofF7gSrRIcrlUJYXfTfw1QUBOKkGFFDsiJpEg4y5pUk1s5Jq9K3SDzNq/W
+  it1oYjOhuGg3al8OOeKFrU6nvNTF1BAvJCl0tr3POai5yXyN5jlK/zPfypmQYxE+
+  TaqQSGBJPVXYt6lrq/PRD9ciZgKLOwEqK8w=
+  -----END CERTIFICATE-----
+  -----BEGIN CERTIFICATE-----
+  MIIB9jCCAV+gAwIBAgIJAPHoWu90gbsgMA0GCSqGSIb3DQEBBQUAMBQxEjAQBgNV
+  BAMMCXZpZmlibm9kZTAeFw0xMjAzMTkyMzIwNTVaFw0xMzAzMTkyMzIwNTVaMBQx
+  EjAQBgNVBAMMCXZpZmlibm9kZTCBnzANBgkqhkiG9w0BAQEFAAOBjQAwgYkCgYEA
+  ozBijpO8PS5RTeKTzA90vi9ezvv4vVjNaguqT4UwP9+O1+i6yq1Y2W5zZxw/Klbn
+  oudyNzie3/wqs9VfPmcyU9ajFzBv/Tobm3obmOqBN0GSYs5fyGw+O9G3//6ZEhf0
+  NinwdKmrRX+d0P5bHewadZWIvlmOupcnVJmkks852BECAwEAAaNQME4wHQYDVR0O
+  BBYEFF9EtgfZZs8L2ZxBJxSiY6eTsTEwMB8GA1UdIwQYMBaAFF9EtgfZZs8L2ZxB
+  JxSiY6eTsTEwMAwGA1UdEwQFMAMBAf8wDQYJKoZIhvcNAQEFBQADgYEAc43YTfc6
+  baSemaMAc/jz8LNLhRE5dLfLOcRSoHda8y0lOrfe4lHT6yP5l8uyWAzLW+g6s3DA
+  Yme/bhX0g51BmI6gjKJo5DoPtiXk/Y9lxwD3p7PWi+RhN+AZQ5rpo8UfwnnN059n
+  yDuimQfvJjBFMVrdn9iP6SfMjxKaGk6gVmI=
+  -----END CERTIFICATE-----
+  -----BEGIN CERTIFICATE-----
   MIIB9jCCAV+gAwIBAgIJAKRvzcy7OH0UMA0GCSqGSIb3DQEBBQUAMBMxETAPBgNV
   BAMMCENPTVAtNzcyMCAXDTEyMDgxMDE1NDI1MVoYDzIxMTIwNzE3MTU0MjUxWjAT
   MREwDwYDVQQDDAhDT01QLTc3MjCBnzANBgkqhkiG9w0BAQEFAAOBjQAwgYkCgYEA
@@ -46,60 +103,74 @@ signature-certificate-list =
   5pW18Ry5Ie7iFK4cQMerZwWPxBodEbAteYlRsI6kePV7Gf735Y1RpuN8qZ2sYL6e
   x2IMeSwJ82BpdEI5niXxB+iT0HxhmR+XaMI=
   -----END CERTIFICATE-----
+  -----BEGIN CERTIFICATE-----
+  MIIB+DCCAWGgAwIBAgIJAKGd0vpks6T/MA0GCSqGSIb3DQEBBQUAMBQxEjAQBgNV
+  BAMMCUNPTVAtMTU4NDAgFw0xMzA2MjAxMjE5MjBaGA8yMTEzMDUyNzEyMTkyMFow
+  FDESMBAGA1UEAwwJQ09NUC0xNTg0MIGfMA0GCSqGSIb3DQEBAQUAA4GNADCBiQKB
+  gQDZTH9etPUC+wMZQ3UIiOwyyCfHsJ+7duCFYjuo1uZrhtDt/fp8qb8qK9ob+df3
+  EEYgA0IgI2j/9jNUEnKbc5+OrfKznzXjrlrH7zU8lKBVNCLzQuqBKRNajZ+UvO8R
+  nlqK2jZCXP/p3HXDYUTEwIR5W3tVCEn/Vda4upTLcPVE5wIDAQABo1AwTjAdBgNV
+  HQ4EFgQU7KXaNDheQWoy5uOU01tn1M5vNkEwHwYDVR0jBBgwFoAU7KXaNDheQWoy
+  5uOU01tn1M5vNkEwDAYDVR0TBAUwAwEB/zANBgkqhkiG9w0BAQUFAAOBgQASmqCU
+  Znbvu6izdicvjuE3aKnBa7G++Fdp2bdne5VCwVbVLYCQWatB+n4crKqGdnVply/u
+  +uZ16u1DbO9rYoKgWqjLk1GfiLw5v86pd5+wZd5I9QJ0/Sbz2vZk5S4ciMIGwArc
+  m711+GzlW5xe6GyH9SZaGOPAdUbI6JTDwLzEgA==
+  -----END CERTIFICATE-----
 
 [versions]
 Werkzeug = 0.9.4
 apache-libcloud = 0.13.2
 async = 0.6.1
 buildout-versions = 1.7
+collective.recipe.template = 1.10
 erp5.util = 0.4.36
 gitdb = 0.5.4
 itsdangerous = 0.23
 lxml = 3.2.3
 meld3 = 0.6.10
 plone.recipe.command = 1.1
-psutil = 1.1.0
+psutil = 1.1.1
 pycrypto = 2.6
 rdiff-backup = 1.0.5
-slapos.cookbook = 0.84
+slapos.cookbook = 0.84.2
 slapos.recipe.cmmi = 0.2
 slapos.recipe.download = 1.0.dev-r4053
-slapos.toolbox = 0.37
+slapos.toolbox = 0.37.4
 smmap = 0.8.2
 websockify = 0.5.1
 z3c.recipe.scripts = 1.0.1
 
 # Required by:
 # slapos.core==0.35.1
-# slapos.toolbox==0.37
+# slapos.toolbox==0.37.4
 Flask = 0.10.1
 
 # Required by:
-# slapos.toolbox==0.37
+# slapos.toolbox==0.37.4
 GitPython = 0.3.2.RC1
 
 # Required by:
-# slapos.toolbox==0.37
+# slapos.toolbox==0.37.4
 atomize = 0.1.1
 
 # Required by:
 # paramiko==1.12.0
-ecdsa = 0.8
+ecdsa = 0.9
 
 # Required by:
-# slapos.toolbox==0.37
+# slapos.toolbox==0.37.4
 feedparser = 5.1.3
 
 # Required by:
-# slapos.cookbook==0.84
+# slapos.cookbook==0.84.2
 inotifyx = 0.2.0-1
 
 # Required by:
-# slapos.cookbook==0.84
+# slapos.cookbook==0.84.2
 lock-file = 2.0
 
 # Required by:
-# slapos.cookbook==0.84
+# slapos.cookbook==0.84.2
 netaddr = 0.7.10
 
 # Required by:
@@ -111,7 +182,7 @@ netifaces = 0.8-1
 numpy = 1.7.1
 
 # Required by:
-# slapos.toolbox==0.37
+# slapos.toolbox==0.37.4
 paramiko = 1.12.0
 
 # Required by:
@@ -119,12 +190,12 @@ paramiko = 1.12.0
 pyflakes = 0.7.3
 
 # Required by:
-# slapos.cookbook==0.84
-pytz = 2013d
+# slapos.cookbook==0.84.2
+pytz = 2013.7
 
 # Required by:
-# slapos.cookbook==0.84
-# slapos.toolbox==0.37
+# slapos.cookbook==0.84.2
+# slapos.toolbox==0.37.4
 slapos.core = 0.35.1
 
 # Required by:
@@ -136,8 +207,8 @@ supervisor = 3.0
 unittest2 = 0.5.1
 
 # Required by:
-# slapos.cookbook==0.84
-# slapos.toolbox==0.37
+# slapos.cookbook==0.84.2
+# slapos.toolbox==0.37.4
 xml-marshaller = 0.9.7
 
 # Required by:

software/kvm/template/kvm-export.sh.in

 #!/bin/bash
 # Create a backup of the disk image of the virtual machine
-QEMU_IMG=${kvm-instance:qemu-img-path}
-SNAPSHOT_NAME=$(date +%s)
-DISK_PATH=${kvm-instance:disk-path}
 BACKUP_PATH=${:backup-disk-path}
+QMP_CLIENT=${buildout:directory}/software_release/bin/qemu-qmp-client
 
 if [ ! -f $DISK_PATH ]; then
   echo "Nothing to backup, disk image doesn't exist yet."
   exit 0;
 fi
 
-$QEMU_IMG snapshot -c $SNAPSHOT_NAME $DISK_PATH
-if [ -f $BACKUP_PATH ]; then
-  rm $BACKUP_PATH
-fi
-$QEMU_IMG convert -f qcow2 -O qcow2 -s $SNAPSHOT_NAME $DISK_PATH $BACKUP_PATH && \
-$QEMU_IMG snapshot -d $SNAPSHOT_NAME $DISK_PATH
+$QMP_CLIENT --socket ${kvm-instance:socket-path} --drive-backup $BACKUP_PATH
+

software/slaprunner/common.cfg

@@ -15,7 +15,6 @@ extends =
   ../../stack/slapos.cfg
 
 parts =
-  rdiff-backup
   template
   eggs
   nginx
@@ -27,6 +26,10 @@ parts =
   instance-runner-export
   slapos-cookbook
 
+# XXX: we have to manually add this for resilience
+  rdiff-backup
+  collective.recipe.template-egg
+
 ####################
 ## Node JS proxy
 ####################

stack/erp5/buildout.cfg

@@ -189,7 +189,7 @@ mode = 640
 [template-kumofs]
 < = template-jinja2-base
 filename = instance-kumofs.cfg
-md5sum = 90a321be12ee977800d590bf941021ef
+md5sum = 40817014a41497bceb696e512436e670
 extra-context =
     key dash_location dash:location
     key dcron_location dcron:location
@@ -430,7 +430,7 @@ eggs =
   paramiko
   ply
   pyflakes
-  pyPdf # should be replaced by PyPDF2, but it is not installable
+  pypdf2
   python-magic
   python-memcached
   pytz
@@ -448,6 +448,8 @@ eggs =
   huBarcode
   qrcode
   spyne
+  httplib2
+  suds
 # Needed for checking ZODB Components source code
   pylint
 
@@ -550,8 +552,8 @@ scripts =
 
 [versions]
 # pin Acquisition and Products.DCWorkflow to Nexedi flavour of eggs
-Acquisition = 2.13.7nxd001
-Products.DCWorkflow = 2.2.3nxd002
+Acquisition = 2.13.8nxd001
+Products.DCWorkflow = 2.2.4nxd001
 
 # specify dev version to be sure that an old released version is not used
 cloudooo = 1.2.5-dev
@@ -613,9 +615,7 @@ GitPython = 0.3.2.RC1
 Jinja2 = 2.6
 MySQL-python = 1.2.4
 PIL = 1.1.7
-Paste = 1.7.5.1
 PasteDeploy = 1.5.0
-PasteScript = 1.7.5
 Products.CMFActionIcons = 2.1.3
 Products.DCWorkflowGraph = 0.4.1
 Products.ExternalEditor = 1.1.0
@@ -655,7 +655,7 @@ inotifyx = 0.2.0
 ipdb = 0.7
 ipython = 0.13.2
 meld3 = 0.6.10
-mr.developer = 1.25
+mr.developer = 1.26
 netaddr = 0.7.10
 netifaces = 0.8_1
 ordereddict = 1.1

stack/erp5/instance-kumofs.cfg.in

@@ -34,8 +34,8 @@ gateway-wrapper = ${basedirectory:services}/kumofs_gateway
 manager-wrapper = ${basedirectory:services}/kumofs_manager
 server-wrapper = ${basedirectory:services}/kumofs_server
 
-# Paths: Data
-data-directory = ${directory:kumofs-data}
+# Paths: Data (with 10M buckets and HDBTLARGE option)
+data-path = ${directory:kumofs-data}/kumodb.tch#bnum=10485760#opts=l
 
 # Paths: Logs
 kumo-gateway-log = ${basedirectory:log}/kumo-gateway.log

stack/resilient/buildout.cfg

 [buildout]
 extends =
+  ../../component/apache/buildout.cfg
+  ../../component/bash/buildout.cfg
   ../../component/dropbear/buildout.cfg
   ../../component/gzip/buildout.cfg
   ../../component/rdiff-backup/buildout.cfg
   ../../component/rsync/buildout.cfg
 
 parts =
-  rdiff-backup
+  collective.recipe.template-egg
   pbsready
   pbsready-import
   pbsready-export
@@ -16,7 +18,12 @@ parts =
 
   # needed tools for resiliency
   gzip
+  rdiff-backup
+  dash
 
+[collective.recipe.template-egg]
+recipe = zc.recipe.egg
+eggs = collective.recipe.template
 
 #----------------
 #--
@@ -30,7 +37,7 @@ parts =
 recipe = slapos.recipe.template
 url = ${:_profile_base_location_}/pbsready.cfg.in
 output = ${buildout:directory}/pbsready.cfg
-md5sum = 570e0b54c97d510befa2ea981c1e90e0
+md5sum = 02a5f1741d6b732519c06b522dbe0d66
 mode = 0644
 
 [pbsready-import]
@@ -39,7 +46,7 @@ mode = 0644
 recipe = slapos.recipe.template
 url = ${:_profile_base_location_}/pbsready-import.cfg.in
 output = ${buildout:directory}/pbsready-import.cfg
-md5sum = cc9c776500ccd07cb51969beb68ffcda
+md5sum = 0f953067aac3e0132f72fc7e1ed38bd4
 mode = 0644
 
 [pbsready-export]
@@ -48,7 +55,7 @@ mode = 0644
 recipe = slapos.recipe.template
 url = ${:_profile_base_location_}/pbsready-export.cfg.in
 output = ${buildout:directory}/pbsready-export.cfg
-md5sum = 25d05b3929fb4c6cf275866bad678d6a
+md5sum = 135638b8c513c7723efb51e3d9182ae9
 mode = 0644
 
 [template-pull-backup]
@@ -61,14 +68,14 @@ mode = 0644
 [template-replicated]
 recipe = slapos.recipe.download
 url = ${:_profile_base_location_}/template-replicated.cfg.in
-md5sum = c762a625f65193bc8a570b4d56a0d08c
+md5sum = 771e1ab7e7e77b35116c57bbae56ba62
 mode = 0644
 destination = ${buildout:directory}/template-replicated.cfg.in
 
 [template-parts]
 recipe = slapos.recipe.download
 url = ${:_profile_base_location_}/template-parts.cfg.in
-md5sum = c942f82552fcb42fc74a5f896e0cd5f3
+md5sum = a3f55a20881c3f1ec4416662146c06f7
 mode = 0644
 destination = ${buildout:directory}/template-parts.cfg.in
 
@@ -81,9 +88,15 @@ url = ${:_profile_base_location_}/instance-frozen.cfg.in
 md5sum = d21472f0e58f928fb827f2cbf22c4d4a
 output = ${buildout:directory}/instance-frozen.cfg
 
+[resilient-web-takeover-cgi-script-download]
+recipe = slapos.recipe.download
+url = ${:_profile_base_location_}/resilient-web-takeover-cgi-script.py.in
+md5sum = e6262c5cf9b1c4d1ea4d959fdcbe3070
+mode = 0644
+destination = ${buildout:directory}/resilient-web-takeover-cgi-script.py.in
+
 [versions]
 # Pin Jinja2 to 2.6, as 2.7 breaks current code
 Jinja2 = 2.6
 # ... And newer s.r.template requires Jinja2 >= 2.7
 slapos.recipe.template = 2.4.2
-

stack/resilient/parameter-schema.json

+{
+  "$schema": "http://json-schema.org/draft-04/schema",
+  "title": "Resiliency Parameters",
+  "description": "List of possible parameters used in the resilient stack",
+  "type": "object",
+
+  "properties": {
+    "-sla-0-computer_guid": {
+      "title": "Target computer for main instance",
+      "description": "Target computer GUID for main instance.",
+      "type": "string"
+    },
+    "-sla-1-computer_guid": {
+      "title": "Target computer for first clone",
+      "description": "Target computer for first clone and PBS.",
+      "type": "string"
+    },
+    "-sla-2-computer_guid": {
+      "title": "Target computer for second clone",
+      "description": "Target computer for second clone and PBS.",
+      "type": "string"
+    },
+    "resiliency-backup-periodicity": {
+      "title": "Periodicity of backup",
+      "description": "Periodicity of backup, in cron format. Default is every hour.",
+      "type": "string"
+    },
+    "remove-backup-older-than": {
+      "title": "Remove backups older than...",
+      "description": "Remove all the backups in PBS that are older than specified value. It should be rdiff-backup-compatible.",
+      "type": "string",
+      "default": "2W"
+    }
+  }
+}

stack/resilient/pbsready-export.cfg.in

@@ -14,6 +14,7 @@ parts =
   sshkeys-authority
   dropbear-server
   sshkeys-dropbear
+  resilient-sshkeys-dropbear-promise
   dropbear-server-pbs-authorized-key
   notifier
 
@@ -52,10 +53,10 @@ pidfile = $${resilient-directory:pid}/$${:name}.pid
 <= cron
 recipe = slapos.cookbook:cron.d
 name = backup
-frequency = $${slap-parameter:resiliency-backup-periodicity} * * * *
+frequency = $${slap-parameter:resiliency-backup-periodicity}
 command = $${notifier-exporter:wrapper}
 
 [slap-parameter]
-# in minutes, modulo 60, in cron.d format (i.e */15 is accepted).
-resiliency-backup-periodicity = 0
+# In cron.d format (i.e things like */15 * * * * are accepted).
+resiliency-backup-periodicity = 0 0 * * *
 

stack/resilient/pbsready-import.cfg.in

@@ -14,14 +14,21 @@ parts =
   sshkeys-authority
   dropbear-server
   sshkeys-dropbear
+  resilient-sshkeys-dropbear-promise
   dropbear-server-pbs-authorized-key
   notifier
 
+  resilient-web-takeover-cgi-script
+  resilient-web-takeover-httpd-wrapper
+  resilient-web-takeover-httpd-promise
+
   import-on-notification
   resilient-publish-connection-parameter
 
 [resilient-publish-connection-parameter]
 notification-url = http://[$${notifier:host}]:$${notifier:port}/notify
+takeover-url = http://[$${resilient-web-takeover-httpd-configuration-file:listening-ip}]:$${resilient-web-takeover-httpd-configuration-file:listening-port}/
+takeover-password = $${resilient-web-takeover-password:passwd}
 
 # Define port of ssh server. It has to be different from import so that it
 # supports export/import using same IP (slaprunner, slapos-in-partition,
@@ -36,3 +43,67 @@ port = 22220
 recipe = slapos.cookbook:notifier.callback
 on-notification-id = $${slap-parameter:on-notification}
 callback = $${importer:wrapper}
+
+###########
+# Deploy a webserver allowing to do takeover from a web browser.
+###########
+[resilient-web-takeover-password]
+recipe = slapos.cookbook:generate.password
+storage-path = $${directory:srv}/passwd
+bytes = 8
+
+[resilient-web-takeover-cgi-script]
+recipe = collective.recipe.template
+input = ${resilient-web-takeover-cgi-script-download:destination}
+output = $${directory:cgi-bin}/web-takeover.cgi
+password = $${resilient-web-takeover-password:passwd}
+mode = 700
+
+# XXX could it be something lighter?
+# XXX Add SSL
+[resilient-web-takeover-httpd-configuration-file]
+recipe = collective.recipe.template
+input = inline:
+  PidFile "$${:pid-file}"
+  Listen [$${:listening-ip}]:$${:listening-port}
+  ServerAdmin someone@email
+  DocumentRoot "$${:document-root}"
+  ErrorLog "$${:error-log}"
+  LoadModule unixd_module modules/mod_unixd.so
+  LoadModule access_compat_module modules/mod_access_compat.so
+  LoadModule authz_core_module modules/mod_authz_core.so
+  LoadModule authz_host_module modules/mod_authz_host.so
+  LoadModule mime_module modules/mod_mime.so
+  LoadModule cgid_module modules/mod_cgid.so
+  LoadModule dir_module modules/mod_dir.so
+  ScriptSock $${:cgid-pid-file}
+  <Directory $${:document-root}>
+    # XXX: security????
+    Options +ExecCGI
+    AddHandler cgi-script .cgi
+    DirectoryIndex web-takeover.cgi
+  </Directory>
+output = $${directory:etc}/resilient-web-takeover-httpd.conf
+# md5sum =
+listening-ip = $${slap-network-information:global-ipv6}
+# XXX: randomize-me
+listening-port = 9263
+htdocs = $${directory:cgi-bin}
+pid-file = $${directory:run}/resilient-web-takeover-httpd.pid
+cgid-pid-file = $${directory:run}/resilient-web-takeover-httpd-cgid.pid
+document-root = $${directory:cgi-bin}
+error-log = $${directory:log}/resilient-web-takeover-httpd-error-log
+
+[resilient-web-takeover-httpd-wrapper]
+recipe = slapos.cookbook:wrapper
+apache-executable = ${apache:location}/bin/httpd
+command-line = $${:apache-executable} -f $${resilient-web-takeover-httpd-configuration-file:output} -DFOREGROUND
+wrapper-path = $${basedirectory:services}/resilient-web-takeover-httpd
+
+[resilient-web-takeover-httpd-promise]
+recipe = slapos.cookbook:check_url_available
+path = $${basedirectory:promises}/resilient-web-takeover-httpd
+url = http://[$${resilient-web-takeover-httpd-configuration-file:listening-ip}]:$${resilient-web-takeover-httpd-configuration-file:listening-port}/
+dash_path = ${dash:location}/bin/dash
+curl_path = ${curl:location}/bin/curl
+

stack/resilient/pbsready.cfg.in

@@ -10,6 +10,7 @@ parts =
   sshkeys-authority
   dropbear-server
   sshkeys-dropbear
+  resilient-sshkeys-dropbear-promise
   dropbear-server-pbs-authorized-key
   notifier
 
@@ -49,6 +50,7 @@ crontabs = $${rootdirectory:etc}/crontabs
 cronstamps = $${rootdirectory:etc}/cronstamps
 logrotate-entries = $${rootdirectory:etc}/logrotate.d
 logrotate-backup = $${basedirectory:backup}/logrotate
+cgi-bin = $${rootdirectory:srv}/cgi-bin
 
 #----------------
 #--
@@ -179,6 +181,27 @@ server-binary = ${buildout:bin-directory}/pubsubserver
 notifier-binary = ${buildout:bin-directory}/pubsubnotifier
 
 
+#----------------
+#--
+#-- Dropbear.
+
+[dropbear-server]
+recipe = slapos.cookbook:dropbear
+host = $${slap-network-information:global-ipv6}
+# Explicitely excludes to define "port" argument. It will be defined in
+# pbs-ready-import.cfg.in and pbs-ready-export.cfg.in
+home = $${directory:ssh}
+wrapper = $${rootdirectory:bin}/raw_sshd
+shell = $${rdiff-backup-server:wrapper}
+rsa-keyfile = $${directory:ssh}/server_key.rsa
+dropbear-binary = ${dropbear:location}/sbin/dropbear
+
+[dropbear-server-pbs-authorized-key]
+<= dropbear-server
+recipe = slapos.cookbook:dropbear.add_authorized_key
+key = $${slap-parameter:authorized-key}
+
+
 #----------------
 #--
 #-- sshkeys
@@ -205,31 +228,21 @@ public-key = $${dropbear-server:rsa-keyfile}.pub
 private-key = $${dropbear-server:rsa-keyfile}
 wrapper = $${basedirectory:services}/sshd
 
-
-#----------------
-#--
-#-- Dropbear.
-
-[dropbear-server]
-recipe = slapos.cookbook:dropbear
-host = $${slap-network-information:global-ipv6}
-# Explicitely excludes to define "port" argument. It will be defined in
-# pbs-ready-import.cfg.in and pbs-ready-export.cfg.in
-home = $${directory:ssh}
-wrapper = $${rootdirectory:bin}/raw_sshd
-shell = $${rdiff-backup-server:wrapper}
-rsa-keyfile = $${directory:ssh}/server_key.rsa
-dropbear-binary = ${dropbear:location}/sbin/dropbear
-
-[dropbear-server-pbs-authorized-key]
-<= dropbear-server
-recipe = slapos.cookbook:dropbear.add_authorized_key
-key = $${slap-parameter:authorized-key}
+[resilient-sshkeys-dropbear-promise]
+# Check that public key file exists and is not empty
+recipe = collective.recipe.template
+input = inline:#!${bash:location}/bin/bash
+  PUBLIC_KEY_CONTENT="$${sshkeys-dropbear:public-key-value}"
+  if [[ ! -n "$PUBLIC_KEY_CONTENT" || "$PUBLIC_KEY_CONTENT" == *None* ]]; then
+    exit 1
+  fi
+output = $${basedirectory:promises}/public-key-existence
+mode = 700
 
 
 #----------------
 #--
-#-- Conncetion informations to re-use.
+#-- Connection informations to re-use.
 # XXX-Cedric: when "aggregation" system is done in libslap, directly publish.
 [resilient-publish-connection-parameter]
 recipe = slapos.cookbook:publish

stack/resilient/resilient-web-takeover-cgi-script.py.in

+#!${buildout:executable}
+import cgi
+import cgitb
+import os
+import subprocess
+import sys
+cgitb.enable()
+
+print "Content-Type: text/html"
+print    
+
+form = cgi.FieldStorage()
+if "password" not in form:
+  print """<html>
+<body>
+  <h1>This is takeover web interface.</h1>
+  <p>Calling takeover will stop and freeze the current main instance, and make this clone instance the new main instance, replacing the old one.</p>
+  <p><b>Warning: submit the form only if you understand what you are doing.</b></p>
+  <p>Note: the password asked here can be found within the parameters of your SlapOS instance page.</p>
+  <form action="/">
+    Password: <input type="text" name="password">
+    <input type="submit" value="Take over" style="background: red;">
+  </form>
+</body>
+</html>"""
+  sys.exit(0)
+
+if form['password'].value != '${:password}':
+  print "<H1>Error</H1>"
+  print "Password is invalid."
+  sys.exit(1)
+
+# XXX hardcoded location
+result = subprocess.check_output([os.path.expanduser("~/bin/takeover")], stderr=subprocess.STDOUT)
+print 'Success.'
+print '<pre>%s</pre>' % result

stack/resilient/template-parts.cfg.in

@@ -4,16 +4,19 @@
 
   request-{{namebase}}
   request-{{namebase}}-2
+  resilient-request-{{namebase}}-public-key-promise
   
-  {% for i in range(1,nbbackup|int) %}
-  request-{{namebase}}-pseudo-replicating-{{i}}
-  request-{{namebase}}-pseudo-replicating-{{i}}-2
+  {% for id in range(1,nbbackup|int) %}
+  request-{{namebase}}-pseudo-replicating-{{id}}
+  request-{{namebase}}-pseudo-replicating-{{id}}-2
+  resilient-request-{{namebase}}-pseudo-replicating-{{id}}-public-key-promise
   {% endfor %}
   
-  {% for i in range(1,nbbackup|int) %}
-  request-pbs-{{namebase}}-{{i}}
-  request-pull-backup-server-{{namebase}}-{{i}}
-  request-pull-backup-server-{{namebase}}-backup-{{i}}
+  {% for id in range(1,nbbackup|int) %}
+  request-pbs-{{namebase}}-{{id}}
+  resilient-request-pbs-{{namebase}}-{{id}}-public-key-promise
+  request-pull-backup-server-{{namebase}}-{{id}}
+  request-pull-backup-server-{{namebase}}-backup-{{id}}
   {% endfor %}
 
 

stack/resilient/template-replicated.cfg.in

@@ -11,6 +11,13 @@
 {% endif -%}
 
 
+[resilient-directory]
+recipe = slapos.cookbook:mkdirectory
+home = ${buildout:directory}
+etc = ${:home}/etc
+promise = ${:etc}/promise
+
+
 ## Tells the Backupable recipe that we want a backup
 [resilient]
 recipe = slapos.cookbook:request
@@ -28,7 +35,6 @@ software-url = ${slap-connection:software-release-url}
 software-type = {{typeexport}}
 name = {{namebase}}0
 return = ssh-public-key ssh-url notification-id ip
-
 config =
 # Resilient related parameters
   number authorized-key notify ip-list namebase
@@ -45,7 +51,10 @@ config-ip-list =
 {% for parameter_name, parameter_value in slapparameter_dict.items() %}config-{{parameter_name}} = {{parameter_value}}
 {% endfor %}
 {% endif %}
-{% if sla_parameter_dict -%}
+{% if sla_parameter_dict == {} -%}
+sla = mode
+sla-mode = unique_by_network
+{% else %}
 {%   set sla_key_main = "-sla-%s%s-" % (namebase, 0) -%}
 {%   set sla_key_secondary = "-sla-%s-" % (0) -%}
 {%   set sla_key_main_length = sla_key_main | length -%}
@@ -86,7 +95,10 @@ config-number = {{id}}
 config-authorized-key = ${request-pbs-{{namebase}}-{{id}}:connection-ssh-key}
 config-on-notification = ${request-pbs-{{namebase}}-{{id}}:connection-feeds-url}${:pbs-notification-id}
 config-ip-list =
-{% if sla_parameter_dict -%}
+{% if sla_parameter_dict == {} -%}
+sla = mode
+sla-mode = unique_by_network
+{% else %}
 {%   set sla_key_main = "-sla-%s%s-" % (namebase, id) -%}
 {%   set sla_key_secondary = "-sla-%s-" % (id) -%}
 {%   set sla_key_main_length = sla_key_main | length -%}
@@ -107,9 +119,9 @@ sla-{{ key }} = {{ value }}
 {%   endif %}
 {% endif %}
 
-
 {% endfor -%}
 
+
 [iplist]
 config-ip-list = ${request-{{namebase}}:connection-ip}{% for j in range(1,nbbackup|int) %} ${request-{{namebase}}-pseudo-replicating-{{j}}:connection-ip}{% endfor %}
 
@@ -117,11 +129,37 @@ config-ip-list = ${request-{{namebase}}:connection-ip}{% for j in range(1,nbback
 <= request-{{namebase}}
    iplist
 
+[resilient-request-{{namebase}}-public-key-promise]
+# Check that public-key-value parameter exists and is not empty
+# XXX: maybe we should consider empty values to be non-nexistent.
+recipe = collective.recipe.template
+# XXX: don't use system executable
+input = inline:#!/bin/sh
+  PUBLIC_KEY_CONTENT="${request-{{namebase}}-2:connection-ssh-public-key})"
+  if [[ ! -n "$PUBLIC_KEY_CONTENT" || "$PUBLIC_KEY_CONTENT" == *None* ]]; then
+    exit 1
+  fi
+output = ${resilient-directory:promise}/resilient-request-{{namebase}}-public-key
+mode = 700
+
 {% for id in range(1,nbbackup|int) %}
 [request-{{namebase}}-pseudo-replicating-{{id}}-2]
 <= request-{{namebase}}-pseudo-replicating-{{id}}
    iplist
 
+[resilient-request-{{namebase}}-pseudo-replicating-{{id}}-public-key-promise]
+# Check that public-key-value parameter exists and is not empty
+# XXX: maybe we should consider empty values to be non-nexistent.
+recipe = collective.recipe.template
+# XXX: don't use system executable
+input = inline:#!/bin/sh
+  PUBLIC_KEY_CONTENT="${request-{{namebase}}-pseudo-replicating-{{id}}-2:connection-ssh-public-key})"
+  if [[ ! -n "$PUBLIC_KEY_CONTENT" || "$PUBLIC_KEY_CONTENT" == *None* ]]; then
+    exit 1
+  fi
+output = ${resilient-directory:promise}/resilient-request-{{namebase}}-pseudo-replicating-{{id}}-public-key
+mode = 700
+
 {% endfor %}
 
 
@@ -146,7 +184,10 @@ software-type = pull-backup
 name = PBS ({{namebase}} / {{id}})
 return = ssh-key notification-url feeds-url
 slave = false
-{% if sla_parameter_dict -%}
+{% if sla_parameter_dict == {} -%}
+sla = mode
+sla-mode = unique_by_network
+{% else %}
 {%   set sla_key_main = "-sla-%s%s-" % ("pbs", id) -%}
 {%   set sla_key_secondary = "-sla-%s-" % (id) -%}
 {%   set sla_key_main_length = sla_key_main | length -%}
@@ -167,6 +208,19 @@ sla-{{ key }} = {{ value }}
 {%   endif %}
 {% endif %}
 
+[resilient-request-pbs-{{namebase}}-{{id}}-public-key-promise]
+# Check that public-key-value parameter exists and is not empty
+# XXX: maybe we should consider empty values to be non-nexistent.
+recipe = collective.recipe.template
+# XXX: don't use system executable
+input = inline:#!/bin/sh
+  PUBLIC_KEY_CONTENT="${request-pbs-{{namebase}}-{{id}}:connection-ssh-key}:connection-ssh-key})"
+  if [[ ! -n "$PUBLIC_KEY_CONTENT" || "$PUBLIC_KEY_CONTENT" == *None* ]]; then
+    exit 1
+  fi
+output = ${resilient-directory:promise}/resilient-request-{{namebase}}-pseudo-replicating-{{id}}-public-key
+mode = 700
+
 
 [request-pull-backup-server-{{namebase}}-{{id}}]
 <= request-pbs-common
@@ -180,7 +234,7 @@ config-notify = ${request-pbs-{{namebase}}-{{id}}:connection-notification-url}
 config-notification-id = ${slap-connection:computer-id}-${slap-connection:partition-id}-{{namebase}}-{{id}}-pull
 config-name = ${slap-connection:computer-id}-${slap-connection:partition-id}-{{namebase}}-{{id}}
 config-title = Pulling from {{namebase}}
-config-remove-backup-older-than = {{ slapparameter_dict.get('remove-backup-older-than', '3B') }}
+config-remove-backup-older-than = {{ slapparameter_dict.get('remove-backup-older-than', '2W') }}
 slave = true
 sla = instance_guid
 sla-instance_guid = ${request-pbs-{{namebase}}-{{id}}:instance_guid}

stack/slapos.cfg

@@ -12,6 +12,7 @@ extensions +=
 # Use shacache and lxml
 extends =
   ../component/lxml-python/buildout.cfg
+  ../component/python-openssl/buildout.cfg
 
 # Separate from site eggs
 allowed-eggs-from-site-packages =
@@ -59,6 +60,7 @@ networkcache-section = networkcache
 recipe = zc.recipe.egg
 eggs =
   ${lxml-python:egg}
+  ${python-openssl:egg}
   slapos.cookbook
   cliff
   hexagonit.recipe.download