diff --git a/CHANGES.txt b/CHANGES.txt
index ac7356d0efc98a9ce543bffbbebe9fbb399569f6..ef1ade22b76132b5d575f32a6249aa0c0e97c207 100644
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -1,6 +1,23 @@
Changes
=======
+0.84.1 (2013-10-03)
+-------------------
+
+ * Resiliency: PBS: promise should NOT bang. [64886cd]
+
+0.84 (2013-09-30)
+-----------------
+
+ * Request.py: improve instance-state handling. [ba5f160]
+ * Resilient recipe: remove hashing of urls/names. [ee2aec8]
+ * Resilient pbs recipe: recover from rdiff-backup failures. [be7f2fc, 92ee0c3]
+ * Resilience: add pidfiles in PBS. [0b3ad5c]
+ * Resilient: don't hide exception, print it. [05b3d64, d2b0494]
+ * Resiliency: Only keep 10 increments of backup. [4e89e33]
+ * KVM SR: add fallback in case of download exception. [de8d796]
+ * slaprunner: don't check certificate for importer. [53dc772]
+
0.83.1 (2013-09-10)
------------------
diff --git a/component/groonga/buildout.cfg b/component/groonga/buildout.cfg
index 6e3f24c5aa7a2ff205b00f036e989fa0e30ff1e8..29f8d315a809b8b8dc60dbc34c10e42dbbdd95d9 100644
--- a/component/groonga/buildout.cfg
+++ b/component/groonga/buildout.cfg
@@ -11,9 +11,9 @@ extends =
[groonga]
recipe = slapos.recipe.cmmi
-version = 3.0.5
+version = 3.0.8
url = http://packages.groonga.org/source/groonga/groonga-${:version}.tar.gz
-md5sum = 2894bbdd2275cb3c62aea14446dc2561
+md5sum = cb32e16126d80ea22604eba4bfb07c9f
configure-options =
--disable-static
--disable-glibtest
diff --git a/component/mariadb/buildout.cfg b/component/mariadb/buildout.cfg
index 6d8787ccd158fe0c911bb785ee5640efd1bc0545..ec36d3d3abf53c9d5f087a7dd620dc0d9c1b76d0 100644
--- a/component/mariadb/buildout.cfg
+++ b/component/mariadb/buildout.cfg
@@ -62,8 +62,8 @@ environment =
# mroonga - a storage engine for MySQL. It provides fast fulltext search feature to all MySQL users.
# http://mroonga.github.com/
recipe = slapos.recipe.cmmi
-url = http://packages.groonga.org/source/mroonga/mroonga-3.05.tar.gz
-md5sum = ba4cbd79274d832b9343a0b2fe7d0787
+url = http://packages.groonga.org/source/mroonga/mroonga-3.08.tar.gz
+md5sum = 7658b94965ae951949b472763d6c8d5a
configure-options =
--with-mysql-source=${mariadb:location}__compile__/mariadb-${mariadb:version}
--with-mysql-config=${mariadb:location}/bin/mysql_config
diff --git a/component/slapos/buildout.cfg b/component/slapos/buildout.cfg
index ec82932c9cfd392bad2f2d2573f76997a8ad0dac..ab15edfeefd8ebf27d73cae4d349511eb639e62b 100644
--- a/component/slapos/buildout.cfg
+++ b/component/slapos/buildout.cfg
@@ -17,6 +17,7 @@ extends =
../pkgconfig/buildout.cfg
../popt/buildout.cfg
../python-2.7/buildout.cfg
+ ../python-openssl/buildout.cfg
../readline/buildout.cfg
../sqlite3/buildout.cfg
../swig/buildout.cfg
@@ -90,10 +91,14 @@ output = ${buildout:directory}/environment.sh
[lxml-python]
python = python2.7
+[python-openssl]
+python = python2.7
+
[slapos]
recipe = z3c.recipe.scripts
python = python2.7
eggs =
+ ${python-openssl:egg}
slapos.libnetworkcache
zc.buildout
${lxml-python:egg}
@@ -131,46 +136,64 @@ scripts = py
[versions]
# Use our own buildout version
-zc.buildout = 1.6.0-dev-SlapOS-010
+zc.buildout = 1.6.0-dev-SlapOS-012
# Force to use zc.recipe.egg 1.x
zc.recipe.egg = 1.3.2
# Use own version of h.r.download to be able to open archives not supported by python2.x: .xz
-hexagonit.recipe.download = 1.6nxd002
+hexagonit.recipe.download = 1.7nxd002
-Jinja2 = 2.7
+Jinja2 = 2.7.1
MarkupSafe = 0.18
-Werkzeug = 0.8.3
+Pygments = 1.6
+Werkzeug = 0.9.4
buildout-versions = 1.7
+cmd2 = 0.6.7
collective.recipe.template = 1.10
-lxml = 3.1.2
+itsdangerous = 0.23
+lxml = 3.2.3
meld3 = 0.6.10
netaddr = 0.7.10
+prettytable = 0.7.2
+pyOpenSSL = 0.13.1
+pyparsing = 2.0.1
+setuptools = 1.1.6
+slapos.core = 1.0.0rc6
slapos.libnetworkcache = 0.13.4
+slapos.recipe.cmmi = 0.1.1
xml-marshaller = 0.9.7
z3c.recipe.scripts = 1.0.1
# Required by:
-# slapos.core==0.35.2-dev
-Flask = 0.9
+# slapos.core==1.0.0rc6
+Flask = 0.10.1
# Required by:
-# slapos.core==0.35.2-dev
+# slapos.core==1.0.0rc6
+bpython = 0.12
+
+# Required by:
+# slapos.core==1.0.0rc6
+cliff = 1.4.5
+
+# Required by:
+# slapos.core==1.0.0rc6
+ipython = 1.1.0
+
+# Required by:
+# slapos.core==1.0.0rc6
netifaces = 0.8
# Required by:
-# slapos.core==0.35.2-dev
-# slapos.libnetworkcache==0.13.3
-# supervisor==3.0b1
-# zc.buildout==1.6.0-dev-SlapOS-010
-# zope.interface==4.0.5
-setuptools = 0.6c12dev-r88846
+# slapos.core==1.0.0rc6
+requests = 2.0.0
# Required by:
-# slapos.core==0.35.2-dev
-supervisor = 3.0b2
+# slapos.core==1.0.0rc6
+supervisor = 3.0
# Required by:
-# slapos.core==0.35.2-dev
+# slapos.core==1.0.0rc6
zope.interface = 4.0.5
+
diff --git a/component/slapos/testing.cfg b/component/slapos/testing.cfg
new file mode 100644
index 0000000000000000000000000000000000000000..58ed21db4e5fa5b7aca12ff6e1b16789ab13d900
--- /dev/null
+++ b/component/slapos/testing.cfg
@@ -0,0 +1,11 @@
+# This file is used to install testing, not-stable-yet, version of SlapOS Node
+[buildout]
+extends =
+ buildout.cfg
+
+# Add hosting location of testing version of slapos.core
+find-links +=
+ http://www.nexedi.org/static/packages/source/slapos.core-testing/
+
+[versions]
+slapos.core =
diff --git a/component/sqlite3/buildout.cfg b/component/sqlite3/buildout.cfg
index 24576703af0e70986d3bf116acbefe06a8464d98..eb2818ba8298a9828c3c734d33805327707aa7cd 100644
--- a/component/sqlite3/buildout.cfg
+++ b/component/sqlite3/buildout.cfg
@@ -5,8 +5,8 @@ parts =
[sqlite3]
recipe = slapos.recipe.cmmi
-url = http://www.sqlite.org/2013/sqlite-autoconf-3071700.tar.gz
-md5sum = 18c285053e9562b848209cb0ee16d4ab
+url = http://www.sqlite.org/2013/sqlite-autoconf-3080002.tar.gz
+md5sum = 6d6cc639a4da04fbbdda7b1a1a01b386
configure-options =
--disable-static
--enable-readline
diff --git a/setup.py b/setup.py
index 1efd3ed6fddf4bbf317d4343fc96e0ce823a90f3..fe4a9ab348fab34395426726ae3244a9d8caa9bc 100755
--- a/setup.py
+++ b/setup.py
@@ -28,7 +28,7 @@ from setuptools import setup, find_packages
import glob
import os
-version = '0.83.1'
+version = '0.84.1'
name = 'slapos.cookbook'
long_description = open("README.txt").read() + "\n" + \
open("CHANGES.txt").read() + "\n"
diff --git a/slapos/recipe/addresiliency/takeover.py b/slapos/recipe/addresiliency/takeover.py
index 7f672ccc6e65fd38c6822e8677101f0b48914abe..024d923acc1d7efad37de1f4814e3274199ecd7d 100644
--- a/slapos/recipe/addresiliency/takeover.py
+++ b/slapos/recipe/addresiliency/takeover.py
@@ -1,6 +1,7 @@
# -*- coding: utf-8 -*-
import logging
import time
+import traceback
import slapos
from slapos.slap.slap import NotFoundError
@@ -65,6 +66,7 @@ def takeover(server_url, key_file, cert_file, computer_guid,
cp_winner.rename(new_name=cp_exporter_ref)
break
except NotFoundError:
+ traceback.print_exc()
log.warning('Impossible to rename. Retrying in a few seconds...')
log.debug('Renamed.')
diff --git a/slapos/recipe/kvm/template/kvm_run.in b/slapos/recipe/kvm/template/kvm_run.in
index d3ec485fdfeaccc06c76d2af7a2d934e4c056f68..09095a77b7d14a9ffb5eae40722e5ab1747fec39 100644
--- a/slapos/recipe/kvm/template/kvm_run.in
+++ b/slapos/recipe/kvm/template/kvm_run.in
@@ -60,7 +60,11 @@ def getSocketStatus(host, port):
# Download existing hard drive if needed at first boot
if not os.path.exists(disk_path) and virtual_hard_drive_url != '':
print('Downloading virtual hard drive...')
- urllib.urlretrieve(virtual_hard_drive_url, disk_path)
+ try:
+ urllib.urlretrieve(virtual_hard_drive_url, disk_path)
+ except:
+ os.remove(disk_path)
+ raise
md5sum = virtual_hard_drive_md5sum.strip()
if md5sum:
print('Checking MD5 checksum...')
diff --git a/slapos/recipe/notifier.py b/slapos/recipe/notifier.py
index 246fc4cb7429087de3ca45cc631fd898c3756dae..7fa27b07bec9459ce0470bbd0a32e4e1d8c86c96 100644
--- a/slapos/recipe/notifier.py
+++ b/slapos/recipe/notifier.py
@@ -50,6 +50,9 @@ class Recipe(GenericBaseRecipe):
class Callback(GenericBaseRecipe):
def createCallback(self, notification_id, callback):
+ # XXX: hashing the name here and in
+ # slapos.toolbox/slapos/pubsub/__init__.py is completely messed up and
+ # prevent any debug.
callback_id = sha512(notification_id).hexdigest()
filepath = os.path.join(self.options['callbacks'], callback_id)
diff --git a/slapos/recipe/pbs.py b/slapos/recipe/pbs.py
index d886b945d83e367ffa0b97daf78e5678fd16a227..6616199ac1e5db9bac67511b7c04350cd5fe5bd5 100644
--- a/slapos/recipe/pbs.py
+++ b/slapos/recipe/pbs.py
@@ -25,43 +25,24 @@
#
##############################################################################
-import hashlib
import json
import os
-import signal
import subprocess
import sys
+import textwrap
import urlparse
from slapos.recipe.librecipe import GenericSlapRecipe
from slapos.recipe.dropbear import KnownHostsFile
from slapos.recipe.notifier import Notify
from slapos.recipe.notifier import Callback
-from slapos import slap as slapmodule
def promise(args):
-
- def failed_ssh():
- sys.stderr.write("SSH Connection failed\n")
- partition = slap.registerComputerPartition(args['computer_id'],
- args['partition_id'])
- partition.bang("SSH Connection failed. rdiff-backup is unusable.")
-
- def sigterm_handler(signum, frame):
- failed_ssh()
-
- signal.signal(signal.SIGTERM, sigterm_handler)
-
- slap = slapmodule.slap()
- slap.initializeConnection(args['server_url'],
- key_file=args.get('key_file'),
- cert_file=args.get('cert_file'))
-
- ssh = subprocess.Popen([args['ssh_client'], '%(user)s@%(host)s/%(port)s' % args],
- stdin=subprocess.PIPE,
- stdout=open(os.devnull, 'w'),
- stderr=open(os.devnull, 'w'))
+ ssh = subprocess.Popen(
+ [args['ssh_client'], '%(user)s@%(host)s/%(port)s' % args],
+ stdin=subprocess.PIPE, stdout=None, stderr=None
+ )
# Rdiff Backup protocol quit command
quitcommand = 'q' + chr(255) + chr(0) * 7
@@ -74,7 +55,7 @@ def promise(args):
if ssh.poll() is None:
return 1
if ssh.returncode != 0:
- failed_ssh()
+ sys.stderr.write("SSH Connection failed\n")
return ssh.returncode
@@ -87,16 +68,19 @@ class Recipe(GenericSlapRecipe, Notify, Callback):
url = entry.get('url')
if not url:
raise ValueError('Missing URL parameter for PBS recipe')
+ parsed_url = urlparse.urlparse(url)
- # We assume that thanks to sha512 there's no collisions
- url_hash = hashlib.sha512(url).hexdigest()
- name_hash = hashlib.sha512(entry['name']).hexdigest()
+ slave_type = entry['type']
+ if not slave_type in ['pull', 'push']:
+ raise ValueError('type parameter must be either pull or push.')
- promise_path = os.path.join(self.options['promises-directory'],
- url_hash)
- parsed_url = urlparse.urlparse(url)
- promise_dict = self.promise_base_dict.copy()
- promise_dict.update(user=parsed_url.username,
+ slave_id = entry['notification-id']
+
+ print 'Processing PBS slave %s with type %s' % (slave_id, slave_type)
+
+ promise_path = os.path.join(self.options['promises-directory'], slave_id)
+ promise_dict = dict(ssh_client=self.options['sshclient-binary'],
+ user=parsed_url.username,
host=parsed_url.hostname,
port=parsed_url.port)
promise = self.createPythonScript(promise_path,
@@ -104,72 +88,109 @@ class Recipe(GenericSlapRecipe, Notify, Callback):
promise_dict)
path_list.append(promise)
-
host = parsed_url.hostname
known_hosts_file[host] = entry['server-key']
- # XXX use -y because the host might not yet be in the
- # trusted hosts file until the next time slapgrid is run.
-
- remote_schema = '%(ssh)s -y -p %%s %(user)s@%(host)s' % \
- {
- 'ssh': self.options['sshclient-binary'],
- 'user': parsed_url.username,
- 'host': parsed_url.hostname,
- }
+ notifier_wrapper_path = os.path.join(self.options['wrappers-directory'], slave_id)
+ rdiff_wrapper_path = notifier_wrapper_path + '_raw'
- parameters = ['--remote-schema', remote_schema]
+ # Create the rdiff-backup wrapper
+ # It is useful to separate it from the notifier so that we can run it
+ # Manually.
+ rdiffbackup_parameter_list = []
+ # XXX use -y because the host might not yet be in the
+ # trusted hosts file until the next time slapgrid is run.
+ rdiffbackup_remote_schema = '%(ssh)s -y -p %%s %(user)s@%(host)s' % {
+ 'ssh': self.options['sshclient-binary'],
+ 'user': parsed_url.username,
+ 'host': parsed_url.hostname,
+ }
remote_directory = '%(port)s::%(path)s' % {'port': parsed_url.port,
'path': parsed_url.path}
-
- local_directory = self.createDirectory(self.options['directory'],
- name_hash)
-
- if entry['type'] == 'push':
- parameters.extend(['--restore-as-of', 'now'])
- parameters.append('--force')
- parameters.extend([local_directory, remote_directory])
- comments = ['','Push data to a PBS *-import instance.','']
- else:
- parameters.extend([remote_directory, local_directory])
- comments = ['','Pull data from a PBS *-export instance.','']
-
- wrapper_basepath = os.path.join(self.options['wrappers-directory'],
- url_hash)
-
- if 'notify' in entry:
- wrapper_path = wrapper_basepath + '_raw'
- else:
- wrapper_path = wrapper_basepath
-
- wrapper = self.createWrapper(name=wrapper_path,
- command=self.options['rdiffbackup-binary'],
- parameters=parameters,
- comments = comments)
- path_list.append(wrapper)
-
- if 'notify' in entry:
- feed_url = '%s/get/%s' % (self.options['notifier-url'],
- entry['notification-id'])
- wrapper = self.createNotifier(notifier_binary=self.options['notifier-binary'],
- wrapper=wrapper_basepath,
- executable=wrapper_path,
- log=os.path.join(self.options['feeds'], entry['notification-id']),
- title=entry.get('title', 'Untitled'),
- notification_url=entry['notify'],
- feed_url=feed_url,
- )
- path_list.append(wrapper)
- #self.setConnectionDict(dict(feed_url=feed_url), entry['slave_reference'])
+ local_directory = self.createDirectory(self.options['directory'], entry['name'])
+
+ if slave_type == 'push':
+ # Create a simple rdiff-backup wrapper that will push
+ rdiffbackup_parameter_list.extend(['--remote-schema', rdiffbackup_remote_schema])
+ rdiffbackup_parameter_list.extend(['--restore-as-of', 'now'])
+ rdiffbackup_parameter_list.append('--force')
+ rdiffbackup_parameter_list.append(local_directory)
+ rdiffbackup_parameter_list.append(remote_directory)
+ comments = ['', 'Push data to a PBS *-import instance.', '']
+ rdiff_wrapper = self.createWrapper(
+ name=rdiff_wrapper_path,
+ command=self.options['rdiffbackup-binary'],
+ parameters=rdiffbackup_parameter_list,
+ comments=comments,
+ pidfile=os.path.join(self.options['run-directory'], '%s_raw.pid' % slave_id),
+ )
+ elif slave_type == 'pull':
+ # Wrap rdiff-backup call into a script that checks consistency of backup
+ # We need to manually escape the remote schema
+ rdiffbackup_parameter_list.extend(['--remote-schema', '"%s"' % rdiffbackup_remote_schema])
+ rdiffbackup_parameter_list.append(remote_directory)
+ rdiffbackup_parameter_list.append(local_directory)
+ comments = ['', 'Pull data from a PBS *-export instance.', '']
+ rdiff_wrapper_template = textwrap.dedent("""\
+ #!/bin/sh
+ # %(comment)s
+ RDIFF_BACKUP="%(rdiffbackup_binary)s"
+ $RDIFF_BACKUP %(rdiffbackup_parameter)s
+ if [ ! $? -eq 0 ]; then
+ # Check the backup, go to the last consistent backup, so that next
+ # run will be okay.
+ echo "Checking backup directory..."
+ $RDIFF_BACKUP --check-destination-dir %(local_directory)s
+ if [ ! $? -eq 0 ]; then
+ # Here, two possiblities:
+ # * The first backup failed. It is safe to remove it since there is nothing valuable there.
+ # * The backup has been complete, but is now in a really weird state. Not safe to remove it.
+ echo "Impossible to check backup: we move it to a safe place."
+ # XXX: bang
+ mv %(local_directory)s %(local_directory)s.$(date +%%s)
+ fi
+ else
+ # Everything's okay, cleaning up...
+ $RDIFF_BACKUP --remove-older-than %(remove_backup_older_than)s --force %(local_directory)s
+ fi
+ """)
+ rdiff_wrapper_content = rdiff_wrapper_template % {
+ 'comment': comments,
+ 'rdiffbackup_binary': self.options['rdiffbackup-binary'],
+ 'local_directory': local_directory,
+ 'rdiffbackup_parameter': ' \\\n '.join(rdiffbackup_parameter_list),
+ # XXX: only 10 increments is not enough by default.
+ 'remove_backup_older_than': entry.get('remove-backup-older-than', '3B')
+ }
+ rdiff_wrapper = self.createFile(
+ name=rdiff_wrapper_path,
+ content=rdiff_wrapper_content,
+ mode=0700
+ )
+
+ path_list.append(rdiff_wrapper)
+
+ # Create notifier wrapper
+ notifier_wrapper = self.createNotifier(
+ notifier_binary=self.options['notifier-binary'],
+ wrapper=notifier_wrapper_path,
+ executable=rdiff_wrapper,
+ log=os.path.join(self.options['feeds'], entry['notification-id']),
+ title=entry.get('title', slave_id),
+ notification_url=entry['notify'],
+ feed_url='%s/get/%s' % (self.options['notifier-url'], entry['notification-id']),
+ pidfile=os.path.join(self.options['run-directory'], '%s.pid' % slave_id)
+ )
+ path_list.append(notifier_wrapper)
if 'on-notification' in entry:
path_list.append(self.createCallback(str(entry['on-notification']),
- wrapper))
+ notifier_wrapper))
else:
- cron_entry = os.path.join(self.options['cron-entries'], url_hash)
+ cron_entry = os.path.join(self.options['cron-entries'], slave_id)
with open(cron_entry, 'w') as cron_entry_file:
- cron_entry_file.write('%s %s' % (entry['frequency'], wrapper))
+ cron_entry_file.write('%s %s' % (entry['frequency'], notifier_wrapper))
path_list.append(cron_entry)
return path_list
@@ -181,20 +202,9 @@ class Recipe(GenericSlapRecipe, Notify, Callback):
if self.optionIsTrue('client', True):
self.logger.info("Client mode")
- slap_connection = self.buildout['slap-connection']
- self.promise_base_dict = {
- 'server_url': slap_connection['server-url'],
- 'computer_id': slap_connection['computer-id'],
- 'cert_file': slap_connection.get('cert-file'),
- 'key_file': slap_connection.get('key-file'),
- 'partition_id': slap_connection['partition-id'],
- 'ssh_client': self.options['sshclient-binary'],
- }
-
slaves = json.loads(self.options['slave-instance-list'])
known_hosts = KnownHostsFile(self.options['known-hosts'])
with known_hosts:
- # XXX this API could be cleaner
for slave in slaves:
path_list.extend(self.add_slave(slave, known_hosts))
else:
@@ -209,4 +219,3 @@ class Recipe(GenericSlapRecipe, Notify, Callback):
path_list.append(wrapper)
return path_list
-
diff --git a/slapos/recipe/request.py b/slapos/recipe/request.py
index 47d9c38258d2208783ffc9a3179b3865b1878374..739b71940362cee068dcf060caf1e080c31d54a9 100644
--- a/slapos/recipe/request.py
+++ b/slapos/recipe/request.py
@@ -119,8 +119,11 @@ class Recipe(object):
))
slave = options.get('slave', 'false').lower() in \
librecipe.GenericBaseRecipe.TRUE_VALUES
+
# By default XXXX Way of doing it is ugly and dangerous
requested_state = options.get('state', buildout['slap-connection'].get('requested','started'))
+ options['requested-state'] = requested_state
+
slap = slapmodule.slap()
slap.initializeConnection(
options['server-url'],
@@ -134,6 +137,7 @@ class Recipe(object):
self._raise_request_exception = None
self._raise_request_exception_formatted = None
self.instance = None
+
# Try to do the request and fetch parameter dict...
try:
self.instance = request(software_url, software_type,
@@ -141,14 +145,17 @@ class Recipe(object):
filter_kw=filter_kw, shared=slave, state=requested_state)
return_parameter_dict = self._getReturnParameterDict(self.instance,
return_parameters)
+ # Fetch the instance-guid and the instance-state
+ # Note: SlapOS Master does not support it for slave instances
if not slave:
try:
options['instance-guid'] = self.instance.getInstanceGuid()
# XXX: deprecated, to be removed
options['instance_guid'] = self.instance.getInstanceGuid()
+ options['instance-state'] = self.instance.getState()
except (slapmodule.ResourceNotReady, AttributeError):
# Backward compatibility. Old SlapOS master and core don't know this.
- self.logger.warning("Impossible to fetch instance GUID.")
+ self.logger.warning("Impossible to fetch instance GUID nor state.")
except (slapmodule.NotFoundError, slapmodule.ServerError, slapmodule.ResourceNotReady) as exc:
self._raise_request_exception = exc
self._raise_request_exception_formatted = traceback.format_exc()
@@ -162,13 +169,6 @@ class Recipe(object):
except KeyError:
if self.failed is None:
self.failed = param
- options['requested-state'] = requested_state
- try:
- options['instance-state'] = self.instance.getState()
- except slapmodule.ResourceNotReady:
- # Odd case: SlapOS Master doesn't send the state of a slave partition.
- # XXX Should be fixed in the SlapOS Master, we should not care here.
- pass
def _filterForStorage(self, partition_parameter_kw):
return partition_parameter_kw
diff --git a/slapos/recipe/slaprunner/backup.py b/slapos/recipe/slaprunner/backup.py
index bec39569c13ef7223b25931373c88dfbb269c139..887761596e67a163860069efbc4274cc7b0f165e 100644
--- a/slapos/recipe/slaprunner/backup.py
+++ b/slapos/recipe/slaprunner/backup.py
@@ -124,8 +124,8 @@ class ImportRecipe(GenericBaseRecipe):
ifs=$IFS IFS=';'
read user pass remaining < %(etc-directory)s/.users
IFS=$ifs
- %(curl-binary)s -vg6L -F clogin="$user" -F cpwd="$pass" --dump-header login_cookie %(backend-url)s/doLogin;
- %(curl-binary)s -vg6LX POST --cookie login_cookie --max-time 5 %(backend-url)s/runSoftwareProfile;
+ %(curl-binary)s --insecure -vg6L -F clogin="$user" -F cpwd="$pass" --dump-header login_cookie %(backend-url)s/doLogin;
+ %(curl-binary)s --insecure -vg6LX POST --cookie login_cookie --max-time 5 %(backend-url)s/runSoftwareProfile;
rm -f login_cookie
""" % self.options)
self.createExecutable(wrapper, content=content)
diff --git a/slapos/recipe/sshkeys_authority.py b/slapos/recipe/sshkeys_authority.py
index 6d3951c2c614d3ce406de3ebb778f2f5de87645d..22725e214309a2a0ba4100db67c8b1eb6b22a891 100644
--- a/slapos/recipe/sshkeys_authority.py
+++ b/slapos/recipe/sshkeys_authority.py
@@ -113,11 +113,11 @@ class Request(GenericBaseRecipe):
hashlib.sha256(options['name']).hexdigest())
self.public_key = self.private_key + '.pub'
+ options['public-key-value'] = ''
if os.path.exists(self.public_key):
- with open(self.public_key) as key:
- options['public-key-value'] = key.read()
- else:
- options['public-key-value'] = ''
+ key_content = open(self.public_key).read()
+ if key_content:
+ options['public-key-value'] = key_content
def install(self):
requests_directory = self.options['request-directory']
diff --git a/software/kvm/README.txt b/software/kvm/README.txt
index 74c7e0f39ebfeb7775be805ae318a989088ef489..ddfa6711554a15d3cbc490ec73732a9e52af5f82 100644
--- a/software/kvm/README.txt
+++ b/software/kvm/README.txt
@@ -16,7 +16,7 @@ The following examples list how to request different possible instances of KVM
Software Release from slap console or command line.
KVM instance (1GB of RAM, 10GB of SSD, one core)
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
Note that the KVM instance will try to request a frontend slave instance in order
to be accessible from IPv4.
@@ -32,22 +32,51 @@ to be accessible from IPv4.
See the instance-kvm-input-schema.json file for more instance parameters (cpu-count, ram-size, disk-size, etc).
-NBD instance
+
+KVM instance (1GB of RAM, 10GB of SSD, one core)
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-This type of instance will allow to host a disk image that will be used by
-any KVM instance.
+Note that the KVM instance will try to request a frontend slave instance in order
+to be accessible from IPv4.
::
- mynbd = request(
+ myawesomekvm = request(
software_release=kvm,
- partition_reference="mynbd",
- software_type="nbd",
+ partition_reference="My awesome KVM",
+ partition_parameter_kw={
+ "nbd-host":"ubuntu-1204.nbd.vifib.net",
+ }
)
+See the instance-kvm-input-schema.json file for more instance parameters (cpu-count, ram-size, disk-size, etc).
+
+Resilient KVM instance
+~~~~~~~~~~~~~~~~~~~~~
+
+Like KVM instance, but backed-up (with history) in two places.
+
+::
+ kvm = 'http://git.erp5.org/gitweb/slapos.git/blob_plain/refs/tags/slapos-0.188:/software/kvm/software.cfg'
+ myresilientkvm = request(
+ software_release=kvm,
+ partition_reference="My resilient KVM",
+ software_type="kvm-resilient",
+ partition_parameter_kw={
+ "-sla-0-computer_guid": "COMP-1000", # Location of the main instance (KVM)
+ "-sla-1-computer_guid": "COMP-1001", # Location of the first clone
+ "-sla-2-computer_guid": "COMP-1002", # Location of the second clone
+ }
+ )
+
+See the instance-kvm-input-schema.json AND instance-kvm-resilient-input-schema.json AND /stack/resilient/README.txt
+files for more instance parameters (cpu-count, ram-size, disk-size, specific location of clones, etc).
+
+Then, if you want one of the two clones to takeover, you need to login into
+the hosting machine, go to the partition of the clone, and invoke bin/takeover.
+
KVM Frontend Master Instance (will host all frontend Slave Instances)
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
This type of instance will allow to host any frontend slave instance requested
by KVM instances. Slave instances (and thus KVM instance) will be accessible
diff --git a/software/kvm/common.cfg b/software/kvm/common.cfg
index e49117b400c4988d08f7946c751e56e233f1c50c..a433527a871c640fddbdf2cc0aaafbf65e630143 100644
--- a/software/kvm/common.cfg
+++ b/software/kvm/common.cfg
@@ -94,15 +94,17 @@ mode = 0644
recipe = hexagonit.recipe.download
url = ${:_profile_base_location_}/instance-kvm-resilient.cfg.jinja2
mode = 644
-md5sum = 6753004b582c0470bd028253ce1964ad
+#md5sum = 6753004b582c0470bd028253ce1964ad
download-only = true
+on-update = true
[template-kvm-resilient-test]
recipe = hexagonit.recipe.download
url = ${:_profile_base_location_}/instance-kvm-resilient-test.cfg.jinja2
-md5sum = 027d68d9decbc6aec59365fa723975d7
+#md5sum = 027d68d9decbc6aec59365fa723975d7
mode = 0644
download-only = true
+on-update = true
[template-kvm-import]
recipe = slapos.recipe.template
@@ -115,7 +117,7 @@ mode = 0644
recipe = hexagonit.recipe.download
url = ${:_profile_base_location_}/template/kvm-import.sh.in
filename = kvm-import.sh.in
-md5sum = a731372420dc59c0b5ba7bc5f39a14ad
+md5sum = e03ed049cddd8d157228b09e1ebc071a
download-only = true
mode = 0755
diff --git a/software/kvm/instance-kvm-input-schema.json b/software/kvm/instance-kvm-input-schema.json
index 98b6ef6b686f946c8e814882e1ed149c2153cbb5..468b4db4ea41929801432bf06e43c979140e2976 100644
--- a/software/kvm/instance-kvm-input-schema.json
+++ b/software/kvm/instance-kvm-input-schema.json
@@ -78,7 +78,6 @@
"description": "MD5 checksum of virtual hard drive, used if virtual-hard-drive-url is specified.",
"type": "string",
},
-virtual-hard-drive-md5sum
"use-tap": {
"title": "Use QEMU TAP network interface",
diff --git a/software/kvm/instance-kvm-resilient-input-schema.json b/software/kvm/instance-kvm-resilient-input-schema.json
new file mode 100644
index 0000000000000000000000000000000000000000..b9e44ebe1c54de66db1230103c7a438c96f261d5
--- /dev/null
+++ b/software/kvm/instance-kvm-resilient-input-schema.json
@@ -0,0 +1,33 @@
+{
+ "type": "object",
+ "$schema": "http://json-schema.org/draft-04/schema",
+ "items": {
+ "allOf": [
+ {
+ "$ref": "instance-kvm-input-schema.json"
+ }
+ ],
+ "title": "Input Parameters",
+ "properties": {
+ "-sla-0-computer_guid": {
+ "title": "Target computer for main instance",
+ "description": "Target computer GUID for main instance.",
+ "type": "string"
+ },
+ "-sla-1-computer_guid": {
+ "title": "Target computer for first clone",
+ "description": "Target computer for first clone and PBS.",
+ "type": "string"
+ },
+ "-sla-2-computer_guid": {
+ "title": "Target computer for second clone",
+ "description": "Target computer for second clone and PBS.",
+ "type": "string"
+ },
+ "resiliency-backup-periodicity": {
+ "title": "Periodicity of backup",
+ "description": "Periodicity of backup, in cron format.",
+ "type": "string"
+ }
+ }
+}
diff --git a/software/kvm/instance-kvm-resilient-test.cfg.jinja2 b/software/kvm/instance-kvm-resilient-test.cfg.jinja2
index e5324b938287181c8f2887f2932b238b9e5c4c65..002b0aef0a2fc8e0297d4074479da0c89720e9cf 100644
--- a/software/kvm/instance-kvm-resilient-test.cfg.jinja2
+++ b/software/kvm/instance-kvm-resilient-test.cfg.jinja2
@@ -42,15 +42,19 @@ recipe = slapos.cookbook:request
software-url = ${slap-connection:software-release-url}
software-type = kvm-resilient
name = Resilient KVM (Root Instance)
-config = virtual-hard-drive-url virtual-hard-drive-md5sum resiliency-backup-periodicity
+{% set cluster_parameter_dict = slapparameter_dict.get('cluster', {}) -%}
+config = virtual-hard-drive-url virtual-hard-drive-md5sum resiliency-backup-periodicity {{ cluster_parameter_dict.keys() | join(' ') }}
+{% for key, value in cluster_parameter_dict.items() -%}
+config-{{ key }} = {{ dumps(value) }}
+{% endfor -%}
config-virtual-hard-drive-url = ${slap-parameter:virtual-hard-drive-url}
config-virtual-hard-drive-md5sum = ${slap-parameter:virtual-hard-drive-md5sum}
config-resiliency-backup-periodicity = */5
# We don't use url parameter, but we want it to be there to make sure root instance is ready.
return = url
# XXX What to do?
-#sla = instance_guid
-#sla-instance_guid = ${slap-parameter:frontend-instance-guid}
+sla = computer_guid
+sla-computer_guid = ${slap-connection:computer-id}
[slap-parameter]
virtual-hard-drive-url = https://softinst43236.host.vifib.net/data/public/8e2138.php?dl=true
diff --git a/software/kvm/instance-kvm-resilient.cfg.jinja2 b/software/kvm/instance-kvm-resilient.cfg.jinja2
index 54fcbc573034c7439f4f1502b73e3cb0bda50086..b100bc6a7e3726f4116f282cc38160e554e0c64b 100644
--- a/software/kvm/instance-kvm-resilient.cfg.jinja2
+++ b/software/kvm/instance-kvm-resilient.cfg.jinja2
@@ -13,7 +13,7 @@ parts +=
{{ parts.replicate("kvm", "3") }}
publish-connection-informations
-{{ replicated.replicate("kvm", "3", "kvm-export", "kvm-import") }}
+{{ replicated.replicate("kvm", "3", "kvm-export", "kvm-import", slapparameter_dict=slapparameter_dict) }}
# Bubble down the parameters of the requested instance to the user
[request-kvm]
diff --git a/software/kvm/instance.cfg.in b/software/kvm/instance.cfg.in
index f2e825b49db88f8c43b5f7b86519d0076cfded16..79e5c4325a7d88887e2e27b559d569e4544fbfe9 100644
--- a/software/kvm/instance.cfg.in
+++ b/software/kvm/instance.cfg.in
@@ -34,6 +34,7 @@ cert = $${slap-connection:cert-file}
recipe = slapos.recipe.template:jinja2
template = ${template-kvm-resilient:location}/instance-kvm-resilient.cfg.jinja2
rendered = $${buildout:directory}/template-kvm-resilient.cfg
+extensions = jinja2.ext.do
context =
key develop_eggs_directory buildout:develop-eggs-directory
key eggs_directory buildout:eggs-directory
diff --git a/software/kvm/software.cfg b/software/kvm/software.cfg
index dfe568239efe035ccced5a11add061f1e86d3ae9..b60f14b8d83cd7e9c9eb284660ddc811dc4861bb 100644
--- a/software/kvm/software.cfg
+++ b/software/kvm/software.cfg
@@ -4,8 +4,13 @@ extends = common.cfg
[networkcache]
# signature certificates of the following uploaders.
# Romain Courteaud
+# Sebastien Robin
+# Kazuhiko Shiozaki
# Cedric de Saint Martin
-# Test Agent
+# Yingjie Xu
+# Gabriel Monnerat
+# Test Agent (Automatic update from tests)
+# Aurélien Calonne
signature-certificate-list =
-----BEGIN CERTIFICATE-----
MIIB4DCCAUkCADANBgkqhkiG9w0BAQsFADA5MQswCQYDVQQGEwJGUjEZMBcGA1UE
@@ -21,6 +26,32 @@ signature-certificate-list =
QUUGLQ==
-----END CERTIFICATE-----
-----BEGIN CERTIFICATE-----
+ MIIB8jCCAVugAwIBAgIJAPu2zchZ2BxoMA0GCSqGSIb3DQEBBQUAMBIxEDAOBgNV
+ BAMMB3RzeGRldjMwHhcNMTExMDE0MTIxNjIzWhcNMTIxMDEzMTIxNjIzWjASMRAw
+ DgYDVQQDDAd0c3hkZXYzMIGfMA0GCSqGSIb3DQEBAQUAA4GNADCBiQKBgQCrPbh+
+ YGmo6mWmhVb1vTqX0BbeU0jCTB8TK3i6ep3tzSw2rkUGSx3niXn9LNTFNcIn3MZN
+ XHqbb4AS2Zxyk/2tr3939qqOrS4YRCtXBwTCuFY6r+a7pZsjiTNddPsEhuj4lEnR
+ L8Ax5mmzoi9nE+hiPSwqjRwWRU1+182rzXmN4QIDAQABo1AwTjAdBgNVHQ4EFgQU
+ /4XXREzqBbBNJvX5gU8tLWxZaeQwHwYDVR0jBBgwFoAU/4XXREzqBbBNJvX5gU8t
+ LWxZaeQwDAYDVR0TBAUwAwEB/zANBgkqhkiG9w0BAQUFAAOBgQA07q/rKoE7fAda
+ FED57/SR00OvY9wLlFEF2QJ5OLu+O33YUXDDbGpfUSF9R8l0g9dix1JbWK9nQ6Yd
+ R/KCo6D0sw0ZgeQv1aUXbl/xJ9k4jlTxmWbPeiiPZEqU1W9wN5lkGuLxV4CEGTKU
+ hJA/yXa1wbwIPGvX3tVKdOEWPRXZLg==
+ -----END CERTIFICATE-----
+ -----BEGIN CERTIFICATE-----
+ MIIB7jCCAVegAwIBAgIJAJWA0jQ4o9DGMA0GCSqGSIb3DQEBBQUAMA8xDTALBgNV
+ BAMMBHg2MXMwIBcNMTExMTI0MTAyNDQzWhgPMjExMTEwMzExMDI0NDNaMA8xDTAL
+ BgNVBAMMBHg2MXMwgZ8wDQYJKoZIhvcNAQEBBQADgY0AMIGJAoGBANdJNiFsRlkH
+ vq2kHP2zdxEyzPAWZH3CQ3Myb3F8hERXTIFSUqntPXDKXDb7Y/laqjMXdj+vptKk
+ 3Q36J+8VnJbSwjGwmEG6tym9qMSGIPPNw1JXY1R29eF3o4aj21o7DHAkhuNc5Tso
+ 67fUSKgvyVnyH4G6ShQUAtghPaAwS0KvAgMBAAGjUDBOMB0GA1UdDgQWBBSjxFUE
+ RfnTvABRLAa34Ytkhz5vPzAfBgNVHSMEGDAWgBSjxFUERfnTvABRLAa34Ytkhz5v
+ PzAMBgNVHRMEBTADAQH/MA0GCSqGSIb3DQEBBQUAA4GBAFLDS7zNhlrQYSQO5KIj
+ z2RJe3fj4rLPklo3TmP5KLvendG+LErE2cbKPqnhQ2oVoj6u9tWVwo/g03PMrrnL
+ KrDm39slYD/1KoE5kB4l/p6KVOdeJ4I6xcgu9rnkqqHzDwI4v7e8/D3WZbpiFUsY
+ vaZhjNYKWQf79l6zXfOvphzJ
+ -----END CERTIFICATE-----
+ -----BEGIN CERTIFICATE-----
MIIB9jCCAV+gAwIBAgIJAO4V/jiMoICoMA0GCSqGSIb3DQEBBQUAMBMxETAPBgNV
BAMMCENPTVAtMjMyMCAXDTEyMDIxNjExMTAyM1oYDzIxMTIwMTIzMTExMDIzWjAT
MREwDwYDVQQDDAhDT01QLTIzMjCBnzANBgkqhkiG9w0BAQEFAAOBjQAwgYkCgYEA
@@ -34,6 +65,32 @@ signature-certificate-list =
If1a2ZoqHRxoNo2yTmm7TSYRORWVS+vvfjY=
-----END CERTIFICATE-----
-----BEGIN CERTIFICATE-----
+ MIIB9jCCAV+gAwIBAgIJAIlBksrZVkK8MA0GCSqGSIb3DQEBBQUAMBMxETAPBgNV
+ BAMMCENPTVAtMzU3MCAXDTEyMDEyNjEwNTUyOFoYDzIxMTIwMTAyMTA1NTI4WjAT
+ MREwDwYDVQQDDAhDT01QLTM1NzCBnzANBgkqhkiG9w0BAQEFAAOBjQAwgYkCgYEA
+ ts+iGUwi44vtIfwXR8DCnLtHV4ydl0YTK2joJflj0/Ws7mz5BYkxIU4fea/6+VF3
+ i11nwBgYgxQyjNztgc9u9O71k1W5tU95yO7U7bFdYd5uxYA9/22fjObaTQoC4Nc9
+ mTu6r/VHyJ1yRsunBZXvnk/XaKp7gGE9vNEyJvPn2bkCAwEAAaNQME4wHQYDVR0O
+ BBYEFKuGIYu8+6aEkTVg62BRYaD11PILMB8GA1UdIwQYMBaAFKuGIYu8+6aEkTVg
+ 62BRYaD11PILMAwGA1UdEwQFMAMBAf8wDQYJKoZIhvcNAQEFBQADgYEAMoTRpBxK
+ YLEZJbofF7gSrRIcrlUJYXfTfw1QUBOKkGFFDsiJpEg4y5pUk1s5Jq9K3SDzNq/W
+ it1oYjOhuGg3al8OOeKFrU6nvNTF1BAvJCl0tr3POai5yXyN5jlK/zPfypmQYxE+
+ TaqQSGBJPVXYt6lrq/PRD9ciZgKLOwEqK8w=
+ -----END CERTIFICATE-----
+ -----BEGIN CERTIFICATE-----
+ MIIB9jCCAV+gAwIBAgIJAPHoWu90gbsgMA0GCSqGSIb3DQEBBQUAMBQxEjAQBgNV
+ BAMMCXZpZmlibm9kZTAeFw0xMjAzMTkyMzIwNTVaFw0xMzAzMTkyMzIwNTVaMBQx
+ EjAQBgNVBAMMCXZpZmlibm9kZTCBnzANBgkqhkiG9w0BAQEFAAOBjQAwgYkCgYEA
+ ozBijpO8PS5RTeKTzA90vi9ezvv4vVjNaguqT4UwP9+O1+i6yq1Y2W5zZxw/Klbn
+ oudyNzie3/wqs9VfPmcyU9ajFzBv/Tobm3obmOqBN0GSYs5fyGw+O9G3//6ZEhf0
+ NinwdKmrRX+d0P5bHewadZWIvlmOupcnVJmkks852BECAwEAAaNQME4wHQYDVR0O
+ BBYEFF9EtgfZZs8L2ZxBJxSiY6eTsTEwMB8GA1UdIwQYMBaAFF9EtgfZZs8L2ZxB
+ JxSiY6eTsTEwMAwGA1UdEwQFMAMBAf8wDQYJKoZIhvcNAQEFBQADgYEAc43YTfc6
+ baSemaMAc/jz8LNLhRE5dLfLOcRSoHda8y0lOrfe4lHT6yP5l8uyWAzLW+g6s3DA
+ Yme/bhX0g51BmI6gjKJo5DoPtiXk/Y9lxwD3p7PWi+RhN+AZQ5rpo8UfwnnN059n
+ yDuimQfvJjBFMVrdn9iP6SfMjxKaGk6gVmI=
+ -----END CERTIFICATE-----
+ -----BEGIN CERTIFICATE-----
MIIB9jCCAV+gAwIBAgIJAKRvzcy7OH0UMA0GCSqGSIb3DQEBBQUAMBMxETAPBgNV
BAMMCENPTVAtNzcyMCAXDTEyMDgxMDE1NDI1MVoYDzIxMTIwNzE3MTU0MjUxWjAT
MREwDwYDVQQDDAhDT01QLTc3MjCBnzANBgkqhkiG9w0BAQEFAAOBjQAwgYkCgYEA
@@ -46,55 +103,73 @@ signature-certificate-list =
5pW18Ry5Ie7iFK4cQMerZwWPxBodEbAteYlRsI6kePV7Gf735Y1RpuN8qZ2sYL6e
x2IMeSwJ82BpdEI5niXxB+iT0HxhmR+XaMI=
-----END CERTIFICATE-----
+ -----BEGIN CERTIFICATE-----
+ MIIB+DCCAWGgAwIBAgIJAKGd0vpks6T/MA0GCSqGSIb3DQEBBQUAMBQxEjAQBgNV
+ BAMMCUNPTVAtMTU4NDAgFw0xMzA2MjAxMjE5MjBaGA8yMTEzMDUyNzEyMTkyMFow
+ FDESMBAGA1UEAwwJQ09NUC0xNTg0MIGfMA0GCSqGSIb3DQEBAQUAA4GNADCBiQKB
+ gQDZTH9etPUC+wMZQ3UIiOwyyCfHsJ+7duCFYjuo1uZrhtDt/fp8qb8qK9ob+df3
+ EEYgA0IgI2j/9jNUEnKbc5+OrfKznzXjrlrH7zU8lKBVNCLzQuqBKRNajZ+UvO8R
+ nlqK2jZCXP/p3HXDYUTEwIR5W3tVCEn/Vda4upTLcPVE5wIDAQABo1AwTjAdBgNV
+ HQ4EFgQU7KXaNDheQWoy5uOU01tn1M5vNkEwHwYDVR0jBBgwFoAU7KXaNDheQWoy
+ 5uOU01tn1M5vNkEwDAYDVR0TBAUwAwEB/zANBgkqhkiG9w0BAQUFAAOBgQASmqCU
+ Znbvu6izdicvjuE3aKnBa7G++Fdp2bdne5VCwVbVLYCQWatB+n4crKqGdnVply/u
+ +uZ16u1DbO9rYoKgWqjLk1GfiLw5v86pd5+wZd5I9QJ0/Sbz2vZk5S4ciMIGwArc
+ m711+GzlW5xe6GyH9SZaGOPAdUbI6JTDwLzEgA==
+ -----END CERTIFICATE-----
[versions]
-Werkzeug = 0.9.3
-apache-libcloud = 0.13.0
+Werkzeug = 0.9.4
+apache-libcloud = 0.13.2
async = 0.6.1
buildout-versions = 1.7
+erp5.util = 0.4.36
gitdb = 0.5.4
-itsdangerous = 0.22
+itsdangerous = 0.23
lxml = 3.2.3
meld3 = 0.6.10
plone.recipe.command = 1.1
+psutil = 1.1.0
pycrypto = 2.6
rdiff-backup = 1.0.5
-slapos.cookbook = 0.79
+slapos.cookbook = 0.84.1
slapos.recipe.cmmi = 0.2
slapos.recipe.download = 1.0.dev-r4053
-slapos.recipe.template = 2.4.2
-slapos.toolbox = 0.35.0
+slapos.toolbox = 0.37.1
smmap = 0.8.2
websockify = 0.5.1
z3c.recipe.scripts = 1.0.1
# Required by:
# slapos.core==0.35.1
-# slapos.toolbox==0.35.0
+# slapos.toolbox==0.37.1
Flask = 0.10.1
# Required by:
-# slapos.toolbox==0.35.0
+# slapos.toolbox==0.37.1
GitPython = 0.3.2.RC1
# Required by:
-# slapos.toolbox==0.35.0
+# slapos.toolbox==0.37.1
atomize = 0.1.1
# Required by:
-# slapos.toolbox==0.35.0
+# paramiko==1.12.0
+ecdsa = 0.9
+
+# Required by:
+# slapos.toolbox==0.37.1
feedparser = 5.1.3
# Required by:
-# slapos.cookbook==0.79
+# slapos.cookbook==0.84.1
inotifyx = 0.2.0-1
# Required by:
-# slapos.cookbook==0.79
+# slapos.cookbook==0.84.1
lock-file = 2.0
# Required by:
-# slapos.cookbook==0.79
+# slapos.cookbook==0.84.1
netaddr = 0.7.10
# Required by:
@@ -105,40 +180,38 @@ netifaces = 0.8-1
# websockify==0.5.1
numpy = 1.7.1
-# Required by:
-# slapos.toolbox==0.35.0
-paramiko = 1.11.0
# Required by:
-# slapos.toolbox==0.35.0
-psutil = 1.0.1
+# slapos.toolbox==0.37.1
+paramiko = 1.12.0
# Required by:
# slapos.core==0.35.1
pyflakes = 0.7.3
# Required by:
-# slapos.cookbook==0.79
-pytz = 2013b
+# slapos.cookbook==0.84.1
+pytz = 2013.7
# Required by:
-# slapos.cookbook==0.79
-# slapos.toolbox==0.35.0
+# slapos.cookbook==0.84.1
+# slapos.toolbox==0.37.1
slapos.core = 0.35.1
# Required by:
# slapos.core==0.35.1
-supervisor = 3.0b2
+supervisor = 3.0
# Required by:
# slapos.core==0.35.1
unittest2 = 0.5.1
# Required by:
-# slapos.cookbook==0.79
-# slapos.toolbox==0.35.0
+# slapos.cookbook==0.84.1
+# slapos.toolbox==0.37.1
xml-marshaller = 0.9.7
# Required by:
# slapos.core==0.35.1
zope.interface = 4.0.5
+
diff --git a/software/kvm/template/kvm-import.sh.in b/software/kvm/template/kvm-import.sh.in
index 66737bf3317b8f2a1bb937d4032afc187615e461..9c1a9b09681a547ccc2d7eef54711b126f9ac192 100644
--- a/software/kvm/template/kvm-import.sh.in
+++ b/software/kvm/template/kvm-import.sh.in
@@ -3,5 +3,5 @@ DISK_PATH=${:disk-path}
BACKUP_PATH=${:backup-disk-path}
# TODO: Use rdiff
-rm $DISK_PATH && \
+rm $DISK_PATH
cp $BACKUP_PATH $DISK_PATH
diff --git a/software/slaprunner/common.cfg b/software/slaprunner/common.cfg
index f8a91f8ae4c663f7af5aff93680ba3cbc11fd087..b93650fac56c32a6f4b26588b335c4dabfc78fc0 100644
--- a/software/slaprunner/common.cfg
+++ b/software/slaprunner/common.cfg
@@ -94,7 +94,7 @@ mode = 0644
recipe = slapos.recipe.template
url = ${:_profile_base_location_}/instance-runner-export.cfg.in
output = ${buildout:directory}/instance-runner-export.cfg
-md5sum = 7e71622c09271790b5cef21c8613b8ac
+md5sum = 9ce3482e64a0c957f7a5f25ad0dc03ae
mode = 0644
[template-resilient]
@@ -107,7 +107,7 @@ mode = 0644
[template-resilient-test]
recipe = slapos.recipe.download
url = ${:_profile_base_location_}/instance-resilient-test.cfg.jinja2
-#md5sum = 0ee2cea5239278a8c1572d7a04798fdc
+md5sum = ac772d3a1cce4072acfabd563df449bb
filename = instance-resilient-test.cfg.jinja2
mode = 0644
diff --git a/software/slaprunner/development.cfg b/software/slaprunner/development.cfg
index 7d02bd51e2ad282db3109be425200b6653a25e76..00d313714ffcf4a35dd46fda2af29a42d4073296 100644
--- a/software/slaprunner/development.cfg
+++ b/software/slaprunner/development.cfg
@@ -39,8 +39,7 @@ git-executable = ${git:location}/bin/git
[erp5.util-repository]
recipe = slapos.recipe.build:gitclone
repository = http://git.erp5.org/repos/erp5.git
-#branch = scalability-master2
-revision = f10da882ab5e1dc03a812f3d0e7390dc8da2b59
+branch = scalability-master2
git-executable = ${git:location}/bin/git
#[slapos.core-repository]
diff --git a/software/slaprunner/instance-resilient-test.cfg.jinja2 b/software/slaprunner/instance-resilient-test.cfg.jinja2
index f30c7aa4423269e33a509ca37d07dea250d995f5..65ec46bfe334f8691d580ff3c9b01c172cd33c3b 100644
--- a/software/slaprunner/instance-resilient-test.cfg.jinja2
+++ b/software/slaprunner/instance-resilient-test.cfg.jinja2
@@ -29,7 +29,7 @@ command-line = {{ bin_directory }}/runResiliencyTest ${:testnode-parameters} ${:
wrapper-path = ${directory:scripts}/runResiliencyTestSuite
[deploy-standalone-resiliency-test]
-# Used to manually run the KVM test if we don't have a running testnode.
+# Used to manually run the resilient test if we don't have a running testnode.
recipe = slapos.cookbook:wrapper
test-suite-title = slaprunner
command-line = {{ bin_directory }}/runStandaloneResiliencyTest --test-suite-title=${:test-suite-title} ${deploy-resiliency-test:test-parameters}
@@ -41,7 +41,11 @@ recipe = slapos.cookbook:request
software-url = ${slap-connection:software-release-url}
software-type = resilient
name = Resilient Instance (Root Instance)
-config = resiliency-backup-periodicity frontend-domain cloud9-frontend-domain
+{% set cluster_parameter_dict = slapparameter_dict.get('cluster', {}) -%}
+config = resiliency-backup-periodicity frontend-domain cloud9-frontend-domain {{ cluster_parameter_dict.keys() | join(' ') }}
+{% for key, value in cluster_parameter_dict.items() -%}
+config-{{ key }} = {{ dumps(value) }}
+{% endfor -%}
config-resiliency-backup-periodicity = *
# XXX hardcoded
config-frontend-domain = google.com
diff --git a/software/slaprunner/instance-runner-export.cfg.in b/software/slaprunner/instance-runner-export.cfg.in
index 8fe0f83112ca9ce8147fbba3cbdb639e5d7ca527..a98c73486d74f132a5d80f77aeee28f8d106762c 100644
--- a/software/slaprunner/instance-runner-export.cfg.in
+++ b/software/slaprunner/instance-runner-export.cfg.in
@@ -22,7 +22,7 @@ parts +=
symlinks
node-frontend-promise
nginx-promise
- urls
+ publish-connection-informations
cron-entry-backup
[exporter]
@@ -36,11 +36,5 @@ rsync-binary = ${rsync:location}/bin/rsync
# Extends publish section with resilient parameters
-[urls]
+[publish-connection-informations]
<= resilient-publish-connection-parameter
-backend_url = $${slaprunner:access-url}
-url = https://$${request-frontend:connection-domain}
-cloud9_backend_url = $${node-frontend:access-url}
-cloud9_url = https://$${request-cloud9-frontend:connection-domain}
-ssh_command = ssh $${dropbear-runner-server:host} -p $${dropbear-runner-server:port}
-password_recovery_code = $${recovery-code:passwd}
diff --git a/stack/resilient/TODO.txt b/stack/resilient/TODO.txt
index 8797b94cdea7f917c94c5e4c491be125d825ed28..415d0e2e4ee6eb6dceacbae89f40d874654ba6b3 100644
--- a/stack/resilient/TODO.txt
+++ b/stack/resilient/TODO.txt
@@ -1,16 +1,17 @@
+* Report, from pbs and from clone, when a backup failed
+* Make sure, when a takeover is done, that "importer" script finishes to run while importer instance is changed into exporter.
+* Test that, after a successful backup/takeover, another backup is possible and will be successful.
+
* PBSs and mirrors should monitor/replace themselves
* Report errors from backup
* If a PBS master is down and then back again, it might want to participate in the ongoing election, then.. what happens?
* If the network is partitioned (the two backups don't see each other, but each can see the slapos master) there will be two concurrent elections taking place, with two winners and two renames.
-* How to know that a backup is working? define "check that it works". Does it deploys? But then, how to ensure data integrity? By application?
* How to ensure "synchronization" between two main instances? example: Wordpress: mysql is down, then replaced, then inconsistency between apache and the new mysql
* How to deal with big data? I.e how to have working backup/restore system of 1TB data with slow connection?
* How to be sure that elected importer contains a/ the latest data and b/ has finished to pull. We should prevent importer not having a/ and b/ to become the main.
-* How to say "I don't want this instance to be here" + Be able to define "here". Allows to automate deployment of PBS and backup instances
-
* Should we crypt backed up data?
* If a PBS is lost, a new PBS should be created from another one, in order ot keep history
diff --git a/stack/resilient/buildout.cfg b/stack/resilient/buildout.cfg
index bbaa549d20b8de2cd89d431f97ee1b4fff486543..8bbb7d45ed46d1f3964d8b4ab1c69888dc117f86 100644
--- a/stack/resilient/buildout.cfg
+++ b/stack/resilient/buildout.cfg
@@ -30,7 +30,7 @@ parts =
recipe = slapos.recipe.template
url = ${:_profile_base_location_}/pbsready.cfg.in
output = ${buildout:directory}/pbsready.cfg
-md5sum = 9f4212a79f10bee8f6d75061943110e2
+md5sum = 570e0b54c97d510befa2ea981c1e90e0
mode = 0644
[pbsready-import]
@@ -39,7 +39,7 @@ mode = 0644
recipe = slapos.recipe.template
url = ${:_profile_base_location_}/pbsready-import.cfg.in
output = ${buildout:directory}/pbsready-import.cfg
-md5sum = 3c2e73f49abdc52282fc045e6d91f3e9
+md5sum = cc9c776500ccd07cb51969beb68ffcda
mode = 0644
[pbsready-export]
@@ -48,20 +48,20 @@ mode = 0644
recipe = slapos.recipe.template
url = ${:_profile_base_location_}/pbsready-export.cfg.in
output = ${buildout:directory}/pbsready-export.cfg
-md5sum = 5e27c391ceafb6a58032f1f87fba7826
+md5sum = 25d05b3929fb4c6cf275866bad678d6a
mode = 0644
[template-pull-backup]
recipe = slapos.recipe.template
url = ${:_profile_base_location_}/instance-pull-backup.cfg.in
output = ${buildout:directory}/instance-pull-backup.cfg
-md5sum = c67a9dad66490ae264f9e7003521bf59
+md5sum = c49e5911b94078d87b94507fb4efc93b
mode = 0644
[template-replicated]
recipe = slapos.recipe.download
url = ${:_profile_base_location_}/template-replicated.cfg.in
-#md5sum = 9e20f283bf709c63c9c6692d5e1f8972
+md5sum = c762a625f65193bc8a570b4d56a0d08c
mode = 0644
destination = ${buildout:directory}/template-replicated.cfg.in
diff --git a/stack/resilient/instance-pull-backup.cfg.in b/stack/resilient/instance-pull-backup.cfg.in
index 574290448057220a1a82d4e264973e34f08f5ecc..5cca1f4d30b6277bc286ed7621b6e3b127badde9 100644
--- a/stack/resilient/instance-pull-backup.cfg.in
+++ b/stack/resilient/instance-pull-backup.cfg.in
@@ -75,7 +75,7 @@ callbacks = $${directory:notifier-callbacks}
id-file = $${rootdirectory:etc}/notifier.id
equeue-socket = $${equeue:socket}
host = $${slap-network-information:global-ipv6}
-port = 8080
+port = 8088
wrapper = $${basedirectory:services}/notifier
server-binary = ${buildout:bin-directory}/pubsubserver
notifier-binary = ${buildout:bin-directory}/pubsubnotifier
@@ -105,6 +105,7 @@ promises-directory = $${basedirectory:promises}
directory = $${directory:pbs-backup}
cron-entries = $${cron:cron-entries}
wrappers-directory = $${directory:pbs-wrappers}
+run-directory = $${basedirectory:run}
# XXX: this should be named "notifier-host"
notifier-url = http://[$${notifier:host}]:$${notifier:port}
slave-instance-list = $${slap-parameter:slave_instance_list}
diff --git a/stack/resilient/pbsready-export.cfg.in b/stack/resilient/pbsready-export.cfg.in
index 712c44d2f5a492a4a2ada53bec149e03db93da32..85b9eb8aa281529b88decb6e41ba6c3452967f06 100644
--- a/stack/resilient/pbsready-export.cfg.in
+++ b/stack/resilient/pbsready-export.cfg.in
@@ -25,6 +25,12 @@ home = $${buildout:directory}
var = $${:home}/var
pid = $${:var}/pid
+# Define port of ssh server. It has to be different from import so that it
+# supports export/import using same IP (slaprunner, slapos-in-partition,
+# ipv4...)
+[dropbear-server]
+port = 22221
+
[resilient-publish-connection-parameter]
notification-id = http://[$${notifier:host}]:$${notifier:port}/get/$${notifier-exporter:name}
diff --git a/stack/resilient/pbsready-import.cfg.in b/stack/resilient/pbsready-import.cfg.in
index 419ba1bed91eab0f3620fd0fb533db542556adbf..5028c05ae82890a8530ca8c9735f18bbc38a4331 100644
--- a/stack/resilient/pbsready-import.cfg.in
+++ b/stack/resilient/pbsready-import.cfg.in
@@ -23,6 +23,12 @@ parts =
[resilient-publish-connection-parameter]
notification-url = http://[$${notifier:host}]:$${notifier:port}/notify
+# Define port of ssh server. It has to be different from import so that it
+# supports export/import using same IP (slaprunner, slapos-in-partition,
+# ipv4...)
+[dropbear-server]
+port = 22220
+
[import-on-notification]
# notifier.callback runs a script when a notification (sent by a parent PBS)
# is received
diff --git a/stack/resilient/pbsready.cfg.in b/stack/resilient/pbsready.cfg.in
index aa0c5aa5bb2965a6e8ab65526024a5ba8f9c7abb..59dc441363e8e9862f77127e408bad3bac95dfc1 100644
--- a/stack/resilient/pbsready.cfg.in
+++ b/stack/resilient/pbsready.cfg.in
@@ -213,7 +213,8 @@ wrapper = $${basedirectory:services}/sshd
[dropbear-server]
recipe = slapos.cookbook:dropbear
host = $${slap-network-information:global-ipv6}
-port = 2222
+# Explicitely excludes to define "port" argument. It will be defined in
+# pbs-ready-import.cfg.in and pbs-ready-export.cfg.in
home = $${directory:ssh}
wrapper = $${rootdirectory:bin}/raw_sshd
shell = $${rdiff-backup-server:wrapper}
diff --git a/stack/resilient/template-replicated.cfg.in b/stack/resilient/template-replicated.cfg.in
index acb934e61e19344ecb491c9e75126109c435bac1..e977505c131a08dfb6b51ff11a3304478ad4c731 100644
--- a/stack/resilient/template-replicated.cfg.in
+++ b/stack/resilient/template-replicated.cfg.in
@@ -79,7 +79,7 @@ software-url = ${slap-connection:software-release-url}
software-type = {{typeimport}}
return = ssh-public-key ssh-url notification-url ip
-pbs-notification-id = ${slap-connection:computer-id}-${slap-connection:partition-id}-{{namebase}}-push
+pbs-notification-id = ${slap-connection:computer-id}-${slap-connection:partition-id}-{{namebase}}-{{id}}-push
config = number authorized-key on-notification ip-list namebase
config-number = {{id}}
@@ -133,8 +133,6 @@ config-ip-list = ${request-{{namebase}}:connection-ip}{% for j in range(1,nbback
## Having 3 backups pulling from the same PBS provides
##only availability, not resiliency
-## WARNING : SLAVES ARE ALLOCATED AT RANDOM, THIS NEEDS TO BE FIXED.
-
[request-pbs-common]
<= slap-connection
recipe = slapos.cookbook:request
@@ -173,15 +171,16 @@ sla-{{ key }} = {{ value }}
[request-pull-backup-server-{{namebase}}-{{id}}]
<= request-pbs-common
name = PBS {{id}} pulling from ${request-{{namebase}}:name}
-config = url name type server-key on-notification notify notification-id title
+config = url name type server-key on-notification notify notification-id title remove-backup-older-than
config-url = ${request-{{namebase}}:connection-ssh-url}
-config-name = ${slap-connection:computer-id}-${slap-connection:partition-id}-{{namebase}}-{{id}}
config-type = pull
config-server-key = ${request-{{namebase}}:connection-ssh-public-key}
config-on-notification = ${request-{{namebase}}:connection-notification-id}
config-notify = ${request-pbs-{{namebase}}-{{id}}:connection-notification-url}
config-notification-id = ${slap-connection:computer-id}-${slap-connection:partition-id}-{{namebase}}-{{id}}-pull
+config-name = ${slap-connection:computer-id}-${slap-connection:partition-id}-{{namebase}}-{{id}}
config-title = Pulling from {{namebase}}
+config-remove-backup-older-than = {{ slapparameter_dict.get('remove-backup-older-than', '3B') }}
slave = true
sla = instance_guid
sla-instance_guid = ${request-pbs-{{namebase}}-{{id}}:instance_guid}
@@ -191,12 +190,12 @@ sla-instance_guid = ${request-pbs-{{namebase}}-{{id}}:instance_guid}
name = PBS pushing on ${request-{{namebase}}-pseudo-replicating-{{id}}:name}
config = url name type server-key on-notification notify notification-id title
config-url = ${request-{{namebase}}-pseudo-replicating-{{id}}:connection-ssh-url}
-config-name = ${request-pull-backup-server-{{namebase}}-{{id}}:config-name}
config-type = push
config-server-key = ${request-{{namebase}}-pseudo-replicating-{{id}}:connection-ssh-public-key}
config-on-notification = ${request-pbs-{{namebase}}-{{id}}:connection-feeds-url}${request-pull-backup-server-{{namebase}}-{{id}}:config-notification-id}
config-notify = ${request-{{namebase}}-pseudo-replicating-{{id}}:connection-notification-url}
config-notification-id = ${request-{{namebase}}-pseudo-replicating-{{id}}:pbs-notification-id}
+config-name = ${slap-connection:computer-id}-${slap-connection:partition-id}-{{namebase}}-{{id}}
config-title = Pushing to {{namebase}} backup {{id}}
slave = true
sla = instance_guid
diff --git a/stack/slapos.cfg b/stack/slapos.cfg
index 633a74d19fbdd28e8698445c170953d1a5044748..61f1321e675923c3462b96145136b7c04e0c5ac7 100644
--- a/stack/slapos.cfg
+++ b/stack/slapos.cfg
@@ -12,6 +12,7 @@ extensions +=
# Use shacache and lxml
extends =
../component/lxml-python/buildout.cfg
+ ../component/python-openssl/buildout.cfg
# Separate from site eggs
allowed-eggs-from-site-packages =
@@ -59,6 +60,7 @@ networkcache-section = networkcache
recipe = zc.recipe.egg
eggs =
${lxml-python:egg}
+ ${python-openssl:egg}
slapos.cookbook
cliff
hexagonit.recipe.download