Merge branch 'master' into openstack

CHANGES.txt

 Changes
 =======
 
+0.81 (2013-08-12)
+-----------------
+
+ * KVM SR: implement resiliency test. [Cedric de Saint Martin]
+
 0.80 (2013-08-06)
 ----------------
 

component/coreutils/buildout.cfg

@@ -19,7 +19,11 @@ environment =
 recipe = plone.recipe.command
 stop-on-error = true
 update-command = ${:command}
-command = ${:test} -x ${:test} -a -x ${:cat} -a -x ${:rm}
+command = ${:test} -x ${:test} -a -x ${:cat} -a -x ${:rm} -a -x ${:echo} -a -x ${:date} -a -x ${:md5sum} -a -x ${:basename}
 test = ${coreutils:location}/bin/test
 cat = ${coreutils:location}/bin/cat
 rm = ${coreutils:location}/bin/rm
+echo = ${coreutils:location}/bin/echo
+date = ${coreutils:location}/bin/date
+md5sum = ${coreutils:location}/bin/md5sum
+basename = ${coreutils:location}/bin/basename

component/nginx/buildout.cfg

@@ -3,12 +3,14 @@ extends =
   ../pcre/buildout.cfg
   ../zlib/buildout.cfg
   ../openssl/buildout.cfg
+  ../coreutils/buildout.cfg
+
+parts = nginx-output
 
 [nginx]
 recipe = slapos.recipe.cmmi
-#url = http://nginx.org/download/nginx-1.2.7.tar.gz
-url = http://nginx.org/download/nginx-1.4.2.tar.gz
-#md5sum = d252f5c689a14a668e241c744ccf5f06
+url = http://nginx.org/download/nginx-1.5.3.tar.gz
+md5sum = 1e735dd6a6ade2b5c20e924b67c3d355
 configure-options=
   --with-ipv6
   --with-http_ssl_module
@@ -17,6 +19,15 @@ configure-options=
   --with-ld-opt="-L ${zlib:location}/lib -L ${openssl:location}/lib -L ${pcre:location}/lib  -Wl,-rpath=${pcre:location}/lib -Wl,-rpath=${zlib:location}/lib -Wl,-rpath=${openssl:location}/lib"
   --with-cc-opt="-I ${pcre:location}/include -I ${openssl:location}/include -I ${zlib:location}/include"
 
+[nginx-output]
+# Shared binary location to ease migration
+recipe = plone.recipe.command
+stop-on-error = true
+update-command = ${:command}
+command = ${coreutils-output:test} -x ${:nginx} -a -f ${:mime}
+nginx = ${nginx:location}/sbin/nginx
+mime = ${nginx:location}/conf/mime.types
+
 [nginx-unstable]
 <= nginx
 url = http://nginx.org/download/nginx-1.3.15.tar.gz

component/openssl/buildout.cfg

@@ -8,9 +8,10 @@ extends =
   ../ca-certificates/buildout.cfg
   ../zlib/buildout.cfg
   ../patch/buildout.cfg
+  ../coreutils/buildout.cfg
 
 parts =
-  openssl
+  openssl-output
 
 [openssl-nodoc.patch]
 # Disable doc generation part in Makefile 
@@ -52,3 +53,11 @@ make-options =
   -j1
 make-targets =
   all install_sw && rm -f ${buildout:parts-directory}/${:_buildout_section_name_}/etc/ssl/certs/* && for i in ${ca-certificates:location}/certs/*/*.crt; do ln -sv $i ${buildout:parts-directory}/${:_buildout_section_name_}/etc/ssl/certs/`${buildout:parts-directory}/${:_buildout_section_name_}/bin/openssl x509 -hash -noout -in $i`.0; done; true
+
+[openssl-output]
+# Shared binary location to ease migration
+recipe = plone.recipe.command
+stop-on-error = true
+update-command = ${:command}
+command = ${coreutils-output:test} -x ${:openssl}
+openssl = ${openssl:location}/bin/openssl

setup.py

@@ -28,7 +28,7 @@ from setuptools import setup, find_packages
 import glob
 import os
 
-version = '0.81-dev'
+version = '0.82-dev'
 name = 'slapos.cookbook'
 long_description = open("README.txt").read() + "\n" + \
     open("CHANGES.txt").read() + "\n"

slapos/recipe/addresiliency/renamer.py

-# -*- coding: utf-8 -*-
-
-import logging
-import time
-
-import slapos
-
-log = logging.getLogger(__name__)
-logging.basicConfig(level=logging.DEBUG)
-
-
-
-class Renamer(object):
-    def __init__(self, server_url, key_file, cert_file, computer_guid,
-                 partition_id, software_release, namebase):
-        self.server_url = server_url
-        self.key_file = key_file
-        self.cert_file = cert_file
-        self.computer_guid = computer_guid
-        self.partition_id = partition_id
-        self.software_release = software_release
-        self.namebase = namebase
-
-
-    def _failover(self):
-        """\
-        This method does
-
-        - retrieve the broken computer partition
-        - change its reference to 'broken-...' and its software type to 'frozen'
-        - retrieve the winner computer partition (attached to this process)
-        - change its reference to replace the broken one.
-          later, slapgrid will change its software_type as well.
-
-        Then, after running slapgrid-cp a few times, the winner takes over and
-        a new cp is created to replace it as an importer.
-        """
-
-        slap = slapos.slap.slap()
-        slap.initializeConnection(self.server_url, self.key_file, self.cert_file)
-
-        # partition that will take over.
-        cp_winner = slap.registerComputerPartition(computer_guid=self.computer_guid,
-                                                   partition_id=self.partition_id)
-        # XXX although we can already rename cp_winner, to change its software type we need to
-        # get hold of the root cp as well
-
-        cp_exporter_ref = self.namebase + '0'       # this is ok. the boss is always number zero.
-
-        # partition to be deactivated
-        cp_broken = cp_winner.request(software_release=self.software_release,
-                                      software_type='frozen',
-                                      state='stopped',
-                                      partition_reference=cp_exporter_ref)
-
-        broken_new_ref = 'broken-{}'.format(time.strftime("%d-%b_%H:%M:%S", time.gmtime()))
-
-        log.debug("Renaming {}: {}".format(cp_broken.getId(), broken_new_ref))
-
-        cp_broken.rename(new_name=broken_new_ref)
-
-        log.debug("Renaming {}: {}".format(cp_winner.getId(), cp_exporter_ref))
-
-        # update name (and later, software type) for the partition that will take over
-
-        cp_winner.rename(new_name=cp_exporter_ref)
-        cp_winner.bang(message='partitions have been renamed!')
-
-
-
-    def failover(self):
-        try:
-            self._failover()
-            log.info('Renaming done')
-        except slapos.slap.ServerError:
-            log.info('Internal server error')
-

slapos/recipe/addresiliency/takeover.py

 # -*- coding: utf-8 -*-
+import logging
+import time
 
-import slapos.recipe.addresiliency.renamer
+import slapos
 
-def run(args):
-    renamer = slapos.recipe.addresiliency.renamer.Renamer(server_url = args.pop('server_url'),
-                                                          key_file = args.pop('key_file'),
-                                                          cert_file = args.pop('cert_file'),
-                                                          computer_guid = args.pop('computer_id'),
-                                                          partition_id = args.pop('partition_id'),
-                                                          software_release = args.pop('software'),
-                                                          namebase = args.pop('namebase'))
+log = logging.getLogger(__name__)
+logging.basicConfig(level=logging.DEBUG)
+
+def takeover(server_url, key_file, cert_file, computer_guid,
+             partition_id, software_release, namebase,
+             winner_instance_suffix = None):
+  """
+  This function does
+
+  - retrieve the broken computer partition
+  - change its reference to 'broken-...' and its software type to 'frozen'
+  - retrieve the winner computer partition (attached to this process)
+  - change its reference to replace the broken one.
+    later, slapgrid will change its software_type as well.
+
+  Then, after running slapgrid-cp a few times, the winner takes over and
+  a new cp is created to replace it as an importer.
+  """
+
+  slap = slapos.slap.slap()
+  slap.initializeConnection(server_url, key_file, cert_file)
+  current_partition = slap.registerComputerPartition(computer_guid=computer_guid,
+                                                     partition_id=partition_id)
+
+  # partition that will take over.
+  if winner_instance_suffix:
+    winner_instance_name = namebase + winner_instance_suffix
+    # XXX: we hardcode a lot of values here, because request is a settergetter, all at once.
+    cp_winner = current_partition.request(software_release=software_release,
+                                  software_type='%s-import' % namebase,
+                                  partition_reference=winner_instance_name)
+  else:
+    # This script is run in the winning partition: use this one as winner
+    cp_winner = current_partition
+  # XXX although we can already rename cp_winner, to change its software type we need to
+  # get hold of the root cp as well
+
+  cp_exporter_ref = namebase + '0'       # this is ok. the boss is always number zero.
 
-    renamer.failover()
+  # partition to be deactivated
+  cp_broken = cp_winner.request(software_release=software_release,
+                                software_type='frozen',
+                                state='stopped',
+                                partition_reference=cp_exporter_ref)
 
+  broken_new_ref = 'broken-{}'.format(time.strftime("%d-%b_%H:%M:%S", time.gmtime()))
+
+  log.debug("Renaming {}: {}".format(cp_broken.getId(), broken_new_ref))
+
+  cp_broken.rename(new_name=broken_new_ref)
+
+  log.debug("Renaming {}: {}".format(cp_winner.getId(), cp_exporter_ref))
+
+  # update name (and later, software type) for the partition that will take over
+
+  cp_winner.rename(new_name=cp_exporter_ref)
+  cp_winner.bang(message='partitions have been renamed!')
+  # Note: Root instance will reconfigure itself the winning instance (software_type
+  # and parameters.)
+
+def run(args):
+  slapos.recipe.addresiliency.takeover.takeover(server_url = args.pop('server_url'),
+                                                key_file = args.pop('key_file'),
+                                                cert_file = args.pop('cert_file'),
+                                                computer_guid = args.pop('computer_id'),
+                                                partition_id = args.pop('partition_id'),
+                                                software_release = args.pop('software'),
+                                                namebase = args.pop('namebase'))
 

slapos/recipe/kvm/__init__.py

@@ -48,16 +48,16 @@ class Recipe(GenericBaseRecipe):
     if not self.isTrueValue(self.options.get('use-tap')):
       # XXX This could be done using Jinja.
       for port in self.options['nat-rules'].split():
-        ipv6_port = int(port) + 10000
+        tunnel_port = int(port) + 10000
         tunnel_path = self.createExecutable(
-            '%s-%sto%s' % (self.options['6tunnel-wrapper-path'], port, ipv6_port),
+            '%s-%s' % (self.options['6tunnel-wrapper-path'], tunnel_port),
             self.substituteTemplate(
                 self.getTemplateFilename('6to4.in'),
                 {
                     'ipv6': self.options['ipv6'],
-                    'ipv6_port': ipv6_port,
+                    'ipv6_port': tunnel_port,
                     'ipv4': self.options['ipv4'],
-                    'ipv4_port': port,
+                    'ipv4_port': tunnel_port,
                     'shell_path': self.options['shell-path'],
                     '6tunnel_path': self.options['6tunnel-path'],
                 },

slapos/recipe/kvm/template/kvm_run.in

@@ -18,7 +18,7 @@ nbd_list = (('%(nbd-host)s', %(nbd-port)s), ('%(nbd2-host)s', %(nbd2-port)s))
 default_disk_image = '%(default-disk-image)s'
 disk_path = '%(disk-path)s'
 virtual_hard_drive_url = '%(virtual-hard-drive-url)s'.strip()
-virtual_hard_drive_md5_url = '%(virtual-hard-drive-md5-url)s'.strip()
+virtual_hard_drive_md5sum = '%(virtual-hard-drive-md5sum)s'.strip()
 nat_rules = '%(nat-rules)s'.strip()
 use_tap = '%(use-tap)s'
 tap_interface = '%(tap-interface)s'
@@ -59,18 +59,27 @@ def getSocketStatus(host, port):
 
 # Download existing hard drive if needed at first boot
 if not os.path.exists(disk_path) and virtual_hard_drive_url != '':
+  print('Downloading virtual hard drive...')
   urllib.urlretrieve(virtual_hard_drive_url, disk_path)
-  local_md5sum = md5Checksum(disk_path)
-  md5sum = urllib.urlopen(virtual_hard_drive_md5_url).read().strip()
-  if local_md5sum != md5sum:
-    os.remove(disk_path)
-    raise Exception('MD5 mismatch.')
+  md5sum = virtual_hard_drive_md5sum.strip()
+  if md5sum:
+    print('Checking MD5 checksum...')
+    local_md5sum = md5Checksum(disk_path)
+    if local_md5sum != md5sum:
+      os.remove(disk_path)
+      raise Exception('MD5 mismatch. MD5 of local file is %%s, Specified MD5 is %%s.' %% (
+          local_md5sum, md5sum))
+    print('MD5sum check passed.')
+  else:
+    print('Warning: not checksum specified.')
 
 # Create disk if doesn't exist
 # XXX: move to Buildout profile
 if not os.path.exists(disk_path):
+  print('Creating virtual hard drive...')
   subprocess.Popen([qemu_img_path, 'create' ,'-f', 'qcow2',
       disk_path, '%%sG' %% disk_size])
+  print('Done.')
 
 # Generate network parameters
 # XXX: use_tap should be a boolean

software/backupserver/instance-pullrdiffbackup.cfg.in

@@ -12,6 +12,7 @@ etc = $${buildout:directory}/etc
 bin = $${buildout:directory}/bin
 srv = $${buildout:directory}/srv
 var = $${buildout:directory}/var
+run = $${:var}/run
 log = $${:var}/log
 # scripts = $${:etc}/run
 services = $${:etc}/service
@@ -20,8 +21,11 @@ cron-lines = $${:etc}/cron.lines
 crontabs = $${:etc}/crontabs
 cronstamps = $${:etc}/cronstamps
 backup = $${:srv}/backup
+status = $${:srv}/status
 backupscript = $${:etc}/backup
+www = $${:srv}/www
 home = $${:etc}/home
+ssl = $${:etc}/ssl
 ssh = $${:home}/.ssh
 
 #################################
@@ -91,6 +95,7 @@ storage-path = {{ '$${' ~ slave_reference }}-backup-public_key:key}
 recipe = slapos.cookbook:publish
 -slave-reference = {{ slave_reference }}
 authorized_key = {{ '$${' ~ slave_reference }}-backup-read-public_key:readline}
+rss = https://[$${nginx-configuration:ip}]:$${nginx-configuration:port}/{{ '$${' ~ slave_reference }}-backup-script:status_name}.rss
 {%   do part_list.append("%s-backup-publish" % slave_reference) -%}
 
 [{{ slave_reference }}-backup-script]
@@ -105,6 +110,8 @@ hostname = {{ hostname }}
 include = {{ include_string }}
 exclude_string = {{ exclude_string }}
 remote_schema = {{ remote_schema }}
+status_name = {{ slave_reference }}_status.txt
+status_log = $${directory:status}/$${:status_name}
 
 [{{ slave_reference }}-backup-crontab-line]
 recipe = slapos.recipe.template
@@ -121,6 +128,25 @@ frequency = {{ frequency }}
 #################################
 # Generate crontab file
 #################################
+[update-rss-script]
+recipe = slapos.recipe.template
+url = ${template-update-rss-script:output}
+output = $${directory:etc}/$${:_buildout_section_name_}
+mode = 0700
+global_rss = $${slap-connection:computer-id}-$${slap-connection:partition-id}.rss
+
+[update-rss-crontab-line]
+recipe = slapos.recipe.template
+url = ${template-crontab-line:output}
+output = $${directory:cron-lines}/$${:_buildout_section_name_}
+mode = 0600
+script = $${update-rss-script:output}
+frequency = */5 * * * *
+{% do crontab_line_list.append("$${update-rss-crontab-line:output}") -%}
+
+[publish-global-rss]
+recipe = slapos.cookbook:publish
+rss = https://[$${nginx-configuration:ip}]:$${nginx-configuration:port}/$${update-rss-script:global_rss}
 
 {% set crontab_line_list_string = " ".join(crontab_line_list) -%}
 [activate-crontab-file]
@@ -129,12 +155,37 @@ recipe = plone.recipe.command
 stop-on-error = true
 command = ${coreutils-output:cat} ${template-crontab:output} {{ crontab_line_list_string }} | ${dcron-output:crontab} -c $${directory:crontabs} -
 
+#################################
+# Nginx service
+#################################
+[nginx-service]
+recipe = slapos.recipe.template
+url = ${template-nginx-service:output}
+output = $${directory:services}/nginx
+mode = 0700
+virtual-depends = 
+  $${nginx-configuration:ip}
+
+[nginx-configuration]
+recipe = slapos.recipe.template
+url = ${template-nginx-configuration:output}
+output = $${directory:etc}/nginx.cfg
+mode = 0600
+access_log = $${directory:log}/nginx-access.log
+error_log = $${directory:log}/nginx-error.log
+ip = $${slap-network-information:global-ipv6}
+port = 9443
+ssl_key = $${directory:ssl}/nginx.key
+ssl_csr = $${directory:ssl}/nginx.csr
+ssl_crt = $${directory:ssl}/nginx.crt
 
 # Add parts generated by template
 [buildout]
 parts =
   dcron-service
+  nginx-service
   activate-crontab-file
+  publish-global-rss
 {% for part in part_list -%}
 {{ '    %s' % part }}
 {% endfor -%}

software/backupserver/software.cfg

@@ -3,7 +3,8 @@ extends =
   ../../component/dash/buildout.cfg
   ../../component/dcron/buildout.cfg
   ../../component/logrotate/buildout.cfg
-#   ../../component/nginx/buildout.cfg
+  ../../component/openssl/buildout.cfg
+  ../../component/nginx/buildout.cfg
   ../../component/rdiff-backup/buildout.cfg
 #   ../../component/duplicity/buildout.cfg
 #   ../../component/git/buildout.cfg
@@ -15,11 +16,13 @@ extends =
   ../../stack/slapos.cfg
 
 parts =
+  extra-eggs
   rdiff-backup
 #   duplicity
   dcron
   logrotate
-#   nginx
+  nginx
+  openssl
 #   git
 #   subversion
   rsync
@@ -30,6 +33,12 @@ parts =
   template-crontab-line
   slapos-cookbook
 
+[extra-eggs]
+recipe = zc.recipe.egg
+interpreter = pythonforrssgen
+eggs =
+  PyRSS2Gen
+
 [networkcache]
 # signature certificates of the following uploaders.
 #   Romain Courteaud
@@ -163,6 +172,20 @@ signature-certificate-list =
 ##########################################################
 # Service startup scripts and configuration files
 ##########################################################
+[template-nginx-service]
+recipe = slapos.recipe.template
+url = ${:_profile_base_location_}/template-nginx-service.sh.in
+md5sum = 5c94d952305552dcbeaeaeb27dd28f3b
+output = ${buildout:directory}/template-nginx-service.sh.in
+mode = 0644
+
+[template-nginx-configuration]
+recipe = slapos.recipe.template
+url = ${:_profile_base_location_}/template-nginx.cfg.in
+md5sum = 1cf80a510a9ba9f6d9f12e2ce828615e
+output = ${buildout:directory}/template-nginx.cfg.in
+mode = 0644
+
 [template-dcron-service]
 recipe = slapos.recipe.template
 url = ${:_profile_base_location_}/template-dcron-service.sh.in
@@ -173,7 +196,7 @@ mode = 0644
 [template-backup-script]
 recipe = slapos.recipe.template
 url = ${:_profile_base_location_}/template-backup-script.sh.in
-md5sum = 19c3a72913cc285bf9745ab2f869018e
+md5sum = ce63dd27aaea8b64f446a7aef9406873
 output = ${buildout:directory}/template-backup-script.sh.in
 mode = 0644
 
@@ -191,13 +214,27 @@ md5sum = 072be0fd04896880c931d44d8eabde37
 output = ${buildout:directory}/template-crontab.in
 mode = 0644
 
+[status2rss]
+recipe = slapos.recipe.template
+url = ${:_profile_base_location_}/status2rss.py
+md5sum = 138c96e0836f2b06414b98ba2643f21c
+output = ${buildout:directory}/status2rss.py
+mode = 0644
+
+[template-update-rss-script]
+recipe = slapos.recipe.template
+url = ${:_profile_base_location_}/template-update-rss.sh.in
+md5sum = 529058c54e873ab26f7920c868b23c50
+output = ${buildout:directory}/template-update-rss.sh.in
+mode = 0644
+
 ##########################################################
 # Buildout instance.cfg templates
 ##########################################################
 [template-pullrdiffbackup]
 recipe = slapos.recipe.template
 url = ${:_profile_base_location_}/instance-pullrdiffbackup.cfg.in
-md5sum = 213256229360f57c778308825b161321
+md5sum = 935ee60f0454ab50df74d0bcc3364d37
 output = ${buildout:directory}/template-pullrdiffbackup.cfg
 mode = 0644
 
@@ -223,6 +260,7 @@ slapos.recipe.build = 0.11.6
 slapos.recipe.cmmi = 0.1.1
 slapos.recipe.template = 2.4.2
 zc.recipe.egg = 1.3.2
+PyRSS2Gen = 1.1
 
 # Required by:
 # slapos.core==0.35.1

software/backupserver/status2rss.py

+import datetime
+import uuid
+import PyRSS2Gen
+import sys
+from email.utils import parsedate_tz, mktime_tz
+import base64
+
+# Based on http://thehelpfulhacker.net/2011/03/27/a-rss-feed-for-your-crontabs/
+
+# ### Defaults
+TITLE = sys.argv[1]
+LINK = sys.argv[2]
+DESCRIPTION = TITLE
+
+items = []
+
+while 1:
+  try:
+    line = sys.stdin.readline()
+  except KeyboardInterrupt:
+    break
+
+  if not line:
+    break
+
+  time, desc = line.split(',', 1)
+
+  rss_item = PyRSS2Gen.RSSItem(
+    title = desc,
+    description = "%s, %s" % (time, desc),
+    link = LINK,
+    pubDate = datetime.datetime.fromtimestamp(mktime_tz(parsedate_tz(time))),
+    guid = PyRSS2Gen.Guid(base64.b64encode("%s, %s" % (time, desc)))
+    )
+  items.append(rss_item)
+
+### Build the rss feed
+rss_feed = PyRSS2Gen.RSS2 (
+  title = TITLE,
+  link = LINK,
+  description = DESCRIPTION,
+  lastBuildDate = datetime.datetime.utcnow(),
+  items = items
+  )
+
+print rss_feed.to_xml()

software/backupserver/template-backup-script.sh.in

@@ -4,7 +4,13 @@
 
 export HOME=$${directory:home}
 
-set -e
+# Clean status file (no history needed)
+${coreutils-output:rm} -f $${:status_log}
+
+# Inform about beginning of backup
+${coreutils-output:echo} "`${coreutils-output:date} -u`, $${:hostname} backup running" >> $${:status_log}
+
+# set -e
 cd $${:datadirectory}
 ${rdiff-backup-output:rdiff-backup} \
   $${:exclude_string} \
@@ -13,5 +19,15 @@ ${rdiff-backup-output:rdiff-backup} \
   --remote-schema '${dropbear-output:ssh} -T -y -i $${:sshkey} %s $${:remote_schema}' \
   $${:connection}::/ ./
 
+RESULT=$?
+ 
+# Inform about backup status
+if [ $RESULT == 0 ]
+then
+  ${coreutils-output:echo} "`${coreutils-output:date} -u`, $${:hostname} backup success" >> $${:status_log}
+else
+  ${coreutils-output:echo} "`${coreutils-output:date} -u`, $${:hostname} backup failed" >> $${:status_log}
+fi
+
 # python scripts/verify_with_sudo.py ./ $${:connection}:/
 

software/backupserver/template-nginx-service.sh.in

+#!${dash-output:dash}
+# BEWARE: This file is operated by slapgrid
+# BEWARE: It will be overwritten automatically
+
+if [ ! -e $${nginx-configuration:ssl_crt} ]
+then
+  ${openssl-output:openssl} genrsa -out $${nginx-configuration:ssl_key} 2048
+  ${openssl-output:openssl} req -new \
+    -subj "/C=AA/ST=Denial/L=Nowhere/O=Dis/CN=$${nginx-configuration:ip}" \
+    -key $${nginx-configuration:ssl_key} -out $${nginx-configuration:ssl_csr}
+  ${openssl-output:openssl} x509 -req -days 365 \
+    -in $${nginx-configuration:ssl_csr} \
+    -signkey $${nginx-configuration:ssl_key} \
+    -out $${nginx-configuration:ssl_crt}
+fi
+
+exec ${nginx-output:nginx} \
+  -c $${nginx-configuration:output}

software/backupserver/template-nginx.cfg.in

+daemon off; # run in the foreground so supervisord can look after it
+
+worker_processes 4;
+pid $${directory:run}/nginx.pid;
+
+events {
+  worker_connections 768;
+  # multi_accept on;
+}
+
+http {  
+  
+  ##
+  # Basic Settings
+  ##
+  
+  sendfile on;
+  tcp_nopush on;
+  tcp_nodelay on;
+  keepalive_timeout 65;
+  types_hash_max_size 2048;
+  server_tokens off;
+
+  # server_names_hash_bucket_size 64;
+  # server_name_in_redirect off;
+
+  default_type application/octet-stream;
+  include ${nginx-output:mime};
+
+  ##
+  # Logging Settings
+  ##
+  
+  access_log $${nginx-configuration:access_log};
+  error_log $${nginx-configuration:error_log};
+
+  ##
+  # Gzip Settings
+  ##
+  
+  gzip on;
+  gzip_disable "msie6";
+
+  # gzip_vary on;
+  # gzip_proxied any;
+  # gzip_comp_level 6;
+  # gzip_buffers 16 8k;
+  # gzip_http_version 1.1;
+  # gzip_types text/plain text/css application/json application/x-javascript text/xml application/xml application/xml+rss text/javascript;
+
+server {
+  listen [$${nginx-configuration:ip}]:$${nginx-configuration:port};
+
+  ssl                 on;
+  ssl_certificate $${nginx-configuration:ssl_crt};
+  ssl_certificate_key $${nginx-configuration:ssl_key};
+
+  ## Only allow GET and HEAD request methods
+  if ($request_method !~ ^(GET|HEAD)$ ) {
+     return 444;
+  }
+
+  ## Serve an error 204 (No Content) for favicon.ico
+  location = /favicon.ico {
+    return 204;
+  }
+  location /
+  {   
+      root $${directory:www};
+#       index index.html;
+  }
+}
+
+}
+

software/backupserver/template-update-rss.sh.in

+#!${dash-output:dash}
+
+STATUS_DIR=$${directory:status}
+RSS_DIR=$${directory:www}
+
+CAT=${coreutils-output:cat}
+PYTHON=${buildout:directory}/bin/${extra-eggs:interpreter}
+STATUS2RSS=${status2rss:output}
+BASENAME=${coreutils-output:basename}
+
+for status in $STATUS_DIR/*
+do
+  NAME=`$BASENAME $status`
+  $CAT $status | $PYTHON $STATUS2RSS "Backup status $NAME" "https://[$${nginx-configuration:ip}]:$${nginx-configuration:port}/$NAME.rss" > $RSS_DIR/$NAME.rss
+done
+
+$CAT $STATUS_DIR/* | $PYTHON $STATUS2RSS "Full backup status $${:global_rss}" "https://[$${nginx-configuration:ip}]:$${nginx-configuration:port}/$${:global_rss}" > $RSS_DIR/$${:global_rss}

software/kvm/common.cfg

@@ -42,6 +42,7 @@ eggs =
   websockify
   slapos.cookbook
   slapos.toolbox
+  erp5.util
 
 [http-proxy]
 # https://github.com/nodejitsu/node-http-proxy
@@ -78,7 +79,7 @@ command =
 [template]
 recipe = slapos.recipe.template
 url = ${:_profile_base_location_}/instance.cfg.in
-md5sum = 0e84223169661462f439c164d62c2a6a
+#md5sum = 461187060d12a6bea4d5b9699687c0e9
 output = ${buildout:directory}/template.cfg
 mode = 0644
 
@@ -91,9 +92,17 @@ mode = 0644
 
 [template-kvm-resilient]
 recipe = hexagonit.recipe.download
-url = ${:_profile_base_location_}/instance-kvm-resilient.cfg.jinja
+url = ${:_profile_base_location_}/instance-kvm-resilient.cfg.jinja2
 mode = 644
-md5sum = 3ee64c654aae503b93b39e9ccd6d3643
+md5sum = 6753004b582c0470bd028253ce1964ad
+download-only = true
+
+[template-kvm-resilient-test]
+recipe = slapos.recipe.template
+url = ${:_profile_base_location_}/instance-kvm-resilient-test.cfg.in
+#md5sum = 9dead7572072307b6c38cdf47052225b
+output = ${buildout:directory}/template-kvm-resilient-test.cfg
+mode = 0644
 
 [template-kvm-import]
 recipe = slapos.recipe.template
@@ -138,3 +147,4 @@ url = ${:_profile_base_location_}/instance-frontend.cfg.in
 md5sum = cdb690495e9eb007d2b7d2f8e12f5c59
 output = ${buildout:directory}/template-frontend.cfg
 mode = 0644
+

software/kvm/development.cfg

@@ -6,11 +6,13 @@ extends =
 parts +=
   slapos.cookbook-repository
   slapos.toolbox-repository
+  erp5.util-repository
   check-recipe
 
 develop =
   ${:parts-directory}/slapos.cookbook-repository
   ${:parts-directory}/slapos.toolbox-repository
+  ${:parts-directory}/erp5.util-repository
 
 [slapos.cookbook-repository]
 recipe = slapos.recipe.build:gitclone
@@ -24,13 +26,18 @@ repository = http://git.erp5.org/repos/slapos.toolbox.git
 branch = master
 git-executable = ${git:location}/bin/git
 
+[erp5.util-repository]
+recipe = slapos.recipe.build:gitclone
+repository = http://git.erp5.org/repos/erp5.git
+branch = scalability-master2
+git-executable = ${git:location}/bin/git
+
 [check-recipe]
 recipe = plone.recipe.command
 stop-on-error = true
 update-command = ${:command}
 command =
   grep parts ${buildout:develop-eggs-directory}/slapos.cookbook.egg-link &&
-  grep parts ${buildout:develop-eggs-directory}/slapos.toolbox.egg-link
+  grep parts ${buildout:develop-eggs-directory}/slapos.toolbox.egg-link &&
+  grep parts ${buildout:develop-eggs-directory}/erp5.util.egg-link
 
-[versions]
-websockify = 0.3.0
\ No newline at end of file

software/kvm/instance-kvm-input-schema.json

@@ -73,6 +73,12 @@
       "format": "uri",
       "type": "string",
     },
+    "virtual-hard-drive-md5sum": {
+      "title": "Checksum of virtual hard drive",
+      "description": "MD5 checksum of virtual hard drive, used if virtual-hard-drive-url is specified.",
+      "type": "string",
+    },
+virtual-hard-drive-md5sum
 
     "use-tap": {
       "title": "Use QEMU TAP network interface",

software/kvm/instance-kvm-resilient-test.cfg.in

+[buildout]
+
+eggs-directory = ${buildout:eggs-directory}
+develop-eggs-directory = ${buildout:develop-eggs-directory}
+offline = true
+
+parts =
+  deploy-resiliency-test
+  request-resilient-kvm
+
+[directory]
+recipe = slapos.cookbook:mkdirectory
+etc = $${buildout:directory}/etc/
+var = $${buildout:directory}/var/
+srv = $${buildout:directory}/srv/
+bin = $${buildout:directory}/bin/
+tmp = $${buildout:directory}/tmp/
+
+services = $${:etc}/service/
+scripts = $${:etc}/run/
+
+[deploy-resiliency-test]
+recipe = slapos.cookbook:wrapper
+#log-path = {directory:log}
+wrapper-path = $${directory:bin}/runKVMResiliencyTestSuite
+#output = $${binary-wrap-launcher:binary-path}
+#site-id = "{site_id}"
+# Suppose that there is a user zope family in the configuration
+#erp5-url = $${erp5-cluster:connection-family-user}{:site-id}
+#parameters-extra = $*
+command-line = ${buildout:bin-directory}/runKVMResiliencyTestSuite --server-url=$${slap-connection:server-url} --key-file=$${slap-connection:key-file} --cert-file=$${slap-connection:cert-file} --computer-id=$${slap-connection:computer-id} --partition-id=$${slap-connection:partition-id} --software=$${slap-connection:software-release-url} --namebase=kvm --kvm-rootinstance-name='$${request-resilient-kvm:name}'
+#return = url
+
+[request-resilient-kvm]
+<= slap-connection
+recipe = slapos.cookbook:request
+software-url = $${slap-connection:software-release-url}
+software-type = kvm-resilient
+name = Resilient KVM (Root Instance)
+config = virtual-hard-drive-url virtual-hard-drive-md5sum resiliency-backup-periodicity
+config-virtual-hard-drive-url = $${slap-parameter:virtual-hard-drive-url}
+config-virtual-hard-drive-md5sum = $${slap-parameter:virtual-hard-drive-md5sum}
+config-resiliency-backup-periodicity = */5
+# We don't use url parameter, but we want it to be there to make sure root instance is ready.
+return = url
+# XXX What to do?
+#sla = instance_guid
+#sla-instance_guid = $${slap-parameter:frontend-instance-guid}
+
+[slap-parameter]
+virtual-hard-drive-url = https://softinst43236.host.vifib.net/data/public/8e2138.php?dl=true
+virtual-hard-drive-md5sum = de0f10c7c6538e9928879332afd9be7a
+# XXX: what to do about pbs-kvm1-computer-guid and kvm1-computer-guid ?
+

software/kvm/instance-kvm-resilient.cfg.jinja2

@@ -10,10 +10,10 @@ offline = true
 
 # += because we need to take up parts (like instance-custom, slapmonitor etc) from the profile we extended
 parts +=
-  {{ parts.replicate("kvm", "2") }}
+  {{ parts.replicate("kvm", "3") }}
   publish-connection-informations
 
-{{ replicated.replicate("kvm", "2", "kvm-export", "kvm-import") }}
+{{ replicated.replicate("kvm", "3", "kvm-export", "kvm-import") }}
 
 # Bubble down the parameters of the requested instance to the user
 [request-kvm]
@@ -22,10 +22,10 @@ return =
 # Resilient related parameters
   url ssh-public-key ssh-url notification-id ip
 # KVM related parameters
-  backend-url url
+  backend-url url ipv6
 
 [publish-connection-informations]
 recipe = slapos.cookbook:publish
 backend-url = ${request-kvm:connection-backend-url}
 url = ${request-kvm:connection-url}
-
+ipv6 = ${request-kvm:connection-ipv6}

software/kvm/instance-kvm.cfg.in

@@ -88,7 +88,7 @@ nat-rules = $${slap-parameter:nat-rules}
 6tunnel-wrapper-path = $${directory:services}/6tunnel
 
 virtual-hard-drive-url = $${slap-parameter:virtual-hard-drive-url}
-virtual-hard-drive-md5-url = $${slap-parameter:virtual-hard-drive-md5-url}
+virtual-hard-drive-md5sum = $${slap-parameter:virtual-hard-drive-md5sum}
 
 shell-path = ${dash:location}/bin/dash
 qemu-path = ${kvm:location}/bin/qemu-system-x86_64
@@ -214,7 +214,7 @@ sla-instance_guid = $${slap-parameter:frontend-instance-guid}
 recipe = slapos.cookbook:publish
 backend-url = https://[$${novnc-instance:ip}]:$${novnc-instance:port}/vnc_auto.html?host=[$${novnc-instance:ip}]&port=$${novnc-instance:port}&encrypt=1&password=$${kvm-instance:vnc-passwd}
 url = $${request-slave-frontend:connection-url}/vnc_auto.html?host=$${request-slave-frontend:connection-domainname}&port=$${request-slave-frontend:connection-port}&encrypt=1&path=$${request-slave-frontend:connection-resource}&password=$${kvm-instance:vnc-passwd}
-
+ipv6 = $${slap-network-information:global-ipv6}
 
 [frontend-promise]
 recipe = slapos.cookbook:check_url_available
@@ -243,4 +243,4 @@ nat-rules = 22 80 443
 use-tap = False
 
 virtual-hard-drive-url =
-virtual-hard-drive-md5-url =
+virtual-hard-drive-md5sum =

software/kvm/instance.cfg.in

@@ -16,6 +16,9 @@ kvm-resilient = $${dynamic-template-kvm-resilient:rendered}
 kvm-import = ${template-kvm-import:output}
 kvm-export = ${template-kvm-export:output}
 
+# Used for the test of resiliency. The system wants a "test" software_type.
+test = ${template-kvm-resilient-test:output}
+
 frozen = ${instance-frozen:output}
 pull-backup = ${template-pull-backup:output}
 
@@ -29,8 +32,7 @@ cert = $${slap-connection:cert-file}
 
 [dynamic-template-kvm-resilient]
 recipe = slapos.recipe.template:jinja2
-template = ${:_profile_base_location_}/instance-kvm-resilient.cfg.jinja2
-md5sum = 1b7a2fcc884649b4d08f238e828899c1
+template = ${template-kvm-resilient:location}/instance-kvm-resilient.cfg.jinja2
 rendered = $${buildout:directory}/template-kvm-resilient.cfg
 context = key buildout buildout:bin-directory
           key develop_eggs_directory buildout:develop-eggs-directory

stack/resilient/buildout.cfg

@@ -48,7 +48,7 @@ mode = 0644
 recipe = slapos.recipe.template
 url = ${:_profile_base_location_}/pbsready-export.cfg.in
 output = ${buildout:directory}/pbsready-export.cfg
-md5sum = a0e22a5de727544c5767d6bee059a77a
+md5sum = ef3861861746d3574f39f1aa3200d74e
 mode = 0644
 
 [template-pull-backup]

stack/resilient/pbsready-export.cfg.in

@@ -39,5 +39,10 @@ notify = $${slap-parameter:notify}
 <= cron
 recipe = slapos.cookbook:cron.d
 name = backup
-frequency = 0 * * * *
+frequency = $${slap-parameter:resiliency-backup-periodicity} * * * *
 command = $${notifier-exporter:wrapper}
+
+[slap-parameter]
+# in minutes, modulo 60, in cron.d format (i.e */15 is accepted).
+resiliency-backup-periodicity = 0
+