Merge remote-tracking branch 'origin/master' into erp5-cluster

CHANGES.txt

 Changes
 =======
 
+0.97 (2015-03-26)
+-----------------
+ * switch softwaretype recipe: the recipe is backward compatible with old slapos node packages.
+ * kvm recipe: Avoid getting wrong storage path when creating kvm external disk
+
+0.96 (2015-03-20)
+-----------------
+ * slap configuration: recipe can read from master network information releated to a tap interface
+ * slap configuration: recipe will setup data folder in DATA directory of computer partition if disk is mounted
+ * switch softwaretype recipe: also generate tap network information when they exist
+ * switch softwaretype recipe: generate configuration for DATA directory when disk is mounted
+
 0.95 (2015-02-14)
 -----------------
 

component/firefox/buildout.cfg

@@ -26,12 +26,12 @@ depends =
   ${liberation-fonts:location}
   ${ipaex-fonts:location}
 
-version = 36.0.1
+version = 36.0.4
 
 # MD5SUMs are available at :
 # https://ftp.mozilla.org/pub/mozilla.org/firefox/releases/${:version}/MD5SUMS
-x86 = http://download-installer.cdn.mozilla.net/pub/firefox/releases/${:version}/linux-i686/en-US/firefox-${:version}.tar.bz2 b079611053993a13decefb2d5a796e41
-x86-64 = http://download-installer.cdn.mozilla.net/pub/firefox/releases/${:version}/linux-x86_64/en-US/firefox-${:version}.tar.bz2 d3ba7c117b7663a3ec644acc80e73811
+x86 = http://download-installer.cdn.mozilla.net/pub/firefox/releases/${:version}/linux-i686/en-US/firefox-${:version}.tar.bz2 6f50b82a6c020d0f29a04cfcd9fe8282
+x86-64 = http://download-installer.cdn.mozilla.net/pub/firefox/releases/${:version}/linux-x86_64/en-US/firefox-${:version}.tar.bz2 119d9369a3b1be282936c35bf036414f
 
 script =
   if not self.options.get('url'): self.options['url'], self.options['md5sum'] = self.options[guessPlatform()].split(' ')

component/libreoffice-bin/buildout.cfg

@@ -9,11 +9,11 @@ parts =
 [libreoffice-bin]
 recipe = slapos.recipe.build
 # here, two %s are used, first one is for directory name (eg. x86_64), and second one is for filename (eg. x86-64).
-version = 4.4.0.3
+version = 4.4.1.2
 url = http://downloadarchive.documentfoundation.org/libreoffice/old/${:version}/rpm/%s/LibreOffice_${:version}_Linux_%s_rpm.tar.gz
 # supported architectures md5sums
-md5sum_x86 = ac2c90333a1b84e6ff0740560d02e8a2
-md5sum_x86-64 = c2848dcc87f6d17e1a2c835a195435bb
+md5sum_x86 = 23b740e84bc2b8774d50ac97dd553afd
+md5sum_x86-64 = 8de6f3d1d4af3826a3a1695a47444948
 
 # where office code can be found?
 officedir = libreoffice4.4

component/re6stnet/buildout.cfg

@@ -28,8 +28,9 @@ stop-on-error = true
 dir = ${re6stnet-repository:location}
 command =
   rm -f "${:dir}/re6stconf.py" && ln -s re6st-conf "${:dir}/re6stconf.py"
-  rm -f "${:dir}/re6stregister.py" && ln -s re6st-conf "${:dir}/re6stregister.py"
-  rm -f "${:dir}/re6stnet.py" && ln -s re6st-conf "${:dir}/re6stnet.py"
+  rm -f "${:dir}/re6stregistry.py" && ln -s re6st-registry "${:dir}/re6stregistry.py"
+  rm -f "${:dir}/re6stnet.py" && ln -s re6stnet "${:dir}/re6stnet.py"
+  sed -i 's#("git",)#("${git:location}/bin/git",)#' ${:dir}/re6st/version.py
 
 update-command = ${:command}
 

setup.py

@@ -28,7 +28,7 @@ from setuptools import setup, find_packages
 import glob
 import os
 
-version = '0.96.dev0'
+version = '0.97.dev'
 name = 'slapos.cookbook'
 long_description = open("README.txt").read() + "\n" + \
     open("CHANGES.txt").read() + "\n"
@@ -174,6 +174,7 @@ setup(name=name,
           'request.serialised = slapos.recipe.request:Serialised',
           'request.edge = slapos.recipe.request:RequestEdge',
           'requestoptional = slapos.recipe.request:RequestOptional',
+          're6stnet.registry = slapos.recipe.re6stnet:Recipe',
           'reverseproxy.nginx = slapos.recipe.reverse_proxy_nginx:Recipe',
           'seleniumrunner = slapos.recipe.seleniumrunner:Recipe',
           'sheepdogtestbed = slapos.recipe.sheepdogtestbed:SheepDogTestBed',

slapos/recipe/kvm/template/kvm_run.in

@@ -32,6 +32,12 @@ tap_mac_address = '%(tap-mac-address)s'
 smp_count = '%(smp-count)s'
 ram_size = '%(ram-size)s'
 pid_file_path = '%(pid-file-path)s'
+external_disk_number = %(external-disk-number)s
+external_disk_size = '%(external-disk-size)s'
+disk_storage_dict = {}
+disk_storage_list = """%(disk-storage-list)s""".split('\n')
+map_storage_list = []
+etc_directory = '%(etc-directory)s'.strip()
 
 def md5Checksum(file_path):
     with open(file_path, 'rb') as fh:
@@ -62,6 +68,22 @@ def getSocketStatus(host, port):
     break
   return s
 
+def getMapStorageList(disk_storage_dict):
+  map_disk_file = os.path.join(etc_directory, '.data-disk-ids')
+  id_list = []
+  if os.path.exists(map_disk_file):
+    with open(map_disk_file, 'r') as mf:
+      # ID are writen in one line: data1 data3 data2 ...
+      content = mf.readline()
+      if content:
+        id_list = [id for id in content.split(' ') if disk_storage_dict.has_key(id)]
+  for key in disk_storage_dict:
+    if not key in id_list:
+      id_list.append(key)
+  with open(map_disk_file, 'w') as mf:
+    mf.write(' '.join(id_list))
+  return id_list
+
 # Download existing hard drive if needed at first boot
 if not os.path.exists(disk_path) and virtual_hard_drive_url != '':
   print('Downloading virtual hard drive...')
@@ -98,6 +120,32 @@ if not os.path.exists(disk_path):
       disk_path, '%%sG' %% disk_size])
   print('Done.')
 
+# Check and create external disk
+additional_disk_list = []
+for storage in sorted(disk_storage_list):
+  if storage:
+    key, val = storage.split(' ')
+    disk_storage_dict[key.strip()] = val.strip()
+
+map_storage_list = getMapStorageList(disk_storage_dict)
+
+if disk_storage_dict:
+  if int(external_disk_number) > 0:
+    index = 0
+    while (index < len(disk_storage_dict)) and (index < external_disk_number):
+      path = disk_storage_dict[map_storage_list[index]]
+      if os.path.exists(path):
+        disk_filepath = os.path.join(path, 'kvm_virtual_disk.qcow2')
+        if not os.path.exists(disk_filepath):
+          print('Creating one additional virtual hard drive...')
+          subprocess.Popen([qemu_img_path, 'create' ,'-f', 'qcow2',
+              disk_filepath, '%%sG' %% external_disk_size])
+        additional_disk_list.append(disk_filepath)
+      else:
+        print('Data folder %%s was not used to create external disk %%r' %% (index +1))
+      index += 1
+
+
 # Generate network parameters
 # XXX: use_tap should be a boolean
 tap_network_parameter = []
@@ -130,6 +178,9 @@ if tap_network_parameter == [] and nat_network_parameter == []:
 else:
   kvm_argument_list += nat_network_parameter + tap_network_parameter
 
+for disk in additional_disk_list:
+  kvm_argument_list.extend([
+            '-drive', 'file=%%s,if=%%s' %% (disk, disk_type)])
 # Try to connect to NBD server (and second nbd if defined).
 # If not available, don't even specify it in qemu command line parameters.
 # Reason: if qemu starts with unavailable NBD drive, it will just crash.

slapos/recipe/re6stnet/__init__.py

+##############################################################################
+#
+# Copyright (c) 2010 Vifib SARL and Contributors. All Rights Reserved.
+#
+# WARNING: This program as such is intended to be used by professional
+# programmers who take the whole responsibility of assessing all potential
+# consequences resulting from its eventual inadequacies and bugs
+# End users who are looking for a ready-to-use solution with commercial
+# guarantees and support are strongly adviced to contract a Free Software
+# Service Company
+#
+# This program is Free Software; you can redistribute it and/or
+# modify it under the terms of the GNU General Public License
+# as published by the Free Software Foundation; either version 3
+# of the License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA  02111-1307, USA.
+#
+##############################################################################
+import subprocess
+
+from slapos.recipe.librecipe import GenericBaseRecipe
+import socket
+import struct
+import os
+import string, random
+import json
+import traceback
+from slapos import slap
+
+class Recipe(GenericBaseRecipe):
+  
+  def __init__(self, buildout, name, options):
+    """Default initialisation"""
+    self.slap = slap.slap()
+
+    # SLAP related information
+    slap_connection = buildout['slap-connection']
+    self.computer_id = slap_connection['computer-id']
+    self.computer_partition_id = slap_connection['partition-id']
+    self.server_url = slap_connection['server-url']
+    self.software_release_url = slap_connection['software-release-url']
+    self.key_file = slap_connection.get('key-file')
+    self.cert_file = slap_connection.get('cert-file')
+    
+    return GenericBaseRecipe.__init__(self, buildout, name, options)
+
+  def getSerialFromIpv6(self, ipv6):
+    prefix = ipv6.split('/')[0].lower()
+    hi, lo = struct.unpack('!QQ', socket.inet_pton(socket.AF_INET6, prefix))
+    ipv6_int = (hi << 64) | lo
+    serial = '0x1%x' % ipv6_int
+
+    # delete non significant part
+    for part in prefix.split(':')[::-1]:
+      if part:
+        for i in ['0']*(4 - len(part)):
+          part = i + part
+        serial = serial.split(part)[0] + part
+        break
+
+    return serial
+
+  def generateCertificate(self):
+    key_file = self.options['key-file'].strip()
+    cert_file = self.options['cert-file'].strip()
+    if not os.path.exists(key_file):
+      serial = self.getSerialFromIpv6(self.options['ipv6-prefix'].strip())
+
+      key_command = [self.options['openssl-bin'], 'genrsa', '-out',
+                            '%s' % key_file, self.options['key-size']]
+
+      #'-config', openssl_configuration
+      cert_command = [self.options['openssl-bin'], 'req', '-nodes', '-new',
+                  '-x509', '-batch', '-key', '%s' % key_file, '-set_serial',
+                  '%s' % serial, '-days', '3650', '-out', '%s' % cert_file]
+
+      subprocess.check_call(key_command)
+      subprocess.check_call(cert_command)
+
+  def generateSlaveTokenList(self, slave_instance_list, token_file):
+    to_remove_dict = {}
+    to_add_dict = {}
+    token_dict = self.loadJsonFile(token_file)
+
+    reference_list = [slave_instance.get('slave_reference') for slave_instance
+                        in slave_instance_list]
+    for reference in reference_list:
+      if not reference in token_dict:
+        # we generate new token
+        number = reference.split('-')[1]
+        new_token = number + ''.join(random.sample(string.ascii_lowercase, 15))
+        token_dict[reference] = new_token
+        to_add_dict[reference] = new_token
+
+    for reference in token_dict.keys():
+      if not reference in reference_list:
+        # This slave instance is destroyed ?
+        to_remove_dict[reference] = token_dict.pop(reference)
+
+    return token_dict, to_add_dict, to_remove_dict
+
+  def loadJsonFile(self, path):
+    if os.path.exists(path):
+      with open(path, 'r') as f:
+        content = f.read()
+      return json.loads(content)
+    else:
+      return {}
+
+  def writeFile(self, path, data):
+    with open(path, 'w') as f:
+      f.write(data)
+    return path
+
+  def readFile(self, path):
+    if os.path.exists(path):
+      with open(path, 'r') as f:
+        content = f.read()
+      return content
+    return ''
+
+  def install(self):
+    path_list = []
+    token_save_path = os.path.join(self.options['conf-dir'], 'token.json')
+    token_list_path = self.options['token-dir']
+
+    self.generateCertificate()
+
+    wrapper = self.createWrapper(name=self.options['wrapper'],
+                            command=self.options['command'],
+                            parameters=['@%s' % self.options['config-file']])
+
+    path_list.append(wrapper)
+
+    slave_list = json.loads(self.options['slave-instance-list'])
+    registry_url = 'http://%s:%s/' % (self.options['ipv4'], self.options['port'])
+    token_dict, add_token_dict, rm_token_dict = self.generateSlaveTokenList(
+                                                    slave_list, token_save_path)
+
+    # write request add token
+    for reference in add_token_dict:
+      path = os.path.join(token_list_path, '%s.add' % reference)
+      if not os.path.exists(path):
+        self.createFile(path, add_token_dict[reference])
+
+    # write request remove token
+    for reference in rm_token_dict:
+      path = os.path.join(token_list_path, '%s.remove' % reference)
+      if not os.path.exists(path):
+        self.createFile(path, rm_token_dict[reference])
+        # remove request add file if exists
+        add_path = os.path.join(token_list_path, '%s.add' % reference)
+        if os.path.exists(add_path):
+          os.unlink(add_path)
+
+    self.createFile(token_save_path, json.dumps(token_dict))
+
+    service_dict = dict(token_base_path=token_list_path,
+                        token_json=token_save_path,
+                        db=self.options['db-path'],
+                        partition_id=self.computer_partition_id,
+                        computer_id=self.computer_id,
+                        registry_url=registry_url)
+    service_dict['server_url'] = self.server_url
+    service_dict['cert_file'] = self.cert_file
+    service_dict['key_file'] = self.key_file
+
+    request_add = self.createPythonScript(
+        self.options['manager-wrapper'].strip(),
+        '%s.re6stnet.manage' % __name__, service_dict
+      )
+    path_list.append(request_add)
+
+    request_drop = self.createPythonScript(
+        self.options['drop-service-wrapper'].strip(),
+        '%s.re6stnet.requestRemoveToken' % __name__, service_dict
+      )
+    path_list.append(request_drop)
+
+    request_check = self.createPythonScript(
+        self.options['check-service-wrapper'].strip(),
+        '%s.re6stnet.checkService' % __name__, service_dict
+      )
+    path_list.append(request_check)
+
+    # Send connection parameters of slave instances
+    if token_dict:
+      self.slap.initializeConnection(self.server_url, self.key_file,
+        self.cert_file)
+      computer_partition = self.slap.registerComputerPartition(
+        self.computer_id,
+        self.computer_partition_id)
+
+      for slave_reference, token in token_dict.iteritems():
+        try:
+          status_file = os.path.join(token_list_path, '%s.status' % slave_reference)
+          status = self.readFile(status_file) or 'New token requested'
+          msg = status
+          if status == 'TOKEN_ADDED':
+            msg = 'Token is ready for use'
+          elif status == 'TOKEN_USED':
+            msg = 'Token not available, it has been used to generate re6stnet certificate.'
+          computer_partition.setConnectionDict(
+              {'token':token, '1_info':msg},
+              slave_reference)
+        except:
+          self.logger.fatal("Error while sending slave %s informations: %s",
+             slave_reference, traceback.format_exc())
+
+    return path_list
+
+

slapos/recipe/re6stnet/re6stnet.py

+# -*- coding: utf-8 -*-
+import logging
+import json
+import os
+import time
+import sqlite3
+import slapos
+
+from re6st import registry
+
+log = logging.getLogger('SLAPOS-RE6STNET')
+logging.basicConfig(level=logging.DEBUG)
+
+def loadJsonFile(path):
+  if os.path.exists(path):
+    with open(path, 'r') as f:
+      content = f.read()
+      return json.loads(content)
+  else:
+    return {}
+
+def writeFile(path, data):
+  with open(path, 'w') as f:
+    f.write(data)
+
+def readFile(path):
+  if os.path.exists(path):
+    with open(path, 'r') as f:
+      content = f.read()
+    return content
+  return ''
+
+def getDb(db_path):
+  db = sqlite3.connect(db_path, isolation_level=None,
+                                                  check_same_thread=False)
+  db.text_factory = str
+  
+  return db.cursor()
+
+def bang(args):
+  computer_guid = args['computer_id']
+  partition_id = args['partition_id']
+  slap = slapos.slap.slap()
+
+  # Redeploy instance to update published information
+  slap.initializeConnection(args['server_url'], args['key_file'],
+                                                          args['cert_file'])
+  partition = slap.registerComputerPartition(computer_guid=computer_guid,
+                                                   partition_id=partition_id)
+  partition.bang(message='Published parameters changed!')
+  log.info("Bang with message 'parameters changed'...")
+
+def requestAddToken(args, can_bang=True):
+
+  time.sleep(3)
+  registry_url = args['registry_url']
+  base_token_path = args['token_base_path']
+  path_list = [x for x in os.listdir(base_token_path) if x.endswith('.add')]
+
+  if not path_list:
+    log.info("No new token to add. Exiting...")
+    return
+
+  client = registry.RegistryClient(registry_url)
+  call_bang = False
+
+  for reference_key in path_list:
+    request_file = os.path.join(base_token_path, reference_key)
+    token = readFile(request_file)
+    if token :
+      reference = reference_key.split('.')[0]
+      email = '%s@slapos' % reference.lower()
+      try:
+        result = client.requestAddToken(token, email)
+      except Exception, e:
+        log.debug('Request add token fail for %s... \n %s' % (request_file,
+                    str(e)))
+        continue
+      if result and result == token:
+        # update information
+        log.info("New token added for slave instance %s. Updating file status..." %
+                            reference)
+        writeFile(os.path.join(base_token_path, '%s.status' % reference),
+                    'TOKEN_ADDED')
+        os.unlink(request_file)
+        call_bang = True
+    else:
+      log.debug('Bad token. Request add token fail for %s...' % request_file)
+  
+  if can_bang and call_bang:
+    bang(args)
+
+def requestRemoveToken(args):
+  base_token_path = args['token_base_path']
+  path_list = [x for x in os.listdir(base_token_path) if x.endswith('.remove')]
+  
+  if not path_list:
+    log.info("No token to delete. Exiting...")
+    return
+  
+  client = registry.RegistryClient(args['registry_url'])
+  for reference_key in path_list:
+    request_file = os.path.join(base_token_path, reference_key)
+    token = readFile(request_file)
+    if token :
+      reference = reference_key.split('.')[0]
+      try:
+        result = client.requestDeleteToken(token)
+      except Exception, e:
+        log.debug('Request delete token fail for %s... \n %s' % (request_file,
+                    str(e)))
+        continue
+      if result == 'True':
+        # update information
+        log.info("Token deleted for slave instance %s. Clean up file status..." %
+                            reference)
+        os.unlink(request_file)
+        status_file = os.path.join(base_token_path, '%s.status' % reference)
+        if os.path.exists(status_file):
+          os.unlink(status_file)
+      else:
+        log.debug('Request delete token fail for %s...' % request_file)
+    else:
+      log.debug('Bad token. Request add token fail for %s...' % request_file)
+
+def checkService(args, can_bang=True):
+  base_token_path = args['token_base_path']
+  token_dict = loadJsonFile(args['token_json'])
+
+  if not token_dict:
+    return
+
+  db = getDb(args['db'])
+  call_bang = False
+
+  computer_guid = args['computer_id']
+  partition_id = args['partition_id']
+  slap = slapos.slap.slap()
+
+  # Check token status
+  for slave_reference, token in token_dict.iteritems():
+    status_file = os.path.join(base_token_path, '%s.status' % slave_reference)
+    if not os.path.exists(status_file):
+      # This token is not added yet!
+      continue
+
+    msg = readFile(status_file)
+    if msg == 'TOKEN_USED':
+      continue
+
+    # Check if token is not in the database
+    status = False
+    try:
+        token_found, = db.execute("SELECT token FROM token WHERE token = ?",
+            (token,)).next()
+        if token_found == token:
+          status = True
+    except StopIteration:
+        pass
+    if not status:
+      # Token is used to register client
+      call_bang = True
+      try:
+        time.sleep(1)
+        writeFile(status_file, 'TOKEN_USED')
+        log.info("Token status of %s updated to 'used'." % slave_reference)
+      except IOError, e:
+        # XXX- this file should always exists
+        log.debug('Error when writing in file %s. Clould not update status of %s...' %
+                              (status_file, slave_reference))
+
+  if call_bang and can_bang:
+    bang(args)
+
+def manage(args):
+  # Request Add new tokens
+  requestAddToken(args)
+
+  # Request delete removed token
+  requestRemoveToken(args)
+
+  # check status of all token
+  checkService(args)

slapos/recipe/shellinabox.py

@@ -69,7 +69,6 @@ def shellinabox(args):
 
   command_line = [
     args['shellinabox'],
-    '-d',
     '-c', certificate_dir,
     '-s', service,
     '--ipv6', args['ipv6'],

slapos/recipe/slapconfiguration.py

@@ -32,6 +32,7 @@ import slapos.slap
 from slapos.recipe.librecipe import unwrap
 from ConfigParser import RawConfigParser
 from netaddr import valid_ipv4, valid_ipv6
+from slapos.util import mkdir_p
 
 class Recipe(object):
   """
@@ -61,6 +62,10 @@ class Recipe(object):
       Partition identifier.
       Example:
         ${slap-connection:partition-id}
+    storage-home
+      Path of folder configured for data storage
+      Example:
+        ${storage-configuration:storage-home}
 
   Output:
     slap-software-type
@@ -75,8 +80,20 @@ class Recipe(object):
       One of the IPv6 addresses.
     tap
       Set of TAP interfaces.
+    tap-network-information-dict
+      Dict of set of all TAP network information
+    tap-ipv4
+      ipv4 allowed for this TAP
+    tap-gateway
+      ipv4 of gateway interface of this TAP
+    tap-netmask
+      ipv4 netmask address of this TAP
+    tap-network
+      ipv4 network address of this TAP
     configuration
       Dict of all parameters.
+    storage-dict
+      Dict of partition data path when it is configured
     configuration.<key>
       One key per partition parameter.
       Partition parameter whose name cannot be represented unambiguously in
@@ -91,7 +108,8 @@ class Recipe(object):
   OPTCRE_match = RawConfigParser.OPTCRE.match
 
   def __init__(self, buildout, name, options):
-      parameter_dict = self.fetch_parameter_dict(options)
+      parameter_dict = self.fetch_parameter_dict(options,
+                                      buildout['buildout']['directory'])
 
       match = self.OPTCRE_match
       for key, value in parameter_dict.iteritems():
@@ -99,7 +117,7 @@ class Recipe(object):
               continue
           options['configuration.' + key] = value
 
-  def fetch_parameter_dict(self, options):
+  def fetch_parameter_dict(self, options, instance_root):
       slap = slapos.slap.slap()
       slap.initializeConnection(
           options['url'],
@@ -134,6 +152,14 @@ class Recipe(object):
       v6_add = ipv6_set.add
       tap_set = set()
       tap_add = tap_set.add
+      route_gw_set = set()
+      route_gw_add = route_gw_set.add
+      route_mask_set = set()
+      route_mask_add = route_mask_set.add
+      route_ipv4_set = set()
+      route_v4_add = route_ipv4_set.add
+      route_network_set = set()
+      route_net_add = route_network_set.add
       for tap, ip in parameter_dict.pop('ip_list'):
           tap_add(tap)
           if valid_ipv4(ip):
@@ -141,6 +167,21 @@ class Recipe(object):
           elif valid_ipv6(ip):
               v6_add(ip)
           # XXX: emit warning on unknown address type ?
+
+      if 'full_ip_list' in parameter_dict:
+        for item in parameter_dict.pop('full_ip_list'):
+          if len(item) == 5:
+            tap, ip, gw, netmask, network = item
+            if  tap.startswith('route_'):
+              if valid_ipv4(gw):
+                route_gw_add(gw)
+              if valid_ipv4(netmask):
+                route_mask_add(netmask)
+              if valid_ipv4(ip):
+                route_v4_add(ip)
+              if valid_ipv4(network):
+                route_net_add(network)
+
       options['ipv4'] = ipv4_set
       options['ipv6'] = ipv6_set
 
@@ -149,6 +190,35 @@ class Recipe(object):
           options['ipv4-random'] = list(ipv4_set)[0].encode('UTF-8')
       if ipv6_set:
           options['ipv6-random'] = list(ipv6_set)[0].encode('UTF-8')
+      if route_ipv4_set:
+        options['tap-ipv4'] = list(route_ipv4_set)[0].encode('UTF-8')
+        options['tap-network-information-dict'] = dict(ipv4=route_ipv4_set,
+                                    netmask=route_mask_set,
+                                    gateway=route_gw_set,
+                                    network=route_network_set)
+      else:
+        options['tap-network-information-dict'] = {}
+      if route_gw_set:
+        options['tap-gateway'] = list(route_gw_set)[0].encode('UTF-8')
+      if route_mask_set:
+        options['tap-netmask'] = list(route_mask_set)[0].encode('UTF-8')
+      if route_network_set:
+        options['tap-network'] = list(route_network_set)[0].encode('UTF-8')
+
+      storage_home = options.get('storage-home')
+      storage_dict = {}
+      if storage_home and os.path.exists(storage_home) and \
+                                  os.path.isdir(storage_home):
+        for filename in os.listdir(storage_home):
+          storage_path = os.path.join(storage_home, filename, 
+                                    options['slap-computer-partition-id'])
+          if os.path.exists(storage_path) and os.path.isdir(storage_path):
+            storage_link = os.path.join(instance_root, 'DATA', filename)
+            mkdir_p(os.path.join(instance_root, 'DATA'))
+            if not os.path.lexists(storage_link):
+              os.symlink(storage_path, storage_link)
+            storage_dict[filename] = storage_link
+      options['storage-dict'] = storage_dict
 
       options['tap'] = tap_set
       return self._expandParameterDict(options, parameter_dict)

slapos/recipe/softwaretype.py

@@ -34,6 +34,7 @@ import subprocess
 import slapos.slap
 import netaddr
 import logging
+import errno
 
 import zc.buildout
 
@@ -54,6 +55,18 @@ class Recipe:
         return ip
     raise AttributeError
 
+  def _getTapIpAddressList(self, test_method):
+    """Internal helper method to fetch full ip address assigned for tap"""
+    if not 'full_ip_list' in self.parameter_dict:
+      return ()
+    for item in self.parameter_dict['full_ip_list']:
+      if len(item) == 5:
+        tap, ip, gw, mask, net = item
+        if tap.startswith('route_') and test_method(ip) and \
+                          test_method(gw) and test_method(mask):
+          return (ip, gw, mask, net)
+    return ()
+
   def getLocalIPv4Address(self):
     """Returns local IPv4 address available on partition"""
     # XXX: Lack checking for locality of address
@@ -64,6 +77,11 @@ class Recipe:
     # XXX: Lack checking for globality of address
     return self._getIpAddress(netaddr.valid_ipv6)
   
+  def getLocalTapIPv4AddressList(self):
+    """Returns global IPv6 address available for tap interface"""
+    # XXX: Lack checking for locality of address
+    return self._getTapIpAddressList(netaddr.valid_ipv4)
+
   def getNetworkInterface(self):
     """Returns the network interface available on partition"""
     if not 'ip_list' in self.parameter_dict:
@@ -73,6 +91,20 @@ class Recipe:
         return name
     raise AttributeError, "Not network interface found"
   
+  def mkdir_p(self, path, mode=0700):
+    """
+    Creates a directory and its parents, if needed.
+    NB: If the directory already exists, it does not change its permission.
+    """
+
+    try:
+        os.makedirs(path, mode)
+    except OSError as exc:
+        if exc.errno == errno.EEXIST and os.path.isdir(path):
+            pass
+        else:
+            raise
+
   def install(self):
     slap = slapos.slap.slap()
     slap_connection = self.buildout['slap_connection']
@@ -81,6 +113,11 @@ class Recipe:
     server_url = slap_connection['server_url']
     key_file = slap_connection.get('key_file')
     cert_file = slap_connection.get('cert_file')
+    instance_root = self.buildout['buildout']['directory']
+    storage_configuration_dict = self.buildout.get('storage-configuration')
+    storage_home = ''
+    if storage_configuration_dict:
+      storage_home = storage_configuration_dict.get('storage-home')
     slap.initializeConnection(server_url, key_file, cert_file)
     self.computer_partition = slap.registerComputerPartition(
       computer_id,
@@ -128,6 +165,14 @@ class Recipe:
                  self.getGlobalIPv6Address())
     buildout.set('slap-network-information', 'network-interface',
                  self.getNetworkInterface())
+    tap_ip_list = self.getLocalTapIPv4AddressList()
+    tap_ipv4 = tap_gateway = tap_netmask = tap_network = ''
+    if tap_ip_list:
+      tap_ipv4, tap_gateway, tap_netmask, tap_network= tap_ip_list
+    buildout.set('slap-network-information', 'tap-ipv4', tap_ipv4)
+    buildout.set('slap-network-information', 'tap-gateway', tap_gateway)
+    buildout.set('slap-network-information', 'tap-netmask', tap_netmask)
+    buildout.set('slap-network-information', 'tap-network', tap_network)
 
     # Copy/paste slap_connection
     buildout.add_section('slap-connection')
@@ -137,6 +182,27 @@ class Recipe:
     # XXX: Needed for lxc. Use non standard API
     buildout.set('slap-connection', 'requested', self.computer_partition._requested_state)
 
+    # setup storage directory
+    buildout.add_section('storage-configuration')
+    buildout.set('storage-configuration', 'storage-home', storage_home)
+    if storage_home and os.path.exists(storage_home) and \
+                                  os.path.isdir(storage_home):
+      # Create folder instance_root/DATA/ if not exist
+      data_home = os.path.join(instance_root, 'DATA')
+      self.mkdir_p(data_home)
+      for filename in os.listdir(storage_home):
+        storage_path = os.path.join(storage_home, filename, computer_partition_id)
+        if os.path.exists(storage_path) and os.path.isdir(storage_path):
+          storage_link = os.path.join(data_home, filename)
+          if os.path.lexists(storage_link):
+            if not os.path.islink(storage_link):
+              raise zc.buildout.UserError(
+                  'Target %r already exists but is not a link' % storage_link)
+              #os.unlink(storage_link)
+          else:
+            os.symlink(storage_path, storage_link)
+          buildout.set('storage-configuration', filename, storage_link)
+
     work_directory = os.path.abspath(self.buildout['buildout'][
       'directory'])
     buildout_filename = os.path.join(work_directory,

slapos/test/recipe/test_re6stnet.py

+
+import os, time
+import shutil
+import sys
+import tempfile
+import unittest
+from slapos.slap.slap import NotFoundError, ConnectionError
+
+from slapos.recipe import re6stnet
+
+
+class Re6stnetTest(unittest.TestCase):
+
+  def setUp(self):
+    self.ssl_dir = tempfile.mkdtemp()
+    self.conf_dir = tempfile.mkdtemp()
+    self.base_dir = tempfile.mkdtemp()
+    self.token_dir = tempfile.mkdtemp()
+    self.dir_list = [self.ssl_dir, self.conf_dir, self.base_dir, self.token_dir]
+    config_file = os.path.join(self.base_dir, 'config')
+    with open(config_file, 'w') as f:
+      f.write('port  9201')
+    self.options = options = {
+                'openssl-bin': '/usr/bin/openssl',
+                'key-file': os.path.join(self.ssl_dir, 'cert.key'),
+                'cert-file': os.path.join(self.ssl_dir, 'cert.crt'),
+                'key-size': '2048',
+                'conf-dir': self.conf_dir,
+                'token-dir': self.token_dir,
+                'wrapper': os.path.join(self.base_dir, 'wrapper'),
+                'config-file': config_file,
+                'ipv4': '127.0.0.1',
+                'port': '9201',
+                'db-path': '/path/to/db',
+                'command': '/path/to/command',
+                'manager-wrapper': os.path.join(self.base_dir, 'manager_wrapper'),
+                'drop-service-wrapper': os.path.join(self.base_dir, 'drop_wrapper'),
+                'check-service-wrapper': os.path.join(self.base_dir, 'check_wrapper'),
+                'slave-instance-list': '{}'
+                }
+    
+  def tearDown(self):
+    for path in self.dir_list:
+      if os.path.exists(path):
+        shutil.rmtree(path)
+    
+  def new_recipe(self):
+      buildout = {
+              'buildout': {
+                  'bin-directory': '',
+                  'find-links': '',
+                  'allow-hosts': '',
+                  'develop-eggs-directory': '',
+                  'eggs-directory': '',
+                  'python': 'testpython',
+                  },
+               'testpython': {
+                   'executable': sys.executable,
+                   },
+               'slap-connection': {
+                   'computer-id': 'comp-test',
+                   'partition-id': 'slappart0',
+                   'server-url': 'http://server.com',
+                   'software-release-url': 'http://software.com',
+                   'key-file': '/path/to/key',
+                   'cert-file': '/path/to/cert'
+                   }
+              }
+
+      options = self.options
+
+      return re6stnet.Recipe(buildout=buildout, name='re6stnet', options=options)
+  
+  def checkWrapper(self, path):
+    self.assertTrue(os.path.exists(path))
+    content = ""
+    token_file = os.path.join(self.options['conf-dir'], 'token.json')
+    with open(path, 'r') as f:
+        content = f.read()
+    self.assertIn("'token_json': '%s'" % token_file, content)
+    self.assertIn("'partition_id': 'slappart0'", content)
+    self.assertIn("'computer_id': 'comp-test'", content)
+    self.assertIn("'key_file': '/path/to/key'", content)
+    self.assertIn("'cert_file': '/path/to/cert'", content)
+    self.assertIn("'server_url': 'http://server.com'", content)
+    self.assertIn("'db': '%s'" % self.options['db-path'], content)
+    self.assertIn("'token_base_path': '%s'" % self.token_dir, content)
+    self.assertIn("'registry_url': 'http://%s:%s/'" % (self.options['ipv4'],
+                                                self.options['port']), content)
+  
+  def checkRegistryWrapper(self):
+    path = os.path.join(self.base_dir, 'wrapper')
+    self.assertTrue(os.path.exists(path))
+    content = ""
+    config_file = os.path.join(self.base_dir, 'config')
+    with open(path, 'r') as f:
+        content = f.read()
+    self.assertIn("@%s" % config_file, content)
+  
+  def test_generateCertificates(self):
+    
+    self.options['ipv6-prefix'] = '2001:db8:24::/48'
+    self.options['key-size'] = '2048'
+    
+    recipe = self.new_recipe()
+    
+    recipe.generateCertificate()
+    
+    self.assertTrue(os.path.exists(self.options['key-file']))
+    self.assertTrue(os.path.exists(self.options['cert-file']))
+    
+    last_time = time.ctime(os.stat(self.options['key-file'])[7])
+    
+    recipe.generateCertificate()
+    
+    self.assertTrue(os.path.exists(self.options['key-file']))
+    this_time = time.ctime(os.stat(self.options['key-file'])[7])
+    
+    self.assertEqual(last_time, this_time)
+  
+  def test_generateCertificates_other_ipv6(self):
+    
+    self.options['ipv6-prefix'] = 'be28:db8:fe6a:d85:4fe:54a:ae:aea/64'
+    
+    recipe = self.new_recipe()
+    
+    recipe.generateCertificate()
+    
+    self.assertTrue(os.path.exists(self.options['key-file']))
+    self.assertTrue(os.path.exists(self.options['cert-file']))
+
+  def test_install(self):
+    recipe = self.new_recipe()
+
+    recipe.options.update({
+        'ipv6-prefix': '2001:db8:24::/48',
+        'slave-instance-list': '''[
+            {"slave_reference":"SOFTINST-58770"},
+            {"slave_reference":"SOFTINST-58778"}
+            ]
+            '''
+        })
+
+    try:
+      recipe.install()
+    except (NotFoundError, ConnectionError):
+      # Recipe will raise not found error when trying to publish slave informations
+      pass
+    
+    self.assertItemsEqual(os.listdir(self.ssl_dir),
+                          ['cert.key', 'cert.crt'])
+    
+    token_file = os.path.join(self.options['conf-dir'], 'token.json')
+    self.assertTrue(os.path.exists(token_file))
+    
+    # token file must contain 2 elements
+    token_content = recipe.readFile(token_file)
+    self.assertIn('SOFTINST-58770', token_content)
+    self.assertIn('SOFTINST-58778', token_content)
+    
+    token_dict = recipe.loadJsonFile(token_file)
+    self.assertEqual(len(token_dict), 2)
+    self.assertTrue(token_dict.has_key('SOFTINST-58770'))
+    self.assertTrue(token_dict.has_key('SOFTINST-58778'))
+        
+    self.assertItemsEqual(os.listdir(self.token_dir),
+                          ['SOFTINST-58770.add', 'SOFTINST-58778.add'])
+
+    first_add = recipe.readFile(os.path.join(self.token_dir, 'SOFTINST-58770.add'))
+    self.assertEqual(token_dict['SOFTINST-58770'], first_add)
+    
+    second_add = recipe.readFile(os.path.join(self.token_dir, 'SOFTINST-58778.add'))
+    self.assertEqual(token_dict['SOFTINST-58778'], second_add)
+    
+    self.checkWrapper(os.path.join(self.base_dir, 'manager_wrapper'))
+    self.checkWrapper(os.path.join(self.base_dir, 'drop_wrapper'))
+    self.checkWrapper(os.path.join(self.base_dir, 'check_wrapper'))
+    self.checkRegistryWrapper()
+    
+    # Remove one element
+    recipe.options.update({
+        "slave-instance-list": """[{"slave_reference":"SOFTINST-58770"}]"""
+        })
+    try:
+      recipe.install()
+    except (NotFoundError, ConnectionError):
+      # Recipe will raise not found error when trying to publish slave informations
+      pass
+    token_dict = recipe.loadJsonFile(token_file)
+    
+    self.assertEqual(len(token_dict), 1)
+    self.assertEqual(token_dict['SOFTINST-58770'], first_add)
+    self.assertItemsEqual(os.listdir(self.token_dir),
+                          ['SOFTINST-58770.add', 'SOFTINST-58778.remove'])
+    
+    second_remove = recipe.readFile(os.path.join(self.token_dir, 'SOFTINST-58778.remove'))
+    self.assertEqual(second_add, second_remove)
+  
+  def test_install_empty_slave(self):
+    recipe = self.new_recipe()
+
+    recipe.options.update({
+        'ipv6-prefix': '2001:db8:24::/48'
+        })
+
+    recipe.install()
+    
+    self.assertItemsEqual(os.listdir(self.ssl_dir),
+                          ['cert.key', 'cert.crt'])
+    
+    token_file = os.path.join(self.options['conf-dir'], 'token.json')
+    self.assertTrue(os.path.exists(token_file))
+    
+    token_content = recipe.readFile(token_file)
+    self.assertEqual(token_content, '{}')
+    self.assertItemsEqual(os.listdir(self.options['token-dir']), [])
+    
+    self.checkWrapper(os.path.join(self.base_dir, 'manager_wrapper'))
+    self.checkWrapper(os.path.join(self.base_dir, 'drop_wrapper'))
+    self.checkWrapper(os.path.join(self.base_dir, 'check_wrapper'))
+    self.checkRegistryWrapper()
+

software/apache-frontend/common.cfg

@@ -96,7 +96,7 @@ mode = 640
 [template-apache-frontend-configuration]
 recipe = slapos.recipe.build:download
 url = ${:_profile_base_location_}/templates/apache.conf.in
-md5sum = ce88924c53f09c9a3ef12ec4d8a8ad16
+md5sum = eb509d5b924464b08e28d296da93b58c
 mode = 640
 
 [template-apache-cached-configuration]
@@ -127,13 +127,13 @@ mode = 640
 [template-default-virtualhost]
 recipe = slapos.recipe.build:download
 url = ${:_profile_base_location_}/templates/000.conf.in
-md5sum = b40ffdab93a80b40046e3bbb2f7a58bc
+md5sum = ed1b680e31e30596bf051682ec0270b4
 mode = 640
 
 [template-default-slave-virtualhost]
 recipe = slapos.recipe.build:download
 url = ${:_profile_base_location_}/templates/default-virtualhost.conf.in
-md5sum = 9bd3eda3c2aad5061f6cd6985e6f18d0
+md5sum = 3671d13456cec8c3347e8a6ad0badbff
 mode = 640
 
 [template-log-access]

software/apache-frontend/templates/000.conf.in

@@ -2,7 +2,8 @@
   ServerName www.example.org
   SSLEngine on
   SSLProxyEngine on
-  SSLProtocol ALL -SSLv2
+  SSLProtocol ALL -SSLv2 -SSLv3
+  SSLCipherSuite HIGH:MEDIUM:!aNULL:!MD5:!RC4
 
   # Rewrite part
   ProxyVia On

software/apache-frontend/templates/apache.conf.in

@@ -115,7 +115,8 @@ SSLSessionCache shmcb:/{{ httpd_mod_ssl_cache_directory }}/ssl_scache(512000)
 SSLSessionCacheTimeout  300
 SSLRandomSeed startup /dev/urandom 256
 SSLRandomSeed connect builtin
-SSLProtocol ALL -SSLv2
+SSLProtocol all -SSLv2 -SSLv3
+SSLCipherSuite HIGH:MEDIUM:!aNULL:!MD5:!RC4
 <FilesMatch "\.(cgi|shtml|phtml|php)$">
       SSLOptions +StdEnvVars
 </FilesMatch>

software/apache-frontend/templates/default-virtualhost.conf.in

@@ -6,9 +6,9 @@
 
   SSLEngine on
   SSLProxyEngine on
-  SSLProtocol -ALL +SSLv3 +TLSv1
-  SSLHonorCipherOrder On
-  SSLCipherSuite RC4-SHA:HIGH:!ADH
+  SSLProtocol all -SSLv2 -SSLv3
+  SSLCipherSuite HIGH:MEDIUM:!aNULL:!MD5:!RC4
+
 
 {% set ssl_configuration_list = [('SSLCertificateFile', 'path_to_ssl_crt'),
        			      	 ('SSLCertificateKeyFile', 'path_to_ssl_key'),

software/kvm/common.cfg

@@ -85,7 +85,7 @@ command =
 [template]
 recipe = slapos.recipe.template
 url = ${:_profile_base_location_}/instance.cfg.in
-md5sum = 5fdeb07b7baaf0dfa9219f0d6ba1b140
+md5sum = d2413a9d4978092e939418748585bbb3
 output = ${buildout:directory}/template.cfg
 mode = 0644
 
@@ -93,7 +93,7 @@ mode = 0644
 recipe = hexagonit.recipe.download
 url = ${:_profile_base_location_}/instance-kvm.cfg.jinja2
 mode = 644
-md5sum = 681cd5a4dddceba1e756e9aa409477a6
+md5sum = 5506e1df6ba32c6ead647636ebece79e
 download-only = true
 on-update = true
 
@@ -101,7 +101,7 @@ on-update = true
 recipe = hexagonit.recipe.download
 url = ${:_profile_base_location_}/instance-kvm-cluster.cfg.jinja2.in
 mode = 644
-md5sum = 0d51e71a7967ead2f88e11cc797037a4
+md5sum = 214c46a9aa7605951b8a1f98572dac28
 download-only = true
 on-update = true
 

software/kvm/instance-for-erp5testnode.cfg.in

@@ -22,6 +22,10 @@ test = $${dynamic-template-kvm-resilient-test:rendered}
 frozen = ${instance-frozen:output}
 pull-backup = ${template-pull-backup:output}
 
+# XXX - If this configuration is not generated by slapgrid, use empty values
+[storage-configuration]
+storage-home = 
+
 [slap-configuration]
 recipe = slapos.cookbook:slapconfiguration.serialised
 computer = $${slap-connection:computer-id}
@@ -29,6 +33,7 @@ partition = $${slap-connection:partition-id}
 url = $${slap-connection:server-url}
 key = $${slap-connection:key-file}
 cert = $${slap-connection:cert-file}
+storage-home = $${storage-configuration:storage-home}
 
 [dynamic-template-kvm]
 recipe = slapos.recipe.template:jinja2
@@ -39,6 +44,8 @@ context =
     key develop_eggs_directory buildout:develop-eggs-directory
     key eggs_directory buildout:eggs-directory
     key slapparameter_dict slap-configuration:configuration
+    key storage_dict slap-configuration:storage-dict
+    key tap_network_dict slap-configuration:tap-network-information-dict
     raw curl_executable_location ${curl:location}/bin/curl
     raw dash_executable_location ${dash:location}/bin/dash
     raw dcron_executable_location ${dcron:location}/sbin/crond

software/kvm/instance-kvm-cluster-input-schema.json

 {
   "type": "object",
   "$schema": "http://json-schema.org/draft-04/schema",
-
   "title": "Input Parameters",
   "properties": {
     "frontend": {
+      "title": "KVM frontend (web socket)",
+      "description": "Front end used to provide VNC.",
+      "properties": {
         "frontend-instance-guid": {
           "title": "Frontend Instance ID",
           "description": "Unique identifier of the frontend instance, like \"SOFTINST-11031\".",
@@ -23,10 +25,14 @@
           "type": "string",
           "format": "uri",
           "default": "http://git.erp5.org/gitweb/slapos.git/blob_plain/refs/tags/slapos-0.92:/software/kvm/software.cfg"
+        }
       },
       "type": "object"
     },
     "slave-frontend": {
+      "title": "Web frontend",
+      "description": "Front end used to provide web access for internal services at the kvm.",
+      "properties": {
         "instance-guid": {
           "title": "Main Frontend Instance ID",
           "description": "Unique identifier of the frontend instance, like \"SOFTINST-11031\".",
@@ -47,7 +53,8 @@
           "default": "http://git.erp5.org/gitweb/slapos.git/blob_plain/HEAD:/software/apache-frontend/software.cfg"
         },
         "slave-frontend-dict": {
-        "description": "kvm instances definition",
+          "title": "Slave Front end definition",
+          "description": "Slave Front end definition",
           "patternProperties": {
             ".*": {
               "properties": {
@@ -55,7 +62,6 @@
                   "title": "Name of the domain to be used",
                   "description": "Name of the domain to be used (example: mydomain.com). Subdomains of this domain will be used for the slave instances (example: instance12345.mydomain.com).",
                   "type": "string",
-                  "format": "uri",
                   "default": ""
                 },
                 "url": {
@@ -96,12 +102,9 @@
                   "default": ""
                 },
                 "service-port": {
-                  "allOf": [{
-                    "$ref": "#/definitions/tcpv4port"
-                  }, {
                   "title": "Specify the port of service to run in VM.",
-                    "description": "This will allow to get URL from defined nat-rules. The port should exist in nat-rules of KVM referenced by 'kvm-partition-name'"
-                  }]
+                  "description": "This will allow to get URL from defined nat-rules. The port should exist in nat-rules of KVM referenced by 'kvm-partition-name'",
+                  "type": "number"
                 },
                 "url-scheme": {
                   "title": "Say If HTTP service to run into the Virtual Machine will use http or https.",
@@ -114,10 +117,12 @@
             }
           },
          "type": "object"
+        }
       },
       "type": "object"
     },
     "kvm-partition-dict": {
+      "title": "kvm instances definition",
       "description": "kvm instances definition",
       "patternProperties": {
         ".*": {
@@ -144,9 +149,16 @@
               "description": "Type of QEMU disk drive.",
               "type": "string",
               "default": "virtio",
-                "enum": ["ide", "scsi", "sd", "mtd", "floppy", "pflash", "virtio"]
+              "enum": [
+                "ide",
+                "scsi",
+                "sd",
+                "mtd",
+                "floppy",
+                "pflash",
+                "virtio"
+              ]
             },
-          
             "cpu-count": {
               "title": "CPU count",
               "description": "Number of CPU cores.",
@@ -154,12 +166,15 @@
               "minimum": 1,
               "maximum": 8
             },
-          
             "nbd-host": {
               "title": "NBD hostname",
               "description": "hostname (or IP) of the NBD server containing the boot image.",
               "type": "string",
-                "format": ["host-name", "ip-address", "ipv6"],
+              "format": [
+                "host-name",
+                "ip-address",
+                "ipv6"
+              ],
               "default": "debian.nbd.vifib.net"
             },
             "nbd-port": {
@@ -170,12 +185,15 @@
               "minimum": 1,
               "maximum": 65535
             },
-          
             "nbd2-host": {
               "title": "Second NBD hostname",
               "description": "hostname (or IP) of the second NBD server (containing drivers for example).",
               "type": "string",
-                "format": ["host-name", "ip-address", "ipv6"]
+              "format": [
+                "host-name",
+                "ip-address",
+                "ipv6"
+              ]
             },
             "nbd2-port": {
               "title": "Second NBD port",
@@ -184,7 +202,6 @@
               "minimum": 1,
               "maximum": 65535
             },
-          
             "virtual-hard-drive-url": {
               "title": "Existing disk image URL",
               "description": "If specified, will download an existing disk image (qcow2, raw, ...), and will use it as main virtual hard drive. Can be used to download and use an already installed and customized virtual hard drive.",
@@ -202,18 +219,43 @@
               "type": "boolean",
               "default": false
             },
-          
+            "external-disk-number": {
+              "title": "Number of additional disk to create for virtual machine",
+              "description": "Specify the number of additional disk to create for virtual machine in data folder of SlapOS Node. Requires instance_storage_home to be configured on SlapOS Node.",
+              "type": "integer",
+              "minimum": 0,
+              "maximum": 4,
+              "default": 0
+            },
+            "external-disk-size": {
+              "title": "Number of additional disk to create for virtual machine, in Gigabytes",
+              "description": "Specify the number of additional disk to create for virtual machine in data folder of SlapOS Node. Requires instance_storage_home to be configured on SlapOS Node.",
+              "type": "integer",
+              "minimum": 10,
+              "maximum": 100,
+              "default": 20
+            },
             "use-tap": {
               "title": "Use QEMU TAP network interface",
-                "description": "Use QEMU TAP network interface, requires a bridge on SlapOS Node. If false, use user-mode network stack (NAT).",
+              "description": "Use QEMU TAP network interface, might require a bridge on SlapOS Node.",
               "type": "boolean",
               "default": false
             },
+            "use-nat": {
+              "title": "Use QEMU USER Mode networking",
+              "description": "Use QEMU user-mode network stack (NAT).",
+              "type": "boolean",
+              "default": true
+            },
             "nat-rules": {
               "title": "List of rules for NAT of QEMU user mode network stack.",
               "description": "List of rules for NAT of QEMU user mode network stack, as comma-separated list of ports. For each port specified, it will redirect port x of the VM (example: 80) to the port x + 10000 of the public IPv6 (example: 10080). Defaults to \"22 80 443\". Ignored if \"use-tap\" parameter is enabled.",
               "type": "array",
-                "default": [22, 80, 443]
+              "default": [
+                22,
+                80,
+                443
+              ]
             }
           },
           "type": "object"

software/kvm/instance-kvm-cluster.cfg.jinja2.in

@@ -47,6 +47,8 @@ config-use-tap = {{ dumps(kvm_parameter_dict.get('use-tap', False)) }}
 config-virtual-hard-drive-url = {{ dumps(kvm_parameter_dict.get('virtual-hard-drive-url', '')) }}
 config-virtual-hard-drive-md5sum = {{ dumps(kvm_parameter_dict.get('virtual-hard-drive-md5sum', '')) }}
 config-virtual-hard-drive-gzipped = {{ dumps(kvm_parameter_dict.get('virtual-hard-drive-gzipped', False)) }}
+config-external-disk-number = {{ dumps(kvm_parameter_dict.get('external-disk-number', 0)) }}
+config-external-disk-size = {{ dumps(kvm_parameter_dict.get('external-disk-size', 20)) }}
 return = 
   backend-url
   url
@@ -110,6 +112,8 @@ recipe = slapos.cookbook:publish
 {% for name, value in publish_dict.items() -%}
 {{   name }} = {{ value }}
 {% endfor %}
+{% set disk_number = len(storage_dict) -%}
+1_info = It is possible to mount up to {{ disk_number }} external disk to your virtual machine. See parameter 'external-disk-number'
 
 [buildout]
 parts = publish

software/kvm/instance-kvm-input-schema.json

@@ -78,13 +78,42 @@
       "description": "MD5 checksum of virtual hard drive, used if virtual-hard-drive-url is specified.",
       "type": "string"
     },
+    "virtual-hard-drive-gzipped": {
+      "title": "Define if virtual hard drive to download is gzipped",
+      "description": "Define if virtual hard drive to download is gzipped using gzip. This help to reduce size of file to download.",
+      "type": "boolean",
+      "default": false
+    },
+
+    "external-disk-number": {
+      "title": "Number of additional disk to create for virtual machine",
+      "description": "Specify the number of additional disk to create for virtual machine in data folder of SlapOS Node. Requires instance_storage_home to be configured on SlapOS Node.",
+      "type": "integer",
+      "minimum": 0,
+      "maximum": 4,
+      "default": 0
+    },
+    "external-disk-size": {
+      "title": "Number of additional disk to create for virtual machine, in Gigabytes",
+      "description": "Specify the number of additional disk to create for virtual machine in data folder of SlapOS Node. Requires instance_storage_home to be configured on SlapOS Node.",
+      "type": "integer",
+      "minimum": 10,
+      "maximum": 100,
+      "default": 20
+    },
 
     "use-tap": {
       "title": "Use QEMU TAP network interface",
-      "description": "Use QEMU TAP network interface, requires a bridge on SlapOS Node. If false, use user-mode network stack (NAT).",
+      "description": "Use QEMU TAP network interface, might require a bridge on SlapOS Node.",
       "type": "boolean",
       "default": false
     },
+    "use-nat": {
+      "title": "Use QEMU USER Mode networking",
+      "description": "Use QEMU user-mode network stack (NAT).",
+      "type": "boolean",
+      "default": true
+    },
     "nat-rules": {
       "title": "List of rules for NAT of QEMU user mode network stack.",
       "description": "List of rules for NAT of QEMU user mode network stack, as comma-separated list of ports. For each port specified, it will redirect port x of the VM (example: 80) to the port x + 10000 of the public IPv6 (example: 10080). Defaults to \"22 80 443\". Ignored if \"use-tap\" parameter is enabled.",

software/kvm/instance-kvm-resilient-input-schema.json

 {
     "type": "object",
     "$schema": "http://json-schema.org/draft-04/schema",
-    "items": {
     "allOf": [
         {
-                "$ref": "instance-kvm-input-schema.json"
-            }
-        ],
-        "title": "Input Parameters",
+            "$ref": "instance-kvm-input-schema.json#/"
+        },
+        {
             "properties": {
                 "-sla-0-computer_guid": {
                     "title": "Target computer for main instance",
                     "description": "Target computer GUID for main instance.",
-                "type": "string"
+                    "type": "string",
+                    "optional": true
                 },
                 "-sla-1-computer_guid": {
                     "title": "Target computer for first clone",
                     "description": "Target computer for first clone and PBS.",
-                "type": "string"
+                    "type": "string",
+                    "optional": true
                 },
                 "-sla-2-computer_guid": {
                     "title": "Target computer for second clone",
                     "description": "Target computer for second clone and PBS.",
-                "type": "string"
+                    "type": "string",
+                    "optional": true
                 },
                 "resiliency-backup-periodicity": {
                     "title": "Periodicity of backup",
                     "description": "Periodicity of backup, in cron format.",
-                "type": "string"
+                    "type": "string",
+                    "optional": true
                 },
                 "remove-backup-older-than": {
                     "title": "Remove backups older than...",
                     "description": "Remove all the backups in PBS that are older than specified value. It should be rdiff-backup-compatible.",
                     "type": "string",
-                "default": "3B"
+                    "default": "3B",
+                    "optional": true
                 },
                 "resilient-clone-number": {
                     "title": "Amount of backup(s) to create",
                     "description": "Amount of backup(s) to create. Each backup consists of a Pull Backup Server and a clone.",
                     "type": "integer",
-                "default": 2
+                    "default": 2,
+                    "optional": true
                 },
                 "ignore-known-hosts-file": {
                     "title": "Ignore known_hosts file",
                     "description": "Set either to fill known_hosts file for ssh or not. Useful if main instance and PBS are using the same IP (slapos proxy, webrunner).",
                     "type": "boolean",
-                "default": false
+                    "default": false,
+                    "optional": true
                 }
             }
         }
+    ]
 }

software/kvm/instance-kvm.cfg.jinja2

@@ -103,6 +103,13 @@ qemu-path =  {{ qemu_executable_location }}
 qemu-img-path = {{ qemu_img_executable_location }}
 6tunnel-path = {{ sixtunnel_executable_location }}
 
+etc-directory = ${directory:etc}
+disk-storage-list = 
+{% for key, path in storage_dict.items() -%}
+{{ '  ' ~ key ~ ' ' ~ path }}
+{% endfor -%}
+external-disk-number = ${slap-parameter:external-disk-number}
+external-disk-size = ${slap-parameter:external-disk-size}
 
 [kvm-vnc-promise]
 recipe = slapos.cookbook:check_port_listening
@@ -230,7 +237,9 @@ recipe = slapos.cookbook:publish
 ipv6 = ${slap-network-information:global-ipv6}
 backend-url = https://[${novnc-instance:ip}]:${novnc-instance:port}/vnc_auto.html?host=[${novnc-instance:ip}]&port=${novnc-instance:port}&encrypt=1&password=${kvm-instance:vnc-passwd}
 url = ${request-slave-frontend:connection-url}/vnc_auto.html?host=${request-slave-frontend:connection-domainname}&port=${request-slave-frontend:connection-port}&encrypt=1&path=${request-slave-frontend:connection-resource}&password=${kvm-instance:vnc-passwd}
+{% set iface = 'eth0' -%}
 {% if slapparameter_dict.get('use-nat', 'True') == 'True' -%}
+{%   set iface = 'eth1' -%}
 # Publish NAT port mapping status
 # XXX: hardcoded value from [slap-parameter]
 {%   set nat_rule_list = slapparameter_dict.get('nat-rules', '22 80 443') %}
@@ -242,6 +251,12 @@ nat-rule-url-{{port}} = [${slap-network-information:global-ipv6}]:{{external_por
 {%     endif -%}
 {%   endfor -%}
 {% endif -%}
+{% if slapparameter_dict.get('use-tap', 'False') == 'True' and tap_network_dict.has_key('ipv4') -%}
+1_info = Use these configurations below to configure interface {{ iface }} in your VM.
+2_info = ifconfig {{ iface }} ${slap-network-information:tap-ipv4} netmask ${slap-network-information:tap-netmask}
+3_info = route add -host ${slap-network-information:tap-gateway} dev {{ iface }}
+4_info = route add -net ${slap-network-information:tap-network} netmask ${slap-network-information:tap-netmask} gw ${slap-network-information:tap-gateway}
+{% endif -%}
 
 
 [slap-parameter]
@@ -268,3 +283,6 @@ use-tap = False
 virtual-hard-drive-url =
 virtual-hard-drive-md5sum =
 virtual-hard-drive-gzipped = False
+
+external-disk-number = 0
+external-disk-size = 20

software/kvm/instance.cfg.in

@@ -23,6 +23,10 @@ test = $${dynamic-template-kvm-resilient-test:rendered}
 frozen = ${instance-frozen:output}
 pull-backup = ${template-pull-backup:output}
 
+# XXX - If this configuration is not generated by slapgrid, use empty values
+[storage-configuration]
+storage-home = 
+
 [slap-configuration]
 recipe = slapos.cookbook:slapconfiguration.serialised
 computer = $${slap-connection:computer-id}
@@ -30,6 +34,7 @@ partition = $${slap-connection:partition-id}
 url = $${slap-connection:server-url}
 key = $${slap-connection:key-file}
 cert = $${slap-connection:cert-file}
+storage-home = $${storage-configuration:storage-home}
 
 [jinja2-template-base]
 recipe = slapos.recipe.template:jinja2
@@ -42,6 +47,8 @@ context =
     key eggs_directory buildout:eggs-directory
     key ipv4 slap-configuration:ipv4
     key ipv6 slap-configuration:ipv6
+    key tap_network_dict slap-configuration:tap-network-information-dict
+    key storage_dict slap-configuration:storage-dict
     key slapparameter_dict slap-configuration:configuration
     key computer_id slap-configuration:computer
     $${:extra-context}
@@ -64,6 +71,8 @@ context =
     key develop_eggs_directory buildout:develop-eggs-directory
     key eggs_directory buildout:eggs-directory
     key slapparameter_dict slap-configuration:configuration
+    key storage_dict slap-configuration:storage-dict
+    key tap_network_dict slap-configuration:tap-network-information-dict
     raw curl_executable_location ${curl:location}/bin/curl
     raw dash_executable_location ${dash:location}/bin/dash
     raw dcron_executable_location ${dcron:location}/sbin/crond

software/kvm/software-for-erp5testnode.cfg

@@ -5,6 +5,6 @@ extends = development.cfg
 [template]
 recipe = slapos.recipe.template
 url = ${:_profile_base_location_}/instance-for-erp5testnode.cfg.in
-md5sum = e8afd5aa5b41df79238e9a84984a6aa5
+md5sum = 5883432c9a004cf505db2718c596ce6a
 output = ${buildout:directory}/template.cfg
 mode = 0644
\ No newline at end of file

software/kvm/software.cfg.json

+{
+    "name": "KVM",
+    "description": "KVM",
+    "serialisation": "xml",
+    "software-type": {
+        "default": {
+            "title": "Default",
+            "description": "Standalone KVM",
+            "request": "instance-kvm-input-schema.json",
+            "response": "instance-kvm-output-schema.json",
+            "index": 0
+        },
+        "resilient": {
+            "title": "Resilient",
+            "description": "Resilient KVM",
+            "request": "instance-kvm-resilient-input-schema.json",
+            "response": "instance-kvm-output-schema.json",
+            "index": 1
+        },
+        "cluster": {
+            "title": "Cluster",
+            "description": "Cluster KVM",
+            "serialisation": "json-in-xml",
+            "request": "instance-kvm-cluster-input-schema.json",
+            "response": "instance-kvm-output-schema.json",
+            "index": 2
+        }
+    }
+}

software/re6stnet/apache.conf.in

+LoadModule unixd_module modules/mod_unixd.so
+LoadModule access_compat_module modules/mod_access_compat.so
+LoadModule authz_core_module modules/mod_authz_core.so
+LoadModule authz_host_module modules/mod_authz_host.so
+LoadModule log_config_module modules/mod_log_config.so
+LoadModule setenvif_module modules/mod_setenvif.so
+LoadModule version_module modules/mod_version.so
+LoadModule proxy_module modules/mod_proxy.so
+LoadModule proxy_http_module modules/mod_proxy_http.so
+LoadModule socache_shmcb_module modules/mod_socache_shmcb.so
+LoadModule ssl_module modules/mod_ssl.so
+LoadModule mime_module modules/mod_mime.so
+#LoadModule dav_module modules/mod_dav.so
+#LoadModule dav_fs_module modules/mod_dav_fs.so
+LoadModule negotiation_module modules/mod_negotiation.so
+LoadModule rewrite_module modules/mod_rewrite.so
+LoadModule headers_module modules/mod_headers.so
+
+PidFile "{{ pid_file }}"
+ServerAdmin admin@
+TypesConfig conf/mime.types
+AddType application/x-compress .Z
+AddType application/x-gzip .gz .tgz
+
+ServerTokens Prod
+ServerSignature Off
+TraceEnable Off
+
+ErrorLog "{{ error_log }}"
+# Default apache log format with request time in microsecond at the end
+LogFormat "%h %l %u %t \"%r\" %>s %b \"%{Referer}i\" \"%{User-Agent}i\" %D" combined
+CustomLog "{{ access_log }}" combined
+
+{% if uri_scheme == 'https'  -%}
+# SSL Configuration
+SSLCertificateFile {{ certificate }}
+SSLCertificateKeyFile {{ key }}
+SSLRandomSeed startup builtin
+SSLRandomSeed connect builtin
+SSLProtocol ALL -SSLv2
+{% endif -%}
+
+<Directory />
+  Options FollowSymLinks
+  AllowOverride None
+  Allow from all
+</Directory>
+
+Listen {{ ipv6 }}:{{ apache_port }}
+<VirtualHost *:{{ apache_port }}>
+{% if uri_scheme == 'https'  -%}
+  SSLEngine On
+  SSLProxyEngine On
+{% endif -%}
+
+  ProxyPass / {{ uri_scheme }}://{{ re6st_ipv4 }}:{{ re6st_port }}/
+</VirtualHost>
\ No newline at end of file

software/re6stnet/instance-logrotate-base.cfg.in

+[buildout]
+parts =
+  cron-entry-logrotate
+
+[cron]
+recipe = slapos.cookbook:cron
+cron-entries = ${logrotate-directory:cron-entries}
+dcrond-binary = {{ dcron_location }}/sbin/crond
+crontabs = ${logrotate-directory:crontabs}
+cronstamps = ${logrotate-directory:cronstamps}
+catcher = ${cron-simplelogger:wrapper}
+binary = ${logrotate-directory:services}/crond
+
+[cron-simplelogger]
+recipe = slapos.cookbook:simplelogger
+wrapper = ${logrotate-directory:bin}/cron_simplelogger
+log = ${logrotate-directory:log}/cron.log
+
+[logrotate]
+recipe = slapos.cookbook:logrotate
+logrotate-entries = ${logrotate-directory:logrotate-entries}
+backup = ${logrotate-directory:logrotate-backup}
+logrotate-binary = {{ logrotate_location }}/usr/sbin/logrotate
+gzip-binary = {{ gzip_location }}/bin/gzip
+gunzip-binary = {{ gzip_location }}/bin/gunzip
+wrapper = ${logrotate-directory:bin}/logrotate
+conf = ${logrotate-directory:etc}/logrotate.conf
+state-file = ${logrotate-directory:srv}/logrotate.status
+
+[cron-entry-logrotate]
+recipe = slapos.cookbook:cron.d
+cron-entries = ${cron:cron-entries}
+name = logrotate
+frequency = 0 0 * * *
+command = ${logrotate:wrapper}
+
+[logrotate-directory]
+recipe = slapos.cookbook:mkdirectory
+cron-entries = ${:etc}/cron.d
+cronstamps = ${:etc}/cronstamps
+crontabs = ${:etc}/crontabs
+logrotate-backup = ${:backup}/logrotate
+logrotate-entries = ${:etc}/logrotate.d
+bin = ${buildout:directory}/bin
+srv = ${buildout:directory}/srv
+backup = ${:srv}/backup
+etc = ${buildout:directory}/etc
+services = ${:etc}/service
+log = ${buildout:directory}/var/log
+

software/re6stnet/instance-re6stnet-input-schema.json

+{
+    "$schema": "http://json-schema.org/draft-04/schema#",
+    "properties": {
+        "ipv6-prefix": {
+            "title": "Ipv6 prefix to use to setup the new re6st network",
+            "description": "Prefix ipv6 used by re6st to setup network. It is something like 2001:db8:42::/48",
+            "type": "string"
+        },
+        "key-size": {
+            "title": "Number of bit to use for certificate generation",
+            "description": "Specify the size of certificate generated by re6st. by default, generate 2048-bit key length",
+            "type": "integer",
+            "minimum": 1024,
+            "default": 2048
+        }
+    }
+}
\ No newline at end of file

software/re6stnet/instance-re6stnet-output-schema.json

+{
+  "$schema": "http://json-schema.org/draft-04/schema#",
+  "description": "Values returned by Re6st Master instanciation",
+  "properties": {
+    "re6stry-url": {
+      "description": "ipv6 url to access your re6st registry service",
+      "type": "string"
+    }
+  },
+  "type": "object"
+}
\ No newline at end of file

software/re6stnet/instance-re6stnet.cfg.in

+
+{% set python_bin = parameter_dict['python-executable'] -%}
+{% set re6st_registry = parameter_dict['re6st-registry'] -%}
+{% set publish_dict = {} -%}
+{% set part_list = [] -%}
+{% set ipv6 = (ipv6_set | list)[0] -%}
+{% set ipv4 = (ipv4_set | list)[0] -%}
+{% set uri_scheme = slapparameter_dict.get('uri-scheme', 'http') -%}
+{% macro section(name) %}{% do part_list.append(name) %}{{ name }}{% endmacro -%}
+
+[directory]
+recipe = slapos.cookbook:mkdirectory
+bin = ${buildout:directory}/bin
+etc = ${buildout:directory}/etc
+srv = ${buildout:directory}/srv
+var = ${buildout:directory}/var
+log = ${:var}/log
+services = ${:etc}/service
+script = ${:etc}/run
+promises = ${:etc}/promise
+run = ${:var}/run
+ca-dir = ${:etc}/ssl
+requests = ${:ca-dir}/requests
+private = ${:ca-dir}/private
+certs = ${:ca-dir}/certs
+newcerts = ${:ca-dir}/newcerts
+crl = ${:ca-dir}/crl
+re6st = ${:srv}/res6stnet
+
+[re6stnet-dirs]
+recipe = slapos.cookbook:mkdirectory
+registry = ${directory:re6st}/registry
+log = ${directory:log}/re6stnet
+conf = ${directory:etc}/re6stnet
+ssl = ${:conf}/ssl
+token = ${:conf}/token
+
+[certificate-authority]
+recipe = slapos.cookbook:certificate_authority
+openssl-binary = {{ openssl_bin }}/openssl
+ca-dir = ${directory:ca-dir}
+requests-directory = ${directory:requests}
+wrapper = ${directory:services}/certificate_authority
+ca-private = ${directory:private}
+ca-certs = ${directory:certs}
+ca-newcerts = ${directory:newcerts}
+ca-crl = ${directory:crl}
+
+
+[apache-conf]
+recipe = slapos.recipe.template:jinja2
+template = {{ parameter_dict['template-apache-conf'] }}
+rendered = ${directory:etc}/apache.conf
+ipv6 = {{ ipv6 }}
+port = 9026
+error-log = ${directory:log}/apache-error.log
+access-log = ${directory:log}/apache-access.log
+pid-file = ${directory:run}/apache.pid
+context = 
+  key apache_port :port
+  key re6st_ipv4 re6st-registry:ipv4
+  key re6st_port re6st-registry:port
+  key access_log :access-log
+  key error_log :error-log
+  key pid_file :pid-file
+  raw certificate ${directory:certs}/apache.crt
+  raw key ${directory:private}/apache.key
+  raw ipv6 {{ ipv6 }}
+  raw uri_scheme {{ uri_scheme }}
+
+{% set apache_wrapper = '${directory:services}/httpd' -%}
+{% if uri_scheme == 'https' -%}
+{% set apache_wrapper = '${directory:bin}/httpd_raw' -%}
+{% endif -%}
+[apache-httpd]
+recipe = slapos.cookbook:wrapper
+wrapper-path = {{ apache_wrapper }}
+command-line = "{{ parameter_dict['apache-location'] }}/bin/httpd" -f "${apache-conf:rendered}" -DFOREGROUND
+
+{% if uri_scheme == 'https' %}
+[{{ section('apache-ca') }}]
+<= certificate-authority
+recipe = slapos.cookbook:certificate_authority.request
+executable = ${apache-httpd:wrapper-path}
+wrapper = ${directory:services}/httpd
+key-file = ${certificate-authority:ca-private}/apache.key
+cert-file = ${certificate-authority:ca-certs}/apache.crt
+{% endif %}
+
+[logrotate-apache]
+< = logrotate-entry-base
+name = apache
+log = ${apache-conf:error-log} ${apache-conf:access-log}
+post = {{ parameter_dict['bin-directory'] }}/slapos-kill --pidfile ${apache-conf:pid-file} -s USR1
+
+[logrotate-entry-base]
+recipe = slapos.cookbook:logrotate.d
+logrotate-entries = ${logrotate:logrotate-entries}
+backup = ${logrotate:backup}
+
+[re6st-registry-conf-dict]
+port = 9201
+ipv4 = {{ ipv4 }}
+ipv6 = {{ ipv6 }}
+db = ${re6stnet-dirs:registry}/registry.db
+ca = ${re6stnet-dirs:ssl}/re6stnet.crt
+key = ${re6stnet-dirs:ssl}/re6stnet.key
+mailhost = 127.0.0.1
+prefix-length = 16
+anonymous-prefix-length = 32
+logfile = ${re6stnet-dirs:log}/registry.log
+verbose = 2
+
+[re6st-registry-conf]
+recipe = slapos.recipe.template:jinja2
+template = {{ parameter_dict['template-re6st-registry-conf'] }}
+rendered = ${directory:etc}/re6st-registry.conf
+context = section parameter_dict re6st-registry-conf-dict
+
+[re6st-registry]
+recipe = slapos.cookbook:re6stnet.registry
+port = ${re6st-registry-conf-dict:port}
+ipv4 = ${re6st-registry-conf-dict:ipv4}
+command = {{ re6st_registry }}
+config-file = ${re6st-registry-conf:rendered}
+db-path = ${re6st-registry-conf-dict:db}
+wrapper = ${directory:services}/re6st-registry
+manager-wrapper = ${directory:bin}/re6stManageToken
+check-service-wrapper = ${directory:bin}/re6stCheckService
+drop-service-wrapper = ${directory:bin}/re6stManageDeleteToken
+key-file = ${re6st-registry-conf-dict:key}
+cert-file = ${re6st-registry-conf-dict:ca}
+openssl-bin = {{ openssl_bin }}/openssl
+python-bin = {{ python_bin }}
+ipv6-prefix = {{ slapparameter_dict.get('ipv6-prefix', '2001:db8:24::/48') }}
+key-size = {{ slapparameter_dict.get('key-size', 2048) }}
+conf-dir = ${re6stnet-dirs:conf}
+token-dir = ${re6stnet-dirs:token}
+
+slave-instance-list = ${slap-parameter:slave_instance_list}
+
+environment = 
+  PATH={{ openssl_bin }}
+
+[re6stnet-manage]
+recipe = slapos.cookbook:wrapper
+wrapper-path = ${directory:script}/re6st-token-manager
+command-line = "{{ python_bin }}" ${re6st-registry:manager-wrapper}
+
+[cron-entry-re6st-check]
+recipe = slapos.cookbook:cron.d
+cron-entries = ${cron:cron-entries}
+name = re6stnet-check-token
+frequency = 0 */1 * * *
+command = {{ python_bin }} ${re6st-registry:check-service-wrapper}
+
+[cron-entry-re6st-drop]
+recipe = slapos.cookbook:cron.d
+cron-entries = ${cron:cron-entries}
+name = re6stnet-drop-token
+frequency = */30 * * * *
+command = {{ python_bin }} ${re6st-registry:drop-service-wrapper}
+
+[logrotate-entry-re6stnet]
+< = logrotate-entry-base
+name = re6stnet
+log = ${re6st-registry-conf-dict:logfile}
+
+[re6st-registry-promise]
+recipe = slapos.cookbook:check_port_listening
+path = ${directory:promises}/re6st-registry
+hostname = ${re6st-registry:ipv4}
+port = ${re6st-registry:port}
+
+[apache-registry-promise]
+recipe = slapos.cookbook:check_port_listening
+path = ${directory:promises}/apache-re6st-registry
+hostname = ${apache-conf:ipv6}
+port = ${apache-conf:port}
+
+{% do publish_dict.__setitem__('re6stry-url', uri_scheme ~ '://[${apache-conf:ipv6}]:${apache-conf:port}') -%}
+[publish]
+recipe = slapos.cookbook:publish
+{% for name, value in publish_dict.items() -%}
+{{   name }} = {{ value }}
+{% endfor -%}
+
+[buildout]
+extends =
+  {{ logrotate_cfg }}
+
+parts = 
+  certificate-authority
+  logrotate-apache
+  logrotate-entry-re6stnet
+  re6stnet-manage
+  cron-entry-logrotate
+  cron-entry-re6st-check
+  cron-entry-re6st-drop
+  apache-httpd
+  publish
+  
+  re6st-registry-promise
+  apache-registry-promise
+# Complete parts with sections
+  {{ part_list | join('\n  ') }}
+
+eggs-directory = {{ eggs_directory }}
+develop-eggs-directory = {{ develop_eggs_directory }}
+offline = true
+
+[slap-parameter]
+slave_instance_list = {}
+

software/re6stnet/instance.cfg.in

+[buildout]
+parts = switch-softwaretype
+eggs-directory = {{ eggs_directory }}
+develop-eggs-directory = {{ develop_eggs_directory }}
+
+[slap-configuration]
+recipe = slapos.cookbook:slapconfiguration.serialised
+computer = ${slap-connection:computer-id}
+partition = ${slap-connection:partition-id}
+url = ${slap-connection:server-url}
+key = ${slap-connection:key-file}
+cert = ${slap-connection:cert-file}
+
+[jinja2-template-base]
+recipe = slapos.recipe.template:jinja2
+rendered = ${buildout:parts-directory}/${:_buildout_section_name_}/${:filename}
+extra-context =
+context =
+    key develop_eggs_directory buildout:develop-eggs-directory
+    key eggs_directory buildout:eggs-directory
+    key ipv6_set slap-configuration:ipv6
+    key ipv4_set slap-configuration:ipv4
+    key slapparameter_dict slap-configuration:configuration
+    key computer_id slap-configuration:computer
+    raw logrotate_cfg {{ template_logrotate_base }}
+    raw dash_binary {{ dash_location }}/bin/dash
+    raw openssl_bin {{ openssl_location}}/bin
+    ${:extra-context}
+
+[dynamic-template-re6stnet-parameters]
+bin-directory = {{ bin_directory }}
+python-executable  = {{ python_with_eggs }}
+re6st-registry = {{ re6stnet_registry }}
+template-apache-conf = {{ template_apache_conf }}
+apache-location = {{ apache_location }}
+template-re6st-registry-conf = {{ template_re6st_registry_conf }}
+
+[dynamic-template-re6stnet]
+< = jinja2-template-base
+template = {{ template_re6stnet }}
+filename = instance-re6stnet.cfg
+extensions = jinja2.ext.do
+extra-context =
+    section parameter_dict dynamic-template-re6stnet-parameters
+
+[switch-softwaretype]
+recipe = slapos.cookbook:softwaretype
+
+default = ${dynamic-template-re6stnet:rendered}
+registry = ${:default}

software/re6stnet/re6st-registry.conf.in

+port  {{ parameter_dict['port'] }}
+4  {{ parameter_dict['ipv4'] }}
+6  {{ parameter_dict['ipv6'] }}
+db  {{ parameter_dict['db'] }}
+ca  {{ parameter_dict['ca'] }}
+key  {{ parameter_dict['key'] }}
+mailhost {{ parameter_dict['mailhost'] }}
+prefix-length  {{ parameter_dict['prefix-length'] }}
+anonymous-prefix-length  {{ parameter_dict['anonymous-prefix-length'] }}
+logfile  {{ parameter_dict['logfile'] }}
+verbose  {{ parameter_dict['verbose'] }}
\ No newline at end of file

software/re6stnet/software.cfg

+[buildout]
+
+extends =
+  ../../component/re6stnet/buildout.cfg
+  ../../component/dash/buildout.cfg
+  ../../component/git/buildout.cfg
+  ../../component/dcron/buildout.cfg
+  ../../component/gzip/buildout.cfg
+  ../../component/openssl/buildout.cfg
+  ../../component/logrotate/buildout.cfg
+  ../../component/apache/buildout.cfg
+  ../../stack/slapos.cfg
+
+develop = 
+  ${:parts-directory}/re6stnet-repository
+  ${:parts-directory}/slapos.cookbook-repository
+
+parts = 
+  slapos-cookbook
+  eggs
+  dash
+  babeld
+  re6stnet-develop
+  re6stnet
+  template
+  slapos.cookbook-repository
+  check-recipe
+
+[eggs]
+recipe = zc.recipe.egg
+eggs =
+  ${lxml-python:egg}
+  slapos.toolbox
+scripts =
+  slapos-kill
+
+[extra-eggs]
+recipe = zc.recipe.egg
+interpreter = pythonwitheggs
+eggs =
+  ${lxml-python:egg}
+  ${python-cffi:egg}
+  ${python-cryptography:egg}
+  pyOpenSSL
+  miniupnpc
+  re6stnet
+
+[re6stnet-repository]
+repository = http://git.erp5.org/repos/re6stnet.git
+branch = re6st-slapos
+
+[slapos.cookbook-repository]
+recipe = slapos.recipe.build:gitclone
+repository = http://git.erp5.org/repos/slapos.git
+branch = re6st-master
+git-executable = ${git:location}/bin/git
+
+[download-base]
+recipe = slapos.recipe.build:download
+url = ${:_profile_base_location_}/${:filename}
+mode = 644
+
+[template-jinja2-base]
+recipe = slapos.recipe.template:jinja2
+template = ${:_profile_base_location_}/${:filename}.in
+rendered = ${buildout:directory}/${:filename}
+# XXX: extra-context is needed because we cannot append to a key of an extended
+# section.
+extra-context =
+context =
+    key bin_directory buildout:bin-directory
+    key develop_eggs_directory buildout:develop-eggs-directory
+    key eggs_directory buildout:eggs-directory
+    ${:extra-context}
+
+
+[template]
+< = template-jinja2-base
+filename = template.cfg
+template = ${:_profile_base_location_}/instance.cfg.in
+md5sum = 0929cf851c4883bcb5c69fc2f918eaeb
+extra-context =
+    key apache_location apache:location
+    key dash_location dash:location
+    key logrotate_location logrotate:location
+    key openssl_location openssl:location
+    key template_apache_conf template-apache-conf:target
+    key template_re6stnet template-re6stnet:target
+    key template_re6st_registry_conf template-re6st-registry-conf:target
+    key template_logrotate_base template-logrotate-base:rendered
+    raw python_with_eggs ${buildout:directory}/bin/${extra-eggs:interpreter}
+    raw re6stnet_registry ${buildout:directory}/bin/re6st-registry
+
+[template-re6stnet]
+< = download-base
+filename = instance-re6stnet.cfg.in
+md5sum = e088fb05ea6e1ceff8a5ac00fd28bd75
+
+[template-logrotate-base]
+< = template-jinja2-base
+filename = instance-logrotate-base.cfg
+md5sum = f28fbd310944f321ccb34b2a34c82005
+extra-context =
+    key dcron_location dcron:location
+    key gzip_location gzip:location
+    key logrotate_location logrotate:location
+
+[template-apache-conf]
+< = download-base
+filename = apache.conf.in
+md5sum = c220229ee37866c8cc404d602edd389d
+
+[template-re6st-registry-conf]
+< = download-base
+filename = re6st-registry.conf.in
+md5sum = ae910e8e154be6575bb19f6eae686a87
+
+[check-recipe]
+recipe = plone.recipe.command
+stop-on-error = true
+update-command = ${:command}
+command =
+  grep parts ${buildout:develop-eggs-directory}/re6stnet.egg-link
+  grep parts ${buildout:develop-eggs-directory}/slapos.cookbook.egg-link
+
+[versions]
+apache-libcloud = 0.17.0
+ecdsa = 0.13
+gitdb = 0.6.4
+plone.recipe.command = 1.1
+pycrypto = 2.6.1
+slapos.recipe.template = 2.6
+slapos.toolbox = 0.47.3
+smmap = 0.9.0
+
+# Required by:
+# slapos.toolbox==0.47.3
+GitPython = 0.3.6
+
+# Required by:
+# slapos.toolbox==0.47.3
+atomize = 0.2.0
+
+# Required by:
+# apache-libcloud==0.17.0
+backports.ssl-match-hostname = 3.4.0.2
+
+# Required by:
+# slapos.toolbox==0.47.3
+feedparser = 5.1.3
+
+# Required by:
+# slapos.toolbox==0.47.3
+lockfile = 0.10.2
+
+# Required by:
+# re6stnet===0-413.gbec6b3c.dirty
+miniupnpc = 1.9
+
+# Required by:
+# slapos.toolbox==0.47.3
+paramiko = 1.15.2
+
+# Required by:
+# slapos.toolbox==0.47.3
+rpdb = 0.1.5
+

software/re6stnet/software.cfg.json

+{
+    "name": "RE6STNET",
+    "description": "Master instance of re6st (Resilient, Scalable, IPv6 Network application)",
+    "serialisation": "xml",
+    "software-type": {
+        "default": {
+            "title": "Default",
+            "description": "Re6st registry",
+            "request": "instance-re6stnet-input-schema.json",
+            "response": "instance-re6stnet-output-schema.json",
+            "index": 0
+        },
+        "registry": {
+            "title": "registry",
+            "description": "Re6st registry",
+            "request": "instance-re6stnet-resilient-input-schema.json",
+            "response": "instance-re6stnet-output-schema.json",
+            "index": 1
+        }
+    }
+}
\ No newline at end of file

software/slaprunner/instance-runner-input-schema.json

@@ -92,6 +92,7 @@
             "exclusiveMinimum": true
         },
         "cpu-usage-ratio": {
+            "title": "CPU Usage Ratio",
             "description": "Ratio of the CPU use for compilation, if value is set to n, compilation will use number-of-cpu/n of cpus (need instance restart)",
             "type": "integer",
             "default" : 4

software/slaprunner/instance-runner-resilient-input-schema.json

+{
+    "type": "object",
+    "$schema": "http://json-schema.org/draft-04/schema",
+    "allOf": [
+        {
+            "$ref": "instance-runner-input-schema.json#/"
+        },
+        {
+            "properties": {
+                "-sla-0-computer_guid": {
+                    "title": "Target computer for main instance",
+                    "description": "Target computer GUID for main instance.",
+                    "type": "string",
+                    "optional": true
+                },
+                "-sla-1-computer_guid": {
+                    "title": "Target computer for first clone",
+                    "description": "Target computer for first clone and PBS.",
+                    "type": "string",
+                    "optional": true
+                },
+                "-sla-2-computer_guid": {
+                    "title": "Target computer for second clone",
+                    "description": "Target computer for second clone and PBS.",
+                    "type": "string",
+                    "optional": true
+                },
+                "resiliency-backup-periodicity": {
+                    "title": "Periodicity of backup",
+                    "description": "Periodicity of backup, in cron format.",
+                    "type": "string",
+                    "optional": true
+                },
+                "remove-backup-older-than": {
+                    "title": "Remove backups older than...",
+                    "description": "Remove all the backups in PBS that are older than specified value. It should be rdiff-backup-compatible.",
+                    "type": "string",
+                    "default": "3B",
+                    "optional": true
+                },
+                "resilient-clone-number": {
+                    "title": "Amount of backup(s) to create",
+                    "description": "Amount of backup(s) to create. Each backup consists of a Pull Backup Server and a clone.",
+                    "type": "integer",
+                    "default": 2,
+                    "optional": true
+                },
+                "ignore-known-hosts-file": {
+                    "title": "Ignore known_hosts file",
+                    "description": "Set either to fill known_hosts file for ssh or not. Useful if main instance and PBS are using the same IP (slapos proxy, webrunner).",
+                    "type": "boolean",
+                    "default": false,
+                    "optional": true
+                }
+            }
+        }
+    ]
+}
\ No newline at end of file

stack/erp5/buildout.cfg

@@ -646,7 +646,7 @@ Products.DCWorkflowGraph = 0.4.1
 Products.ExternalEditor = 1.1.0
 Products.GenericSetup = 1.7.5
 Products.LongRequestLogger = 1.1.0
-Products.MimetypesRegistry = 2.0.6
+Products.MimetypesRegistry = 2.0.7
 Products.PluginRegistry = 1.3
 Products.TIDStorage = 5.4.9
 PyPDF2 = 1.24
@@ -685,7 +685,7 @@ pyflakes = 0.8.1
 pylint = 1.4.3
 python-ldap = 2.4.19
 python-magic = 0.4.6
-python-memcached = 1.53
+python-memcached = 1.54
 qrcode = 5.1
 restkit = 4.2.2
 rtjp-eventlet = 0.3.2

stack/monitor/buildout.cfg

@@ -57,7 +57,7 @@ mode = 0644
 recipe = hexagonit.recipe.download
 url = ${:_profile_base_location_}/${:filename}
 download-only = true
-md5sum = 2d48f8b8e01fa0fdde964ed1c1547f05
+md5sum = 93e1dda50cb71bfe29966b2946c02dd1
 filename = cgi-httpd.conf.in
 mode = 0644
 

stack/monitor/cgi-httpd.conf.in

 PidFile "{{ httpd_configuration.get('pid-file') }}"
+
+StartServers 1
+ServerLimit 1
+ThreadLimit 4
+ThreadsPerChild 4
+
 ServerName example.com
 ServerAdmin someone@email
 <IfDefine !MonitorPort>

stack/slapos.cfg

@@ -116,7 +116,7 @@ buildout-versions = 1.7
 cffi = 0.9.2
 collective.recipe.template = 1.11
 cmd2 = 0.6.8
-cryptography = 0.8
+cryptography = 0.8.1
 inotifyx = 0.2.2
 itsdangerous = 0.24
 lxml = 3.4.2
@@ -128,13 +128,13 @@ prettytable = 0.7.2
 psutil = 2.2.1
 pyOpenSSL = 0.14
 pyparsing = 2.0.3
-pytz = 2014.10
+pytz = 2015.2
 requests = 2.6.0
 setuptools = 12.0.4
 simplejson = 3.6.5
 six = 1.9.0
-slapos.cookbook = 0.95
-slapos.core = 1.3.8
+slapos.cookbook = 0.97
+slapos.core = 1.3.9
 slapos.libnetworkcache = 0.14.2
 slapos.recipe.build = 0.20
 slapos.recipe.cmmi = 0.2
@@ -142,7 +142,7 @@ stevedore = 1.3.0
 xml-marshaller = 0.9.7
 
 # Required by:
-# slapos.core==1.3.8
+# slapos.core==1.3.9
 Flask = 0.10.1
 
 # Required by:
@@ -151,7 +151,7 @@ Flask = 0.10.1
 argparse = 1.3.0
 
 # Required by:
-# slapos.core==1.3.8
+# slapos.core==1.3.9
 cliff = 1.10.1
 
 # Required by:
@@ -159,11 +159,11 @@ cliff = 1.10.1
 enum34 = 1.0.4
 
 # Required by:
-# slapos.cookbook==0.95
+# slapos.cookbook==0.97
 jsonschema = 2.4.0
 
 # Required by:
-# slapos.cookbook==0.95
+# slapos.cookbook==0.97
 lock-file = 2.0
 
 # Required by:
@@ -183,15 +183,15 @@ pyasn1 = 0.1.7
 pycparser = 2.10
 
 # Required by:
-# slapos.core==1.3.8
+# slapos.core==1.3.9
 supervisor = 3.1.3
 
 # Required by:
-# slapos.core==1.3.8
+# slapos.core==1.3.9
 uritemplate = 0.6
 
 # Required by:
-# slapos.core==1.3.8
+# slapos.core==1.3.9
 zope.interface = 4.1.2
 
 [networkcache]