Merge branch 'kvm' into slaprunner

component/cloud9/buildout.cfg

+[buildout]
+extends =
+  ../dcron/buildout.cfg
+  ../libxml2/buildout.cfg
+  ../logrotate/buildout.cfg
+  ../rdiff-backup/buildout.cfg
+  ../nodejs/buildout.cfg
+
+parts =
+  nodejs
+  npm
+  cloud9
+
+[cloud9]
+<= cloud9-git
+
+[cloud9-git]
+# Online IDE written in javascript/node.js
+# URL : c9.io
+# You can use it using the following command :
+# NODE_PATH=${:destination}/node_modules ${nodejs:node_location} ${:cloud9_js_location} 
+recipe = plone.recipe.command
+stop-on-error = true
+commit = 97db1467c517d265438684bd2a70b0b76ee282f6
+repository = https://github.com/ajaxorg/cloud9.git
+location = ${buildout:parts-directory}/${:_buildout_section_name_}
+git-binary = ${git:location}/bin/git
+npm-binary = ${nodejs-0.4:location}/bin/node ${npm:location}/bin/npm
+command = export GIT_SSL_NO_VERIFY=true; (${:git-binary} clone --quiet ${:repository} ${:location} && cd ${:location} && ${:git-binary} reset --hard ${:commit} && ${:git-binary} submodule update --init && cd support/jsdav && PATH=/opt/slapgrid/c86b6d036d0004572b96e10de1a1b5e8/parts/nodejs-0.4/bin:$PATH LDFLAGS=-L${libxml2:location}/lib ${:npm-binary} install) || (rm -fr ${:location}; exit 1)
+update-command =
+
+[cloud9-npm]
+# Online IDE written in javascript/node.js
+# URL : c9.io 
+# You can use it using the following command :
+# NODE_PATH=${:destination}/node_modules ${nodejs:node_location} ${:cloud9_js_location} 
+recipe = slapos.recipe.npm
+# Node part has to be specified, otherwise system node is used.
+node = nodejs-0.6
+# List of packages to install
+packages =
+  cloud9
+# Specify environment jsDAV (dependency of cloud9) needs libxml2
+environment = 
+  LDFLAGS=-L${libxml2:location}/lib -Wl,-rpath=${libxml2:location}/lib

component/gettext/buildout.cfg

@@ -20,6 +20,7 @@ configure-options =
   --without-emacs
   --disable-acl
   --disable-openmp
+  --without-git
 
 environment =
   CPPFLAGS=-I${libxml2:location}/include -I${zlib:location}/include -I${ncurses:location}/include

component/noVNC/buildout.cfg

@@ -3,6 +3,6 @@ parts =
   noVNC
 
 [noVNC]
-recipe = hexagonit.recipe.download
+recipe = slapos.recipe.build:download-unpacked
 url = https://github.com/kanaka/noVNC/tarball/master
 strip-top-level-dir = true

component/nodejs/buildout.cfg

+[buildout]
+extends =
+  ../git/buildout.cfg
+  ../pkgconfig/buildout.cfg
+  ../openssl/buildout.cfg
+  ../python-2.7/buildout.cfg
+  ../zlib/buildout.cfg
+
+parts =
+  nodejs
+
+[nodejs]
+# Server-side Javascript.
+recipe = hexagonit.recipe.cmmi
+url = http://nodejs.org/dist/v0.6.10/node-v0.6.10.tar.gz
+md5sum = 8a74fd5d48c2c7c64abc60b2b8f3fbc7
+configure-options =
+  --openssl-includes=${openssl:location}/include
+  --openssl-libpath=${openssl:location}/lib
+environment =
+  PATH=${pkgconfig:location}/bin:${python2.7:location}/bin:%(PATH)s
+  PKG_CONFIG_PATH=${openssl:location}/lib/pkgconfig/
+  CPPFLAGS=-I${zlib:location}/include
+  LDFLAGS=-Wl,-rpath=${openssl:location}/lib -L${zlib:location}/lib -Wl,-rpath=${zlib:location}/lib
+
+[nodejs-0.4]
+recipe = hexagonit.recipe.cmmi
+url = http://nodejs.org/dist/node-v0.4.12.tar.gz
+md5sum = a6375eaa43db5356bf443e25b828ae16
+configure-options =
+  --openssl-includes=${openssl:location}/include
+  --openssl-libpath=${openssl:location}/lib
+environment =
+  PATH=${pkgconfig:location}/bin:${python2.7:location}/bin:%(PATH)s
+  PKG_CONFIG_PATH=${openssl:location}/lib/pkgconfig/
+  CPPFLAGS=-I${zlib:location}/include
+  LDFLAGS=-Wl,-rpath=${openssl:location}/lib -L${zlib:location}/lib -Wl,-rpath=${zlib:location}/lib
+
+[npm]
+# Node.js Package Manager
+# Deprecated. Included in node >= 0.6.3.
+recipe = plone.recipe.command
+location = ${buildout:parts-directory}/${:_buildout_section_name_}
+stop-on-error = true
+commit = 3136abc5c6b3ed332c4700ece24450fada63639b
+origin = https://github.com/isaacs/npm.git
+git-bin = ${git:location}/bin/git
+node-bin = ${nodejs-0.4:location}/bin/node
+command = (GIT_SSL_NO_VERIFY=true ${:git-bin} clone --quiet ${:origin} ${:location} && cd ${:location} && ${:git-bin} reset --hard ${:commit} && ${:location}/configure --prefix=${:location} && GIT_SSL_NO_VERIFY=true ${:git-bin} submodule update --init --recursive && ${:node_bin} cli.js install npm@1.0.106 -g -f) || (rm -fr ${:location}; exit 1)
+update-command =
+

setup.py

@@ -45,6 +45,7 @@ setup(name=name,
           'apache.zope.backend = slapos.recipe.apache_zope_backend:Recipe',
           'certificate_authority = slapos.recipe.certificate_authority:Recipe',
           'certificate_authority.request = slapos.recipe.certificate_authority:Request',
+          'check_port_listening = slapos.recipe.check_port_listening:Recipe',
           'cron = slapos.recipe.dcron:Recipe',
           'cron.d = slapos.recipe.dcron:Part',
           'davstorage = slapos.recipe.davstorage:Recipe',
@@ -55,14 +56,19 @@ setup(name=name,
           'erp5scalabilitytestbed = slapos.recipe.erp5scalabilitytestbed:Recipe',
           'equeue = slapos.recipe.equeue:Recipe',
           'erp5testnode = slapos.recipe.erp5testnode:Recipe',
+          'generate.mac = slapos.recipe.generatemac:Recipe',
+          'generic.kvm.frontend = slapos.recipe.generic_kvm_frontend:Recipe',
+          'nbdserver = slapos.recipe.nbdserver:Recipe',
+          'generic.novnc = slapos.recipe.generic_novnc:Recipe',
+          'generic.onetimeupload = slapos.recipe.generic_onetimeupload:Recipe',
           'helloworld = slapos.recipe.helloworld:Recipe',
           'generic.cloudooo = slapos.recipe.generic_cloudooo:Recipe',
           'fontconfig = slapos.recipe.fontconfig:Recipe',
           'java = slapos.recipe.java:Recipe',
           'kumofs = slapos.recipe.kumofs:Recipe',
+          'kvm = slapos.recipe.kvm:Recipe',
           'generic.kumofs = slapos.recipe.generic_kumofs:Recipe',
           'haproxy = slapos.recipe.haproxy:Recipe',
-          'kvm = slapos.recipe.kvm:Recipe',
           'libcloud = slapos.recipe.libcloud:Recipe',
           'libcloudrequest = slapos.recipe.libcloudrequest:Recipe',
           'lockfile = slapos.recipe.lockfile:Recipe',
@@ -72,7 +78,6 @@ setup(name=name,
           'mydumper = slapos.recipe.mydumper:Recipe',
           'generic.mysql = slapos.recipe.generic_mysql:Recipe',
           'mkdirectory = slapos.recipe.mkdirectory:Recipe',
-          'nbdserver = slapos.recipe.nbdserver:Recipe',
           'nosqltestbed = slapos.recipe.nosqltestbed:NoSQLTestBed',
           'notifier = slapos.recipe.notifier:Recipe',
           'notifier.callback = slapos.recipe.notifier:Callback',

slapos/recipe/README.kvm_frontend.txt

+kvm_frontend
+===
+
+Introduction
+------------
+
+The ``slapos.recipe.kvm_frontend`` aims to provide proxy server to KVM instances.
+
+It allows HTTPS IPv4/IPv6 proxying (with or without domain name), and supports
+the WebSocket technology needed for VNC-in-webapplication noVNC.
+
+It works following the master/slave instances system. A master instance is
+created, containing all what is needed to run the proxy. Slave instances
+are later created, adding one line in the master instance's proxy configuration
+that specify the IP/port to proxy to the KVM.
+
+The slave instance (kvm) is then accessible from
+http://[masterinstanceIPorhostname]/[randomgeneratednumber]
+
+
+Instance parameters
+------------
+
+Incoming master instance parameters
++++++++
+
+``port``                - Port of server, optional, defaults to 4443.
+``domain``              - domain name to use, optional, default to
+                          "host.vifib.net".
+``redirect_plain_http`` - if value is one of ['y', 'yes', '1', 'true'], instance
+                          will try to create a simple http server on port 80
+                          redirecting to the proxy. Optional.
+
+Incoming slave instance parameters
++++++++
+
+``host``    - KVM instance IP or hostname. Mandatory.
+``port``    - KVM instance port, Mandatory.
+``https``   - if value is one of ['n', 'no', '0', 'false'], will try to connect
+              to target in plain http. Optional.
+
+Connection parameters
+-------------
+
+Outgoing master connection parameters
++++++++
+
+``domain_ipv6_address``  - Proxy IP
+``site_url``             - Proxy URL
+
+Outgoing slave connection parameters are :
++++++++
+
+``site_url``             - URL of instance
+``domain_name``          - Domain name of proxy
+``port``                 - Port of proxy

slapos/recipe/check_port_listening/__init__.py

+##############################################################################
+#
+# Copyright (c) 2011 Vifib SARL and Contributors. All Rights Reserved.
+#
+# WARNING: This program as such is intended to be used by professional
+# programmers who take the whole responsibility of assessing all potential
+# consequences resulting from its eventual inadequacies and bugs
+# End users who are looking for a ready-to-use solution with commercial
+# guarantees and support are strongly adviced to contract a Free Software
+# Service Company
+#
+# This program is Free Software; you can redistribute it and/or
+# modify it under the terms of the GNU General Public License
+# as published by the Free Software Foundation; either version 3
+# of the License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA  02111-1307, USA.
+#
+##############################################################################
+from slapos.recipe.librecipe import GenericBaseRecipe
+import sys
+
+class Recipe(GenericBaseRecipe):
+  """
+  Check listening port promise
+  """
+
+  def install(self):
+    config = dict(
+      hostname=self.options['hostname'],
+      port=self.options['port'],
+      python_path=sys.executable,
+    )
+
+    vnc_promise = self.createExecutable(
+      self.options['path'],
+      self.substituteTemplate(
+        self.getTemplateFilename('socket_connection_attempt.py.in'),
+        config))
+
+    return [vnc_promise]

slapos/recipe/check_port_listening/template/socket_connection_attempt.py.in

+#!%(python_path)s
+# BEWARE: This file is operated by slapgrid
+# BEWARE: It will be overwritten automatically
+import socket
+import sys
+
+hostname = "%(hostname)s"
+port = %(port)s
+
+connection_okay = False
+
+try:
+  s = socket.create_connection((hostname, port))
+  connection_okay = True
+  s.close()
+except (socket.error, socket.timeout):
+  connection_okay = False
+
+if not connection_okay:
+  print >> sys.stderr, "%(port)s on %(hostname)s isn't listening"
+  sys.exit(127)

slapos/recipe/generatemac.py

+
+##############################################################################
+#
+# Copyright (c) 2010 Vifib SARL and Contributors. All Rights Reserved.
+#
+# WARNING: This program as such is intended to be used by professional
+# programmers who take the whole responsibility of assessing all potential
+# consequences resulting from its eventual inadequacies and bugs
+# End users who are looking for a ready-to-use solution with commercial
+# guarantees and support are strongly adviced to contract a Free Software
+# Service Company
+#
+# This program is Free Software; you can redistribute it and/or
+# modify it under the terms of the GNU General Public License
+# as published by the Free Software Foundation; either version 3
+# of the License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA  02111-1307, USA.
+#
+##############################################################################
+import random
+
+from slapos.recipe.librecipe import GenericBaseRecipe
+
+class Recipe(GenericBaseRecipe):
+
+  def __init__(self, buildout, name, options):
+    # First octet has to represent a locally administered address
+    octet_list = [254] + [random.randint(0x00, 0xff) for x in range(5)]
+    options['mac-address'] = ':'.join(['%02x' % x for x in octet_list])
+
+  def install(self):
+    return []

slapos/recipe/generic_kvm_frontend/__init__.py

+##############################################################################
+#
+# Copyright (c) 2011 Vifib SARL and Contributors. All Rights Reserved.
+#
+# WARNING: This program as such is intended to be used by professional
+# programmers who take the whole responsibility of assessing all potential
+# consequences resulting from its eventual inadequacies and bugs
+# End users who are looking for a ready-to-use solution with commercial
+# guarantees and support are strongly adviced to contract a Free Software
+# Service Company
+#
+# This program is Free Software; you can redistribute it and/or
+# modify it under the terms of the GNU General Public License
+# as published by the Free Software Foundation; either version 3
+# of the License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA  02111-1307, USA.
+#
+##############################################################################
+from slapos.recipe.librecipe import GenericBaseRecipe
+from json import loads as unjson
+
+class Recipe(GenericBaseRecipe):
+  """
+  kvm frontend instance configuration.
+  """
+
+  def _getRewriteRuleContent(self, slave_instance_list):
+    """Generate rewrite rules list from slaves list"""
+    rewrite_rule_list = []
+    for slave_instance in slave_instance_list:
+      current_slave_dict = dict()
+      # Get host, and if IPv6 address, remove "[" and "]"
+      current_slave_dict['host'] = current_slave_dict['host'].\
+          replace('[', '').replace(']', '')
+      current_slave_dict['port'] = slave_instance['port']
+      if current_slave_dict['host'] is None \
+          or current_slave_dict['port'] is None:
+        # XXX-Cedric: should raise warning because slave seems badly configured
+        continue
+      # Check if target is https or http
+      current_slave_dict['https'] = slave_instance.get('https', 'true')
+      if current_slave_dict['https'] in FALSE_VALUE_LIST:
+        current_slave_dict['https'] = 'false'
+      # Set reference and resource url
+      # Reference is raw reference from SlapOS Master, resource is
+      # URL-compatible name
+      reference = slave_instance.get('slave_reference')
+      current_slave_dict['reference'] = reference
+      current_slave_dict['resource'] = reference.replace('-', '')
+      rewrite_rule_list.append(current_slave_dict)
+    return rewrite_rule_list
+
+  def _getProxyTableContent(self, rewrite_rule_list):
+    """Generate proxy table file content from rewrite rules list"""
+    proxy_table_content = '{'
+    for rewrite_rule in rewrite_rule_list:
+      rewrite_part = self.substituteTemplate(
+         self.getTemplateFilename('proxytable-resource-snippet.json.in'),
+         rewrite_rule)
+      proxy_table_content += "%s," % rewrite_part
+
+    proxy_table_content += '}\n'
+    return proxy_table_content
+
+  def install(self):
+    # Generate rewrite rules
+    rewrite_rule_list = self._getRewriteRuleContent(
+      unjson(self.options['slave-instance-list']))
+    # Create Map
+    map_content = self._getProxyTableContent(rewrite_rule_list)
+    map_file = self.createFile(self.options['map-path'], map_content)
+
+    # Create configuration
+    conf = open(self.getTemplateFilename('kvm-proxy.js'), 'r')
+    conf_file = self.createFile(self.options['conf-path'], conf.read())
+    conf.close()
+
+    config = dict(
+      ip=self.options['ip'],
+      port=self.options['port'],
+      key=self.options['ssl-key-path'],
+      certificate=self.options['ssl-cert-path'],
+      name=self.options['domain'],
+      shell_path=self.options['shell-path'],
+      node_path=self.options['node-binary'],
+      node_env=self.options['node-env'],
+      conf_path=conf_file,
+      map_path=map_file,
+      plain_http='',
+    )
+
+    runner_path = self.createExecutable(
+      self.options['wrapper-path'],
+      self.substituteTemplate(self.getTemplateFilename('nodejs_run.in'),
+                              config))
+
+    return [map_file, conf_file, runner_path]

slapos/recipe/generic_kvm_frontend/template/kvm-proxy.js

+var fs = require('fs'),
+    util = require('util'),
+    colors = require('colors'),
+    http = require('http'),
+    httpProxy = require('http-proxy'),
+    proxyByUrl = require('proxy-by-url');
+
+var listenInterface = process.argv[2],
+    port = process.argv[3],
+    sslKeyFile = process.argv[4],
+    sslCertFile = process.argv[5],
+    proxyTable = process.argv[6],
+    redirect = process.argv[7] || false,
+    isRawIPv6;
+
+if (process.argv.length < 7) {
+    console.error("Too few arguments. Exiting.");
+  process.exit(1);
+}
+
+isRawIPv6 = function checkipv6(str) {
+  // Inspired by http://forums.intermapper.com/viewtopic.php?t=452
+  return (/^\s*((([0-9A-Fa-f]{1,4}:){7}([0-9A-Fa-f]{1,4}|:))|(([0-9A-Fa-f]{1,4}:){6}(:[0-9A-Fa-f]{1,4}|((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3})|:))|(([0-9A-Fa-f]{1,4}:){5}(((:[0-9A-Fa-f]{1,4}){1,2})|:((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3})|:))|(([0-9A-Fa-f]{1,4}:){4}(((:[0-9A-Fa-f]{1,4}){1,3})|((:[0-9A-Fa-f]{1,4})?:((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3}))|:))|(([0-9A-Fa-f]{1,4}:){3}(((:[0-9A-Fa-f]{1,4}){1,4})|((:[0-9A-Fa-f]{1,4}){0,2}:((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3}))|:))|(([0-9A-Fa-f]{1,4}:){2}(((:[0-9A-Fa-f]{1,4}){1,5})|((:[0-9A-Fa-f]{1,4}){0,3}:((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3}))|:))|(([0-9A-Fa-f]{1,4}:){1}(((:[0-9A-Fa-f]{1,4}){1,6})|((:[0-9A-Fa-f]{1,4}){0,4}:((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3}))|:))|(:(((:[0-9A-Fa-f]{1,4}){1,7})|((:[0-9A-Fa-f]{1,4}){0,5}:((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3}))|:)))(%.+)?\s*$/.test(str));
+}(listenInterface);
+
+/**
+ * Dummy middleware that throws 404 not found. Does not contain websocket
+ * middleware.
+ */
+var middlewareNotFound = function(req, res, proxy) {
+  res.statusCode = 404;
+  res.setHeader('Content-Type', 'text/plain');
+  res.end('This URL is not known. Please check your URL or contact your ' +
+      'SlapOS administrator.');
+};
+
+/**
+ * Create server
+ */
+var proxyServer = httpProxy.createServer(
+  // We declare our proxyByUrl middleware
+  proxyByUrl(proxyTable),
+  // Then we add your dummy middleware, called when proxyByUrl doesn't find url.
+  middlewareNotFound,
+  // And we set HTTPS options for server. HTTP will be forbidden.
+  {
+    https: {
+      key: fs.readFileSync(
+        sslKeyFile,
+        'utf8'
+      ),
+      cert: fs.readFileSync(
+        sslCertFile,
+        'utf8'
+      )
+    },
+    source: {
+    host: listenInterface,
+    port: port
+  }}
+);
+
+// Release the beast.
+proxyServer.listen(port, listenInterface);
+console.log('HTTPS server started and listening at ' + listenInterface + ':' +
+            port);
+
+// Dummy HTTP server redirecting to HTTPS. Only has sense if we can use port 80
+if (redirect === '1') {
+  try {
+    var httpPort = 80;
+    http.createServer(function(req, res) {
+      var url;
+      if (isRawIPv6 === true) {
+        url = 'https://[' + listenInterface + ']';
+      } else {
+        url = 'https://' + listenInterface;
+      }
+      // If non standard port : need to specify it
+      if (port !== 443) {
+        url = url + ':' + port;
+      }
+      // Add last part of URL
+      url = url + req.url;
+      console.log(url);
+      // Anwser "permanently redirected"
+      res.statusCode = 301;
+      res.setHeader('Location', url);
+      res.end();
+    }).listen(httpPort, listenInterface);
+    console.log('HTTP redirect server started and listening at ' +
+                listenInterface + ':' + httpPort);
+  } catch (error) {
+    console.log('Couldn\'t start plain HTTP redirection server : ' + error)
+  }
+}

slapos/recipe/kvm/template/slapmonitor_run.in → slapos/recipe/generic_kvm_frontend/template/nodejs_run.in

-#!/bin/sh
+#!%(shell_path)s
 # BEWARE: This file is operated by slapgrid
 # BEWARE: It will be overwritten automatically
-exec %(python_path)s %(slapmonitor_path)s %(pid_file_path)s %(database_path)s
+export NODE_PATH=%(node_env)s
+exec %(node_path)s %(conf_path)s %(ip)s %(port)s %(key)s %(certificate)s %(map_path)s %(plain_http)s

slapos/recipe/generic_kvm_frontend/template/proxytable-resource-snippet.json.in

+
+  "/%(resource)s": {
+    "port": %(port)s,
+    "host": "%(host)s",
+    "https": %(https)s
+  }

slapos/recipe/generic_novnc/__init__.py

+##############################################################################
+#
+# Copyright (c) 2011 Vifib SARL and Contributors. All Rights Reserved.
+#
+# WARNING: This program as such is intended to be used by professional
+# programmers who take the whole responsibility of assessing all potential
+# consequences resulting from its eventual inadequacies and bugs
+# End users who are looking for a ready-to-use solution with commercial
+# guarantees and support are strongly adviced to contract a Free Software
+# Service Company
+#
+# This program is Free Software; you can redistribute it and/or
+# modify it under the terms of the GNU General Public License
+# as published by the Free Software Foundation; either version 3
+# of the License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA  02111-1307, USA.
+#
+##############################################################################
+from slapos.recipe.librecipe import GenericBaseRecipe
+import binascii
+import os
+import sys
+
+class Recipe(GenericBaseRecipe):
+  """
+  novnc instance configuration.
+  """
+
+  def install(self):
+    runner_path = self.createPythonScript(
+      self.options['path'],
+      'slapos.recipe.librecipe.execute.execute_wait',
+      [[
+        self.options['websockify-path'],
+        '--web',
+        self.options['novnc-location'],
+        '--key=%s' % self.options['ssl-key-path'],
+        '--cert=%s' % self.options['ssl-cert-path'],
+        '--ssl-only',
+        '%s:%s' % (self.options['ip'], self.options['port']),
+        '%s:%s' % (self.options['vnc-ip'], self.options['vnc-port']),
+      ],
+      [self.options['ssl-key-path'], self.options['ssl-cert-path']]],
+    )
+
+    return [runner_path]

slapos/recipe/generic_onetimeupload/__init__.py

+##############################################################################
+#
+# Copyright (c) 2011 Vifib SARL and Contributors. All Rights Reserved.
+#
+# WARNING: This program as such is intended to be used by professional
+# programmers who take the whole responsibility of assessing all potential
+# consequences resulting from its eventual inadequacies and bugs
+# End users who are looking for a ready-to-use solution with commercial
+# guarantees and support are strongly adviced to contract a Free Software
+# Service Company
+#
+# This program is Free Software; you can redistribute it and/or
+# modify it under the terms of the GNU General Public License
+# as published by the Free Software Foundation; either version 3
+# of the License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA  02111-1307, USA.
+#
+##############################################################################
+from slapos.recipe.librecipe import GenericBaseRecipe
+import binascii
+import os
+import sys
+
+class Recipe(GenericBaseRecipe):
+  """
+  kvm instance configuration.
+  """
+
+  def __init__(self, buildout, name, options):
+    options['key'] = binascii.hexlify(os.urandom(24))
+    return GenericBaseRecipe.__init__(self, buildout, name, options)
+
+  def install(self):
+    config = dict(
+      ip=self.options['ip'],
+      port=self.options['port'],
+      onetimeupload_path=self.options['onetimeupload-path'],
+      shell_path=self.options['shell-path'],
+      log_path=self.options['log-path'],
+      image=self.options['image-path'],
+      key=self.options['key'],
+    )
+
+    # Runners
+    runner_path = self.createExecutable(
+      self.options['path'],
+      self.substituteTemplate(self.getTemplateFilename('onetimeupload_run.in'),
+                              config))
+
+    return [runner_path]
+

slapos/recipe/nbdserver/template/onetimeupload_run.in → slapos/recipe/generic_onetimeupload/template/onetimeupload_run.in

-#!/bin/sh
+#!%(shell_path)s
 # BEWARE: This file is operated by slapgrid
 # BEWARE: It will be overwritten automatically
 exec %(onetimeupload_path)s -l %(log_path)s %(ip)s %(port)s %(image)s %(key)s 

slapos/recipe/kvm/__init__.py

 ##############################################################################
 #
-# Copyright (c) 2010 Vifib SARL and Contributors. All Rights Reserved.
+# Copyright (c) 2011 Vifib SARL and Contributors. All Rights Reserved.
 #
 # WARNING: This program as such is intended to be used by professional
 # programmers who take the whole responsibility of assessing all potential
@@ -24,333 +24,53 @@
 # Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA  02111-1307, USA.
 #
 ##############################################################################
+from slapos.recipe.librecipe import GenericBaseRecipe
+import binascii
 import os
 import sys
-from slapos.recipe.librecipe import BaseSlapRecipe
-import subprocess
-import binascii
-import random
-import zc.buildout
-import pkg_resources
-import ConfigParser
-import hashlib
-
-class Recipe(BaseSlapRecipe):
-
-  # To avoid magic numbers
-  VNC_BASE_PORT = 5900
-
-  def _install(self):
-    """
-    Set the connection dictionnary for the computer partition and create a list
-    of paths to the different wrappers
-
-    Parameters : none
-
-    Returns    : List path_list
-    """
-    self.path_list = []
-
-    self.requirements, self.ws           = self.egg.working_set()
-    self.cron_d                          = self.installCrond()
-
-    self.ca_conf                         = self.installCertificateAuthority()
-    self.key_path, self.certificate_path = self.requestCertificate('noVNC')
-
-    # Install the socket_connection_attempt script
-    catcher = zc.buildout.easy_install.scripts(
-      [('check_port_listening', 'slapos.recipe.kvm.socket_connection_attempt',
-        'connection_attempt')],
-      self.ws,
-      sys.executable,
-      self.bin_directory,
-    )
-    # Save the check_port_listening script path
-    check_port_listening_script = catcher[0]
-    # Get the port_listening_promise template path, and save it
-    self.port_listening_promise_path = pkg_resources.resource_filename(
-      __name__, 'template/port_listening_promise.in')
-    self.port_listening_promise_conf = dict(
-     check_port_listening_script=check_port_listening_script,
-    )
 
-    kvm_conf = self.installKvm(vnc_ip = self.getLocalIPv4Address())
-
-    vnc_port = Recipe.VNC_BASE_PORT + kvm_conf['vnc_display']
-
-    noVNC_conf = self.installNoVnc(source_ip   = self.getGlobalIPv6Address(),
-                                   source_port = 6080,
-                                   target_ip   = kvm_conf['vnc_ip'],
-                                   target_port = vnc_port)
-
-    self.linkBinary()
-    self.computer_partition.setConnectionDict(dict(
-        url = "https://[%s]:%s/vnc_auto.html?host=[%s]&port=%s&encrypt=1" % (
-            noVNC_conf['source_ip'],
-            noVNC_conf['source_port'],
-            noVNC_conf['source_ip'],
-            noVNC_conf['source_port']),
-        password = kvm_conf['vnc_passwd']))
-
-    return self.path_list
-
-  def installKvm(self, vnc_ip):
+class Recipe(GenericBaseRecipe):
   """
-    Create kvm configuration dictionnary and instanciate a wrapper for kvm and
-    kvm controller
-
-    Parameters : IP the vnc server is listening on
-
-    Returns    : Dictionnary kvm_conf
+  kvm instance configuration.
   """
-    kvm_conf = dict(vnc_ip = vnc_ip)
 
-    connection_found = False
-    for tap_interface, dummy in self.parameter_dict['ip_list']:
-      # Get an ip associated to a tap interface
-      if tap_interface:
-        connection_found = True
-    if not connection_found:
-      raise NotImplementedError("Do not support ip without tap interface")
-
-    kvm_conf['tap_interface'] = tap_interface
-
-    # Disk path
-    kvm_conf['disk_path'] = os.path.join(self.data_root_directory,
-        'virtual.qcow2')
-    kvm_conf['socket_path'] = os.path.join(self.var_directory, 'qmp_socket')
+  def __init__(self, buildout, name, options):
+    options['passwd'] = binascii.hexlify(os.urandom(4))
+    return GenericBaseRecipe.__init__(self, buildout, name, options)
+
+  def install(self):
+    config = dict(
+      tap_interface=self.options['tap'],
+      vnc_ip=self.options['vnc-ip'],
+      vnc_port=self.options['vnc-port'],
+      nbd_ip=self.options['nbd-ip'],
+      nbd_port=self.options['nbd-port'],
+      disk_path=self.options['disk-path'],
+      disk_size=self.options['disk-size'],
+      mac_address=self.options['mac-address'],
+      smp_count=self.options['smp-count'],
+      ram_size=self.options['ram-size'],
+      socket_path=self.options['socket-path'],
+      pid_file_path=self.options['pid-path'],
+      python_path=sys.executable,
+      shell_path=self.options['shell-path'],
+      qemu_path=self.options['qemu-path'],
+      qemu_img_path=self.options['qemu-img-path'],
       # XXX Weak password
-    ##XXX -Vivien: add an option to generate one password for all instances
-    # and/or to input it yourself
-    kvm_conf['vnc_passwd'] = binascii.hexlify(os.urandom(4))
-
-    #XXX pid_file path, database_path, path to python binary and xml path
-    kvm_conf['pid_file_path'] = os.path.join(self.run_directory, 'pid_file')
-    kvm_conf['database_path'] = os.path.join(self.data_root_directory,
-        'slapmonitor_database')
-    kvm_conf['python_path']   = sys.executable
-    kvm_conf['qemu_path']     = self.options['qemu_path']
-    #xml_path = os.path.join(self.var_directory, 'slapreport.xml' )
-
-    # Create disk if needed
-    if not os.path.exists(kvm_conf['disk_path']):
-      retcode = subprocess.call(["%s create -f qcow2 %s %iG" % (
-          self.options['qemu_img_path'], kvm_conf['disk_path'],
-          int(self.options['disk_size']))], shell=True)
-      if retcode != 0:
-        raise OSError, "Disk creation failed!"
-
-    # Options nbd_ip and nbd_port are provided by slapos master
-    kvm_conf['nbd_ip']   = self.parameter_dict['nbd_ip']
-    kvm_conf['nbd_port'] = self.parameter_dict['nbd_port']
-
-    # First octet has to represent a locally administered address
-    octet_list         = [254] + [random.randint(0x00, 0xff) for x in range(5)]
-    kvm_conf['mac_address'] = ':'.join(['%02x' % x for x in octet_list])
-
-    kvm_conf['hostname']    = "slaposkvm"
-    kvm_conf['smp_count']   = self.options['smp_count']
-    kvm_conf['ram_size']    = self.options['ram_size']
-
-    kvm_conf['vnc_display'] = 1
-
-    # Instanciate KVM
-    kvm_template_location = pkg_resources.resource_filename(
-                                             __name__, os.path.join(
-                                             'template', 'kvm_run.in'))
-
-    kvm_runner_path = self.createRunningWrapper("kvm",
-          self.substituteTemplate(kvm_template_location,
-                                  kvm_conf))
-
-    self.path_list.append(kvm_runner_path)
-
-    # Instanciate KVM controller
-    kvm_controller_template_location = pkg_resources.resource_filename(
-                                             __name__, os.path.join(
-                                             'template',
-                                             'kvm_controller_run.in' ))
-
-    kvm_controller_runner_path = self.createRunningWrapper("kvm_controller",
-          self.substituteTemplate(kvm_controller_template_location,
-                                  kvm_conf))
-
-    self.path_list.append(kvm_controller_runner_path)
-
-    # Instanciate Slapmonitor
-    ##slapmonitor_runner_path = self.instanciate_wrapper("slapmonitor",
-    #    [database_path, pid_file_path, python_path])
-    # Instanciate Slapreport
-    ##slapreport_runner_path = self.instanciate_wrapper("slapreport",
-    #    [database_path, python_path])
-
-    # Add VNC promise
-    self.port_listening_promise_conf.update(
-      hostname=kvm_conf['vnc_ip'],
-      port=Recipe.VNC_BASE_PORT + kvm_conf['vnc_display'],
-    )
-    self.createPromiseWrapper("vnc_promise",
-        self.substituteTemplate(self.port_listening_promise_path,
-                                self.port_listening_promise_conf,
-                               )
-                             )
-
-    return kvm_conf
-
-  def installNoVnc(self, source_ip, source_port, target_ip, target_port):
-    """
-    Create noVNC configuration dictionnary and instanciate Websockify proxy
-
-    Parameters : IP of the proxy, port on which is situated the proxy,
-                 IP of the vnc server, port on which is situated the vnc server,
-                 path to python binary
-
-    Returns    : noVNC configuration dictionnary
-    """
-
-    noVNC_conf = {}
-
-    noVNC_conf['source_ip']   = source_ip
-    noVNC_conf['source_port'] = source_port
-
-    execute_arguments = [[
-        self.options['websockify'].strip(),
-        '--web',
-        self.options['noVNC_location'],
-        '--key=%s' % (self.key_path),
-        '--cert=%s' % (self.certificate_path),
-        '--ssl-only',
-        '%s:%s' % (source_ip, source_port),
-        '%s:%s' % (target_ip, target_port)],
-        [self.certificate_path, self.key_path]]
-
-    self.path_list.extend(zc.buildout.easy_install.scripts([('websockify',
-      'slapos.recipe.librecipe.execute', 'execute_wait')], self.ws, sys.executable,
-      self.wrapper_directory, arguments=execute_arguments))
-
-    # Add noVNC promise
-    self.port_listening_promise_conf.update(hostname=noVNC_conf['source_ip'],
-                                            port=noVNC_conf['source_port'],
-                                           )
-    self.createPromiseWrapper("novnc_promise",
-        self.substituteTemplate(self.port_listening_promise_path,
-                                self.port_listening_promise_conf,
-                               )
+      vnc_passwd=self.options['passwd']
     )
 
-    return noVNC_conf
+    # Runners
+    runner_path = self.createExecutable(
+      self.options['runner-path'],
+      self.substituteTemplate(self.getTemplateFilename('kvm_run.in'),
+                              config))
 
-  def linkBinary(self):
-    """Links binaries to instance's bin directory for easier exposal"""
-    for linkline in self.options.get('link_binary_list', '').splitlines():
-      if not linkline:
-        continue
-      target = linkline.split()
-      if len(target) == 1:
-        target = target[0]
-        path, linkname = os.path.split(target)
-      else:
-        linkname = target[1]
-        target = target[0]
-      link = os.path.join(self.bin_directory, linkname)
-      if os.path.lexists(link):
-        if not os.path.islink(link):
-          raise zc.buildout.UserError(
-              'Target link already %r exists but it is not link' % link)
-        os.unlink(link)
-      os.symlink(target, link)
-      self.logger.debug('Created link %r -> %r' % (link, target))
-      self.path_list.append(link)
+    controller_path = self.createExecutable(
+      self.options['controller-path'],
+      self.substituteTemplate(self.getTemplateFilename('kvm_controller_run.in'),
+                              config))
 
-  def installCertificateAuthority(self, ca_country_code='XX',
-      ca_email='xx@example.com', ca_state='State', ca_city='City',
-      ca_company='Company'):
-    backup_path = self.createBackupDirectory('ca')
-    self.ca_dir = os.path.join(self.data_root_directory, 'ca')
-    self._createDirectory(self.ca_dir)
-    self.ca_request_dir = os.path.join(self.ca_dir, 'requests')
-    self._createDirectory(self.ca_request_dir)
-    config = dict(ca_dir=self.ca_dir, request_dir=self.ca_request_dir)
-    self.ca_private = os.path.join(self.ca_dir, 'private')
-    self.ca_certs = os.path.join(self.ca_dir, 'certs')
-    self.ca_crl = os.path.join(self.ca_dir, 'crl')
-    self.ca_newcerts = os.path.join(self.ca_dir, 'newcerts')
-    self.ca_key_ext = '.key'
-    self.ca_crt_ext = '.crt'
-    for d in [self.ca_private, self.ca_crl, self.ca_newcerts, self.ca_certs]:
-      self._createDirectory(d)
-    for f in ['crlnumber', 'serial']:
-      if not os.path.exists(os.path.join(self.ca_dir, f)):
-        open(os.path.join(self.ca_dir, f), 'w').write('01')
-    if not os.path.exists(os.path.join(self.ca_dir, 'index.txt')):
-      open(os.path.join(self.ca_dir, 'index.txt'), 'w').write('')
-    openssl_configuration = os.path.join(self.ca_dir, 'openssl.cnf')
-    config.update(
-        working_directory=self.ca_dir,
-        country_code=ca_country_code,
-        state=ca_state,
-        city=ca_city,
-        company=ca_company,
-        email_address=ca_email,
-    )
-    self._writeFile(openssl_configuration, pkg_resources.resource_string(
-      __name__, 'template/openssl.cnf.ca.in') % config)
-    self.path_list.extend(zc.buildout.easy_install.scripts([
-      ('certificate_authority',
-        __name__ + '.certificate_authority', 'runCertificateAuthority')],
-        self.ws, sys.executable, self.wrapper_directory, arguments=[dict(
-          openssl_configuration=openssl_configuration,
-          openssl_binary=self.options['openssl_binary'],
-          certificate=os.path.join(self.ca_dir, 'cacert.pem'),
-          key=os.path.join(self.ca_private, 'cakey.pem'),
-          crl=os.path.join(self.ca_crl),
-          request_dir=self.ca_request_dir
-          )]))
-    # configure backup
-    backup_cron = os.path.join(self.cron_d, 'ca_rdiff_backup')
-    open(backup_cron, 'w').write(
-        '''0 0 * * * %(rdiff_backup)s %(source)s %(destination)s'''%dict(
-          rdiff_backup=self.options['rdiff_backup_binary'],
-          source=self.ca_dir,
-          destination=backup_path))
-    self.path_list.append(backup_cron)
-
-    return dict(
-      ca_certificate=os.path.join(config['ca_dir'], 'cacert.pem'),
-      ca_crl=os.path.join(config['ca_dir'], 'crl'),
-      certificate_authority_path=config['ca_dir']
-    )
 
-  def requestCertificate(self, name):
-    hash = hashlib.sha512(name).hexdigest()
-    key = os.path.join(self.ca_private, hash + self.ca_key_ext)
-    certificate = os.path.join(self.ca_certs, hash + self.ca_crt_ext)
-    parser = ConfigParser.RawConfigParser()
-    parser.add_section('certificate')
-    parser.set('certificate', 'name', name)
-    parser.set('certificate', 'key_file', key)
-    parser.set('certificate', 'certificate_file', certificate)
-    parser.write(open(os.path.join(self.ca_request_dir, hash), 'w'))
-    return key, certificate
+    return [runner_path, controller_path]
 
-  def installCrond(self):
-    timestamps = self.createDataDirectory('cronstamps')
-    cron_output = os.path.join(self.log_directory, 'cron-output')
-    self._createDirectory(cron_output)
-    catcher = zc.buildout.easy_install.scripts([('catchcron',
-      __name__ + '.catdatefile', 'catdatefile')], self.ws, sys.executable,
-      self.bin_directory, arguments=[cron_output])[0]
-    self.path_list.append(catcher)
-    cron_d = os.path.join(self.etc_directory, 'cron.d')
-    crontabs = os.path.join(self.etc_directory, 'crontabs')
-    self._createDirectory(cron_d)
-    self._createDirectory(crontabs)
-    # Use execute from erp5.
-    wrapper = zc.buildout.easy_install.scripts([('crond',
-      'slapos.recipe.librecipe.execute', 'execute')], self.ws, sys.executable,
-      self.wrapper_directory, arguments=[
-        self.options['dcrond_binary'].strip(), '-s', cron_d, '-c', crontabs,
-        '-t', timestamps, '-f', '-l', '5', '-M', catcher]
-      )[0]
-    self.path_list.append(wrapper)
-    return cron_d

slapos/recipe/kvm/certificate_authority.py

-import os
-import subprocess
-import time
-import ConfigParser
-import uuid
-
-
-def popenCommunicate(command_list, input=None):
-  subprocess_kw = dict(stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
-  if input is not None:
-    subprocess_kw.update(stdin=subprocess.PIPE)
-  popen = subprocess.Popen(command_list, **subprocess_kw)
-  result = popen.communicate(input)[0]
-  if popen.returncode is None:
-    popen.kill()
-  if popen.returncode != 0:
-    raise ValueError('Issue during calling %r, result was:\n%s' % (
-      command_list, result))
-  return result
-
-
-class CertificateAuthority:
-  def __init__(self, key, certificate, openssl_binary,
-      openssl_configuration, request_dir):
-    self.key = key
-    self.certificate = certificate
-    self.openssl_binary = openssl_binary
-    self.openssl_configuration = openssl_configuration
-    self.request_dir = request_dir
-
-  def checkAuthority(self):
-    file_list = [ self.key, self.certificate ]
-    ca_ready = True
-    for f in file_list:
-      if not os.path.exists(f):
-        ca_ready = False
-        break
-    if ca_ready:
-      return
-    for f in file_list:
-      if os.path.exists(f):
-        os.unlink(f)
-    try:
-      # no CA, let us create new one
-      popenCommunicate([self.openssl_binary, 'req', '-nodes', '-config',
-          self.openssl_configuration, '-new', '-x509', '-extensions', 'v3_ca',
-          '-keyout', self.key, '-out', self.certificate, '-days', '10950'],
-          # Authority name will be random, so no instance has the same issuer
-          'Certificate Authority %s\n' % uuid.uuid1())
-    except:
-      try:
-        for f in file_list:
-          if os.path.exists(f):
-            os.unlink(f)
-      except:
-        # do not raise during cleanup
-        pass
-      raise
-
-  def _checkCertificate(self, common_name, key, certificate):
-    file_list = [key, certificate]
-    ready = True
-    for f in file_list:
-      if not os.path.exists(f):
-        ready = False
-        break
-    if ready:
-      return False
-    for f in file_list:
-      if os.path.exists(f):
-        os.unlink(f)
-    csr = certificate + '.csr'
-    try:
-      popenCommunicate([self.openssl_binary, 'req', '-config',
-        self.openssl_configuration, '-nodes', '-new', '-keyout',
-        key, '-out', csr, '-days', '3650'],
-        common_name + '\n')
-      try:
-        popenCommunicate([self.openssl_binary, 'ca', '-batch', '-config',
-          self.openssl_configuration, '-out', certificate,
-          '-infiles', csr])
-      finally:
-        if os.path.exists(csr):
-          os.unlink(csr)
-    except:
-      try:
-        for f in file_list:
-          if os.path.exists(f):
-            os.unlink(f)
-      except:
-        # do not raise during cleanup
-        pass
-      raise
-    else:
-      return True
-
-  def checkRequestDir(self):
-    for request_file in os.listdir(self.request_dir):
-      parser = ConfigParser.RawConfigParser()
-      parser.readfp(open(os.path.join(self.request_dir, request_file), 'r'))
-      if self._checkCertificate(parser.get('certificate', 'name'),
-          parser.get('certificate', 'key_file'), parser.get('certificate',
-            'certificate_file')):
-        print 'Created certificate %r' % parser.get('certificate', 'name')
-
-def runCertificateAuthority(args):
-  ca_conf = args[0]
-  ca = CertificateAuthority(ca_conf['key'], ca_conf['certificate'],
-      ca_conf['openssl_binary'], ca_conf['openssl_configuration'],
-      ca_conf['request_dir'])
-  while True:
-    ca.checkAuthority()
-    ca.checkRequestDir()
-    time.sleep(60)

slapos/recipe/kvm/socket_connection_attempt.py

-import socket
-import sys
-
-def connection_attempt():
-
-  try:
-    hostname, port = sys.argv[1:3]
-  except ValueError:
-    print >> sys.stderr, """Bad command line.
-  Usage: %s hostname|ip port""" % sys.argv[0]
-    sys.exit(1)
-
-  connection_okay = False
-
-  try:
-    s = socket.create_connection((hostname, port))
-    connection_okay = True
-    s.close()
-  except (socket.error, socket.timeout):
-    connection_okay = False
-
-  if not connection_okay:
-    print >> sys.stderr, "%(port)s on %(ip)s isn't listening" % {
-      'port': port, 'ip': hostname
-    }
-    sys.exit(127)

slapos/recipe/kvm/template/kvm_run.in

-#!/bin/sh
+#!%(python_path)s
 # BEWARE: This file is operated by slapgrid
 # BEWARE: It will be overwritten automatically
 
-# TODO: -net nic,model=virtio, but OS installer has to provide the virtio_net
-# module
-exec %(qemu_path)s \
-  -net nic,macaddr=%(mac_address)s \
-  -net tap,ifname=%(tap_interface)s,script=no,downscript=no \
-  -smp %(smp_count)s \
-  -m %(ram_size)s \
-  -cdrom nbd:[%(nbd_ip)s]:%(nbd_port)s \
-  -drive file=%(disk_path)s,if=virtio,boot=on \
-  -vnc %(vnc_ip)s:1,ipv4,password \
-  -boot menu=on \
-  -qmp unix:%(socket_path)s,server \
-  -pidfile %(pid_file_path)s
+# Echo client program
+import os
+import socket
+import subprocess
+
+def getSocketStatus(host, port):
+  s = None
+  for res in socket.getaddrinfo(host, port,
+      socket.AF_UNSPEC, socket.SOCK_STREAM):
+    af, socktype, proto, canonname, sa = res
+    try:
+      s = socket.socket(af, socktype, proto)
+    except socket.error, msg:
+      s = None
+      continue
+    try:
+      s.connect(sa)
+    except socket.error, msg:
+      s.close()
+      s = None
+      continue
+    break
+  return s
+
+# create disk if doesn't exist
+disk_path = '%(disk_path)s'
+if not os.path.exists(disk_path):
+  subprocess.Popen(['%(qemu_img_path)s', 'create' ,'-f', 'qcow2',
+      '%(disk_path)s', '%(disk_size)sG'])
+
+kvm_argument_list = ['-net', 'nic,macaddr=%(mac_address)s',
+  '-net', 'tap,ifname=%(tap_interface)s,script=no,downscript=no',
+  '-smp', '%(smp_count)s',
+  '-m', '%(ram_size)s',
+  '-drive', 'file=%(disk_path)s,if=virtio,boot=on',
+  '-vnc', '%(vnc_ip)s:1,ipv4,password',
+  '-boot', 'menu=on',
+  '-qmp', 'unix:%(socket_path)s,server',
+  '-pidfile', '%(pid_file_path)s',
+  '-no-kvm']
+
+# Try to connect to NBD server
+s = getSocketStatus('%(nbd_ip)s', %(nbd_port)s)
+if s is None:
+  # NBD is not available : launch kvm without it
+  print 'Warning : Nbd is not available.'
+  os.execv('%(qemu_path)s', kvm_argument_list)
+else:
+  # NBD is available
+  kvm_argument_list.extend(['-cdrom' 'nbd:[%(nbd_ip)s]:%(nbd_port)s'])
+  os.execv('%(qemu_path)s', kvm_argument_list)

slapos/recipe/kvm/template/port_listening_promise.in

-#!/usr/bin/env sh
-
-"%(check_port_listening_script)s" "%(hostname)s" "%(port)s"
-exit $?

slapos/recipe/kvm/template/slapreport_run.in

-#!/bin/sh
-# BEWARE: This file is operated by slapgrid
-# BEWARE: It will be overwritten automatically
-exec %(python_path)s %(slapreport_path)s $1 %(database_path)s

slapos/recipe/kvm_frontend/__init__.py

+##############################################################################
+#
+# Copyright (c) 2010 Vifib SARL and Contributors. All Rights Reserved.
+#
+# WARNING: This program as such is intended to be used by professional
+# programmers who take the whole responsibility of assessing all potential
+# consequences resulting from its eventual inadequacies and bugs
+# End users who are looking for a ready-to-use solution with commercial
+# guarantees and support are strongly adviced to contract a Free Software
+# Service Company
+#
+# This program is Free Software; you can redistribute it and/or
+# modify it under the terms of the GNU General Public License
+# as published by the Free Software Foundation; either version 3
+# of the License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA  02111-1307, USA.
+#
+##############################################################################
+from slapos.recipe.librecipe import BaseSlapRecipe
+import os
+import pkg_resources
+import hashlib
+import string
+import sys
+import zc.buildout
+import zc.recipe.egg
+import ConfigParser
+
+TRUE_VALUE_LIST = ['y', 'yes', '1', 'true']
+FALSE_VALUE_LIST = ['n', 'no', '0', 'false']
+
+class Recipe(BaseSlapRecipe):
+
+  def getTemplateFilename(self, template_name):
+    return pkg_resources.resource_filename(__name__,
+        'template/%s' % template_name)
+
+  def _install(self):
+    self.path_list = []
+    self.requirements, self.ws = self.egg.working_set()
+
+#     frontend_port_number = self.parameter_dict.get("port", 4443)
+#     frontend_domain_name = self.parameter_dict.get("domain",
+#         "host.vifib.net")
+
+    # Create http server redirecting (302) to https proxy?
+    redirect_plain_http = self.parameter_dict.get("redirect_plain_http", '')
+    if redirect_plain_http in TRUE_VALUE_LIST:
+      redirect_plain_http = '1'
+    
+    # Cert stuffs
+    valid_certificate_str = self.parameter_dict.get('domain_ssl_ca_cert')
+    valid_key_str = self.parameter_dict.get('domain_ssl_ca_key')
+    if valid_certificate_str is None and valid_key_str is None:
+      ca_conf = self.installCertificateAuthority()
+      key, certificate = self.requestCertificate(frontend_domain_name)
+    else:
+      ca_conf = self.installValidCertificateAuthority(
+          frontend_domain_name, valid_certificate_str, valid_key_str)
+      key = ca_conf.pop('key')
+      certificate = ca_conf.pop('certificate')
+
+    # Install node + js script
+#     node_parameter_dict = self.installFrontendNode(
+#         ip=self.getGlobalIPv6Address(),
+#         port=frontend_port_number,
+#         plain_http=redirect_plain_http,
+#         name=frontend_domain_name,
+#         slave_instance_list=self.parameter_dict.get('slave_instance_list', []),
+#         key=key, certificate=certificate)
+
+    # Send connection parameters of master instance
+    site_url = node_parameter_dict['site_url']
+    self.setConnectionDict(
+      dict(site_url=site_url,
+           domain_ipv6_address=self.getGlobalIPv6Address()))
+    # Send connection parameters of slave instances
+    for slave in node_parameter_dict['rewrite_rule_list']:
+      self.setConnectionDict(
+          dict(site_url="%s%s" % (site_url, slave['resource']),
+               domainname=frontend_domain_name,
+               port=frontend_port_number,
+               resource=slave['resource']),
+          slave['reference'])
+
+    return self.path_list
+
+
+  def installFrontendNode(self, ip, port, key, certificate, plain_http,
+                            name, slave_instance_list):
+
+    return dict(site_url="https://%s:%s/" % (name, port),
+                rewrite_rule_list=rewrite_rule_list)

slapos/recipe/kvm_frontend/template/logrotate_entry.in

+%(file_list)s {
+  daily
+  dateext
+  rotate 30
+  compress
+  notifempty
+  sharedscripts
+  create
+  postrotate
+    %(postrotate)s
+  endscript
+  olddir %(olddir)s
+}

slapos/recipe/kvm/template/openssl.cnf.ca.in → slapos/recipe/kvm_frontend/template/openssl.cnf.ca.in

@@ -71,7 +71,7 @@ cert_opt      = ca_default		  # Certificate field options
 # crl_extensions	= crl_ext
 
 default_days	= 3650			# how long to certify for
-default_crl_days = 30			      # how long before next CRL
+default_crl_days= 30			# how long before next CRL
 default_md	= default		# use public key default MD
 preserve	= no			# keep passed DN ordering
 

slapos/recipe/kvm_frontend/template/proxytable-vifib-snippet.json.in

+
+  "/hosting": {
+    "port": 15000,
+    "host": "2a01:e35:2e27:460:e2cb:4eff:fed9:ddd6",
+    "https": true
+  }
+

slapos/recipe/nbdserver/__init__.py

 ##############################################################################
 #
-# Copyright (c) 2010 Vifib SARL and Contributors. All Rights Reserved.
+# Copyright (c) 2011 Vifib SARL and Contributors. All Rights Reserved.
 #
 # WARNING: This program as such is intended to be used by professional
 # programmers who take the whole responsibility of assessing all potential
@@ -24,59 +24,29 @@
 # Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA  02111-1307, USA.
 #
 ##############################################################################
-import os
+from slapos.recipe.librecipe import GenericBaseRecipe
 import binascii
-from slapos.recipe.librecipe import BaseSlapRecipe
-
-import pkg_resources
-
-class Recipe(BaseSlapRecipe):
-
-  def _install(self):
-
-    # Image path
-    cdrom_iso = os.path.join(self.data_root_directory, 'cdrom.iso')
-
-    #Get the IP list
-    ip = self.getGlobalIPv6Address()
-    http_port = 9999
-    nbd_port = 1024
-
-    # Instanciate onetimeupload
-    onetimeupload_config = {}
-    onetimeupload_config.update(self.options)
-    onetimeupload_config['port'] = http_port
-    onetimeupload_config['ip'] = ip
-    onetimeupload_config['image'] = cdrom_iso
-    onetimeupload_config['key'] = binascii.hexlify(os.urandom(24))
-    onetimeupload_config['log_path'] = os.path.join(self.log_directory, 
-                                                    'onetimeupload.log')
-
-    wrapper_template_location = pkg_resources.resource_filename(
-                                       __name__, os.path.join(
-                                       'template', 'onetimeupload_run.in'))
-    onetimeupload_runner_path = self.createRunningWrapper("onetimeupload",
-        self.substituteTemplate(wrapper_template_location, 
-                                onetimeupload_config))
-
-    # Instanciate qemu
-    qemu_config = {}
-    qemu_config.update(self.options)
-    qemu_config['ip'] = ip
-    qemu_config['port'] = nbd_port
-    qemu_config['image'] = cdrom_iso
-
-    wrapper_template_location = pkg_resources.resource_filename(
-                                             __name__, os.path.join(
-                                             'template', 'nbdserver_run.in'))
-    nbdserver_runner_path = self.createRunningWrapper("nbdserver",
-        self.substituteTemplate(wrapper_template_location, qemu_config))
-
-    # Publish connection dict
-    self.computer_partition.setConnectionDict(dict(
-      upload_connection_string="https://[%s]:%s/" % (ip, http_port),
-      upload_key=onetimeupload_config['key'],
-      nbd_connection_string="nbd:[%s]:%s" % (ip, nbd_port),
-      ))
-
-    return [onetimeupload_runner_path, nbdserver_runner_path]
+import os
+import sys
+
+class Recipe(GenericBaseRecipe):
+  """
+  nbd instance configuration.
+  """
+
+  def install(self):
+    config = dict(
+      ip=self.options['ip'],
+      port=self.options['port'],
+      image_path=self.options['image-path'],
+      qemu_path=self.options['qemu-path'],
+      shell_path=self.options['shell-path'],
+    )
+
+    # Runners
+    runner_path = self.createExecutable(
+      self.options['path'],
+      self.substituteTemplate(self.getTemplateFilename('nbdserver_run.in'),
+                              config))
+
+    return [runner_path]

slapos/recipe/nbdserver/template/nbdserver_run.in

-#!/bin/sh
+#!%(shell_path)s
 # BEWARE: This file is operated by slapgrid
 # BEWARE: It will be overwritten automatically
 
 # 32767 is the maximum number of connections allowed by the nbd server
-exec %(qemu_path)s -b %(ip)s %(image)s -r -t -p %(port)s -e 32767
+exec %(qemu_path)s -b %(ip)s %(image_path)s -r -t -p %(port)s -e 32767

slapos/recipe/softwaretype.py

@@ -64,6 +64,15 @@ class Recipe:
     # XXX: Lack checking for globality of address
     return self._getIpAddress(netaddr.valid_ipv6)
 
+  def getNetworkInterface(self):
+    """Returns the network interface available on partition"""
+    if not 'ip_list' in self.parameter_dict:
+      raise AttributeError
+    for name, ip in self.parameter_dict['ip_list']:
+      if name:
+        return name
+    raise AttributeError, "Not network interface found"
+
   def install(self):
     slap = slapos.slap.slap()
     slap_connection = self.buildout['slap_connection']
@@ -111,6 +120,8 @@ class Recipe:
                  self.getLocalIPv4Address())
     buildout.set('slap-network-information', 'global-ipv6',
                  self.getGlobalIPv6Address())
+    buildout.set('slap-network-information', 'network-interface', 
+                 self.getNetworkInterface())
 
     # Copy/paste slap_connection
     buildout.add_section('slap-connection')

software/kvm/instance-frontend.cfg

+#############################
+#
+# Instanciate kvm frontend
+#
+#############################
+[buildout]
+parts =
+  logrotate
+#   logrotate-entry-frontend
+  cron
+  cron-entry-logrotate
+  ca-frontend
+  certificate-authority
+  frontend-promise
+
+eggs-directory = ${buildout:eggs-directory}
+develop-eggs-directory = ${buildout:develop-eggs-directory}
+offline = true
+
+[rootdirectory]
+recipe = slapos.cookbook:mkdirectory
+etc = $${buildout:directory}/etc
+bin = $${buildout:directory}/bin
+srv = $${buildout:directory}/srv
+var = $${buildout:directory}/var
+
+[basedirectory]
+recipe = slapos.cookbook:mkdirectory
+services = $${rootdirectory:etc}/run
+promises = $${rootdirectory:etc}/promise
+nodejs-conf = $${rootdirectory:etc}/nodejs
+run = $${rootdirectory:var}/run
+log = $${rootdirectory:var}/log
+ca-dir = $${rootdirectory:srv}/ssl
+backup = $${rootdirectory:srv}/backup
+
+[directory]
+recipe = slapos.cookbook:mkdirectory
+cron-entries = $${rootdirectory:etc}/cron.d
+crontabs = $${rootdirectory:etc}/crontabs
+cronstamps = $${rootdirectory:etc}/cronstamps
+ca-dir = $${rootdirectory:srv}/ssl
+logrotate-backup = $${basedirectory:backup}/logrotate
+logrotate-entries = $${rootdirectory:etc}/logrotate.d
+
+[frontend-instance]
+recipe = slapos.cookbook:generic.kvm.frontend
+domain = $${ca-frontend:name}
+# port = $${slap-parameter:port}
+ip = $${slap-network-information:global-ipv6}
+port = 4443
+# http-port = 
+# https-port = 
+ssl-key-path = $${ca-frontend:key-file}
+ssl-cert-path = $${ca-frontend:cert-file}
+slave-instance-list = $${slap-parameter:slave_instance_list}
+map-path = $${basedirectory:nodejs-conf}/proxy_table.json
+conf-path = $${basedirectory:nodejs-conf}/kvm-proxy.js
+wrapper-path = $${rootdirectory:bin}/kvm_frontend
+node-binary = ${nodejs:location}/bin/node
+node-env = ${buildout:parts-directory}:${npm-modules:location}/node_modules
+shell-path = ${dash:location}/bin/dash
+
+[frontend-promise]
+recipe = slapos.cookbook:check_port_listening
+path = $${basedirectory:promises}/frontend_promise
+hostname = $${frontend-instance:ip}
+port = $${frontend-instance:port}
+
+[certificate-authority]
+recipe = slapos.cookbook:certificate_authority
+openssl-binary = ${openssl:location}/bin/openssl
+ca-dir = $${basedirectory:ca-dir}
+requests-directory = $${cadirectory:requests}
+wrapper = $${basedirectory:services}/certificate_authority
+ca-private = $${cadirectory:private}
+ca-certs = $${cadirectory:certs}
+ca-newcerts = $${cadirectory:newcerts}
+ca-crl = $${cadirectory:crl}
+
+[cadirectory]
+recipe = slapos.cookbook:mkdirectory
+requests = $${basedirectory:ca-dir}/requests/
+private = $${basedirectory:ca-dir}/private/
+certs = $${basedirectory:ca-dir}/certs/
+newcerts = $${basedirectory:ca-dir}/newcerts/
+crl = $${basedirectory:ca-dir}/crl/
+
+[ca-frontend]
+<= certificate-authority
+recipe = slapos.cookbook:certificate_authority.request
+key-file = $${basedirectory:nodejs-conf}/nodejs.key
+cert-file = $${basedirectory:nodejs-conf}/nodejs.crt
+executable = $${frontend-instance:wrapper-path}
+wrapper = $${basedirectory:services}/nodejs
+# Put domain name
+name = $${slap-parameter:domain}
+
+[cron]
+recipe = slapos.cookbook:cron
+dcrond-binary = ${dcron:location}/sbin/crond
+cron-entries = $${directory:cron-entries}
+crontabs = $${directory:crontabs}
+cronstamps = $${directory:cronstamps}
+catcher = $${cron-simplelogger:wrapper}
+binary = $${basedirectory:services}/crond
+
+[cron-simplelogger]
+recipe = slapos.cookbook:simplelogger
+wrapper = $${rootdirectory:bin}/cron_simplelogger
+log = $${basedirectory:log}/cron.log
+
+[cron-entry-logrotate]
+<= cron
+recipe = slapos.cookbook:cron.d
+name = logrotate
+frequency = 0 0 * * *
+command = $${logrotate:wrapper}
+
+[logrotate]
+recipe = slapos.cookbook:logrotate
+# Binaries
+logrotate-binary = ${logrotate:location}/usr/sbin/logrotate
+gzip-binary = ${gzip:location}/bin/gzip
+gunzip-binary = ${gzip:location}/bin/gunzip
+# Directories
+wrapper = $${rootdirectory:bin}/logrotate
+conf = $${rootdirectory:etc}/logrotate.conf
+logrotate-entries = $${directory:logrotate-entries}
+backup = $${directory:logrotate-backup}
+state-file = $${rootdirectory:srv}/logrotate.status
+
+# [logrotate-entry-frontend]
+# <= logrotate
+# recipe = slapos.cookbook:logrotate.d
+# name = frontend
+# log = $${mariadb-instance:error-log} $${mariadb-instance:slow-query-log}
+# post = $${mariadb-instance:mysql-binary} --no-defaults -B --socket=$${mariadb-instance:socket} -e "FLUSH LOGS"

software/kvm/instance-kvm.cfg

+#############################
+#
+# Instanciate kvm
+#
+#############################
+[buildout]
+parts =
+  request-slave-frontend
+  certificate-authority
+  kvm-promise
+  novnc-promise
+  publish-kvm-connection-information
+
+eggs-directory = ${buildout:eggs-directory}
+develop-eggs-directory = ${buildout:develop-eggs-directory}
+offline = true
+
+[rootdirectory]
+recipe = slapos.cookbook:mkdirectory
+etc = $${buildout:directory}/etc
+bin = $${buildout:directory}/bin
+srv = $${buildout:directory}/srv
+var = $${buildout:directory}/var
+
+[basedirectory]
+recipe = slapos.cookbook:mkdirectory
+services = $${rootdirectory:etc}/run
+promises = $${rootdirectory:etc}/promise
+novnc-conf = $${rootdirectory:etc}/novnc
+run = $${rootdirectory:var}/run
+ca-dir = $${rootdirectory:srv}/ssl
+
+[create-mac]
+recipe = slapos.cookbook:generate.mac
+
+[kvm-instance]
+recipe = slapos.cookbook:kvm
+vnc-ip = $${slap-network-information:local-ipv4}
+vnc-port = 5901
+nbd-ip = $${slap-parameter:nbd_ip}
+nbd-port = $${slap-parameter:nbd_port}
+tap = $${slap-network-information:network-interface}
+disk-path = $${rootdirectory:srv}/virtual.qcow2
+disk-size = 10
+socket-path = $${rootdirectory:var}/qmp_socket
+pid-path = $${basedirectory:run}/pid_file
+smp-count = 1
+ram-size = 1024
+mac-address = $${create-mac:mac-address}
+runner-path = $${basedirectory:services}/kvm
+controller-path = $${basedirectory:services}/kvm_controller
+shell-path = ${dash:location}/bin/dash
+qemu-path = ${kvm:location}/bin/qemu-system-x86_64
+qemu-img-path = ${kvm:location}/bin/qemu-img
+
+[kvm-promise]
+recipe = slapos.cookbook:check_port_listening
+path = $${basedirectory:promises}/vnc_promise
+hostname = $${kvm-instance:vnc-ip}
+port = $${kvm-instance:vnc-port}
+
+[novnc-instance]
+recipe = slapos.cookbook:generic.novnc
+path = $${ca-novnc:executable}
+ip = $${slap-network-information:global-ipv6}
+port = 6080
+vnc-ip = $${kvm-instance:vnc-ip}
+vnc-port = $${kvm-instance:vnc-port}
+novnc-location = ${noVNC:location}
+websockify-path = ${buildout:directory}/bin/websockify
+ssl-key-path = $${ca-novnc:key-file}
+ssl-cert-path = $${ca-novnc:cert-file}
+
+[certificate-authority]
+recipe = slapos.cookbook:certificate_authority
+openssl-binary = ${openssl:location}/bin/openssl
+ca-dir = $${basedirectory:ca-dir}
+requests-directory = $${cadirectory:requests}
+wrapper = $${basedirectory:services}/certificate_authority
+ca-private = $${cadirectory:private}
+ca-certs = $${cadirectory:certs}
+ca-newcerts = $${cadirectory:newcerts}
+ca-crl = $${cadirectory:crl}
+
+[cadirectory]
+recipe = slapos.cookbook:mkdirectory
+requests = $${basedirectory:ca-dir}/requests/
+private = $${basedirectory:ca-dir}/private/
+certs = $${basedirectory:ca-dir}/certs/
+newcerts = $${basedirectory:ca-dir}/newcerts/
+crl = $${basedirectory:ca-dir}/crl/
+
+[ca-novnc]
+<= certificate-authority
+recipe = slapos.cookbook:certificate_authority.request
+key-file = $${basedirectory:novnc-conf}/novnc.key
+cert-file = $${basedirectory:novnc-conf}/novnc.crt
+executable = $${rootdirectory:bin}/novnc
+wrapper = $${basedirectory:services}/websockify
+
+[novnc-promise]
+recipe = slapos.cookbook:check_port_listening
+path = $${basedirectory:promises}/novnc_promise
+hostname = $${novnc-instance:ip}
+port = $${novnc-instance:port}
+
+[kvm-monitor]
+recipe = slapos.cookbook:generic.slapmonitor
+db-path = $${rootdirectory:srv}/slapmonitor_database
+
+[request-common]
+recipe = slapos.cookbook:request
+software-url = $${slap-connection:software-release-url}
+sla = computer_guid
+sla-computer_guid = $${slap-connection:computer-id}
+server-url = $${slap-connection:server-url}
+key-file = $${slap-connection:key-file}
+cert-file = $${slap-connection:cert-file}
+computer-id = $${slap-connection:computer-id}
+partition-id = $${slap-connection:partition-id}
+
+[request-slave-frontend]
+<=request-common
+name = SlaveFrontend
+software-type = frontend
+slave = true
+return = frontend_url
+
+[publish-kvm-connection-information]
+recipe = slapos.cookbook:publish
+backend_url = https://[$${novnc-instance:ip}]:$${novnc-instance:port}/vnc_auto.html?host=[$${novnc-instance:ip}]&port=$${novnc-instance:port}&encrypt=1
+frontend_url = $${request-slave-frontend:frontend_url}
+vnc_passwd = $${kvm-instance:passwd}

software/kvm/instance-kvmplus.cfg

+#############################
+#
+# Instanciate kvm+
+#
+#############################
+[buildout]
+extends = instance-kvm.cfg
+
+[kvm-instance]
+recipe = slapos.cookbook:kvm
+vnc-ip = $${slap-network-information:local-ipv4}
+vnc-port = 5901
+nbd-ip = $${slap-parameter:nbd_ip}
+nbd-port = $${slap-parameter:nbd_port}
+tap = $${slap-network-information:network-interface}
+disk-path = $${rootdirectory:srv}/virtual.qcow2
+disk-size = 20
+socket-path = $${rootdirectory:var}/qmp_socket
+pid-path = $${basedirectory:run}/pid_file
+smp-count = 2
+ram-size = 2048
+mac-address = $${create-mac:mac-address}
+runner-path = $${basedirectory:services}/kvm
+controller-path = $${basedirectory:services}/kvm_controller
+shell-path = ${dash:location}/bin/dash
+qemu-path = ${kvm:location}/bin/qemu-system-x86_64
+qemu-img-path = ${kvm:location}/bin/qemu-img

software/kvm/instance-nbd.cfg

+#############################
+#
+# Instanciate nbdserver
+#
+#############################
+[buildout]
+parts =
+  nbd-promise
+  onetimeupload-promise
+  publish-connection-information
+
+eggs-directory = ${buildout:eggs-directory}
+develop-eggs-directory = ${buildout:develop-eggs-directory}
+offline = true
+
+[rootdirectory]
+recipe = slapos.cookbook:mkdirectory
+etc = $${buildout:directory}/etc
+srv = $${buildout:directory}/srv
+log = $${buildout:directory}/log
+
+[basedirectory]
+recipe = slapos.cookbook:mkdirectory
+services = $${rootdirectory:etc}/run
+promises = $${rootdirectory:etc}/promise
+
+[nbd-instance]
+recipe = slapos.cookbook:nbdserver
+ip = $${slap-network-information:global-ipv6}
+port = 1024
+image-path = $${onetimeupload-instance:image-path}
+qemu-path = ${kvm:location}/bin/qemu-nbd
+shell-path = ${dash:location}/bin/dash
+# XXX TODO: Wait for the iso to be uploaded (execute_wait)
+path = $${basedirectory:services}/nbdserver
+
+[nbd-promise]
+recipe = slapos.cookbook:check_port_listening
+path = $${basedirectory:promises}/nbd_promise
+hostname = $${nbd-instance:ip}
+port = $${nbd-instance:port}
+
+[onetimeupload-instance]
+recipe = slapos.cookbook:generic.onetimeupload
+ip = $${slap-network-information:global-ipv6}
+port = 9999
+image-path = $${rootdirectory:srv}/cdrom.iso
+log-path = $${rootdirectory:log}/onetimeupload.log
+shell-path = ${dash:location}/bin/dash
+onetimeupload-path = ${buildout:bin-directory}/onetimeupload
+path = $${basedirectory:services}/onetimeupload
+
+[onetimeupload-promise]
+recipe = slapos.cookbook:check_port_listening
+path = $${basedirectory:promises}/onetimeupload_promise
+hostname = $${onetimeupload-instance:ip}
+port = $${onetimeupload-instance:port}
+
+[publish-connection-information]
+recipe = slapos.cookbook:publish
+nbd_url = nbd://[$${nbd-instance:ip}]:$${nbd-instance:port}
+upload_url = http://[$${onetimeupload-instance:ip}]:$${onetimeupload-instance:port}/
+upload_key = $${onetimeupload-instance:key}

software/kvm/instance.cfg

 [buildout]
 parts =
-  kvminstance
+  switch-softwaretype
 
 eggs-directory = ${buildout:eggs-directory}
 develop-eggs-directory = ${buildout:develop-eggs-directory}
+offline = true
 
-[kvminstance]
-recipe = slapos.cookbook:kvm
-qemu_path = ${kvm:location}/bin/qemu-system-x86_64
-qemu_img_path = ${kvm:location}/bin/qemu-img
-#slapmonitor_path = ${buildout:bin-directory}/slapmonitor
-#slapreport_path = ${buildout:bin-directory}/slapreport
-websockify = ${buildout:directory}/bin/websockify
-noVNC_location = ${noVNC:location}
-openssl_binary = ${openssl:location}/bin/openssl
-rdiff_backup_binary = ${buildout:bin-directory}/rdiff-backup
-dcrond_binary = ${dcron:location}/sbin/crond
+[switch-softwaretype]
+recipe = slapos.cookbook:softwaretype
+default = ${template-kvm:output}
+kvm = ${template-kvm:output}
+kvm+ = ${template-kvmplus:output}
+nbd = ${template-nbd:output}
+frontend = ${template-frontend:output}
 
-smp_count = 1
-ram_size = 1024
-disk_size = 10
+[slap-connection]
+# part to migrate to new - separated words
+computer-id = $${slap_connection:computer_id}
+partition-id = $${slap_connection:partition_id}
+server-url = $${slap_connection:server_url}
+software-release-url = $${slap_connection:software_release_url}
+key-file = $${slap_connection:key_file}
+cert-file = $${slap_connection:cert_file}

software/kvm/software.cfg

@@ -3,26 +3,37 @@ extensions =
   buildout-versions
   
 extends =
+  ../../component/gzip/buildout.cfg
   ../../component/dcron/buildout.cfg
+  ../../component/logrotate/buildout.cfg
   ../../component/git/buildout.cfg
   ../../component/gnutls/buildout.cfg
   ../../component/libpng/buildout.cfg
   ../../component/libuuid/buildout.cfg
-  ../../component/lxml-python/buildout.cfg
   ../../component/noVNC/buildout.cfg
   ../../component/openssl/buildout.cfg
-  ../../component/python-2.7/buildout.cfg
-  ../../component/rdiff-backup/buildout.cfg
+  ../../component/dash/buildout.cfg
+  ../../component/lxml-python/buildout.cfg
+  ../../stack/nodejs.cfg
   ../../stack/shacache-client.cfg
 
 develop =
   ${:parts-directory}/websockify
+  /opt/slapdev
 
 parts =
   template
+  dash
   kvm
   eggs
   check-local-eggs
+  check-local-eggs2
+  nodejs
+  http-proxy
+  proxy-by-url
+  npm-modules
+  dcron
+  logrotate
 
 find-links +=
   http://www.nexedi.org/static/packages/source/slapos.buildout/
@@ -42,6 +53,12 @@ allow-hosts =
   psutil.googlecode.com
   www.dabeaz.com
   www.owlfish.com
+  launchpad.net
+
+# XXX: Workaround of SlapOS limitation
+# Unzippig of eggs is required, as SlapOS do not yet provide nicely working
+# development / fast switching environment for whole software
+unzip = true
 
 #XXX-Cedric : Currently, one can only access to KVM using noVNC.
 #             Ideally one should be able to access KVM by using either NoVNC or VNC.
@@ -63,6 +80,9 @@ allow-hosts =
 [kvm]
 recipe = hexagonit.recipe.cmmi
 url = http://downloads.sourceforge.net/project/kvm/qemu-kvm/0.15.1/qemu-kvm-0.15.1.tar.gz
+# XXX-Cedric : Upgrade to 1.0
+# url = http://downloads.sourceforge.net/project/kvm/qemu-kvm/1.0/qemu-kvm-1.0.tar.gz
+# md5sum = 00a825db46a70ba8ef9fc95da9cc7c1e
 md5sum = 8800a7d6b3aa4a168ea7f78dc66c0320
 configure-options =
   --disable-sdl
@@ -86,8 +106,8 @@ environment =
 # XXX-Cedric : use official egg from pypi when it is released
 recipe = plone.recipe.command
 stop-on-error = true
-commit = e7363f43443deb9982bdb5c3db50eec475584b06
-repository = https://github.com/desaintmartin/websockify.git
+commit = 301f3ae580557da47fa5ea2050aa671ce9c5a1a0
+repository = https://github.com/SlapOS/websockify.git
 location = ${buildout:parts-directory}/${:_buildout_section_name_}
 git-binary = ${git:location}/bin/git
 command = export GIT_SSL_NO_VERIFY=true; (${:git-binary} clone --quiet ${:repository} ${:location} && cd ${:location} && ${:git-binary} reset --hard ${:commit}) || (rm -fr ${:location}; exit 1)
@@ -100,84 +120,119 @@ update-command = ${:command}
 command = grep parts ${buildout:develop-eggs-directory}/websockify.egg-link
 depends = ${eggs:dummy}
 
+[check-local-eggs2]
+recipe = plone.recipe.command
+stop-on-error = true
+update-command = ${:command}
+command = grep slapdev ${buildout:develop-eggs-directory}/slapos.cookbook.egg-link
+depends = ${eggs:dummy}
+
 [eggs]
-python = python2.7
 recipe = z3c.recipe.scripts
 dummy =
   ${websockify:location}
 eggs =
   ${lxml-python:egg}
-  slapos.cookbook
   websockify
+  slapos.cookbook
+  slapos.toolbox
+
+[http-proxy]
+# https://github.com/nodejitsu/node-http-proxy
+recipe = slapos.recipe.build:download-unpacked
+#XXX-Cedric : use upstream when merged
+url = https://nodeload.github.com/desaintmartin/node-http-proxy/zipball/master
+md5sum = 20204d0b29c2cef26e1c91e99eedca6b
+
+[proxy-by-url]
+# https://github.com/dominictarr/proxy-by-url
+recipe = slapos.recipe.build:download-unpacked
+#XXX-Cedric : use upstream when merged
+url = https://nodeload.github.com/desaintmartin/proxy-by-url/zipball/master
+md5sum = f6a7c4f8f01f3049086a1dbde2cd141c
+
+[npm-modules]
+recipe = plone.recipe.command
+destination = ${buildout:parts-directory}/${:_buildout_section_name_}
+location = ${buildout:parts-directory}/${:_buildout_section_name_}
+command =
+  rm -fr ${:destination} &&
+  mkdir -p ${:destination} &&
+  cd ${:destination} &&
+  ${nodejs:location}/bin/node ${nodejs:location}/bin/npm install colors@0.6.0-1 &&
+  ${nodejs:location}/bin/node ${nodejs:location}/bin/npm install socket.io@0.8.7 &&
+  ${nodejs:location}/bin/node ${nodejs:location}/bin/npm install socket.io-client@0.8.7 &&
+  ${nodejs:location}/bin/node ${nodejs:location}/bin/npm install optimist@0.3.1 &&
+  ${nodejs:location}/bin/node ${nodejs:location}/bin/npm install pkginfo@0.2.3
+
+
+[template-kvm]
+recipe = slapos.recipe.template
+url = ${:_profile_base_location_}/instance-kvm.cfg
+md5sum = ce62abe1edabc78a4baa574a39d7faa0
+output = ${buildout:directory}/template-kvm.cfg
+mode = 0644
+
+[template-kvmplus]
+recipe = slapos.recipe.template
+url = ${:_profile_base_location_}/instance-kvmplus.cfg
+md5sum = 2e35c5b2ac9ee51d8f98fb1199f011c4
+output = ${buildout:directory}/template-kvmplus.cfg
+mode = 0644
+
+[template-nbd]
+recipe = slapos.recipe.template
+url = ${:_profile_base_location_}/instance-nbd.cfg
+md5sum = 7691fadfc8d4392c58ac1bf0ebd5aaf2
+output = ${buildout:directory}/template-nbd.cfg
+mode = 0644
+
+[template-frontend]
+recipe = slapos.recipe.template
+url = ${:_profile_base_location_}/instance-frontend.cfg
+md5sum = 16a28991ce59f215f11d7cc61906d64e
+output = ${buildout:directory}/template-frontend.cfg
+mode = 0644
 
 [template]
 recipe = slapos.recipe.template
 url = ${:_profile_base_location_}/instance.cfg
-md5sum = 298b146e4efce41bfd58b3f85d064ff1
+md5sum = 68788763d23f70f24b9e575871c903a8
 output = ${buildout:directory}/template.cfg
 mode = 0644
 
 [versions]
-zc.buildout = 1.5.3-dev-SlapOS-010
-
-slapos.cookbook = 0.37
-Jinja2 = 2.6
-Werkzeug = 0.8.1
-buildout-versions = 1.7
-hexagonit.recipe.cmmi = 1.5.0
-lxml = 2.3.2
-meld3 = 0.6.7
-plone.recipe.command = 1.1
-slapos.recipe.template = 2.2
-z3c.recipe.scripts = 1.0.1
-
-# Required by:
-# slapos.core==0.20
-Flask = 0.8
-
-# Required by:
-# slapos.cookbook==0.37
-PyXML = 0.8.4
-
-# Required by:
-# hexagonit.recipe.cmmi==1.5.0
-hexagonit.recipe.download = 1.5.0
-
-# Required by:
-# slapos.cookbook==0.37
-netaddr = 0.7.6
-
-# Required by:
-# slapos.core==0.20
-netifaces = 0.6
-
-# Required by:
-# websockify==0.1-dev
+# Use SlapOS patched zc.buildout
+zc.buildout = 1.6.0-dev-SlapOS-004
+
+[networkcache]
+# signature certificates of the following uploaders.
+#   Romain Courteaud
+#   Cedric de Saint Martin
+signature-certificate-list =
+  -----BEGIN CERTIFICATE-----
+  MIIB4DCCAUkCADANBgkqhkiG9w0BAQsFADA5MQswCQYDVQQGEwJGUjEZMBcGA1UE
+  CBMQRGVmYXVsdCBQcm92aW5jZTEPMA0GA1UEChMGTmV4ZWRpMB4XDTExMDkxNTA5
+  MDAwMloXDTEyMDkxNTA5MDAwMlowOTELMAkGA1UEBhMCRlIxGTAXBgNVBAgTEERl
+  ZmF1bHQgUHJvdmluY2UxDzANBgNVBAoTBk5leGVkaTCBnzANBgkqhkiG9w0BAQEF
+  AAOBjQAwgYkCgYEApYZv6OstoqNzxG1KI6iE5U4Ts2Xx9lgLeUGAMyfJLyMmRLhw
+  boKOyJ9Xke4dncoBAyNPokUR6iWOcnPHtMvNOsBFZ2f7VA28em3+E1JRYdeNUEtX
+  Z0s3HjcouaNAnPfjFTXHYj4um1wOw2cURSPuU5dpzKBbV+/QCb5DLheynisCAwEA
+  ATANBgkqhkiG9w0BAQsFAAOBgQBCZLbTVdrw3RZlVVMFezSHrhBYKAukTwZrNmJX
+  mHqi2tN8tNo6FX+wmxUUAf3e8R2Ymbdbn2bfbPpcKQ2fG7PuKGvhwMG3BlF9paEC
+  q7jdfWO18Zp/BG7tagz0jmmC4y/8akzHsVlruo2+2du2freE8dK746uoMlXlP93g
+  QUUGLQ==
+  -----END CERTIFICATE-----
+  -----BEGIN CERTIFICATE-----
+  MIIB9jCCAV+gAwIBAgIJAO4V/jiMoICoMA0GCSqGSIb3DQEBBQUAMBMxETAPBgNV
+  BAMMCENPTVAtMjMyMCAXDTEyMDIxNjExMTAyM1oYDzIxMTIwMTIzMTExMDIzWjAT
+  MREwDwYDVQQDDAhDT01QLTIzMjCBnzANBgkqhkiG9w0BAQEFAAOBjQAwgYkCgYEA
+  wi/3Z8W9pUiegUXIk/AiFDQ0UJ4JFAwjqr+HSRUirlUsHHT+8DzH/hfcTDX1I5BB
+  D1ADk+ydXjMm3OZrQcXjn29OUfM5C+g+oqeMnYQImN0DDQIOcUyr7AJc4xhvuXQ1
+  P2pJ5NOd3tbd0kexETa1LVhR6EgBC25LyRBRae76qosCAwEAAaNQME4wHQYDVR0O
+  BBYEFMDmW9aFy1sKTfCpcRkYnP6zUd1cMB8GA1UdIwQYMBaAFMDmW9aFy1sKTfCp
+  cRkYnP6zUd1cMAwGA1UdEwQFMAMBAf8wDQYJKoZIhvcNAQEFBQADgYEAskbFizHr
+  b6d3iIyN+wffxz/V9epbKIZVEGJd/6LrTdLiUfJPec7FaxVCWNyKBlCpINBM7cEV
+  Gn9t8mdVQflNqOlAMkOlUv1ZugCt9rXYQOV7rrEYJBWirn43BOMn9Flp2nibblby
+  If1a2ZoqHRxoNo2yTmm7TSYRORWVS+vvfjY=
+  -----END CERTIFICATE-----
\ No newline at end of file
-numpy = 1.6.1
-
-# Required by:
-# slapos.cookbook==0.37
-# slapos.core==0.20
-# zc.buildout==1.5.3-dev-SlapOS-010
-# zc.recipe.egg==1.3.2
-setuptools = 0.6c12dev-r88846
-
-# Required by:
-# slapos.cookbook==0.37
-slapos.core = 0.20
-
-# Required by:
-# slapos.core==0.20
-supervisor = 3.0a10
-
-# Required by:
-# slapos.cookbook==0.37
-xml-marshaller = 0.9.7
-
-# Required by:
-# slapos.cookbook==0.37
-zc.recipe.egg = 1.3.2
-
-# Required by:
-# slapos.core==0.20
-zope.interface = 3.8.0
\ No newline at end of file

software/nbd/instance.cfg

-[buildout]
-parts =
-  nbdserverinstance
-
-eggs-directory = ${buildout:eggs-directory}
-develop-eggs-directory = ${buildout:develop-eggs-directory} 
-
-[nbdserverinstance]
-recipe = ${instance-recipe:egg}:${instance-recipe:module}
-qemu_path = ${nbdserver:location}/bin/qemu-nbd
-onetimeupload_path = ${buildout:bin-directory}/onetimeupload

software/nbd/software.cfg

-[buildout]
-
-extends =
-  ../../stack/nbd.cfg
-  ../../stack/shacache-client.cfg
-
-parts +=
-  template
-
-[template]
-recipe = slapos.recipe.template
-url = ${:_profile_base_location_}/instance.cfg
-md5sum = 82e948e1c0cb0d5540ef185edeef3ec3
-output = ${buildout:directory}/template.cfg
-mode = 0644
-
-[versions]
-# XXX-CEDRIC Quick and dirty workaround to avoid m2crypto problems.
-# should not be used elsewhere unless for urgent cases. 
-slapos.libnetworkcache = 0.2

stack/nodejs.cfg

+[buildout]
+extends =
+  ../component/nodejs/buildout.cfg
+  ../stack/shacache-client.cfg
+
+versions = versions
+
+find-links +=
+    http://www.nexedi.org/static/packages/source/slapos.buildout/
+
+# Use only quite well working sites.
+allow-hosts =
+  *.nexedi.org
+  *.python.org
+  *.sourceforge.net
+  alastairs-place.net
+  dist.repoze.org
+  effbot.org
+  github.com
+  peak.telecommunity.com
+  psutil.googlecode.com
+  www.dabeaz.com
+
+parts =
+  eggs
+  nodejs
+  npm
+
+[eggs]
+recipe = zc.recipe.egg
+eggs =
+  slapos.cookbook