Merge branch 'master' into apache

Conflicts:
	CHANGES.txt
	component/slapos/buildout.cfg
	setup.py
	slapos/recipe/kumofs/__init__.py
	slapos/recipe/memcached/__init__.py
	slapos/recipe/mysql/__init__.py
	software/kumofs/software.cfg
	software/mysql-5.1/software.cfg

CHANGES.txt

-0.13 (unreleased)
-================
+Changes
+=======
 
- * No change yet.
+0.16 (unreleased)
+-----------------
+
+ * No changes yet.
+
+0.15 (2011-07-13)
+-----------------
+
+ * Encrypt connection by default. [Vivien Alger]
+
+0.14 (2011-07-13)
+-----------------
+
+ * Provide new way to instantiate kvm. [Cedric de Saint Martin, Vivien Alger]
+
+0.13 (2011-07-13)
+-----------------
+
+ * Implement generic execute_wait wrapper, which allows to wait for some files
+   to appear before starting service depending on it. [Łukasz Nowak]
 
 0.12 (2011-07-11)
-=================
+-----------------
 
  * Fix slaprunner, phpmyadmin software releases, added
    wordpress software release. [Cedric de Saint Martin]
 
 0.11 (2011-07-07)
-=================
+-----------------
 
  * Enable test suite runner for vifib.
 
 0.10 (2011-07-01)
-=================
+-----------------
 
  * Add PHPMyAdmin software release used in SlapOS tutorials
  [Cedric de Saint Martin]
  * Add slaprunner software release [Cedric de Saint Martin]
 
+
 0.9 (2011-06-24)
-================
+----------------
 
  * mysql recipe : Changing slapos.recipe.erp5.execute to
    slapos.recipe.librecipe.execute [Cedric de Saint Martin]
 
 0.8 (2011-06-15)
-================
+----------------
 
  * Add MySQL and MariaDB standalone software release and recipe
    [Cedric de Saint Martin]
  * Fixed slapos.recipe.erp5testnode instantiation [Sebastien Robin]
 
 0.7 (2011-06-14)
-================
+----------------
 
  * Fix slapos.recipe.erp5 package by providing site.zcml in it. [Łukasz Nowak]
  * Improve slapos.recipe.erp5testnode partition instantiation error reporting
    [Sebastien Robin]
 
 0.6 (2011-06-13)
-================
+----------------
 
  * Fixed slapos.recipe.erp5 instantiation. [Łukasz Nowak]
 
 0.5 (2011-06-13)
-================
+----------------
 
  * Implement zabbix agent instantiation. [Łukasz Nowak]
  * Drop dependency on Zope2. [Łukasz Nowak]
  * Share more in slapos.recipe.librecipe module. [Łukasz Nowak]
 
 0.4 (2011-06-09)
-================
+----------------
 
  * Remove reference to slapos.tool.networkcache as it was removed from pypi. [Łukasz Nowak]
  * Add Kumofs standalone software release and recipe [Cedric de Saint Martin]
  * Add Memcached standalone software release and recipe [Cedric de Saint Martin]
 
 0.3 (2011-06-09)
-================
+----------------
 
  * Moved out template and build to separate distributions [Łukasz Nowak]
  * Depend on slapos.core instead of depracated slapos.slap [Romain Courteaud]
@@ -69,11 +89,11 @@
  * Allow to control full environment in erp5 module [Łukasz Nowak]
 
 0.2 (2011-05-30)
-================
+----------------
 
   * Allow to pass zope_environment in erp5 entry point [Łukasz Nowak]
 
 0.1 (2011-05-27)
-================
+----------------
 
   * All slapos.recipe.* became slapos.cookbook:* [Łukasz Nowak]

README.txt

 slapos.cookbook
 ===============
+
+Cookbook of SlapOS recipes.

component/gzip/buildout.cfg

+[buildout]
+parts =
+  gzip
+
+[gzip]
+recipe = hexagonit.recipe.cmmi
+url = ftp://ftp.gnu.org/pub/gnu/gzip/gzip-1.4.tar.gz
+md5sum = e381b8506210c794278f5527cba0e765

component/noVNC/buildout.cfg

+[buildout]
+parts =
+  noVNC
+
+[noVNC]
+recipe = hexagonit.recipe.download
+url = https://github.com/kanaka/noVNC/tarball/master
+strip-top-level-dir = true

component/slapos/buildout.cfg

@@ -12,6 +12,15 @@ find-links =
 
 versions = versions
 
+allow-hosts =
+  *.googlecode.com
+  *.nexedi.org
+  *.python.org
+  alastairs-place.net
+  code.google.com
+  github.com
+  peak.telecommunity.com
+
 # separate from system python
 include-site-packages = false
 exec-sitecustomize = false

setup.py

@@ -2,7 +2,7 @@ from setuptools import setup, find_packages
 import glob
 import os
 
-version = '0.13dev'
+version = '0.16-dev'
 name = 'slapos.cookbook'
 long_description = open("README.txt").read() + "\n" + \
     open("CHANGES.txt").read() + "\n"

slapos/recipe/kumofs/__init__.py

@@ -220,9 +220,10 @@ class Recipe(BaseSlapRecipe):
         self.substituteTemplate(template_filename,
           stunnel_conf))
     wrapper = zc.buildout.easy_install.scripts([('stunnel',
-      'slapos.recipe.librecipe.execute', 'execute')], self.ws, sys.executable,
-      self.wrapper_directory, arguments=[
-        self.options['stunnel_binary'].strip(), stunnel_conf_path]
+      'slapos.recipe.librecipe.execute', 'execute_wait')], self.ws,
+      sys.executable, self.wrapper_directory, arguments=[
+        [self.options['stunnel_binary'].strip(), stunnel_conf_path],
+        [ca_certificate, key]]
       )[0]
     self.path_list.append(wrapper)
 
@@ -267,4 +268,4 @@ class Recipe(BaseSlapRecipe):
         config['kumo_gateway_port']),
       kumo_gateway_ip=config['kumo_gateway_ip'],
       kumo_gateway_port=config['kumo_gateway_port'],
-    )
\ No newline at end of file
+    )

slapos/recipe/kvm/certificate_authority.py

+import os
+import subprocess
+import time
+import ConfigParser
+
+
+def popenCommunicate(command_list, input=None):
+  subprocess_kw = dict(stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
+  if input is not None:
+    subprocess_kw.update(stdin=subprocess.PIPE)
+  popen = subprocess.Popen(command_list, **subprocess_kw)
+  result = popen.communicate(input)[0]
+  if popen.returncode is None:
+    popen.kill()
+  if popen.returncode != 0:
+    raise ValueError('Issue during calling %r, result was:\n%s' % (
+      command_list, result))
+  return result
+
+
+class CertificateAuthority:
+  def __init__(self, key, certificate, openssl_binary,
+      openssl_configuration, request_dir):
+    self.key = key
+    self.certificate = certificate
+    self.openssl_binary = openssl_binary
+    self.openssl_configuration = openssl_configuration
+    self.request_dir = request_dir
+
+  def checkAuthority(self):
+    file_list = [ self.key, self.certificate ]
+    ca_ready = True
+    for f in file_list:
+      if not os.path.exists(f):
+        ca_ready = False
+        break
+    if ca_ready:
+      return
+    for f in file_list:
+      if os.path.exists(f):
+        os.unlink(f)
+    try:
+      # no CA, let us create new one
+      popenCommunicate([self.openssl_binary, 'req', '-nodes', '-config',
+          self.openssl_configuration, '-new', '-x509', '-extensions',
+          'v3_ca', '-keyout', self.key, '-out', self.certificate,
+          '-days', '10950'], 'Automatic Certificate Authority\n')
+    except:
+      try:
+        for f in file_list:
+          if os.path.exists(f):
+            os.unlink(f)
+      except:
+        # do not raise during cleanup
+        pass
+      raise
+
+  def _checkCertificate(self, common_name, key, certificate):
+    file_list = [key, certificate]
+    ready = True
+    for f in file_list:
+      if not os.path.exists(f):
+        ready = False
+        break
+    if ready:
+      return False
+    for f in file_list:
+      if os.path.exists(f):
+        os.unlink(f)
+    csr = certificate + '.csr'
+    try:
+      popenCommunicate([self.openssl_binary, 'req', '-config',
+        self.openssl_configuration, '-nodes', '-new', '-keyout',
+        key, '-out', csr, '-days', '3650'],
+        common_name + '\n')
+      try:
+        popenCommunicate([self.openssl_binary, 'ca', '-batch', '-config',
+          self.openssl_configuration, '-out', certificate,
+          '-infiles', csr])
+      finally:
+        if os.path.exists(csr):
+          os.unlink(csr)
+    except:
+      try:
+        for f in file_list:
+          if os.path.exists(f):
+            os.unlink(f)
+      except:
+        # do not raise during cleanup
+        pass
+      raise
+    else:
+      return True
+
+  def checkRequestDir(self):
+    for request_file in os.listdir(self.request_dir):
+      parser = ConfigParser.RawConfigParser()
+      parser.readfp(open(os.path.join(self.request_dir, request_file), 'r'))
+      if self._checkCertificate(parser.get('certificate', 'name'),
+          parser.get('certificate', 'key_file'), parser.get('certificate',
+            'certificate_file')):
+        print 'Created certificate %r' % parser.get('certificate', 'name')
+
+def runCertificateAuthority(args):
+  ca_conf = args[0]
+  ca = CertificateAuthority(ca_conf['key'], ca_conf['certificate'],
+      ca_conf['openssl_binary'], ca_conf['openssl_configuration'],
+      ca_conf['request_dir'])
+  while True:
+    ca.checkAuthority()
+    ca.checkRequestDir()
+    time.sleep(60)

slapos/recipe/kvm/template/kvm_controller_run.in

@@ -11,7 +11,7 @@ so = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM)
 connected = False
 while not connected:
   try:
-    so.connect('%(qmp_socket)s')
+    so.connect('%(socket_path)s')
   except socket.error:
     time.sleep(1)
   else:

slapos/recipe/kvm/template/kvm_run.in

@@ -10,8 +10,8 @@ exec %(qemu_path)s \
   -smp %(smp_count)s \
   -m %(ram_size)s \
   -cdrom nbd:[%(nbd_ip)s]:%(nbd_port)s \
-  -drive file=%(image)s,if=virtio,boot=on \
-  -vnc [%(vnc_ip)s]:1,ipv6,tls,password \
+  -drive file=%(disk_path)s,if=virtio,boot=on \
+  -vnc %(vnc_ip)s:1,ipv4,password \
   -boot menu=on \
-  -qmp unix:%(qmp_socket)s,server \
-  -pidfile %(pid_file)s
+  -qmp unix:%(socket_path)s,server \
+  -pidfile %(pid_file_path)s

slapos/recipe/kvm/template/slapmonitor_run.in

 #!/bin/sh
 # BEWARE: This file is operated by slapgrid
 # BEWARE: It will be overwritten automatically
-exec %(python_path)s %(slapmonitor_path)s %(pid_file)s %(database_path)s
+exec %(python_path)s %(slapmonitor_path)s %(pid_file_path)s %(database_path)s

slapos/recipe/librecipe/execute.py

@@ -10,6 +10,23 @@ def execute(args):
   # Note: Candidate for slapos.lib.recipe
   os.execv(args[0], args + sys.argv[1:])
 
+def execute_wait(args):
+  """Execution but after all files in args[1] exists"""
+  exec_list = list(args[0])
+  file_list = list(args[1])
+  sleep = 60
+  while True:
+    ready = True
+    for f in file_list:
+      if not os.path.exists(f):
+        print 'File %r does not exists, sleeping for %s' % (f, sleep)
+        ready = False
+    if ready:
+      break
+    time.sleep(sleep)
+  os.execv(exec_list[0], exec_list + sys.argv[1:])
+
+
 child_pg = None
 
 

slapos/recipe/memcached/__init__.py

@@ -220,9 +220,10 @@ class Recipe(BaseSlapRecipe):
         self.substituteTemplate(template_filename,
           stunnel_conf))
     wrapper = zc.buildout.easy_install.scripts([('stunnel',
-      'slapos.recipe.librecipe.execute', 'execute')], self.ws, sys.executable,
-      self.wrapper_directory, arguments=[
-        self.options['stunnel_binary'].strip(), stunnel_conf_path]
+      'slapos.recipe.librecipe.execute', 'execute_wait')], self.ws,
+      sys.executable, self.wrapper_directory, arguments=[
+        [self.options['stunnel_binary'].strip(), stunnel_conf_path],
+        [ca_certificate, key]]
       )[0]
     self.path_list.append(wrapper)
     return stunnel_conf

slapos/recipe/mysql/__init__.py

@@ -224,9 +224,10 @@ class Recipe(BaseSlapRecipe):
         self.substituteTemplate(template_filename,
           stunnel_conf))
     wrapper = zc.buildout.easy_install.scripts([('stunnel',
-      'slapos.recipe.librecipe.execute', 'execute')], self.ws, sys.executable,
-      self.wrapper_directory, arguments=[
-        self.options['stunnel_binary'].strip(), stunnel_conf_path]
+      'slapos.recipe.librecipe.execute', 'execute_wait')], self.ws,
+      sys.executable, self.wrapper_directory, arguments=[
+        [self.options['stunnel_binary'].strip(), stunnel_conf_path],
+        [ca_certificate, key]]
       )[0]
     self.path_list.append(wrapper)
     return stunnel_conf

software/kumofs/software.cfg

 [buildout]
-
-extensions =
-  slapos.zcbworkarounds
-  slapos.rebootstrap
-
 find-links +=
     http://www.nexedi.org/static/packages/source/slapos.buildout/
 
@@ -29,18 +24,6 @@ allow-hosts =
   psutil.googlecode.com
   www.dabeaz.com
 
-# Use only quite well working sites.
-allow-hosts =
-  *.nexedi.org
-  *.python.org
-  *.sourceforge.net
-  dist.repoze.org
-  effbot.org
-  github.com
-  peak.telecommunity.com
-  psutil.googlecode.com
-  www.dabeaz.com
-
 versions = versions
 
 parts +=
@@ -56,11 +39,6 @@ parts +=
 # development / fast switching environment for whole software
 unzip = true
 
-[rebootstrap]
-# Default first version of rebootstrapped python
-version = 2
-section = python2.7
-
 [instance-recipe]
 egg = slapos.cookbook
 module = kumofs
@@ -85,13 +63,13 @@ output = ${buildout:directory}/template.cfg
 mode = 0644
 
 [versions]
-slapos.cookbook = 0.7
+slapos.cookbook = 0.13
 
 erp5.recipe.cmmiforcei686 = 0.1.1
 hexagonit.recipe.cmmi = 1.5.0
 hexagonit.recipe.download = 1.5.0
 
-# Required by slapos.cookbook==0.7
+# Required by slapos.cookbook==0.13
 slapos.core = 0.2
 collective.recipe.template = 1.8
 netaddr = 0.7.5

software/kvm/instance.cfg

+[buildout]
+parts =
+  kvminstance
+
+eggs-directory = ${buildout:eggs-directory}
+develop-eggs-directory = ${buildout:develop-eggs-directory} 
+
+[kvminstance]
+recipe = slapos.cookbook:kvm
+qemu_path = ${kvm:location}/bin/qemu-system-x86_64
+qemu_img_path = ${kvm:location}/bin/qemu-img
+#slapmonitor_path = ${buildout:bin-directory}/slapmonitor
+#slapreport_path = ${buildout:bin-directory}/slapreport
+websockify_path = ${noVNC:location}/utils/wsproxy.py
+noVNC_location = ${noVNC:location}
+openssl_binary = ${openssl:location}/bin/openssl
+rdiff_backup_binary = ${buildout:bin-directory}/rdiff-backup
+dcrond_binary = ${dcron:location}/sbin/crond
+
+smp_count = 1
+ram_size = 1024
+disk_size = 10

software/kvm/software.cfg

+[buildout]
+extends =
+  ../../stack/kvm.cfg
+
+[template]
+recipe = slapos.recipe.template
+url = ${:_profile_base_location_}/instance.cfg
+md5sum = d899f2111aab18ad25776f35ed49a91b
+output = ${buildout:directory}/template.cfg
+mode = 0644
+
+[kvmsource]
+command =
+  (${git:location}/bin/git clone --quiet http://git.erp5.org/repos/slapos.kvm.git ${:location} && cd ${:location} && git reset --hard 94ee45cc02e69798cac8209d2296fd1751125018) || (rm -fr ${:location} ; exit 1)
+update-command =
+
+[versions]
+Jinja2 = 2.5.5
+Werkzeug = 0.6.2
+hexagonit.recipe.cmmi = 1.5.0
+lxml = 2.3
+meld3 = 0.6.7
+plone.recipe.command = 1.1
+slapos.cookbook = 0.15
+slapos.recipe.template = 1.1
+z3c.recipe.scripts = 1.0.1
+
+# Required by:
+# slapos.core==0.9
+Flask = 0.7.2
+
+# Required by:
+# slapos.cookbook==0.15
+PyXML = 0.8.4
+
+# Required by:
+# slapos.recipe.template==1.1
+collective.recipe.template = 1.8
+
+# Required by:
+# hexagonit.recipe.cmmi==1.5.0
+hexagonit.recipe.download = 1.5.0
+
+# Required by:
+# slapos.cookbook==0.15
+netaddr = 0.7.5
+
+# Required by:
+# slapos.core==0.9
+netifaces = 0.5
+
+# Required by:
+# slapos.cookbook==0.15
+# slapos.core==0.9
+# zc.buildout==1.5.3-dev-SlapOS-005
+# zc.recipe.egg==1.3.2
+setuptools = 0.6c12dev-r88846
+
+# Required by:
+# slapos.cookbook==0.15
+slapos.core = 0.9
+
+# Required by:
+# slapos.core==0.9
+supervisor = 3.0a10
+
+# Required by:
+# slapos.cookbook==0.15
+xml-marshaller = 0.9.7
+
+# Required by:
+# slapos.cookbook==0.15
+zc.recipe.egg = 1.3.2
+
+# Required by:
+# slapos.core==0.9
+zope.interface = 3.6.4

software/mysql-5.1/software.cfg

 [buildout]
-
-extensions =
-  slapos.zcbworkarounds
-  slapos.rebootstrap
-
 find-links +=
     http://www.nexedi.org/static/packages/source/slapos.buildout/
 
@@ -47,11 +42,6 @@ parts +=
 # development / fast switching environment for whole software
 unzip = true
 
-[rebootstrap]
-# Default first version of rebootstrapped python
-version = 2
-section = python2.7
-
 [instance-recipe]
 egg = slapos.cookbook
 module = mysql
@@ -76,9 +66,9 @@ output = ${buildout:directory}/template.cfg
 mode = 0644
 
 [versions]
-slapos.cookbook = 0.9
+slapos.cookbook = 0.13
 
-# Required by slapos.cookbook==0.9
+# Required by slapos.cookbook==0.13
 slapos.core = 0.4
 collective.recipe.template = 1.8
 netaddr = 0.7.5

stack/kvm.cfg

+[buildout]
+extends =
+  shacache-client.cfg
+  ../component/python-2.7/buildout.cfg
+  ../component/lxml-python/buildout.cfg
+  ../component/git/buildout.cfg
+  ../component/zlib/buildout.cfg
+  ../component/readline/buildout.cfg
+  ../component/ncurses/buildout.cfg
+  ../component/libuuid/buildout.cfg
+  ../component/noVNC/buildout.cfg
+  ../component/openssl/buildout.cfg
+  ../component/rdiff-backup/buildout.cfg
+  ../component/dcron/buildout.cfg
+
+parts =
+  template
+  gnutls
+  kvm
+  eggs
+
+find-links +=
+  http://www.nexedi.org/static/packages/source/slapos.buildout/
+
+versions = versions
+
+[gpg-error]
+recipe = hexagonit.recipe.cmmi
+url = ftp://ftp.gnupg.org/gcrypt/libgpg-error/libgpg-error-1.10.tar.gz
+md5sum = 7c2710ef439f82ac429b88fec88e9a4c
+
+[gcrypt]
+recipe = hexagonit.recipe.cmmi
+url = ftp://ftp.gnupg.org/gcrypt/libgcrypt/libgcrypt-1.4.6.tar.gz
+md5sum = bfd45922eefb8a24d598af77366220d4
+configure-options =
+  --with-gpg-error-prefix=${gpg-error:location}
+environment =
+  CPPFLAGS=-I${gpg-error:location}/include
+  LDFLAGS=-Wl,-rpath -Wl,${gpg-error:location}/lib -Wl,${gpg-error:location}/lib/libgpg-error.so.0
+
+[gnutls]
+recipe = hexagonit.recipe.cmmi
+url = ftp://ftp.gnupg.org/gcrypt/gnutls/gnutls-2.8.6.tar.bz2
+md5sum = eb0a6d7d3cb9ac684d971c14f9f6d3ba
+configure-options =
+  --with-libgcrypt-prefix=${gcrypt:location}
+environment =
+  CPPFLAGS=-I${zlib:location}/include -I${readline:location}/include -I${ncurses:location}/include -I${ncurses:location}/include/ncursesw -I${gcrypt:location}/include -I${gpg-error:location}/include
+  LDFLAGS=-L${readline:location}/lib -L${ncurses:location}/lib -L${gcrypt:location}/lib -Wl,-rpath -Wl,${zlib:location}/lib -Wl,-rpath -Wl,${readline:location}/lib -Wl,-rpath -Wl,${ncurses:location}/lib -Wl,-rpath -Wl,${gcrypt:location}/lib -Wl,-rpath -Wl,${gpg-error:location}/lib -Wl,${gcrypt:location}/lib/libgcrypt.so.11
+  PKG_CONFIG=${zlib:location}/lib/pkgconfig
+
+[kvm]
+recipe = hexagonit.recipe.cmmi
+path = ${kvmsource:location}/
+configure-options =
+  --disable-sdl
+  --disable-xen
+  --enable-vnc-tls
+  --disable-vnc-sasl
+  --disable-curses
+  --disable-curl
+  --enable-kvm
+  --disable-docs
+  --enable-vnc-png
+  --disable-vnc-jpeg
+  --extra-cflags="-I${gnutls:location}/include -I${libuuid:location}/include -I${zlib:location}/include"
+  --extra-ldflags="-Wl,-rpath -Wl,${gnutls:location}/lib -L${libuuid:location}/lib -Wl,-rpath -Wl,${libuuid:location}/lib -L${zlib:location}/lib -Wl,-rpath -Wl,${zlib:location}/lib"
+  --disable-werror
+environment =
+  PKG_CONFIG_PATH=${gnutls:location}/lib/pkgconfig
+
+[kvmsource]
+recipe=plone.recipe.command
+location = ${buildout:parts-directory}/${:_buildout_section_name_}
+stop-on-error = true
+#tag = slapos-v0.1
+command =
+  (${git:location}/bin/git clone --quiet http://git.erp5.org/repos/slapos.kvm.git ${:location} ) || (rm -fr ${:location} ; exit 1)
+update-command =
+ cd ${:location} && ${git:location}/bin/git pull --quiet origin master
+
+[eggs]
+python = python2.7
+recipe = z3c.recipe.scripts
+eggs =
+  ${lxml-python:egg}
+  slapos.cookbook
+
+[versions]
+zc.buildout = 1.5.3-dev-SlapOS-005