Merge master in lamp-mohamadou branch

CHANGES.txt

 Changes
 =======
 
-0.48 (Unreleased)
+0.49 (Unreleased)
 -----------------
 
-  * No change yet.
+  * Slap Test Agent [Yingjie Xu]
+
+0.48 (2012-04-26)
+-----------------
+
+  * New utility recipe: slapos.recipe.generate_output_if_input_not_null.
+    [Cedric de Saint Martin]
+  * New promise recipe: slapos.recipe.url_available: check if url returns http
+    code 200. [Cedric de Saint Martin]
+  * Fix: slapos.recipe.request won't raise anymore if instance is not ready.
+    [Cedric de Saint Martin]
+  * Fix: slapos.recipe.request won't assume instance reference if not
+    specified. [Cedric de Saint Martin]
 
 0.47 (2012-04-19)
 -----------------

setup.py

@@ -2,7 +2,7 @@ from setuptools import setup, find_packages
 import glob
 import os
 
-version = '0.48-dev'
+version = '0.49-dev'
 name = 'slapos.cookbook'
 long_description = open("README.txt").read() + "\n" + \
     open("CHANGES.txt").read() + "\n"
@@ -48,6 +48,7 @@ setup(name=name,
           'certificate_authority = slapos.recipe.certificate_authority:Recipe',
           'certificate_authority.request = slapos.recipe.certificate_authority:Request',
           'check_port_listening = slapos.recipe.check_port_listening:Recipe',
+          'check_url_available = slapos.recipe.check_url_available:Recipe',
           'cron = slapos.recipe.dcron:Recipe',
           'cron.d = slapos.recipe.dcron:Part',
           'davstorage = slapos.recipe.davstorage:Recipe',
@@ -58,6 +59,7 @@ setup(name=name,
           'erp5scalabilitytestbed = slapos.recipe.erp5scalabilitytestbed:Recipe',
           'equeue = slapos.recipe.equeue:Recipe',
           'erp5testnode = slapos.recipe.erp5testnode:Recipe',
+          'generate_output_if_input_not_null = slapos.recipe.generate_output_if_input_not_null:Recipe',
           'generate.mac = slapos.recipe.generatemac:Recipe',
           'nbdserver = slapos.recipe.nbdserver:Recipe',
           'generic.onetimeupload = slapos.recipe.generic_onetimeupload:Recipe',

slapos/recipe/check_url_available/__init__.py

+##############################################################################
+#
+# Copyright (c) 2011 Vifib SARL and Contributors. All Rights Reserved.
+#
+# WARNING: This program as such is intended to be used by professional
+# programmers who take the whole responsibility of assessing all potential
+# consequences resulting from its eventual inadequacies and bugs
+# End users who are looking for a ready-to-use solution with commercial
+# guarantees and support are strongly adviced to contract a Free Software
+# Service Company
+#
+# This program is Free Software; you can redistribute it and/or
+# modify it under the terms of the GNU General Public License
+# as published by the Free Software Foundation; either version 3
+# of the License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA  02111-1307, USA.
+#
+##############################################################################
+from slapos.recipe.librecipe import GenericBaseRecipe
+import sys
+
+class Recipe(GenericBaseRecipe):
+  """
+  Create script that will check if "url" is available (e.g page answers 200 OK).
+  """
+
+  def install(self):
+    config = {
+      'url': self.options['url'],
+      'shell_path': self.options['dash_path'],
+    }
+
+    # XXX-Cedric in this script, curl won't check certificate
+    promise = self.createExecutable(
+      self.options['path'],
+      self.substituteTemplate(self.getTemplateFilename('check_url.in'), config)
+    )
+
+    return [promise]

slapos/recipe/check_url_available/template/check_url.in

+#!%(shell_path)s
+# BEWARE: This file is operated by slapgrid
+# BEWARE: It will be overwritten automatically
+
+URL="%(url)s"
+CODE=$(curl -k -sL $URL -w %%{http_code} -o /dev/null)
+
+if [ $CODE -eq 000 ]; then
+  echo "$URL is not available (server not reachable)" >&2
+  exit 1
+fi
+
+if ! [ $CODE -eq 200 ]; then
+  echo "$URL is not available (returned $CODE)" >&2
+  exit 2
+fi

slapos/recipe/generate_output_if_input_not_null.py

+##############################################################################
+#
+# Copyright (c) 2010 Vifib SARL and Contributors. All Rights Reserved.
+#
+# WARNING: This program as such is intended to be used by professional
+# programmers who take the whole responsibility of assessing all potential
+# consequences resulting from its eventual inadequacies and bugs
+# End users who are looking for a ready-to-use solution with commercial
+# guarantees and support are strongly adviced to contract a Free Software
+# Service Company
+#
+# This program is Free Software; you can redistribute it and/or
+# modify it under the terms of the GNU General Public License
+# as published by the Free Software Foundation; either version 3
+# of the License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA  02111-1307, USA.
+#
+##############################################################################
+from slapos.recipe.librecipe import GenericBaseRecipe
+
+class Recipe(GenericBaseRecipe):
+  """Generate an output from one or several input and a template.
+  
+  Take "input-list" buildout parameter as input.
+  Each input of the list is separated by \n
+  Each input contains :
+    1/ The parameter to use (like mybuildoutpart:myparameter)
+    2/ The name of the input to use as key.
+
+  If all parameters in input are found, create an "output" parameter from a
+  "template" parameter. The "template" parameter is just a string containing
+  python parameters (like %(mykey)s).
+
+  Will produce nothing if one element of "input_list" doesn't exist.
+  Will raise if any input reference non-existent buildout part.
+
+  Example : 
+  [get-output]
+  recipe = slapos.cookbook:generate_output_if_input_not_null
+  input-list =
+    firstkey mybuildoutpart:myparameter
+    otherkey myotherbuildoutpart:myotherparameter
+  template = I want to get %(key)s and %(otherkey)s
+
+  This example will produce an "output" parameter if myparameter and
+  myotherparameter are defined.
+  """
+  def __init__(self, buildout, name, options):
+    # Get all inputs
+    input_dict = {}
+    for line in options['input-list'].strip().split('\n'):
+      key, buildout_parameter = line.split(' ')
+      buildout_part, parameter_name = buildout_parameter.split(':')
+      parameter_value = buildout[buildout_part].get(parameter_name)
+      # If any parameter is not defined, don't do anything
+      if not parameter_value:
+        return
+      input_dict[key] = parameter_value
+    # Generate output
+    options['output'] = options['template'] % input_dict
+
+  def install(self):
+    return []

slapos/recipe/request.py

@@ -73,7 +73,7 @@ class Recipe(object):
             options['config-%s' % config_parameter]
 
     self.instance = self.request(options['software-url'], software_type,
-      options.get('name', name), partition_parameter_kw=partition_parameter_kw,
+      options['name'], partition_parameter_kw=partition_parameter_kw,
       filter_kw=filter_kw, shared=self.isSlave)
 
     self.failed = None
@@ -96,7 +96,7 @@ class Recipe(object):
       # XXX-Cedric : currently raise an error. So swallow it...
       except AttributeError:
         status = "unknown"
-      raise KeyError("Connection parameter %s not found. "
+      self.logger.error("Connection parameter %s not found. "
           "Status of requested instance is : %s." % (self.failed, status))
     return []
 

software/kvm/instance-kvm.cfg

@@ -9,6 +9,7 @@ parts =
   certificate-authority
   kvm-promise
   novnc-promise
+  frontend-promise
   publish-kvm-connection-information
 
 eggs-directory = ${buildout:eggs-directory}
@@ -59,6 +60,7 @@ path = $${basedirectory:promises}/vnc_promise
 hostname = $${kvm-instance:vnc-ip}
 port = $${kvm-instance:vnc-port}
 
+
 [novnc-instance]
 recipe = slapos.cookbook:novnc
 path = $${ca-novnc:executable}
@@ -104,23 +106,20 @@ path = $${basedirectory:promises}/novnc_promise
 hostname = $${novnc-instance:ip}
 port = $${novnc-instance:port}
 
+
 [kvm-monitor]
 recipe = slapos.cookbook:generic.slapmonitor
 db-path = $${rootdirectory:srv}/slapmonitor_database
 
-[request-common]
+
+[request-slave-frontend]
 recipe = slapos.cookbook:request
 software-url = $${slap-connection:software-release-url}
-sla = computer_guid
-sla-computer_guid = $${slap-connection:computer-id}
 server-url = $${slap-connection:server-url}
 key-file = $${slap-connection:key-file}
 cert-file = $${slap-connection:cert-file}
 computer-id = $${slap-connection:computer-id}
 partition-id = $${slap-connection:partition-id}
-
-[request-slave-frontend]
-<=request-common
 name = SlaveFrontend
 software-type = frontend
 slave = true
@@ -129,8 +128,27 @@ config-host = $${novnc-instance:ip}
 config-port = $${novnc-instance:port}
 return = url resource port domainname
 
+# Will generate, if existing, URL to reach KVM using frontend
+[get-slave-connection-url]
+recipe = slapos.cookbook:generate_output_if_input_not_null
+input-list =
+  frontend-url request-slave-frontend:connection-url
+  frontend-port request-slave-frontend:connection-port
+  frontend-resource request-slave-frontend:connection-resource
+  frontend-domainname request-slave-frontend:connection-domainname
+template = %(frontend-url)s/vnc_auto.html?host=%(frontend-domainname)s&port=%(frontend-port)s&encrypt=1&path=%(frontend-resource)s
+# This is default output, if slave is not ready yet
+output = Not ready yet. Please use backend URL if possible or wait a few minutes.
+
+[frontend-promise]
+recipe = slapos.cookbook:check_url_available
+path = $${basedirectory:promises}/frontend_promise
+url = $${get-slave-connection-url:output}
+dash_path = ${dash:location}/bin/dash
+
+
 [publish-kvm-connection-information]
 recipe = slapos.cookbook:publish
 backend_url = https://[$${novnc-instance:ip}]:$${novnc-instance:port}/vnc_auto.html?host=[$${novnc-instance:ip}]&port=$${novnc-instance:port}&encrypt=1
-url = $${request-slave-frontend:connection-url}/vnc_auto.html?host=$${request-slave-frontend:connection-domainname}&port=$${request-slave-frontend:connection-port}&encrypt=1&path=$${request-slave-frontend:connection-resource}
+url = $${get-slave-connection-url:output}
 password = $${kvm-instance:passwd}

software/kvm/software.cfg

@@ -132,7 +132,7 @@ command =
 [template-kvm]
 recipe = slapos.recipe.template
 url = ${:_profile_base_location_}/instance-kvm.cfg
-md5sum = b6572c018e44d4676e76805116bcade0
+md5sum = 8d67a6cabe4fbce2bd44aa006a0d0cf8
 output = ${buildout:directory}/template-kvm.cfg
 mode = 0644
 
@@ -199,53 +199,53 @@ signature-certificate-list =
 [versions]
 Jinja2 = 2.6
 Werkzeug = 0.8.3
-apache-libcloud = 0.8.0
+apache-libcloud = 0.9.1
 async = 0.6.1
 buildout-versions = 1.7
 gitdb = 0.5.4
 hexagonit.recipe.cmmi = 1.5.0
-lxml = 2.3.3
+lxml = 2.3.4
 meld3 = 0.6.8
 plone.recipe.command = 1.1
 pycrypto = 2.5
-slapos.cookbook = 0.41
+slapos.cookbook = 0.48
 slapos.recipe.build = 0.7
-slapos.recipe.template = 2.2
-slapos.toolbox = 0.18
+slapos.recipe.template = 2.3
+slapos.toolbox = 0.24
 smmap = 0.8.2
 z3c.recipe.scripts = 1.0.1
 
 # Required by:
-# slapos.core==0.23
-# slapos.toolbox==0.18
+# slapos.core==0.24
+# slapos.toolbox==0.24
 Flask = 0.8
 
 # Required by:
-# slapos.toolbox==0.18
+# slapos.toolbox==0.24
 GitPython = 0.3.2.RC1
 
 # Required by:
-# slapos.cookbook==0.41
+# slapos.cookbook==0.48
 PyXML = 0.8.4
 
 # Required by:
-# slapos.toolbox==0.18
+# slapos.toolbox==0.24
 atomize = 0.1.1
 
 # Required by:
-# slapos.toolbox==0.18
+# slapos.toolbox==0.24
 feedparser = 5.1.1
 
 # Required by:
-# slapos.cookbook==0.41
+# slapos.cookbook==0.48
 inotifyx = 0.2.0
 
 # Required by:
-# slapos.cookbook==0.41
+# slapos.cookbook==0.48
 netaddr = 0.7.6
 
 # Required by:
-# slapos.core==0.23
+# slapos.core==0.24
 netifaces = 0.8
 
 # Required by:
@@ -253,37 +253,37 @@ netifaces = 0.8
 numpy = 1.6.1
 
 # Required by:
-# slapos.toolbox==0.18
+# slapos.toolbox==0.24
 paramiko = 1.7.7.1
 
 # Required by:
-# slapos.toolbox==0.18
+# slapos.toolbox==0.24
 psutil = 0.4.1
 
 # Required by:
-# slapos.cookbook==0.41
-# slapos.core==0.23
-# slapos.toolbox==0.18
+# slapos.cookbook==0.48
+# slapos.core==0.24
+# slapos.toolbox==0.24
 setuptools = 0.6c12dev-r88846
 
 # Required by:
-# slapos.cookbook==0.41
-# slapos.toolbox==0.18
-slapos.core = 0.23
+# slapos.cookbook==0.48
+# slapos.toolbox==0.24
+slapos.core = 0.24
 
 # Required by:
-# slapos.core==0.23
+# slapos.core==0.24
 supervisor = 3.0a12
 
 # Required by:
-# slapos.cookbook==0.41
-# slapos.toolbox==0.18
+# slapos.cookbook==0.48
+# slapos.toolbox==0.24
 xml-marshaller = 0.9.7
 
 # Required by:
-# slapos.cookbook==0.41
+# slapos.cookbook==0.48
 zc.recipe.egg = 1.3.2
 
 # Required by:
-# slapos.core==0.23
+# slapos.core==0.24
 zope.interface = 3.8.0

stack/slapos.cfg

@@ -6,9 +6,10 @@
 extensions +=
   buildout-versions
 
-# Use shacache
+# Use shacache and lxml
 extends =
   shacache-client.cfg
+  ../component/lxml-python/buildout.cfg
   
 # Separate from site eggs
 allowed-eggs-from-site-packages =
@@ -40,8 +41,17 @@ allow-hosts +=
 # development / fast switching environment for whole software
 unzip = true
 
+parts += slapos-cookbook
+
 versions = versions
 
+# Install slapos.cookbook containing all officials recipes
+[slapos-cookbook]
+recipe = zc.recipe.egg
+eggs =
+  ${lxml-python:egg}
+  slapos.cookbook
+
 [versions]
 # Use patched hexagonit.recipe.download from
 # https://github.com/SlapOS/hexagonit.recipe.download