Update release candidate

component/file/buildout.cfg

@@ -6,6 +6,13 @@ parts = file
 extends =
   ../zlib/buildout.cfg
 
+[file-msooxml]
+recipe = slapos.recipe.build:download
+url = ${:_profile_base_location_}/msooxml
+md5sum = c889ad135cbfb343db36b729a3897432
+location = ${buildout:parts-directory}/${:_buildout_section_name_}
+filename = msooxml
+
 [file]
 recipe = slapos.recipe.cmmi
 url = http://ftp.icm.edu.pl/packages/file/file-5.32.tar.gz
@@ -15,3 +22,7 @@ configure-options =
 environment =
   CPPFLAGS=-I${zlib:location}/include
   LDFLAGS=-L${zlib:location}/lib -Wl,-rpath=${zlib:location}/lib
+pre-configure =
+# patch for fix msooxml files detect correctly
+  test -f ./magic/Magdir/msooxml
+  cp ${file-msooxml:location}/msooxml ./magic/Magdir/

component/file/msooxml

+#------------------------------------------------------------------------------
+# $File: msooxml,v 1.5 2014/08/05 07:38:45 christos Exp $
+# msooxml:  file(1) magic for Microsoft Office XML
+# From: Ralf Brown <ralf.brown@gmail.com>
+
+# .docx, .pptx, and .xlsx are XML plus other files inside a ZIP
+#   archive.  The first member file is normally "[Content_Types].xml".
+#   but some libreoffice generated files put this later. Perhaps skip
+#   the "[Content_Types].xml" test?
+# Since MSOOXML doesn't have anything like the uncompressed "mimetype"
+#   file of ePub or OpenDocument, we'll have to scan for a filename
+#   which can distinguish between the three types
+
+# start by checking for ZIP local file header signature
+0               string          PK\003\004
+!:strength +10
+# make sure the first file is correct
+>0x1E           regex           \\[Content_Types\\]\\.xml|_rels/\\.rels
+# skip to the second local file header
+# since some documents include a 520-byte extra field following the file
+# header, we need to scan for the next header
+>>(18.l+49)     search/2000     PK\003\004
+# now skip to the *third* local file header; again, we need to scan due to a
+# 520-byte extra field following the file header
+>>>&26          search/1000     PK\003\004
+# and check the subdirectory name to determine which type of OOXML
+# file we have.  Correct the mimetype with the registered ones:
+# http://technet.microsoft.com/en-us/library/cc179224.aspx
+>>>>&26         string          word/           Microsoft Word 2007+
+!:mime application/vnd.openxmlformats-officedocument.wordprocessingml.document
+>>>>&26         string          ppt/            Microsoft PowerPoint 2007+
+!:mime application/vnd.openxmlformats-officedocument.presentationml.presentation
+>>>>&26         string          xl/             Microsoft Excel 2007+
+!:mime application/vnd.openxmlformats-officedocument.spreadsheetml.sheet
+
+>>1104          search/300      PK\003\004
+# and check the subdirectory name to determine which type of OOXML
+# file we have.  Correct the mimetype with the registered ones:
+# http://technet.microsoft.com/en-us/library/cc179224.aspx
+>>>&26          string          word/           Microsoft Word 2007+
+!:mime application/vnd.openxmlformats-officedocument.wordprocessingml.document
+>>>&26          string          ppt/            Microsoft PowerPoint 2007+
+!:mime application/vnd.openxmlformats-officedocument.presentationml.presentation
+>>>&26          string          xl/             Microsoft Excel 2007+
+!:mime application/vnd.openxmlformats-officedocument.spreadsheetml.sheet

component/gdal/buildout.cfg

@@ -32,7 +32,7 @@ configure-options =
   --with-png=${libpng:location}
   --with-static-proj4=${proj4:location}
   --with-sqlite3=${sqlite3:location}
-  --with-wepb=${webp:location}
+  --with-webp=${webp:location}
   --with-xml2=${libxml2:location}/bin/xml2-config
 environment =
   PATH=${xz-utils:location}/bin:%(PATH)s

component/icu/buildout.cfg

 [buildout]
-
+extends =
+  ../gcc/buildout.cfg
 parts = icu4c
 
 [icu4c]
+# need for couchdb
 recipe = slapos.recipe.cmmi
 location = ${buildout:parts-directory}/${:_buildout_section_name_}
 url = http://download.icu-project.org/files/icu4c/58.2/icu4c-58_2-src.tgz
@@ -13,6 +15,14 @@ configure-options =
   --disable-static
   --enable-rpath
 
+[icu4c-slaposgcc]
+# need for onlyoffice-core
+<= icu4c
+environment =
+  PATH=${gcc:location}/bin:%(PATH)s
+  LD_LIBRARY_PATH=${gcc:location}/lib:${gcc:location}/lib64
+  LDFLAGS=-Wl,-rpath=${gcc:location}/lib -Wl,-rpath=${gcc:location}/lib64
+
 [icu]
 <= icu4c
 

component/java/buildout.cfg

@@ -36,11 +36,20 @@ slapos_promisee =
   directory:plugin
   file:lib/rt.jar
   file:bin/java
-# http://java.com/en/download/manual_java7.jsp
-x86 = http://javadl.sun.com/webapps/download/AutoDL?BundleId=97358 22d970566c418499d331a2099d77c548
-x86-64 = http://javadl.sun.com/webapps/download/AutoDL?BundleId=97360 f4f7f7335eaf2e7b5ff455abece9d5ed
+# https://www.java.com/en/download/manual.jsp
+# Update 161
+x86 = http://javadl.oracle.com/webapps/download/AutoDL?BundleId=230530_2f38c3b165be4555a1fa6e98c45e0808 32db95dd417fd7949922206b2a61aa19
+x86-64 = http://javadl.oracle.com/webapps/download/AutoDL?BundleId=230532_2f38c3b165be4555a1fa6e98c45e0808 4385bc121b085862be623f4a31e7e0b4
 script =
   if not self.options.get('url'): self.options['url'], self.options['md5sum'] = self.options[guessPlatform()].split(' ')
   extract_dir = self.extract(self.download(self.options['url'], self.options.get('md5sum')))
   workdir = guessworkdir(extract_dir)
   self.copyTree(workdir, "%(location)s")
+
+[java-re-8-output]
+# Shared binary location to ease migration
+recipe = plone.recipe.command
+stop-on-error = true
+update-command = ${:command}
+command = ${coreutils-output:test} -x ${:keytool}
+keytool = ${java-re-8:location}/bin/keytool

component/logrotate/buildout.cfg

@@ -11,7 +11,7 @@ url = https://github.com/logrotate/logrotate/releases/download/3.11.0/logrotate-
 md5sum = 3a9280e4caeb837427a2d54518fbcdac
 # BBB this is only for backward-compatibility.
 post-install =
-  ln -s . ${buildout:parts-directory}/${:_buildout_section_name_}/usr
+  ln -sf . ${buildout:parts-directory}/${:_buildout_section_name_}/usr
 environment =
   PATH=${xz-utils:location}/bin:%(PATH)s
   CPPFLAGS=-I${popt:location}/include

component/mariadb/buildout.cfg

@@ -16,6 +16,7 @@ extends =
   ../readline/buildout.cfg
   ../xz-utils/buildout.cfg
   ../zlib/buildout.cfg
+  ../unixodbc/buildout.cfg
 # The following lines are only for mariarocks.cfg
 # to be extended last without touching 'parts'.
   ../gcc/buildout.cfg
@@ -62,16 +63,18 @@ configure-options =
   -DCMAKE_C_FLAGS="${:CMAKE_CFLAGS}"
   -DCMAKE_CXX_FLAGS="${:CMAKE_CFLAGS}"
   -DCMAKE_INSTALL_RPATH=${:CMAKE_LIBRARY_PATH}
-CMAKE_CFLAGS = -I${bzip2:location}/include -I${jemalloc:location}/include -I${libaio:location}/include -I${libxml2:location}/include -I${ncurses:location}/include -I${openssl:location}/include -I${readline5:location}/include -I${xz-utils:location}/include -I${zlib:location}/include ${:extra_cflags}
-CMAKE_LIBRARY_PATH = ${bzip2:location}/lib:${jemalloc:location}/lib:${libaio:location}/lib:${libxml2:location}/lib:${ncurses:location}/lib:${openssl:location}/lib:${readline5:location}/lib:${xz-utils:location}/lib:${zlib:location}/lib${:extra_library_path}
+  -DCMAKE_INCLUDE_PATH=${unixodbc:location}/include
+  -DCMAKE_LIBRARY_PATH=${unixodbc:location}/lib
+CMAKE_CFLAGS = -I${bzip2:location}/include -I${jemalloc:location}/include -I${libaio:location}/include -I${libxml2:location}/include -I${ncurses:location}/include -I${openssl:location}/include -I${readline5:location}/include -I${xz-utils:location}/include -I${zlib:location}/include -I${unixodbc:location}/include ${:extra_cflags}
+CMAKE_LIBRARY_PATH = ${bzip2:location}/lib:${jemalloc:location}/lib:${libaio:location}/lib:${libxml2:location}/lib:${ncurses:location}/lib:${openssl:location}/lib:${readline5:location}/lib:${xz-utils:location}/lib:${zlib:location}/lib:${unixodbc:location}/lib:${:extra_library_path}
 extra_cflags =
 extra_include_path =
 extra_library_path =
 environment =
   CMAKE_PROGRAM_PATH=${cmake:location}/bin
-  CMAKE_INCLUDE_PATH=${bzip2:location}/include:${libaio:location}/include:${libaio:location}/include:${libxml2:location}/include:${ncurses:location}/include:${openssl:location}/include:${readline5:location}/include:${xz-utils:location}/include:${zlib:location}/include${:extra_include_path}
+  CMAKE_INCLUDE_PATH=${bzip2:location}/include:${libaio:location}/include:${libaio:location}/include:${libxml2:location}/include:${ncurses:location}/include:${openssl:location}/include:${readline5:location}/include:${xz-utils:location}/include:${zlib:location}/include:${unixodbc:location}/include:${:extra_include_path}
   CMAKE_LIBRARY_PATH=${:CMAKE_LIBRARY_PATH}
-  LDFLAGS=-L${bzip2:location}/lib -L${jemalloc:location}/lib -L${libaio:location}/lib -L${xz-utils:location}/lib -L${zlib:location}/lib
+  LDFLAGS=-L${bzip2:location}/lib -L${jemalloc:location}/lib -L${libaio:location}/lib -L${xz-utils:location}/lib -L${zlib:location}/lib -L${unixodbc:location}/lib
   PATH=${patch:location}/bin:%(PATH)s
 post-install =
   mkdir -p ${:location}/include/wsrep &&

component/mariadb/mariarocks.cfg

 # Do not extend any file that touch buildout:parts.
 
 [mariadb]
-version = 10.2.11
-md5sum = 954088299fe5f11b4fda3b540558adbd
+version = 10.2.13
+md5sum = 20c61bd4059ba287e54cfb2862bae81d
 stable-patches =
 configure-options +=
-# force build of TokuDB due to a regression in 10.2.11
-  -DTOKUDB_OK=1
   -DCMAKE_C_COMPILER=${gcc:location}/bin/gcc
   -DCMAKE_CXX_COMPILER=${gcc:location}/bin/g++
 extra_cflags = -I${zstd:location}/include

component/nodejs/buildout.cfg

@@ -5,6 +5,7 @@ extends =
   ../pkgconfig/buildout.cfg
   ../openssl/buildout.cfg
   ../zlib/buildout.cfg
+  ../python-2.7/buildout.cfg
 
 parts =
   nodejs
@@ -12,6 +13,24 @@ parts =
 [nodejs]
 <= nodejs-0.12
 
+[nodejs-8.6.0]
+# Server-side Javascript.
+recipe = slapos.recipe.cmmi
+version = v8.6.0
+url = https://nodejs.org/dist/${:version}/node-${:version}.tar.gz
+md5sum = 0c95e08220667d8a18b97ecec8218ac6
+configure-options =
+  --shared-openssl
+  --shared-openssl-includes=${openssl:location}/include
+  --shared-openssl-libpath=${openssl:location}/lib
+environment =
+  HOME=${buildout:parts-directory}/${:_buildout_section_name_}
+  PATH=${pkgconfig:location}/bin:${python2.7:location}/bin/:%(PATH)s
+  PKG_CONFIG_PATH=${openssl:location}/lib/pkgconfig/
+  CPPFLAGS=-I${zlib:location}/include
+  LDFLAGS=-Wl,-rpath=${openssl:location}/lib -L${zlib:location}/lib -Wl,-rpath=${zlib:location}/lib
+  LD_LIBRARY_PATH=${openssl:location}/lib
+
 [nodejs-5]
 # Server-side Javascript.
 recipe = slapos.recipe.cmmi

component/onlyoffice-core/buildout.cfg

+[buildout]
+extends =
+  ../gcc/buildout.cfg
+  ../libxml2/buildout.cfg
+  ../zlib/buildout.cfg
+  ../icu/buildout.cfg
+# for qmake
+  ../qt/buildout.cfg
+
+parts +=
+  onlyoffice-core
+
+[onlyoffice-core]
+recipe = slapos.recipe.cmmi
+location = ${buildout:parts-directory}/${:_buildout_section_name_}
+# This url contains the hash provided by the DocumentServer core submodule hash.
+# https://github.com/ONLYOFFICE/DocumentServer/
+url = https://lab.nexedi.com/bk/onlyoffice_core/repository/archive.tar.bz2?ref=b051e75b179b3599c09937668fbbd2d7e2c50683
+md5sum = b2713373d687dd1c7121c286fa156626
+configure-command = true
+make-targets = lib bin
+environment =
+  PATH=${gcc:location}/bin:${qt5-qmake:location}/bin:%(PATH)s
+  CXXFLAGS=-I${libxml2:location}/include -I${zlib:location}/include -I${icu4c-slaposgcc:location}/include -I${boost-lib:location}/include -Wno-comment -Wno-deprecated-declarations -Wno-endif-labels -Wno-parentheses -Wno-reorder -Wno-sign-compare -Wno-switch -Wno-unknown-pragmas -Wno-unused
+  LDFLAGS=-L${gcc:location}/lib -Wl,-rpath=${gcc:location}/lib -L${gcc:location}/lib64 -Wl,-rpath=${gcc:location}/lib64 -L${libxml2:location}/lib -Wl,-rpath=${libxml2:location}/lib -L${zlib:location}/lib -Wl,-rpath=${zlib:location}/lib -L${icu4c-slaposgcc:location}/lib -Wl,-rpath=${icu4c-slaposgcc:location}/lib -L${boost-lib:location}/lib -Wl,-rpath=${boost-lib:location}/lib -Wl,-rpath=${:location}/lib
+post-install =
+  set -e -x
+  mkdir -p ${:location}/bin ${:location}/lib
+  mv -t ${:location}/lib build/lib/*/*.so
+  mv -t ${:location}/bin build/bin/*/*
+# the binary linux_64 in build/bin/AllFontsGen is renamed AllFontsGen here.
+#  mv build/bin/AllFontsGen/* ${:location}/bin/AllFontsGen

component/onlyoffice-x2t/buildout.cfg

-[buildout]
-extends =
-  ../dash/buildout.cfg
-  ../curl/buildout.cfg
-
-parts +=
-  onlyoffice-x2t
-
-[onlyoffice-x2t]
-recipe = slapos.recipe.build
-
-url = https://lab.nexedi.com/tc/bin/raw/fc3af671d3b19e9d25b40326373222b601f23edc/onlyoffice-x2t-part.tar.gz
-md5sum = 3e08a8b1345c301078cdce3a7f7360b2
-
-# script to install.
-script =
-  location = %(location)r
-  self.failIfPathExists(location)
-  import sys
-  extract_dir = self.extract(self.download(self.options['url'], self.options.get('md5sum')))
-  shutil.move(extract_dir, location)
-  wrapper_location = os.path.join("%(location)s", "x2t")
-  wrapper = open(wrapper_location, 'w')
-  wrapper.write('''#!${dash:location}/bin/dash
-  export LD_LIBRARY_PATH=%(location)s/lib:${curl:location}/lib
-  exec %(location)s/bin/x2t "$@"''')
-  wrapper.close()
-  os.chmod(wrapper_location, 0755)
-  os.chmod(location, 0750)

component/qemu-kvm/buildout.cfg

@@ -57,22 +57,27 @@ location = ${buildout:parts-directory}/${:_buildout_section_name_}
 <= debian-netinst-base
 arch = amd64
 
+[debian-amd64-wheezy-netinst.iso]
+<= debian-amd64-netinst-base
+version = 7.11.0
+md5sum = 096c1c18b44c269808bd815d58c53c8f
+
 [debian-amd64-jessie-netinst.iso]
-# Download the installer of Debian 8 (Jessie)
 <= debian-amd64-netinst-base
 release = archive
 version = 8.10.0
 md5sum = 19dcfc381bd3e609c6056216d203f5bc
 
 [debian-amd64-netinst.iso]
+# Download the installer of Debian 9 (Stretch)
 <= debian-amd64-netinst-base
 release = release/current
-version = 9.3.0
-md5sum = db8ab7871bc2b7d456c4746e706fb5d3
+version = 9.4.0
+md5sum = 73bd8aaaeb843745ec939f6ae3906e48
 
 [debian-amd64-testing-netinst.iso]
-# Download the installer of Debian Stretch
+# Download the installer of Debian Buster
 <= debian-amd64-netinst-base
-release = stretch_di_alpha7
-version = stretch-DI-alpha7
-md5sum = 3fe53635b904553b26588491e1473e99
+release = buster_di_alpha2
+version = buster-DI-alpha2
+md5sum = fbdc192f8857e2bd884e41481ed0fc09

component/qt/buildout.cfg

 [buildout]
+extends =
+  ../xorg/buildout.cfg
+  ../gcc/buildout.cfg
 
 parts =
-  qt
-  
-[qt]
-recipe = slapos.recipe.build
-slapos_promisee =
-  file:plop
-
-# Online installer
-x86 = http://get.qt.nokia.com/qtsdk/Qt_SDK_Lin32_online_v1_1_3_en.run eae2e2a1396fec1369b66c71d7df6eab
-x86-64 =  http://get.qt.nokia.com/qtsdk/Qt_SDK_Lin64_online_v1_1_3_en.run a4d929bc4d6511290c07c3745477b77b
-# Offline installer
-#x86 = http://get.qt.nokia.com/qtsdk/Qt_SDK_Lin32_offline_v1_1_3_en.run 106fdae4ec8947c491ab0a827a02da12
-#x86-64 = http://get.qt.nokia.com/qtsdk/Qt_SDK_Lin64_offline_v1_1_3_en.run 8c280beb11ee763840464572ed80e8b8
-
-# Needs many dependencies.
-
-script =
-  if not self.options.get('url'): self.options['url'], self.options['md5sum'] = self.options[guessPlatform()].split(' ')
-  download_file = self.download(self.options['url'], self.options.get('md5sum'))
-  extract_dir = tempfile.mkdtemp(self.name)
-  os.chdir(extract_dir)
-  (download_dir, filename) = os.path.split(download_file)
-  auto_extract_bin = os.path.join(extract_dir, filename)
-  shutil.move(download_file, auto_extract_bin)
-  os.chmod(auto_extract_bin, 0755)
-  subprocess.call([auto_extract_bin])
-  self.cleanup_list.append(extract_dir)
-  workdir = guessworkdir(extract_dir)
-  import pdb; pdb.set_trace()
-  self.copyTree(os.path.join(workdir, "jre1.6.0_27"), "%(location)s")
-
-
-
-#ldd qt.run
-#	linux-gate.so.1 =>  (0xb7827000)
-#	libutil.so.1 => /lib/i686/cmov/libutil.so.1 (0xb781c000)
-#	libgobject-2.0.so.0 => not found
-#	libSM.so.6 => not found
-#	libICE.so.6 => not found
-#	libXrender.so.1 => not found
-#	libfontconfig.so.1 => not found
-#	libfreetype.so.6 => /usr/lib/libfreetype.so.6 (0xb77a4000)
-#	libz.so.1 => /usr/lib/libz.so.1 (0xb778f000)
-#	libXext.so.6 => /usr/lib/libXext.so.6 (0xb7780000)
-#	libX11.so.6 => /usr/lib/libX11.so.6 (0xb7663000)
-#	libdl.so.2 => /lib/i686/cmov/libdl.so.2 (0xb765f000)
-#	libgthread-2.0.so.0 => not found
-#	librt.so.1 => /lib/i686/cmov/librt.so.1 (0xb7655000)
-#	libglib-2.0.so.0 => not found
-#	libpthread.so.0 => /lib/i686/cmov/libpthread.so.0 (0xb763c000)
-#	libstdc++.so.6 => /usr/lib/libstdc++.so.6 (0xb754d000)
-#	libm.so.6 => /lib/i686/cmov/libm.so.6 (0xb7527000)
-#	libgcc_s.so.1 => /lib/libgcc_s.so.1 (0xb7508000)
-#	libc.so.6 => /lib/i686/cmov/libc.so.6 (0xb73c2000)
-#	libxcb.so.1 => /usr/lib/libxcb.so.1 (0xb73a9000)
-#	/lib/ld-linux.so.2 (0xb7828000)
-#	libXau.so.6 => /usr/lib/libXau.so.6 (0xb73a6000)
-#	libXdmcp.so.6 => /usr/lib/libXdmcp.so.6 (0xb73a1000)
+  qt4-qmake
+
+[qt5-qmake]
+# XXX work on all systems needs check
+recipe = slapos.recipe.cmmi
+location = ${buildout:parts-directory}/${:_buildout_section_name_}
+url = http://download.qt.io/official_releases/qt/5.6/5.6.2/submodules/qtbase-opensource-src-5.6.2.tar.gz
+md5sum = 7aa5841b50c411e23e31e8a6cc1c6981
+configure-command = ./configure
+configure-options =
+  --prefix=${:location}
+  -v
+  -no-separate-debug-info
+  -release
+  -confirm-license
+  -opensource
+  -no-opengl
+  -nomake examples
+environment =
+  PATH=${gcc:location}/bin:%(PATH)s
+  CPPFLAGS=-I${libX11:location}/include -I${xproto:location}/include -I${libXext:location}/include
+  LDFLAGS=-L${gcc:location}/lib -Wl,-rpath=${gcc:location}/lib -L${gcc:location}/lib64 -Wl,-rpath=${gcc:location}/lib64 -L${libX11:location}/lib -Wl,-rpath=${libX11:location}/lib -L${xproto:location}/lib -Wl,-rpath=${xproto:location}/lib -L${libXext:location}/lib -Wl,-rpath=${libXext:location}/lib
+make-binary = true
+post-install =
+  mkdir -p ${:location}/bin
+  mv -t ${:location}/bin bin/qmake
+  mv -t ${:location} mkspecs
+
+[qt5.6-qmake]
+<= qt5-qmake
+[qt5.6.2-qmake]
+<= qt5.6-qmake
+
+
+[qt4-qmake]
+# building [qmake] will download the full qt source anyway ~200MB
+# qmake binary can be reached directly from ${qt:location}/bin/qmake if [qt] is fully built
+recipe = slapos.recipe.cmmi
+location = ${buildout:parts-directory}/${:_buildout_section_name_}
+url = http://download.qt.io/official_releases/qt/4.8/4.8.7/qt-everywhere-opensource-src-4.8.7.tar.gz
+md5sum = d990ee66bf7ab0c785589776f35ba6ad
+# see https://github.com/NixOS/nixpkgs/blob/3e387c3e005c87566b5403d24c86f71f4945a79b/pkgs/development/libraries/qt-4.x/4.8/default.nix#L101
+pre-configure =
+  set -e -x
+  sed 's,/usr/X11R6/lib64,${libX11:location}/lib64 ${xproto:location}/lib64 ${libXext:location}/lib64,g' -i mkspecs/*/*.conf
+  sed 's,/usr/X11R6/lib,${libX11:location}/lib ${xproto:location}/lib ${libXext:location}/lib,g' -i mkspecs/*/*.conf
+  sed 's,/usr/X11R6/include,${libX11:location}/include ${xproto:location}/include ${libXext:location}/include,g' -i mkspecs/*/*.conf
+configure-command = ./configure --prefix=${:location} -v -no-separate-debug-info -release -no-fast -confirm-license -opensource
+make-targets = qmake
+post-install =
+  cp -rt ${:location} *
+
+[qt4.8-qmake]
+<= qt4-qmake
+[qt4.8.7-qmake]
+<= qt4.8-qmake
\ No newline at end of file

component/tomcat/buildout.cfg

@@ -20,5 +20,13 @@ md5sum = 3dde098fd0b3a08d3f2867e4a95591ba
 recipe = hexagonit.recipe.download
 ignore-existing = true
 strip-top-level-dir = true
-url = http://apache.multidist.com/tomcat/tomcat-7/v7.0.34/bin/apache-tomcat-7.0.34.tar.gz
-md5sum = 0f50494425c24450b4f66dfd4d2aecca
+url = http://www-us.apache.org/dist/tomcat/tomcat-7/v7.0.84/bin/apache-tomcat-7.0.84.tar.gz
+md5sum = 1c6f2c06a90bd7d8a19522749c219a2a
+
+[tomcat7-output]
+# Shared binary location to ease migration
+recipe = plone.recipe.command
+stop-on-error = true
+update-command = ${:command}
+command = ${coreutils-output:test} -x ${:catalina}
+catalina = ${tomcat7:location}/bin/catalina.sh
\ No newline at end of file

component/unixodbc/buildout.cfg

+[buildout]
+parts=
+  unixodbc
+
+[unixodbc]
+recipe = slapos.recipe.cmmi
+url = http://www.unixodbc.org/unixODBC-2.3.5.tar.gz
+md5sum = abf14cf943f1f8c5e63a24cb26d54fd9

component/vnu/buildout.cfg

+# nu - The Nu Html Checker (v.Nu) is an ongoing experiment in better HTML checking.
+# https://validator.w3.org/nu/
+
+[buildout]
+parts =
+  vnu
+
+[vnu]
+recipe = hexagonit.recipe.download
+ignore-existing = true
+strip-top-level-dir = true
+url = https://github.com/validator/validator/releases/download/17.11.1/vnu.war_17.11.1.zip
+md5sum = 2af6dec153a5011cd6fcc85ce5fb599d
+
+[vnu-output]
+# Shared binary location to ease migration
+recipe = plone.recipe.command
+stop-on-error = true
+update-command = ${:command}
+command = ${coreutils-output:test} -r ${:war}
+war = ${vnu:location}/vnu.war

component/zstd/buildout.cfg

@@ -5,7 +5,7 @@ parts =
 [zstd]
 recipe = slapos.recipe.cmmi
 location = ${buildout:parts-directory}/${:_buildout_section_name_}
-url = https://github.com/facebook/zstd/archive/v1.3.1.tar.gz
-md5sum = e849ceef2f090240f690c13fba6ca70b
+url = https://github.com/facebook/zstd/archive/v1.3.3.tar.gz
+md5sum = 187f8df17a75a74f78a23ea4806ac65f
 configure-command = :
 make-options = PREFIX=${:location}

setup.py

@@ -28,7 +28,7 @@ from setuptools import setup, find_packages
 import glob
 import os
 
-version = '1.0.54.dev0'
+version = '1.0.58'
 name = 'slapos.cookbook'
 long_description = open("README.rst").read() + "\n" + \
     open("CHANGES.rst").read() + "\n"
@@ -72,7 +72,6 @@ setup(name=name,
         'zc.buildout': [
           'addresiliency = slapos.recipe.addresiliency:Recipe',
           'accords = slapos.recipe.accords:Recipe',
-          'apache.zope.backend = slapos.recipe.apache_zope_backend:Recipe',
           'apacheperl = slapos.recipe.apacheperl:Recipe',
           'apachephp = slapos.recipe.apachephp:Recipe',
           'apachephpconfigure = slapos.recipe.apachephpconfigure:Recipe',
@@ -118,14 +117,11 @@ setup(name=name,
           'generic.mysql.wrap_update_mysql = slapos.recipe.generic_mysql:WrapUpdateMySQL',
           'generic.mysql.wrap_mysqld = slapos.recipe.generic_mysql:WrapMySQLd',
           'generic.varnish = slapos.recipe.generic_varnish:Recipe',
-          'generic.zope = slapos.recipe.generic_zope:Recipe',
-          'generic.zope.zeo.client = slapos.recipe.generic_zope_zeo_client:Recipe',
           'gitinit = slapos.recipe.gitinit:Recipe',
           'haproxy = slapos.recipe.haproxy:Recipe',
           'ipv4toipv6 = slapos.recipe.6tunnel:FourToSix',
           'ipv6toipv4 = slapos.recipe.6tunnel:SixToFour',
           'jsondump = slapos.recipe.jsondump:Recipe',
-          'kumofs = slapos.recipe.kumofs:Recipe',
           'kvm.frontend = slapos.recipe.kvm_frontend:Recipe',
           'lamp = slapos.recipe.lamp:Request',
           'lamp.generic = slapos.recipe.lampgeneric:Recipe',
@@ -136,7 +132,6 @@ setup(name=name,
           'libcloudrequest = slapos.recipe.libcloudrequest:Recipe',
           'logrotate = slapos.recipe.logrotate:Recipe',
           'logrotate.d = slapos.recipe.logrotate:Part',
-          'memcached = slapos.recipe.memcached:Recipe',
           'mkdirectory = slapos.recipe.mkdirectory:Recipe',
           'mioga.instantiate = slapos.recipe.mioga.instantiate:Recipe',
           'mydumper = slapos.recipe.mydumper:Recipe',
@@ -196,7 +191,6 @@ setup(name=name,
           'urlparse = slapos.recipe._urlparse:Recipe',
           'uuid = slapos.recipe._uuid:Recipe',
           'userinfo = slapos.recipe.userinfo:Recipe',
-          'waitfor = slapos.recipe.waitfor:Recipe',
           'webchecker = slapos.recipe.web_checker:Recipe',
           'wrapper = slapos.recipe.wrapper:Recipe',
           'xvfb = slapos.recipe.xvfb:Recipe',

slapos/recipe/accords/__init__.py

@@ -91,8 +91,8 @@ class Recipe(GenericSlapRecipe):
 
     # Generate wrapper
     wrapper_location = self.createPythonScript(self.options['accords-wrapper'],
-        '%s.accords.runAccords' % __name__,
-        parameter_dict)
+        __name__ + '.accords.runAccords',
+        (parameter_dict,))
     path_list.append(wrapper_location)
 
     # Generate helper for debug

slapos/recipe/addresiliency/__init__.py

@@ -38,26 +38,18 @@ class Recipe(GenericSlapRecipe):
     """
 
     def _install(self):
-        path_list = []
-
         slap_connection = self.buildout['slap-connection']
 
-        takeover_wrapper = self.createPythonScript(
-            name=self.options['wrapper-takeover'],
-            absolute_function='slapos.recipe.addresiliency.takeover.run',
-            arguments={
+        return self.createPythonScript(
+            self.options['wrapper-takeover'],
+            __name__ + '.takeover.takeover',
+            kw={
                 'server_url': slap_connection['server-url'],
                 'key_file': slap_connection.get('key-file'),
                 'cert_file': slap_connection.get('cert-file'),
-                'computer_id': slap_connection['computer-id'],
+                'computer_guid': slap_connection['computer-id'],
                 'partition_id': slap_connection['partition-id'],
-                'software': slap_connection['software-release-url'],
+                'software_release': slap_connection['software-release-url'],
                 'namebase': self.parameter_dict['namebase'],
                 'takeover_triggered_file_path': self.options['takeover-triggered-file-path'],
             })
-
-        path_list.append(takeover_wrapper)
-
-        return path_list
-
-

slapos/recipe/addresiliency/takeover.py

@@ -78,14 +78,3 @@ def takeover(server_url, key_file, cert_file, computer_guid,
   # Create "lock" file preventing equeue to run import scripts
   # XXX hardcoded
   open(takeover_triggered_file_path, 'w').write('')
-
-def run(args):
-  slapos.recipe.addresiliency.takeover.takeover(server_url = args.pop('server_url'),
-                                                key_file = args.pop('key_file'),
-                                                cert_file = args.pop('cert_file'),
-                                                computer_guid = args.pop('computer_id'),
-                                                partition_id = args.pop('partition_id'),
-                                                software_release = args.pop('software'),
-                                                namebase = args.pop('namebase'),
-                                                takeover_triggered_file_path = args.pop('takeover_triggered_file_path'))
-

slapos/recipe/apache_zope_backend/__init__.py

-##############################################################################
-#
-# Copyright (c) 2011 Vifib SARL and Contributors. All Rights Reserved.
-#
-# WARNING: This program as such is intended to be used by professional
-# programmers who take the whole responsibility of assessing all potential
-# consequences resulting from its eventual inadequacies and bugs
-# End users who are looking for a ready-to-use solution with commercial
-# guarantees and support are strongly adviced to contract a Free Software
-# Service Company
-#
-# This program is Free Software; you can redistribute it and/or
-# modify it under the terms of the GNU General Public License
-# as published by the Free Software Foundation; either version 3
-# of the License, or (at your option) any later version.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with this program; if not, write to the Free Software
-# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA  02111-1307, USA.
-#
-##############################################################################
-from slapos.recipe.librecipe import GenericBaseRecipe
-import pkg_resources
-
-class Recipe(GenericBaseRecipe):
-  def install(self):
-    try:
-      backend_list = self.options['backend-list']
-    except KeyError:
-      backend_list = [(self.options['port'], self.options['backend'])]
-
-    scheme = self.options['scheme']
-    if scheme == 'http':
-      required_path_list = []
-      ssl_enable = ssl_snippet = ''
-    elif scheme == 'https':
-      key = self.options['key-file']
-      certificate = self.options['cert-file']
-      required_path_list = [key, certificate]
-      ssl_snippet = self.substituteTemplate(self.getTemplateFilename('snippet.ssl.in'), {
-        'key': key,
-        'certificate': certificate,
-        'ssl_session_cache': self.options['ssl-session-cache'],
-      })
-      if 'ssl-authentication' in self.options and self.optionIsTrue(
-          'ssl-authentication'):
-        ssl_snippet += self.substituteTemplate(self.getTemplateFilename('snippet.ssl.ca.in'), {
-          'ca_certificate': self.options['ssl-authentication-certificate'],
-          'ca_crl': self.options['ssl-authentication-crl'],
-        })
-      ssl_enable = 'SSLEngine on'
-    else:
-      raise ValueError('Unsupported scheme %s' % scheme)
-
-    ip_list = self.options['ip']
-    if isinstance(ip_list, basestring):
-      ip_list = [ip_list]
-    backend_path = self.options.get('backend-path', '/')
-    vhost_template_name = self.getTemplateFilename('vhost.in')
-    apache_config_file = self.createFile(
-      self.options['configuration-file'],
-      self.substituteTemplate(
-        self.getTemplateFilename('apache.zope.conf.in'),
-        {
-          'path': '/',
-          'server_admin': 'admin@',
-          'pid_file': self.options['pid-file'],
-          'lock_file': self.options['lock-file'],
-          'error_log': self.options['error-log'],
-          'access_log': self.options['access-log'],
-          'access_control_string': self.options['access-control-string'],
-          'ssl_snippet': ssl_snippet,
-          'vhosts': ''.join(self.substituteTemplate(vhost_template_name, {
-            'ip': ip,
-            'port': port,
-            'backend': ('%s/%s' % (backend.rstrip('/'), backend_path.strip('/'))).rstrip('/'),
-            'ssl_enable': ssl_enable,
-          }) for (port, backend) in backend_list for ip in ip_list),
-        },
-      )
-    )
-    return [
-      apache_config_file,
-      self.createPythonScript(
-        self.options['wrapper'],
-        __name__ + '.apache.runApache',
-        [
-          {
-            'required_path_list': required_path_list,
-            'binary': self.options['apache-binary'],
-            'config': apache_config_file,
-          },
-        ],
-      ),
-    ]

slapos/recipe/apache_zope_backend/apache.py

-import os
-import sys
-import time
-
-
-def runApache(args):
-  sleep = 60
-  conf = args[0]
-  while True:
-    ready = True
-    for f in conf.get('required_path_list', []):
-      if not os.path.exists(f):
-        print 'File %r does not exists, sleeping for %s' % (f, sleep)
-        ready = False
-    if ready:
-      break
-    time.sleep(sleep)
-  apache_wrapper_list = [conf['binary'], '-f', conf['config'], '-DFOREGROUND']
-  apache_wrapper_list.extend(sys.argv[1:])
-  sys.stdout.flush()
-  sys.stderr.flush()
-  os.execl(apache_wrapper_list[0], *apache_wrapper_list)

slapos/recipe/apache_zope_backend/template/apache.zope.conf.in

-# Apache configuration file for Zope
-# Automatically generated
-
-# List of modules
-LoadModule unixd_module modules/mod_unixd.so
-LoadModule access_compat_module modules/mod_access_compat.so
-LoadModule authz_core_module modules/mod_authz_core.so
-LoadModule authz_host_module modules/mod_authz_host.so
-LoadModule log_config_module modules/mod_log_config.so
-LoadModule setenvif_module modules/mod_setenvif.so
-LoadModule version_module modules/mod_version.so
-LoadModule proxy_module modules/mod_proxy.so
-LoadModule proxy_http_module modules/mod_proxy_http.so
-LoadModule socache_shmcb_module modules/mod_socache_shmcb.so
-LoadModule ssl_module modules/mod_ssl.so
-LoadModule mime_module modules/mod_mime.so
-LoadModule dav_module modules/mod_dav.so
-LoadModule dav_fs_module modules/mod_dav_fs.so
-LoadModule negotiation_module modules/mod_negotiation.so
-LoadModule rewrite_module modules/mod_rewrite.so
-LoadModule headers_module modules/mod_headers.so
-
-# Basic server configuration
-PidFile "%(pid_file)s"
-ServerAdmin %(server_admin)s
-TypesConfig conf/mime.types
-AddType application/x-compress .Z
-AddType application/x-gzip .gz .tgz
-
-ServerTokens Prod
-ServerSignature Off
-TraceEnable Off
-
-# Apache 2.4's default value (60 seconds) can be a bit too short
-TimeOut 300
-
-# As backend is trusting REMOTE_USER header unset it always
-RequestHeader unset REMOTE_USER
-
-%(ssl_snippet)s
-
-# Log configuration
-ErrorLog "%(error_log)s"
-# Default apache log format with request time in microsecond at the end
-LogFormat "%%h %%l %%u %%t \"%%r\" %%>s %%b \"%%{Referer}i\" \"%%{User-Agent}i\" %%D" combined
-CustomLog "%(access_log)s" combined
-
-# Directory protection
-<Directory />
-    Options FollowSymLinks
-    AllowOverride None
-    Order deny,allow
-    Deny from all
-</Directory>
-
-# Path protected
-<Location %(path)s>
-  Order Deny,Allow
-  Deny from all
-  Allow from %(access_control_string)s
-</Location>
-
-# Magic of Zope related rewrite
-RewriteEngine On
-%(vhosts)s

slapos/recipe/apache_zope_backend/template/snippet.ssl.ca.in

-SSLVerifyClient require
-RequestHeader set REMOTE_USER %%{SSL_CLIENT_S_DN_CN}s
-SSLCACertificateFile %(ca_certificate)s
-SSLCARevocationPath %(ca_crl)s

slapos/recipe/apache_zope_backend/template/snippet.ssl.in

-SSLCertificateFile %(certificate)s
-SSLCertificateKeyFile %(key)s
-SSLRandomSeed startup builtin
-SSLRandomSeed connect builtin
-SSLProtocol all -SSLv2 -SSLv3
-SSLCipherSuite ECDH+AESGCM:DH+AESGCM:ECDH+AES256:DH+AES256:ECDH+AES128:DH+AES:ECDH+3DES:DH+3DES:RSA+AESGCM:RSA+AES:RSA+3DES:HIGH:!aNULL:!MD5
-SSLHonorCipherOrder on
-SSLSessionCache shmcb:%(ssl_session_cache)s(512000)
-SSLProxyEngine On

slapos/recipe/apache_zope_backend/template/vhost.in

-Listen %(ip)s:%(port)s
-<VirtualHost *:%(port)s>
-  %(ssl_enable)s
-  RewriteRule ^/(.*) %(backend)s/$1 [L,P]
-</VirtualHost>

slapos/recipe/apacheperl/__init__.py

@@ -57,10 +57,9 @@ class Recipe(GenericBaseRecipe):
     )
     path_list.append(httpd_conf)
 
-    wrapper = self.createPythonScript(self.options['wrapper'],
-        'slapos.recipe.librecipe.execute.execute',
-        [self.options['httpd-binary'], '-f', self.options['httpd-conf'],
-         '-DFOREGROUND']
+    wrapper = self.createWrapper(self.options['wrapper'],
+        (self.options['httpd-binary'], '-f', self.options['httpd-conf'],
+          '-DFOREGROUND'),
     )
     path_list.append(wrapper)
 

slapos/recipe/apachephp/__init__.py

@@ -92,14 +92,13 @@ class Recipe(GenericBaseRecipe):
       )
       path_list.append(httpd_conf)
 
-    wrapper = self.createWrapper(name=self.options['wrapper'],
-                                 command=self.options['httpd-binary'],
-                                 parameters=[
+    wrapper = self.createWrapper(self.options['wrapper'],
+                                 (self.options['httpd-binary'],
                                      '-f',
                                      self.options['httpd-conf'],
                                      '-DFOREGROUND'
-                                     ],
-                                 environment=self.environ)
+                                     ),
+                                 self.environ)
     path_list.append(wrapper)
 
     secret_key_filename = os.path.join(self.buildout['buildout']['directory'],

slapos/recipe/apachephpconfigure/__init__.py

@@ -118,7 +118,7 @@ class Recipe(GenericBaseRecipe):
     configureinstall_wrapper_path = self.createPythonScript(
         self.options['configureinstall-location'],
         __name__ + '.runner.executeRunner',
-        [argument, delete, rename, chmod, data]
+        (argument, delete, rename, chmod, data)
     )
     
     #TODO finish to port this and remove upper one

slapos/recipe/apachephpconfigure/runner.py

 import subprocess
 
-def executeRunner(args):
+def executeRunner(arguments, delete, rename, chmod, data):
   """Start the instance configure. this may run a python script, move or/and rename
   file or directory when dondition is filled. the condition may be when file exist or when an entry
   exist into database.
   """
-  arguments, delete, rename, chmod, data = args
-  if delete != []:
+  if delete:
     print "Calling lampconfigure with 'delete' arguments"
-    result = subprocess.Popen(arguments + delete)
-    result.wait()
-  if rename != []:
+    subprocess.call(arguments + delete)
+  if rename:
     for parameters in rename:
       print "Calling lampconfigure with 'rename' arguments"
-      result = subprocess.Popen(arguments + parameters)
-      result.wait()
-  if chmod != []:
+      subprocess.call(arguments + parameters)
+  if chmod:
     print "Calling lampconfigure with 'chmod' arguments"
-    result = subprocess.Popen(arguments + chmod)
-    result.wait()
-  if data != []:
+    subprocess.call(arguments + chmod)
+  if data:
     print "Calling lampconfigure with 'run' arguments"
     print arguments + data
-    result = subprocess.Popen(arguments + data)
-    result.wait()
-    return
+    subprocess.call(arguments + data)
 

slapos/recipe/apacheproxy/__init__.py

@@ -49,13 +49,12 @@ class Recipe(GenericBaseRecipe):
     )
     path_list.append(httpd_conf)
 
-    wrapper = self.createWrapper(name=self.options['wrapper'],
-                                 command=self.options['httpd-binary'],
-                                 parameters=[
+    wrapper = self.createWrapper(self.options['wrapper'],
+                                 (self.options['httpd-binary'],
                                      '-f',
                                      self.options['httpd-conf'],
                                      '-DFOREGROUND',
-                                     ])
+                                     ))
 
     path_list.append(wrapper)
 

slapos/recipe/boinc/__init__.py

@@ -124,12 +124,15 @@ class Recipe(GenericBaseRecipe):
 
     #Generate wrapper for php
     wrapperphp = os.path.join(self.home, 'bin/php')
-    php_wrapper = self.createPythonScript(wrapperphp,
-        'slapos.recipe.librecipe.execute.executee',
-        ([self.phpbin, '-c', self.phpini], os.environ)
+    php_wrapper = self.createWrapper(wrapperphp,
+        (self.phpbin, '-c', self.phpini),
     )
     path_list.append(php_wrapper)
 
+    mysql_dict = dict(db=self.database,
+        host=self.mysqlhost, port=self.mysqlport,
+        user=self.username, passwd=self.password)
+
     #Generate python script for MySQL database test (starting)
     file_status = os.path.join(self.home, '.boinc_config')
     if os.path.exists(file_status):
@@ -137,11 +140,7 @@ class Recipe(GenericBaseRecipe):
     mysql_wrapper = self.createPythonScript(
       os.path.join(self.wrapperdir, 'start_config'),
       '%s.configure.checkMysql' % __name__,
-      dict(mysql_port=self.mysqlport, mysql_host=self.mysqlhost,
-          mysql_user=self.username, mysql_password=self.password,
-          database=self.database,
-          file_status=file_status, environment=environment
-      )
+      (environment, mysql_dict, file_status)
     )
 
     # Generate make project wrapper file
@@ -164,8 +163,7 @@ class Recipe(GenericBaseRecipe):
     install_wrapper = self.createPythonScript(
       os.path.join(self.wrapperdir, 'make_project'),
       '%s.configure.makeProject' % __name__,
-      dict(launch_args=launch_args, request_file=request_make_boinc,
-      make_sig=file_status, env=environment)
+      (file_status, launch_args, request_make_boinc, environment)
     )
     path_list.append(install_wrapper)
 
@@ -197,7 +195,7 @@ class Recipe(GenericBaseRecipe):
     )
     start_service = self.createPythonScript(
       os.path.join(self.wrapperdir, 'config_project'),
-      '%s.configure.services' % __name__, parameter
+      '%s.configure.services' % __name__, (parameter,)
     )
     path_list.append(start_service)
 
@@ -208,14 +206,12 @@ class Recipe(GenericBaseRecipe):
       os.unlink(start_boinc)
     boinc_parameter = dict(service_status=service_status,
         installroot=self.installroot, drop_install=drop_install,
-        mysql_port=self.mysqlport, mysql_host=self.mysqlhost,
-        mysql_user=self.username, mysql_password=self.password,
-        database=self.database, environment=environment,
+        mysql_dict=mysql_dict, environment=environment,
         start_boinc=start_boinc)
     start_wrapper = self.createPythonScript(os.path.join(self.wrapperdir,
         'start_boinc'),
         '%s.configure.restart_boinc' % __name__,
-        boinc_parameter
+        (boinc_parameter,)
     )
     path_list.append(start_wrapper)
 
@@ -362,7 +358,7 @@ class App(GenericBaseRecipe):
         )
         deploy_app = self.createPythonScript(
           os.path.join(wrapperdir, 'boinc_%s' % appname),
-          '%s.configure.deployApp' % __name__, parameter
+          '%s.configure.deployApp' % __name__, (parameter,)
         )
         path_list.append(deploy_app)
 
@@ -404,17 +400,15 @@ class Client(GenericBaseRecipe):
       cc_cmd = '--read_cc_config'
     cmd = self.createPythonScript(cmd_wrapper,
         '%s.configure.runCmd' % __name__,
-        dict(base_cmd=base_cmd, cc_cmd=cc_cmd, installdir=installdir,
-        project_url=url, key=key)
+        (base_cmd, cc_cmd, installdir, url, key)
     )
     path_list.append(cmd)
 
     #Generate BOINC client wrapper
-    boinc = self.createPythonScript(boinc_wrapper,
-            'slapos.recipe.librecipe.execute.execute',
-            [boincbin, '--allow_multiple_clients', '--gui_rpc_port',
+    boinc = self.createWrapper(boinc_wrapper,
+            (boincbin, '--allow_multiple_clients', '--gui_rpc_port',
               str(self.options['rpc-port']), '--allow_remote_gui_rpc',
-              '--dir', installdir, '--redirectio', '--check_all_logins']
+              '--dir', installdir, '--redirectio', '--check_all_logins'),
     )
     path_list.append(boinc)
 

slapos/recipe/boinc/configure.py

@@ -35,27 +35,21 @@ import filecmp
 
 from lock_file import LockFile
 
-def checkMysql(args):
-  sys.path += args['environment']['PYTHONPATH'].split(':')
+def checkMysql(environment, connect_kw, file_status=None):
+  sys.path += environment['PYTHONPATH'].split(':')
   import MySQLdb
   #Sleep until mysql server becomes available
   while True:
     try:
-      conn = MySQLdb.connect(host = args['mysql_host'],
-                          user = args['mysql_user'],
-                          port = int(args['mysql_port']),
-                          passwd = args['mysql_password'],
-                          db = args['database'])
-      conn.close()
-      print "Successfully connect to MySQL database... "
-      if args.has_key('file_status'):
-        writeFile(args['file_status'], "starting")
+      MySQLdb.connect(**connect_kw).close()
       break
     except Exception, ex:
       print "The result is: \n" + ex.message
       print "Could not connect to MySQL database... sleep for 2 secondes"
       time.sleep(2)
-
+  print "Successfully connect to MySQL database... "
+  if file_status:
+    writeFile(file_status, "starting")
 
 def checkFile(file, stime):
   """Loop until 'file' is created (exist)"""
@@ -70,18 +64,16 @@ def checkFile(file, stime):
 
 def restart_boinc(args):
   """Stop (if currently is running state) and start all Boinc service"""
+  environment = args['environment']
   if args['drop_install']:
     checkFile(args['service_status'], 3)
   else:
-    checkMysql(args)
+    checkMysql(environment, args['mysql_dict'], args.get('file_status'))
   print "Restart Boinc..."
-  env = os.environ
-  env['PATH'] = args['environment']['PATH']
-  env['PYTHONPATH'] = args['environment']['PYTHONPATH']
-  binstart = os.path.join(args['installroot'], 'bin/start')
-  binstop = os.path.join(args['installroot'], 'bin/stop')
-  os.system(binstop)
-  os.system(binstart)
+  env = os.environ.copy()
+  env.update(environment)
+  subprocess.call((os.path.join(args['installroot'], 'bin', 'stop'),), env=env)
+  subprocess.call((os.path.join(args['installroot'], 'bin', 'start'),), env=env)
   writeFile(args['start_boinc'], "started")
   print "Done."
 
@@ -122,17 +114,16 @@ def startProcess(launch_args, env=None, cwd=None, stdout=subprocess.PIPE):
     return False
   return True
 
-def makeProject(args):
+def makeProject(make_sig, launch_args, request_file, extra_environ):
   """Run BOINC make_project script but once only"""
   #Wait for DateBase initialization...
-  checkFile(args['make_sig'], 3)
+  checkFile(make_sig, 3)
   print "Cheking if needed to run BOINC make_project..."
-  if os.path.exists(args['request_file']):
-    env = os.environ
-    env['PATH'] = args['env']['PATH']
-    env['PYTHONPATH'] = args['env']['PYTHONPATH']
-    if startProcess(args['launch_args'], env=env):
-      os.unlink(args['request_file'])
+  if os.path.exists(request_file):
+    env = os.environ.copy()
+    env.update(extra_environ)
+    if startProcess(launch_args, env=env):
+      os.unlink(request_file)
     print "Finished running BOINC make_projet...Ending"
   else:
     print "No new request for make_project. Exiting..."
@@ -155,9 +146,8 @@ def services(args):
     return
 
   print "execute script xadd..."
-  env = os.environ
-  env['PATH'] = args['environment']['PATH']
-  env['PYTHONPATH'] = args['environment']['PYTHONPATH']
+  env = os.environ.copy()
+  env.update(args['environment'])
   if not startProcess([os.path.join(args['installroot'], 'bin/xadd')], env):
     return
   print "Update files and directories permissions..."
@@ -212,9 +202,8 @@ def deployManagement(args):
     newInstall = True
   #Sleep until file .start_boinc exist (File indicate that BOINC has been started)
   checkFile(args['start_boinc'], 3)
-  env = os.environ
-  env['PATH'] = args['environment']['PATH']
-  env['PYTHONPATH'] = args['environment']['PYTHONPATH']
+  env = os.environ.copy()
+  env.update(args['environment'])
 
   print "setup directories..."
   numversion = args['version'].replace('.', '')
@@ -263,7 +252,7 @@ def deployManagement(args):
     privateKeyFile = os.path.join(args['installroot'], 'keys/code_sign_private')
     output = open(binary + '.sig', 'w')
     p_sign = subprocess.Popen([sign, binary, privateKeyFile], stdout=output,
-            stderr=subprocess.STDOUT)
+            stderr=subprocess.STDOUT, env=env)
     result = p_sign.communicate()[0]
     if p_sign.returncode is None or p_sign.returncode != 0:
       print "Failed to execute bin/sign_executable.\nThe error was: %s" % result
@@ -290,10 +279,8 @@ def deployManagement(args):
   create_wu(args, env)
 
   print "Restart Boinc..."
-  binstart = os.path.join(args['installroot'], 'bin/start')
-  binstop = os.path.join(args['installroot'], 'bin/stop')
-  os.system(binstop)
-  os.system(binstart)
+  subprocess.call((os.path.join(args['installroot'], 'bin', 'stop'),), env=env)
+  subprocess.call((os.path.join(args['installroot'], 'bin', 'start'),), env=env)
 
   print "Boinc Application deployment is done... writing end signal file..."
   writeFile(token, str(args['wu_number']))
@@ -315,22 +302,21 @@ def create_wu(args, env):
     startProcess(launch_args, env, args['installroot'])
 
 
-def runCmd(args):
+def runCmd(base_cmd, cc_cmd, installdir, url, key):
   """Wait for Boinc Client started and run boinc cmd"""
-  client_config = os.path.join(args['installdir'], 'client_state.xml')
+  client_config = os.path.join(installdir, 'client_state.xml')
   checkFile(client_config, 5)
   time.sleep(10)
   #Scan client state xml to find client ipv4 adress
   host = re.search("<ip_addr>([\w\d\.:]+)</ip_addr>",
                 open(client_config, 'r').read()).group(1)
-  args['base_cmd'][2] = host + ':' + args['base_cmd'][2]
-  print "Run boinccmd with host at %s " % args['base_cmd'][2]
-  project_args = args['base_cmd'] + ['--project_attach', args['project_url'],
-                      args['key']]
-  startProcess(project_args, cwd=args['installdir'])
-  if args['cc_cmd'] != '':
+  base_cmd[2] = host + ':' + base_cmd[2]
+  print "Run boinccmd with host at %s " % base_cmd[2]
+  project_args = base_cmd + ['--project_attach', url, key]
+  startProcess(project_args, cwd=installdir)
+  if cc_cmd:
     #Load or reload cc_config file
-    startProcess(args['base_cmd'] + [args['cc_cmd']], cwd=args['installdir'])
+    startProcess(base_cmd + [cc_cmd], cwd=installdir)
 
 
 def writeFile(file, content):

slapos/recipe/bonjourgrid/__init__.py

@@ -62,8 +62,8 @@ class Recipe(GenericBaseRecipe):
                       condor_wrapper_list=condor_wrapper_list,
                       boinc_wrapper_list=boinc_wrapper_list)
     bonjourGrid_wrapper = self.createPythonScript(grid_wrapper,
-        '%s.configure.launchScript' % __name__,
-        parameters
+        __name__ + '.configure.launchScript',
+        (parameters,)
     )
     path_list.append(bonjourGrid_wrapper)
 
@@ -73,16 +73,15 @@ class Recipe(GenericBaseRecipe):
     bg_wrapper = self.options['wrapper'].strip()
     log = self.options['log_file'].strip()
     pid_file = self.options['pid_file'].strip()
-    wrapper = self.createPythonScript(bg_wrapper,
-        'slapos.recipe.librecipe.execute.execute',
-        ([python, bonjourgrid_master, '--log_file', log,
+    wrapper = self.createWrapper(bg_wrapper,
+        (python, bonjourgrid_master, '--log_file', log,
           '--pid_file', pid_file,
           '--master_wrapper', grid_wrapper,
           '--directory', self.options['work_dir'].strip(),
           '--server', self.options['redis-url'].strip(),
           '--port', self.options['redis-port'].strip(),
           '--num_workers', self.options['nworkers'].strip(),
-        ])
+         ),
     )
     path_list.append(wrapper)
 
@@ -113,9 +112,8 @@ class Client(GenericBaseRecipe):
     bg_wrapper = self.options['wrapper'].strip()
     log = self.options['log_file'].strip()
     pid_file = self.options['pid_file'].strip()
-    wrapper = self.createPythonScript(bg_wrapper,
-        'slapos.recipe.librecipe.execute.execute',
-        ([python, bonjourgrid_client, '--log_file', log,
+    wrapper = self.createWrapper(bg_wrapper,
+        (python, bonjourgrid_client, '--log_file', log,
           '--pid_file', pid_file,
           '--boinc_wrapper', boinc_script,
           '--condor_wrapper', condor_script,
@@ -123,7 +121,7 @@ class Client(GenericBaseRecipe):
           '--install_directory', self.options['install_dir'].strip(),
           '--server', self.options['redis-url'].strip(),
           '--port', self.options['redis-port'].strip(),
-        ])
+         ),
     )
     path_list.append(wrapper)
 

slapos/recipe/certificate_authority/__init__.py

@@ -40,13 +40,10 @@ class Recipe(GenericBaseRecipe):
     self.ca_private = self.options['ca-private']
     self.ca_certs = self.options['ca-certs']
     self.ca_newcerts = self.options['ca-newcerts']
-    self.ca_crl = self.options['ca-crl']
     self.ca_key_ext = '.key'
     self.ca_crt_ext = '.crt'
 
   def install(self):
-    path_list = []
-
     ca_country_code = self.options.get('country-code', 'XX')
     ca_email = self.options.get('email', 'xx@example.com')
     # XXX-BBB: State by mistake has been configured as string "('State',)"
@@ -77,21 +74,15 @@ class Recipe(GenericBaseRecipe):
     self.createFile(openssl_configuration, self.substituteTemplate(
       self.getTemplateFilename('openssl.cnf.ca.in'), config))
 
-    ca_wrapper = self.createPythonScript(
+    return self.createPythonScript(
       self.options['wrapper'],
-      '%s.certificate_authority.runCertificateAuthority' % __name__,
-      dict(
-        openssl_configuration=openssl_configuration,
-        openssl_binary=self.options['openssl-binary'],
-        certificate=os.path.join(self.ca_dir, 'cacert.pem'),
-        key=os.path.join(self.ca_private, 'cakey.pem'),
-        crl=self.ca_crl,
-        request_dir=self.request_directory
-      )
+      __name__ + '.certificate_authority.runCertificateAuthority',
+      (os.path.join(self.ca_private, 'cakey.pem'),
+       os.path.join(self.ca_dir, 'cacert.pem'),
+       self.options['openssl-binary'],
+       openssl_configuration,
+       self.request_directory)
     )
-    path_list.append(ca_wrapper)
-
-    return path_list
 
 class Request(Recipe):
 
@@ -146,11 +137,10 @@ class Request(Recipe):
 
     path_list = [key_file, cert_file]
     if request_needed:
-      wrapper = self.createPythonScript(
+      wrapper = self.createWrapper(
         self.options['wrapper'],
-        'slapos.recipe.librecipe.execute.execute_wait',
-        [ [self.options['executable']],
-          [certificate, key] ],
+        (self.options['executable'],),
+        wait_list=(certificate, key),
       )
       path_list.append(wrapper)
 

slapos/recipe/certificate_authority/certificate_authority.py

@@ -102,10 +102,8 @@ class CertificateAuthority:
             'certificate_file')):
         print 'Created certificate %r' % parser.get('certificate', 'name')
 
-def runCertificateAuthority(ca_conf):
-  ca = CertificateAuthority(ca_conf['key'], ca_conf['certificate'],
-      ca_conf['openssl_binary'], ca_conf['openssl_configuration'],
-      ca_conf['request_dir'])
+def runCertificateAuthority(*args):
+  ca = CertificateAuthority(*args)
   while True:
     ca.checkAuthority()
     ca.checkRequestDir()

slapos/recipe/cloud9/__init__.py

@@ -42,8 +42,6 @@ class Recipe(GenericBaseRecipe):
     options['access-url'] = 'http://[%s]:%s' % (self.ip, self.port)
 
   def install(self):
-    path_list = []
-
     environment = {
         'PATH': os.path.dirname(self.git) + ':' + os.environ['PATH'],
     }
@@ -51,10 +49,4 @@ class Recipe(GenericBaseRecipe):
     cloud9_args = [self.node_executable, self.cloud9, '-l', self.ip, '-p',
         self.port, '-w', self.workdir]
 
-    wrapper = self.createPythonScript(self.wrapper,
-        'slapos.recipe.librecipe.execute.executee',
-        (cloud9_args, environment)
-    )
-    path_list.append(wrapper)
-
-    return path_list
+    return self.createWrapper(self.wrapper, cloud9_args, environment)

slapos/recipe/condor/__init__.py

@@ -178,12 +178,11 @@ class Recipe(GenericBaseRecipe):
       os.chmod(wrapper_location, 0744)
 
     #generate script for start condor
-    start_condor = os.path.join(self.wrapperdir, 'start_condor')
-    start_bin = os.path.join(self.wrapper_sbin, 'condor_master')
-    condor_reconfig = os.path.join(self.wrapper_sbin, 'condor_reconfig')
-    wrapper = self.createPythonScript(start_condor,
-        '%s.configure.condorStart' % __name__,
-        dict(start_bin=start_bin, condor_reconfig=condor_reconfig)
+    wrapper = self.createPythonScript(
+        os.path.join(self.wrapperdir, 'start_condor'),
+        __name__ + '.configure.condorStart',
+        (os.path.join(self.wrapper_sbin, 'condor_reconfig'),
+         os.path.join(self.wrapper_sbin, 'condor_master'))
     )
     path_list.append(wrapper)
     return path_list
@@ -276,13 +275,11 @@ class AppSubmit(GenericBaseRecipe):
           os.unlink(destination)
         os.symlink(app_list[appname]['files'][file], destination)
       #generate wrapper for submitting job
-      condor_submit = os.path.join(self.options['bin'].strip(), 'condor_submit')
-      parameter = dict(submit=condor_submit, sig_install=sig_install,
-                      submit_file='submit',
-                      appname=appname, appdir=appdir)
       submit_job = self.createPythonScript(
         os.path.join(self.options['wrapper-dir'].strip(), appname),
-        '%s.configure.submitJob' % __name__, parameter
+        __name__ + '.configure.submitJob',
+        (os.path.join(self.options['bin'].strip(), 'condor_submit'),
+         'submit', appdir, appname, sig_install)
       )
       path_list.append(submit_job)
     return path_list
\ No newline at end of file

slapos/recipe/condor/configure.py

@@ -29,27 +29,25 @@ import os
 import subprocess
 import time
 
-def submitJob(args):
+def submitJob(submit, submit_file, appdir, appname, sig_install):
   """Run condor_submit (if needed) for job deployment"""
   time.sleep(10)
-  print "Check if needed to submit %s job's" % args['appname']
-  if not os.path.exists(args['sig_install']):
+  print "Check if needed to submit %s job's" % appname
+  if not os.path.exists(sig_install):
     print "Nothing for install or update...Exited"
     return
   # '-a', "log = out.log", '-a', "error = error.log",
-  launch_args = [args['submit'], '-verbose', args['submit_file']]
+  launch_args = submit, '-verbose', submit_file
   process = subprocess.Popen(launch_args, stdout=subprocess.PIPE,
-              stderr=subprocess.STDOUT, cwd=args['appdir'])
+              stderr=subprocess.STDOUT, cwd=appdir)
   result = process.communicate()[0]
   if process.returncode is None or process.returncode != 0:
     print "Failed to execute condor_submit.\nThe error was: %s" % result
   else:
-    os.unlink(args['sig_install'])
+    os.unlink(sig_install)
 
-def condorStart(args):
+def condorStart(condor_reconfig, start_bin):
   """Start Condor if deamons is currently stopped"""
-  result = os.system(args['condor_reconfig'])
-  if result != 0:
+  if subprocess.call(condor_reconfig):
     #process failled to reconfig condor that mean that condor deamons is not curently started
-    os.system(args['start_bin'])
-  
+    subprocess.call(start_bin)
\ No newline at end of file

slapos/recipe/davstorage/__init__.py

@@ -98,26 +98,19 @@ class Recipe(GenericBaseRecipe):
     )
     path_list.append(config_file)
 
-    wrapper = self.createPythonScript(self.options['wrapper'],
-      'slapos.recipe.librecipe.execute.execute',
-      [self.options['apache-binary'], '-f', config_file, '-DFOREGROUND'])
+    wrapper = self.createWrapper(self.options['wrapper'],
+      (self.options['apache-binary'], '-f', config_file, '-DFOREGROUND'))
     path_list.append(wrapper)
 
     promise = self.createPythonScript(self.options['promise'],
       __name__ + '.promise',
-      dict(host=self.options['ip'], port=int(self.options['port_webdav']),
-           user=self.options['user'], password=self.options['password'])
-                                     )
+      (self.options['ip'], int(self.options['port_webdav']),
+       self.options['user'], self.options['password']))
     path_list.append(promise)
 
     return path_list
 
-def promise(args):
-  host = args['host']
-  port = args['port']
-  user = args['user']
-  password = args['password']
-
+def promise(host, port, user, password):
   connection = httplib.HTTPSConnection(host, port)
   auth = base64.b64encode('%s:%s' % (user, password))
   connection.request('OPTIONS', '/',

slapos/recipe/dcron.py

@@ -35,15 +35,14 @@ class Recipe(GenericBaseRecipe):
     self.logger.info("Installing dcron...")
 
     options = self.options
-    script = self.createWrapper(name=options['binary'],
-                                command=options['dcrond-binary'].strip(),
-                                parameters=[
+    script = self.createWrapper(options['binary'],
+                                (options['dcrond-binary'].strip(),
                                     '-s', options['cron-entries'],
                                     '-c', options['crontabs'],
                                     '-t', options['cronstamps'],
                                     '-f', '-l', '5',
                                     '-M', options['catcher']
-                                    ])
+                                    ))
 
     self.logger.debug('Main cron executable created at : %r', script)
 

slapos/recipe/dropbear.py

@@ -57,8 +57,6 @@ class KnownHostsFile(dict):
 class Recipe(GenericBaseRecipe):
 
   def install(self):
-    path_list = []
-
     dropbear_cmd = [self.options['dropbear-binary']]
     # Don't fork into background
     dropbear_cmd.append('-F')
@@ -95,19 +93,12 @@ class Recipe(GenericBaseRecipe):
     if 'shell' in self.options:
       env['DROPBEAR_OVERRIDE_SHELL'] = self.options['shell']
 
-    wrapper = self.createPythonScript(
-      self.options['wrapper'],
-      'slapos.recipe.librecipe.execute.executee',
-      (dropbear_cmd, env, )
-    )
-    path_list.append(wrapper)
-
-    return path_list
+    return self.createWrapper(self.options['wrapper'], dropbear_cmd, env)
 
 class Client(GenericBaseRecipe):
 
   def install(self):
-    env = dict()
+    env = {}
 
     if 'home' in self.options:
       env['HOME'] = self.options['home']
@@ -120,13 +111,7 @@ class Client(GenericBaseRecipe):
     if 'identity-file' in self.options:
       dropbear_cmd.extend(['-i', self.options['identity-file']])
 
-    wrapper = self.createPythonScript(
-      self.options['wrapper'],
-      'slapos.recipe.librecipe.execute.executee',
-      (dropbear_cmd, env, )
-    )
-
-    return [wrapper]
+    return self.createWrapper(self.options['wrapper'], dropbear_cmd, env)
 
 
 class AddAuthorizedKey(GenericBaseRecipe):

slapos/recipe/duplicity.py

@@ -46,7 +46,4 @@ class Recipe(GenericBaseRecipe):
       cmd.extend(options)
       cmd.extend([backup_directory, remote_url])
 
-    wrapper = self.createPythonScript(self.options['wrapper'],
-      'slapos.recipe.librecipe.execute.execute', cmd)
-
-    return [wrapper]
+    return self.createWrapper(self.options['wrapper'], cmd)

slapos/recipe/equeue.py

@@ -30,23 +30,20 @@ class Recipe(GenericBaseRecipe):
 
   def install(self):
 
-    parameters = [
+    args = [
+      self.options['equeue-binary'],
       '--database', self.options['database'],
       '--logfile', self.options['log'],
       '--lockfile', self.options['lockfile']
     ]
 
     if 'takeover-triggered-file-path' in self.options:
-      parameters.extend(['--takeover-triggered-file-path', self.options['takeover-triggered-file-path']])
+      args += ('--takeover-triggered-file-path',
+               self.options['takeover-triggered-file-path'])
 
     if 'loglevel' in self.options:
-      parameters.extend(['--loglevel', self.options['loglevel']])
+      args += '--loglevel', self.options['loglevel']
 
-    parameters.append(self.options['socket'])
-
-    wrapper = self.createWrapper(name=self.options['wrapper'],
-                                 command=self.options['equeue-binary'],
-                                 parameters=parameters)
-
-    return [wrapper]
+    args.append(self.options['socket'])
 
+    return self.createWrapper(self.options['wrapper'], args)

slapos/recipe/erp5_test/__init__.py

@@ -67,7 +67,7 @@ class Recipe(GenericBaseRecipe):
         openssl_binary=self.options['openssl-binary'],
         test_ca_path=self.options['certificate-authority-path'],
     )
-    common_list = [
+    common_list = (
       '--conversion_server_url=' + cloudooo_url,
       # BBB: We still have test suites that only accept the following 2 options.
       '--conversion_server_hostname=%s' % cloudooo_parsed.hostname,
@@ -76,19 +76,19 @@ class Recipe(GenericBaseRecipe):
       '--volatile_memcached_server_port=%s' % memcached_parsed.port,
       '--persistent_memcached_server_hostname=%s' % kumofs_parsed.hostname,
       '--persistent_memcached_server_port=%s' % kumofs_parsed.port,
-    ]
+    )
     path_list.append(self.createPythonScript(self.options['run-unit-test'],
-        __name__ + '.test.runUnitTest', [dict(
-        call_list=[self.options['run-unit-test-binary'],
+        __name__ + '.test.runUnitTest',
+        ((self.options['run-unit-test-binary'],
           '--erp5_sql_connection_string', mysql_connection_string,
           '--extra_sql_connection_string_list', ','.join(
             mysql_connection_string_list),
-          ] + common_list, **common_dict)]))
+          ) + common_list, common_dict)))
     path_list.append(self.createPythonScript(self.options['run-test-suite'],
-        __name__ + '.test.runTestSuite', [dict(
-        call_list=[self.options['run-test-suite-binary'],
+        __name__ + '.test.runTestSuite',
+        ((self.options['run-test-suite-binary'],
           '--db_list', ','.join(mysql_connection_string_list),
-          ] + common_list, **common_dict)]))
+          ) + common_list, common_dict)))
 
     return path_list
 
@@ -98,20 +98,18 @@ class CloudoooRecipe(GenericBaseRecipe):
     common_dict = dict(
         prepend_path=self.options['prepend-path'],
     )
-    common_list = [
+    common_list = (
            "--paster_path", self.options['ooo-paster'],
            self.options['configuration-file']
-          ]
-    run_unit_test_path = self.createPythonScript(self.options['run-unit-test'],
-        __name__ + '.test.runUnitTest', [dict(
-        call_list=[self.options['run-unit-test-binary'],
-          ] + common_list, **common_dict)])
-
-    path_list.append(run_unit_test_path)
+          )
+    path_list.append(self.createPythonScript(self.options['run-unit-test'],
+        __name__ + '.test.runUnitTest',
+        ((self.options['run-unit-test-binary'],
+          ) + common_list, common_dict)))
     path_list.append(self.createPythonScript(self.options['run-test-suite'],
-        __name__ + '.test.runTestSuite', [dict(
-        call_list=[self.options['run-test-suite-binary'],
-          ], **common_dict)]))
+        __name__ + '.test.runTestSuite',
+        ((self.options['run-test-suite-binary'],
+          ), common_dict)))
 
     return path_list
 
@@ -121,32 +119,20 @@ class EggTestRecipe(GenericBaseRecipe):
   off a list of Python eggs.
   """
   def install(self):
-    path_list = []
     test_list = self.options['test-list'].strip().replace('\n', ',')
-    common_dict = {}
 
-    environment_dict = {}
+    common_dict = {}
     if self.options.get('environment'):
       environment_part = self.buildout.get(self.options['environment'])
       if environment_part:
-        for key, value in environment_part.iteritems():
-          environment_dict[key] = value
-
-    common_list = [ "--source_code_path_list", test_list]
+        common_dict['environment'] = dict(environment_part)
 
-    argument_dict = dict(
-        call_list=[self.options['run-test-suite-binary'],] + common_list,
-        environment=environment_dict,
-        **common_dict
-    )
     if 'prepend-path' in self.options:
-      argument_dict['prepend_path'] = self.options['prepend-path']
+      common_dict['prepend_path'] = self.options['prepend-path']
 
-    run_test_suite_script = self.createPythonScript(
+    return self.createPythonScript(
         self.options['run-test-suite'], __name__ + '.test.runTestSuite',
-        [argument_dict]
+        ((self.options['run-test-suite-binary'],
+          "--source_code_path_list", test_list),
+         common_dict)
     )
-
-    path_list.append(run_test_suite_script)
-
-    return path_list

slapos/recipe/erp5_test/test.py

@@ -26,9 +26,9 @@
 ##############################################################################
 import os
 import sys
-def runTestSuite(args):
+
+def runTestSuite(args, d):
   env = os.environ.copy()
-  d = args[0]
   if 'openssl_binary' in d:
     env['OPENSSL_BINARY'] = d['openssl_binary']
   if 'test_ca_path' in d:
@@ -47,17 +47,15 @@ def runTestSuite(args):
     env.update(d['environment'])
 
   # Deal with Shebang size limitation
-  executable_filepath = d['call_list'][0]
-  file_object = open(executable_filepath, 'r')
-  line = file_object.readline()
-  file_object.close()
+  executable_filepath = args[0]
+  with open(executable_filepath, 'r') as f:
+    line = f.readline()
   argument_list = []
   if line[:2] == '#!':
     executable_filepath = line[2:].strip()
     argument_list.append(executable_filepath)
-  argument_list.extend(d['call_list'])
-  argument_list.extend(sys.argv[1:])
-  argument_list.append(env)
-  os.execle(executable_filepath, *argument_list)
+  argument_list += args
+  argument_list += sys.argv[1:]
+  os.execve(executable_filepath, argument_list, env)
 
 runUnitTest = runTestSuite

slapos/recipe/erp5testnode/__init__.py

@@ -68,17 +68,9 @@ class Recipe(GenericBaseRecipe):
     )
     self.path_list.append(configuration_file)
     self.path_list.append(
-      self.createPythonScript(
-        self.options['wrapper'],
-        'slapos.recipe.librecipe.execute.executee',
-        [ # Executable
-          [ self.options['testnode'], '-l', self.options['log-file'],
-            configuration_file],
-          # Environment
-          {
-            'GIT_SSL_NO_VERIFY': '1',
-          }
-        ],
+      self.createWrapper(self.options['wrapper'],
+          ( self.options['testnode'], '-l', self.options['log-file'],
+            configuration_file)
       )
     )
     self.installApache()
@@ -106,9 +98,8 @@ class Recipe(GenericBaseRecipe):
                                apache_config)
     )
     self.path_list.append(config_file)
-    wrapper = self.createPythonScript(self.options['httpd-wrapper'],
-      'slapos.recipe.librecipe.execute.execute',
-      [self.options['apache-binary'], '-f', config_file, '-DFOREGROUND'])
+    wrapper = self.createWrapper(self.options['httpd-wrapper'],
+      (self.options['apache-binary'], '-f', config_file, '-DFOREGROUND'))
     self.path_list.append(wrapper)
     # create empty html page to not allow listing of /
     page = open(os.path.join(self.options['log-directory'], "index.html"), "w")

slapos/recipe/generic_cloudooo/__init__.py

@@ -76,6 +76,12 @@ default_mimetype_entry_list = [
   "application/x-asc-text application/vnd.openxmlformats-officedocument.wordprocessingml.document x2t",
   "application/x-asc-spreadsheet application/vnd.openxmlformats-officedocument.spreadsheetml.sheet x2t",
   "application/x-asc-presentation application/vnd.openxmlformats-officedocument.presentationml.presentation x2t",
+  "application/vnd.oasis.opendocument.text application/x-asc-text x2t",
+  "application/vnd.oasis.opendocument.spreadsheet application/x-asc-spreadsheet x2t",
+  "application/vnd.oasis.opendocument.presentation application/x-asc-presentation x2t",
+  "application/x-asc-text application/vnd.oasis.opendocument.text x2t",
+  "application/x-asc-spreadsheet application/vnd.oasis.opendocument.spreadsheet x2t",
+  "application/x-asc-presentation application/vnd.oasis.opendocument.presentation x2t",
 ]
 
 class Recipe(GenericBaseRecipe):
@@ -118,5 +124,5 @@ class Recipe(GenericBaseRecipe):
     path_list.append(config_file)
     path_list.append(self.createPythonScript(self.options['wrapper'],
      'slapos.recipe.librecipe.execute.execute_with_signal_translation',
-      [self.options['ooo-paster'].strip(), 'serve', config_file]))
+      ((self.options['ooo-paster'].strip(), 'serve', config_file),)))
     return path_list

slapos/recipe/generic_mysql/__init__.py

@@ -154,7 +154,7 @@ class Recipe(GenericBaseRecipe):
        )]
     )
     path_list.append(mysqld)
-    environment = dict(PATH='%s' % self.options['bin-directory'])
+    environment = {'PATH': self.options['bin-directory']}
     # TODO: move to a separate recipe (ack'ed by Cedric)
     if 'backup-script' in self.options:
       # backup configuration
@@ -165,9 +165,13 @@ class Recipe(GenericBaseRecipe):
           '--defaults-file=%s' % mysql_conf_file,
           '--socket=%s' % socket.strip(), '--user=root',
           '--ibbackup=%s'% self.options['xtrabackup-binary']]
-      innobackupex_incremental = self.createPythonScript(self.options['innobackupex-incremental'], 'slapos.recipe.librecipe.execute.executee', [innobackupex_argument_list + ['--incremental'], environment])
+      innobackupex_incremental = self.createWrapper(
+        self.options['innobackupex-incremental'],
+        innobackupex_argument_list + ['--incremental'], environment)
       path_list.append(innobackupex_incremental)
-      innobackupex_full = self.createPythonScript(self.options['innobackupex-full'], 'slapos.recipe.librecipe.execute.executee', [innobackupex_argument_list, environment])
+      innobackupex_full = self.createWrapper(
+        self.options['innobackupex-full'],
+        innobackupex_argument_list, environment)
       path_list.append(innobackupex_full)
       backup_controller = self.createPythonScript(self.options['backup-script'], __name__ + '.innobackupex.controller', [innobackupex_incremental, innobackupex_full, full_backup, incremental_backup])
       path_list.append(backup_controller)
@@ -215,7 +219,9 @@ class Recipe(GenericBaseRecipe):
           '--defaults-file=%s' % mysql_conf_file,
           '--socket=%s' % socket.strip(), '--user=root',
           ]
-      pt_exe = self.createPythonScript(os.path.join(self.options['bin-directory'], pt_script_name), 'slapos.recipe.librecipe.execute.executee', [pt_argument_list, environment])
+      pt_exe = self.createWrapper(
+        os.path.join(self.options['bin-directory'], pt_script_name),
+        pt_argument_list, environment)
       path_list.append(pt_exe)
 
     return path_list

slapos/recipe/generic_mysql/innobackupex.py

 import os
 import glob
-def controller(args):
+def controller(innobackupex_incremental, innobackupex_full,
+               full_backup, incremental_backup):
   """Creates full or incremental backup
 
   If no full backup is done, it is created
@@ -9,8 +10,6 @@ def controller(args):
 
   base is the newest (according to date) full or incremental backup
   """
-  innobackupex_incremental, innobackupex_full, full_backup, incremental_backup \
-      = args
   if len(os.listdir(full_backup)) == 0:
     print 'Doing full backup in %r' % full_backup
     os.execv(innobackupex_full, [innobackupex_full, full_backup])

slapos/recipe/generic_mysql/mysql.py

@@ -5,9 +5,8 @@ import sys
 import pytz
 
 
-def runMysql(args):
+def runMysql(conf):
   sleep = 60
-  conf = args[0]
   mysqld_wrapper_list = [conf['mysqld_binary'], '--defaults-file=%s' %
       conf['configuration_file']]
   # we trust mysql_install that if mysql directory is available mysql was
@@ -54,8 +53,7 @@ def runMysql(args):
   os.execl(mysqld_wrapper_list[0], *mysqld_wrapper_list)
 
 
-def updateMysql(args):
-  conf = args[0]
+def updateMysql(conf):
   sleep = 30
   is_succeed = False
   try:

slapos/recipe/generic_zope/__init__.py

-##############################################################################
-#
-# Copyright (c) 2011 Vifib SARL and Contributors. All Rights Reserved.
-#
-# WARNING: This program as such is intended to be used by professional
-# programmers who take the whole responsibility of assessing all potential
-# consequences resulting from its eventual inadequacies and bugs
-# End users who are looking for a ready-to-use solution with commercial
-# guarantees and support are strongly adviced to contract a Free Software
-# Service Company
-#
-# This program is Free Software; you can redistribute it and/or
-# modify it under the terms of the GNU General Public License
-# as published by the Free Software Foundation; either version 3
-# of the License, or (at your option) any later version.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with this program; if not, write to the Free Software
-# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA  02111-1307, USA.
-#
-##############################################################################
-from slapos.recipe.librecipe import GenericBaseRecipe
-import binascii
-import hashlib
-import os
-import re
-import zc.buildout
-
-_isurl = re.compile('([a-zA-Z0-9+.-]+)://').match
-
-# based on Zope2.utilities.mkzopeinstance.write_inituser
-def Zope2InitUser(path, username, password):
-  # Set password only once
-  # Currently, rely on existence of a simple file:
-  # Create it the first time, then next time, detect this file and do no-op.
-  inituser_done_path = '%s_done' % path
-  if os.path.exists(inituser_done_path):
-    return
-
-  if os.path.exists(path):
-    return
-  open(path, 'w').write('')
-  os.chmod(path, 0600)
-  open(path, 'w').write('%s:{SHA}%s\n' % (
-    username,binascii.b2a_base64(hashlib.sha1(password).digest())[:-1]))
-
-  open(inituser_done_path, 'w').write('"inituser" file already created once.')
-
-class Recipe(GenericBaseRecipe):
-  def _options(self, options):
-    options['password'] = self.generatePassword()
-    options['deadlock-password'] = self.generatePassword()
-
-  def install(self):
-    """
-    All zope have to share file created by portal_classes
-    (until everything is integrated into the ZODB).
-    So, do not request zope instance and create multiple in the same partition.
-    """
-    path_list = []
-    Zope2InitUser(self.options['inituser'], self.options['user'],
-      self.options['password'])
-
-    # Symlink to BT5 repositories defined in instance config.
-    # Those paths will eventually end up in the ZODB, and having symlinks
-    # inside the XXX makes it possible to reuse such ZODB with another software
-    # release[ version].
-    # Note: this path cannot be used for development, it's really just a
-    # read-only repository.
-    repository_path = self.options['bt5-repository']
-
-    self.bt5_repository_list = []
-    append = self.bt5_repository_list.append
-    for repository in self.options.get('bt5-repository-list', '').split():
-      repository = repository.strip()
-      if not repository:
-        continue
-
-      if _isurl(repository) and not repository.startswith("file://"):
-        # XXX: assume it's a valid URL
-        append(repository)
-        continue
-
-      if repository.startswith('file://'):
-        repository = repository.replace('file://', '', '')
-
-      if os.path.isabs(repository):
-        repo_id = hashlib.sha1(repository).hexdigest()
-        link = os.path.join(repository_path, repo_id)
-        if os.path.lexists(link):
-          if not os.path.islink(link):
-            raise zc.buildout.UserError(
-              'Target link already %r exists but it is not link' % link)
-          os.unlink(link)
-        os.symlink(repository, link)
-        self.logger.debug('Created link %r -> %r' % (link, repository_path))
-        # Always provide a URL-Type
-        append("file://" + link)
-
-    # Create zope configuration file
-    zope_config = dict(
-        thread_amount=self.options['thread-amount'],
-        zodb_root_path=self.options['zodb-path'],
-        zodb_cache_size=int(self.options['zodb-cache-size']),
-    )
-    zope_environment = dict(
-      TMP=self.options['tmp-path'],
-      TMPDIR=self.options['tmp-path'],
-      HOME=self.options['tmp-path'],
-      PATH=self.options['bin-path']
-    )
-    # configure default Zope2 zcml
-    open(self.options['site-zcml'], 'w').write(open(self.getTemplateFilename(
-        'site.zcml')).read())
-    zope_config['instance'] = self.options['instance-path']
-    zope_config['event_log'] = self.options['event-log']
-    zope_config['z2_log'] = self.options['z2-log']
-    zope_config['pid-filename'] = self.options['pid-file']
-    zope_config['lock-filename'] = self.options['lock-file']
-    zope_config['products'] = 'products %s' % self.options['instance-products']
-    zope_config['address'] = '%s:%s' % (self.options['ip'], self.options['port'])
-    zope_config.update(dump_url=self.options['deadlock-path'],
-      secret=self.options['deadlock-password'])
-
-    zope_wrapper_template_location = self.getTemplateFilename('zope.conf.in')
-    zope_conf_content = self.substituteTemplate(zope_wrapper_template_location,
-      zope_config)
-
-    if ('promise-path' in self.options) and ('site-id' in self.options):
-      zope_conf_content += self.substituteTemplate(self.getTemplateFilename(
-          'zope.conf.promise.in'), {
-            'site-id': self.options['site-id'],
-            'promise-path': self.options['promise-path'],
-            })
-
-    zope_conf_path = self.createFile(self.options['configuration-file'], zope_conf_content)
-    path_list.append(zope_conf_path)
-    # Create init script
-    path_list.append(self.createPythonScript(self.options['wrapper'], 'slapos.recipe.librecipe.execute.executee', [[self.options['runzope-binary'].strip(), '-C', zope_conf_path], zope_environment]))
-    return path_list

slapos/recipe/generic_zope/template/site.zcml

-<configure
-    xmlns="http://namespaces.zope.org/zope"
-    xmlns:meta="http://namespaces.zope.org/meta"
-    xmlns:five="http://namespaces.zope.org/five">
-
-  <include package="Products.Five" />
-  <meta:redefinePermission from="zope2.Public" to="zope.Public" />
-
-
-  <!-- Load the meta -->
-  <include files="package-includes/*-meta.zcml" />
-  <five:loadProducts file="meta.zcml"/>
-
-  <!-- Load the configuration -->
-  <include files="package-includes/*-configure.zcml" />
-  <five:loadProducts />
-
-  <!-- Load the configuration overrides-->
-  <includeOverrides files="package-includes/*-overrides.zcml" />
-  <five:loadProductsOverrides />
-
-
-  <securityPolicy
-      component="Products.Five.security.FiveSecurityPolicy" />
-
-</configure>

slapos/recipe/generic_zope/template/zope.conf.in

-## Zope 2 configuration file generated by SlapOS
-
-# Some defines
-%%define INSTANCE %(instance)s
-instancehome $INSTANCE
-
-# Used products
-%(products)s
-
-# Environment is setup in running wrapper script
-# Reason: zope.conf is read too late for some componets
-
-# No need to debug
-debug-mode off
-
-# One thread is safe enough
-zserver-threads %(thread_amount)s
-
-# File location
-pid-filename %(pid-filename)s
-lock-filename %(lock-filename)s
-
-# Encoding
-rest-input-encoding utf-8
-rest-output-encoding utf-8
-default-zpublisher-encoding utf-8
-
-# Disable ownership checking to execute codes generated by alarm
-skip-ownership-checking on
-
-# Temporary storage database (for sessions)
-<zodb_db temporary>
-     <temporarystorage>
-       name temporary storage for sessioning
-     </temporarystorage>
-     mount-point /temp_folder
-     container-class Products.TemporaryFolder.TemporaryContainer
-</zodb_db>
-
-# Logging configuration
-<eventlog>
-  <logfile>
-    dateformat
-    path %(event_log)s
-  </logfile>
-</eventlog>
-<logger access>
-  <logfile>
-    dateformat
-    path %(z2_log)s
-  </logfile>
-</logger>
-
-# Serving configuration
-<http-server>
-  address %(address)s
-</http-server>
-
-# ZODB configuration
-<zodb_db root>
-    cache-size %(zodb_cache_size)d
-    <filestorage>
-      path %(zodb_root_path)s
-    </filestorage>
-    mount-point /
-</zodb_db>
-<zoperunner>
-    program $INSTANCE/bin/runzope
-</zoperunner>
-
-# DeadlockDebugger configuration
-<product-config DeadlockDebugger>
-  dump_url %(dump_url)s
-  secret %(secret)s
-</product-config>
-
-# ERP5 Timer Service
-%%import timerserver
-<timer-server>
-  interval 5
-</timer-server>

slapos/recipe/generic_zope/template/zope.conf.promise.in

-
-# ERP5 promise
-<product-config /%(site-id)s>
-  promise_path %(promise-path)s
-</product-config>

slapos/recipe/generic_zope_zeo_client/__init__.py

-##############################################################################
-#
-# Copyright (c) 2011 Vifib SARL and Contributors. All Rights Reserved.
-#
-# WARNING: This program as such is intended to be used by professional
-# programmers who take the whole responsibility of assessing all potential
-# consequences resulting from its eventual inadequacies and bugs
-# End users who are looking for a ready-to-use solution with commercial
-# guarantees and support are strongly adviced to contract a Free Software
-# Service Company
-#
-# This program is Free Software; you can redistribute it and/or
-# modify it under the terms of the GNU General Public License
-# as published by the Free Software Foundation; either version 3
-# of the License, or (at your option) any later version.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with this program; if not, write to the Free Software
-# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA  02111-1307, USA.
-#
-##############################################################################
-from slapos.recipe.librecipe import GenericBaseRecipe
-import binascii
-import hashlib
-import os
-import re
-import zc.buildout
-
-_isurl = re.compile('([a-zA-Z0-9+.-]+)://').match
-
-# based on Zope2.utilities.mkzopeinstance.write_inituser
-def Zope2InitUser(path, username, password):
-  # Set password only once
-  # Currently, rely on existence of a simple file:
-  # Create it the first time, then next time, detect this file and do no-op.
-  inituser_done_path = '%s_done' % path
-  if os.path.exists(inituser_done_path):
-    return
-
-  if os.path.exists(path):
-    return
-  open(path, 'w').write('')
-  os.chmod(path, 0600)
-  open(path, 'w').write('%s:{SHA}%s\n' % (
-    username,binascii.b2a_base64(hashlib.sha1(password).digest())[:-1]))
-
-  open(inituser_done_path, 'w').write('"inituser" file already created once.')
-
-class Recipe(GenericBaseRecipe):
-  def _options(self, options):
-    if 'password' not in options:
-      options['password'] = self.generatePassword()
-
-  def install(self):
-    """
-    All zope have to share file created by portal_classes
-    (until everything is integrated into the ZODB).
-    So, do not request zope instance and create multiple in the same partition.
-    """
-    path_list = []
-    Zope2InitUser(self.options['inituser'], self.options['user'],
-      self.options['password'])
-
-    # Symlink to BT5 repositories defined in instance config.
-    # Those paths will eventually end up in the ZODB, and having symlinks
-    # inside the XXX makes it possible to reuse such ZODB with another software
-    # release[ version].
-    # Note: this path cannot be used for development, it's really just a
-    # read-only repository.
-    repository_path = self.options['bt5-repository']
-
-    self.bt5_repository_list = []
-    append = self.bt5_repository_list.append
-    for repository in self.options.get('bt5-repository-list', '').split():
-      repository = repository.strip()
-      if not repository:
-        continue
-
-      if _isurl(repository) and not repository.startswith("file://"):
-        # XXX: assume it's a valid URL
-        append(repository)
-        continue
-
-      if repository.startswith('file://'):
-        repository = repository.replace('file://', '', '')
-
-      if os.path.isabs(repository):
-        repo_id = hashlib.sha1(repository).hexdigest()
-        link = os.path.join(repository_path, repo_id)
-        if os.path.lexists(link):
-          if not os.path.islink(link):
-            raise zc.buildout.UserError(
-              'Target link already %r exists but it is not link' % link)
-          os.unlink(link)
-        os.symlink(repository, link)
-        self.logger.debug('Created link %r -> %r' % (link, repository_path))
-        # Always provide a URL-Type
-        append("file://" + link)
-
-    zope_environment = {
-      'TMP': self.options['tmp-path'],
-      'TMPDIR': self.options['tmp-path'],
-      'HOME': self.options.get('home-path', self.options.get('tmp-path')),
-      'PATH': self.options['bin-path'],
-      'TZ': self.options['timezone'],
-    }
-    instance_home = self.options.get("instancehome-path", None)
-    if instance_home:
-      zope_environment["INSTANCE_HOME"] = instance_home
-
-    # configure default Zope2 zcml
-    open(self.options['site-zcml'], 'w').write(open(self.getTemplateFilename(
-        'site.zcml')).read())
-
-    # Create init script
-    path_list.append(self.createPythonScript(self.options['wrapper'], 'slapos.recipe.librecipe.execute.executee', [[self.options['runzope-binary'].strip(), '-C', self.options['configuration-file']], zope_environment]))
-    return path_list

slapos/recipe/generic_zope_zeo_client/template/site.zcml

-<configure
-    xmlns="http://namespaces.zope.org/zope"
-    xmlns:meta="http://namespaces.zope.org/meta"
-    xmlns:five="http://namespaces.zope.org/five">
-
-  <include package="Products.Five" />
-  <meta:redefinePermission from="zope2.Public" to="zope.Public" />
-
-
-  <!-- Load the meta -->
-  <include files="package-includes/*-meta.zcml" />
-  <five:loadProducts file="meta.zcml"/>
-
-  <!-- Load the configuration -->
-  <include files="package-includes/*-configure.zcml" />
-  <five:loadProducts />
-
-  <!-- Load the configuration overrides-->
-  <includeOverrides files="package-includes/*-overrides.zcml" />
-  <five:loadProductsOverrides />
-
-
-  <securityPolicy
-      component="Products.Five.security.FiveSecurityPolicy" />
-
-</configure>

slapos/recipe/haproxy/__init__.py

@@ -120,12 +120,11 @@ class Recipe(GenericBaseRecipe):
          'server_text': server_snippet},
       )
     )
-    wrapper_path = self.createPythonScript(
+    wrapper_path = self.createWrapper(
       self.options['wrapper-path'],
-      'slapos.recipe.librecipe.execute.execute',
-      arguments=[self.options['binary-path'].strip(), '-f', configuration_path],)
+      (self.options['binary-path'].strip(), '-f', configuration_path))
     ctl_path = self.createPythonScript(
       self.options['ctl-path'],
-      '%s.haproxy.haproxyctl' % __name__,
-      {'socket_path':self.options['socket-path']})
+      __name__ + '.haproxy.haproxyctl',
+      (self.options['socket-path'],))
     return [configuration_path, wrapper_path, ctl_path]

slapos/recipe/haproxy/haproxy.py

@@ -4,7 +4,7 @@ try:
 except ImportError:
   pass
 
-def haproxyctl(conf):
+def haproxyctl(socket_path):
   while True:
     try:
       l = raw_input('> ')
@@ -14,7 +14,7 @@ def haproxyctl(conf):
     if l == 'quit':
       break
     s = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM)
-    s.connect(conf['socket_path'])
+    s.connect(socket_path)
     s.send('%s\n' % l)
     while True:
       r = s.recv(1024)

slapos/recipe/kumofs/certificate_authority.py

-import os
-import subprocess
-import time
-import ConfigParser
-
-
-def popenCommunicate(command_list, input=None):
-  subprocess_kw = dict(stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
-  if input is not None:
-    subprocess_kw.update(stdin=subprocess.PIPE)
-  popen = subprocess.Popen(command_list, **subprocess_kw)
-  result = popen.communicate(input)[0]
-  if popen.returncode is None:
-    popen.kill()
-  if popen.returncode != 0:
-    raise ValueError('Issue during calling %r, result was:\n%s' % (
-      command_list, result))
-  return result
-
-
-class CertificateAuthority:
-  def __init__(self, key, certificate, openssl_binary,
-      openssl_configuration, request_dir):
-    self.key = key
-    self.certificate = certificate
-    self.openssl_binary = openssl_binary
-    self.openssl_configuration = openssl_configuration
-    self.request_dir = request_dir
-
-  def checkAuthority(self):
-    file_list = [ self.key, self.certificate ]
-    ca_ready = True
-    for f in file_list:
-      if not os.path.exists(f):
-        ca_ready = False
-        break
-    if ca_ready:
-      return
-    for f in file_list:
-      if os.path.exists(f):
-        os.unlink(f)
-    try:
-      # no CA, let us create new one
-      popenCommunicate([self.openssl_binary, 'req', '-nodes', '-config',
-          self.openssl_configuration, '-new', '-x509', '-extensions',
-          'v3_ca', '-keyout', self.key, '-out', self.certificate,
-          '-days', '10950'], 'Automatic Certificate Authority\n')
-    except:
-      try:
-        for f in file_list:
-          if os.path.exists(f):
-            os.unlink(f)
-      except:
-        # do not raise during cleanup
-        pass
-      raise
-
-  def _checkCertificate(self, common_name, key, certificate):
-    file_list = [key, certificate]
-    ready = True
-    for f in file_list:
-      if not os.path.exists(f):
-        ready = False
-        break
-    if ready:
-      return False
-    for f in file_list:
-      if os.path.exists(f):
-        os.unlink(f)
-    csr = certificate + '.csr'
-    try:
-      popenCommunicate([self.openssl_binary, 'req', '-config',
-        self.openssl_configuration, '-nodes', '-new', '-keyout',
-        key, '-out', csr, '-days', '3650'],
-        common_name + '\n')
-      try:
-        popenCommunicate([self.openssl_binary, 'ca', '-batch', '-config',
-          self.openssl_configuration, '-out', certificate,
-          '-infiles', csr])
-      finally:
-        if os.path.exists(csr):
-          os.unlink(csr)
-    except:
-      try:
-        for f in file_list:
-          if os.path.exists(f):
-            os.unlink(f)
-      except:
-        # do not raise during cleanup
-        pass
-      raise
-    else:
-      return True
-
-  def checkRequestDir(self):
-    for request_file in os.listdir(self.request_dir):
-      parser = ConfigParser.RawConfigParser()
-      parser.readfp(open(os.path.join(self.request_dir, request_file), 'r'))
-      if self._checkCertificate(parser.get('certificate', 'name'),
-          parser.get('certificate', 'key_file'), parser.get('certificate',
-            'certificate_file')):
-        print 'Created certificate %r' % parser.get('certificate', 'name')
-
-def runCertificateAuthority(args):
-  ca_conf = args[0]
-  ca = CertificateAuthority(ca_conf['key'], ca_conf['certificate'],
-      ca_conf['openssl_binary'], ca_conf['openssl_configuration'],
-      ca_conf['request_dir'])
-  while True:
-    ca.checkAuthority()
-    ca.checkRequestDir()
-    time.sleep(60)

slapos/recipe/kumofs/template/kumo_gateway.in

-#!/bin/sh
-exec %(kumo_gateway_binary)s -F -E -m %(kumo_manager_ip)s:%(kumo_manager_port)s -t %(kumo_gateway_ip)s:%(kumo_gateway_port)s -o %(kumo_gateway_log)s

slapos/recipe/kumofs/template/kumo_manager.in

-#!/bin/sh
-exec %(kumo_manager_binary)s -a -l %(kumo_manager_ip)s:%(kumo_manager_port)s -o %(kumo_manager_log)s

slapos/recipe/kumofs/template/kumo_server.in

-#!/bin/sh
-exec %(kumo_server_binary)s -l %(kumo_server_ip)s:%(kumo_server_port)s -L %(kumo_server_listen_port)s -m %(kumo_manager_ip)s:%(kumo_manager_port)s -s %(kumo_server_storage)s -o %(kumo_server_log)s

slapos/recipe/kumofs/template/stunnel.conf.in

-foreground = yes
-output = %(log)s
-pid = %(pid_file)s
-syslog = no
-CApath = %(ca_path)s
-key = %(key)s
-CRLpath = %(ca_crl)s
-cert = %(cert)s
-
-[service]
-accept = %(public_ip)s:%(public_port)s
-connect = %(private_ip)s:%(private_port)s

slapos/recipe/lamp/__init__.py

@@ -298,9 +298,9 @@ class Request(BaseRecipe):
         'local_host': local_host, 'local_port': local_port,
       }))
     wrapper = zc.buildout.easy_install.scripts([('stunnel',
-      'slapos.recipe.librecipe.execute', 'execute')], self.ws,
-      sys.executable, self.wrapper_directory, arguments=[
-        self.options['stunnel_binary'].strip(), stunnel_conf_path]
+      'slapos.recipe.librecipe.execute', 'generic_exec')], self.ws,
+      sys.executable, self.wrapper_directory, arguments='%r, %r'
+        % (self.options['stunnel_binary'].strip(), stunnel_conf_path)
       )[0]
     self.path_list.append(wrapper)
     return (local_host, local_port,)

slapos/recipe/lampgeneric/__init__.py

@@ -71,13 +71,12 @@ class Recipe(GenericBaseRecipe):
     )
     path_list.append(httpd_conf)
 
-    wrapper = self.createWrapper(name=self.options['wrapper'],
-                                 command=self.options['httpd-binary'],
-                                 parameters=[
+    wrapper = self.createWrapper(self.options['wrapper'],
+                                 (self.options['httpd-binary'],
                                      '-f',
                                      self.options['httpd-conf'],
                                      '-DFOREGROUND'
-                                     ])
+                                     ))
     path_list.append(wrapper)
 
 

slapos/recipe/librecipe/execute.py

@@ -2,7 +2,6 @@ import sys
 import os
 import signal
 import subprocess
-import time
 from collections import defaultdict
 from inotify_simple import INotify, flags
 
@@ -29,65 +28,96 @@ def _wait_files_creation(file_list):
         if event.name in directory:
           directory[event.name] = event.mask & (flags.CREATE | flags.MOVED_TO)
 
-def execute(args):
-  """Portable execution with process replacement"""
-  # XXX: Kept for backward compatibility
-  generic_exec([args, None, None])
-
-def execute_wait(args):
-  """Execution but after all files in args[1] exists"""
-  # XXX: Kept for backward compatibility
-  generic_exec([args[0], args[1], None])
-
+def _libc():
+  from ctypes import CDLL, get_errno, c_char_p, c_int, c_ulong, util
+  libc = CDLL(util.find_library('c'), use_errno=True)
+  libc_mount = libc.mount
+  libc_mount.argtypes = c_char_p, c_char_p, c_char_p, c_ulong, c_char_p
+  def mount(source, target, filesystemtype, mountflags, data):
+    if libc_mount(source, target, filesystemtype, mountflags, data):
+      e = get_errno()
+      raise OSError(e, os.strerror(e))
+  libc_unshare = libc.unshare
+  libc_unshare.argtypes = c_int,
+  def unshare(flags):
+    if libc_unshare(flags):
+      e = get_errno()
+      raise OSError(e, os.strerror(e))
+  return mount, unshare
+
+def generic_exec(args, extra_environ=None, wait_list=None,
+                 pidfile=None, reserve_cpu=False, private_dev_shm=None,
+                 #shebang_workaround=False, # XXX: still needed ?
+                 ):
+  args = list(args)
+
+  if pidfile:
+    import psutil
+    try:
+      with open(pidfile) as f:
+        pid = int(f.read())
+      running = psutil.Process(pid).cmdline()
+    except Exception:
+      pass
+    else:
+      # With chained shebangs, several paths may be inserted at the beginning.
+      n = len(args)
+      for i in xrange(1+len(running)-n):
+        if args == running[i:n+i]:
+          sys.exit("Already running with pid %s." % pid)
+    with open(pidfile, 'w') as f:
+      f.write(str(os.getpid()))
+
+  args += sys.argv[1:]
+
+  if reserve_cpu:
+    # If the CGROUPS cpuset is available (and prepared by slap format),
+    # request an exclusive CPU core for this process.
+    with open(os.path.expanduser('~/.slapos-cpu-exclusive'), 'a') as f:
+      f.write('%s\n' % os.getpid())
+
+  if wait_list:
+    _wait_files_creation(wait_list)
+
+  if private_dev_shm:
+    mount, unshare = _libc()
+    CLONE_NEWNS   = 0x00020000
+    CLONE_NEWUSER = 0x10000000
+    uid = os.getuid()
+    gid = os.getgid()
+    unshare(CLONE_NEWUSER |CLONE_NEWNS)
+    with open('/proc/self/setgroups', 'wb') as f: f.write('deny')
+    with open('/proc/self/uid_map',   'wb') as f: f.write('%s %s 1' % (uid, uid))
+    with open('/proc/self/gid_map',   'wb') as f: f.write('%s %s 1' % (gid, gid))
+    mount('tmpfs', '/dev/shm', 'tmpfs', 0, 'size=' + private_dev_shm)
+
+  if extra_environ:
+    env = os.environ.copy()
+    env.update(extra_environ)
+    os.execve(args[0], args, env)
+  else:
+    os.execv(args[0], args)
 
 child_pg = None
 
-
-def executee(args):
-  """Portable execution with process replacement and environment manipulation"""
-  # XXX: Kept for backward compatibility
-  generic_exec([args[0], None, args[1]])
-
-def executee_wait(args):
-  """Portable execution with process replacement and environment manipulation"""
-  # XXX: Kept for backward compatibility
-  generic_exec(args)
-
-def generic_exec(args):
-  exec_list = list(args[0])
-  file_list = args[1]
-  environment_overriding = args[2]
-
-  exec_env = os.environ.copy()
-  if environment_overriding is not None:
-      exec_env.update(environment_overriding)
-
-  if file_list is not None:
-      _wait_files_creation(file_list)
-
-  os.execve(exec_list[0], exec_list + sys.argv[1:], exec_env)
-
 def sig_handler(sig, frame):
   print 'Received signal %r, killing children and exiting' % sig
   if child_pg is not None:
     os.killpg(child_pg, signal.SIGHUP)
     os.killpg(child_pg, signal.SIGTERM)
-  sys.exit(0)
-
-signal.signal(signal.SIGINT, sig_handler)
-signal.signal(signal.SIGQUIT, sig_handler)
-signal.signal(signal.SIGTERM, sig_handler)
-
+  sys.exit()
 
 def execute_with_signal_translation(args):
   """Run process as children and translate from SIGTERM to another signal"""
   global child_pg
+  signal.signal(signal.SIGINT, sig_handler)
+  signal.signal(signal.SIGQUIT, sig_handler)
+  signal.signal(signal.SIGTERM, sig_handler)
   child = subprocess.Popen(args, close_fds=True, preexec_fn=os.setsid)
   child_pg = child.pid
   try:
     print 'Process %r started' % args
-    while True:
-      time.sleep(10)
+    signal.pause()
   finally:
     os.killpg(child_pg, signal.SIGHUP)
     os.killpg(child_pg, signal.SIGTERM)

slapos/recipe/librecipe/generic.py

@@ -33,7 +33,6 @@ import sys
 import inspect
 import re
 import shutil
-from textwrap import dedent
 import urllib
 import urlparse
 
@@ -116,92 +115,60 @@ class GenericBaseRecipe(object):
       with io.open(filepath, 'w+', encoding=encoding) as f:
         f.write(u'\n'.join(lines))
 
-  def createPythonScript(self, name, absolute_function, arguments=''):
+  def createPythonScript(self, name, absolute_function, args=(), kw={}):
     """Create a python script using zc.buildout.easy_install.scripts
 
      * function should look like 'module.function', or only 'function'
        if it is a builtin function."""
-    absolute_function = tuple(absolute_function.rsplit('.', 1))
-    if len(absolute_function) == 1:
-      absolute_function = ('__builtin__',) + absolute_function
-    if len(absolute_function) != 2:
-      raise ValueError("A non valid function was given")
-
-    module, function = absolute_function
+    function = absolute_function.rsplit('.', 1)
+    if len(function) == 1:
+      module = '__builtin__'
+      function, = function
+    else:
+      module, function = function
     path, filename = os.path.split(os.path.abspath(name))
 
-    script = zc.buildout.easy_install.scripts(
+    assert not isinstance(args, (basestring, dict)), args
+    args = map(repr, args)
+    args += map('%s=%r'.__mod__, kw.iteritems())
+
+    return zc.buildout.easy_install.scripts(
       [(filename, module, function)], self._ws, sys.executable,
-      path, arguments=arguments)[0]
-    return script
+      path, arguments=', '.join(args))[0]
 
-  def createWrapper(self, name, command, parameters, comments=[],
-      parameters_extra=False, environment=None,
-      pidfile=None, reserve_cpu=False
-  ):
-    """
-    Creates a shell script for process replacement.
-    Takes care of quoting.
-    Takes care of #! line limitation when the wrapped command is a script.
-    if pidfile parameter is specified, then it will make the wrapper a singleton,
-    accepting to run only if no other instance is running.
+  def createWrapper(self, path, args, env=None, **kw):
+    """Create a wrapper script for process replacement"""
+    assert args
+    if kw:
+      return self.createPythonScript(path,
+        'slapos.recipe.librecipe.execute.generic_exec',
+        (args, env) if env else (args,), kw)
 
-    :param reserve_cpu: bool, try to reserve one core for the `command`
-    """
+    # Simple case: creates a basic shell script for process replacement.
+    # This must be kept minimal to avoid code duplication with generic_exec.
+    # In particular, do not implement workaround for shebang size limitation
+    # here (note that this can't be done correctly with a POSIX shell, because
+    # the process can't be given a name).
+
+    lines = ['#!/bin/sh']
+
+    if env:
+      for k, v in sorted(env.iteritems()):
+        lines.append('export %s=%s' % (k, shlex.quote(v)))
+
+    lines.append('exec')
 
-    lines = [ '#!/bin/sh' ]
-
-    if comments:
-      lines += '# ', '\n# '.join(comments), '\n'
-
-    lines.append('COMMAND=' + shlex.quote(command))
-
-    for key in environment or ():
-      lines.append('export %s=%s' % (key, environment[key]))
-
-    if pidfile:
-      lines.append(dedent("""
-          # Check for other instances
-          pidfile=%s
-          if [ -s $pidfile ]; then
-            if pid=`pgrep -F $pidfile -f "$COMMAND" 2>/dev/null`; then
-              echo "Already running with pid $pid."
-              exit 1
-            fi
-          fi
-          echo $$ > $pidfile""" % shlex.quote(pidfile)))
-
-    if reserve_cpu:
-      # if the CGROUPS cpuset is available (and prepared by slap format)
-      # request an exclusive CPU core for this process
-      lines.append(dedent("""
-        # put own PID into waiting list for exclusive CPU-core access
-        echo $$ >> ~/.slapos-cpu-exclusive
-        """))
-
-    lines.append(dedent('''
-    # If the wrapped command uses a shebang, execute the referenced
-    # executable passing the script path as first argument.
-    # This is to workaround the limitation of 127 characters in #!
-    [ ! -f "$COMMAND" ] || {
-      [ "`head -c2`" != "#!" ] || read -r EXE ARG
-    } < "$COMMAND"
-
-    exec $EXE ${ARG:+"$ARG"} "$COMMAND"'''))
-
-    parameters = map(shlex.quote, parameters)
-    if parameters_extra:
-      # pass-through further parameters
-      parameters.append('"$@"')
-    for param in parameters:
+    args = map(shlex.quote, args)
+    args.append('"$@"')
+    for arg in args:
       if len(lines[-1]) < 40:
-        lines[-1] += ' ' + param
+        lines[-1] += ' ' + arg
       else:
         lines[-1] += ' \\'
-        lines.append('\t' + param)
+        lines.append('\t' + arg)
 
     lines.append('')
-    return self.createFile(name, '\n'.join(lines), 0700)
+    return self.createFile(path, '\n'.join(lines), 0700)
 
   def createDirectory(self, parent, name, mode=0700):
     path = os.path.join(parent, name)

slapos/recipe/logrotate.py

@@ -46,10 +46,10 @@ class Recipe(GenericBaseRecipe):
 
     state_file = self.options['state-file']
 
-    logrotate = self.createPythonScript(
+    logrotate = self.createWrapper(
       self.options['wrapper'],
-      'slapos.recipe.librecipe.execute.execute',
-      [self.options['logrotate-binary'], '-s', state_file, logrotate_conf_file, ]
+      (self.options['logrotate-binary'],
+        '-s', state_file, logrotate_conf_file),
     )
 
     return [logrotate, logrotate_conf_file]

slapos/recipe/memcached/certificate_authority.py

-import os
-import subprocess
-import time
-import ConfigParser
-
-
-def popenCommunicate(command_list, input=None):
-  subprocess_kw = dict(stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
-  if input is not None:
-    subprocess_kw.update(stdin=subprocess.PIPE)
-  popen = subprocess.Popen(command_list, **subprocess_kw)
-  result = popen.communicate(input)[0]
-  if popen.returncode is None:
-    popen.kill()
-  if popen.returncode != 0:
-    raise ValueError('Issue during calling %r, result was:\n%s' % (
-      command_list, result))
-  return result
-
-
-class CertificateAuthority:
-  def __init__(self, key, certificate, openssl_binary,
-      openssl_configuration, request_dir):
-    self.key = key
-    self.certificate = certificate
-    self.openssl_binary = openssl_binary
-    self.openssl_configuration = openssl_configuration
-    self.request_dir = request_dir
-
-  def checkAuthority(self):
-    file_list = [ self.key, self.certificate ]
-    ca_ready = True
-    for f in file_list:
-      if not os.path.exists(f):
-        ca_ready = False
-        break
-    if ca_ready:
-      return
-    for f in file_list:
-      if os.path.exists(f):
-        os.unlink(f)
-    try:
-      # no CA, let us create new one
-      popenCommunicate([self.openssl_binary, 'req', '-nodes', '-config',
-          self.openssl_configuration, '-new', '-x509', '-extensions',
-          'v3_ca', '-keyout', self.key, '-out', self.certificate,
-          '-days', '10950'], 'Automatic Certificate Authority\n')
-    except:
-      try:
-        for f in file_list:
-          if os.path.exists(f):
-            os.unlink(f)
-      except:
-        # do not raise during cleanup
-        pass
-      raise
-
-  def _checkCertificate(self, common_name, key, certificate):
-    file_list = [key, certificate]
-    ready = True
-    for f in file_list:
-      if not os.path.exists(f):
-        ready = False
-        break
-    if ready:
-      return False
-    for f in file_list:
-      if os.path.exists(f):
-        os.unlink(f)
-    csr = certificate + '.csr'
-    try:
-      popenCommunicate([self.openssl_binary, 'req', '-config',
-        self.openssl_configuration, '-nodes', '-new', '-keyout',
-        key, '-out', csr, '-days', '3650'],
-        common_name + '\n')
-      try:
-        popenCommunicate([self.openssl_binary, 'ca', '-batch', '-config',
-          self.openssl_configuration, '-out', certificate,
-          '-infiles', csr])
-      finally:
-        if os.path.exists(csr):
-          os.unlink(csr)
-    except:
-      try:
-        for f in file_list:
-          if os.path.exists(f):
-            os.unlink(f)
-      except:
-        # do not raise during cleanup
-        pass
-      raise
-    else:
-      return True
-
-  def checkRequestDir(self):
-    for request_file in os.listdir(self.request_dir):
-      parser = ConfigParser.RawConfigParser()
-      parser.readfp(open(os.path.join(self.request_dir, request_file), 'r'))
-      if self._checkCertificate(parser.get('certificate', 'name'),
-          parser.get('certificate', 'key_file'), parser.get('certificate',
-            'certificate_file')):
-        print 'Created certificate %r' % parser.get('certificate', 'name')
-
-def runCertificateAuthority(args):
-  ca_conf = args[0]
-  ca = CertificateAuthority(ca_conf['key'], ca_conf['certificate'],
-      ca_conf['openssl_binary'], ca_conf['openssl_configuration'],
-      ca_conf['request_dir'])
-  while True:
-    ca.checkAuthority()
-    ca.checkRequestDir()
-    time.sleep(60)

slapos/recipe/memcached/template/memcached.in

-#!/bin/sh
-exec %(memcached_binary)s -p %(memcached_port)s -U %(memcached_port)s -l %(memcached_ip)s

slapos/recipe/memcached/template/stunnel.conf.in

-foreground = yes
-output = %(log)s
-pid = %(pid_file)s
-syslog = no
-CApath = %(ca_path)s
-key = %(key)s
-CRLpath = %(ca_crl)s
-cert = %(cert)s
-
-[service]
-accept = %(public_ip)s:%(public_port)s
-connect = %(private_ip)s:%(private_port)s

slapos/recipe/mioga/instantiate.py

@@ -203,11 +203,10 @@ Include conf/extra/httpd-autoindex.conf
 
     services_dir = self.options['services_dir']
 
-    httpd_wrapper = self.createPythonScript(
+    httpd_wrapper = self.createWrapper(
       os.path.join(services_dir, 'httpd_wrapper'),
-      'slapos.recipe.librecipe.execute.execute',
-      [self.options['httpd_binary'], '-f', self.options['httpd_conf'],
-       '-DFOREGROUND']
+      (self.options['httpd_binary'],
+        '-f', self.options['httpd_conf'], '-DFOREGROUND'),
     )
     path_list.append(httpd_wrapper)
 
@@ -220,19 +219,17 @@ Include conf/extra/httpd-autoindex.conf
 
     site_perl_bin = os.path.join(self.options['site_perl'], 'bin')
     mioga_conf_path = os.path.join(mioga_base, 'conf', 'Mioga.conf')
-    notifier_wrapper = self.createPythonScript(
+    notifier_wrapper = self.createWrapper(
       os.path.join(services_dir, 'notifier'),
-      'slapos.recipe.librecipe.execute.execute',
-      [ os.path.join(site_perl_bin, 'notifier.pl'),
-        mioga_conf_path ]
+      (os.path.join(site_perl_bin, 'notifier.pl'),
+        mioga_conf_path),
     )
     path_list.append(notifier_wrapper)
 
-    searchengine_wrapper = self.createPythonScript(
+    searchengine_wrapper = self.createWrapper(
       os.path.join(services_dir, 'searchengine'),
-      'slapos.recipe.librecipe.execute.execute',
-      [ os.path.join(site_perl_bin, 'searchengine.pl'),
-        mioga_conf_path ]
+      (os.path.join(site_perl_bin, 'searchengine.pl'),
+        mioga_conf_path),
     )
     path_list.append(searchengine_wrapper)
 

slapos/recipe/mydumper.py

@@ -24,7 +24,7 @@
 # Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA  02111-1307, USA.
 #
 ##############################################################################
-import subprocess
+import os
 
 from slapos.recipe.librecipe import GenericBaseRecipe
 
@@ -58,14 +58,14 @@ def do_export(args):
 
   cmd.extend(['-o', args['directory']])
 
-  subprocess.check_call(cmd)
+  os.execv(cmd[0], cmd)
 
 
 def do_import(args):
   cmd = _mydumper_base_cmd(**args)
   cmd.append('--overwrite-tables')
   cmd.extend(['-d', args['directory']])
-  subprocess.check_call(cmd)
+  os.execv(cmd[0], cmd)
 
 
 class Recipe(GenericBaseRecipe):
@@ -95,9 +95,5 @@ class Recipe(GenericBaseRecipe):
       config['compression'] = self.optionIsTrue('compression', default=False)
       config['rows'] = self.options.get('rows')
 
-    wrapper = self.createPythonScript(name=self.options['wrapper'],
-                                      absolute_function = '%s.%s' % (__name__, function.func_name),
-                                      arguments=config)
-
-    return [wrapper]
-
+    return self.createPythonScript(self.options['wrapper'],
+      '%s.%s' % (function.__module__, function.__name__), (config,))

slapos/recipe/neoppod.py

@@ -42,7 +42,8 @@ class NeoBaseRecipe(GenericBaseRecipe):
       # Only then can this recipe start succeeding and actually doing anything
       # useful, as per NEO deploying constraints.
       raise UserError('"masters" parameter is mandatory')
-    option_list = [
+    args = [
+      options['binary'],
       # Keep the -l option first, as expected by logrotate snippets.
       '-l', options['logfile'],
       '-m', options['masters'],
@@ -53,17 +54,13 @@ class NeoBaseRecipe(GenericBaseRecipe):
     ]
     if options['ssl']:
       etc = os.path.join(self.buildout['buildout']['directory'], 'etc', '')
-      option_list += (
+      args += (
         '--ca', etc + 'ca.crt',
         '--cert', etc + 'neo.crt',
         '--key', etc + 'neo.key',
         )
-    option_list.extend(self._getOptionList())
-    return [self.createWrapper(
-      options['wrapper'],
-      options['binary'],
-      option_list
-    )]
+    args += self._getOptionList()
+    return self.createWrapper(options['wrapper'], args)
 
   def _getBindingAddress(self):
     options = self.options

slapos/recipe/redis/promise.py

-#! /usr/bin/env python
-# -*- coding: utf-8 -*-
-
-import slapos.recipe.redis.MyRedis2410 as redis
-import sys
-
-def main(args):
-  host = args['host']
-  port = int(args['port'])
-  unixsocket = args['unixsocket']
-  try:
-    r = redis.Redis(host=host, port=port, unix_socket_path=unixsocket, db=0)
-    r.publish("Promise-Service","SlapOS Promise")
-    r.connection_pool.disconnect()
-    sys.exit(0)
-  except Exception, e:
-    print str(e)
-    sys.exit(1)
\ No newline at end of file