Merge branch 'master' into 1.0

Conflicts:
	stack/slapos.cfg

CHANGES.rst

 Changes
 =======
 
+1.0.45 (2017-01-09)
+-------------------
+
+ * recipe: set default timeout of check url promise to 20 seconds
+
 1.0.44 (2016-12-30)
 -------------------
 

setup.py

@@ -28,7 +28,7 @@ from setuptools import setup, find_packages
 import glob
 import os
 
-version = '1.0.44'
+version = '1.0.45'
 name = 'slapos.cookbook'
 long_description = open("README.rst").read() + "\n" + \
     open("CHANGES.rst").read() + "\n"

slapos/recipe/check_url_available/__init__.py

@@ -26,6 +26,7 @@
 ##############################################################################
 from slapos.recipe.librecipe import GenericBaseRecipe
 import sys
+import os
 
 class Recipe(GenericBaseRecipe):
   """
@@ -33,12 +34,14 @@ class Recipe(GenericBaseRecipe):
   """
 
   def install(self):
+    timeout_file = os.path.join(os.getcwd(), 'etc/promise_timeout')
     config = {
       'url': self.options['url'],
       'shell_path': self.options['dash_path'],
       'curl_path': self.options['curl_path'],
       'check_secure': self.options.get('check-secure', 0),
       'http_code': self.options.get('http_code', '200'),
+      'time_out': self.options.get('timeout-file-path', timeout_file),
     }
 
     # XXX-Cedric in this script, curl won't check certificate

slapos/recipe/check_url_available/template/check_url.in

@@ -3,13 +3,24 @@
 # BEWARE: It will be overwritten automatically
 
 URL="%(url)s"
+F_TIMEOUT="%(time_out)s"
+TIMEOUT=20
 
 if [ -z "$URL" ]; then
   echo "No URL specified." >&2
   exit 3
 fi
 
-CODE=$(%(curl_path)s -g -k -sL $URL -w %%{http_code} --max-time 10 -o /dev/null)
+if [ -s "$F_TIMEOUT" ]; then
+  TIMEOUT=$(cat $F_TIMEOUT)
+  result=$(echo $TIMEOUT | grep -E ^[0-9]+$)
+  if [ -z $result ]; then
+    # Not an integer
+    TIMEOUT=20
+  fi
+fi
+
+CODE=$(%(curl_path)s -g -k -sL $URL -w %%{http_code} --max-time $TIMEOUT -o /dev/null)
 
 if [ $? -eq 3 ]; then
   echo "URL malformed: $URL." >&2

software/agent/software.cfg

@@ -51,29 +51,29 @@ pycrypto = 2.6.1
 pycurl = 7.43.0
 slapos.recipe.download = 1.0
 slapos.recipe.template = 2.8
-slapos.toolbox = 0.62
+slapos.toolbox = 0.63
 smmap = 0.9.0
 
 # Required by:
-# slapos.toolbox = 0.62
+# slapos.toolbox = 0.63
 GitPython = 2.0.8
 
 # Required by:
-# slapos.toolbox = 0.62
+# slapos.toolbox = 0.63
 atomize = 0.2.0
 
 # Required by:
-# slapos.toolbox = 0.62
+# slapos.toolbox = 0.63
 feedparser = 5.2.1
 
 # Required by:
-# slapos.toolbox = 0.62
+# slapos.toolbox = 0.63
 lockfile = 0.12.2
 
 # Required by:
-# slapos.toolbox = 0.62
+# slapos.toolbox = 0.63
 paramiko = 2.0.1
 
 # Required by:
-# slapos.toolbox = 0.62
+# slapos.toolbox = 0.63
 rpdb = 0.1.5

software/apache-frontend/software.cfg

@@ -15,7 +15,7 @@ plone.recipe.command = 1.1
 pycrypto = 2.6.1
 rdiff-backup = 1.0.5
 slapos.recipe.template = 2.8
-slapos.toolbox = 0.62
+slapos.toolbox = 0.63
 smmap = 0.9.0
 numpy = 1.11.2
 
@@ -34,29 +34,29 @@ erp5.util = 0.4.46
 passlib = 1.6.5
 
 # Required by:
-# slapos.toolbox = 0.62
+# slapos.toolbox = 0.63
 GitPython = 2.0.8
 
 # Required by:
-# slapos.toolbox = 0.62
+# slapos.toolbox = 0.63
 atomize = 0.2.0
 
 # Required by:
-# slapos.toolbox = 0.62
+# slapos.toolbox = 0.63
 feedparser = 5.2.1
 
 # Required by:
-# slapos.toolbox = 0.62
+# slapos.toolbox = 0.63
 lockfile = 0.12.2
 
 # Required by:
-# slapos.toolbox = 0.62
+# slapos.toolbox = 0.63
 paramiko = 2.0.1
 
 # Required by:
-# slapos.toolbox = 0.62
+# slapos.toolbox = 0.63
 pycurl = 7.43.0
 
 # Required by:
-# slapos.toolbox = 0.62
+# slapos.toolbox = 0.63
 rpdb = 0.1.5

software/gitlab/software.cfg

@@ -62,6 +62,10 @@ parts =
 [slapos.cookbook-repository]
 revision = 571d6514f7290e8faa9439c4b86aa2f6c87df261
 
+[git]
+# TODO stop using custom git revision once git 2.11 is supported by Gitlab
+url = https://www.kernel.org/pub/software/scm/git/git-2.10.2.tar.xz
+md5sum = 3cd1dca37be60668f482545716923b72
 
 ############################
 #   Software compilation   #

software/kvm/software.cfg

@@ -5,7 +5,7 @@ extends = common.cfg
 # XXX - use websockify = 0.5.1 for compatibility with kvm frontend
 websockify = 0.5.1
 
-slapos.toolbox = 0.62
+slapos.toolbox = 0.63
 erp5.util = 0.4.46
 apache-libcloud = 1.1.0
 collective.recipe.environment = 0.2.0

software/monitor/instance-monitor.cfg.jinja2

@@ -10,7 +10,6 @@ parts =
   monitor-collect-csv-wrapper
   fluentd-wrapper
   monitor-base
-  promise-check-slapgrid
   publish-connection-information
 
 
@@ -106,17 +105,6 @@ recipe = slapos.cookbook:publish
 monitor-setup-url = https://monitor.app.officejs.com/#page=settings_configurator&url=${publish:monitor-url}&username=${publish:monitor-user}&password=${publish:monitor-password}
 server_log_url = ${publish:monitor-base-url}/${slap-configuration:private-hash}/ 
 
-[promise-check-slapgrid]
-recipe = collective.recipe.template
-error-log-file = ${buildout:directory}/.slapgrid-${slap-connection:partition-id}-error.log
-input = inline:#!/bin/sh
-  if [ -f "${:error-log-file}" ]; then
-    >&2 cat ${:error-log-file}
-    exit 1
-  fi
-output = ${monitor-directory:promises}/slapgrid-${slap-connection:partition-id}-status
-mode = 700
-
 [slap-configuration]
 recipe = slapos.cookbook:slapconfiguration.serialised
 computer = ${slap-connection:computer-id}

software/monitor/software.cfg

@@ -32,7 +32,7 @@ mode = 0644
 recipe = slapos.recipe.build:download 
 url = ${:_profile_base_location_}/instance-monitor.cfg.jinja2
 destination = ${buildout:directory}/template-base-monitor.cfg
-md5sum = ec16fdaa0e6e13dcc7d3007908182c20
+md5sum = 20738a1b16de234e8bb7e88c822552b9
 mode = 0644
 
 [template-monitor-distributor]
@@ -90,6 +90,12 @@ eggs +=
 scripts =
   networkbench
   onetimedownload
+  monitor.bootstrap
+  monitor.collect
+  monitor.runpromise
+  monitor.genstatus
+  monitor.genrss
+  monitor.configwrite
 
 [fluentd]
 gems += 

software/neoppod/software-common.cfg

@@ -113,8 +113,8 @@ transaction = 1.7.0
 zdaemon = 4.1.0
 zodbpickle = 0.6.0
 zodbtools = 0.0.0.dev3
-# Required by slapos.toolbox = 0.62
-slapos.toolbox = 0.62
+# Required by slapos.toolbox = 0.63
+slapos.toolbox = 0.63
 PyRSS2Gen = 1.1
 apache-libcloud = 1.2.1
 atomize = 0.2.0

software/re6stnet/software.cfg

@@ -115,15 +115,15 @@ plone.recipe.command = 1.1
 pycrypto = 2.6.1
 pycurl = 7.43.0
 slapos.recipe.template = 2.7
-slapos.toolbox = 0.62
+slapos.toolbox = 0.63
 smmap = 0.9.0
 
 # Required by:
-# slapos.toolbox = 0.62
+# slapos.toolbox = 0.63
 GitPython = 2.0.8
 
 # Required by:
-# slapos.toolbox = 0.62
+# slapos.toolbox = 0.63
 atomize = 0.2.0
 
 # Required by:
@@ -131,11 +131,11 @@ atomize = 0.2.0
 backports.ssl-match-hostname = 3.4.0.2
 
 # Required by:
-# slapos.toolbox = 0.62
+# slapos.toolbox = 0.63
 feedparser = 5.1.3
 
 # Required by:
-# slapos.toolbox = 0.62
+# slapos.toolbox = 0.63
 lockfile = 0.12.2
 
 # Required by:
@@ -143,10 +143,10 @@ lockfile = 0.12.2
 miniupnpc = 1.9
 
 # Required by:
-# slapos.toolbox = 0.62
+# slapos.toolbox = 0.63
 paramiko = 2.0.1
 
 # Required by:
-# slapos.toolbox = 0.62
+# slapos.toolbox = 0.63
 rpdb = 0.1.5
 

software/slaprunner/common.cfg

@@ -84,7 +84,7 @@ recipe = hexagonit.recipe.download
 ignore-existing = true
 url = ${:_profile_base_location_}/template/runner-export.sh.jinja2
 download-only = true
-md5sum = e2d472ade09c11c70dbea080932e80bd
+md5sum = 2e96b9daa5cd20fc77e9573a9ce3fe52
 filename = runner-export.sh.jinja2
 mode = 0644
 

software/slaprunner/software.cfg

@@ -17,7 +17,7 @@ gunicorn = 19.4.5
 prettytable = 0.7.2
 pycurl = 7.43.0
 slapos.recipe.template = 2.9
-slapos.toolbox = 0.62
+slapos.toolbox = 0.63
 smmap = 0.9.0
 
 # Required by:

software/slaprunner/template/runner-export.sh.jinja2

@@ -72,8 +72,9 @@ remove_tmp_files () {
 }
 trap remove_tmp_files EXIT
 
-cd $srv_directory
-
+# Getting files from runner backup directory, as instance backup files may be
+# explicitely excluded from the backup, using the srv/exporter.exclude
+cd {{ directory['backup'] }}
 backup_directory_path=$(find . -path "./runner/instance/slappart*/srv/backup/*" -type f)
 
 # If no backup found, it's over

stack/monitor/buildout.cfg

@@ -96,7 +96,7 @@ recipe = slapos.recipe.template:jinja2
 filename = template-monitor.cfg
 template = ${:_profile_base_location_}/instance-monitor.cfg.jinja2.in
 rendered = ${buildout:directory}/template-monitor.cfg
-md5sum = 5b6b7a96bfa1e0ca83310caa48ed7b20
+md5sum = 04b4555615c48fb615dd452294c185e0
 context =
     key apache_location apache:location
     key gzip_location gzip:location
@@ -125,5 +125,5 @@ depends =
 PyRSS2Gen = 1.1
 cns.recipe.symlink = 0.2.3
 pycurl = 7.43.0
-slapos.toolbox = 0.62
+slapos.toolbox = 0.63
 

stack/monitor/instance-monitor.cfg.jinja2.in

@@ -138,6 +138,7 @@ logrotate-folder = ${logrotate:logrotate-entries}
 promise-runner = {{ monitor_runpromise }}
 promise-folder = ${directory:promises}
 monitor-promise-folder = ${monitor-directory:promises}
+promises-timeout-file = ${monitor-promise-timeout-file:file}
 pid-file = ${monitor-directory:pids}/monitor-bootstrap.pid
 
 public-path-list = 
@@ -152,6 +153,7 @@ parameter-file-path = ${monitor-instance-parameter:configuration-file-path}
 parameter-list = 
   raw monitor-user ${monitor-instance-parameter:username}
   htpasswd monitor-password ${monitor-htpassword-file:password-file} ${monitor-instance-parameter:username} ${httpd-monitor-htpasswd:htpasswd-path}
+  file promise-timeout ${monitor-promise-timeout-file:file}
   ${monitor-instance-parameter:instance-configuration}
 # htpasswd entry:  htpasswd key password-file username htpasswd-file
 
@@ -161,6 +163,14 @@ python = {{ python_with_eggs }}
 
 promise-output-file = ${directory:monitor}/monitor-bootstrap-status
 
+[monitor-promise-timeout-file]
+recipe = plone.recipe.command
+file = ${directory:etc}/promise_timeout
+command = 
+  if [ ! -s "${:file}" ]; then
+    echo "20" > ${:file}
+  fi
+
 [monitor-conf]
 recipe = slapos.recipe.template:jinja2
 template = {{ monitor_conf_template }}
@@ -335,7 +345,7 @@ monitor-base-url = ${monitor-frontend-promise:url}
 #monitor-base-url = ${monitor-httpd-conf-parameter:url}
 root-instance-title = ${slap-configuration:root-instance-title}
 monitor-url-list =
-cors-domains = monitor.app.officejs.com 
+cors-domains = monitor.app.officejs.com
 # XXX Hard coded parameter
 collector-db = /srv/slapgrid/var/data-log/collector.db
 # Credentials
@@ -390,6 +400,17 @@ input = inline:#!{{ dash_executable_location }}
 output = ${directory:promises}/monitor-bootstrap-status
 mode = 700
 
+[promise-check-slapgrid]
+recipe = collective.recipe.template
+error-log-file = ${buildout:directory}/.slapgrid-${slap-connection:partition-id}-error.log
+input = inline:#!/bin/sh
+  if [ -f "${:error-log-file}" ]; then
+    >&2 cat ${:error-log-file}
+    exit 1
+  fi
+output = ${monitor-directory:promises}/buildout-${slap-connection:partition-id}-status
+mode = 700
+
 [monitor-base]
 # create dependencies between required monitor parts
 recipe = plone.recipe.command
@@ -407,6 +428,7 @@ depends =
   ${monitor-httpd-promise:filename}
   ${monitor-status2rss-cron-entry:name}
   ${monitor-bootstrap-promise:file}
+  ${promise-check-slapgrid:output}
 
 [monitor-publish]
 monitor-base-url = ${publish:monitor-base-url}

stack/slapos.cfg

@@ -133,7 +133,7 @@ pytz = 2016.7
 requests = 2.11.1
 setuptools = 19.6.2
 six = 1.10.0
-slapos.cookbook = 1.0.44
+slapos.cookbook = 1.0.45
 slapos.core = 1.3.18
 slapos.extension.strip = 0.1
 slapos.libnetworkcache = 0.14.5
@@ -164,11 +164,11 @@ functools32 = 3.2.3.post2
 ipaddress = 1.0.17
 
 # Required by:
-# slapos.cookbook==1.0.44
+# slapos.cookbook==1.0.45
 jsonschema = 2.5.1
 
 # Required by:
-# slapos.cookbook==1.0.44
+# slapos.cookbook==1.0.45
 lock-file = 2.0
 
 # Required by: