Update Release Candidate

component/openssl/buildout.cfg

@@ -17,8 +17,8 @@ parts =
 [openssl]
 recipe = slapos.recipe.cmmi
 shared = true
-url = https://www.openssl.org/source/openssl-1.1.1i.tar.gz
-md5sum = 08987c3cf125202e2b0840035efb392c
+url = https://www.openssl.org/source/openssl-1.1.1j.tar.gz
+md5sum = cccaa064ed860a2b4d1303811bf5c682
 location = @@LOCATION@@
 # 'prefix' option to override --openssldir/--prefix (which is useful
 # when combined with DESTDIR). Used by slapos.package.git/obs

slapos/recipe/6tunnel/template/4to6.in

 #!%(shell_path)s
-# BEWARE: This file is operated by slapgrid
+# BEWARE: This file is operated by slapos node
 # BEWARE: It will be overwritten automatically
 exec %(6tunnel_path)s -d -l %(ipv4)s %(ipv4_port)s %(ipv6)s %(ipv6_port)s

slapos/recipe/6tunnel/template/6to4.in

 #!%(shell_path)s
-# BEWARE: This file is operated by slapgrid
+# BEWARE: This file is operated by slapos node
 # BEWARE: It will be overwritten automatically
 exec %(6tunnel_path)s -6 -4 -d -l %(ipv6)s %(ipv6_port)s %(ipv4)s %(ipv4_port)s

slapos/recipe/check_page_content/template/check_page_content.in

 #!%(shell_path)s
-# BEWARE: This file is operated by slapgrid
+# BEWARE: This file is operated by slapos node
 # BEWARE: It will be overwritten automatically
 
 URL="%(url)s"

slapos/recipe/check_parameter/template/check_ipv4.py.in

 #!%(python_path)s
-# BEWARE: This file is operated by slapgrid
+# BEWARE: This file is operated by slapos node
 # BEWARE: It will be overwritten automatically
 import socket
 

slapos/recipe/check_parameter/template/check_ipv6.py.in

 #!%(python_path)s
-# BEWARE: This file is operated by slapgrid
+# BEWARE: This file is operated by slapos node
 # BEWARE: It will be overwritten automatically
 import socket
 import sys

slapos/recipe/check_parameter/template/check_parameter.py.in

 #!%(python_path)s
-# BEWARE: This file is operated by slapgrid
+# BEWARE: This file is operated by slapos node
 # BEWARE: It will be overwritten automatically
 import socket
 import sys

slapos/recipe/check_port_listening.py

@@ -30,7 +30,7 @@ from zc.buildout.easy_install import _safe_arg, script_header
 import sys
 
 template = script_header + r"""
-# BEWARE: This file is operated by slapgrid
+# BEWARE: This file is operated by slapos node
 # BEWARE: It will be overwritten automatically
 import socket
 import sys

slapos/recipe/check_url_available/template/check_url.in

 #!%(shell_path)s
-# BEWARE: This file is operated by slapgrid
+# BEWARE: This file is operated by slapos node
 # BEWARE: It will be overwritten automatically
 
 URL="%(url)s"

slapos/recipe/erp5scalabilitytestbed/template/erp5tester_manager_run.in

 #!/bin/sh
-# BEWARE: This file is operated by slapgrid
+# BEWARE: This file is operated by slapos node
 # BEWARE: It will be overwritten automatically
 exec %(scalability_tester_manager_binary)s %(plugin_name)s -a %(address)s \
   -r %(report_path)s -m %(nb_tester_init)s -t %(nb_tester_max)s \

slapos/recipe/erp5scalabilitytestbed/template/nosqltester_run.in

 #!/bin/sh
-# BEWARE: This file is operated by slapgrid
+# BEWARE: This file is operated by slapos node
 # BEWARE: It will be overwritten automatically
 cd %(benchmark_suite_path)s && \
 exec %(scalability_tester_binary)s -m %(host_address)s -a %(tester_address)s \

slapos/recipe/kvm_frontend/template/nodejs_run.in

 #!%(shell_path)s
-# BEWARE: This file is operated by slapgrid
+# BEWARE: This file is operated by slapos node
 # BEWARE: It will be overwritten automatically
 export NODE_PATH=%(node_env)s
 exec %(node_path)s %(conf_path)s %(ipv6)s %(ipv4)s %(port)s %(key)s %(certificate)s %(map_path)s %(plain_http)s

slapos/recipe/nbdserver/template/nbdserver_run.in

 #!%(shell_path)s
-# BEWARE: This file is operated by slapgrid
+# BEWARE: This file is operated by slapos node
 # BEWARE: It will be overwritten automatically
 
 # 32767 is the maximum number of connections allowed by the nbd server

slapos/recipe/nosqltestbed/kumo/template/kumo_gateway_run.in

 #!/bin/sh
-# BEWARE: This file is operated by slapgrid
+# BEWARE: This file is operated by slapos node
 # BEWARE: It will be overwritten automatically
 exec %(gateway_binary)s -F -E -m %(manager_address)s:%(manager_port)s \
   -t %(gateway_address)s:%(gateway_port)s --verbose -o %(gateway_log)s

slapos/recipe/nosqltestbed/kumo/template/kumo_manager_run.in

 #!/bin/sh
-# BEWARE: This file is operated by slapgrid
+# BEWARE: This file is operated by slapos node
 # BEWARE: It will be overwritten automatically
 exec %(manager_binary)s -a -l %(manager_address)s:%(manager_port)s \
   --verbose -o %(manager_log)s

slapos/recipe/nosqltestbed/kumo/template/kumo_server_run.in

 #!/bin/sh
-# BEWARE: This file is operated by slapgrid
+# BEWARE: This file is operated by slapos node
 # BEWARE: It will be overwritten automatically
 exec %(server_binary)s -l %(server_address)s:%(server_port)s \
   -L %(server_listen_port)s -m %(manager_address)s:%(manager_port)s \

slapos/recipe/nosqltestbed/kumo/template/kumotester_manager_run.in

 #!/bin/sh
-# BEWARE: This file is operated by slapgrid
+# BEWARE: This file is operated by slapos node
 # BEWARE: It will be overwritten automatically
 exec %(nosqltester_manager_binary)s %(plugin_name)s -a %(address)s \
   -r %(report_path)s -s %(max_server)s -t %(max_tester)s \

slapos/recipe/nosqltestbed/kumo/template/memstrike_run.in

 #!/bin/sh
-# BEWARE: This file is operated by slapgrid
+# BEWARE: This file is operated by slapos node
 # BEWARE: It will be overwritten automatically
 exec %(memstrike_binary)s -s -l %(gateway_address)s -p %(gateway_port)s -t %(nb_thread)s %(nb_request)s

slapos/recipe/nosqltestbed/template/nosqltester_manager_run.in

 #!/bin/sh
-# BEWARE: This file is operated by slapgrid
+# BEWARE: This file is operated by slapos node
 # BEWARE: It will be overwritten automatically
 exec %(nosqltester_manager_binary)s %(plugin_name)s -a %(address)s \
   -r %(report_path)s -s %(max_server)s -t %(max_tester)s \

slapos/recipe/nosqltestbed/template/nosqltester_run.in

 #!/bin/sh
-# BEWARE: This file is operated by slapgrid
+# BEWARE: This file is operated by slapos node
 # BEWARE: It will be overwritten automatically
 exec %(nosqltester_binary)s -m %(host_address)s -a %(tester_address)s \
   -r %(report_path)s -b "%(binary)s" -l %(log_directory)s \

slapos/recipe/onetimeupload/template/onetimeupload_run.in

 #!%(shell_path)s
-# BEWARE: This file is operated by slapgrid
+# BEWARE: This file is operated by slapos node
 # BEWARE: It will be overwritten automatically
 exec %(onetimeupload_path)s -l %(log_path)s %(ip)s %(port)s %(image)s %(key)s 

slapos/recipe/sheepdogtestbed/template/nosqltester_manager_run.in

 #!/bin/sh
-# BEWARE: This file is operated by slapgrid
+# BEWARE: This file is operated by slapos node
 # BEWARE: It will be overwritten automatically
 exec %(nosqltester_manager_binary)s -a %(address)s -r %(report_path)s -s %(nb_server_max)s -t %(nb_tester_max)s %(software_release_url)s %(server_url)s "%(key_file)s" "%(cert_file)s" %(computer_id)s %(computer_partition_id)s %(plugin_name)s

slapos/recipe/sheepdogtestbed/template/nosqltester_run.in

 #!/bin/sh
-# BEWARE: This file is operated by slapgrid
+# BEWARE: This file is operated by slapos node
 # BEWARE: It will be overwritten automatically
 exec %(nosqltester_binary)s -h %(host_address)s -a %(tester_address)s -r %(report_path)s -b "%(binary)s"

slapos/recipe/zimbra_kvm/template/kvm_controller_run.in

 #!%(python_path)s
-# BEWARE: This file is operated by slapgrid
+# BEWARE: This file is operated by slapos node
 # BEWARE: It will be overwritten automatically
 
 # Echo client program

slapos/recipe/zimbra_kvm/template/kvm_run.in

 #!%(python_path)s
-# BEWARE: This file is operated by slapgrid
+# BEWARE: This file is operated by slapos node
 # BEWARE: It will be overwritten automatically
 
 # Echo client program

slapos/test/test_json_schema.py

@@ -59,7 +59,16 @@ def createSoftwareCfgValidatorTest(path, software_cfg_schema):
   # which is defined in schema.json in this directory
   def run(self, *args, **kwargs):
     with open(path, "r") as json_file:
-      jsonschema.validate(json.load(json_file), software_cfg_schema)
+      schema = json.load(json_file)
+      jsonschema.validate(schema, software_cfg_schema)
+
+      # also make sure request and response schemas can be resolved
+      schema.setdefault('$id', 'file://' + path)
+      resolver = jsonschema.RefResolver.from_schema(schema)
+      for software_type_definition in schema['software-type'].itervalues():
+        resolver.resolve(software_type_definition['request'])
+        resolver.resolve(software_type_definition['response'])
+
   return run
 
 

software/backupserver/buildout.hash.cfg

@@ -14,7 +14,7 @@
 # not need these here).
 [template-nginx-service]
 filename = template-nginx-service.sh.in
-md5sum = 5c94d952305552dcbeaeaeb27dd28f3b
+md5sum = 458870b70c33a1621b68961ae2372ad5
 
 [template-nginx-configuration]
 filename = template-nginx.cfg.in
@@ -22,7 +22,7 @@ md5sum = fc4f258e3ae56e0c9596484d736ccfed
 
 [template-dcron-service]
 filename = template-dcron-service.sh.in
-md5sum = 1372441dac23e4fa7d2dc773a74725ea
+md5sum = 851262d7174da868805cb7c8e1ced7c0
 
 [template-backup-script]
 filename = template-backup-script.sh.in

software/backupserver/template-dcron-service.sh.in

 #!${dash-output:dash}
-# BEWARE: This file is operated by slapgrid
+# BEWARE: This file is operated by slapos node
 # BEWARE: It will be overwritten automatically
 
 exec ${dcron-output:crond} \

software/backupserver/template-nginx-service.sh.in

 #!${dash-output:dash}
-# BEWARE: This file is operated by slapgrid
+# BEWARE: This file is operated by slapos node
 # BEWARE: It will be overwritten automatically
 
 if [ ! -e $${nginx-configuration:ssl_crt} ]

software/cdn-me/instance-cdn-me-output-schema.json

+{
+  "$schema": "http://json-schema.org/draft-07/schema#",
+  "description": "Values returned by CDN ME instanciation",
+  "properties": {
+    "url-https": {
+      "description": "URL to access instance",
+      "type": "string"
+    },
+    "url-http": {
+      "description": "URL to access instance over unencrypted HTTP",
+      "type": "string"
+    },
+    "token": {
+      "description": "Token for Re6st",
+      "type": "string"
+    },
+    "ipv6": {
+      "description": "IPv6 address of the instance",
+      "type": "string"
+    },
+    "ipv4": {
+      "description": "IPv4 address of the instance",
+      "type": "string"
+    },
+    "monitor_url": {
+      "description": "URL to access monitoring interface",
+      "type": "string"
+    }
+  },
+  "type": "object"
+}

software/cdn-me/instance-cdn-request.cfg.jinja2

 [buildout]
 parts =
   request-monitor-test-distributor-slave 
-  publish-connection-informations
+  publish-connection-information
   connection-parameter-http-checker
   connection-parameter-checker
 
@@ -135,7 +135,7 @@ slave = true
 config-url = $${request-monitor-test-distributor-slave:connection-site_url}
 return = site_url domain
 
-[publish-connection-informations]
+[publish-connection-information]
 recipe = slapos.cookbook:publish
 url-https =  https://$${request-frontend-token-slave:connection-domain}
 url = http://$${request-frontend-token-slave:connection-domain}
@@ -156,6 +156,6 @@ expected-value =
 [connection-parameter-http-checker]
 recipe = slapos.cookbook:check_parameter
 path = $${directory:promise}/check_re6stnet_http
-value = $${publish-connection-informations:url}
+value = $${publish-connection-information:url}
 expected-not-value = http://
 expected-value =

software/erp5/upgrade_test/test.py

@@ -39,7 +39,7 @@ from slapos.testing.testcase import installSoftwareUrlList
 from slapos.testing.testcase import SlapOSNodeCommandError
 from slapos.grid.utils import md5digest
 
-old_software_release_url = 'https://lab.nexedi.com/nexedi/slapos/raw/1.0.167/software/erp5/software.cfg'
+old_software_release_url = 'https://lab.nexedi.com/nexedi/slapos/raw/1.0.167.1/software/erp5/software.cfg'
 new_software_release_url = os.path.abspath(
     os.path.join(os.path.dirname(__file__), '..', 'software.cfg'))
 

software/galene/buildout.hash.cfg

@@ -15,4 +15,4 @@
 
 [instance-cfg]
 filename = instance.cfg.in
-md5sum = 270b39f448ec553fa9e203c5fbb49856
+md5sum = 8341d1d8c1839a1c623dd0fd771ef104

software/galene/instance.cfg.in

@@ -67,17 +67,21 @@ input = inline:{
 output = $${directory:groups}/public.json
 
 [galene-wrapper]
-recipe = slapos.cookbook:wrapper
+recipe = collective.recipe.template
 port = 8443
 ip = $${slap-configuration:ipv6-random}
-command-line = 
-  ${galene:location}/bin/galene
-    -static ${galene-repository:location}/static
-    -recordings $${directory:recordings}
-    -groups $${directory:groups}
-    -data $${directory:data}
-    -http [$${:ip}]:$${:port}
-wrapper-path = $${directory:services}/galene
+input = inline:
+  #!/bin/bash
+  ulimit -n $(ulimit -Hn)
+  exec ${galene:location}/bin/galene \
+    -static ${galene-repository:location}/static \
+    -recordings $${directory:recordings} \
+    -groups $${directory:groups} \
+    -data $${directory:data} \
+    -http [$${:ip}]:$${:port} \
+    -turn ""
+output = $${directory:services}/galene
+mode = 0755
 depends =
   $${ice-servers.json:recipe}
   $${groups-json:recipe}

software/galene/software.cfg

@@ -22,7 +22,7 @@ eggs    =
 [galene-repository]
 <= git-repository
 repository = https://lab.nexedi.com/nexedi/galene.git
-revision = galene-0.2
+revision = galene-0.3
 location = ${buildout:parts-directory}/galene-repository
 
 [galene]

software/html5as-base/README.md

+# Html5 Application Server #
+
+## Presentation ##
+
+* Fast hosting software for static website (html5)
+
+* Use Nginx server
+
+## Parameter ##
+
+download_url (string) :required
+
+Details :
+
+* Only tarball (tar) is supported
+
+* Compressed format is gunzip (optional)
+
+* Tarball must contain an index.html at its root
+
+## How it works ##
+
+Each time you (re)start your instance or update parameters, html5as will remove previous website then download tarball and extract it in docroot directory.

software/html5as-base/instance.cfg.in

+[buildout]
+parts =
+  switch-softwaretype
+
+eggs-directory = {{ buildout['eggs-directory'] }}
+develop-eggs-directory = {{ buildout['develop-eggs-directory'] }}
+offline = true
+
+[profile-common]
+nginx_location = {{ nginx_location }}
+dash_location = {{ dash_location }}
+template_nginx_conf = {{ template_nginx_conf_target }}
+template_mime_types = {{ template_mime_types_target }} 
+template_launcher = {{ template_launcher_target }} 
+
+[instance-html5as]
+recipe = slapos.recipe.template:jinja2
+template = {{ template_instance_html5as_target }}
+rendered = ${buildout:directory}/${:filename}
+filename = instance-html5as.cfg
+context = 
+  section buildout buildout
+  section parameter_list profile-common
+
+[switch-softwaretype]
+recipe = slapos.cookbook:softwaretype
+default = ${instance-html5as:rendered}

software/html5as-base/instance_html5as.cfg.in

+#############################
+#
+# Deploy html5as instance
+#
+#############################
+[buildout]
+parts =
+    nginx_conf
+    downloader
+    mime_types
+    launcher
+    publish-connection-information
+
+# Define egg directories to be the one from Software Release
+# (/opt/slapgrid/...)
+# Always the same.
+eggs-directory = {{ buildout['eggs-directory'] }}
+develop-eggs-directory = {{ buildout['develop-eggs-directory'] }}
+offline = true
+
+# partition tree
+# /
+# |- etc/
+# |    |- nginx.conf
+# |    |- run/
+# |         |- html5as (binary)
+# |- var/
+# |    |- run/
+# |    |    |- nginx.pid
+# |    |- log/
+# |    |    |- nginx.log
+# |    |    |- nginx.access.log
+# |- srv/
+# |    |- html5as/ (doc root)
+# |    |        |- index.html
+# |    |- backup/
+
+# Create all needed directories, depending on your needs
+[directory]
+recipe = slapos.cookbook:mkdirectory
+home = ${buildout:directory}
+etc = ${:home}/etc
+var = ${:home}/var
+srv = ${:home}/srv
+
+[basedirectory]
+recipe = slapos.cookbook:mkdirectory
+# Executables put here will be started but not monitored (for startup scripts)
+script = ${directory:etc}/run
+# Executables put here will be started and monitored (for daemons)
+service = ${directory:etc}/service
+log = ${directory:var}/log
+run = ${directory:var}/run
+backup = ${directory:srv}/backup
+data = ${directory:srv}/html5as
+
+[tempdirectory]
+recipe = slapos.cookbook:mkdirectory
+tmp = ${directory:home}/tmp
+client_body_temp_path = ${:tmp}/client_body_temp_path
+proxy_temp_path = ${:tmp}/proxy_temp_path
+fastcgi_temp_path = ${:tmp}/fastcgi_temp_path
+uwsgi_temp_path = ${:tmp}/uwsgi_temp_path
+scgi_temp_path = ${:tmp}/scgi_temp_path
+
+# List of options for html5as configuration
+# It will run a simple nginx serving the content of srv/html5as
+[html5as]
+# Options
+nb_workers = 2
+
+# Network
+ip = ${slap-network-information:global-ipv6}
+port = 8081
+access_url = http://[${:ip}]:${:port}
+
+# Paths
+# Log
+path_pid = ${basedirectory:run}/nginx.pid
+path_log = ${basedirectory:log}/nginx.log
+path_access_log = ${basedirectory:log}/nginx.access.log
+path_error_log = ${basedirectory:log}/nginx.error.log
+path_tmp = ${tempdirectory:tmp}
+# Docroot
+docroot = ${basedirectory:data}
+default_index = ${basedirectory:data}/index.html
+# Config files
+path_nginx_conf = ${directory:etc}/nginx.conf
+path_mime_types = ${directory:etc}/mime_types
+# Binaries
+path_shell = {{ parameter_list['dash_location'] }}/bin/dash
+
+# Executables
+bin_launcher = ${basedirectory:script}/launcher
+
+# Utils
+path_nginx = {{ parameter_list['nginx_location'] }}/sbin/nginx
+
+# Render nginx conf
+[nginx_conf]
+recipe = slapos.recipe.template:jinja2
+template = {{ parameter_list['template_nginx_conf'] }}
+rendered = ${html5as:path_nginx_conf}
+context = 
+    section param_html5as html5as
+    section param_tempdir tempdirectory
+
+# Render necessary mime types file for nginx
+[mime_types]
+recipe = slapos.recipe.template:jinja2
+template = {{ parameter_list['template_mime_types'] }}
+rendered = ${html5as:path_mime_types}
+
+# Render the script launching nginx
+[launcher]
+recipe = slapos.recipe.template:jinja2
+template = {{ parameter_list['template_launcher'] }}
+rendered = ${html5as:bin_launcher}
+mode = 700
+context = 
+    section param_html5as html5as
+
+# Simple command to put content in the docroot
+[downloader]
+recipe = plone.recipe.command
+command = rm -r ${html5as:docroot}/*; echo "Hello World!" >  ${html5as:docroot}/index.html
+
+# Publish nginx address
+[publish-connection-information]
+recipe = slapos.cookbook:publish
+server_url = ${html5as:access_url}

software/html5as-base/software.cfg

+[buildout]
+extends =
+# "slapos" stack describes basic things needed for 99.9% of SlapOS Software
+# Releases
+  ../../stack/slapos.cfg
+# Extend here component profiles, like openssl, apache, mariadb, curl...
+# Or/and extend a stack (lamp, tomcat) that does most of the work for you
+# In this example we extend needed components for html5as.
+  ../../component/nginx/buildout.cfg
+  ../../component/dash/buildout.cfg
+
+parts =
+# Call installation of slapos.cookbook egg defined in stack/slapos.cfg (needed
+# in 99,9% of Slapos Software Releases)
+    slapos-cookbook
+# Call creation of instance.cfg file that will be called for deployment of
+# instance
+    template-cfg
+
+# Download instance.cfg.in (buildout profile used to deployment of instance),
+# replace all {{ foo_bar }} parameters by real values
+# The recipe, template and mode are fetched from jijna-template
+[template-cfg]
+recipe = slapos.recipe.template:jinja2
+rendered = ${buildout:directory}/template.cfg
+template = ${:_profile_base_location_}/${:filename}
+filename = instance.cfg.in
+md5sum = 5a6ebc126bcad3cdff1b51fb51f82a35
+mode = 0644
+context =
+  section buildout buildout
+  key nginx_location nginx:location
+  key dash_location dash:location
+  key template_nginx_conf_target template_nginx_conf:target
+  key template_mime_types_target template_mime_types:target
+  key template_launcher_target template_launcher:target
+  key template_instance_html5as_target instance_html5as:target
+
+# Download instance_html5as.cfg.in
+[instance_html5as]
+recipe = slapos.recipe.build:download
+url = ${:_profile_base_location_}/${:_update_hash_filename_}
+_update_hash_filename_ = instance_html5as.cfg.in
+md5sum = 4a8c98cc5ca141f78f14fb9cec203cb8
+mode = 0644
+
+[template_nginx_conf]
+recipe = slapos.recipe.build:download
+url = ${:_profile_base_location_}/${:_update_hash_filename_}
+_update_hash_filename_ = templates/nginx_conf.in
+md5sum = 61dc4c82bf48563228ce4dea6c5c6319
+mode = 0644
+
+[template_launcher]
+recipe = slapos.recipe.build:download
+url = ${:_profile_base_location_}/${:_update_hash_filename_}
+_update_hash_filename_ = templates/launcher.in
+md5sum = 8d4d3152f5125f73d0c3f3ea1c3699dd
+mode = 0644
+
+[template_mime_types]
+recipe = slapos.recipe.build:download
+url = ${:_profile_base_location_}/${:_update_hash_filename_}
+_update_hash_filename_ = templates/mime_types.in
+md5sum = 4ef94a7b458d885cd79ba0b930a5727e
+mode = 0644
+
+# Pin versions of eggs used that are not already pinned by stack/slapos.cfg
+[versions]
+slapos.recipe.template = 4.4

software/html5as-base/templates/launcher.in

+#! {{ param_html5as['path_shell'] }}
+# BEWARE: This file is operated by slapos node
+# BEWARE: It will be overwritten automatically
+
+# Run nginx
+exec {{ param_html5as['path_nginx'] }} -c {{ param_html5as['path_nginx_conf'] }}

software/html5as-base/templates/mime_types.in

+types {
+	text/html				html htm shtml;
+	text/css				css;
+	text/xml				xml rss;
+	image/gif				gif;
+	image/jpeg				jpeg jpg;
+	application/x-javascript		js;
+	application/atom+xml			atom;
+
+	text/mathml				mml;
+	text/plain				txt;
+	text/vnd.sun.j2me.app-descriptor	jad;
+	text/vnd.wap.wml			wml;
+	text/x-component			htc;
+
+	image/png				png;
+	image/tiff				tif tiff;
+	image/vnd.wap.wbmp			wbmp;
+	image/x-icon				ico;
+	image/x-jng				jng;
+	image/x-ms-bmp				bmp;
+	image/svg+xml				svg svgz;
+
+	application/java-archive		jar war ear;
+	application/mac-binhex40		hqx;
+	application/msword			doc;
+	application/pdf				pdf;
+	application/postscript			ps eps ai;
+	application/rtf				rtf;
+	application/vnd.ms-excel		xls;
+	application/vnd.ms-powerpoint		ppt;
+	application/vnd.wap.wmlc		wmlc;
+	application/vnd.google-earth.kml+xml	kml;
+	application/vnd.google-earth.kmz	kmz;
+	application/x-7z-compressed		7z;
+	application/x-cocoa			cco;
+	application/x-java-archive-diff		jardiff;
+	application/x-java-jnlp-file		jnlp;
+	application/x-makeself			run;
+	application/x-perl			pl pm;
+	application/x-pilot			prc pdb;
+	application/x-rar-compressed		rar;
+	application/x-redhat-package-manager	rpm;
+	application/x-sea			sea;
+	application/x-shockwave-flash		swf;
+	application/x-stuffit			sit;
+	application/x-tcl			tcl tk;
+	application/x-x509-ca-cert		der pem crt;
+	application/x-xpinstall			xpi;
+	application/xhtml+xml			xhtml;
+	application/zip				zip;
+
+	application/octet-stream		bin exe dll;
+	application/octet-stream		deb;
+	application/octet-stream		dmg;
+	application/octet-stream		eot;
+	application/octet-stream		iso img;
+	application/octet-stream		msi msp msm;
+	application/ogg				ogx;
+
+	audio/midi				mid midi kar;
+	audio/mpeg				mpga mpega mp2 mp3 m4a;
+	audio/ogg				oga ogg spx;
+	audio/x-realaudio			ra;
+	audio/webm				weba;
+
+	video/3gpp				3gpp 3gp;
+	video/mp4				mp4;
+	video/mpeg				mpeg mpg mpe;
+	video/ogg				ogv;
+	video/quicktime				mov;
+	video/webm				webm;
+	video/x-flv				flv;
+	video/x-mng				mng;
+	video/x-ms-asf				asx asf;
+	video/x-ms-wmv				wmv;
+	video/x-msvideo				avi;
+}

software/html5as-base/templates/nginx_conf.in

+worker_processes {{ param_html5as['nb_workers'] }};
+
+pid {{ param_html5as['path_pid'] }};
+error_log {{ param_html5as['path_error_log'] }};
+
+daemon off;
+
+events {
+	worker_connections 1024;
+	accept_mutex off;
+}
+
+http {
+	include {{ param_html5as['path_mime_types'] }};
+	default_type application/octet-stream;
+	types_hash_bucket_size 64;
+    access_log {{ param_html5as['path_access_log'] }} combined;
+	index index.html;
+
+    server {
+        listen [{{ param_html5as['ip'] }}]:{{ param_html5as['port'] }};
+        server_name _;
+
+        keepalive_timeout 5;
+        client_body_temp_path {{ param_tempdir['client_body_temp_path'] }};
+        proxy_temp_path {{ param_tempdir['proxy_temp_path'] }};
+        fastcgi_temp_path {{ param_tempdir['fastcgi_temp_path'] }};
+        uwsgi_temp_path {{ param_tempdir['uwsgi_temp_path'] }};
+        scgi_temp_path {{ param_tempdir['scgi_temp_path'] }};
+
+        # path for static files
+        root {{ param_html5as['docroot'] }};
+    }
+}

software/html5as/buildout.hash.cfg

+# To learn more about how to generate this file read 
+# ../../README.update-hash.rst
+# THIS IS NOT A BUILDOUT FILE, despite purposedly using a compatible syntax.
+# The only allowed lines here are (regexes):
+# - "^#" comments, copied verbatim
+# - "^[" section beginings, copied verbatim
+# - lines containing an "=" sign which must fit in the following categorie.
+#   - "^\s*filename\s*=\s*path\s*$" where "path" is relative to this file
+#     Copied verbatim.
+#   - "^\s*hashtype\s*=.*" where "hashtype" is one of the values supported
+#     by the re-generation script.
+#     Re-generated.
+# - other lines are copied verbatim
+# Substitution (${...:...}), extension ([buildout] extends = ...) and
+# section inheritance (< = ...) are NOT supported (but you should really
+# not need these here).
+
+[template-cfg]
+filename = instance.cfg.in
+md5sum = 8a1383f7508a911c0c2d81445b5d4250
+
+[instance_html5as]
+_update_hash_filename_ = instance_html5as.cfg.in
+md5sum = 44a0f9f0a7682e800e56d8ece60b46b4
+
+[template_nginx_conf]
+_update_hash_filename_ = templates/nginx_conf.in
+md5sum = 61dc4c82bf48563228ce4dea6c5c6319
+
+[template_launcher]
+_update_hash_filename_ = templates/launcher.in
+md5sum = 8d4d3152f5125f73d0c3f3ea1c3699dd
+
+[template_mime_types]
+_update_hash_filename_ = templates/mime_types.in
+md5sum = 4ef94a7b458d885cd79ba0b930a5727e
+
+[template_index_html]
+_update_hash_filename_ = templates/index.html.in
+md5sum = d57cb01df5941e139b02a2f7bdabcdc8
+
+[template_graceful]
+_update_hash_filename_ = templates/graceful.in
+md5sum = 1c0ee16966e1fcdb3fd11c09f12ee2b2

software/html5as/instance.cfg.in

@@ -9,9 +9,14 @@ offline = true
 [profile-common]
 nginx_location = {{ nginx_location }}
 dash_location = {{ dash_location }}
+tar_location = {{ tar_location }}
+curl_location = {{ curl_location }}
 template_nginx_conf = {{ template_nginx_conf_target }}
 template_mime_types = {{ template_mime_types_target }} 
 template_launcher = {{ template_launcher_target }} 
+template_index_html = {{ template_index_html_target }}
+template_graceful  = {{ template_graceful_target }}
+template_monitor = {{ template_monitor }}
 
 [instance-html5as]
 recipe = slapos.recipe.template:jinja2

software/html5as/instance_html5as.cfg.in

@@ -9,6 +9,9 @@ parts =
     downloader
     mime_types
     launcher
+    nginx-graceful
+    port-listening-promise
+    logrotate-entry-nginx
     publish-connection-information
 
 # Define egg directories to be the one from Software Release
@@ -18,6 +21,9 @@ eggs-directory = {{ buildout['eggs-directory'] }}
 develop-eggs-directory = {{ buildout['develop-eggs-directory'] }}
 offline = true
 
+# Instance profile extends monitoring stack
+extends = {{ parameter_list['template_monitor'] }}
+
 # partition tree
 # /
 # |- etc/
@@ -71,7 +77,7 @@ nb_workers = 2
 
 # Network
 ip = ${slap-network-information:global-ipv6}
-port = 8081
+port = ${slap-parameter:port}
 access_url = http://[${:ip}]:${:port}
 
 # Paths
@@ -89,9 +95,11 @@ path_nginx_conf = ${directory:etc}/nginx.conf
 path_mime_types = ${directory:etc}/mime_types
 # Binaries
 path_shell = {{ parameter_list['dash_location'] }}/bin/dash
+curl-binary = {{ parameter_list['curl_location'] }}/bin/curl
+tar-binary = {{ parameter_list['tar_location'] }}/bin/tar
 
 # Executables
-bin_launcher = ${basedirectory:script}/launcher
+bin_launcher = ${basedirectory:service}/launcher
 
 # Utils
 path_nginx = {{ parameter_list['nginx_location'] }}/sbin/nginx
@@ -120,12 +128,71 @@ mode = 700
 context = 
     section param_html5as html5as
 
-# Simple command to put content in the docroot
+# Command to put content in the docroot
 [downloader]
 recipe = plone.recipe.command
-command = rm -r ${html5as:docroot}/*; echo "Hello World!" >  ${html5as:docroot}/index.html
+# This section will fail if the command fails.
+stop-on-error = true
+# If a tarball is passed as a parameter in download url
+# it's content will be served by the instance.
+# If the parameter is not provided it fallback to the default template
+command = 
+  rm -rf ${html5as:docroot}/*; 
+  URL="${slap-parameter:download_url}"; 
+  if [ -n "$URL" ];
+  then
+    ${html5as:curl-binary} -Lks $URL | ${html5as:tar-binary} xzv -C ${html5as:docroot} --strip-components 1;
+  else
+    cp ${default_index_html:rendered} ${html5as:docroot}/;
+  fi
+
+[default_index_html]
+recipe = slapos.recipe.template:jinja2
+template = {{ parameter_list['template_index_html'] }}
+rendered = ${directory:srv}/index.html
+context =
+  key title slap-parameter:title
+
+### Nginx Graceful
+[nginx-graceful]
+recipe = slapos.recipe.template:jinja2
+template = {{ parameter_list['template_graceful'] }}
+rendered = ${basedirectory:script}/nginx-graceful
+mode = 0700
+context =
+    section param_html5as html5as
+
+# Port Listening checking promise
+[port-listening-promise]
+<= monitor-promise-base
+module = check_port_listening
+name = nginx-port-listening.py
+config-hostname = ${html5as:ip}
+config-port = ${html5as:port}
+
+# Use a port different from the default one in order to be able to
+# use it in a SlapOS webrunner or a Theia SlapOS Runner
+[monitor-instance-parameter]
+monitor-httpd-port = 8197
+
+# Monitor Stack also provides logrotate stack. We only need to extend
+# the logrotate-entry-base defined in instance-logrotate-base.cfg.in .
+# More parameters can be added following the logrotate-entry-base section  
+[logrotate-entry-nginx]
+<= logrotate-entry-base
+name = nginx
+log = ${html5as:path_access_log} ${html5as:path_error_log}
 
 # Publish nginx address
 [publish-connection-information]
 recipe = slapos.cookbook:publish
+# By extending monitor publish, all the section deploying monitoring will
+# be deployed. The parameters needed for accessing monitoring will be published
+<= monitor-publish
 server_url = ${html5as:access_url}
+title = Title ${slap-parameter:title}!
+
+[slap-parameter]
+title =
+download_url =
+port = 8081

software/html5as/software.cfg

 [buildout]
 extends =
+# buildout.hash.cfg is used for automated hash calculation of managed
+# instance files by calling update-hash
+  buildout.hash.cfg
 # "slapos" stack describes basic things needed for 99.9% of SlapOS Software
 # Releases
   ../../stack/slapos.cfg
+# Extend monitoring stack to provide necessary tools for monitoring
+  ../../stack/monitor/buildout.cfg
 # Extend here component profiles, like openssl, apache, mariadb, curl...
 # Or/and extend a stack (lamp, tomcat) that does most of the work for you
 # In this example we extend needed components for html5as.
   ../../component/nginx/buildout.cfg
   ../../component/dash/buildout.cfg
+  ../../component/tar/buildout.cfg
+  ../../component/curl/buildout.cfg
 
 parts =
 # Call installation of slapos.cookbook egg defined in stack/slapos.cfg (needed
@@ -24,46 +31,50 @@ parts =
 recipe = slapos.recipe.template:jinja2
 rendered = ${buildout:directory}/template.cfg
 template = ${:_profile_base_location_}/${:filename}
-filename = instance.cfg.in
-md5sum = 5a6ebc126bcad3cdff1b51fb51f82a35
 mode = 0644
 context =
   section buildout buildout
   key nginx_location nginx:location
   key dash_location dash:location
+  key curl_location curl:location
+  key tar_location tar:location
   key template_nginx_conf_target template_nginx_conf:target
   key template_mime_types_target template_mime_types:target
   key template_launcher_target template_launcher:target
   key template_instance_html5as_target instance_html5as:target
+  key template_index_html_target template_index_html:target
+  key template_graceful_target template_graceful:target
+# Monitor stack also provides a template for the instance
+  key template_monitor monitor2-template:rendered
 
-# Download instance_html5as.cfg.in
-[instance_html5as]
+# Have one shared section to define the default behaviour to download
+# templates. Sections inheriting from this one won't need to redefine
+# shared parameters
+[download-base]
 recipe = slapos.recipe.build:download
 url = ${:_profile_base_location_}/${:_update_hash_filename_}
-_update_hash_filename_ = instance_html5as.cfg.in
-md5sum = 4a8c98cc5ca141f78f14fb9cec203cb8
 mode = 0644
 
+# Download instance_html5as.cfg.in
+[instance_html5as]
+# This section inherit from download-base
+<= download-base
+# Filename and md5sum is defined in buildout.hash.cfg
+
 [template_nginx_conf]
-recipe = slapos.recipe.build:download
-url = ${:_profile_base_location_}/${:_update_hash_filename_}
-_update_hash_filename_ = templates/nginx_conf.in
-md5sum = 61dc4c82bf48563228ce4dea6c5c6319
-mode = 0644
+<= download-base
 
 [template_launcher]
-recipe = slapos.recipe.build:download
-url = ${:_profile_base_location_}/${:_update_hash_filename_}
-_update_hash_filename_ = templates/launcher.in
-md5sum = 8d4d3152f5125f73d0c3f3ea1c3699dd
-mode = 0644
+<= download-base
 
 [template_mime_types]
-recipe = slapos.recipe.build:download
-url = ${:_profile_base_location_}/${:_update_hash_filename_}
-_update_hash_filename_ = templates/mime_types.in
-md5sum = 4ef94a7b458d885cd79ba0b930a5727e
-mode = 0644
+<= download-base
+
+[template_index_html]
+<= download-base
+
+[template_graceful]
+<= download-base
 
 # Pin versions of eggs used that are not already pinned by stack/slapos.cfg
 [versions]

software/html5as/templates/graceful.in

+#! {{ param_html5as['path_shell'] }}
+# BEWARE: This file is operated by slapos node
+# BEWARE: It will be overwritten automatically
+
+# Run graceful
+exec kill -s SIGHUP $(cat {{ param_html5as['path_pid'] }})

software/html5as/templates/index.html.in

+{% if title %}
+<h1>{{ title }}</h1>
+{% endif %}
+<p>Hello World</p>

software/html5as/templates/launcher.in

 #! {{ param_html5as['path_shell'] }}
-# BEWARE: This file is operated by slapgrid
+# BEWARE: This file is operated by slapos node
 # BEWARE: It will be overwritten automatically
 
 # Run nginx

software/htmlvalidatorserver/buildout.hash.cfg

@@ -18,7 +18,7 @@ md5sum = fd2562b9ac0c52a2e9cc02e3ef3cef78
 
 [template-tomcat-service]
 filename = template-tomcat-service.sh.in
-md5sum = 09803fb71404edbccb32c44a0040dae4
+md5sum = 87781e6bcb523bb8434888d5f984f36c
 
 [template-validator]
 filename = instance-validator.cfg.in

software/htmlvalidatorserver/template-tomcat-service.sh.in

 #!${dash-output:dash}
-# BEWARE: This file is operated by slapgrid
+# BEWARE: This file is operated by slapos node
 # BEWARE: It will be overwritten automatically
 
 export JRE_HOME=${java-re-8:location}

software/jscrawler/buildout.hash.cfg

@@ -14,13 +14,13 @@
 # not need these here).
 
 [instance]
-md5sum = 6c17361a49cfc47564063b867aab6e8c
 filename = instance.cfg.in
+md5sum = 6c17361a49cfc47564063b867aab6e8c
 
 [template-jscrawler]
-md5sum = 8fa425275120e8ba5c466aff00b48e7b
 filename = instance-jscrawler.cfg.jinja2.in
+md5sum = bdf9e67077cd5c3c140974bcc56422ad
 
 [template-jscrawler-builder]
-md5sum = c5e8f8b3983d5f572a564b34fa0f7499
 filename = template-jscrawler.builder.sh.in
+md5sum = c5e8f8b3983d5f572a564b34fa0f7499

software/jscrawler/instance-jscrawler.cfg.jinja2.in

@@ -98,7 +98,7 @@ command = ${jscrawler-build-wrapper:rendered}
 
 [publish-connection-information]
 <= monitor-publish
-recipe = slapos.cookbook:publish.serialised
+recipe = slapos.cookbook:publish
 url = ${request-jscrawler-frontend:connection-secure_access}
 
 [buildout]

software/jstestnode/buildout.hash.cfg

@@ -19,7 +19,7 @@ md5sum = 342fcf3fc18e33fa4f940518dab6af5c
 
 [template-nginx-service]
 filename = template-nginx-service.sh.in
-md5sum = 5c94d952305552dcbeaeaeb27dd28f3b
+md5sum = 458870b70c33a1621b68961ae2372ad5
 
 [template-nginx-configuration]
 filename = template-nginx.cfg.in
@@ -27,4 +27,4 @@ md5sum = 98faa5ad8cfb23a11d97a459078a1d05
 
 [template-runTestSuite]
 filename = runTestSuite.in
-md5sum = 73ef758825563b7a6a1d660d4d5229b8
+md5sum = 3e2fbde4b6a1cc202b5fd2a0f14413fd

software/jstestnode/runTestSuite.in

 #!${buildout:directory}/bin/${eggs:interpreter}
-# BEWARE: This file is operated by slapgrid
+# BEWARE: This file is operated by slapos node
 # BEWARE: It will be overwritten automatically
 """
   Script to run jIO/renderJS test suite using Nexedi's test node framework.

software/jstestnode/template-nginx-service.sh.in

 #!${dash-output:dash}
-# BEWARE: This file is operated by slapgrid
+# BEWARE: This file is operated by slapos node
 # BEWARE: It will be overwritten automatically
 
 if [ ! -e $${nginx-configuration:ssl_crt} ]

software/jupyter/buildout.hash.cfg

@@ -19,7 +19,7 @@ md5sum = de37ec3d4adb0be4c67bcc7397f27c91
 
 [instance-jupyter]
 filename = instance-jupyter.cfg.in
-md5sum = 9340498841caa5771f40f8c9e561eacd
+md5sum = cbc90e517ae3680ab8bef04c6f503af5
 
 [jupyter-notebook-config]
 filename = jupyter_notebook_config.py.jinja

software/jupyter/instance-jupyter-input-schema.json

+{
+  "$schema": "http://json-schema.org/draft-04/schema#",
+  "description": "Parameters to instantiate Jupyter",
+  "properties": {
+    "frontend-instance-guid": {
+      "title": "Frontend Instance ID",
+      "description": "Unique identifier of the frontend instance, like \"SOFTINST-11031\".",
+      "type": "string"
+    },
+    "frontend-software-type": {
+      "title": "Frontend Software Type",
+      "description": "Type of the frontend instance, like \"frontend\".",
+      "type": "string",
+      "default": "RootSoftwareInstance"
+    },
+    "frontend-software-url": {
+      "title": "Frontend Software URL",
+      "description": "Software Release URL of the frontend instance, like \"http://example.com/path/to/software.cfg\".",
+      "type": "string",
+      "format": "uri",
+      "default": "http://git.erp5.org/gitweb/slapos.git/blob_plain/HEAD:/software/apache-frontend/software.cfg"
+    },
+    "frontend-additional-instance-guid": {
+      "title": "Additional Frontend Instance ID",
+      "description": "Unique identifier of the additional frontend instance, like \"SOFTINST-11031\", if empty won't be requested.",
+      "type": "string"
+    },
+    "frontend-additional-software-type": {
+      "title": "Additional Frontend Software Type",
+      "description": "Type of the frontend instance, like \"frontend\".",
+      "type": "string",
+      "default": "RootSoftwareInstance"
+    },
+    "frontend-additional-software-url": {
+      "title": "Additional Frontend Software URL",
+      "description": "Software Release URL of the frontend instance, like \"http://example.com/path/to/software.cfg\".",
+      "type": "string",
+      "format": "uri",
+      "default": "http://git.erp5.org/gitweb/slapos.git/blob_plain/HEAD:/software/apache-frontend/software.cfg"
+    }
+  }
+}

software/jupyter/instance-jupyter-output-schema.json

+{
+  "$schema": "http://json-schema.org/draft-04/schema#",
+  "description": "Connection parameters for the Jupyter Notebook",
+  "properties": {
+    "url": {
+      "description": "Jupyter Notebook URL",
+      "type": "string"
+    },
+    "jupyterlab-url": {
+      "description": "JupyterLab URL",
+      "type": "string"
+    },
+    "password": {
+      "description": "Password",
+      "type": "string"
+    }
+  }
+}

software/jupyter/instance-jupyter.cfg.in

@@ -51,6 +51,10 @@ frontend-additional-software-url = http://git.erp5.org/gitweb/slapos.git/blob_pl
 frontend-additional-instance-guid =
 frontend-additional-instance-name = Jupyter Frontend Additional
 
+{% for k, v in slapparameter_dict.items() -%}
+{{ k }} = {{ v }}
+{% endfor -%}
+
 [dynamic-jinja2-template-base]
 recipe = slapos.recipe.template:jinja2
 mode = 0644
@@ -171,7 +175,7 @@ instance-configuration =
   raw jupyter-password ${jupyter-password:passwd}
 
 [publish-connection-parameter]
-recipe = slapos.cookbook:publish.serialised
+recipe = slapos.cookbook:publish
 jupyter-classic-url = ${request-slave-frontend:connection-secure_access}/tree
 url = ${:jupyter-classic-url}
 jupyterlab-url = ${request-slave-frontend:connection-secure_access}/lab

software/jupyter/software.cfg.json

+{
+  "name": "Jupyter",
+  "description": "Jupyter Notebook",
+  "serialisation": "json-in-xml",
+  "software-type": {
+    "default": {
+      "title": "Default",
+      "software-type": "default",
+      "description": "Default",
+      "request": "instance-jupyter-input-schema.json",
+      "response": "instance-jupyter-ouput-schema.json",
+      "index": 1
+    }
+  }
+}

software/jupyter/test/test.py

@@ -30,6 +30,8 @@ import http.client
 import json
 import os
 import requests
+import sqlite3
+
 
 from slapos.testing.testcase import makeModuleSetUpAndTestCaseClass
 
@@ -41,12 +43,7 @@ setUpModule, InstanceTestCase = makeModuleSetUpAndTestCaseClass(
 class TestJupyter(InstanceTestCase):
 
   def test(self):
-    parameter_dict = self.computer_partition.getConnectionParameterDict()
-    self.assertTrue('_' in parameter_dict)
-    try:
-      connection_dict = json.loads(parameter_dict['_'])
-    except Exception as e:
-      self.fail("Can't parse json in %s, error %s" % (parameter_dict['_'], e))
+    connection_dict = self.computer_partition.getConnectionParameterDict()
 
     self.assertTrue('password' in connection_dict)
     password = connection_dict['password']
@@ -85,29 +82,6 @@ class TestJupyter(InstanceTestCase):
     )
 
 
-class TestJupyterPassword(InstanceTestCase):
-  def test(self):
-    parameter_dict = self.computer_partition.getConnectionParameterDict()
-    self.assertTrue('_' in parameter_dict)
-    try:
-      connection_dict = json.loads(parameter_dict['_'])
-    except Exception as e:
-      self.fail("Can't parse json in %s, error %s" % (parameter_dict['_'], e))
-
-    url = connection_dict['url']
-    with requests.Session() as s:
-      resp = s.get(url, verify=False)
-      result = s.post(
-        resp.url,
-        verify = False,
-        data={"_xsrf": s.cookies["_xsrf"], "password": connection_dict['password']}
-      )
-      self.assertEqual(
-        [http.client.OK, url],
-        [result.status_code, result.url]
-      )
-
-
 class TestJupyterAdditional(InstanceTestCase):
 
   @classmethod
@@ -117,12 +91,7 @@ class TestJupyterAdditional(InstanceTestCase):
     }
 
   def test(self):
-    parameter_dict = self.computer_partition.getConnectionParameterDict()
-    self.assertTrue('_' in parameter_dict)
-    try:
-      connection_dict = json.loads(parameter_dict['_'])
-    except Exception as e:
-      self.fail("Can't parse json in %s, error %s" % (parameter_dict['_'], e))
+    connection_dict = self.computer_partition.getConnectionParameterDict()
 
     result = requests.get(
       connection_dict['url'], verify=False, allow_redirects=False)
@@ -169,3 +138,139 @@ class TestJupyterAdditional(InstanceTestCase):
       [http.client.FOUND, True, '/login?next=%2Flab'],
       [result.status_code, result.is_redirect, result.headers['Location']]
     )
+
+
+class TestJupyterPassword(InstanceTestCase):
+  def test(self):
+    connection_dict = self.computer_partition.getConnectionParameterDict()
+
+    url = connection_dict['url']
+    with requests.Session() as s:
+      resp = s.get(url, verify=False)
+      result = s.post(
+        resp.url,
+        verify = False,
+        data={"_xsrf": s.cookies["_xsrf"], "password": connection_dict['password']}
+      )
+      self.assertEqual(
+        [http.client.OK, url],
+        [result.status_code, result.url]
+      )
+
+
+class SelectMixin(object):
+  def sqlite3_connect(self):
+    sqlitedb_file = os.path.join(
+      os.path.abspath(
+        os.path.join(
+          self.slap.instance_directory, os.pardir
+        )
+      ), 'var', 'proxy.db'
+    )
+    return sqlite3.connect(sqlitedb_file)
+
+  def select(self, fields, table, where={}):
+    connection = self.sqlite3_connect()
+
+    def dict_factory(cursor, row):
+      d = {}
+      for idx, col in enumerate(cursor.description):
+        d[col[0]] = row[idx]
+      return d
+    connection.row_factory = dict_factory
+    cursor = connection.cursor()
+
+    condition = " AND ".join("%s='%s'" % (k, v) for k, v in where.items())
+    cursor.execute(
+      "SELECT %s FROM %s%s"
+      % (
+        ", ".join(fields),
+        table,
+        " WHERE %s" % condition if where else "",
+      )
+    )
+    return cursor.fetchall()
+
+
+class TestJupyterCustomFrontend(SelectMixin, InstanceTestCase):
+  instance_parameter_dict = {}
+  frontend_software_url = 'hello://frontend.url'
+  frontend_software_type = 'hello-type'
+  frontend_instance_name = 'Hello Frontend'
+
+  @classmethod
+  def getInstanceParameterDict(cls):
+    return cls.instance_parameter_dict
+
+  def test(self):
+
+    # create a fake master instance for the frontend slave request
+    r = self.slap.request(
+        software_release=self.frontend_software_url,
+        software_type=self.frontend_software_type,
+        partition_reference= "Fake master instance",
+    )
+
+    # update the request parameters of the test instance
+    self.instance_parameter_dict.update({
+      'frontend-software-url': self.frontend_software_url,
+      'frontend-software-type': self.frontend_software_type,
+      'frontend-instance-name': self.frontend_instance_name,
+      'frontend-instance-guid': r._partition_id,
+    })
+    self.requestDefaultInstance()
+
+    # wait for the instance to converge to the new state
+    try:
+      self.slap.waitForInstance()
+    except Exception:
+      pass
+
+    selection = self.select(fields=["*"], table = "slave14", where = {"hosted_by": r._partition_id})
+
+    self.assertEqual(len(selection), 1)
+
+    # clean up the fake master
+    r.destroyed()
+
+
+class TestJupyterCustomAdditional(SelectMixin, InstanceTestCase):
+  instance_parameter_dict = {}
+  frontend_additional_software_url = 'hello://frontend.url'
+  frontend_additional_software_type = 'hello-type'
+  frontend_additional_instance_name = 'Hello Frontend'
+
+  @classmethod
+  def getInstanceParameterDict(cls):
+    return cls.instance_parameter_dict
+
+  def test(self):
+
+    # create a fake master instance for the frontend slave request
+    r = self.slap.request(
+        software_release=self.frontend_additional_software_url,
+        software_type=self.frontend_additional_software_type,
+        partition_reference= "Fake master instance",
+    )
+
+    # update the request parameters of the test instance
+    self.instance_parameter_dict.update({
+      'frontend-additional-software-url': self.frontend_additional_software_url,
+      'frontend-additional-software-type': self.frontend_additional_software_type,
+      'frontend-additional-instance-name': self.frontend_additional_instance_name,
+      'frontend-additional-instance-guid': r._partition_id,
+    })
+    self.requestDefaultInstance()
+
+    # wait for the instance to converge to the new state
+    try:
+      self.slap.waitForInstance()
+    except Exception:
+      pass
+
+    selection = self.select(fields=["*"], table = "slave14", where = {"hosted_by": r._partition_id})
+
+    self.assertEqual(len(selection), 1)
+
+    # clean up the fake master
+    r.destroyed()

software/kvm/buildout.hash.cfg

@@ -55,11 +55,11 @@ md5sum = b7e87479a289f472b634a046b44b5257
 
 [template-kvm-run]
 filename = template/template-kvm-run.in
-md5sum = a97b8462955dd422a30fbe02d6906172
+md5sum = cc1b748c82ca15022744558dab05995c
 
 [template-kvm-controller]
 filename = template/kvm-controller-run.in
-md5sum = 3827b4f7624de190cf5f5d37e3b72e86
+md5sum = 36c6e4fd3550639f534e7f55b2ca9a3f
 
 [template-apache-conf]
 filename = template/apache.conf.in
@@ -83,7 +83,7 @@ md5sum = d657884d02105deffddee0edae4b50a6
 
 [image-download-controller]
 _update_hash_filename_ = template/image-download-controller.py
-md5sum = 7e4b54f8172c364bd12d28b07f8b1600
+md5sum = 9c67058edcc4edae0b57956c0932a9fc
 
 [image-download-config-creator]
 _update_hash_filename_ = template/image-download-config-creator.py

software/kvm/instance-kvm-cluster-input-schema.json

@@ -505,7 +505,7 @@
             },
             "boot-image-url-list": {
               "title": "Boot image list",
-              "description": "The list shall be list of direct URLs to images, followed by hash (#), then by image MD5SUM. Each image shall appear on newline, like: \"https://example.com/image.iso#06226c7fac5bacfa385872a19bb99684<newline>https://example.com/another-image.iso#31b40d58b18e038498ddb46caea1361c\". They will be provided in KVM image list according to the order on the list. After updating the list, the instance has to be restarted to refresh it. Amount of images is limited to 4, and one image can be maximum 5G. Image will be downloaded and checked against its MD5SUM 4 times, then it will be considered as impossible to download with given MD5SUM. Each image has to be downloaded in time shorter than 4 hours, so in case of very slow images to access, it can take up to 16 hours to download all of them. Note: The instance has to be restarted in order to update the list of available images in the VM. Note: Maximum 3 ISOs are supported.",
+              "description": "The list shall be list of direct URLs to images, followed by hash (#), then by image MD5SUM. Each image shall appear on newline, like: \"https://example.com/image.iso#06226c7fac5bacfa385872a19bb99684<newline>https://example.com/another-image.iso#31b40d58b18e038498ddb46caea1361c\". They will be provided in KVM image list according to the order on the list. After updating the list, the instance has to be restarted to refresh it. Amount of images is limited to 4, and one image can be maximum 10G. Image will be downloaded and checked against its MD5SUM 4 times, then it will be considered as impossible to download with given MD5SUM. Each image has to be downloaded in time shorter than 4 hours, so in case of very slow images to access, it can take up to 16 hours to download all of them. Note: The instance has to be restarted in order to update the list of available images in the VM. Note: Maximum 3 ISOs are supported.",
               "type": "string",
               "textarea": true
             },

software/kvm/instance-kvm-input-schema.json

@@ -368,7 +368,7 @@
     },
     "boot-image-url-list": {
       "title": "Boot image list",
-      "description": "The list shall be list of direct URLs to images, followed by hash (#), then by image MD5SUM. Each image shall appear on newline, like: \"https://example.com/image.iso#06226c7fac5bacfa385872a19bb99684<newline>https://example.com/another-image.iso#31b40d58b18e038498ddb46caea1361c\". They will be provided in KVM image list according to the order on the list. After updating the list, the instance has to be restarted to refresh it. Amount of images is limited to 4, and one image can be maximum 5G. Image will be downloaded and checked against its MD5SUM 4 times, then it will be considered as impossible to download with given MD5SUM. Each image has to be downloaded in time shorter than 4 hours, so in case of very slow images to access, it can take up to 16 hours to download all of them. Note: The instance has to be restarted in order to update the list of available images in the VM. Note: Maximum 3 ISOs are supported.",
+      "description": "The list shall be list of direct URLs to images, followed by hash (#), then by image MD5SUM. Each image shall appear on newline, like: \"https://example.com/image.iso#06226c7fac5bacfa385872a19bb99684<newline>https://example.com/another-image.iso#31b40d58b18e038498ddb46caea1361c\". They will be provided in KVM image list according to the order on the list. After updating the list, the instance has to be restarted to refresh it. Amount of images is limited to 4, and one image can be maximum 10G. Image will be downloaded and checked against its MD5SUM 4 times, then it will be considered as impossible to download with given MD5SUM. Each image has to be downloaded in time shorter than 4 hours, so in case of very slow images to access, it can take up to 16 hours to download all of them. Note: The instance has to be restarted in order to update the list of available images in the VM. Note: Maximum 3 ISOs are supported.",
       "type": "string",
       "textarea": true
     },

software/kvm/software.cfg

@@ -50,7 +50,7 @@ eggs =
   ${python-cffi:egg}
   ${lxml-python:egg}
   websockify
-  slapos.cookbook
+  ${slapos-cookbook:eggs}
   erp5.util
 # BBB: eggs used as recipe should be kept otherwise sections depending
 # on it can't be uninstalled

software/kvm/template/image-download-controller.py

@@ -93,7 +93,7 @@ if __name__ == "__main__":
         '--location',  # follow redirects
         '--no-progress-meter',  # do not tell too much
         '--max-time', '14400',  # maximum time for download is 4 hours
-        '--max-filesize', '5368709120',  # maximum 5G for an image
+        '--max-filesize', '10737418240',  # maximum 10G for an image
         '--output', destination_tmp, image['url']],
         stderr=subprocess.STDOUT)
     except subprocess.CalledProcessError as e:

software/kvm/template/kvm-controller-run.in

 #!{{ parameter_dict.get('python-path') }}
-# BEWARE: This file is operated by slapgrid
+# BEWARE: This file is operated by slapos node
 # BEWARE: It will be overwritten automatically
 
 # Echo client program

software/kvm/template/template-kvm-run.in

 #!{{ parameter_dict.get('python-path') }}
-# BEWARE: This file is operated by slapgrid
+# BEWARE: This file is operated by slapos node
 # BEWARE: It will be overwritten automatically
 
 import hashlib

software/nginx-push-stream/template-nginx-service.sh.in

 #!${dash-output:dash}
-# BEWARE: This file is operated by slapgrid
+# BEWARE: This file is operated by slapos node
 # BEWARE: It will be overwritten automatically
 
 exec ${nginx-push-stream-output:nginx} \

software/slaprunner/buildout.hash.cfg

@@ -46,7 +46,7 @@ md5sum = 1d983937ad0909b3ef4d35d190645635
 
 [template_launcher]
 filename = launcher.in
-md5sum = 525e37ea8b2acf6209869999b15071a6
+md5sum = 7a2f5e25a818cb29964666ada7852a5c
 
 [template-slapos-cfg]
 filename = template/slapos.cfg.in

software/slaprunner/launcher.in

 #! {{ param_nginx_frontend['path_shell'] }}
-# BEWARE: This file is operated by slapgrid
+# BEWARE: This file is operated by slapos node
 # BEWARE: It will be overwritten automatically
 
 # Run nginx

software/srslte/instance-srslte-output-schema.json

+{
+  "$schema": "http://json-schema.org/draft-07/schema#",
+  "description": "Values returned by srsLTE instanciation",
+  "properties": {},
+  "type": "object"
+}

software/srslte/software.cfg.json

@@ -8,7 +8,7 @@
       "software-type": "default",
       "description": "Default",
       "request": "instance-srslte-input-schema.json",
-      "response": "instance-srslte-schema.json",
+      "response": "instance-srslte-output-schema.json",
       "index": 1
     }
   }

software/theia/buildout.hash.cfg

@@ -15,7 +15,7 @@
 
 [instance]
 filename = instance.cfg.in
-md5sum = a3e4cb7d28daa7816f04359c8aa3445b
+md5sum = 397a2e0ac0d7f8813e0b594af6785300
 
 [yarn.lock]
 filename = yarn.lock

software/theia/instance-input-schema.json

+{
+  "$schema": "http://json-schema.org/draft-04/schema#",
+  "description": "Parameters to instantiate Theia",
+  "additionalProperties": false,
+  "properties": {
+    "autorun": {
+      "title": "autorun",
+      "description": "State of the autorun service",
+      "type": "string",
+      "enum": [
+        "running",
+        "stopped",
+        "disabled"
+      ],
+      "default": "running"
+    },
+    "embedded-sr": {
+      "title": "Embedded Software URL",
+      "description": "Optional URL of a software to be embedded",
+      "type": "string"
+    },
+    "embedded-sr-type": {
+      "title": "Embedded Software Type",
+      "description": "Type of the optional embedded software",
+      "type": "string"
+    }
+  }
+}

software/theia/instance-output-schema.json

+{
+  "$schema": "http://json-schema.org/draft-04/schema#",
+  "description": "Values returned by Theia instanciation",
+  "properties": {
+    "url": {
+      "description": "Theia Frontend URL",
+      "type": "string"
+    },
+    "username": {
+      "description": "Theia username",
+      "type": "string"
+    },
+    "password": {
+      "description": "Theia password",
+      "type": "string"
+    }
+  },
+  "type": "object"
+}

software/theia/instance.cfg.in

@@ -316,6 +316,8 @@ template =
       $${slapos-standalone-config:ipv6} \
       $${slapos-standalone-config:port} \
       $${slapos-standalone-config:computer-id} \
+      --sr='$${instance-parameter:configuration.embedded-sr}' \
+      --srtype='$${instance-parameter:configuration.embedded-sr-type}' \
       $${slap-connection:server-url} \
       $${slap-connection:computer-id} \
       $${slap-connection:partition-id} \
@@ -338,6 +340,7 @@ instance-promises =
   $${frontend-listen-promise:name}
   $${apache-frontend-url-available-promise:name}
   $${slapos-standalone-listen-promise:name}
+  $${slapos-autorun-promise:name}
 
 [theia-listen-promise]
 <= monitor-promise-base
@@ -368,6 +371,14 @@ name = standalone-listen-promise.py
 config-hostname = $${slapos-standalone-instance:hostname}
 config-port = $${slapos-standalone-instance:port}
 
+[slapos-autorun-promise]
+<= monitor-promise-base
+module = check_service_state
+# XXX promise plugins can not contain "slapos" in their names
+name = autorun-state-promise.py
+config-service = $${slapos-autorun:service-name}
+config-expect = $${slapos-autorun:autorun}
+
 [apache-frontend]
 <= slap-connection
 recipe = slapos.cookbook:requestoptional
@@ -382,6 +393,7 @@ config-websocket-path-list = /services /file-upload
 return = domain secure_access
 
 [publish-connection-parameter]
+<= monitor-publish
 recipe = slapos.cookbook:publish
 url = $${apache-frontend:connection-secure_access}
 username = $${frontend-instance-password:username}
@@ -394,6 +406,21 @@ partition = $${slap-connection:partition-id}
 url = $${slap-connection:server-url}
 key = $${slap-connection:key-file}
 cert = $${slap-connection:cert-file}
+configuration.autorun = running
+configuration.embedded-sr =
+configuration.embedded-sr-type =
+
+[slapos-autorun]
+recipe = plone.recipe.command
+command =
+  case $${:autorun} in
+    ( running ) ${buildout:bin-directory}/supervisorctl -c $${:supervisor-conf} start $${:service-name};;
+    ( stopped ) ${buildout:bin-directory}/supervisorctl -c $${:supervisor-conf} stop $${:service-name};;
+  esac
+update-command = $${:command}
+service-name = slapos-node-auto
+supervisor-conf = $${directory:runner}/etc/supervisord.conf
+autorun = $${instance-parameter:configuration.autorun}
 
 [slapos-repository]
 recipe = slapos.recipe.build:gitclone

software/theia/software.cfg

@@ -71,6 +71,8 @@ initialization =
   parser.add_argument('ipv6')
   parser.add_argument('server_port', type=int)
   parser.add_argument('computer_id')
+  parser.add_argument('--sr')
+  parser.add_argument('--srtype')
   forwarded_arguments = parser.add_argument_group('forwarded')
   forwarded_arguments.add_argument('master_url')
   forwarded_arguments.add_argument('computer')
@@ -119,6 +121,11 @@ initialization =
   except slapos.slap.standalone.SlapOSNodeCommandError as e:
     print("Error instanciating: {}".format(e))
 
+  if args.sr:
+    print("Supplying and Requesting Embedded Software {}".format(args.sr))
+    standalone.supply(args.sr)
+    standalone.request(args.sr, "Embedded Instance", args.srtype or None)
+
   quit_requested = []
   def signal_handler(signum, frame):
     print("Signal {signum} received".format(signum=signum))

software/theia/software.cfg.json

+{
+  "name": "Theia",
+  "description": "Theia",
+  "serialisation": "xml",
+  "software-type": {
+    "default": {
+      "title": "Default",
+      "software-type": "default",
+      "description": "Default",
+      "request": "instance-input-schema.json",
+      "response": "instance-ouput-schema.json",
+      "index": 1
+    }
+  }
+}

software/theia/test/test.py

@@ -38,6 +38,7 @@ from six.moves.urllib.parse import urlparse, urljoin
 import pexpect
 import psutil
 import requests
+import sqlite3
 
 from slapos.testing.testcase import makeModuleSetUpAndTestCaseClass
 from slapos.grid.svcbackend import getSupervisorRPC
@@ -184,3 +185,59 @@ class TestTheiaEmbeddedSlapOSShutdown(SlapOSInstanceTestCase):
 
     # the supervisor controlling instances is also stopped
     self.assertFalse(embedded_slapos_process.is_running())
+
+
+class SQLiteDB(object):
+  def __init__(self, sqlitedb_file):
+    self.sqlitedb_file = sqlitedb_file
+
+  def select(self, fields, table, where={}):
+    connection = sqlite3.connect(self.sqlitedb_file)
+
+    def dict_factory(cursor, row):
+      d = {}
+      for idx, col in enumerate(cursor.description):
+        d[col[0]] = row[idx]
+      return d
+    connection.row_factory = dict_factory
+    cursor = connection.cursor()
+
+    condition = " AND ".join("%s='%s'" % (k, v) for k, v in where.items())
+    cursor.execute(
+      "SELECT %s FROM %s%s"
+      % (
+        ", ".join(fields),
+        table,
+        " WHERE %s" % condition if where else "",
+      )
+    )
+    return cursor.fetchall()
+
+
+class TestTheiaWithSR(SlapOSInstanceTestCase):
+  __partition_reference__ = 'T' # for sockets in included slapos
+
+  srurl = 'bogus/software.cfg'
+  srtype = 'bogus'
+
+  @classmethod
+  def getInstanceParameterDict(cls):
+    return {
+      'embedded-sr': cls.srurl,
+      'embedded-sr-type': cls.srtype,
+    }
+
+  def test(self):
+    db = SQLiteDB(os.path.join(self.computer_partition_root_path, 'srv', 'runner', 'var', 'proxy.db'))
+    supplied = db.select(
+      fields=["*"],
+      table = "software14",
+      where={'url': self.srurl}
+    )
+    self.assertEqual(len(supplied), 1)
+    requested = db.select(
+      fields=["*"],
+      table = "partition14",
+      where={'software_release': self.srurl, 'software_type': self.srtype}
+    )
+    self.assertEqual(len(requested), 1)

software/unstable/pyodide/buildout.hash.cfg

@@ -18,4 +18,4 @@ md5sum = e4e96f0f08b8de6807a24f43fce34b40
 
 [template-runTestSuite]
 filename = runTestSuite.in
-md5sum = 28fb6c00c7568a6349e9ff1a9b09fc53
\ No newline at end of file
+md5sum = 77cad4d124464c644f17ecb076d16786
\ No newline at end of file

software/unstable/pyodide/runTestSuite.in

 #!${buildout:directory}/bin/${eggs:interpreter}
-# BEWARE: This file is operated by slapgrid
+# BEWARE: This file is operated by slapos node
 # BEWARE: It will be overwritten automatically
 """
   Script to run Cython test suite using Nexedi's test node framework.
@@ -120,4 +120,4 @@ def main():
     # XXX: inform test node master of error
     raise EnvironmentError(result)
 if __name__ == "__main__":
-    main()
\ No newline at end of file
+    main()

stack/erp5/buildout.hash.cfg

@@ -18,7 +18,7 @@ md5sum = 85ce1e2f3d251aa435fef8118dca8a63
 
 [mariadb-slow-query-report-script]
 filename = mysql-querydigest.sh.in
-md5sum = 7b14d2b81f2c864e47682ddb03b1b663
+md5sum = 6457ab192d709aa2c9014e9a3e91ca20
 
 [mariadb-start-clone-from-backup]
 filename = instance-mariadb-start-clone-from-backup.sh.in
@@ -66,7 +66,7 @@ md5sum = 0969fbb25b05c02ef3c2d437b2f4e1a0
 
 [template-run-zelenium]
 filename = run-zelenium-test.py.in
-md5sum = cc19560b9400cecbd23064d55c501eec
+md5sum = 7a14019abf48ca100eb94d9add20f5ae
 
 [template]
 filename = instance.cfg.in

stack/erp5/mysql-querydigest.sh.in

 #!{{dash}}
-# BEWARE: This file is operated by slapgrid
+# BEWARE: This file is operated by slapos node
 
 set -e
 

stack/erp5/run-zelenium-test.py.in

 #!{{ bin_path }}
-# BEWARE: This file is operated by slapgrid
+# BEWARE: This file is operated by slapos node
 # BEWARE: It will be overwritten automatically
 
 import argparse, os, sys, traceback

stack/slapos.cfg

@@ -92,6 +92,11 @@ recipe = zc.recipe.egg
 eggs = slapos.cookbook
 depends = ${slapos-cookbook-dependencies:eggs}
 
+[pyrsistent]
+recipe = zc.recipe.egg:custom
+egg = pyrsistent
+setup-eggs = pytest-runner
+
 [slapos-cookbook-dependencies]
 recipe = zc.recipe.egg
 eggs =
@@ -101,6 +106,7 @@ eggs =
   ${python-cachecontrol:egg}
   ${python-cliff:egg}
   ${python-cryptography:egg}
+  ${pyrsistent:egg}
   ${jsonschema:egg}
 
 # slapos.toolbox containing utilities
@@ -192,7 +198,7 @@ setuptools-dso = 1.7
 rubygemsrecipe  = 0.3.0
 six = 1.12.0
 slapos.cookbook = 1.0.171
-slapos.core = 1.6.3
+slapos.core = 1.6.4
 slapos.extension.strip = 0.4
 slapos.extension.shared = 1.0
 slapos.libnetworkcache = 0.20
@@ -200,7 +206,7 @@ slapos.rebootstrap = 4.5
 slapos.recipe.build = 0.46
 slapos.recipe.cmmi = 0.16
 slapos.recipe.template = 4.5
-slapos.toolbox = 0.116
+slapos.toolbox = 0.117
 stevedore = 1.21.0
 subprocess32 = 3.5.3
 unicodecsv = 0.14.1
@@ -262,18 +268,28 @@ erp5.util = 0.4.69
 feedparser = 5.2.1
 
 # Required by:
-# jsonschema==2.6.0
+# jsonschema==3.0.2
 functools32 = 3.2.3.post2
 
 # Required by:
-# jsonschema==2.6.0
+# jsonschema==3.0.2
+attrs = 18.2.0
+
+# Required by:
+# jsonschema==3.0.2
+pyrsistent = 0.16.1
+
+pytest-runner = 5.2.0
+
+# Required by:
+# jsonschema==3.0.2
 setuptools-scm = 3.5.0
 
 ipaddress = 1.0.23
 
 # Required by:
 # slapos.cookbook==1.0.143
-jsonschema = 2.6.0
+jsonschema = 3.0.2
 
 # Required by:
 # slapos.toolbox==0.94