Merge remote-tracking branch 'origin' into zope4

component/haproxy/buildout.cfg

@@ -13,8 +13,8 @@ parts = haproxy
 [haproxy]
 recipe = slapos.recipe.cmmi
 shared = true
-url = http://www.haproxy.org/download/2.0/src/haproxy-2.0.17.tar.gz
-md5sum = 786a967c73cc1455c938d42fbe333bfe
+url = http://www.haproxy.org/download/2.0/src/haproxy-2.0.20.tar.gz
+md5sum = 9f85ea9e6fd7d49a11cdc4c6269e10dd
 configure-command = true
 # for Linux kernel 2.6.28 and above, we use "linux-glibc" as the TARGET,
 # otherwise use "generic".

component/trafficserver/buildout.cfg

@@ -28,7 +28,7 @@ md5sum = 4f4d1e7de19c77157be0c2a825b31026
 shared = true
 patch-options = -p1
 patches =
-  ${:_profile_base_location_}/trafficserver-8.1-stale-negative-cache-not-returnable.patch#e1a2f8a23f00cee1301ccf1a84e46763
+  https://github.com/apache/trafficserver/commit/254e9e22181ca369673407bd3fcd93e7287275ac.patch#1c324f76464d33d334a15a69490ccd55
 configure-options =
   --with-openssl=${openssl:location}
   --with-pcre=${pcre:location}

component/trafficserver/trafficserver-8.1-stale-negative-cache-not-returnable.patch

---- trafficserver-8.1.1/proxy/http/HttpTransact.cc.orig	2020-12-01 00:30:26.000000000 +0100
-+++ trafficserver-8.1.1/proxy/http/HttpTransact.cc	2021-01-11 11:35:41.946893735 +0100
-@@ -5752,6 +5752,7 @@
- HttpTransact::is_stale_cache_response_returnable(State *s)
- {
-   HTTPHdr *cached_response = s->cache_info.object_read->response_get();
-+  HTTPStatus cache_response_code = cached_response->status_get();
- 
-   // First check if client allows cached response
-   // Note does_client_permit_lookup was set to
-@@ -5760,6 +5761,12 @@
-   if (!s->cache_info.directives.does_client_permit_lookup) {
-     return false;
-   }
-+  // We don't serve stale negative cache.
-+  if (cache_response_code == HTTP_STATUS_INTERNAL_SERVER_ERROR || cache_response_code == HTTP_STATUS_GATEWAY_TIMEOUT ||
-+      cache_response_code == HTTP_STATUS_BAD_GATEWAY || cache_response_code == HTTP_STATUS_SERVICE_UNAVAILABLE) {
-+    TxnDebug("http_trans", "[is_stale_cache_response_returnable] stale negative cache");
-+    return false;
-+  }
-   // Spec says that we can not serve a stale document with a
-   //   "must-revalidate header"
-   // How about "s-maxage" and "no-cache" directives?

release-sr

 #!/bin/bash
 
-git fetch
+set -e
+
+git fetch --all
+
+# Go to master
+git checkout master
+
+# Clean up 1.0
+git branch -D 1.0 || echo
+
+# Checkout 1.0
+git checkout origin/1.0 -b 1.0
 
-git checkout 1.0
 git reset origin/1.0 --hard
 
 # for now this script is hardcoded to release on 1.0.x versions intentionally

software/caddy-frontend/README.caddy_frontend.rst

@@ -60,14 +60,12 @@ This is to deploy an entire frontend server with a public IPv4.  If you want to
 First, you will need to request a "master" instance of Caddy Frontend with:
 
   * A ``domain`` parameter where the frontend will be available
-  * A ``public-ipv4`` parameter to state which public IPv4 will be used
 
 like::
 
   <?xml version='1.0' encoding='utf-8'?>
   <instance>
    <parameter id="domain">moulefrite.org</parameter>
-   <parameter id="public-ipv4">xxx.xxx.xxx.xxx</parameter>
   </instance>
 
 Then, it is possible to request many slave instances (currently only from slapconsole, UI doesn't work yet) of Caddy Frontend, like::
@@ -189,10 +187,6 @@ Name of the domain to be used (example: mydomain.com). Sub domains of this domai
 
 Using the IP given by the Master Instance.  "domain" is a mandatory Parameter.
 
-public-ipv4
-~~~~~~~~~~~
-Public ipv4 of the frontend (the one Caddy will be indirectly listening to)
-
 port
 ~~~~
 Port used by Caddy. Optional parameter, defaults to 4443.

software/caddy-frontend/buildout.hash.cfg

@@ -14,7 +14,7 @@
 # not need these here).
 [template]
 filename = instance.cfg.in
-md5sum = 28bf0c4c75c028bed79fc38786831b3e
+md5sum = de69a8c408ce4f228fc22eacb7e96657
 
 [profile-common]
 filename = instance-common.cfg.in
@@ -22,15 +22,15 @@ md5sum = 5784bea3bd608913769ff9a8afcccb68
 
 [profile-caddy-frontend]
 filename = instance-apache-frontend.cfg.in
-md5sum = e8db3179e3278c6390a786cdcc947173
+md5sum = a6a626fd1579fd1d4b80ea67433ca16a
 
 [profile-caddy-replicate]
 filename = instance-apache-replicate.cfg.in
-md5sum = 2329022227099971a57f710832509153
+md5sum = 7cb8157d2b368ab3b281ea42f743eb9c
 
 [profile-slave-list]
 _update_hash_filename_ = templates/apache-custom-slave-list.cfg.in
-md5sum = 2cbcdff6fe75ec469ab7d6accd72f83c
+md5sum = 772c04c165fdae91299fd909e061f926
 
 [profile-replicate-publish-slave-information]
 _update_hash_filename_ = templates/replicate-publish-slave-information.cfg.in
@@ -46,11 +46,11 @@ md5sum = 88af61e7abbf30dc99a1a2526161128d
 
 [template-default-slave-virtualhost]
 _update_hash_filename_ = templates/default-virtualhost.conf.in
-md5sum = bd9e269130bac989faa639e0903814e2
+md5sum = a0ae858a3db8825c22d33d323392f588
 
 [template-backend-haproxy-configuration]
 _update_hash_filename_ = templates/backend-haproxy.cfg.in
-md5sum = 5c807d34198f334b143cfa9263f6bc4e
+md5sum = 0923a9227c131d2f1e11d7ddd5b15673
 
 [template-empty]
 _update_hash_filename_ = templates/empty.in
@@ -62,7 +62,7 @@ md5sum = 975177dedf677d24e14cede5d13187ce
 
 [template-trafficserver-records-config]
 _update_hash_filename_ = templates/trafficserver/records.config.jinja2
-md5sum = a6a6e21036f8c56e69ad7ac8a1dfd656
+md5sum = b99403e02d1aff471a7d5ebd0afbdb6c
 
 [template-trafficserver-storage-config]
 _update_hash_filename_ = templates/trafficserver/storage.config.jinja2

software/caddy-frontend/instance-apache-frontend.cfg.in

@@ -287,6 +287,7 @@ extra-context =
     key backend_client_caucase_url :backend-client-caucase-url
     import urlparse_module urlparse
     import furl_module furl
+    import urllib_module urllib
     key master_key_download_url :master_key_download_url
     key autocert caddy-directory:autocert
     key caddy_log_directory caddy-directory:slave-log

software/caddy-frontend/instance-apache-replicate.cfg.in

@@ -15,13 +15,13 @@
         'enable-http2-by-default',
         'global-disable-http2',
         'mpm-graceful-shutdown-timeout',
-        'public-ipv4',
         're6st-verification-url',
         'backend-connect-timeout',
         'backend-connect-retries',
         'ciphers',
         'request-timeout',
         'authenticate-to-backend',
+        'strict-transport-security',
     ]
 %}
 {% set aikc_enabled = slapparameter_dict.get('automatic-internal-kedifa-caucase-csr', 'true').lower() in TRUE_VALUES %}
@@ -135,23 +135,30 @@ context =
 {%     if backend_active_check_http_version not in ['HTTP/1.1', 'HTTP/1.0'] %}
 {%       do slave_error_list.append('Wrong backend-active-check-http-version %s' % (backend_active_check_http_version,)) %}
 {%     endif %}
-{%     set backend_active_check_timeout = (slave.get('backend-active-check-timeout') or '2') | int(False) %}
-{%     if backend_active_check_timeout in [False] or backend_active_check_timeout <= 0 %}
+{%     set backend_active_check_timeout = (slave.get('backend-active-check-timeout') or '2') | int(false) %}
+{%     if backend_active_check_timeout is false or backend_active_check_timeout <= 0 %}
 {%       do slave_error_list.append('Wrong backend-active-check-timeout %s' % (slave.get('backend-active-check-timeout'),)) %}
 {%     endif %}
-{%     set backend_active_check_interval = (slave.get('backend-active-check-interval') or '5') | int(False) %}
-{%     if backend_active_check_interval in [False] or backend_active_check_interval <= 0 %}
+{%     set backend_active_check_interval = (slave.get('backend-active-check-interval') or '5') | int(false) %}
+{%     if backend_active_check_interval is false or backend_active_check_interval <= 0 %}
 {%       do slave_error_list.append('Wrong backend-active-check-interval %s' % (slave.get('backend-active-check-interval'),)) %}
 {%     endif %}
-{%     set backend_active_check_rise = (slave.get('backend-active-check-rise') or '1') | int(False) %}
-{%     if backend_active_check_rise in [False] or backend_active_check_rise <= 0 %}
+{%     set backend_active_check_rise = (slave.get('backend-active-check-rise') or '1') | int(false) %}
+{%     if backend_active_check_rise is false or backend_active_check_rise <= 0 %}
 {%       do slave_error_list.append('Wrong backend-active-check-rise %s' % (slave.get('backend-active-check-rise'),)) %}
 {%     endif %}
-{%     set backend_active_check_fall = (slave.get('backend-active-check-fall') or '1') | int(False) %}
-{%     if backend_active_check_fall in [False] or backend_active_check_fall <= 0 %}
+{%     set backend_active_check_fall = (slave.get('backend-active-check-fall') or '1') | int(false) %}
+{%     if backend_active_check_fall is false or backend_active_check_fall <= 0 %}
 {%       do slave_error_list.append('Wrong backend-active-check-fall %s' % (slave.get('backend-active-check-fall'),)) %}
 {%     endif %}
 {%   endif %}
+{#   Check virtualhostroot-http-port and virtualhostroot-https-port #}
+{%   for key in ['virtualhostroot-http-port', 'virtualhostroot-https-port'] %}
+{%     set value = (slave.get(key) or '1') | int(false) %}
+{%     if value is false or value < 0 %}
+{%       do slave_error_list.append('Wrong %s %r' % (key, slave.get(key))) %}
+{%     endif %}
+{%   endfor %}
 {#   Check ciphers #}
 {%   set slave_cipher_list = slave.get('ciphers', '').strip().split() %}
 {%   if slave_cipher_list %}
@@ -161,6 +168,11 @@ context =
 {%       endif %}
 {%     endfor %}
 {%   endif %}
+{#   Check strict-transport-security #}
+{%     set strict_transport_security = (slave.get('strict-transport-security') or '0') | int(false) %}
+{%     if strict_transport_security is false or strict_transport_security < 0 %}
+{%       do slave_error_list.append('Wrong strict-transport-security %s' % (slave.get('strict-transport-security'),)) %}
+{%     endif %}
 {%   set custom_domain = slave.get('custom_domain') %}
 {%   if custom_domain and custom_domain in used_host_list %}
 {%      do slave_error_list.append('custom_domain %r clashes' % (custom_domain,)) %}
@@ -264,7 +276,7 @@ config-monitor-username = ${monitor-instance-parameter:username}
 config-monitor-password = ${monitor-htpasswd:passwd}
 
 software-type = {{frontend_type}}
-return = private-ipv4 public-ipv4 slave-instance-information-list monitor-base-url backend-client-csr_id-url csr_id-url csr_id-certificate backend-haproxy-statistic-url
+return = slave-instance-information-list monitor-base-url backend-client-csr_id-url csr_id-url csr_id-certificate backend-haproxy-statistic-url
 
 {#- Send only needed parameters to frontend nodes #}
 {%- set base_node_configuration_dict = {} %}

software/caddy-frontend/instance-caddy-input-schema.json

@@ -50,17 +50,6 @@
       "title": "Duration of the graceful shutdown period. Warning: Changing the parameter will result in restarting Caddy process.",
       "type": "integer"
     },
-    "nginx-domain": {
-      "description": "Base Domain for create subdomains (ie.: example2.com) for websocket, notebook and eventsource.",
-      "pattern": "^([a-zA-Z0-9]([a-zA-Z0-9\\-]{0,61}[a-zA-Z0-9])?\\.)+[a-zA-Z]{2,6}$",
-      "title": "[NOT IMPLEMENTED] Nginx Domain",
-      "type": "string"
-    },
-    "public-ipv4": {
-      "description": "Public ipv4 of the frontend (the one Caddy will be indirectly listening to).",
-      "title": "Public IPv4",
-      "type": "string"
-    },
     "re6st-verification-url": {
       "description": "Url to verify if the internet and/or re6stnet is working.",
       "title": "Test Verification URL",
@@ -118,6 +107,12 @@
       ],
       "title": "Authenticate to backend",
       "type": "string"
+    },
+    "strict-transport-security": {
+      "title": "Strict Transport Security",
+      "description": "Enables Strict Transport Security (HSTS) on the slave, the default 0 results with option disabled. More information: https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Strict-Transport-Security",
+      "default": "0",
+      "type": "integer"
     }
   },
   "title": "Input Parameters",

software/caddy-frontend/instance-slave-caddy-input-schema.json

@@ -285,6 +285,32 @@
       "description": "Amount of bad responses from the backend to consider it down.",
       "default": "1",
       "type": "integer"
+    },
+    "strict-transport-security": {
+      "title": "Strict Transport Security",
+      "description": "Enables Strict Transport Security (HSTS) on the slave, the default 0 results with option disabled. More information: https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Strict-Transport-Security",
+      "default": "0",
+      "type": "integer"
+    },
+    "strict-transport-security-sub-domains": {
+      "title": "Strict Transport Security Sub Domains",
+      "description": "Configures Strict Transport Security for sub domains.",
+      "enum": [
+        "false",
+        "true"
+      ],
+      "type": "string",
+      "default": "false"
+    },
+    "strict-transport-security-preload": {
+      "title": "Strict Transport Security Preload",
+      "description": "Configures Strict Transport Security preload mechanism.",
+      "enum": [
+        "false",
+        "true"
+      ],
+      "type": "string",
+      "default": "false"
     }
   },
   "title": "Input Parameters",

software/caddy-frontend/instance-slave-output-schema.json

@@ -18,10 +18,6 @@
       "description": "List of URLs to access logs",
       "type": "array"
     },
-    "public-ipv4": {
-      "description": "Public IPv4 to be included on DNS",
-      "type": "string"
-    },
     "replication_number": {
       "description": "Number of nodes the slave is replicated",
       "type": "integer"

software/caddy-frontend/instance.cfg.in

@@ -78,7 +78,6 @@ cert = ${slap-connection:cert-file}
 # All parameters are available through the configuration.XX syntax.
 # All possible parameters should have a default.
 configuration.domain = example.org
-configuration.public-ipv4 =
 configuration.port = 4443
 configuration.plain_http_port = 8080
 configuration.plain_nginx_port = 8081
@@ -107,3 +106,4 @@ configuration.backend-haproxy-statistic-port = 21444
 configuration.authenticate-to-backend = False
 configuration.rotate-num = 4000
 configuration.slave-introspection-https-port = 22443
+configuration.strict-transport-security = 0

software/caddy-frontend/software.cfg

@@ -31,7 +31,7 @@ parts +=
 recipe = slapos.recipe.build:gitclone
 repository = https://lab.nexedi.com/nexedi/kedifa.git
 git-executable = ${git:location}/bin/git
-revision = d6bbd7db215e12871c1536f22a8fbf994227270c
+revision = 1e38bcb3bf56f544262780be7e8858f3e6ceedc5
 
 [kedifa-develop]
 recipe = zc.recipe.egg:develop

software/caddy-frontend/templates/backend-haproxy.cfg.in

@@ -18,12 +18,8 @@ defaults
 {%- macro frontend_entry(slave_instance, scheme, wildcard) %}
 {#-   wildcard switch allows to put dangerous entries in the end, as haproxy parses with first match #}
 {%-   if slave_instance[SCHEME_PREFIX_MAPPING[scheme]]['hostname'] and slave_instance[SCHEME_PREFIX_MAPPING[scheme]]['port'] %}
-{%-     set host_list = (slave_instance.get('server-alias') or  '').split() %}
-{%-     if slave_instance.get('custom_domain') not in host_list %}
-{%-       do host_list.append(slave_instance.get('custom_domain')) %}
-{%-     endif %}
 {%-     set matched = {'count': 0} %}
-{%-     for host in host_list %}
+{%-     for host in slave_instance['host_list'] %}
 {#-       Match up to the end or optional port (starting with ':') #}
 {#-       Please note that this matching is quite sensitive to changes and hard to test, so avoid needless changes #}
 {%-       if wildcard and host.startswith('*.') %}
@@ -80,10 +76,9 @@ frontend https-backend
 {%-           do ssl_list.append('crt %s' % (configuration['certificate'],)) %}
 {%-         endif %}
 {%-         do ssl_list.append('ssl verify') %}
-{%-         set path_to_ssl_proxy_ca_crt = slave_instance.get('path_to_ssl_proxy_ca_crt') %}
 {%-         if slave_instance['ssl_proxy_verify'] %}
-{%-           if path_to_ssl_proxy_ca_crt %}
-{%-             do ssl_list.append('required ca-file %s' % (path_to_ssl_proxy_ca_crt,)) %}
+{%-           if slave_instance['path_to_ssl_proxy_ca_crt']  %}
+{%-             do ssl_list.append('required ca-file %s' % (slave_instance['path_to_ssl_proxy_ca_crt'],)) %}
 {%-           else %}
 {#-           Backend SSL shall be verified, but not CA provided, disallow connection #}
 {#-           Simply dropping hostname from the dict will result with ignoring it... #}

software/caddy-frontend/templates/trafficserver/records.config.jinja2

@@ -23,9 +23,19 @@ CONFIG proxy.config.log.logfile_dir STRING {{ ats_directory['log'] }}
 # Implement RFC 5861 with core
 CONFIG proxy.config.http.cache.open_write_fail_action INT 2
 CONFIG proxy.config.body_factory.template_sets_dir STRING  {{ ats_configuration['templates-dir'] }}
-# Support stale-if-error by returning cached content on backend 5xx or unavailability
+# Simulate stale-if-error (not supported by TrafficServer), by using internal
+# mechanism
+# This results with replying last know non-5xx response until max_stale_age is reached
+# ignoring max-age returned by the server 
 CONFIG proxy.config.http.negative_revalidating_enabled INT 1
-CONFIG proxy.config.http.negative_revalidating_lifetime INT 86400
+# max_stale_age set here means that for 1 week since last correct response
+# the response will be sent by the system
+CONFIG proxy.config.http.cache.max_stale_age INT 604800
+# negative_revalidating_lifetime just adds Expires header calculated as
+#          Expires = Date + negative_revalidating_lifetime
+# for case when backend replies 5xx, and Age > max-age and Age < max_stale_age
+# and that's not needed, so drop this behaviour
+CONFIG proxy.config.http.negative_revalidating_lifetime INT 0
 
 ##############################################################################
 # Thread configurations. Docs:

software/caddy-frontend/test/test_data/test.TestRe6stVerificationUrlSlave.test_file_list_etc_cron_d-CADDY.txt

+T-0/etc/cron.d/logrotate
+T-0/etc/cron.d/monitor-configurator
+T-0/etc/cron.d/monitor-globalstate
+T-0/etc/cron.d/monitor_collect
+T-1/etc/cron.d/logrotate
+T-1/etc/cron.d/monitor-configurator
+T-1/etc/cron.d/monitor-globalstate
+T-1/etc/cron.d/monitor_collect
+T-2/etc/cron.d/logrotate
+T-2/etc/cron.d/monitor-configurator
+T-2/etc/cron.d/monitor-globalstate
+T-2/etc/cron.d/monitor_collect
+T-2/etc/cron.d/trafficserver-logrotate

software/caddy-frontend/test/test_data/test.TestRe6stVerificationUrlSlave.test_file_list_log-CADDY.txt

+T-0/var/log/monitor-httpd-access.log
+T-0/var/log/monitor-httpd-error.log
+T-0/var/log/slapgrid-T-0-error.log
+T-1/var/log/expose-csr_id.log
+T-1/var/log/kedifa.log
+T-1/var/log/monitor-httpd-access.log
+T-1/var/log/monitor-httpd-error.log
+T-2/var/log/backend-haproxy.log
+T-2/var/log/expose-csr_id.log
+T-2/var/log/frontend-access.log
+T-2/var/log/frontend-error.log
+T-2/var/log/httpd/_default_access_log
+T-2/var/log/httpd/_default_backend_log
+T-2/var/log/httpd/_default_error_log
+T-2/var/log/monitor-httpd-access.log
+T-2/var/log/monitor-httpd-error.log
+T-2/var/log/slave-introspection-access.log
+T-2/var/log/slave-introspection-error.log
+T-2/var/log/trafficserver/manager.log

software/caddy-frontend/test/test_data/test.TestRe6stVerificationUrlSlave.test_file_list_plugin-CADDY.txt

+T-0/etc/plugin/__init__.py
+T-0/etc/plugin/aibcc-user-caucase-updater.py
+T-0/etc/plugin/aikc-user-caucase-updater.py
+T-0/etc/plugin/buildout-T-0-status.py
+T-0/etc/plugin/caucased-backend-client.py
+T-0/etc/plugin/check-backend-haproxy-statistic-url-caddy-frontend-1.py
+T-0/etc/plugin/check-free-disk-space.py
+T-0/etc/plugin/monitor-bootstrap-status.py
+T-0/etc/plugin/monitor-http-frontend.py
+T-0/etc/plugin/monitor-httpd-listening-on-tcp.py
+T-0/etc/plugin/rejected-slave-publish-ip-port-listening.py
+T-0/etc/plugin/rejected-slave.py
+T-1/etc/plugin/__init__.py
+T-1/etc/plugin/buildout-T-1-status.py
+T-1/etc/plugin/caucased.py
+T-1/etc/plugin/check-free-disk-space.py
+T-1/etc/plugin/expose-csr_id-ip-port-listening.py
+T-1/etc/plugin/kedifa-http-reply.py
+T-1/etc/plugin/monitor-bootstrap-status.py
+T-1/etc/plugin/monitor-http-frontend.py
+T-1/etc/plugin/monitor-httpd-listening-on-tcp.py
+T-1/etc/plugin/promise-logrotate-setup.py
+T-2/etc/plugin/__init__.py
+T-2/etc/plugin/backend-client-caucase-updater.py
+T-2/etc/plugin/backend-haproxy-configuration.py
+T-2/etc/plugin/backend_haproxy_http.py
+T-2/etc/plugin/backend_haproxy_https.py
+T-2/etc/plugin/buildout-T-2-status.py
+T-2/etc/plugin/caddy_frontend_ipv4_http.py
+T-2/etc/plugin/caddy_frontend_ipv4_https.py
+T-2/etc/plugin/caddy_frontend_ipv6_http.py
+T-2/etc/plugin/caddy_frontend_ipv6_https.py
+T-2/etc/plugin/caucase-updater.py
+T-2/etc/plugin/check-free-disk-space.py
+T-2/etc/plugin/expose-csr_id-ip-port-listening.py
+T-2/etc/plugin/frontend-caddy-configuration-promise.py
+T-2/etc/plugin/monitor-bootstrap-status.py
+T-2/etc/plugin/monitor-http-frontend.py
+T-2/etc/plugin/monitor-httpd-listening-on-tcp.py
+T-2/etc/plugin/promise-logrotate-setup.py
+T-2/etc/plugin/re6st-connectivity.py
+T-2/etc/plugin/slave-introspection-configuration.py
+T-2/etc/plugin/slave_introspection_https.py
+T-2/etc/plugin/trafficserver-cache-availability.py
+T-2/etc/plugin/trafficserver-port-listening.py

software/caddy-frontend/test/test_data/test.TestRe6stVerificationUrlSlave.test_file_list_run-CADDY.txt

+T-0/var/run/monitor-httpd.pid
+T-1/var/run/kedifa.pid
+T-1/var/run/monitor-httpd.pid
+T-2/var/run/backend-haproxy-rsyslogd.pid
+T-2/var/run/backend-haproxy.pid
+T-2/var/run/backend_haproxy_configuration_last_state
+T-2/var/run/backend_haproxy_graceful_configuration_state_signature
+T-2/var/run/bhlog.sck
+T-2/var/run/graceful_configuration_state_signature
+T-2/var/run/httpd.pid
+T-2/var/run/monitor-httpd.pid
+T-2/var/run/slave-introspection.pid
+T-2/var/run/slave_introspection_configuration_last_state
+T-2/var/run/slave_introspection_graceful_configuration_state_signature

software/caddy-frontend/test/test_data/test.TestRe6stVerificationUrlSlave.test_supervisor_state-CADDY.txt

+T-0:aibcc-user-caucase-updater-on-watch RUNNING
+T-0:aikc-user-caucase-updater-on-watch RUNNING
+T-0:bootstrap-monitor EXITED
+T-0:caucased-backend-client-{hash-generic}-on-watch RUNNING
+T-0:certificate_authority-{hash-generic}-on-watch RUNNING
+T-0:crond-{hash-generic}-on-watch RUNNING
+T-0:monitor-httpd-{hash-generic}-on-watch RUNNING
+T-0:monitor-httpd-graceful EXITED
+T-0:rejected-slave-publish-{hash-rejected-slave-publish}-on-watch RUNNING
+T-1:bootstrap-monitor EXITED
+T-1:caucase-updater-on-watch RUNNING
+T-1:caucased-{hash-generic}-on-watch RUNNING
+T-1:certificate_authority-{hash-generic}-on-watch RUNNING
+T-1:crond-{hash-generic}-on-watch RUNNING
+T-1:expose-csr_id-{hash-generic}-on-watch RUNNING
+T-1:kedifa-{hash-generic}-on-watch RUNNING
+T-1:kedifa-reloader EXITED
+T-1:monitor-httpd-{hash-generic}-on-watch RUNNING
+T-1:monitor-httpd-graceful EXITED
+T-2:6tunnel-11080-{hash-generic}-on-watch RUNNING
+T-2:6tunnel-11443-{hash-generic}-on-watch RUNNING
+T-2:backend-client-login-certificate-caucase-updater-on-watch RUNNING
+T-2:backend-haproxy-{hash-generic}-on-watch RUNNING
+T-2:backend-haproxy-rsyslogd-{hash-generic}-on-watch RUNNING
+T-2:backend-haproxy-safe-graceful EXITED
+T-2:bootstrap-monitor EXITED
+T-2:certificate_authority-{hash-generic}-on-watch RUNNING
+T-2:crond-{hash-generic}-on-watch RUNNING
+T-2:expose-csr_id-{hash-generic}-on-watch RUNNING
+T-2:frontend-caddy-safe-graceful EXITED
+T-2:frontend_caddy-{hash-caddy-T-2}-on-watch RUNNING
+T-2:kedifa-login-certificate-caucase-updater-on-watch RUNNING
+T-2:kedifa-updater-{hash-generic}-on-watch RUNNING
+T-2:monitor-httpd-{hash-generic}-on-watch RUNNING
+T-2:monitor-httpd-graceful EXITED
+T-2:slave-instrospection-nginx-{hash-generic}-on-watch RUNNING
+T-2:slave-introspection-safe-graceful EXITED
+T-2:trafficserver-{hash-generic}-on-watch RUNNING
+T-2:trafficserver-reload EXITED

software/caddy-frontend/test/test_data/test.TestReplicateSlave.test_file_list_etc_cron_d-CADDY.txt

+T-0/etc/cron.d/logrotate
+T-0/etc/cron.d/monitor-configurator
+T-0/etc/cron.d/monitor-globalstate
+T-0/etc/cron.d/monitor_collect
+T-1/etc/cron.d/logrotate
+T-1/etc/cron.d/monitor-configurator
+T-1/etc/cron.d/monitor-globalstate
+T-1/etc/cron.d/monitor_collect
+T-2/etc/cron.d/logrotate
+T-2/etc/cron.d/monitor-configurator
+T-2/etc/cron.d/monitor-globalstate
+T-2/etc/cron.d/monitor_collect
+T-2/etc/cron.d/trafficserver-logrotate
+T-3/etc/cron.d/logrotate
+T-3/etc/cron.d/monitor-configurator
+T-3/etc/cron.d/monitor-globalstate
+T-3/etc/cron.d/monitor_collect
+T-3/etc/cron.d/trafficserver-logrotate

software/caddy-frontend/test/test_data/test.TestReplicateSlave.test_file_list_log-CADDY.txt

+T-0/var/log/monitor-httpd-access.log
+T-0/var/log/monitor-httpd-error.log
+T-0/var/log/slapgrid-T-0-error.log
+T-1/var/log/expose-csr_id.log
+T-1/var/log/kedifa.log
+T-1/var/log/monitor-httpd-access.log
+T-1/var/log/monitor-httpd-error.log
+T-2/var/log/backend-haproxy.log
+T-2/var/log/expose-csr_id.log
+T-2/var/log/frontend-access.log
+T-2/var/log/frontend-error.log
+T-2/var/log/httpd/_replicate_access_log
+T-2/var/log/httpd/_replicate_backend_log
+T-2/var/log/httpd/_replicate_error_log
+T-2/var/log/monitor-httpd-access.log
+T-2/var/log/monitor-httpd-error.log
+T-2/var/log/slave-introspection-access.log
+T-2/var/log/slave-introspection-error.log
+T-2/var/log/trafficserver/manager.log
+T-3/var/log/backend-haproxy.log
+T-3/var/log/expose-csr_id.log
+T-3/var/log/frontend-access.log
+T-3/var/log/frontend-error.log
+T-3/var/log/httpd/_replicate_access_log
+T-3/var/log/httpd/_replicate_error_log
+T-3/var/log/monitor-httpd-access.log
+T-3/var/log/monitor-httpd-error.log
+T-3/var/log/slave-introspection-access.log
+T-3/var/log/slave-introspection-error.log
+T-3/var/log/trafficserver/manager.log

software/caddy-frontend/test/test_data/test.TestReplicateSlave.test_file_list_plugin-CADDY.txt

+T-0/etc/plugin/__init__.py
+T-0/etc/plugin/aibcc-user-caucase-updater.py
+T-0/etc/plugin/aikc-user-caucase-updater.py
+T-0/etc/plugin/buildout-T-0-status.py
+T-0/etc/plugin/caucased-backend-client.py
+T-0/etc/plugin/check-backend-haproxy-statistic-url-caddy-frontend-1.py
+T-0/etc/plugin/check-backend-haproxy-statistic-url-caddy-frontend-2.py
+T-0/etc/plugin/check-free-disk-space.py
+T-0/etc/plugin/monitor-bootstrap-status.py
+T-0/etc/plugin/monitor-http-frontend.py
+T-0/etc/plugin/monitor-httpd-listening-on-tcp.py
+T-0/etc/plugin/rejected-slave-publish-ip-port-listening.py
+T-0/etc/plugin/rejected-slave.py
+T-1/etc/plugin/__init__.py
+T-1/etc/plugin/buildout-T-1-status.py
+T-1/etc/plugin/caucased.py
+T-1/etc/plugin/check-free-disk-space.py
+T-1/etc/plugin/expose-csr_id-ip-port-listening.py
+T-1/etc/plugin/kedifa-http-reply.py
+T-1/etc/plugin/monitor-bootstrap-status.py
+T-1/etc/plugin/monitor-http-frontend.py
+T-1/etc/plugin/monitor-httpd-listening-on-tcp.py
+T-1/etc/plugin/promise-logrotate-setup.py
+T-2/etc/plugin/__init__.py
+T-2/etc/plugin/backend-client-caucase-updater.py
+T-2/etc/plugin/backend-haproxy-configuration.py
+T-2/etc/plugin/backend_haproxy_http.py
+T-2/etc/plugin/backend_haproxy_https.py
+T-2/etc/plugin/buildout-T-2-status.py
+T-2/etc/plugin/caddy_frontend_ipv4_http.py
+T-2/etc/plugin/caddy_frontend_ipv4_https.py
+T-2/etc/plugin/caddy_frontend_ipv6_http.py
+T-2/etc/plugin/caddy_frontend_ipv6_https.py
+T-2/etc/plugin/caucase-updater.py
+T-2/etc/plugin/check-free-disk-space.py
+T-2/etc/plugin/expose-csr_id-ip-port-listening.py
+T-2/etc/plugin/frontend-caddy-configuration-promise.py
+T-2/etc/plugin/monitor-bootstrap-status.py
+T-2/etc/plugin/monitor-http-frontend.py
+T-2/etc/plugin/monitor-httpd-listening-on-tcp.py
+T-2/etc/plugin/promise-logrotate-setup.py
+T-2/etc/plugin/re6st-connectivity.py
+T-2/etc/plugin/slave-introspection-configuration.py
+T-2/etc/plugin/slave_introspection_https.py
+T-2/etc/plugin/trafficserver-cache-availability.py
+T-2/etc/plugin/trafficserver-port-listening.py
+T-3/etc/plugin/__init__.py
+T-3/etc/plugin/backend-client-caucase-updater.py
+T-3/etc/plugin/backend-haproxy-configuration.py
+T-3/etc/plugin/backend_haproxy_http.py
+T-3/etc/plugin/backend_haproxy_https.py
+T-3/etc/plugin/buildout-T-3-status.py
+T-3/etc/plugin/caddy_frontend_ipv4_http.py
+T-3/etc/plugin/caddy_frontend_ipv4_https.py
+T-3/etc/plugin/caddy_frontend_ipv6_http.py
+T-3/etc/plugin/caddy_frontend_ipv6_https.py
+T-3/etc/plugin/caucase-updater.py
+T-3/etc/plugin/check-free-disk-space.py
+T-3/etc/plugin/expose-csr_id-ip-port-listening.py
+T-3/etc/plugin/frontend-caddy-configuration-promise.py
+T-3/etc/plugin/monitor-bootstrap-status.py
+T-3/etc/plugin/monitor-http-frontend.py
+T-3/etc/plugin/monitor-httpd-listening-on-tcp.py
+T-3/etc/plugin/promise-logrotate-setup.py
+T-3/etc/plugin/re6st-connectivity.py
+T-3/etc/plugin/slave-introspection-configuration.py
+T-3/etc/plugin/slave_introspection_https.py
+T-3/etc/plugin/trafficserver-cache-availability.py
+T-3/etc/plugin/trafficserver-port-listening.py

software/caddy-frontend/test/test_data/test.TestReplicateSlave.test_file_list_run-CADDY.txt

+T-0/var/run/monitor-httpd.pid
+T-1/var/run/kedifa.pid
+T-1/var/run/monitor-httpd.pid
+T-2/var/run/backend-haproxy-rsyslogd.pid
+T-2/var/run/backend-haproxy.pid
+T-2/var/run/backend_haproxy_configuration_last_state
+T-2/var/run/backend_haproxy_graceful_configuration_state_signature
+T-2/var/run/bhlog.sck
+T-2/var/run/graceful_configuration_state_signature
+T-2/var/run/httpd.pid
+T-2/var/run/monitor-httpd.pid
+T-2/var/run/slave-introspection.pid
+T-2/var/run/slave_introspection_configuration_last_state
+T-2/var/run/slave_introspection_graceful_configuration_state_signature
+T-3/var/run/backend-haproxy.pid
+T-3/var/run/backend_haproxy_configuration_last_state
+T-3/var/run/backend_haproxy_graceful_configuration_state_signature
+T-3/var/run/graceful_configuration_state_signature
+T-3/var/run/slave_introspection_configuration_last_state
+T-3/var/run/slave_introspection_graceful_configuration_state_signature

software/caddy-frontend/test/test_data/test.TestReplicateSlave.test_supervisor_state-CADDY.txt

+T-0:aibcc-user-caucase-updater-on-watch RUNNING
+T-0:aikc-user-caucase-updater-on-watch RUNNING
+T-0:bootstrap-monitor EXITED
+T-0:caucased-backend-client-{hash-generic}-on-watch RUNNING
+T-0:certificate_authority-{hash-generic}-on-watch RUNNING
+T-0:crond-{hash-generic}-on-watch RUNNING
+T-0:monitor-httpd-{hash-generic}-on-watch RUNNING
+T-0:monitor-httpd-graceful EXITED
+T-0:rejected-slave-publish-{hash-rejected-slave-publish}-on-watch RUNNING
+T-1:bootstrap-monitor EXITED
+T-1:caucase-updater-on-watch RUNNING
+T-1:caucased-{hash-generic}-on-watch RUNNING
+T-1:certificate_authority-{hash-generic}-on-watch RUNNING
+T-1:crond-{hash-generic}-on-watch RUNNING
+T-1:expose-csr_id-{hash-generic}-on-watch RUNNING
+T-1:kedifa-{hash-generic}-on-watch RUNNING
+T-1:kedifa-reloader EXITED
+T-1:monitor-httpd-{hash-generic}-on-watch RUNNING
+T-1:monitor-httpd-graceful EXITED
+T-2:6tunnel-11080-{hash-generic}-on-watch RUNNING
+T-2:6tunnel-11443-{hash-generic}-on-watch RUNNING
+T-2:backend-client-login-certificate-caucase-updater-on-watch RUNNING
+T-2:backend-haproxy-{hash-generic}-on-watch RUNNING
+T-2:backend-haproxy-rsyslogd-{hash-generic}-on-watch RUNNING
+T-2:backend-haproxy-safe-graceful EXITED
+T-2:bootstrap-monitor EXITED
+T-2:certificate_authority-{hash-generic}-on-watch RUNNING
+T-2:crond-{hash-generic}-on-watch RUNNING
+T-2:expose-csr_id-{hash-generic}-on-watch RUNNING
+T-2:frontend-caddy-safe-graceful EXITED
+T-2:frontend_caddy-{hash-caddy-T-2}-on-watch RUNNING
+T-2:kedifa-login-certificate-caucase-updater-on-watch RUNNING
+T-2:kedifa-updater-{hash-generic}-on-watch RUNNING
+T-2:monitor-httpd-{hash-generic}-on-watch RUNNING
+T-2:monitor-httpd-graceful EXITED
+T-2:slave-instrospection-nginx-{hash-generic}-on-watch RUNNING
+T-2:slave-introspection-safe-graceful EXITED
+T-2:trafficserver-{hash-generic}-on-watch RUNNING
+T-2:trafficserver-reload EXITED
+T-3:6tunnel-11080-{hash-generic}-on-watch STOPPED
+T-3:6tunnel-11443-{hash-generic}-on-watch STOPPED
+T-3:backend-client-login-certificate-caucase-updater-on-watch STOPPED
+T-3:backend-haproxy-{hash-generic}-on-watch STOPPED
+T-3:backend-haproxy-rsyslogd-{hash-generic}-on-watch STOPPED
+T-3:backend-haproxy-safe-graceful EXITED
+T-3:bootstrap-monitor EXITED
+T-3:certificate_authority-{hash-generic}-on-watch STOPPED
+T-3:crond-{hash-generic}-on-watch STOPPED
+T-3:expose-csr_id-{hash-generic}-on-watch STOPPED
+T-3:frontend-caddy-safe-graceful EXITED
+T-3:frontend_caddy-{hash-caddy-T-3}-on-watch STOPPED
+T-3:kedifa-login-certificate-caucase-updater-on-watch STOPPED
+T-3:kedifa-updater-{hash-generic}-on-watch STOPPED
+T-3:monitor-httpd-{hash-generic}-on-watch STOPPED
+T-3:monitor-httpd-graceful EXITED
+T-3:slave-instrospection-nginx-{hash-generic}-on-watch STOPPED
+T-3:slave-introspection-safe-graceful EXITED
+T-3:trafficserver-{hash-generic}-on-watch STOPPED
+T-3:trafficserver-reload EXITED

software/erp5/test/test/test_balancer.py

@@ -145,7 +145,15 @@ class BalancerTestCase(ERP5InstanceTestCase):
         # XXX what is this ? should probably not be needed here
         'name': cls.__name__,
         'monitor-passwd': 'secret',
-        'apachedex-configuration': '--erp5-base +erp5 .*/VirtualHostRoot/erp5(/|\\?|$) --base +other / --skip-user-agent Zabbix --error-detail --js-embed --quiet',
+        'apachedex-configuration': [
+            '--logformat', '%h %l %u %t "%r" %>s %O "%{Referer}i" "%{User-Agent}i" %{ms}T',
+            '--erp5-base', '+erp5', '.*/VirtualHostRoot/erp5(/|\\?|$)',
+            '--base', '+other', '/',
+            '--skip-user-agent', 'Zabbix',
+            '--error-detail',
+            '--js-embed',
+            '--quiet',
+          ],
         'apachedex-promise-threshold': 100,
         'haproxy-server-check-path': '/',
         'zope-family-dict': {

software/jupyter/buildout.hash.cfg

@@ -13,17 +13,17 @@
 # section inheritance (< = ...) are NOT supported (but you should really
 # not need these here).
 
-[instance-jupyter-notebook]
+[instance]
 filename = instance.cfg.in
-md5sum = 659e0d583acd70ebb8d815c22735d91b
+md5sum = de37ec3d4adb0be4c67bcc7397f27c91
+
+[instance-jupyter]
+filename = instance-jupyter.cfg.in
+md5sum = 9340498841caa5771f40f8c9e561eacd
 
 [jupyter-notebook-config]
 filename = jupyter_notebook_config.py.jinja
-md5sum = 4514df152740dbd7990da9705f40e978
-
-[jupyter-set-password]
-filename = jupyter_set_password.cgi.jinja
-md5sum = b8d31441780b524a7e52d1710dd78385
+md5sum = 089e4c511a3c7b110471bf41ca2695a4
 
 [erp5-kernel]
 filename = ERP5kernel.py

software/jupyter/instance-jupyter.cfg.in

+{% set additional_frontend = slapparameter_dict.get('frontend-additional-instance-guid') %}
+
+[buildout]
+parts =
+  instance
+  publish-connection-parameter
+  jupyter-notebook-config
+  erp5-kernel
+  kernel-json
+  custom-js
+  frontend-promise
+  {% if additional_frontend %}
+  frontend-additional-promise
+  {% endif %}
+  monitor-base
+extends =
+  {{ monitor_template_rendered }}/template-monitor.cfg
+
+
+eggs-directory = {{ eggs_directory }}
+develop-eggs-directory = {{ develop_eggs_directory }}
+offline = true
+
+[slapconfiguration]
+recipe = slapos.cookbook:slapconfiguration.serialised
+computer = ${slap-connection:computer-id}
+partition = ${slap-connection:partition-id}
+url = ${slap-connection:server-url}
+key = ${slap-connection:key-file}
+cert = ${slap-connection:cert-file}
+
+# ERP5 URL to use in Jupyter by default
+# default value is empty - which means no default ERP5 URL
+configuration.erp5-url =
+
+[instance-parameter]
+port = 8888
+host = ${slapconfiguration:ipv6-random}
+cert_file = ${generate-certificate:cert_file}
+key_file = ${generate-certificate:key_file}
+logfile = ${directory:log}/jupyter_notebook.log
+notebook_dir = ${directory:notebook_dir}
+
+[slap-parameter]
+frontend-software-type = RootSoftwareInstance
+frontend-software-url = http://git.erp5.org/gitweb/slapos.git/blob_plain/HEAD:/software/apache-frontend/software.cfg
+frontend-instance-guid =
+frontend-instance-name = Jupyter Frontend
+frontend-additional-software-type = RootSoftwareInstance
+frontend-additional-software-url = http://git.erp5.org/gitweb/slapos.git/blob_plain/HEAD:/software/apache-frontend/software.cfg
+frontend-additional-instance-guid =
+frontend-additional-instance-name = Jupyter Frontend Additional
+
+[dynamic-jinja2-template-base]
+recipe = slapos.recipe.template:jinja2
+mode = 0644
+
+[generate-certificate]
+; TODO: there is a slapos recipe to generate certificates. Use it instead
+recipe = plone.recipe.command
+command =
+  if [ ! -e ${instance-parameter:key_file} ]
+  then
+    {{ openssl_output }} req -x509 -nodes -days 3650 \
+      -subj "/C=AA/ST=X/L=X/O=Dis/CN=${instance-parameter:host}" \
+      -newkey rsa:1024 -keyout ${instance-parameter:key_file} \
+      -out ${instance-parameter:cert_file}
+  fi
+update-command = ${:command}
+cert_file = ${directory:etc}/jupyter_cert.crt
+key_file = ${directory:etc}/jupyter_cert.key
+
+[instance]
+recipe = slapos.cookbook:wrapper
+command-line =
+  {{ bin_directory }}/jupyter-lab
+  --no-browser
+  --ip=${instance-parameter:host}
+  --port=${instance-parameter:port}
+  --port-retries=0
+  --certfile=${instance-parameter:cert_file}
+  --keyfile=${instance-parameter:key_file}
+  --notebook-dir=${instance-parameter:notebook_dir}
+  --log-level="DEBUG"
+wrapper-path = ${directory:service}/jupyter-lab
+environment =
+  JUPYTER_PATH=${directory:jupyter_dir}
+  JUPYTER_CONFIG_DIR=${directory:jupyter_config_dir}
+  JUPYTER_RUNTIME_DIR=${directory:jupyter_runtime_dir}
+  LANG=C.UTF-8
+
+[jupyter-password]
+recipe = slapos.cookbook:generate.password
+bytes = 10
+
+[jupyter-notebook-config]
+<= dynamic-jinja2-template-base
+template = {{ jupyter_config_location }}/{{ jupyter_config_filename }}
+rendered = ${directory:jupyter_config_dir}/jupyter_notebook_config.py
+mode = 0744
+context =
+  key password jupyter-password:passwd
+  raw gcc_location {{ gcc_location }}
+  raw cythonplus_repository {{ cythonplus_repository }}
+
+[directory]
+recipe = slapos.cookbook:mkdirectory
+home = ${buildout:directory}
+etc = ${:home}/etc
+var = ${:home}/var
+script = ${:etc}/run/
+service = ${:etc}/service
+log = ${:var}/log
+notebook_dir = ${:var}/notebooks
+# Add folders to explicitly define jupyter directory
+jupyter_dir = ${:home}/jupyter
+jupyter_config_dir = ${:jupyter_dir}/etc
+jupyter_kernel_dir = ${:jupyter_dir}/kernels
+jupyter_runtime_dir = ${:jupyter_dir}/runtime
+jupyter_custom_dir = ${:jupyter_config_dir}/custom
+jupyter_nbextensions_dir = ${:jupyter_dir}/nbextensions
+erp5_kernel_dir = ${:jupyter_kernel_dir}/ERP5
+
+[request-slave-frontend-base]
+recipe = slapos.cookbook:requestoptional
+server-url = ${slap-connection:server-url}
+key-file = ${slap-connection:key-file}
+cert-file = ${slap-connection:cert-file}
+computer-id = ${slap-connection:computer-id}
+partition-id = ${slap-connection:partition-id}
+slave = true
+config-type = notebook
+config-url = https://[${instance-parameter:host}]:${instance-parameter:port}
+return = secure_access
+
+[request-slave-frontend]
+<= request-slave-frontend-base
+software-url = ${slap-parameter:frontend-software-url}
+software-type = ${slap-parameter:frontend-software-type}
+name = ${slap-parameter:frontend-instance-name}
+sla-instance_guid = ${slap-parameter:frontend-instance-guid}
+
+[frontend-promise]
+<= monitor-promise-base
+module = check_url_available
+name = frontend_promise.py
+config-url = ${publish-connection-parameter:url}
+
+{% if additional_frontend %}
+[request-slave-frontend-additional]
+<= request-slave-frontend-base
+software-url = ${slap-parameter:frontend-additional-software-url}
+software-type = ${slap-parameter:frontend-additional-software-type}
+name = ${slap-parameter:frontend-additional-instance-name}
+sla-instance_guid = ${slap-parameter:frontend-additional-instance-guid}
+
+[frontend-additional-promise]
+<= monitor-promise-base
+module = check_url_available
+name = frontend_additional_promise.py
+config-url = ${publish-connection-parameter:url-additional}
+{% endif %}
+
+[monitor-instance-parameter]
+monitor-base-url = ${monitor-frontend-promise:url}
+# In case you're using a developer instance you should edit these in:
+# monitor-base-url = ${monitor-httpd-conf-parameter:url}
+# cors-domains = softinstXXXXX.host.vifib.net (or equivalent)
+# interface-url = https://softinstXXXXX.host.vifib.net/erp5/web_site_module/monitoring_rjs_unsafe
+instance-configuration =
+  raw jupyter-password ${jupyter-password:passwd}
+
+[publish-connection-parameter]
+recipe = slapos.cookbook:publish.serialised
+jupyter-classic-url = ${request-slave-frontend:connection-secure_access}/tree
+url = ${:jupyter-classic-url}
+jupyterlab-url = ${request-slave-frontend:connection-secure_access}/lab
+{% if additional_frontend %}
+jupyter-classic-url-additional = ${request-slave-frontend-additional:connection-secure_access}/tree
+url-additional = ${:jupyter-classic-url-additional}
+jupyterlab-url-additional = ${request-slave-frontend-additional:connection-secure_access}/lab
+{% endif %}
+password = ${jupyter-password:passwd}
+
+[erp5-kernel]
+recipe = slapos.cookbook:symbolic.link
+link-binary = {{ erp5_kernel_location }}/{{ erp5_kernel_filename }}
+target-directory = ${directory:erp5_kernel_dir}
+
+[kernel-json]
+<= dynamic-jinja2-template-base
+template = {{ kernel_json_location }}/{{ kernel_json_filename }}
+rendered = ${directory:erp5_kernel_dir}/kernel.json
+# Use python2.7 executable bin file for kernel config
+context =
+  raw python_executable {{ python_executable }}
+  raw kernel_dir ${erp5-kernel:target-directory}/{{ erp5_kernel_filename }}
+  key erp5_url slapconfiguration:configuration.erp5-url 
+  raw display_name ERP5
+  raw language_name python
+
+[custom-js]
+recipe = slapos.cookbook:symbolic.link
+target-directory = ${directory:jupyter_custom_dir}
+link-binary = {{ custom_js_location }}/custom.js

software/jupyter/instance.cfg.in

 [buildout]
 parts =
-  instance
-  jupyter_notebook
-  read-knowledge0
-  publish-connection-parameter
-  jupyter-notebook-config
-  erp5-kernel
-  kernel-json
-  custom-js
-  monitor-base
-extends =
-  {{ monitor_template_rendered }}/template-monitor.cfg
+  switch_softwaretype
 
+eggs-directory = ${buildout:eggs-directory}
+develop-eggs-directory = ${buildout:develop-eggs-directory}
 
-eggs-directory = {{ eggs_directory }}
-develop-eggs-directory = {{ develop_eggs_directory }}
-offline = true
+[switch_softwaretype]
+recipe = slapos.cookbook:switch-softwaretype
+default = $${:jupyter}
+jupyter = instance-jupyter:rendered
+RootSoftwareInstance = $${:default}
 
-[slapconfiguration]
+[slap-configuration]
 recipe = slapos.cookbook:slapconfiguration.serialised
-computer = ${slap-connection:computer-id}
-partition = ${slap-connection:partition-id}
-url = ${slap-connection:server-url}
-key = ${slap-connection:key-file}
-cert = ${slap-connection:cert-file}
+computer = $${slap-connection:computer-id}
+partition = $${slap-connection:partition-id}
+url = $${slap-connection:server-url}
+key = $${slap-connection:key-file}
+cert = $${slap-connection:cert-file}
 
-# ERP5 URL to use in Jupyter by default
-# default value is empty - which means no default ERP5 URL
-configuration.erp5-url =
-
-[instance-parameter]
-port = 8888
-host = ${slapconfiguration:ipv6-random}
-cert_file = ${generate-certificate:cert_file}
-key_file = ${generate-certificate:key_file}
-logfile = ${directory:log}/jupyter_notebook.log
-notebook_dir = ${directory:notebook_dir}
-
-[dynamic-jinja2-template-base]
+[instance-jupyter]
 recipe = slapos.recipe.template:jinja2
+filename = instance-jupyter.cfg.in
+template = ${instance-jupyter:location}/$${:filename}
+rendered = $${buildout:directory}/template.cfg
 mode = 0644
-
-[generate-certificate]
-; TODO: there is a slapos recipe to generate certificates. Use it instead
-recipe = plone.recipe.command
-command =
-  if [ ! -e ${instance-parameter:key_file} ]
-  then
-    {{ openssl_output }} req -x509 -nodes -days 3650 \
-      -subj "/C=AA/ST=X/L=X/O=Dis/CN=${instance-parameter:host}" \
-      -newkey rsa:1024 -keyout ${instance-parameter:key_file} \
-      -out ${instance-parameter:cert_file}
-  fi
-update-command = ${:command}
-cert_file = ${directory:etc}/jupyter_cert.crt
-key_file = ${directory:etc}/jupyter_cert.key
-
-[instance]
-recipe = slapos.cookbook:wrapper
-command-line =
-  {{ bin_directory }}/jupyter-lab
-  --no-browser
-  --ip=${instance-parameter:host}
-  --port=${instance-parameter:port}
-  --port-retries=0
-  --certfile=${instance-parameter:cert_file}
-  --keyfile=${instance-parameter:key_file}
-  --notebook-dir=${instance-parameter:notebook_dir}
-  --log-level="DEBUG"
-wrapper-path = ${directory:service}/jupyter-lab
-environment =
-  JUPYTER_PATH=${directory:jupyter_dir}
-  JUPYTER_CONFIG_DIR=${directory:jupyter_config_dir}
-  JUPYTER_RUNTIME_DIR=${directory:jupyter_runtime_dir}
-  LANG=C.UTF-8
-
-[jupyter-notebook-config]
-<= dynamic-jinja2-template-base
-template = {{ jupyter_config_location }}/{{ jupyter_config_filename }}
-rendered = ${directory:jupyter_config_dir}/jupyter_notebook_config.py
-mode = 0744
-context =
-  raw config_cfg ${buildout:directory}/knowledge0.cfg
-  raw gcc_location {{ gcc_location }}
-  raw cythonplus_repository {{ cythonplus_repository }}
-
-[directory]
-recipe = slapos.cookbook:mkdirectory
-home = ${buildout:directory}
-etc = ${:home}/etc
-var = ${:home}/var
-script = ${:etc}/run/
-service = ${:etc}/service
-log = ${:var}/log
-notebook_dir = ${:var}/notebooks
-# Add folders to explicitly define jupyter directory
-jupyter_dir = ${:home}/jupyter
-jupyter_config_dir = ${:jupyter_dir}/etc
-jupyter_kernel_dir = ${:jupyter_dir}/kernels
-jupyter_runtime_dir = ${:jupyter_dir}/runtime
-jupyter_custom_dir = ${:jupyter_config_dir}/custom
-jupyter_nbextensions_dir = ${:jupyter_dir}/nbextensions
-erp5_kernel_dir = ${:jupyter_kernel_dir}/ERP5
-
-[jupyter_notebook]
-# This part is called like this because knowledge0.write uses the part name for
-# the section name in the config file.
-recipe = slapos.cookbook:zero-knowledge.write
-password = 
-filename = knowledge0.cfg
-
-[read-knowledge0]
-recipe = slapos.cookbook:zero-knowledge.read
-filename = knowledge0.cfg
-password =
-
-[monitor-instance-parameter]
-monitor-base-url = ${monitor-frontend-promise:url}
-# In case you're using a developer instance you should edit these in:
-# monitor-base-url = ${monitor-httpd-conf-parameter:url}
-# cors-domains = softinstXXXXX.host.vifib.net (or equivalent)
-# interface-url = https://softinstXXXXX.host.vifib.net/erp5/web_site_module/monitoring_rjs_unsafe
-instance-configuration =
-  raw jupyter-password ${read-knowledge0:password}
-
-[publish-connection-parameter]
-recipe = slapos.cookbook:publish.serialised
-jupyter-classic-url = https://[${instance-parameter:host}]:${instance-parameter:port}/tree
-url = ${:jupyter-classic-url}
-jupyterlab-url = https://[${instance-parameter:host}]:${instance-parameter:port}/lab
-
-[erp5-kernel]
-recipe = slapos.cookbook:symbolic.link
-link-binary = {{ erp5_kernel_location }}/{{ erp5_kernel_filename }}
-target-directory = ${directory:erp5_kernel_dir}
-
-[kernel-json]
-<= dynamic-jinja2-template-base
-template = {{ kernel_json_location }}/{{ kernel_json_filename }}
-rendered = ${directory:erp5_kernel_dir}/kernel.json
-# Use python2.7 executable bin file for kernel config
 context =
-  raw python_executable {{ python_executable }}
-  raw kernel_dir ${erp5-kernel:target-directory}/{{ erp5_kernel_filename }}
-  key erp5_url slapconfiguration:configuration.erp5-url 
-  raw display_name ERP5
-  raw language_name python
-
-[custom-js]
-recipe = slapos.cookbook:symbolic.link
-target-directory = ${directory:jupyter_custom_dir}
-link-binary = {{ custom_js_location }}/custom.js
+  key slapparameter_dict slap-configuration:configuration
+  raw bin_directory ${buildout:bin-directory}
+  raw develop_eggs_directory ${buildout:develop-eggs-directory}
+  raw eggs_directory ${buildout:eggs-directory}
+  raw openssl_output ${openssl-output:openssl}
+  raw python_executable ${jupyter:python_executable}
+  raw jupyter_config_location ${jupyter-notebook-config:location}
+  raw jupyter_config_filename ${jupyter-notebook-config:filename}
+  raw erp5_kernel_location ${erp5-kernel:location}
+  raw erp5_kernel_filename ${erp5-kernel:filename}
+  raw kernel_json_location ${kernel-json:location}
+  raw kernel_json_filename ${kernel-json:filename}
+  raw custom_js_location ${custom-js:location}
+  raw custom_js_filename ${custom-js:filename}
+  raw monitor_template_rendered ${buildout:directory}
+  raw cythonplus_repository ${cythonplus-repository:location}
+  raw gcc_location ${gcc:prefix}

software/jupyter/jupyter_notebook_config.py.jinja

@@ -3,39 +3,19 @@
   things. It is run by IPython hence why it can use functions like get_config().
 '''
 import configparser
-import random
 from notebook.auth import passwd
 import os
 
-def random_password(length = 10):
-  result = ""
-  for i in range(0, length):
-    result = result + chr(random.randint(0, 25) + ord('a'))
-  return result
-
-knowledge_0 = '{{ config_cfg }}'
-
-if not os.path.exists(knowledge_0):
-  print("Your software does <b>not</b> embed 0-knowledge. \
-  This interface is useless in this case</body></html>")
-  exit(0)
-
 c = get_config()
 
-parser = configparser.ConfigParser()
-parser.read(knowledge_0)
-
-if not parser.has_section("jupyter_notebook"):
-  parser.add_section("jupyter_notebook")
-
-if not parser.has_option("jupyter_notebook", "password") or \
-  parser.get("jupyter_notebook", "password") == "":
-  parser.set("jupyter_notebook", "password", random_password())
-
-c.NotebookApp.password = passwd(parser.get("jupyter_notebook", "password"))
+c.NotebookApp.password = passwd("{{ password }}")
 
-with open(knowledge_0, 'w') as file:
-  parser.write(file)
+try:
+  os.environ['PATH'] = "{{ gcc_location }}/bin" + os.pathsep + os.environ['PATH']
+except KeyError:
+  os.environ['PATH'] = "{{ gcc_location }}/bin"
 
-os.environ['PATH'] = "{{ gcc_location }}/bin" + os.pathsep + os.environ['PATH']
-os.environ['PYTHONPATH'] = "{{ cythonplus_repository }}" + os.pathsep + os.environ['PYTHONPATH']
+try:
+  os.environ['PYTHONPATH'] = "{{ cythonplus_repository }}" + os.pathsep + os.environ['PYTHONPATH']
+except KeyError:
+  os.environ['PYTHONPATH'] = "{{ cythonplus_repository }}"

software/jupyter/jupyter_set_password.cgi.jinja

-#!{{ python_executable }}
-
-import cgi
-import cgitb
-import ConfigParser
-import os
-import re
-import subprocess
-
-from IPython.lib import passwd
-
-#cgitb.enable(display=0, logdir="/tmp/cgi.log")
-cgitb.enable()
-form = cgi.FieldStorage()
-
-config_file = "{{ config_cfg }}"
-
-
-if not os.path.exists(config_file):
-  print "Your software does <b>not</b> embed 0-knowledge. \
-  This interface is useless in this case</body></html>"
-  exit(0)
-
-
-parser = ConfigParser.ConfigParser()
-parser.read(config_file)
-if not parser.has_section("jupyter_notebook"):
-  parser.add_section("jupyter_notebook")
-
-if not parser.has_option("jupyter_notebook", "password"):
-  parser.set("jupyter_notebook", "password", "")
-
-if "password" in form:
-  parser.set("jupyter_notebook", "password", passwd(form["password"].value))
-  # subprocess.call('{{ httpd_graceful }}')
-  # TODO: we should restart jupyter
-
-with open(config_file, 'w') as file:
-  parser.write(file)
-
-# TODO cleanup
-print "<html><head>"
-print "<link rel=\"stylesheet\" href=\"static/pure-min.css\">"
-print "<link rel=\"stylesheet\" href=\"static/style.css\">"
-print "</head><body>"
-print "<h1>Jupyter Notebook Password :</h1>"
-print "<form action=\"/index.cgi\" method=\"post\" class=\"pure-form-aligned\">"
-print "<input type=\"hidden\" name=\"posting-script\" value=\"{{ pwd }}/{{ this_file }}\">"
-
-print """<div class="pure-control-group">
-<label for="password">Password*:</label>
-<input placeholder="Set your password" type="password" name="password" id="password"></br>
-</div><div class="pure-control-group">
-<label for="password">Verify Password*:</label>
-<input placeholder="Verify password" type="password" name="password_2" id="password_2"></br>
-</div><p id="validate-status" style="color:red"></p>
-<div class="pure-controls">
-<button id="register-button" type="submit" class="pure-button pure-button-primary" disabled>Access</button></div>
-</form>
-<script type="text/javascript" src="static/jquery-1.10.2.min.js"></script>
-<script type="text/javascript" src="static/monitor-register.js"></script>
-</body></html>
-"""
-

software/jupyter/software.cfg

@@ -10,7 +10,7 @@ parts +=
   slapos-cookbook
   jupyter
   jupyter-notebook-initialized-scripts
-  instance-jupyter-notebook
+  instance
 
 [python]
 part = python3
@@ -31,9 +31,6 @@ mode = 0644
 [jupyter-notebook-config]
 <= download-file-base
 
-[jupyter-set-password]
-<= download-file-base
-
 [erp5-kernel]
 <= download-file-base
 
@@ -43,30 +40,14 @@ mode = 0644
 [custom-js]
 <= download-file-base
 
-[instance-jupyter-notebook]
-recipe = slapos.recipe.template:jinja2
-template = ${:_profile_base_location_}/${:filename}
-rendered = ${buildout:directory}/template.cfg
+[instance]
+recipe = slapos.recipe.template
+url =  ${:_profile_base_location_}/${:filename}
+output = ${buildout:directory}/template.cfg
 mode = 0644
-context =
-  key bin_directory buildout:bin-directory
-  key develop_eggs_directory buildout:develop-eggs-directory
-  key eggs_directory buildout:eggs-directory
-  key openssl_output openssl-output:openssl
-  key python_executable jupyter:python_executable
-  key jupyter_config_location jupyter-notebook-config:location
-  key jupyter_config_filename jupyter-notebook-config:filename
-  key jupyter_set_password_location jupyter-set-password:location
-  key jupyter_set_password_filename jupyter-set-password:filename
-  key erp5_kernel_location erp5-kernel:location
-  key erp5_kernel_filename erp5-kernel:filename
-  key kernel_json_location kernel-json:location
-  key kernel_json_filename kernel-json:filename
-  key custom_js_location custom-js:location
-  key custom_js_filename custom-js:filename
-  key monitor_template_rendered buildout:directory
-  key cythonplus_repository cythonplus-repository:location
-  key gcc_location gcc:prefix
+
+[instance-jupyter]
+<= download-file-base
 
 [versions]
 Pygments = 2.7.2

software/jupyter/test/test.py

@@ -48,10 +48,14 @@ class TestJupyter(InstanceTestCase):
     except Exception as e:
       self.fail("Can't parse json in %s, error %s" % (parameter_dict['_'], e))
 
+    self.assertTrue('password' in connection_dict)
+    password = connection_dict['password']
+
     self.assertEqual(
       {
         'jupyter-classic-url': 'https://[%s]:8888/tree' % (self._ipv6_address, ),
         'jupyterlab-url': 'https://[%s]:8888/lab' % (self._ipv6_address, ),
+        'password': '%s' % (password, ),
         'url': 'https://[%s]:8888/tree' % (self._ipv6_address, )
       },
       connection_dict
@@ -79,3 +83,89 @@ class TestJupyter(InstanceTestCase):
       [http.client.FOUND, True, '/login?next=%2Flab'],
       [result.status_code, result.is_redirect, result.headers['Location']]
     )
+
+
+class TestJupyterPassword(InstanceTestCase):
+  def test(self):
+    parameter_dict = self.computer_partition.getConnectionParameterDict()
+    self.assertTrue('_' in parameter_dict)
+    try:
+      connection_dict = json.loads(parameter_dict['_'])
+    except Exception as e:
+      self.fail("Can't parse json in %s, error %s" % (parameter_dict['_'], e))
+
+    url = connection_dict['url']
+    with requests.Session() as s:
+      resp = s.get(url, verify=False)
+      result = s.post(
+        resp.url,
+        verify = False,
+        data={"_xsrf": s.cookies["_xsrf"], "password": connection_dict['password']}
+      )
+      self.assertEqual(
+        [http.client.OK, url],
+        [result.status_code, result.url]
+      )
+
+
+class TestJupyterAdditional(InstanceTestCase):
+
+  @classmethod
+  def getInstanceParameterDict(cls):
+    return {
+      'frontend-additional-instance-guid': 'SOMETHING'
+    }
+
+  def test(self):
+    parameter_dict = self.computer_partition.getConnectionParameterDict()
+    self.assertTrue('_' in parameter_dict)
+    try:
+      connection_dict = json.loads(parameter_dict['_'])
+    except Exception as e:
+      self.fail("Can't parse json in %s, error %s" % (parameter_dict['_'], e))
+
+    result = requests.get(
+      connection_dict['url'], verify=False, allow_redirects=False)
+    self.assertEqual(
+      [http.client.FOUND, True, '/login?next=%2Ftree'],
+      [result.status_code, result.is_redirect, result.headers['Location']]
+    )
+
+    result = requests.get(
+      connection_dict['jupyter-classic-url'],
+      verify=False, allow_redirects=False)
+    self.assertEqual(
+      [http.client.FOUND, True, '/login?next=%2Ftree'],
+      [result.status_code, result.is_redirect, result.headers['Location']]
+    )
+
+    result = requests.get(
+      connection_dict['jupyterlab-url'],
+      verify=False, allow_redirects=False)
+    self.assertEqual(
+      [http.client.FOUND, True, '/login?next=%2Flab'],
+      [result.status_code, result.is_redirect, result.headers['Location']]
+    )
+
+    result = requests.get(
+      connection_dict['url-additional'], verify=False, allow_redirects=False)
+    self.assertEqual(
+      [http.client.FOUND, True, '/login?next=%2Ftree'],
+      [result.status_code, result.is_redirect, result.headers['Location']]
+    )
+
+    result = requests.get(
+      connection_dict['jupyter-classic-url-additional'],
+      verify=False, allow_redirects=False)
+    self.assertEqual(
+      [http.client.FOUND, True, '/login?next=%2Ftree'],
+      [result.status_code, result.is_redirect, result.headers['Location']]
+    )
+
+    result = requests.get(
+      connection_dict['jupyterlab-url-additional'],
+      verify=False, allow_redirects=False)
+    self.assertEqual(
+      [http.client.FOUND, True, '/login?next=%2Flab'],
+      [result.status_code, result.is_redirect, result.headers['Location']]
+    )

software/slapos-master/buildout.hash.cfg

@@ -14,11 +14,11 @@
 # not need these here).
 [template-erp5]
 filename = instance-erp5.cfg.in
-md5sum = 2ef0ddc206c6b0982a37cfc21f23e423
+md5sum = 4d0839e359c98ba3cd516903b72f798b
 
 [template-balancer]
 filename = instance-balancer.cfg.in
-md5sum = 4998e62351f54700ee23a2ca8cd89329
+md5sum = 9c67c77eab5195b2674e340fb44c48a2
 
 [template-apache-backend-conf]
 filename = apache-backend.conf.in

software/slapos-master/instance-balancer.cfg.in

@@ -259,14 +259,24 @@ command = ${monitor-generate-apachedex-report-wrapper:wrapper-path}
 [monitor-generate-apachedex-report-wrapper]
 recipe = slapos.cookbook:wrapper
 wrapper-path = ${directory:bin}/${:command}
-command-line = "{{ parameter_dict['run-apachedex-location'] }}" "{{ parameter_dict['apachedex-location'] }}" "${directory:apachedex}" ${monitor-publish-parameters:monitor-base-url}/private/apachedex --apache-log-list "${apachedex-parameters:apache-log-list}" --configuration "${apachedex-parameters:configuration}"
+command-line = "{{ parameter_dict['run-apachedex-location'] }}" "{{ parameter_dict['apachedex-location'] }}" "${directory:apachedex}" ${monitor-publish-parameters:monitor-base-url}/private/apachedex --apache-log-list "${apachedex-parameters:apache-log-list}" --configuration ${apachedex-parameters:configuration}
 command = generate-apachedex-report
 
+[monitor-apachedex-report-config]
+recipe = slapos.recipe.template:jinja2
+rendered = ${directory:etc}/${:_buildout_section_name_}
+template = inline:
+  {% for line in slapparameter_dict['apachedex-configuration'] %}
+    {# apachedex config files use shlex.split, so we need to quote the arguments. #}
+    {# BBB: in python 3 we can use shlex.quote instead. #}
+    {{ repr(line.encode('utf-8')) }}
+  {% endfor %}
+
 [apachedex-parameters]
 # XXX - Sample log file with curent date: apache_access.log-%(date)s.gz
 # which will be equivalent to apache_access.log-20150112.gz if the date is 2015-01-12
 apache-log-list = ${apache-conf-parameter-dict:access-log}
-configuration = {{ slapparameter_dict['apachedex-configuration'] }}
+configuration = ${monitor-apachedex-report-config:rendered}
 promise-threshold = {{ slapparameter_dict['apachedex-promise-threshold'] }}
 
 [{{ section('monitor-promise-apachedex-result') }}]

software/slapos-master/instance-erp5.cfg.in

@@ -374,8 +374,17 @@ config-shared-certificate-authority-path = ${directory:ca-dir}
 config-backend-path-dict = {{ dumps(zope_backend_path_dict) }}
 config-ssl-authentication-dict = {{ dumps(ssl_authentication_dict) }}
 config-apachedex-promise-threshold = {{ dumps(monitor_dict.get('apachedex-promise-threshold', 70)) }}
-config-apachedex-configuration = {{ dumps(monitor_dict.get('apachedex-configuration',
-'--erp5-base +erp5 .*/VirtualHostRoot/erp5(/|\\?|$) --base +other / --skip-user-agent Zabbix --error-detail --js-embed --quiet')) }}
+config-apachedex-configuration = {{
+  dumps(
+    monitor_dict.get(
+      'apachedex-configuration',
+      [
+        '--erp5-base', '+erp5', '.*/VirtualHostRoot/erp5(/|\\?|$)',
+        '--base', '+other', '/',
+        '--skip-user-agent', 'Zabbix',
+        '--error-detail',
+        '--js-embed',
+        '--quiet'])) }}
 
 [request-frontend-base]
 {% if has_frontend -%}

software/slapos-testing/software.cfg

@@ -203,22 +203,29 @@ output = ${buildout:directory}/template.cfg
 mode = 640
 
 [versions]
-# clear version pins of tested eggs, to make sure buildout always use the
-# git checkout version.
+# When possible, clear version pins of tested eggs, to make sure buildout
+# always use the git checkout version.
+# This is not possible for buildout extensions, or for buildout recipes
+# that are also used in this profile, so we keep version pins for these.
+# We need to keep the versions in the profiles (in stack/slapos.cfg, or
+# maybe here during development) in sync with the latest version from their
+# setup.py , because if the version in setup.py is newer from the version
+# pin, buildout will install the egg from pypi and run the test against
+# released egg
 caucase =
 erp5.util =
 kedifa =
 slapos.cookbook =
 slapos.core =
-slapos.libnetworkcache =
-slapos.rebootstrap =
-slapos.recipe.build =
-slapos.recipe.cmmi =
-slapos.recipe.template =
+# slapos.libnetworkcache =
+# slapos.rebootstrap =
+# slapos.recipe.build =
+# slapos.recipe.cmmi =
+# slapos.recipe.template =
 slapos.toolbox =
 rubygemsrecipe =
 
-# All depencies should be pinned.
+# All other depencies should be pinned.
 Pygments = 2.1.3
 zc.lockfile = 1.4
 bcrypt = 3.1.4

software/theia/buildout.hash.cfg

@@ -19,7 +19,7 @@ md5sum = a3e4cb7d28daa7816f04359c8aa3445b
 
 [yarn.lock]
 filename = yarn.lock
-md5sum = 5a89742266a9f9d4115efa6d641fd5bb
+md5sum = e4b8d436916e48d354342894d6ffecb7
 
 [python-language-server-requirements.txt]
 filename = python-language-server-requirements.txt

software/theia/generate_download_plugins_cfg.py

+import configparser
+import requests
+import hashlib
+
+urls = []
+
+for plugin_and_version in '''\
+    vscode/bat/latest
+    vscode/clojure/latest
+    vscode/coffeescript/latest
+    vscode/configuration-editing/latest
+    vscode/cpp/latest
+    vscode/csharp/latest
+    vscode/css/latest
+    vscode/css-language-features/latest
+    vscode/debug-auto-launch/latest
+    vscode/docker/latest
+    vscode/emmet/latest
+    vscode/fsharp/latest
+    vscode/go/latest
+    vscode/groovy/latest
+    vscode/grunt/latest
+    vscode/gulp/latest
+    vscode/handlebars/latest
+    vscode/hlsl/latest
+    vscode/html/latest
+    vscode/html-language-features/latest
+    vscode/ini/latest
+    vscode/jake/latest
+    vscode/java/latest
+    vscode/javascript/latest
+    ms-vscode/js-debug/latest
+    vscode/json/latest
+# latest json-language-features does offer completions with .theia/settings.json
+    vscode/json-language-features/1.45.1
+    vscode/less/latest
+    vscode/log/latest
+    vscode/lua/latest
+    vscode/make/latest
+    vscode/markdown/latest
+# https://github.com/eclipse-theia/theia/issues/7780
+    vscode/markdown-language-features/1.39.1
+    vscode/merge-conflict/latest
+    vscode/npm/latest
+    ms-vscode/node-debug/latest
+    ms-vscode/node-debug2/latest
+    vscode/objective-c/latest
+    vscode/perl/latest
+    vscode/powershell/latest
+    vscode/pug/latest
+    vscode/python/latest
+    vscode/r/latest
+    vscode/razor/latest
+    vscode/ruby/latest
+    vscode/rust/latest
+    vscode/scss/latest
+    vscode/shaderlab/latest
+    vscode/shellscript/latest
+    vscode/sql/latest
+    vscode/swift/latest
+    vscode/theme-abyss/latest
+    vscode/theme-defaults/latest
+    vscode/theme-kimbie-dark/latest
+    vscode/theme-monokai/latest
+    vscode/theme-monokai-dimmed/latest
+    vscode/theme-quietlight/latest
+    vscode/theme-red/latest
+    vscode/theme-solarized-dark/latest
+    vscode/theme-tomorrow-night-blue/latest
+    vscode/typescript/latest
+    vscode/typescript-language-features/latest
+    vscode/vb/latest
+    vscode/vscode-theme-seti/latest
+    vscode/xml/latest
+    vscode/yaml/latest
+    EditorConfig/EditorConfig/latest
+    dbaeumer/vscode-eslint/latest
+    ms-vscode/references-view/latest
+    golang/Go/0.16.2
+    vscjava/vscode-java-debug/0.29.0
+    redhat/java/0.61.0
+    vscjava/vscode-java-test/0.26.0
+    ms-python/python/2020.9.112786
+    perrinjerome/vscode-zc-buildout/latest
+    jebbs/plantuml/2.13.12
+    rafaelmaiolla/diff/latest
+    perrinjerome/git-commit-syntax/latest
+    perrinjerome/git-rebase-syntax/latest
+'''.splitlines():
+  plugin_and_version = plugin_and_version.strip()
+  if not plugin_and_version or plugin_and_version.startswith('#'):
+    continue
+  publisher, extension_name, version = plugin_and_version.split('/')
+
+  api_url = f'https://open-vsx.org/api/{publisher}/{extension_name}/{version}'
+  download_url = requests.get(api_url).json()['files']['download']
+  md5sum = hashlib.md5(requests.get(download_url).content).hexdigest()
+  urls.append(f'{publisher}-{extension_name} {download_url} {md5sum}')
+
+cfg = configparser.ConfigParser()
+cfg.add_section('theia-download-plugins')
+cfg.set('theia-download-plugins', 'urls', '\n'.join(urls))
+
+with open('download-plugins.cfg', 'w') as f:
+  f.write(f"""\
+# This file is automatically generated from {__file__}
+# Do not edit directly.
+""")
+  cfg.write(f)

stack/erp5/buildout.cfg

@@ -623,7 +623,7 @@ zope.app.publication = 3.14.0
 zope.app.testing = 3.8.1
 
 # Pinned versions
-APacheDEX = 1.6.2
+APacheDEX = 1.8
 Pillow = 6.2.2
 Products.CMFActionIcons = 2.1.3
 Products.DCWorkflowGraph = 0.4.1

stack/erp5/dummy.cfg

-

stack/erp5/instance-balancer.cfg.in

@@ -360,12 +360,22 @@ command = ${monitor-generate-apachedex-report-wrapper:wrapper-path}
 [monitor-generate-apachedex-report-wrapper]
 recipe = slapos.cookbook:wrapper
 wrapper-path = ${directory:bin}/${:command}
-command-line = "{{ parameter_dict['run-apachedex-location'] }}" "{{ parameter_dict['apachedex-location'] }}" "${directory:apachedex}" ${monitor-publish-parameters:monitor-base-url}/private/apachedex --apache-log-list "${apachedex-parameters:apache-log-list}" --configuration "${apachedex-parameters:configuration}"
+command-line = "{{ parameter_dict['run-apachedex-location'] }}" "{{ parameter_dict['apachedex-location'] }}" "${directory:apachedex}" ${monitor-publish-parameters:monitor-base-url}/private/apachedex --apache-log-list "${apachedex-parameters:apache-log-list}" --configuration ${apachedex-parameters:configuration}
 command = generate-apachedex-report
 
+[monitor-apachedex-report-config]
+recipe = slapos.recipe.template:jinja2
+rendered = ${directory:etc}/${:_buildout_section_name_}
+template = inline:
+  {% for line in slapparameter_dict['apachedex-configuration'] %}
+    {# apachedex config files use shlex.split, so we need to quote the arguments. #}
+    {# BBB: in python 3 we can use shlex.quote instead. #}
+    {{ repr(line.encode('utf-8')) }}
+  {% endfor %}
+
 [apachedex-parameters]
 apache-log-list = ${rsyslogd-cfg-parameter-dict:access-log-file}
-configuration = {{ slapparameter_dict['apachedex-configuration'] }}
+configuration = ${monitor-apachedex-report-config:rendered}
 promise-threshold = {{ slapparameter_dict['apachedex-promise-threshold'] }}
 
 [{{ section('monitor-promise-apachedex-result') }}]

stack/erp5/instance-erp5.cfg.in

@@ -64,9 +64,8 @@ config-name = {{ name }}
 
 [directory]
 recipe = slapos.cookbook:mkdirectory
-etc = ${buildout:directory}/etc
-plugin = ${:etc}/plugin
-service-on-watch = ${:etc}/service
+bin = ${buildout:directory}/bin
+service-on-watch = ${buildout:directory}/etc/service
 srv = ${buildout:directory}/srv
 backup-caucased = ${:srv}/backup/caucased
 
@@ -351,8 +350,18 @@ config-name = ${:name}
 config-backend-path-dict = {{ dumps(zope_backend_path_dict) }}
 config-ssl-authentication-dict = {{ dumps(ssl_authentication_dict) }}
 config-apachedex-promise-threshold = {{ dumps(monitor_dict.get('apachedex-promise-threshold', 70)) }}
-config-apachedex-configuration = {{ dumps(monitor_dict.get('apachedex-configuration',
-'--erp5-base +erp5 .*/VirtualHostRoot/erp5(/|\\?|$) --base +other / --skip-user-agent Zabbix --error-detail --js-embed --quiet')) }}
+config-apachedex-configuration = {{
+  dumps(
+    monitor_dict.get(
+      'apachedex-configuration',
+      [
+        '--logformat', '%h %l %u %t "%r" %>s %O "%{Referer}i" "%{User-Agent}i" %{ms}T',
+        '--erp5-base', '+erp5', '.*/VirtualHostRoot/erp5(/|\\?|$)',
+        '--base', '+other', '/',
+        '--skip-user-agent', 'Zabbix',
+        '--error-detail',
+        '--js-embed',
+        '--quiet'])) }}
 
 [request-frontend-base]
 {% if has_frontend -%}

stack/erp5/zope-versions.cfg

@@ -5,7 +5,6 @@ Acquisition = 2.13.12
 DateTime = 2.12.8
 DocumentTemplate = 2.13.6
 ExtensionClass = 2.13.2
-Jinja2 = 2.8.1
 MarkupSafe = 1.1.1
 Missing = 2.13.1
 MultiMapping = 2.13.0

stack/erp5/ztk-versions.cfg

@@ -74,7 +74,6 @@ zope.documenttemplate = 3.4.3
 ClientForm = 0.2.10
 distribute = 0.6.49
 docutils = 0.7
-Jinja2 = 2.5.5
 # Newer versions of mechanize are not fully py24 compatible.
 mechanize = 0.1.11
 Paste = 1.7.5.1

stack/slapos.cfg

@@ -142,7 +142,7 @@ zc.recipe.egg = 2.0.3+slapos003
 # Use own version of h.r.download to be able to open .xz and .lz archives
 hexagonit.recipe.download = 1.7.post4
 
-Jinja2 = 2.9.5
+Jinja2 = 2.11.2
 Importing = 1.10
 MarkupSafe = 1.0
 PyYAML = 3.13
@@ -200,7 +200,7 @@ slapos.rebootstrap = 4.5
 slapos.recipe.build = 0.46
 slapos.recipe.cmmi = 0.16
 slapos.recipe.template = 4.5
-slapos.toolbox = 0.112
+slapos.toolbox = 0.115
 stevedore = 1.21.0
 subprocess32 = 3.5.3
 unicodecsv = 0.14.1

update-rc

 #!/bin/bash
 
+set -e
+
+# Go to master
+git checkout master
+
+# Clean up 1.0
+git branch -D 1.0 || echo
+
 # Checkout 1.0
-git checkout 1.0
+git checkout origin/1.0 -b 1.0
 
 # Reset and Clean
 git reset --hard origin/1.0