Update Release Candidate

component/apache/buildout.cfg

@@ -38,9 +38,9 @@ md5sum = 2202b18f269ad606d70e1864857ed93c
 # inspired on http://old.aclark.net/team/aclark/blog/a-lamp-buildout-for-wordpress-and-other-php-apps/
 recipe = slapos.recipe.cmmi
 shared = false
-version = 2.4.37
+version = 2.4.38
 url = https://archive.apache.org/dist/httpd/httpd-${:version}.tar.bz2
-md5sum = 6a36e742180ee74bff97b28eee90c3f7
+md5sum = 4a2b87ca55e42017d21f18724c560084
 pre-configure =
   cp -ar ${apr:location}/apr-${apr:version} srclib/apr/ &&
   cp -ar ${apr-util:location}/apr-util-${apr-util:version} srclib/apr-util

component/python-2.7/buildout.cfg

@@ -29,9 +29,9 @@ python = python2.7
 [python2.7]
 recipe = slapos.recipe.cmmi
 shared = true
-package_version = 2.7.14
+package_version = 2.7.15
 package_version_suffix =
-md5sum = 1f6db41ad91d9eb0a6f0c769b8613c5b
+md5sum = a80ae3cc478460b922242f43a1b4094d
 
 # This is actually the default setting for prefix, but we can't use it in
 # other settings in this part if we don't set it explicitly here.
@@ -42,7 +42,7 @@ executable = ${:prefix}/bin/python${:version}
 patch-options = -p1
 patches =
   ${:_profile_base_location_}/fix_compiler_module_issue_20613.patch#94443a77f903e9de880a029967fa6aa7
-  ${:_profile_base_location_}/pytracemalloc_pep445.patch#46662cf0ccc7cb7cfb8289bbfd68b21a
+  ${:_profile_base_location_}/pytracemalloc_pep445.patch#3dfad79654af9671325f988c36fb6be2
   ${:_profile_base_location_}/disabled_module_list.patch#71ad30d32bcdbc50c19cf48675b1246e
   ${:_profile_base_location_}/asyncore_poll_insteadof_select.patch#ab6991c0ee6e25aeb8951e71f280a2f1
 url =

component/ruby/buildout.cfg

@@ -21,7 +21,7 @@ configure-options =
 environment =
   PATH=${xz-utils:location}/bin:%(PATH)s
   CPPFLAGS=-I${gdbm:location}/include -I${libffi:location}/include -I${libyaml:location}/include -I${ncurses:location}/include -I${ncurses:location}/include/ncursesw -I${openssl-1.0:location}/include -I${readline:location}/include -I${zlib:location}/include
-  LDFLAGS=-L${gdbm:location}/lib -L${libffi:location}/lib -L${libyaml:location}/lib -L${ncurses:location}/lib -L${openssl-1.0:location}/lib -L${readline:location}/lib -L${zlib:location}/lib -Wl,-rpath=${gdbm:location}/lib  -Wl,-rpath=${libffi:location}/lib -Wl,-rpath=${libyaml:location}/lib -Wl,-rpath=${ncurses:location}/lib -Wl,-rpath=${openssl:location}/lib -Wl,-rpath=${readline:location}/lib -Wl,-rpath=${zlib:location}/lib
+  LDFLAGS=-L${gdbm:location}/lib -L${libffi:location}/lib -L${libyaml:location}/lib -L${ncurses:location}/lib -L${openssl-1.0:location}/lib -L${readline:location}/lib -L${zlib:location}/lib -Wl,-rpath=${gdbm:location}/lib  -Wl,-rpath=${libffi:location}/lib -Wl,-rpath=${libyaml:location}/lib -Wl,-rpath=${ncurses:location}/lib -Wl,-rpath=${openssl-1.0:location}/lib -Wl,-rpath=${readline:location}/lib -Wl,-rpath=${zlib:location}/lib
   PKG_CONFIG_PATH=${libyaml:location}/lib/
 
 

component/zstd/buildout.cfg

@@ -4,8 +4,8 @@ parts =
 
 [zstd]
 recipe = slapos.recipe.cmmi
-url = https://github.com/facebook/zstd/archive/v1.3.5.tar.gz
-md5sum = d2fc97be4852d666f086282053c02319
+url = https://github.com/facebook/zstd/archive/v1.3.8.tar.gz
+md5sum = 0e85b062003729776c6114cd9632cf75
 shared = true
 location = @@LOCATION@@
 configure-command = :

software/apache-frontend/instance-slave-apache-input-schema.json

@@ -97,6 +97,7 @@
       "title": "SSL Backend Authority's Certificate",
       "description": "Content of the SSL Certificate Authority file of the backend (to be used with ssl-proxy-verify)",
       "type": "string",
+      "textarea": true,
       "default": ""
     },
     "monitor-ipv6-test": {

software/caddy-frontend/buildout.hash.cfg

@@ -14,7 +14,7 @@
 # not need these here).
 [template]
 filename = instance.cfg.in
-md5sum = de7e30546a952e306e2a74f8492ab419
+md5sum = 2747f9125c8dffa0c27b79a6902a55cb
 
 [template-common]
 filename = instance-common.cfg.in
@@ -22,15 +22,15 @@ md5sum = c801b7f9f11f0965677c22e6bbe9281b
 
 [template-apache-frontend]
 filename = instance-apache-frontend.cfg.in
-md5sum = 2c8e1dd8df4a225b4a8d8f70688a85ca
+md5sum = da7b5804d50c727412885e8b818ca433
 
 [template-apache-replicate]
 filename = instance-apache-replicate.cfg.in
-md5sum = 6a86edb96b171fbd0a59d0adc9cc906b
+md5sum = 86f15c85054800a55a7093d37b03cdce
 
 [template-slave-list]
 filename = templates/apache-custom-slave-list.cfg.in
-md5sum = 4b06918875e889f850c5aca0d8c57796
+md5sum = 5b91c0079b42780054da004f8fc6f26b
 
 [template-slave-configuration]
 filename = templates/custom-virtualhost.conf.in
@@ -58,11 +58,11 @@ md5sum = f20d6c3d2d94fb685f8d26dfca1e822b
 
 [template-default-slave-virtualhost]
 filename = templates/default-virtualhost.conf.in
-md5sum = a0b5a3dbf7b1d6622a52173ef5a90e72
+md5sum = e57b9ae012f777482295698a23b7e850
 
 [template-cached-slave-virtualhost]
 filename = templates/cached-virtualhost.conf.in
-md5sum = c64f8ac7ec439460877ce5a5c5ccf1f7
+md5sum = 907372828d1ceb05c41240078196f439
 
 [template-log-access]
 filename = templates/template-log-access.conf.in
@@ -94,7 +94,7 @@ md5sum = 176cbca2070734a185a7ae5a4d1181c5
 
 [template-nginx-notebook-slave-virtualhost]
 filename = templates/nginx-notebook-slave.conf.in
-md5sum = 2b765db72191197122554df17ad471d1
+md5sum = ee3b5c23f1c81aa43ce7cd8f8e327f70
 
 [template-apache-lazy-script-call]
 filename = templates/apache-lazy-script-call.sh.in

software/caddy-frontend/instance-apache-frontend.cfg.in

@@ -202,6 +202,8 @@ extra-context =
     key login_ca_crt ca-custom-frontend:rendered
     key enable_http2_by_default configuration:enable-http2-by-default
     key global_disable_http2 configuration:global-disable-http2
+    key proxy_try_duration configuration:proxy-try-duration
+    key proxy_try_interval configuration:proxy-try-interval
     key access_log caddy-configuration:access-log
     key error_log caddy-configuration:error-log
     key sixtunnel_executable :sixtunnel_executable

software/caddy-frontend/instance-apache-replicate.cfg.in

@@ -80,19 +80,19 @@ context =
 {%      do used_host_list.append(custom_domain) %}
 {%   endif %}
 {%   if slave.get('server-alias') %}
-{%     for slave_alias in slave['server-alias'].split() %}
+{%     for slave_alias in ('' ~ slave['server-alias']).split() %}
 {%       if slave_alias.startswith('*.') %}
 {%         set clean_slave_alias = slave_alias[2:] %}
 {%       else %}
 {%         set clean_slave_alias = slave_alias %}
 {%       endif %}
 {%       if not validators.domain(clean_slave_alias) %}
-{%         do slave_error_list.append('server-alias %r not valid' % (slave_alias,)) %}
+{%         do slave_error_list.append('server-alias \'%s\' not valid' % (slave_alias,)) %}
 {%       else %}
 {%         if slave_alias in slave_server_alias_unclashed or slave_alias == custom_domain %}
 {#           optionally do something about reporting back that server-alias has been unclashed #}
 {%         elif slave_alias in used_host_list %}
-{%           do slave_error_list.append('server-alias %r clashes' % (slave_alias,)) %}
+{%           do slave_error_list.append('server-alias \'%s\' clashes' % (slave_alias,)) %}
 {%         else %}
 {%           do slave_server_alias_unclashed.append(slave_alias) %}
 {%           do used_host_list.append(slave_alias) %}
@@ -107,18 +107,18 @@ context =
 {%         if not unauthorized_message in slave_error_list %}
 {%           do slave_error_list.append(unauthorized_message) %}
 {%         endif %}
-{%       elif subprocess_module.call([caddy_custom_http_validator, slave[key]]) == 1 %}
+{%       elif subprocess_module.call([caddy_custom_http_validator, '' ~ slave[key]]) == 1 %}
 {%         do slave_error_list.append('slave %s configuration invalid' % (key,)) %}
 {%       endif %}
 {%     endif %}
 {%   endfor %} {# for key in ['caddy_custom_http', 'caddy_custom_https', 'apache_custom_http', 'apache_custom_https'] #}
 {%   if slave.get('url') %}
-{%     if subprocess_module.call([caddy_backend_url_validator, slave['url']]) == 1 %}
+{%     if subprocess_module.call([caddy_backend_url_validator, '' ~ slave['url']]) == 1 %}
 {%       do slave_error_list.append('slave url %r invalid' % (slave['url'],)) %}
 {%     endif %}
 {%   endif %}
 {%   if slave.get('https-url') %}
-{%     if subprocess_module.call([caddy_backend_url_validator, slave['https-url']]) == 1 %}
+{%     if subprocess_module.call([caddy_backend_url_validator, '' ~ slave['https-url']]) == 1 %}
 {%       do slave_error_list.append('slave https-url %r invalid' % (slave['https-url'],)) %}
 {%     endif %}
 {%   endif %}
@@ -128,17 +128,18 @@ context =
 {%   if slave.get('ssl_key') and slave.get('ssl_crt') %}
 {%     set key_popen = popen([openssl, 'rsa', '-noout', '-modulus']) %}
 {%     set crt_popen = popen([openssl, 'x509', '-noout', '-modulus']) %}
-{%     set key_modulus = key_popen.communicate(slave['ssl_key'])[0] | trim %}
-{%     set crt_modulus = crt_popen.communicate(slave['ssl_crt'])[0] | trim %}
+{%     set key_modulus = key_popen.communicate('' ~ slave['ssl_key'])[0] | trim %}
+{%     set crt_modulus = crt_popen.communicate('' ~ slave['ssl_crt'])[0] | trim %}
 {%     if not key_modulus or key_modulus != crt_modulus  %}
 {%       do slave_error_list.append('slave ssl_key and ssl_crt does not match')  %}
 {%     endif %}
 {%   endif %}
 {%   if slave.get('custom_domain') %}
-{%     if slave['custom_domain'].startswith('*.') %}
-{%       set clean_custom_domain = slave['custom_domain'][2:] %}
+{%     set slave_custom_domain = '' ~ slave['custom_domain'] %}
+{%     if slave_custom_domain.startswith('*.') %}
+{%       set clean_custom_domain = slave_custom_domain[2:] %}
 {%     else %}
-{%       set clean_custom_domain = slave['custom_domain'] %}
+{%       set clean_custom_domain = slave_custom_domain %}
 {%     endif %}
 {%     if not validators.domain(clean_custom_domain) %}
 {%       do slave_error_list.append('custom_domain %r invalid' % (slave['custom_domain'],)) %}

software/caddy-frontend/instance-caddy-input-schema.json

@@ -86,6 +86,18 @@
       ],
       "title": "Enable QUIC",
       "type": "string"
+    },
+    "proxy-try-duration": {
+      "default": 5,
+      "description": "A time during which Caddy will try to establish connection with a backend. Setting it to 0 will result with immediate return of 502 EOF error to the browser, even if it would be possible to (re)connect to the backend during few moments. More info in https://caddyserver.com/docs/proxy try_durtion.",
+      "title": "Duration in seconds of trying a backend",
+      "type": "integer"
+    },
+    "proxy-try-interval": {
+      "default": 250,
+      "description": "How often Caddy will try to establish connection with a backend during proxy-try-duration. More info in https://caddyserver.com/docs/proxy try_interval",
+      "title": "Interval in milliseconds of tries during proxy-try-duration",
+      "type": "integer"
     }
   },
   "title": "Input Parameters",

software/caddy-frontend/instance-slave-caddy-input-schema.json

@@ -163,6 +163,7 @@
     "ssl_proxy_ca_crt": {
       "default": "",
       "description": "Content of the SSL Certificate Authority file of the backend (to be used with ssl-proxy-verify)",
+      "textarea": true,
       "title": "SSL Backend Authority's Certificate",
       "type": "string"
     },

software/caddy-frontend/instance.cfg.in

@@ -102,3 +102,5 @@ configuration.enable-quic = false
 configuration.mpm-graceful-shutdown-timeout = 5
 configuration.monitor-httpd-port = 8072
 configuration.frontend-name =
+configuration.proxy-try-duration = 5
+configuration.proxy-try-interval = 250

software/caddy-frontend/templates/apache-custom-slave-list.cfg.in

@@ -66,6 +66,10 @@ crl = {{ custom_ssl_directory }}/crl/
 {%   do slave_instance.__setitem__('enable_http2_by_default', enable_http2_by_default) %}
 {%   do slave_instance.__setitem__('global_disable_http2', global_disable_http2) %}
 
+{#   Pass proxy_try_duration and proxy_try_interval #}
+{%   do slave_instance.__setitem__('proxy_try_duration', proxy_try_duration) %}
+{%   do slave_instance.__setitem__('proxy_try_interval', proxy_try_interval) %}
+
 {#   Set Up log files #}
 {%   do slave_parameter_dict.__setitem__('access_log', '/'.join([caddy_log_directory, '%s_access_log' % slave_reference])) %}
 {%   do slave_parameter_dict.__setitem__('error_log', '/'.join([caddy_log_directory, '%s_error_log' % slave_reference])) %}
@@ -158,7 +162,7 @@ bytes = 8
 
 {# Set ssl certificates for each slave #}
 {%   for cert_name in ('ssl_csr', 'ssl_proxy_ca_crt')%}
-{%     if cert_name in slave_instance %}
+{%     if slave_instance.get(cert_name) %}
 {%       set cert_title = '%s-%s' % (slave_reference, cert_name.replace('ssl_', '')) %}
 {%       set cert_file = '/'.join([custom_ssl_directory, cert_title.replace('-','.')]) %}
 {%       do part_list.append(cert_title) %}
@@ -183,7 +187,7 @@ value = {{ dumps(slave_instance.get(cert_name)) }}
 {%   do slave_instance.__setitem__('login_ca_crt', login_ca_crt) %}
 {%   do slave_parameter_dict.__setitem__('ssl_crt', login_certificate) %}
 {%   do slave_parameter_dict.__setitem__('ssl_key', login_key) %}
-{%   if 'ssl_key' in slave_instance and 'ssl_crt' in slave_instance %}
+{%   if slave_instance.get('ssl_key') and slave_instance.get('ssl_crt') %}
 {%     set cert_title = '%s-crt' % (slave_reference) %}
 {%     set key_title = '%s-key' % (slave_reference) %}
 {%     set cert_file = '/'.join([custom_ssl_directory, cert_title.replace('-','.')]) %}
@@ -207,7 +211,7 @@ extra-context =
 < = jinja2-template-base
 template = {{ empty_template }}
 rendered = {{ cert_file }}
-cert-content = {{ dumps(slave_instance.get('ssl_crt') + '\n' + slave_instance.get('ssl_ca_crt', '')) }}
+cert-content = {{ dumps('%s\n%s' % (slave_instance.get('ssl_crt'), slave_instance.get('ssl_ca_crt', '') or '')) }}
 extra-context =
     key content :cert-content
 {%     endif %}
@@ -215,21 +219,33 @@ extra-context =
 {# ########################################## #}
 {# Set Slave Configuration                    #}
 [{{ slave_configuration_section_name }}]
-https_port = {{ dumps(https_port) }}
-http_port = {{ dumps(http_port) }}
-local_ipv4 = {{ dumps(local_ipv4) }}
-nginx_http_port = {{ dumps(nginx_http_port) }}
-nginx_https_port = {{ dumps(nginx_https_port) }}
-cached_port = {{ dumps(cached_port) }}
-ssl_cached_port = {{ (ssl_cached_port) }}
+https_port = {{ dumps('' ~ https_port) }}
+http_port = {{ dumps('' ~ http_port) }}
+local_ipv4 = {{ dumps('' ~ local_ipv4) }}
+nginx_http_port = {{ dumps('' ~ nginx_http_port) }}
+nginx_https_port = {{ dumps('' ~ nginx_https_port) }}
+cached_port = {{ dumps('' ~ cached_port) }}
+ssl_cached_port = {{ ('' ~ ssl_cached_port) }}
 {# BBB: apache_custom_https and apache_custom_http #}
-{%     set caddy_custom_http = ((slave_instance.pop('caddy_custom_http', slave_instance.pop('apache_custom_http', ''))) % slave_parameter_dict) %}
-{%     set caddy_custom_https = ((slave_instance.pop('caddy_custom_https', slave_instance.pop('apache_custom_https', ''))) % slave_parameter_dict) %}
+{%     set caddy_custom_http_template = slave_instance.pop('caddy_custom_http', slave_instance.pop('apache_custom_http', '')) %}
+{%     set caddy_custom_https_template = slave_instance.pop('caddy_custom_https', slave_instance.pop('apache_custom_https', '')) %}
+{%     if caddy_custom_http_template is not none %}
+{%       set caddy_custom_http = ('' ~ caddy_custom_http_template) % slave_parameter_dict %}
 caddy_custom_http = {{ dumps(caddy_custom_http) }}
+{%     else %}
+{%       set caddy_custom_http = None %}
+{%     endif %}
+{%     if caddy_custom_https_template is not none %}
+{%       set caddy_custom_https = ('' ~ caddy_custom_https_template) % slave_parameter_dict %}
 caddy_custom_https = {{ dumps(caddy_custom_https) }}
+{%     else %}
+{%       set caddy_custom_https = None %}
+{%     endif %}
 {{ '\n' }}
 {%     for key, value in slave_instance.iteritems() %}
-{{ key }} = {{ dumps(value) }}
+{%       if value is not none %}
+{{ key }} = {{ dumps('' ~ value) }}
+{%       endif %}
 {%     endfor %}
 
 [{{ slave_section_title }}]
@@ -295,7 +311,7 @@ filename = {{ monitor_ipv4_section_title }}
 wrapper-path = {{ promise_directory }}/${:filename}
 {%   endif %}
 
-{%   set re6st_optimal_test = slave_instance.get('re6st-optimal-test', '') %}
+{%   set re6st_optimal_test = '' ~ slave_instance.get('re6st-optimal-test', '') %}
 {%   set re6st_ipv6 = None %}
 {%   set re6st_ipv4 = None %}
 {%   if ',' in re6st_optimal_test %}

software/caddy-frontend/templates/cached-virtualhost.conf.in

@@ -21,6 +21,8 @@
   gzip
 # Rewrite part
   proxy / {{ slave_parameter.get('backend_url', '') }} {
+    try_duration {{ slave_parameter['proxy_try_duration'] }}s
+    try_interval {{ slave_parameter['proxy_try_interval'] }}ms
     # As backend is trusting REMOTE_USER header unset it always
     header_upstream -REMOTE_USER
 
@@ -42,6 +44,8 @@
   # Compress the output
   gzip
   proxy / {{ slave_parameter.get('https_backend_url', '') }} {
+    try_duration {{ slave_parameter['proxy_try_duration'] }}s
+    try_interval {{ slave_parameter['proxy_try_interval'] }}ms
     # As backend is trusting REMOTE_USER header unset it always
     header_upstream -REMOTE_USER
     transparent

software/caddy-frontend/templates/default-virtualhost.conf.in

 {%- set TRUE_VALUES = ['y', 'yes', '1', 'true'] %}
-{%- set disable_no_cache_header = ('' ~ slave_parameter.get('disable-no-cache-request', '')).lower() in TRUE_VALUES %}
-{%- set disable_via_header = ('' ~ slave_parameter.get('disable-via-header', '')).lower() in TRUE_VALUES %}
-{%- set prefer_gzip = ('' ~ slave_parameter.get('prefer-gzip-encoding-to-backend', '')).lower() in TRUE_VALUES %}
+{%- set disable_no_cache_header = slave_parameter.get('disable-no-cache-request', '').lower() in TRUE_VALUES %}
+{%- set disable_via_header = slave_parameter.get('disable-via-header', '').lower() in TRUE_VALUES %}
+{%- set prefer_gzip = slave_parameter.get('prefer-gzip-encoding-to-backend', '').lower() in TRUE_VALUES %}
 {%- set proxy_append_list = [('', 'Default proxy configuration')] %}
 {%- if prefer_gzip %}
 {%- do proxy_append_list.append(('prefer-gzip', 'Proxy which always overrides Accept-Encoding to gzip if such is found')) %}
 {%- endif %} {#- if prefer_gzip #}
 {%- set server_alias_list =  slave_parameter.get('server-alias', '').split() %}
-{%- set enable_h2 = slave_parameter['global_disable_http2'].lower() not in TRUE_VALUES and ('' ~ slave_parameter.get('enable-http2', slave_parameter['enable_http2_by_default'])).lower() in TRUE_VALUES %}
-{%- set ssl_proxy_verify = ('' ~ slave_parameter.get('ssl-proxy-verify', '')).lower() in TRUE_VALUES %}
+{%- set enable_h2 = slave_parameter['global_disable_http2'].lower() not in TRUE_VALUES and slave_parameter.get('enable-http2', slave_parameter['enable_http2_by_default']).lower() in TRUE_VALUES %}
+{%- set ssl_proxy_verify = slave_parameter.get('ssl-proxy-verify', '').lower() in TRUE_VALUES %}
 {%- set disabled_cookie_list =  slave_parameter.get('disabled-cookie-list', '').split() %}
-{%- set https_only = ('' ~ slave_parameter.get('https-only', '')).lower() in TRUE_VALUES %}
+{%- set https_only = slave_parameter.get('https-only', '').lower() in TRUE_VALUES %}
 {%- set slave_type = slave_parameter.get('type', '') %}
 {%- set host_list = server_alias_list %}
 {%- if slave_parameter.get('custom_domain') not in host_list %}
@@ -67,6 +67,8 @@
 {%-   for (proxy_name, proxy_comment) in proxy_append_list %}
   # {{ proxy_comment }}
   proxy /{{ proxy_name }} {{ backend_url }} {
+    try_duration {{ slave_parameter['proxy_try_duration'] }}s
+    try_interval {{ slave_parameter['proxy_try_interval'] }}ms
 {%-     if proxy_name == 'prefer-gzip' %}
     without /prefer-gzip
     header_upstream Accept-Encoding gzip
@@ -138,6 +140,8 @@
 {%-   for (proxy_name, proxy_comment) in proxy_append_list %}
   # {{ proxy_comment }}
   proxy /{{ proxy_name }} {{ backend_url }} {
+    try_duration {{ slave_parameter['proxy_try_duration'] }}s
+    try_interval {{ slave_parameter['proxy_try_interval'] }}ms
 {%-     if proxy_name == 'prefer-gzip' %}
     without /prefer-gzip
     header_upstream Accept-Encoding gzip
@@ -216,6 +220,8 @@
 {%-   for (proxy_name, proxy_comment) in proxy_append_list %}
   # {{ proxy_comment }}
   proxy /{{ proxy_name }} {{ backend_url }} {
+    try_duration {{ slave_parameter['proxy_try_duration'] }}s
+    try_interval {{ slave_parameter['proxy_try_interval'] }}ms
 {%-     if proxy_name == 'prefer-gzip' %}
     without /prefer-gzip
     header_upstream Accept-Encoding gzip
@@ -281,6 +287,8 @@
 {%-   for (proxy_name, proxy_comment) in proxy_append_list %}
   # {{ proxy_comment }}
   proxy /{{ proxy_name }} {{ slave_parameter.get('url', '') }} {
+    try_duration {{ slave_parameter['proxy_try_duration'] }}s
+    try_interval {{ slave_parameter['proxy_try_interval'] }}ms
 {%-     if proxy_name == 'prefer-gzip' %}
     without /prefer-gzip
     header_upstream Accept-Encoding gzip

software/caddy-frontend/templates/nginx-notebook-slave.conf.in

@@ -17,6 +17,8 @@ https://{{ slave_parameter.get('custom_domain') }}:{{ slave_parameter['nginx_htt
   }
 
   proxy / {{ https_upstream }} {
+    try_duration {{ slave_parameter['proxy_try_duration'] }}s
+    try_interval {{ slave_parameter['proxy_try_interval'] }}ms
     transparent
     insecure_skip_verify
   }
@@ -25,6 +27,8 @@ https://{{ slave_parameter.get('custom_domain') }}:{{ slave_parameter['nginx_htt
     to /proxy/{1}
   }
   proxy /proxy/ {{ https_upstream }} {
+    try_duration {{ slave_parameter['proxy_try_duration'] }}s
+    try_interval {{ slave_parameter['proxy_try_interval'] }}ms
     header_upstream X-Real-IP {remote}
     header_upstream Host {host}
     websocket
@@ -42,6 +46,8 @@ http://{{ slave_parameter.get('custom_domain') }}:{{ slave_parameter['nginx_http
   errors {{ slave_parameter.get('error_log') }}
 
   proxy / {{ upstream }} {
+    try_duration {{ slave_parameter['proxy_try_duration'] }}s
+    try_interval {{ slave_parameter['proxy_try_interval'] }}ms
     transparent
     insecure_skip_verify
   }
@@ -50,6 +56,8 @@ http://{{ slave_parameter.get('custom_domain') }}:{{ slave_parameter['nginx_http
     to /proxy/{1}
   }
   proxy /proxy/ {{ upstream }} {
+    try_duration {{ slave_parameter['proxy_try_duration'] }}s
+    try_interval {{ slave_parameter['proxy_try_interval'] }}ms
     header_upstream X-Real-IP {remote}
     header_upstream Host {host}
     websocket

software/caddy-frontend/test/test.py

@@ -187,6 +187,18 @@ class CertificateAuthority(object):
     return certificate, certificate.public_bytes(serialization.Encoding.PEM)
 
 
+def subprocess_output(*args, **kwargs):
+  prc = subprocess.Popen(
+    stdout=subprocess.PIPE,
+    stderr=subprocess.STDOUT,
+    *args,
+    **kwargs
+  )
+
+  out, err = prc.communicate()
+  return out
+
+
 def isHTTP2(domain, ip):
   curl_command = 'curl --http2 -v -k -H "Host: %(domain)s" ' \
     'https://%(domain)s:%(https_port)s/ '\
@@ -456,6 +468,8 @@ class TestMasterRequestDomain(HttpFrontendTestCase, TestDataMixin):
 
 class TestHandler(BaseHTTPRequestHandler):
   def do_GET(self):
+    timeout = int(self.headers.dict.get('Timeout', '0'))
+    time.sleep(timeout)
     self.send_response(200)
     self.send_header("Content-type", "application/json")
     self.send_header('Set-Cookie', 'secured=value;secure')
@@ -1157,7 +1171,9 @@ http://apachecustomhttpsaccepted.example.com:%%(http_port)s {
 
     result = self.fakeHTTPSResult(
       parameter_dict['domain'], parameter_dict['public-ipv4'],
-      'test-path/deep/.././deeper')
+      'test-path/deep/.././deeper',
+      headers={'Timeout': '10'}  # more than default proxy-try-duration == 5
+    )
 
     self.assertEqual(
       self.certificate_pem,
@@ -1171,6 +1187,8 @@ http://apachecustomhttpsaccepted.example.com:%%(http_port)s {
       raise ValueError('JSON decode problem in:\n%s' % (result.text,))
     self.assertFalse('remote_user' in j['Incoming Headers'].keys())
 
+    self.assertEqual(j['Incoming Headers']['timeout'], '10')
+
     self.assertFalse('Content-Encoding' in result.headers)
 
     self.assertEqual(
@@ -1196,6 +1214,14 @@ http://apachecustomhttpsaccepted.example.com:%%(http_port)s {
       result_http.headers['Set-Cookie']
     )
 
+    # check that try_duration == 5 in the test_url slave
+    slave_configuration_file = glob.glob(os.path.join(
+      self.instance_path, '*', 'etc', '*slave-conf.d', '_url.conf'))[0]
+    with open(slave_configuration_file) as fh:
+      content = fh.read()
+      self.assertTrue('try_duration 5s' in content)
+      self.assertTrue('try_interval 250ms' in content)
+
   @skip('Feature postponed')
   def test_url_ipv6_access(self):
     parameter_dict = self.parseSlaveParameterDict('url')
@@ -2395,7 +2421,7 @@ http://apachecustomhttpsaccepted.example.com:%%(http_port)s {
         'check-_monitor-ipv6-test-ipv6-packet-list-test'))[0]
     self.assertEqual(
       '-a monitor-ipv6-test',
-      subprocess.check_output(monitor_file).strip()
+      subprocess_output(monitor_file).strip()
     )
 
   def test_monitor_ipv4_test(self):
@@ -2437,7 +2463,7 @@ http://apachecustomhttpsaccepted.example.com:%%(http_port)s {
         'check-_monitor-ipv4-test-ipv4-packet-list-test'))[0]
     self.assertEqual(
       '-4 -a monitor-ipv4-test',
-      subprocess.check_output(monitor_file).strip()
+      subprocess_output(monitor_file).strip()
     )
 
   def test_re6st_optimal_test(self):
@@ -2480,7 +2506,7 @@ http://apachecustomhttpsaccepted.example.com:%%(http_port)s {
         'check-_re6st-optimal-test-re6st-optimal-test'))[0]
     self.assertEqual(
       '-4 ipv4 -6 ipv6',
-      subprocess.check_output(monitor_file).strip()
+      subprocess_output(monitor_file).strip()
     )
 
   def test_enable_cache(self):
@@ -3825,7 +3851,7 @@ https://www.google.com {}""",
     #       correctly passed to the script.
     self.assertEqual(
       '-4 newline [s${esection:eoption} -6 new line;rm -fr ~;',
-      subprocess.check_output(monitor_file).strip()
+      subprocess_output(monitor_file).strip()
     )
 
   def test_re6st_optimal_test_nocomma(self):
@@ -4014,7 +4040,7 @@ https://www.google.com {}""",
         'check-_monitor-ipv4-test-unsafe-ipv4-packet-list-test'))[0]
     self.assertEqual(
       '-4 -a ${section:option} afternewline ipv4',
-      subprocess.check_output(monitor_file).strip()
+      subprocess_output(monitor_file).strip()
     )
 
   def test_monitor_ipv6_test_unsafe(self):
@@ -4056,7 +4082,7 @@ https://www.google.com {}""",
         'check-_monitor-ipv6-test-unsafe-ipv6-packet-list-test'))[0]
     self.assertEqual(
       '-a ${section:option} afternewline ipv6',
-      subprocess.check_output(monitor_file).strip()
+      subprocess_output(monitor_file).strip()
     )
 
   def test_ssl_key_ssl_crt_unsafe(self):

software/kvm/buildout.hash.cfg

+# THIS IS NOT A BUILDOUT FILE, despite purposedly using a compatible syntax.
+# The only allowed lines here are (regexes):
+# - "^#" comments, copied verbatim
+# - "^[" section beginings, copied verbatim
+# - lines containing an "=" sign which must fit in the following categorie.
+#   - "^\s*filename\s*=\s*path\s*$" where "path" is relative to this file
+#     Copied verbatim.
+#   - "^\s*hashtype\s*=.*" where "hashtype" is one of the values supported
+#     by the re-generation script.
+#     Re-generated.
+# - other lines are copied verbatim
+# Substitution (${...:...}), extension ([buildout] extends = ...) and
+# section inheritance (< = ...) are NOT supported (but you should really
+# not need these here).
+
+[template]
+filename = instance.cfg.in
+md5sum = 028b6a6456d744c11b1bb2c51ecd51b2
+
+[template-kvm]
+filename = instance-kvm.cfg.jinja2
+md5sum = e30912b1256bb093329ee3c108666f05
+
+[template-kvm-cluster]
+filename = instance-kvm-cluster.cfg.jinja2.in
+md5sum = 63fa784d8946d0b6e3fbd6381e1ea9f4
+
+[template-kvm-resilient]
+filename = instance-kvm-resilient.cfg.jinja2
+md5sum = c189bce8b196a3b5bcb01526a3f9b2aa
+
+[template-kvm-import]
+filename = instance-kvm-import.cfg.jinja2.in
+md5sum = dc3f3ad9ebd8b3b5c3ded57b91cee9c7
+
+[template-kvm-import-script]
+filename = template/kvm-import.sh.jinja2
+md5sum = cd0008f1689dfca9b77370bc4d275b70
+
+[template-kvm-export]
+filename = instance-kvm-export.cfg.jinja2
+md5sum = 00ce5e6da3c833d9d9d1825311f11a81
+
+[template-kvm-export-script]
+filename = template/kvm-export.sh.jinja2
+md5sum = b617d64de73de1eed518185f310bbc82
+
+[template-nbd]
+filename = instance-nbd.cfg.in
+md5sum = f634a5249b773658b7a7bc9fa9bb0368
+
+[template-frontend]
+filename = instance-frontend.cfg.in
+md5sum = cdb690495e9eb007d2b7d2f8e12f5c59
+
+[template-ansible-promise]
+filename = template/ansible-promise.in
+md5sum = 2036bf145f472f62ef8dee5e729328fd
+
+[template-kvm-run]
+filename = template/template-kvm-run.in
+md5sum = 9e40246b4bc4f968f0631016c939b014
+
+[template-kvm-controller]
+filename = template/kvm-controller-run.in
+md5sum = b96cba47c97f277c857176d69e086a12
+
+[template-apache-conf]
+filename = template/apache.conf.in
+md5sum = ac97f6a52e1c5a19a646242ef85abb8a
+
+[template-content]
+filename = template/template-content.in
+md5sum = 822737e483864bf255ad1259237bef2a
+
+[template-qemu-ready]
+filename = template/qemu-is-ready.in
+md5sum = b304eec8e2cb71f10ea83cac22f6db12
+
+[file-download-script]
+filename = template/download_file.in
+md5sum = 599dbbbd438fe7801e3f8642ae9e9a78
+
+[template-httpd]
+filename = instance-kvm-http.cfg.in
+md5sum = 26a181a48046ce88570adb32334747ef

software/kvm/software.cfg

@@ -12,6 +12,7 @@ extends =
   ../../stack/slapos.cfg
   ../../component/nodejs/buildout.cfg
   ../../stack/resilient/buildout.cfg
+  buildout.hash.cfg
 
 # stacks are listed from most generic to most specific,
 # to avoid versioning issues
@@ -86,158 +87,101 @@ command =
 
 
 # Create all templates that will be used to deploy instances
+[download-base]
+recipe = hexagonit.recipe.download
+url = ${:_profile_base_location_}/${:filename}
+mode = 0644
 
-[template]
+[download-file-base]
+<= download-base
+ignore-existing = true
+download-only = true
+
+[download-template-base]
+<= download-file-base
+url = ${:_profile_base_location_}/template/${:path}
+path = ${:filename}
+
+[template-file-base]
 recipe = slapos.recipe.template
-url = ${:_profile_base_location_}/instance.cfg.in
-md5sum = 028b6a6456d744c11b1bb2c51ecd51b2
-output = ${buildout:directory}/template.cfg
+url = ${:_profile_base_location_}/${:filename}
 mode = 0644
 
+[template]
+<= template-file-base
+output = ${buildout:directory}/template.cfg
+
 [template-kvm]
-recipe = hexagonit.recipe.download
-ignore-existing = true
-url = ${:_profile_base_location_}/instance-kvm.cfg.jinja2
-mode = 644
-md5sum = e30912b1256bb093329ee3c108666f05
-download-only = true
+<= download-file-base
 on-update = true
 
 [template-kvm-cluster]
-recipe = hexagonit.recipe.download
-ignore-existing = true
-url = ${:_profile_base_location_}/instance-kvm-cluster.cfg.jinja2.in
-mode = 644
-md5sum = 63fa784d8946d0b6e3fbd6381e1ea9f4
-download-only = true
+<= download-file-base
 on-update = true
 
 [template-kvm-resilient]
-recipe = hexagonit.recipe.download
-ignore-existing = true
-url = ${:_profile_base_location_}/instance-kvm-resilient.cfg.jinja2
-mode = 644
-md5sum = c189bce8b196a3b5bcb01526a3f9b2aa
-download-only = true
+<= download-file-base
 on-update = true
 
 [template-kvm-import]
-recipe = hexagonit.recipe.download
-ignore-existing = true
-url = ${:_profile_base_location_}/instance-kvm-import.cfg.jinja2.in
-md5sum = dc3f3ad9ebd8b3b5c3ded57b91cee9c7
-mode = 0644
-download-only = true
+<= download-file-base
 on-update = true
 
 [template-kvm-import-script]
-recipe = hexagonit.recipe.download
-ignore-existing = true
-url = ${:_profile_base_location_}/template/kvm-import.sh.jinja2
+<= download-template-base
 filename = kvm-import.sh.jinja2
-md5sum = cd0008f1689dfca9b77370bc4d275b70
-download-only = true
 mode = 0755
 
 [template-kvm-export]
-recipe = hexagonit.recipe.download
-ignore-existing = true
-url = ${:_profile_base_location_}/instance-kvm-export.cfg.jinja2
-mode = 644
-md5sum = 00ce5e6da3c833d9d9d1825311f11a81
-download-only = true
+<= download-file-base
 on-update = true
 
 [template-kvm-export-script]
-recipe = hexagonit.recipe.download
-ignore-existing = true
-url = ${:_profile_base_location_}/template/kvm-export.sh.jinja2
+<= download-template-base
 filename = kvm-export.sh.jinja2
-md5sum = b617d64de73de1eed518185f310bbc82
-download-only = true
 mode = 0755
 
 [template-nbd]
-recipe = slapos.recipe.template
-url = ${:_profile_base_location_}/instance-nbd.cfg.in
-md5sum = f634a5249b773658b7a7bc9fa9bb0368
+<= template-file-base
 output = ${buildout:directory}/template-nbd.cfg
-mode = 0644
 
 [template-frontend]
-recipe = slapos.recipe.template
-url = ${:_profile_base_location_}/instance-frontend.cfg.in
-md5sum = cdb690495e9eb007d2b7d2f8e12f5c59
+<= template-file-base
 output = ${buildout:directory}/template-frontend.cfg
-mode = 0644
 
 [template-ansible-promise]
-recipe = hexagonit.recipe.download
-ignore-existing = true
-url = ${:_profile_base_location_}/template/ansible-promise.in
-md5sum = 2036bf145f472f62ef8dee5e729328fd
-mode = 0644
-download-only = true
+<= download-template-base
 filename = ansible-promise.in
 
 [template-kvm-run]
-recipe = hexagonit.recipe.download
-ignore-existing = true
-url = ${:_profile_base_location_}/template/template-kvm-run.in
-mode = 644
+<= download-template-base
 filename = template-kvm-run.in
-md5sum = 9e40246b4bc4f968f0631016c939b014
-download-only = true
 on-update = true
 
 [template-kvm-controller]
-recipe = hexagonit.recipe.download
-ignore-existing = true
-url = ${:_profile_base_location_}/template/kvm-controller-run.in
-mode = 644
+<= download-template-base
 filename = kvm-controller-run.in
-md5sum = b96cba47c97f277c857176d69e086a12
-download-only = true
 on-update = true
 
 [template-apache-conf]
-recipe = hexagonit.recipe.download
-ignore-existing = true
-url = ${:_profile_base_location_}/template/apache.conf.in
-mode = 644
+<= download-template-base
 filename = apache.conf.in
-md5sum = ac97f6a52e1c5a19a646242ef85abb8a
-download-only = true
 on-update = true
 
 [template-content]
-recipe = hexagonit.recipe.download
-ignore-existing = true
-url = ${:_profile_base_location_}/template/template-content.in
-mode = 644
+<= download-template-base
 filename = template-content.in
-md5sum = 822737e483864bf255ad1259237bef2a
-download-only = true
 on-update = true
 
 [template-qemu-ready]
-recipe = hexagonit.recipe.download
-ignore-existing = true
-url = ${:_profile_base_location_}/template/qemu-is-ready.in
-mode = 644
+<= download-template-base
 filename = qemu-is-ready.in
-md5sum = b304eec8e2cb71f10ea83cac22f6db12
-download-only = true
 on-update = true
 
 [file-download-script]
-recipe = hexagonit.recipe.download
-ignore-existing = true
-url = ${:_profile_base_location_}/template/download_file.in
-mode = 644
+<= download-template-base
+path = download_file.in
 filename = download_file
-md5sum = 599dbbbd438fe7801e3f8642ae9e9a78
-download-only = true
 on-update = true
 
 [template-httpd]

software/neotest/gowork.cfg

@@ -5,6 +5,7 @@
 depends_gitfetch  =
     ${go_github.com_DataDog_czlib:recipe}
     ${go_github.com_cznic_strutil:recipe}
+    ${go_github.com_fsnotify_fsnotify:recipe}
     ${go_github.com_golang_glog:recipe}
     ${go_github.com_gwenn_gosqlite:recipe}
     ${go_github.com_gwenn_yacr:recipe}
@@ -15,6 +16,7 @@ depends_gitfetch  =
     ${go_github.com_soheilhy_cmux:recipe}
     ${go_github.com_someonegg_gocontainer:recipe}
     ${go_github.com_someonegg_gox:recipe}
+    ${go_github.com_stretchr_testify:recipe}
     ${go_golang.org_x_crypto:recipe}
     ${go_golang.org_x_net:recipe}
     ${go_golang.org_x_perf:recipe}
@@ -38,6 +40,12 @@ go.importpath = github.com/cznic/strutil
 repository    = https://github.com/cznic/strutil
 revision      = 529a34b1c1
 
+[go_github.com_fsnotify_fsnotify]
+<= go-git-package
+go.importpath = github.com/fsnotify/fsnotify
+repository    = https://github.com/fsnotify/fsnotify
+revision      = ccc981bf80
+
 [go_github.com_golang_glog]
 <= go-git-package
 go.importpath = github.com/golang/glog
@@ -60,7 +68,7 @@ revision      = 77093bdc7e
 <= go-git-package
 go.importpath = github.com/kisielk/og-rek
 repository    = https://github.com/kisielk/og-rek
-revision      = dd41cde712
+revision      = 8b25c4cefd
 
 [go_github.com_kylelemons_godebug]
 <= go-git-package
@@ -98,6 +106,12 @@ go.importpath = github.com/someonegg/gox
 repository    = https://github.com/someonegg/gox
 revision      = 4915b7fd7c
 
+[go_github.com_stretchr_testify]
+<= go-git-package
+go.importpath = github.com/stretchr/testify
+repository    = https://github.com/stretchr/testify
+revision      = 363ebb24d0
+
 [go_golang.org_x_crypto]
 <= go-git-package
 go.importpath = golang.org/x/crypto

software/slapos-sr-testing/buildout.hash.cfg

@@ -15,4 +15,4 @@
 
 [template]
 filename = instance.cfg
-md5sum = 30f0e0c5b88b518047197f251f0aea8d
+md5sum = ceb421decde6512ffee3abea463c4c7a

software/slapos-sr-testing/instance.cfg

@@ -39,7 +39,7 @@ command-line =
   --source_code_path_list=$${test-list:path_list}
 
 environment =
-  PATH=${quic_client-bin:location}:${curl:location}/bin/:/usr/bin/:/bin
+  PATH=${buildout:bin-directory}:${quic_client-bin:location}:${curl:location}/bin/:/usr/bin/:/bin
   SLAPOS_TEST_IPV4=$${slap-configuration:ipv4-random}
   SLAPOS_TEST_IPV6=$${slap-configuration:ipv6-random}
   SLAPOS_TEST_WORKING_DIR=$${directory:working-dir}

software/slapos-sr-testing/software.cfg

@@ -134,6 +134,9 @@ output = ${buildout:directory}/template.cfg
 mode = 640
 
 [versions]
+# Use slapos.core <= 1.4.16 for correctly working slapos proxy
+slapos.core = 1.4.15
+
 # clear the version of tested eggs, to make sure we installed the developped ones
 erp5.util =
 

stack/slapos.cfg

@@ -100,7 +100,7 @@ eggs =
 [versions]
 setuptools = 40.4.3
 # Use SlapOS patched zc.buildout
-zc.buildout = 2.5.2+slapos013
+zc.buildout = 2.5.2+slapos014
 # Use SlapOS patched zc.recipe.egg (zc.recipe.egg 2.x is for Buildout 2)
 zc.recipe.egg = 2.0.3+slapos003
 # Use own version of h.r.download to be able to open .xz and .lz archives
@@ -135,14 +135,14 @@ pytz = 2016.10
 requests = 2.13.0
 six = 1.11.0
 slapos.cookbook = 1.0.85
-slapos.core = 1.4.16
+slapos.core = 1.4.18
 slapos.extension.strip = 0.4
 slapos.extension.shared = 1.0
 slapos.libnetworkcache = 0.16
 slapos.rebootstrap = 4.1
 slapos.recipe.build = 0.40
 slapos.recipe.cmmi = 0.10
-slapos.toolbox = 0.85
+slapos.toolbox = 0.86
 stevedore = 1.21.0
 subprocess32 = 3.5.3
 unicodecsv = 0.14.1
@@ -150,11 +150,11 @@ xml-marshaller = 0.9.7
 paramiko = 2.1.3
 
 # Required by:
-# slapos.core==1.4.15
+# slapos.core==1.4.18
 Flask = 0.12
 
 # Required by:
-# slapos.toolbox==0.81
+# slapos.toolbox==0.86
 GitPython = 2.1.11
 
 # Required by:
@@ -166,23 +166,23 @@ gitdb2 = 2.0.5
 smmap2 = 2.0.5
 
 # Required by:
-# slapos.toolbox==0.81
+# slapos.toolbox==0.86
 PyRSS2Gen = 1.1
 
 # Required by:
-# slapos.toolbox==0.81
+# slapos.toolbox==0.86
 apache-libcloud = 2.4.0
 
 # Required by:
-# slapos.toolbox==0.81
+# slapos.toolbox==0.86
 atomize = 0.2.0
 
 # Required by:
-# slapos.toolbox==0.82
+# slapos.toolbox==0.86
 croniter = 0.3.25
 
 # Required by:
-# slapos.toolbox==0.81
+# slapos.toolbox==0.86
 dnspython = 1.15.0
 
 # Required by:
@@ -190,11 +190,11 @@ dnspython = 1.15.0
 enum34 = 1.1.6
 
 # Required by:
-# slapos.toolbox==0.81
+# slapos.toolbox==0.86
 erp5.util = 0.4.51
 
 # Required by:
-# slapos.toolbox==0.81
+# slapos.toolbox==0.86
 feedparser = 5.2.1
 
 # Required by:
@@ -218,11 +218,11 @@ ipaddress = 1.0.18
 jsonschema = 3.0.0a3
 
 # Required by:
-# slapos.toolbox==0.81
+# slapos.toolbox==0.86
 lockfile = 0.12.2
 
 # Required by:
-# slapos.core==1.4.15
+# slapos.core==1.4.18
 # XXX 'slapos node format' raises an exception with netifaces 0.10.5.
 netifaces = 0.10.4
 
@@ -231,11 +231,11 @@ netifaces = 0.10.4
 packaging = 16.8
 
 # Required by:
-# slapos.toolbox==0.81
+# slapos.toolbox==0.86
 passlib = 1.7.1
 
 # Required by:
-# slapos.toolbox==0.81
+# slapos.toolbox==0.86
 pyasn1 = 0.4.5
 
 # Required by:
@@ -243,31 +243,31 @@ pyasn1 = 0.4.5
 pycparser = 2.17
 
 # Required by:
-# slapos.toolbox==0.81
+# slapos.toolbox==0.86
 pycurl = 7.43.0
 
 # Required by:
-# slapos.toolbox==0.82
+# slapos.toolbox==0.86
 python-dateutil = 2.7.3
 
 # Required by:
-# slapos.toolbox==0.81
+# slapos.toolbox==0.86
 rpdb = 0.1.5
 
 # Required by:
-# slapos.core==1.4.15
+# slapos.core==1.4.18
 supervisor = 3.3.3
 
 # Required by:
-# slapos.toolbox==0.84
+# slapos.toolbox==0.86
 tzlocal = 1.5.1
 
 # Required by:
-# slapos.core==1.4.15
+# slapos.core==1.4.18
 uritemplate = 3.0.0
 
 # Required by:
-# slapos.core==1.4.15
+# slapos.core==1.4.18
 zope.interface = 4.3.3
 
 [networkcache]