slapos-master: Follow up recent changes on erp5 stack

  Apply commits:
   stack.erp5: Drop unneeded executable permissions. (fe7ea950)
   stack.logrotate: Fix support for stopped processes.
   stack.erp5: Use an iterator to produce port numbers.
   software/erp5: if wendelin-core-zblk-fmt is not given, then use wendelin.core's default.
   software/erp5 & stack/erp5: Add a new parameter wendelin-core-zblk-fmt.

software/slapos-master/apache.conf.in → software/slapos-master/apache-backend.conf.in

+{# This file configures apache to redirect requests from ports to specific urls.
+ # It provides SSL support for server and optionaly for client.
+ #
+ # All parameters are given through the `parameter_dict` variable, see the
+ # list entries :
+ #
+ #     parameter_dict = {
+ #       #  The path given to "PidFile"
+ #       "pid-file": "<file_path>",
+ #
+ #       #  The number given to "TimeOut"
+ #       "timeout": 300,
+ #
+ #       #  The path given to "SSLCertificateFile"
+ #       "cert": "<file_path>",
+ #
+ #       #  The path given to "SSLCertificateKeyFile"
+ #       "key": "<file_path>",
+ #
+ #       #  The value given to "SSLCipherSuite" (can be empty)
+ #       "cipher": "",
+ #
+ #       #  The path given to "SSLSessionCache shmcb:<folder_path>(512000)"
+ #       "ssl-session-cache": "<folder_path>",
+ #
+ #       #  The path given to "SSLCACertificateFile" (can be empty)
+ #       #  If this value is not empty, it enables client certificate check.
+ #       #  (Enabling "SSLVerifyClient require")
+ #       "ca-cert": "<file_path>",
+ #
+ #       #  The path given to "SSLCARevocationFile" (used if ca-cert is not
+ #       #  empty)
+ #       "crl": "<file_path>",
+ #
+ #       #  The path given to "ErrorLog"
+ #       "error-log": "<file_path>",
+ #
+ #       #  The path given to "AccessLog"
+ #       "access-log": "<file_path>",
+ #
+ #       #  The list of ip which apache will listen to.
+ #       "ip-list": [
+ #         "0.0.0.0",
+ #         "[::1]",
+ #       ],
+ #
+ #       #  The list of backends which apache should redirect to.
+ #       "backend-list": [
+ #         # (port, unused, internal_scheme)
+ #         (8000, _, "http://10.0.0.10:8001"),
+ #         (8002, _, "http://10.0.0.10:8003"),
+ #       ],
+ #     }
+ #
+ # This sample of `parameter_dict` will make apache listening to :
+ # - 0.0.0.0:8000 redirecting internaly to http://10.0.0.10:8001
+ # - [::1]:8000 redirecting internaly to http://10.0.0.10:8001
+ # - 0.0.0.0:8002 redirecting internaly to http://10.0.0.10:8003
+ # - [::1]:8002 redirecting internaly to http://10.0.0.10:8003
+-#}
 LoadModule unixd_module modules/mod_unixd.so
 LoadModule access_compat_module modules/mod_access_compat.so
 LoadModule authz_core_module modules/mod_authz_core.so
@@ -18,7 +78,7 @@ LoadModule headers_module modules/mod_headers.so
 LoadModule deflate_module modules/mod_deflate.so
 LoadModule filter_module modules/mod_filter.so
 
-AddOutputFilterByType DEFLATE text/cache-manifest text/html text/plain text/css application/hal+json application/json application/x-javascript text/xml application/xml application/rss+xml text/javascript image/svg+xml
+AddOutputFilterByType DEFLATE text/cache-manifest text/html text/plain text/css application/hal+json application/json application/x-javascript text/xml application/xml application/rss+xml text/javascript image/svg+xml application/x-font-ttf application/font-woff application/font-woff2 application/x-font-opentype
 
 PidFile "{{ parameter_dict['pid-file'] }}"
 ServerAdmin admin@
@@ -68,7 +128,7 @@ CustomLog "{{ parameter_dict['access-log'] }}" combined
 </Directory>
 
 RewriteEngine On
-{% for family_name, (port, _, backend, enable_authentication) in parameter_dict['backend-list'].items() -%}
+{% for port, _, backend, enable_authentication in parameter_dict['backend-list'] -%}
 {%   for ip in parameter_dict['ip-list'] -%}
 Listen {{ ip }}:{{ port }}
 {%   endfor -%}
@@ -82,8 +142,8 @@ Listen {{ ip }}:{{ port }}
   LogFormat "%h %l %{REMOTE_USER}i %t \"%r\" %>s %b \"%{Referer}i\" \"%{User-Agent}i\" %D" combined
   
   # We would like to separate the the authentificated logs.
-  ErrorLog "{{ parameter_dict['log-dir'] }}/apache-{{ family_name }}-error.log"
-  CustomLog "{{ parameter_dict['log-dir'] }}/apache-{{ family_name }}-access.log" combined
+  ErrorLog "{{ parameter_dict['log-dir'] }}/apache-service-error.log"
+  CustomLog "{{ parameter_dict['log-dir'] }}/apache-service-access.log" combined
 {% endif -%}
   SSLEngine on
   RewriteRule ^/(.*) {{ backend }}/$1 [L,P]

software/slapos-master/buildout.hash.cfg

+# THIS IS NOT A BUILDOUT FILE, despite purposedly using a compatible syntax.
+# The only allowed lines here are (regexes):
+# - "^#" comments, copied verbatim
+# - "^[" section beginings, copied verbatim
+# - lines containing an "=" sign which must fit in the following categorie.
+#   - "^\s*filename\s*=\s*path\s*$" where "path" is relative to this file
+#     But avoid directories, they are not portable.
+#     Copied verbatim.
+#   - "^\s*hashtype\s*=.*" where "hashtype" is one of the values supported
+#     by the re-generation script.
+#     Re-generated.
+# - other lines are copied verbatim
+# Substitution (${...:...}), extension ([buildout] extends = ...) and
+# section inheritance (< = ...) are NOT supported (but you should really
+# not need these here).
+[template-erp5]
+filename = instance-erp5.cfg.in
+md5sum = 1489091851f27c41243eeaa4967cc91c
+
+[template-balancer]
+filename = instance-balancer.cfg.in
+md5sum = a410f10fe9766c9321ffd7b18ace5a4f
+
+[template-apache-backend-conf]
+filename = apache-backend.conf.in
+md5sum = ea77222f440bb72fee4939fe1b72976e
+
+[template-create-erp5-site-real]
+filename = instance-create-erp5-site-real.cfg.in
+md5sum = 86a2b244341218cd0c4b6d398c61ee20

software/slapos-master/instance-balancer.cfg.in

@@ -17,8 +17,12 @@ per partition. No more (undefined result), no less (IndexError).
 {%   do apache_ip_list.append('[' ~ ipv6 ~ ']') -%}
 {% endif -%}
 
-[simplefile]
+[jinja2-template-base]
 recipe = slapos.recipe.template:jinja2
+mode = 644
+
+[simplefile]
+< = jinja2-template-base
 template = inline:{{ '{{ content }}' }}
 
 {% macro simplefile(section_name, file_path, content, mode='') -%}
@@ -44,7 +48,7 @@ ipv4 = {{ ipv4 }}
 {% endif -%}
 {% set haproxy_dict = {} -%}
 {% set apache_dict = {} -%}
-{% set next_port = slapparameter_dict['tcpv4-port'] -%}
+{% set next_port = itertools.count(slapparameter_dict['tcpv4-port']).next -%}
 {% for family_name, parameter_id_list in sorted(
   slapparameter_dict['zope-family-dict'].iteritems()) -%}
 {%   set zope_family_address_list = [] -%}
@@ -56,14 +60,14 @@ ipv4 = {{ ipv4 }}
 {%         do has_webdav.append(None) %}
 {%       endif -%}
 {%       if use_ipv6 -%}
-[{{ section('zope-tunnel-' ~ next_port) }}]
+{%         set current_port = next_port() -%}
+[{{ section('zope-tunnel-' ~ current_port) }}]
 < = zope-tunnel-base
-base-name = {{ 'zeo-tunnel-' ~ next_port }}
-ipv4-port = {{ next_port }}
+base-name = {{ 'zeo-tunnel-' ~ current_port }}
+ipv4-port = {{ current_port }}
 ipv6-port = {{ zope_address.split(']:')[1] }}
 ipv6 = {{ zope_address.split(']:')[0][1:] }}
-{%         set zope_effective_address = ipv4 ~ ":" ~ next_port -%}
-{%         set next_port = next_port + 1 -%}
+{%         set zope_effective_address = ipv4 ~ ":" ~ current_port -%}
 {%       else -%}
 {%         set zope_effective_address = zope_address -%}
 {%       endif -%}
@@ -77,8 +81,7 @@ ipv6 = {{ zope_address.split(']:')[0][1:] }}
  # do a no-op getitem.
 -#}
 {%   do zope_family_address_list[0][0] -%}
-{%   set haproxy_port = next_port -%}
-{%   set next_port = next_port + 1 -%}
+{%   set haproxy_port = next_port() -%}
 {%   do haproxy_dict.__setitem__(family_name, (haproxy_port, zope_family_address_list)) -%}
 {%   if has_webdav -%}
 {%     set internal_scheme = 'http' -%}{# mod_rewrite does not recognise webdav scheme -#}
@@ -89,8 +92,7 @@ ipv6 = {{ zope_address.split(']:')[0][1:] }}
 {%   endif -%}
 {%   set backend_path = slapparameter_dict['backend-path-dict'][family_name] -%}
 {%   set ssl_authentication = slapparameter_dict['ssl-authentication-dict'][family_name] -%}
-{%   do apache_dict.__setitem__(family_name, (next_port, external_scheme, internal_scheme ~ '://' ~ ipv4 ~ ':' ~ haproxy_port ~ backend_path, ssl_authentication)) -%}
-{%   set next_port = next_port + 1 -%}
+{%   do apache_dict.__setitem__(family_name, (next_port(), external_scheme, internal_scheme ~ '://' ~ ipv4 ~ ':' ~ haproxy_port ~ backend_path, ssl_authentication)) -%}
 {% endfor -%}
 
 [haproxy-cfg-parameter-dict]
@@ -100,7 +102,7 @@ backend-dict = {{ dumps(haproxy_dict) }}
 ip = {{ ipv4 }}
 
 [haproxy-cfg]
-recipe = slapos.recipe.template:jinja2
+< = jinja2-template-base
 template = {{ parameter_dict['template-haproxy-cfg'] }}
 rendered = ${directory:etc}/haproxy.cfg
 context = section parameter_dict haproxy-cfg-parameter-dict
@@ -120,7 +122,7 @@ ca-cert =  ${directory:apache-conf}/ca.crt
 crl = ${directory:apache-conf}/crl.pem
 
 [apache-conf-parameter-dict]
-backend-list = {{ dumps(apache_dict) }}
+backend-list = {{ dumps(apache_dict.values()) }}
 ip-list = {{ dumps(apache_ip_list) }}
 pid-file = ${directory:run}/apache.pid
 error-log = ${directory:log}/apache-error.log
@@ -145,7 +147,7 @@ shared-crl = {{ shared_ca_path }}/crl
 
 
 [apache-conf]
-recipe = slapos.recipe.template:jinja2
+< = jinja2-template-base
 template = {{ parameter_dict['template-apache-conf'] }}
 rendered = ${directory:apache-conf}/apache.conf
 context = section parameter_dict apache-conf-parameter-dict
@@ -201,22 +203,18 @@ crl =
 {%     do part_list.append('logrotate-' ~ base_name) -%}
 {%     do apache_service_log_list.__setitem__(family_name, base_name) -%}
 [logrotate-{{ base_name }}]
-recipe = slapos.cookbook:logrotate.d
-logrotate-entries = ${logrotate:logrotate-entries}
-backup = ${logrotate:backup}
+ < = logrotate-entry-base
 name = {{ base_name }}
 log = ${apache-conf-parameter-dict:log-dir}/{{ base_name }}-error.log ${apache-conf-parameter-dict:log-dir}/{{ base_name }}-access.log
-post = {{ parameter_dict['bin-directory'] }}/slapos-kill --pidfile ${apache-conf-parameter-dict:pid-file} -s USR1
+post = test ! -s ${apache-conf-parameter-dict:pid-file} || {{ parameter_dict['bin-directory'] }}/slapos-kill --pidfile ${apache-conf-parameter-dict:pid-file} -s USR1
 {%   endif -%}
 {% endfor -%}
 
 [logrotate-apache]
-recipe = slapos.cookbook:logrotate.d
-logrotate-entries = ${logrotate:logrotate-entries}
-backup = ${logrotate:backup}
+< = logrotate-entry-base
 name = apache
 log = ${apache-conf-parameter-dict:error-log} ${apache-conf-parameter-dict:access-log}
-post = {{ parameter_dict['bin-directory'] }}/slapos-kill --pidfile ${apache-conf-parameter-dict:pid-file} -s USR1
+post = test ! -s ${apache-conf-parameter-dict:pid-file} || {{ parameter_dict['bin-directory'] }}/slapos-kill --pidfile ${apache-conf-parameter-dict:pid-file} -s USR1
 
 [directory]
 recipe = slapos.cookbook:mkdirectory
@@ -237,7 +235,7 @@ crl = ${:ca-dir}/crl
 
 [monitor-instance-parameter]
 monitor-httpd-ipv6 = {{ (ipv6_set | list)[0] }}
-monitor-httpd-port = {{ next_port }}
+monitor-httpd-port = {{ next_port() }}
 monitor-title = Balancer monitor
 
 [buildout]

software/slapos-master/instance-erp5.cfg.in

@@ -117,6 +117,10 @@ name = neo-${gen-neo-cluster-base:passwd}
 [gen-smtpd-sasl-password]
 < = gen-password
 
+{% set zope_partition_dict = slapparameter_dict.get('zope-partition-dict', {'1': {}}) -%}
+{% set zope_address_list_id_dict = {} -%}
+{% if zope_partition_dict -%}
+
 [request-zope-base]
 <= request-common
 return =
@@ -142,6 +146,7 @@ config-mysql-url-list = ${request-mariadb:connection-database-list}
 config-site-id = {{ dumps(site_id) }}
 config-smtp-url = ${request-smtp:connection-url}
 config-timezone = {{ dumps(slapparameter_dict.get('timezone', 'UTC')) }}
+config-wendelin-core-zblk-fmt = {{ dumps(slapparameter_dict.get('wendelin-core-zblk-fmt', '')) }} 
 config-ca-path = ${directory:ca-dir}
 config-zodb-dict = {{ dumps(zodb_dict) }}
 {% for server_type, server_dict in storage_dict.iteritems() -%}
@@ -161,7 +166,8 @@ software-type = zope
 {% set jupyter_zope_family_default = [] -%}
 {% set zope_backend_path_dict = {} -%}
 {% set ssl_authentication_dict = {} -%}
-{% for custom_name, zope_parameter_dict in slapparameter_dict.get('zope-partition-dict', {'1': {}}).items() -%}
+
+{% for custom_name, zope_parameter_dict in  zope_partition_dict.items() -%}
 {%   set partition_name = 'zope-' ~ custom_name -%}
 {%   set section_name = 'request-' ~ partition_name -%}
 {%   set backend_path = zope_parameter_dict.get('backend-path', '/') % {'site-id': site_id} %}
@@ -314,6 +320,8 @@ config-{{ name }} = {{ value }}
 return = site_url
 {% endif -%}
 
+{% endif -%}{# if zope_partition_dict -#}
+
 [publish]
 recipe = slapos.cookbook:publish.serialised
 -extends = publish-early
@@ -321,11 +329,13 @@ recipe = slapos.cookbook:publish.serialised
 neo-masters = ${neo-0-final:connection-masters}
 neo-admins = ${neo-0-final:connection-admins}
 {% endif  -%}
+{% if zope_address_list_id_dict -%}
 {#
 Pick any published hosts-dict, they are expected to be identical - and there is
 no way to check here.
 -#}
 hosts-dict = {{ '${' ~ zope_address_list_id_dict.keys()[0] ~ ':connection-hosts-dict}' }}
+{% endif -%}
 {% for name, value in publish_dict.items() -%}
 {{   name }} = {{ value }}
 {% endfor -%}

software/slapos-master/software.cfg

 [buildout]
 extends =
   ../../software/erp5/software.cfg
+  buildout.hash.cfg
 
 parts +=
   vifib-fix-products-paths
@@ -52,22 +53,18 @@ mode = 644
 [template-erp5]
 < = download-base-part
 filename = instance-erp5.cfg.in
-md5sum = 073b96888388a39721a8cef3e90c5c5b
 
 [template-balancer]
 < = download-base-part
 filename = instance-balancer.cfg.in
-md5sum = 02c258e51ff4619efe258bbf24b9ceed 
 
-[template-apache-conf]
-< = download-base-part
-filename = apache.conf.in
-md5sum = 6a9426138d46ba5de75a86199be4f8d1
+[template-apache-backend-conf]
+url = ${:_profile_base_location_}/${:filename}
+filename = apache-backend.conf.in 
 
 [template-create-erp5-site-real]
 < = download-base-part
 filename = instance-create-erp5-site-real.cfg.in
-md5sum = 86a2b244341218cd0c4b6d398c61ee20
 
 [versions]
 python-memcached = 1.47