slapos-testing: Re-introduce changes for support portal_slap

  This changes were squashed into a single commit and it includes:

    - install slapos_configurator by default
    - Allow define ssl authentication for a certain zope family
    - Allow define custom zope path per zope family

  Those changes are originally written by Alain Takoudjou and rewrited by Rafael Monnerat

software/slapos-master/apache.conf.in

@@ -66,12 +66,25 @@ CustomLog "{{ parameter_dict['access-log'] }}" combined
 </Directory>
 
 RewriteEngine On
-{% for port, _, backend in parameter_dict['backend-list'] -%}
+{% for family_name, (port, _, backend, enable_authentication) in parameter_dict['backend-list'].items() -%}
 {%   for ip in parameter_dict['ip-list'] -%}
 Listen {{ ip }}:{{ port }}
 {%   endfor -%}
 <VirtualHost *:{{ port }}>
+{% if enable_authentication -%}
+  SSLVerifyClient require
+  RequestHeader set REMOTE_USER %{SSL_CLIENT_S_DN_CN}s
+  SSLCACertificateFile {{ parameter_dict['single-ca-cert'] }}
+  SSLCARevocationPath {{ parameter_dict['single-crl'] }}
+  
+  LogFormat "%h %l %{REMOTE_USER}i %t \"%r\" %>s %b \"%{Referer}i\" \"%{User-Agent}i\" %D" combined
+  
+  # We would like to separate the the authentificated logs.
+  ErrorLog "{{ parameter_dict['log-dir'] }}/apache-{{ family_name }}-error.log"
+  CustomLog "{{ parameter_dict['log-dir'] }}/apache-{{ family_name }}-access.log" combined
+{% endif -%}
   SSLEngine on
   RewriteRule ^/(.*) {{ backend }}/$1 [L,P]
 </VirtualHost>
 {% endfor -%}
+

software/slapos-master/instance-balancer.cfg.in

@@ -2,6 +2,7 @@
 {% set ssl_parameter_dict = slapparameter_dict.get('ssl', {}) %}
 {% macro section(name) %}{% do part_list.append(name) %}{{ name }}{% endmacro -%}
 {% set use_ipv6 = slapparameter_dict.get('use-ipv6', False) -%}
+{% set shared_ca_path = slapparameter_dict['shared-certificate-authority-path'] -%}
 {#
 XXX: This template only supports exactly one IPv4 and (if ipv6 is used) one IPv6
 per partition. No more (undefined result), no less (IndexError).
@@ -86,7 +87,9 @@ ipv6 = {{ zope_address.split(']:')[0][1:] }}
 {%     set internal_scheme = 'http' -%}
 {%     set external_scheme = 'https' -%}
 {%   endif -%}
-{%   do apache_dict.__setitem__(family_name, (next_port, external_scheme, internal_scheme ~ '://' ~ ipv4 ~ ':' ~ haproxy_port ~ slapparameter_dict['backend-path'])) -%}
+{%   set backend_path = slapparameter_dict['backend-path-dict'][family_name] -%}
+{%   set ssl_authentication = slapparameter_dict['ssl-authentication-dict'][family_name] -%}
+{%   do apache_dict.__setitem__(family_name, (next_port, external_scheme, internal_scheme ~ '://' ~ ipv4 ~ ':' ~ haproxy_port ~ backend_path, ssl_authentication)) -%}
 {%   set next_port = next_port + 1 -%}
 {% endfor -%}
 
@@ -117,11 +120,12 @@ ca-cert =  ${directory:apache-conf}/ca.crt
 crl = ${directory:apache-conf}/crl.pem
 
 [apache-conf-parameter-dict]
-backend-list = {{ dumps(apache_dict.values()) }}
+backend-list = {{ dumps(apache_dict) }}
 ip-list = {{ dumps(apache_ip_list) }}
 pid-file = ${directory:run}/apache.pid
 error-log = ${directory:log}/apache-error.log
 access-log = ${directory:log}/apache-access.log
+log-dir = ${directory:log} 
 # Apache 2.4's default value (60 seconds) can be a bit too short
 timeout = 300
 # Basic SSL server configuration
@@ -132,6 +136,11 @@ ssl-session-cache = ${directory:log}/apache-ssl-session-cache
 # Client x509 auth
 ca-cert = ${apache-ssl-client:cert}
 crl = ${apache-ssl-client:crl}
+{% if ssl_parameter_dict.get('single-ca-cert') and ssl_parameter_dict.get('single-ca-crl') -%}
+single-ca-cert = {{ dumps(ssl_parameter_dict.get('single-ca-cert')) }}
+single-crl = {{ dumps(ssl_parameter_dict.get('single-ca-crl')) }}
+{% endif -%}
+
 
 [apache-conf]
 recipe = slapos.recipe.template:jinja2
@@ -153,7 +162,7 @@ port = {{ apache_dict.values()[0][0] }}
 
 [publish]
 recipe = slapos.cookbook:publish.serialised
-{% for family_name, (apache_port, scheme, _) in apache_dict.items() -%}
+{% for family_name, (apache_port, scheme, _, _) in apache_dict.items() -%}
 {{   family_name ~ '-v6' }} = {% if ipv6_set %}{{ scheme ~ '://[' ~ ipv6 ~ ']:' ~ apache_port }}{% endif %}
 {{   family_name }} = {{ scheme ~ '://' ~ ipv4 ~ ':' ~ apache_port }}
 {% endfor -%}
@@ -182,6 +191,23 @@ cert =
 crl =
 {%- endif %}
 
+
+{% set apache_service_log_list = {} -%}
+{% for family_name, (_, _, _, authentication) in apache_dict.items() -%}
+{%   if authentication -%}
+{%     set base_name = 'apache-' ~ family_name -%}
+{%     do part_list.append('logrotate-' ~ base_name) -%}
+{%     do apache_service_log_list.__setitem__(family_name, base_name) -%}
+[logrotate-{{ base_name }}]
+recipe = slapos.cookbook:logrotate.d
+logrotate-entries = ${logrotate:logrotate-entries}
+backup = ${logrotate:backup}
+name = {{ base_name }}
+log = ${apache-conf-parameter-dict:log-dir}/{{ base_name }}-error.log ${apache-conf-parameter-dict:log-dir}/{{ base_name }}-access.log
+post = {{ parameter_dict['bin-directory'] }}/slapos-kill --pidfile ${apache-conf-parameter-dict:pid-file} -s USR1
+{%   endif -%}
+{% endfor -%}
+
 [logrotate-apache]
 recipe = slapos.cookbook:logrotate.d
 logrotate-entries = ${logrotate:logrotate-entries}

software/slapos-master/instance-create-erp5-site-real.cfg.in

@@ -16,9 +16,10 @@ runner-path = ${directory:services}/erp5-bootstrap
 {% set mysql_parsed = urlparse.urlparse(publish['mariadb-database-list'][0]) -%}
 mysql-url = {{ dumps(urlparse.urlunparse(mysql_parsed[:1] + (mysql_parsed.username + ":" + mysql_parsed.password + "@" + reverse_hosts.get(mysql_parsed.hostname, mysql_parsed.hostname) + ':' ~ mysql_parsed.port, ) + mysql_parsed[2:])) }}
 {# Pick the first http[s] family found, they should be all equivalent anyway. -#}
+{# Don't pick the https[s] configurated with ssl-authenticat=true. By convention, this family name contain 'service'. -#}
 {% set family_list = [] -%}
 {% for key, value in publish.items() -%}
-{%   if key.startswith('family-') and value.startswith('http') -%}
+{%   if key.startswith('family-') and value.startswith('http') and not 'service' in key -%}
 {%     do family_list.append(value.split('://', 1)) -%}
 {%   endif -%}
 {%  endfor -%}

software/slapos-master/instance-erp5.cfg.in

@@ -122,7 +122,7 @@ name = neo-${gen-neo-cluster-base:passwd}
 return =
   zope-address-list
   hosts-dict
-{% set bt5_default_list = 'erp5_full_text_myisam_catalog erp5_configurator_standard erp5_configurator_maxma_demo erp5_configurator_ung erp5_configurator_run_my_doc' -%}
+{% set bt5_default_list = 'erp5_full_text_myisam_catalog erp5_configurator_standard erp5_configurator_maxma_demo erp5_configurator_ung erp5_configurator_run_my_doc slapos_configurator' -%}
 {% if has_jupyter -%}
 {%   set bt5_default_list = bt5_default_list + ' erp5_data_notebook' -%}
 {% endif -%}
@@ -142,6 +142,7 @@ config-mysql-url-list = ${request-mariadb:connection-database-list}
 config-site-id = {{ dumps(site_id) }}
 config-smtp-url = ${request-smtp:connection-url}
 config-timezone = {{ dumps(slapparameter_dict.get('timezone', 'UTC')) }}
+config-ca-path = ${directory:ca-dir}
 config-zodb-dict = {{ dumps(zodb_dict) }}
 {% for server_type, server_dict in storage_dict.iteritems() -%}
 {%   if server_type == 'neo' -%}
@@ -158,9 +159,12 @@ software-type = zope
 
 {% set zope_family_dict = {} -%}
 {% set jupyter_zope_family_default = [] -%}
+{% set zope_backend_path_dict = {} -%}
+{% set ssl_authentication_dict = {} -%}
 {% for custom_name, zope_parameter_dict in slapparameter_dict.get('zope-partition-dict', {'1': {}}).items() -%}
 {%   set partition_name = 'zope-' ~ custom_name -%}
 {%   set section_name = 'request-' ~ partition_name -%}
+{%   set backend_path = zope_parameter_dict.get('backend-path', '/') % {'site-id': site_id} %}
 {%   set zope_family = zope_parameter_dict.get('family', 'default') -%}
 {#   # default jupyter zope family is first zope family. -#}
 {#   # use list.append() to update it, because in jinja2 set changes only local scope. -#}
@@ -168,6 +172,8 @@ software-type = zope
 {%     do jupyter_zope_family_default.append(zope_family) -%}
 {%   endif -%}
 {%   do zope_family_dict.setdefault(zope_family, []).append(section_name) -%}
+{%   do zope_backend_path_dict.setdefault(zope_parameter_dict.get('family', 'default'), backend_path) -%}
+{%   do ssl_authentication_dict.setdefault(zope_parameter_dict.get('family', 'default'), zope_parameter_dict.get('ssl-authentication', False)) -%}
 [{{ section_name }}]
 <= request-zope-base
 name = {{ partition_name }}
@@ -224,6 +230,47 @@ config-url = ${request-jupyter:connection-url}
 {%   endif -%}
 {%- endif %}
 
+[directory]
+recipe = slapos.cookbook:mkdirectory
+{% if slapparameter_dict.get('shared-certificate-authority-path', '') -%}
+ca-dir = {{ slapparameter_dict.get('shared-certificate-authority-path') }}
+{% else -%}
+ca-dir = ${buildout:directory}/srv/ssl
+{% endif -%}
+bin = ${buildout:directory}/bin
+etc = ${buildout:directory}/etc
+services = ${:etc}/run
+requests = ${:ca-dir}/requests
+private = ${:ca-dir}/private
+certs = ${:ca-dir}/certs
+newcerts = ${:ca-dir}/newcerts
+crl = ${:ca-dir}/crl
+
+[apache-certificate-authority]
+recipe = slapos.cookbook:certificate_authority
+openssl-binary = {{ openssl_location }}/bin/openssl
+ca-dir = ${directory:ca-dir}
+requests-directory = ${directory:requests}
+wrapper = ${directory:services}/service-ca
+ca-private = ${directory:private}
+ca-certs = ${directory:certs}
+ca-newcerts = ${directory:newcerts}
+ca-crl = ${directory:crl}
+country-code = {{ dumps(slapparameter_dict.get('country-code', 'ZZ')) }}
+email = {{ dumps(slapparameter_dict.get('email', 'nobody@example.com')) }}
+state = {{ dumps(slapparameter_dict.get('state', "('State',)")) }}
+city = {{ dumps(slapparameter_dict.get('city', 'City')) }}
+company = {{ dumps(slapparameter_dict.get('company', 'Compagny')) }}
+
+# XXX - Big hack: Change access for certificate authority configuration
+#       To allow apache to read openssl.cnf in this folder
+[fix-ca-folder]
+recipe = plone.recipe.command
+stop-on-error = true
+command =
+  chmod 644 ${apache-certificate-authority:ca-dir}/openssl.cnf
+update-command = ${:command}
+
 {% set balancer_dict = slapparameter_dict.get('balancer', {}) -%}
 [request-balancer]
 <= request-common
@@ -281,3 +328,10 @@ hosts-dict = {{ '${' ~ zope_address_list_id_dict.keys()[0] ~ ':connection-hosts-
 {% endfor -%}
 
 {{ root_common.common_section() }}
+
+
+[buildout]
+parts += 
+  apache-certificate-authority
+  fix-ca-folder
+  publish

software/slapos-master/software.cfg

@@ -64,22 +64,22 @@ mode = 644
 [template-erp5]
 < = download-base-part
 filename = instance-erp5.cfg.in
-md5sum = 66edf64eeaecded8977459acb26f4424 
+md5sum = c7dc552383ab56b8a616b1bbc426b90b 
 
 [template-balancer]
 < = download-base-part
 filename = instance-balancer.cfg.in
-md5sum = ec9321514674c084e509ca070763b4a1
+md5sum = 40b9bcb1a15049ba923146cd10a19cb7
 
 [template-apache-conf]
 < = download-base-part
 filename = apache.conf.in
-md5sum = 713b22938d7212c8506449bc0508452b 
+md5sum = b76e00ca343031423f7d724f68d19f76
 
 [template-create-erp5-site-real]
 < = download-base-part
 filename = instance-create-erp5-site-real.cfg.in
-md5sum = 79f789360e71146486c82a7a10834bae
+md5sum = 86a2b244341218cd0c4b6d398c61ee20
 
 [versions]
 python-memcached = 1.47