Merge remote-tracking branch 'nexedi/master'

software/slapos-master/apache-backend.conf.in

@@ -23,14 +23,14 @@
  #       #  The path given to "SSLSessionCache shmcb:<folder_path>(512000)"
  #       "ssl-session-cache": "<folder_path>",
  #
- #       #  The path given to "SSLCACertificatePath" (can be empty)
+ #       #  The path given to "SSLCACertificateFile" (can be empty)
  #       #  If this value is not empty, it enables client certificate check.
  #       #  (Enabling "SSLVerifyClient require")
- #       "ca-cert-dir": "<directory_path>",
+ #       "ca-cert": "<file_path>",
  #
- #       #  The path given to "SSLCARevocationPath" (used if ca-cert-dir is not
+ #       #  The path given to "SSLCARevocationFile" (used if ca-cert is not
  #       #  empty)
- #       "crl-dir": "<directory_path>",
+ #       "crl": "<file_path>",
  #
  #       #  The path given to "ErrorLog"
  #       "error-log": "<file_path>",
@@ -69,7 +69,7 @@
  #  From to `backend-list`:
  #   - 0.0.0.0:8000 redirecting internaly to http://10.0.0.10:8001 and
  #   - [::1]:8000 redirecting internaly to http://10.0.0.10:8001
- #  only accepting requests from clients who provide a valid SSL certificate trusted in `ca-cert-dir`.
+ #  only accepting requests from clients who provide a valid SSL certificate trusted in `ca-cert`.
  #   - 0.0.0.0:8002 redirecting internaly to http://10.0.0.10:8003
  #   - [::1]:8002 redirecting internaly to http://10.0.0.10:8003
  #  accepting requests from any client.
@@ -83,8 +83,6 @@
  #  For more details, refer to
  #  https://docs.zope.org/zope2/zope2book/VirtualHosting.html#using-virtualhostroot-and-virtualhostbase-together
 -#}
-{% set ca_cert_dir = parameter_dict.get('ca-cert-dir') -%}
-{% set crl_dir = parameter_dict.get('crl-dir') -%}
 LoadModule unixd_module modules/mod_unixd.so
 LoadModule access_compat_module modules/mod_access_compat.so
 LoadModule authz_core_module modules/mod_authz_core.so
@@ -135,16 +133,17 @@ SSLProxyEngine On
 
 # As backend is trusting Remote-User header unset it always
 RequestHeader unset Remote-User
-{% if ca_cert_dir -%}
+{% if parameter_dict['ca-cert'] -%}
 SSLVerifyClient optional
 RequestHeader set Remote-User %{SSL_CLIENT_S_DN_CN}s
-RequestHeader unset X-Forwarded-For "expr=%{SSL_CLIENT_VERIFY} != 'SUCCESS'"
-SSLCACertificatePath {{ ca_cert_dir }}
-{%   if crl_dir -%}
+SSLCACertificateFile {{ parameter_dict['ca-cert'] }}
+{%   if not parameter_dict['shared-ca-cert'] %}
+{%     if parameter_dict['crl'] -%}
 SSLCARevocationCheck chain
-SSLCARevocationPath {{ crl_dir }}
-{%   endif -%}
-{% endif -%}
+SSLCARevocationFile {{ parameter_dict['crl'] }}
+{%-     endif %}
+{%-   endif %}
+{%- endif %}
 
 ErrorLog "{{ parameter_dict['error-log'] }}"
 # Default apache log format with request time in microsecond at the end
@@ -164,8 +163,12 @@ Listen {{ ip }}:{{ port }}
 {%   endfor -%}
 <VirtualHost *:{{ port }}>
   SSLEngine on
-{% if enable_authentication and ca_cert_dir -%}
+{% if enable_authentication and parameter_dict['shared-ca-cert'] and parameter_dict['shared-crl'] -%}
   SSLVerifyClient require
+  # Custom block we use for now different parameters.
+  RequestHeader set Remote-User %{SSL_CLIENT_S_DN_CN}s
+  SSLCACertificateFile {{ parameter_dict['shared-ca-cert'] }}
+  SSLCARevocationPath {{ parameter_dict['shared-crl'] }}
 
   LogFormat "%h %l %{REMOTE_USER}i %t \"%r\" %>s %b \"%{Referer}i\" \"%{User-Agent}i\" %D" combined
 
@@ -183,8 +186,11 @@ Listen {{ ip }}:{{ port }}
 <VirtualHost {{ ip }}:{{ port }}>
   SSLEngine on
   Timeout 3600
-{%   if enable_authentication and ca_cert_dir -%}
+{%   if enable_authentication and parameter_dict['ca-cert'] and parameter_dict['crl'] -%}
   SSLVerifyClient require
+  SSLCACertificateFile {{ parameter_dict['ca-cert'] }}
+  SSLCARevocationCheck chain
+  SSLCARevocationFile {{ parameter_dict['crl'] }}
 
   LogFormat "%h %l %{REMOTE_USER}i %t \"%r\" %>s %b \"%{Referer}i\" \"%{User-Agent}i\" %D" combined
 

software/slapos-master/buildout.hash.cfg

@@ -18,8 +18,8 @@ md5sum = 2ef0ddc206c6b0982a37cfc21f23e423
 
 [template-balancer]
 filename = instance-balancer.cfg.in
-md5sum = d10a5ddfffa67b8ca01b3e38315bae2f
+md5sum = ef86e09e44ac67a9b15939df0ab4a466
 
 [template-apache-backend-conf]
 filename = apache-backend.conf.in
-md5sum = ba8f878083b79535c6a2d10de496d90f
+md5sum = 9d7104ce18f79a7a84988efc11f5ed23

software/slapos-master/instance-balancer.cfg.in

@@ -2,7 +2,6 @@
 {% set part_list = [] -%}
 {% macro section(name) %}{% do part_list.append(name) %}{{ name }}{% endmacro -%}
 {% set ssl_parameter_dict = slapparameter_dict['ssl'] -%}
-{% set frontend_caucase_url_list = ssl_parameter_dict.get('frontend_caucase_url_list', []) -%}
 {% set shared_ca_path = slapparameter_dict.get('shared-certificate-authority-path') -%}
 {#
 XXX: This template only supports exactly one IPv4 and (if ipv6 is used) one IPv6
@@ -29,8 +28,8 @@ mode = 644
      url=ssl_parameter_dict['caucase-url'],
      data_dir='${directory:srv}/caucase-updater',
      crt_path='${apache-conf-ssl:caucase-cert}',
-     ca_path='${directory:srv}/caucase-updater/ca.crt',
-     crl_path='${directory:srv}/caucase-updater/crl.pem',
+     ca_path='${apache-conf-ssl:ca-cert}',
+     crl_path='${apache-conf-ssl:crl}',
      key_path='${apache-conf-ssl:caucase-key}',
      on_renew='${apache-graceful:output}',
      max_sleep=ssl_parameter_dict.get('max-crl-update-delay', 1.0),
@@ -38,69 +37,6 @@ mode = 644
      openssl=parameter_dict['openssl'] ~ '/bin/openssl',
 )}}
 {% do section('caucase-updater') -%}
-{% do section('caucase-updater-promise') -%}
-
-{% set frontend_caucase_url_hash_list = [] -%}
-{% for frontend_caucase_url in frontend_caucase_url_list -%}
-{%   set hash = hashlib.md5(frontend_caucase_url).hexdigest() -%}
-{%   do frontend_caucase_url_hash_list.append(hash) -%}
-{%   set data_dir = '${directory:srv}/client-cert-ca/%s' % hash -%}
-{{   caucase.updater(
-       prefix='caucase-updater-%s' % hash,
-       buildout_bin_directory=parameter_dict['bin-directory'],
-       updater_path='${directory:services-on-watch}/caucase-updater-%s' % hash,
-       url=frontend_caucase_url,
-       data_dir=data_dir,
-       ca_path='%s/ca.crt' % data_dir,
-       crl_path='%s/crl.pem' % data_dir,
-       on_renew='${caucase-updater-housekeeper:output}; ${apache-graceful:output}',
-       max_sleep=ssl_parameter_dict.get('max-crl-update-delay', 1.0),
-       openssl=parameter_dict['openssl'] ~ '/bin/openssl',
-     )}}
-{%   do section('caucase-updater-%s' % hash) -%}
-{% endfor -%}
-
-{% if frontend_caucase_url_hash_list -%}
-[caucase-updater-housekeeper]
-recipe = collective.recipe.template
-output = ${directory:bin}/caucase-updater-housekeeper
-mode = 700
-input =
-  inline:
-  #!${buildout:executable}
-  import glob
-  import os
-  import subprocess
-  hash_list = {{ repr(frontend_caucase_url_hash_list) }}
-  crt_list = ['%s.crt' % e for e in hash_list]
-  crl_list = ['%s.crl' % e for e in hash_list]
-{% if shared_ca_path -%}
-  crt_list.append('{{ shared_ca_path }}/cacert.pem')
-  crl_list.append('{{ shared_ca_path }}/crl')
-{% endif -%}
-  for path in glob.glob('${apache-conf-ssl:ca-cert-dir}/*.crt'):
-    if os.path.basename(path) not in crt_list:
-      os.unlink(path)
-  for path in glob.glob('${apache-conf-ssl:crl-dir}/*.crl'):
-    if os.path.basename(path) not in crl_list:
-      os.unlink(path)
-  for hash in hash_list:
-    crt = '${directory:srv}/client-cert-ca/%s/ca.crt' % hash
-    crt_link = '${apache-conf-ssl:ca-cert-dir}/%s.crt' % hash
-    crl = '${directory:srv}/client-cert-ca/%s/crl.pem' % hash
-    crl_link = '${apache-conf-ssl:crl-dir}/%s.crl' % hash
-    if os.path.isfile(crt) and not os.path.islink(crt_link):
-      os.symlink(crt, crt_link)
-    if os.path.isfile(crl) and not os.path.islink(crl_link):
-      os.symlink(crl, crl_link)
-  subprocess.check_call(['{{ parameter_dict["openssl"] }}/bin/c_rehash', '${apache-conf-ssl:ca-cert-dir}'])
-  subprocess.check_call(['{{ parameter_dict["openssl"] }}/bin/c_rehash', '${apache-conf-ssl:crl-dir}'])
-
-[caucase-updater-housekeeper-run]
-recipe = plone.recipe.command
-command = ${caucase-updater-housekeeper:output}
-update-command = ${:command}
-{% endif -%}
 
 {% set haproxy_dict = {} -%}
 {% set apache_dict = {} -%}
@@ -187,27 +123,8 @@ key = ${directory:apache-conf}/apache.pem
 # XXX caucase certificate is not supported by caddy for now
 caucase-cert = ${directory:apache-conf}/apache-caucase.crt
 caucase-key = ${directory:apache-conf}/apache-caucase.pem
-{% if frontend_caucase_url_list -%}
-depends = ${caucase-updater-housekeeper-run:recipe}
-ca-cert-dir = ${directory:apache-ca-cert-dir}
-crl-dir = ${directory:apache-crl-dir}
-{%- endif %}
-
-[simplefile]
-< = jinja2-template-base
-template = inline:{{ '{{ content }}' }}
-
-{% macro simplefile(section_name, file_path, content, mode='') -%}
-{%   set content_section_name = section_name ~ '-content' -%}
-[{{  content_section_name }}]
-content = {{ dumps(content) }}
-
-[{{  section(section_name) }}]
-< = simplefile
-rendered = {{ file_path }}
-context = key content {{content_section_name}}:content
-mode = {{ mode }}
-{%- endmacro %}
+ca-cert =  ${directory:apache-conf}/ca.crt
+crl = ${directory:apache-conf}/crl.pem
 
 [apache-ssl]
 {% if ssl_parameter_dict.get('key') -%}
@@ -237,10 +154,13 @@ cert = ${apache-ssl:cert}
 key = ${apache-ssl:key}
 cipher =
 ssl-session-cache = ${directory:log}/apache-ssl-session-cache
-{% if frontend_caucase_url_list -%}
 # Client x509 auth
-ca-cert-dir = ${apache-conf-ssl:ca-cert-dir}
-crl-dir = ${apache-conf-ssl:crl-dir}
+ca-cert = ${apache-conf-ssl:ca-cert}
+crl = ${apache-conf-ssl:crl}
+
+{% if shared_ca_path -%}
+shared-ca-cert = {{ shared_ca_path }}/cacert.pem
+shared-crl = {{ shared_ca_path }}/crl 
 {%- endif %}
 
 [apache-conf]
@@ -266,8 +186,8 @@ input = inline:
   kill -USR1 "$(cat '${apache-conf-parameter-dict:pid-file}')"
 
 [{{ section('apache-promise') }}]
-<= monitor-promise-base
 # Check any apache port in ipv4, expect other ports and ipv6 to behave consistently
+<= monitor-promise-base
 module = check_port_listening
 name = apache.py
 config-hostname = {{ ipv4 }}
@@ -308,10 +228,6 @@ post = test ! -s ${apache-conf-parameter-dict:pid-file} || {{ parameter_dict['bi
 [directory]
 recipe = slapos.cookbook:mkdirectory
 apache-conf = ${:etc}/apache
-{% if frontend_caucase_url_list -%}
-apache-ca-cert-dir = ${:apache-conf}/ssl.crt
-apache-crl-dir = ${:apache-conf}/ssl.crl
-{% endif -%}
 bin = ${buildout:directory}/bin
 etc = ${buildout:directory}/etc
 services = ${:etc}/run

stack/monitor/README.md

@@ -94,20 +94,18 @@ New promises should be placed into the folder etc/plugin, legacy promises are in
 You will use slapos.cookbook:promise.plugin to generate your promise script into `etc/plugin` directory. Adding a promise will look like this:
 
     [promise-check-site]
-    recipe = slapos.cookbook:promise.plugin
-    eggs =
-      slapos.toolbox
-    output = ${directory:plugins}/promise-check-mysite-status.py
-    content = 
-      from slapos.promise.plugin.check_site_state import RunPromise
-    config-site-url = ${publish:site-url}
-    config-connection-timeout = 20
+    <= monitor-promise-base
+    module = check_port_listening
+    name = check_site.py
+    config-hostname = ${publish:ipv6}
+    config-port = 2020
     config-foo = bar
-    mode = 600
+
+The section `monitor-promise-base` is defined in the monitor stack, `name` is the filename of the script that will be generated under `etc/plugin` directory, `module` is the name of your promise module (you can find a list of existing module in https://lab.nexedi.com/nexedi/slapos.toolbox/tree/master/slapos/promise/plugin).
 
 Then you will have to add `promise-check-site` section to buildout parts, so it will be installed.
 
-In your promise code, you will be able to call `self.getConfig('site-url')`, `self.getConfig('connection-timeout')` and `self.getConfig('foo')`. The returned value is `None` if the config parameter is not set.
+In your promise code, you will be able to call `self.getConfig('hostname')`, `self.getConfig('port')` and `self.getConfig('foo')`. The returned value is `None` if the config parameter is not set.
 
 Slapgrid will run each promise every time a partition is processed (every minutes in theory), if the partition is up to date, slapgrid will only run promises anomaly check and save the result in a json file. Here is an exemple of promise result: