instance-balancer.cfg.in

{% import "caucase" as caucase with context %}
{% set part_list = [] -%}
{% macro section(name) %}{% do part_list.append(name) %}{{ name }}{% endmacro -%}
{% set ssl_parameter_dict = slapparameter_dict['ssl'] -%}
{% set caucase_url = ssl_parameter_dict.get('caucase-url') %}
{% set with_embedded_caucased = caucase_url is none %}
{% if not caucase_url -%}
{%   if ipv6_set -%}
{%     set caucase_host = '[' ~ (ipv6_set | list)[0] ~ ']' %}
{%-  else -%}
{%     set caucase_host = (ipv4_set | list)[0] %}
{%-  endif %}
{%   set caucase_http_port = 2198 -%}{# caucase_https_port is implicitly 2199 #}
{%   set caucase_netloc = caucase_host ~ ':' ~ caucase_http_port -%}
{%   set caucase_url = 'http://' ~ caucase_netloc %}
{% endif %}
{# Caucase related parts include a hash of the caucase_url, to make is possible to re-request with another caucase url. #}
{% set caucase_haproxy_certificate_suffix = '-' + hashlib.md5(six.ensure_binary(caucase_url)).hexdigest()[:6] -%}
{% set frontend_caucase_url_list = ssl_parameter_dict.get('frontend-caucase-url-list', []) -%}
{#
XXX: This template only supports exactly one IPv4 and (if ipv6 is used) one IPv6
per partition. No more (undefined result), no less (IndexError).
-#}
{% set ipv4 = (ipv4_set | list)[0] -%}
{% if ipv6_set -%}
{%   set ipv6 = (ipv6_set | list)[0] -%}
{% endif -%}

[jinja2-template-base]
recipe = slapos.recipe.template:jinja2

[simplefile]
< = jinja2-template-base
inline = {{ '{{ content }}' }}

{% macro simplefile(section_name, file_path, content, mode='') -%}
{%   set content_section_name = section_name ~ '-content' -%}
[{{  content_section_name }}]
content = {{ dumps(content) }}

[{{  section(section_name) }}]
< = simplefile
output = {{ file_path }}
context = key content {{content_section_name}}:content
mode = {{ mode }}
{%- endmacro %}

{% if with_embedded_caucased -%}
{{   caucase.caucased(
       prefix='caucased-haproxy-certificate',
       buildout_bin_directory=bin_directory,
       caucased_path='${directory:services-on-watch}/caucased' + caucase_haproxy_certificate_suffix,
       backup_dir='${directory:backup-caucased-haproxy-certificate}',
       data_dir='${directory:srv-caucased-haproxy-certificate}',
       netloc=caucase_netloc,
       tmp='${directory:tmp}',
       service_auto_approve_count=ssl_parameter_dict.get('service-auto-approve-amount', 1),
       user_auto_approve_count=ssl_parameter_dict.get('user-auto-approve-amount', 0),
       key_len=ssl_parameter_dict.get('key-length', 2048),
)}}
{%   do section('caucased-haproxy-certificate') -%}
{%   do section('caucased-haproxy-certificate-promise') -%}
{% endif -%}

[haproxy-certificate]
cert-and-key-file = ${directory:etc}/${:_buildout_section_name_}{{ caucase_haproxy_certificate_suffix }}-cert-and-key.pem
ca-file = ${directory:etc}/${:_buildout_section_name_}{{ caucase_haproxy_certificate_suffix }}.ca.crt
crl-file = ${directory:etc}/${:_buildout_section_name_}{{ caucase_haproxy_certificate_suffix }}.crl

[haproxy-certificate-csr-config]
recipe = slapos.recipe.template
inline =
  [ req ]
  prompt = no
  req_extensions = req_ext
  distinguished_name = dn
  [ dn ]
  CN = haproxy
  [ req_ext ]
  subjectAltName = @alt_names
  [ alt_names ]
  IP.1 = {{ ipv4 }}
{% if ipv6_set %}
  IP.2 = {{ ipv6 }}
{% endif %}
output = ${buildout:parts-directory}/${:_buildout_section_name_}/${:_buildout_section_name_}.txt

[haproxy-certificate-csr]
recipe = plone.recipe.command
command =
  if [ ! -f '${:csr}' ] ; then
   {{ parameter_dict['openssl'] }}/bin/openssl req \
      -newkey rsa \
      -batch \
      -new \
      -sha256 \
      -nodes \
      -keyout /dev/null \
      -config '${haproxy-certificate-csr-config:output}' \
      -out '${:csr}'
  fi
stop-on-error = true
csr = ${directory:srv}/${:_buildout_section_name_}{{ caucase_haproxy_certificate_suffix }}.csr.pem

{% if ssl_parameter_dict.get('csr') %}
{{ simplefile(
    'haproxy-certificate-csr-from-parameters',
    '${directory:etc}/haproxy-certificate-csr-from-parameters.pem',
    ssl_parameter_dict['csr']) }}
[haproxy-certificate-csr]
csr = ${haproxy-certificate-csr-from-parameters:output}
{% endif %}

{{ caucase.updater(
     prefix='caucase-updater-haproxy-certificate',
     buildout_bin_directory=parameter_dict['bin-directory'],
     updater_path='${directory:services-on-watch}/caucase-updater-haproxy-certificate' + caucase_haproxy_certificate_suffix,
     url=caucase_url,
     data_dir='${directory:caucase-updater-haproxy-certificate}',
     crt_path='${haproxy-certificate:cert-and-key-file}',
     ca_path='${haproxy-certificate:ca-file}',
     crl_path='${haproxy-certificate:crl-file}',
     key_path='${haproxy-certificate:cert-and-key-file}',
     on_renew='${haproxy-reload:output}',
     max_sleep=ssl_parameter_dict.get('max-crl-update-delay', 1.0),
     template_csr='${haproxy-certificate-csr:csr}',
     openssl=parameter_dict['openssl'] ~ '/bin/openssl',
)}}
{% do section('caucase-updater-haproxy-certificate') -%}

{% set frontend_caucase_url_hash_list = [] -%}
{% for frontend_caucase_url in frontend_caucase_url_list -%}
{%   set hash = hashlib.md5(six.ensure_binary(frontend_caucase_url)).hexdigest() -%}
{%   do frontend_caucase_url_hash_list.append(hash) -%}
{%   set data_dir = '${directory:client-cert-ca}/%s' % hash -%}
{{   caucase.updater(
       prefix='caucase-updater-%s' % hash,
       buildout_bin_directory=parameter_dict['bin-directory'],
       updater_path='${directory:services-on-watch}/caucase-updater-%s' % hash,
       url=frontend_caucase_url,
       data_dir=data_dir,
       ca_path='%s/ca.crt' % data_dir,
       crl_path='%s/crl.pem' % data_dir,
       on_renew='${caucase-updater-housekeeper:output}',
       max_sleep=ssl_parameter_dict.get('max-crl-update-delay', 1.0),
       openssl=parameter_dict['openssl'] ~ '/bin/openssl',
     )}}
{%   do section('caucase-updater-%s' % hash) -%}
{% endfor -%}

{% if frontend_caucase_url_hash_list -%}
[caucase-updater-housekeeper]
recipe = collective.recipe.template
output = ${directory:bin}/caucase-updater-housekeeper
mode = 700
input =
  inline:
  #!${buildout:executable}
  import glob
  import os
  import subprocess
  hash_list = {{ repr(frontend_caucase_url_hash_list) }}
  crt_list = ['%s.crt' % e for e in hash_list]
  for path in glob.glob('${haproxy-conf-ssl:ca-cert-dir}/*.crt'):
    if os.path.basename(path) not in crt_list:
      os.unlink(path)
  crl_list = ['%s.crl' % e for e in hash_list]
  for path in glob.glob('${haproxy-conf-ssl:crl-dir}/*.crl'):
    if os.path.basename(path) not in crl_list:
      os.unlink(path)

  for hash in hash_list:
    crt = '${directory:client-cert-ca}/%s/ca.crt' % hash
    crt_link = '${haproxy-conf-ssl:ca-cert-dir}/%s.crt' % hash
    crl = '${directory:client-cert-ca}/%s/crl.pem' % hash
    crl_link = '${haproxy-conf-ssl:crl-dir}/%s.crl' % hash
    if os.path.isfile(crt) and not os.path.islink(crt_link):
      os.symlink(crt, crt_link)
    if os.path.isfile(crl) and not os.path.islink(crl_link):
      os.symlink(crl, crl_link)
  subprocess.check_call(['{{ parameter_dict["openssl"] }}/bin/c_rehash', '${haproxy-conf-ssl:ca-cert-dir}'])
  subprocess.check_call(['{{ parameter_dict["openssl"] }}/bin/c_rehash', '${haproxy-conf-ssl:crl-dir}'])

  # assemble all CA and all CRLs in one file for haproxy
  with open('${haproxy-conf-ssl:ca-cert}.tmp', 'w') as f:
    for path in glob.glob('${haproxy-conf-ssl:ca-cert-dir}/*.crt'):
      with open(path) as in_f:
        f.write('#{}\n'.format(path))
        f.write(in_f.read() + '\n')
  with open('${haproxy-conf-ssl:crl}.tmp', 'w') as f:
    for path in glob.glob('${haproxy-conf-ssl:crl-dir}/*.crl'):
      with open(path) as in_f:
        f.write('#{}\n'.format(path))
        f.write(in_f.read() + '\n')

  if os.path.exists('${haproxy-conf-ssl:ca-cert}'):
    os.unlink('${haproxy-conf-ssl:ca-cert}')
  if os.path.exists('${haproxy-conf-ssl:crl}'):
    os.unlink('${haproxy-conf-ssl:crl}')
  os.rename('${haproxy-conf-ssl:ca-cert}.tmp', '${haproxy-conf-ssl:ca-cert}')
  os.rename('${haproxy-conf-ssl:crl}.tmp', '${haproxy-conf-ssl:crl}')

  subprocess.check_call(['${haproxy-reload:output}'])


[caucase-updater-housekeeper-run]
recipe = plone.recipe.command
command = ${caucase-updater-housekeeper:output}
update-command = ${:command}
{% endif -%}

[haproxy-cfg-parameter-dict]
recipe = slapos.recipe.build
ipv4 = {{ ipv4 }}
ipv6 = {{ ipv6 }}
cert = ${haproxy-conf-ssl:certificate}
{% if frontend_caucase_url_list -%}
ca-cert = ${haproxy-conf-ssl:ca-cert}
crl = ${haproxy-conf-ssl:crl}
{% endif %}
stats-socket = ${directory:run}/ha.sock
path-routing-list = {{ dumps(slapparameter_dict['path-routing-list']) }}
family-path-routing-dict = {{ dumps(slapparameter_dict['family-path-routing-dict']) }}
pidfile = ${directory:run}/haproxy.pid
log-socket = ${rsyslogd-cfg-parameter-dict:log-socket}
server-check-path = {{ dumps(slapparameter_dict['haproxy-server-check-path']) }}
slapparameter-dict = {{ dumps(slapparameter_dict) }}
ports-state-file = ${buildout:directory}/.${:_buildout_section_name_}-ports.json

init =
  import functools
  import itertools
  import json
  import os
  import shutil
  import six
  from zc.buildout import UserError

  slapparameter_dict = options['slapparameter-dict']
  ipv4 = options['ipv4']
  ipv6 = options['ipv6']

  # read port state file
  port_dict = {}
  previous_port_dict = None
  if os.path.exists(options['ports-state-file']):
    with open(options['ports-state-file']) as f:
      port_dict = json.load(f)
      previous_port_dict = dict(port_dict)

  _next_port = functools.partial(next, itertools.count(slapparameter_dict['tcpv4-port']))
  def get_port(name):
    if name in port_dict:
      return port_dict[name]
    port = _next_port()
    while port in port_dict.values():
      port = _next_port()
    port_dict[name] = port
    return port

  backend_dict = {}
  frontend_dict = {}
  zope_virtualhost_monster_backend_dict = {}
  for family_name, parameter_id_list in sorted(
      six.iteritems(slapparameter_dict['zope-family-dict'])):
    zope_family_address_list = []
    ssl_authentication = slapparameter_dict['ssl-authentication-dict'][family_name]
    for parameter_id in parameter_id_list:
      zope_family_address_list.extend(slapparameter_dict[parameter_id])

      # Generate entries with rewrite rule for test runnners
      test_runner_address_list = slapparameter_dict.get(parameter_id + '-test-runner-address-list', [])
      if test_runner_address_list:
        test_runner_backend_mapping = {}
        test_runner_balancer_url_list = []
        for i, (test_runner_internal_ip, test_runner_internal_port) in enumerate(test_runner_address_list):
          test_runner_backend_mapping['unit_test_%s' % i] = \
            'http://%s:%s' % (test_runner_internal_ip, test_runner_internal_port)
          test_runner_balancer_url_list.append(
            'https://%s:%s/unit_test_%s/' % (ipv4, get_port('test-runner-' + family_name), i))
        zope_virtualhost_monster_backend_dict[(ipv4, get_port('test-runner-' + family_name))] =\
          ( ssl_authentication, test_runner_backend_mapping )
        self.buildout['publish'][family_name + '-test-runner-url-list'] = test_runner_balancer_url_list

    if not zope_family_address_list:
      raise UserError('No zope defined for family %s (maybe not ready)' % family_name)

    # consume a port for compatibility when were using apache + haproxy
    get_port('apache-compatibility-' + family_name)

    legacy_port = get_port('legacy-' + family_name)
    # a port for monitoring promise (which port is not important, the promise checks
    # that haproxy is healthy enough to listen on a port)
    options['haproxy-promise-port'] = legacy_port
    frontend_dict['legacy-frontend-' + family_name] = {
      'port': legacy_port,
      'client-cert-required': ssl_authentication,
      'backend-name': family_name,
      'request-path-prepend': None,
    }
    backend_dict[family_name] = {
      'timeout': slapparameter_dict['timeout-dict'][family_name],
      'backend-list': zope_family_address_list,
    }
    external_scheme = 'webdavs' if any(a[2] for a in zope_family_address_list) else 'https'
    self.buildout['publish'][family_name] = "{external_scheme}://{ipv4}:{legacy_port}".format(**locals())
    self.buildout['publish'][family_name + "-v6"] = "{external_scheme}://[{ipv6}]:{legacy_port}".format(**locals())

  for frontend_name, frontend in six.iteritems(slapparameter_dict['frontend-parameter-dict']):
    frontend_port = get_port('frontend-' + frontend_name)
    family_name = frontend['zope-family']
    frontend_dict['frontend-' + frontend_name] = {
      'port': frontend_port,
      'client-cert-required': slapparameter_dict['ssl-authentication-dict'][family_name],
      'backend-name': family_name,
      'request-path-prepend': frontend['internal-path'],
    }
    self.buildout['publish']['url-backend-' + frontend_name] = "https://[{ipv6}]:{frontend_port}".format(**locals())

  options['backend-dict'] = backend_dict
  options['frontend-dict'] = frontend_dict
  options['zope-virtualhost-monster-backend-dict'] = zope_virtualhost_monster_backend_dict

  if port_dict != previous_port_dict:
    with open(options['ports-state-file'] + '.tmp', 'w') as f:
      json.dump(port_dict, f, indent=True)
    shutil.move(options['ports-state-file'] + '.tmp', options['ports-state-file'])

[haproxy-cfg]
< = jinja2-template-base
url = {{ parameter_dict['template-haproxy-cfg'] }}
output = ${directory:etc}/haproxy.cfg
context =
  section parameter_dict haproxy-cfg-parameter-dict
  import urllib_parse six.moves.urllib.parse
extensions = jinja2.ext.do

[haproxy-reload]
recipe = collective.recipe.template
output = ${directory:bin}/${:_buildout_section_name_}
mode = 700
input =
  inline:
  #!/bin/sh
  kill -USR2 $(cat "${haproxy-cfg-parameter-dict:pidfile}")

[{{ section('haproxy') }}]
recipe = slapos.cookbook:wrapper
wrapper-path = ${directory:services-on-watch}/haproxy
command-line = "{{ parameter_dict['haproxy'] }}/sbin/haproxy" -f "${haproxy-cfg:output}"
hash-files = ${haproxy-cfg:output}

[{{ section('haproxy-socat-stats')}}]
recipe = slapos.cookbook:wrapper
wrapper-path = ${directory:bin}/${:_buildout_section_name_}
command-line = "{{ parameter_dict['socat'] }}/bin/socat" unix-connect:${haproxy-cfg-parameter-dict:stats-socket} stdio

[rsyslogd-cfg-parameter-dict]
log-socket = ${directory:run}/log.sock
access-log-file = ${directory:log}/apache-access.log
error-log-file = ${directory:log}/apache-error.log
pid-file = ${directory:run}/rsyslogd.pid
spool-directory = ${directory:rsyslogd-spool}

[rsyslogd-cfg]
<= jinja2-template-base
url = {{ parameter_dict['template-rsyslogd-cfg'] }}
output = ${directory:etc}/rsyslogd.conf
context = section parameter_dict rsyslogd-cfg-parameter-dict

[{{ section ('rsyslogd') }}]
recipe = slapos.cookbook:wrapper
command-line = {{ parameter_dict['rsyslogd'] }}/sbin/rsyslogd -i ${rsyslogd-cfg-parameter-dict:pid-file} -n -f ${rsyslogd-cfg:output}
wrapper-path = ${directory:services-on-watch}/rsyslogd
hash-existing-files = ${buildout:directory}/software_release/buildout.cfg
hash-files = ${rsyslogd-cfg:output}

[{{ section ('rsyslogd-listen-promise') }}]
<= monitor-promise-base
promise = check_command_execute
name = rsyslogd_listen_promise.py
config-command = test -S ${rsyslogd-cfg-parameter-dict:log-socket}


[haproxy-conf-ssl]
certificate = ${haproxy-certificate:cert-and-key-file}
{% if frontend_caucase_url_list -%}
ca-cert = ${directory:etc}/frontend-ca.pem
ca-cert-dir = ${directory:ca-cert}
crl = ${directory:etc}/frontend-crl.pem
crl-dir = ${directory:crl}
depends = ${caucase-updater-housekeeper-run:recipe}
{%- endif %}

# BBB cert and key as ssl parameters. Use caucase instead.
{% if ssl_parameter_dict.get('key') -%}
{{ simplefile(
    'haproxy-conf-ssl-certificate-and-key-from-parameters',
    '${directory:etc}/certificate-and-key-from-parameters.pem',
    ssl_parameter_dict['cert'] ~ "\n" ~ ssl_parameter_dict['key']) }}
[haproxy-conf-ssl]
certificate = ${haproxy-conf-ssl-certificate-and-key-from-parameters:output}
{% endif %}

[{{ section('haproxy-promise') }}]
<= monitor-promise-base
# Check any haproxy port in ipv4, expect other ports and ipv6 to behave consistently
promise = check_socket_listening
name = haproxy.py
config-host = {{ ipv4 }}
config-port = ${haproxy-cfg-parameter-dict:haproxy-promise-port}

[{{ section('publish') }}]
recipe = slapos.cookbook:publish.serialised
# note: some values are pushed by haproxy-cfg-parameter-dict
caucase-http-url = {{ caucase_url }}
monitor-base-url = ${monitor-publish-parameters:monitor-base-url}

[{{ section('logrotate-rsyslogd') }}]
< = logrotate-entry-base
name = rsyslogd
log = ${rsyslogd-cfg-parameter-dict:access-log-file} ${rsyslogd-cfg-parameter-dict:error-log-file}
post = test ! -s ${rsyslogd-cfg-parameter-dict:pid-file} || kill -HUP $(cat ${rsyslogd-cfg-parameter-dict:pid-file})


[directory]
recipe = slapos.cookbook:mkdirectory
bin = ${buildout:directory}/bin
etc = ${buildout:directory}/etc
services = ${:etc}/run
services-on-watch = ${:etc}/service
var = ${buildout:directory}/var
run = ${:var}/run
log = ${:var}/log
srv = ${buildout:directory}/srv
{% if with_embedded_caucased %}
srv-caucased-haproxy-certificate = ${:srv}/caucased-haproxy-certificate{{ caucase_haproxy_certificate_suffix }}/
backup-caucased-haproxy-certificate = ${:srv}/backup/caucased{{ caucase_haproxy_certificate_suffix }}
{% endif %}
caucase-updater-haproxy-certificate = ${:srv}/caucase-updater-haproxy-certificate
tmp = ${buildout:directory}/tmp
apachedex = ${monitor-directory:private}/apachedex
rsyslogd-spool = ${:run}/rsyslogd-spool
{% if frontend_caucase_url_list -%}
ca-cert = ${:etc}/ssl.crt
crl = ${:etc}/ssl.crl
client-cert-ca = ${:srv}/client-cert-ca
{% endif -%}

[{{ section('resiliency-exclude-file') }}]
# Generate rdiff exclude file in case of resiliency
< = jinja2-template-base
inline = {{ '{{ "${directory:log}/**\\n" }}' }}
output = ${directory:srv}/exporter.exclude

[{{ section('monitor-generate-apachedex-report') }}]
recipe = slapos.cookbook:cron.d
cron-entries = ${cron:cron-entries}
name = generate-apachedex-report
# The goal is to be executed before logrotate log rotation.
# Here, logrotate-entry-base:frequency = daily, so we run at 23 o'clock every day.
frequency = 0 23 * * *
command = ${monitor-generate-apachedex-report-wrapper:wrapper-path}

[monitor-generate-apachedex-report-wrapper]
recipe = slapos.cookbook:wrapper
wrapper-path = ${directory:bin}/${:command}
command-line = "{{ parameter_dict['run-apachedex-location'] }}" "{{ parameter_dict['apachedex-location'] }}" "${directory:apachedex}" ${monitor-publish-parameters:monitor-base-url}/private/apachedex --apache-log-list "${apachedex-parameters:apache-log-list}" --configuration ${apachedex-parameters:configuration}
command = generate-apachedex-report

[monitor-apachedex-report-config]
recipe = slapos.recipe.template
output = ${directory:etc}/${:_buildout_section_name_}
inline =
  {% for line in slapparameter_dict['apachedex-configuration'] %}
    {# apachedex config files use shlex.split, so we need to quote the arguments. #}
    {# BBB: in python 3 we can use shlex.quote instead. #}
    {{ repr(line.encode('utf-8')) }}
  {% endfor %}

[apachedex-parameters]
apache-log-list = ${rsyslogd-cfg-parameter-dict:access-log-file}
configuration = ${monitor-apachedex-report-config:output}
promise-threshold = {{ slapparameter_dict['apachedex-promise-threshold'] }}

[{{ section('monitor-promise-apachedex-result') }}]
<= monitor-promise-base
promise = check_command_execute
name = check-apachedex-result.py
config-command = "{{ parameter_dict['promise-check-apachedex-result'] }}" --apachedex_path "${directory:apachedex}" --status_file ${monitor-directory:private}/apachedex.report.json --threshold "${apachedex-parameters:promise-threshold}"

[{{ section('promise-check-computer-memory') }}]
<= monitor-promise-base
promise = check_command_execute
name = check-computer-memory.py
config-command = "{{ parameter_dict["check-computer-memory-binary"] }}" -db ${monitor-instance-parameter:collector-db} --threshold "{{ slapparameter_dict["computer-memory-percent-threshold"] }}" --unit percent

[monitor-instance-parameter]
monitor-httpd-ipv6 = {{ (ipv6_set | list)[0] }}
monitor-httpd-port = 2197
monitor-title = {{ slapparameter_dict['name'] }}
password = {{ slapparameter_dict['monitor-passwd'] }}

[buildout]
extends =
  {{ template_monitor }}
parts +=
  {{ part_list | join('\n  ') }}