instance-zeo.cfg.in

{% set next_port = itertools.count(slapparameter_dict['tcpv4-port']).next -%}
{% set ipv4 = (ipv4_set | list)[0] -%}
{% set backup_periodicity = slapparameter_dict.get('backup-periodicity', 'daily') -%}
{% set part_list = [] -%}
{% macro section(name) %}{% do part_list.append(name) %}{{ name }}{% endmacro -%}
{% set storage_dict = {} -%}
{% set default_zodb_path = buildout_directory ~ '/srv/zodb' -%}
{% set default_backup_path = buildout_directory ~ '/srv/backup/zodb' -%}
{% set bin_directory = parameter_dict['buildout-bin-directory'] -%}

[zeo-base]
recipe = slapos.cookbook:zeo
log-path = ${directory:log}/${:base-name}.log
pid-path = ${directory:run}/${:base-name}.pid
conf-path = ${directory:etc}/${:base-name}.conf
wrapper-path = ${directory:services}/${:base-name}
binary-path = {{ bin_directory }}/runzeo
ip = {{ ipv4 }}

{% set known_tid_storage_identifier_dict = {} -%}
{% set zodb_dict = {} -%}
{% for name, zodb in slapparameter_dict['zodb-dict'].iteritems() -%}
{%   do zodb_dict.setdefault(zodb.get('family', 'default').lower(), []).append((name, zodb)) -%}
{% endfor -%}
{% set tidstorage_port = slapparameter_dict['tcpv4-port'] + len(zodb_dict) -%}
{% for family, zodb in zodb_dict.iteritems() -%}
{%   set storage_list = [] -%}
{%   set current_port = next_port() -%}
{%   set known_tid_storage_identifier_host = (ipv4, current_port), -%}
{%   for name, zodb in zodb -%}
{%     do storage_dict.__setitem__(name, {'server': ipv4 ~ ':' ~ current_port, 'storage': name}) %}
{%     set path = zodb.get('path', '%(zodb)s/%(name)s.fs') % {'zodb': default_zodb_path, 'name': name} -%}
{%     do storage_list.append((name, path)) -%}
{%     set backup_directory = zodb.get('backup', '%(backup)s/%(name)s') % {'backup': default_backup_path, 'name': name} -%}
{# BBB: No mount-point specified because they're meaningless for ZEO and
        TIDStorage. Pass '' for compatibility, and not None
        because this would disable TIDStorage bootstrapping. -#}
{%     do known_tid_storage_identifier_dict.__setitem__(json_module.dumps(
         (known_tid_storage_identifier_host, name)),
         (path, backup_directory, '')) -%}
{%   endfor -%}

{%   set zeo_section_name = 'zeo-' ~ family %}
[{{ zeo_section_name }}]
< = zeo-base
base-name = zeo-{{ family }}
port = {{ current_port }}
storage = {{ dumps(storage_list) }}

[{{ section("logrotate-" ~ zeo_section_name) }}]
< = logrotate-entry-base
name = {{ "${" ~ zeo_section_name ~ ":base-name}" }}
log = {{ "${" ~ zeo_section_name ~ ":log-path}" }}
post = test ! -s {{ "${" ~ zeo_section_name ~":pid-path}" }} || {{ bin_directory }}/slapos-kill --pidfile {{ "${" ~ zeo_section_name ~ ":pid-path}" }} -s USR2

[{{ section(zeo_section_name ~ "-promise") }}]
recipe = slapos.cookbook:check_port_listening
hostname = {{ "${" ~ zeo_section_name ~ ":ip}" }}
port = {{ "${" ~ zeo_section_name ~ ":port}" }}
path = ${directory:promises}/zeo-{{ family }}

{% endfor -%}

{% if backup_periodicity == 'never' -%}
{%   set known_tid_storage_identifier_dict = () %}
{%   set tidstorage_repozo_path = '' -%}
{% else -%}

[tidstorage]
recipe = slapos.cookbook:tidstorage
known-tid-storage-identifier-dict = {{ dumps(known_tid_storage_identifier_dict) }}
configuration-path = ${directory:etc}/tidstorage.py
ip = {{ ipv4 }}
port = {{ tidstorage_port }}
{% set tidstorage_repozo_path = slapparameter_dict.get('tidstorage-repozo-path', buildout_directory ~ '/srv/backup/tidstorage') -%}
timestamp-file-path = {{ tidstorage_repozo_path ~ '/repozo_tidstorage_timestamp.log' }}
{# BBB: recipe requires logfile-name for nothing because tidstorage runs in foreground mode -#}
logfile-name =
pidfile-name = ${directory:run}/tidstorage.pid
{# TODO: Add support for backup status file, so that the status file can be close to the ZODB (rather than close to the backup files). And do it efficiently, to not copy the whole status file every time. -#}
status-file = {{ tidstorage_repozo_path ~ '/tidstorage.tid' }}
tidstorage-repozo-binary = {{ bin_directory }}/tidstorage_repozo
tidstoraged-binary = {{ bin_directory }}/tidstoraged
repozo-binary = {{ bin_directory }}/repozo
repozo-wrapper = ${buildout:bin-directory}/tidstorage-repozo
{% if len(known_tid_storage_identifier_dict) > 1 -%}
tidstorage-wrapper = ${directory:services}/tidstoraged

[{{ section("promise-tidstorage") }}]
recipe = slapos.cookbook:check_port_listening
hostname = ${tidstorage:ip}
port = ${tidstorage:port}
path = ${directory:promises}/tidstorage
{% endif -%}

[{{ section("cron-entry-tidstorage-backup") }}]
# TODO:
# - configurable full/incremental
# - configurable retention
recipe = slapos.cookbook:cron.d
cron-entries = ${cron:cron-entries}
name = tidstorage
time = {{ dumps(backup_periodicity) }}
command = ${tidstorage:repozo-wrapper}

# Used for ERP5 resiliency or (more probably)
# webrunner resiliency with erp5 inside.
[{{ section("resiliency-exclude-file") }}]
# Generate rdiff exclude file
recipe = slapos.recipe.template:jinja2
mode = 644
template = {{ 'inline:{{ "${directory:zodb}/**\\n${directory:log}\\n" }}' }}
rendered = ${directory:srv}/exporter.exclude

[{{ section("resiliency-identity-signature-script")}}]
# Generate identity script used by webrunner to check data integrity
# It excludes repozo files as they already include a hash function
# used to check backups when rebuilding the datafs
recipe = slapos.cookbook:wrapper
command-line = {{ bin_directory }}/backup-identity-script-excluding-path --exclude-path "srv/backup/logrotate" --exclude-path "srv/backup/zodb"
wrapper-path = ${directory:srv}/.backup_identity_script
mode = 770

[{{ section("resiliency-after-import-script") }}]
# Generate after import script used by importer instance of webrunner
recipe = collective.recipe.template
input = inline: #!/bin/sh
  # DO NOT RUN THIS SCRIPT ON PRODUCTION INSTANCE
  # OR ZODB DATA WILL BE ERASED.

  # This script will restore the repozo backup to the real
  # zodb location. It is launched by the clone (importer) instance of webrunner
  # in the end of the import script.

  # Depending on the output, it will create a file containing
  # the status of the restoration (success or failure).
  zodb_directory="${directory:zodb}"
  zodb_backup_directory="{{ default_backup_path }}"
  repozo="${tidstorage:repozo-binary}"
  EXIT_CODE=0
  {% for family, zodb in zodb_dict.iteritems() -%}
  {%   for name, zodb in zodb -%}
  {%   set zeo_section_name = 'zeo-' ~ family %}
  storage_name="{{ name }}"
  zodb_path="$storage_name.fs"
  pid_file={{ "${" ~ zeo_section_name ~ ":pid-path}" }}
  if [ -e "$pid_file" ]; then
    pid=$(cat $pid_file) > /dev/null 2>&1
    if kill -0 "$pid"; then
      echo "Zeo is already running with pid $pid. Aborting."
      exit 1
    fi
  fi
  echo "Removing $zodb_path..."
  echo "Restoring $storage_name into $zodb_path..."
  $repozo --verify --repository="$zodb_backup_directory/$storage_name"
  CURRENT_EXIT_CODE=$?
  if [ ! "$CURRENT_EXIT_CODE"="0" ]; then
    echo "$storage_name Backup verification failed. Backup data is inconsistent."
    exit "$CURRENT_EXIT_CODE"
  fi
  $repozo --recover --output="$zodb_directory/$zodb_path" --repository="$zodb_backup_directory/$storage_name"
  CURRENT_EXIT_CODE=$?
  if [ ! "$CURRENT_EXIT_CODE"="0" ]; then
    EXIT_CODE="$CURRENT_EXIT_CODE"
    echo "$storage_name Backup restoration failed."
  fi
  {%   endfor -%}
  {% endfor -%}
  exit $EXIT_CODE
output = ${directory:srv}/runner-import-restore
mode = 755

{% endif -%}

[{{ section('promise-check-computer-memory') }}]
recipe = slapos.cookbook:wrapper
wrapper-path = ${directory:promises}/check-computer-memory
command-line = "{{ parameter_dict["check-computer-memory-binary"] }}" -db ${monitor-instance-parameter:collector-db} --threshold "{{ slapparameter_dict["computer-memory-percent-threshold"] }}" --unit percent

[publish]
recipe = slapos.cookbook:publish.serialised
storage-dict = {{ dumps(storage_dict) }}
{% if len(known_tid_storage_identifier_dict) > 1 -%}
tidstorage-ip = ${tidstorage:ip}
tidstorage-port = ${tidstorage:port}
{% else -%}
tidstorage-ip =
tidstorage-port =
{% endif -%}
monitor-base-url = ${monitor-publish-parameters:monitor-base-url}

[directory]
recipe = slapos.cookbook:mkdirectory
bin = ${buildout:directory}/bin
etc = ${buildout:directory}/etc
services = ${:etc}/run
promises = ${:etc}/promise
srv = ${buildout:directory}/srv
var = ${buildout:directory}/var
log = ${:var}/log
run = ${:var}/run
backup-zodb = {{ default_backup_path }}
zodb = {{ default_zodb_path }}
tidstorage = {{ tidstorage_repozo_path }}

[monitor-instance-parameter]
monitor-httpd-ipv6 = {{ (ipv6_set | list)[0] }}
monitor-httpd-port = {{ next_port() }}
monitor-title = {{ slapparameter_dict['name'] }}
password = {{ slapparameter_dict['monitor-passwd'] }}

[buildout]
extends =
  {{ logrotate_cfg }}
  {{ parameter_dict['template-monitor'] }}
parts +=
  {{ part_list | join('\n  ') }}
  publish