instance-zeo.cfg.in 8.66 KB
Newer Older
1
{% set next_port = itertools.count(slapparameter_dict['tcpv4-port']).next -%}
2 3
{% set ipv4 = (ipv4_set | list)[0] -%}
{% set backup_periodicity = slapparameter_dict.get('backup-periodicity', 'daily') -%}
4
{% set part_list = [] -%}
5
{% macro section(name) %}{% do part_list.append(name) %}{{ name }}{% endmacro -%}
6 7
{% set storage_dict = {} -%}
{% set default_zodb_path = buildout_directory ~ '/srv/zodb' -%}
8
{% set default_backup_path = buildout_directory ~ '/srv/backup/zodb' -%}
9 10 11 12 13 14 15 16 17 18 19
{% set bin_directory = parameter_dict['buildout-bin-directory'] -%}

[zeo-base]
recipe = slapos.cookbook:zeo
log-path = ${directory:log}/${:base-name}.log
pid-path = ${directory:run}/${:base-name}.pid
conf-path = ${directory:etc}/${:base-name}.conf
wrapper-path = ${directory:services}/${:base-name}
binary-path = {{ bin_directory }}/runzeo
ip = {{ ipv4 }}

20 21 22 23
{% set known_tid_storage_identifier_dict = {} -%}
{% set zodb_dict = {} -%}
{% for name, zodb in slapparameter_dict['zodb-dict'].iteritems() -%}
{%   do zodb_dict.setdefault(zodb.get('family', 'default').lower(), []).append((name, zodb)) -%}
24
{% endfor -%}
25
{% set tidstorage_port = slapparameter_dict['tcpv4-port'] + len(zodb_dict) -%}
26 27
{% for family, zodb in zodb_dict.iteritems() -%}
{%   set storage_list = [] -%}
28 29
{%   set current_port = next_port() -%}
{%   set known_tid_storage_identifier_host = (ipv4, current_port), -%}
30
{%   for name, zodb in zodb -%}
31
{%     do storage_dict.__setitem__(name, {'server': ipv4 ~ ':' ~ current_port, 'storage': name}) %}
32 33 34 35 36 37 38 39 40
{%     set path = zodb.get('path', '%(zodb)s/%(name)s.fs') % {'zodb': default_zodb_path, 'name': name} -%}
{%     do storage_list.append((name, path)) -%}
{%     set backup_directory = zodb.get('backup', '%(backup)s/%(name)s') % {'backup': default_backup_path, 'name': name} -%}
{# BBB: No mount-point specified because they're meaningless for ZEO and
        TIDStorage. Pass '' for compatibility, and not None
        because this would disable TIDStorage bootstrapping. -#}
{%     do known_tid_storage_identifier_dict.__setitem__(json_module.dumps(
         (known_tid_storage_identifier_host, name)),
         (path, backup_directory, '')) -%}
41
{%   endfor -%}
42 43

{%   set zeo_section_name = 'zeo-' ~ family %}
44 45
[{{ zeo_section_name }}]
< = zeo-base
46
base-name = zeo-{{ family }}
47
port = {{ current_port }}
48 49 50
storage = {{ dumps(storage_list) }}

[{{ section("logrotate-" ~ zeo_section_name) }}]
51
< = logrotate-entry-base
52 53
name = {{ "${" ~ zeo_section_name ~ ":base-name}" }}
log = {{ "${" ~ zeo_section_name ~ ":log-path}" }}
54
post = test ! -s {{ "${" ~ zeo_section_name ~":pid-path}" }} || {{ bin_directory }}/slapos-kill --pidfile {{ "${" ~ zeo_section_name ~ ":pid-path}" }} -s USR2
55 56 57 58

[{{ section(zeo_section_name ~ "-promise") }}]
recipe = slapos.cookbook:check_port_listening
hostname = {{ "${" ~ zeo_section_name ~ ":ip}" }}
59 60
port = {{ "${" ~ zeo_section_name ~ ":port}" }}
path = ${directory:promises}/zeo-{{ family }}
61 62 63

{% endfor -%}

64 65
{% if backup_periodicity == 'never' -%}
{%   set known_tid_storage_identifier_dict = () %}
66
{%   set tidstorage_repozo_path = '' -%}
67
{% else -%}
68

69 70 71 72
[tidstorage]
recipe = slapos.cookbook:tidstorage
known-tid-storage-identifier-dict = {{ dumps(known_tid_storage_identifier_dict) }}
configuration-path = ${directory:etc}/tidstorage.py
73
ip = {{ ipv4 }}
74
port = {{ tidstorage_port }}
75 76
{% set tidstorage_repozo_path = slapparameter_dict.get('tidstorage-repozo-path', buildout_directory ~ '/srv/backup/tidstorage') -%}
timestamp-file-path = {{ tidstorage_repozo_path ~ '/repozo_tidstorage_timestamp.log' }}
77 78
{# BBB: recipe requires logfile-name for nothing because tidstorage runs in foreground mode -#}
logfile-name =
79
pidfile-name = ${directory:run}/tidstorage.pid
80 81
{# TODO: Add support for backup status file, so that the status file can be close to the ZODB (rather than close to the backup files). And do it efficiently, to not copy the whole status file every time. -#}
status-file = {{ tidstorage_repozo_path ~ '/tidstorage.tid' }}
82 83 84 85
tidstorage-repozo-binary = {{ bin_directory }}/tidstorage_repozo
tidstoraged-binary = {{ bin_directory }}/tidstoraged
repozo-binary = {{ bin_directory }}/repozo
repozo-wrapper = ${buildout:bin-directory}/tidstorage-repozo
86 87
{% if len(known_tid_storage_identifier_dict) > 1 -%}
tidstorage-wrapper = ${directory:services}/tidstoraged
88 89 90 91 92 93

[{{ section("promise-tidstorage") }}]
recipe = slapos.cookbook:check_port_listening
hostname = ${tidstorage:ip}
port = ${tidstorage:port}
path = ${directory:promises}/tidstorage
94
{% endif -%}
95 96 97 98 99 100 101 102

[{{ section("cron-entry-tidstorage-backup") }}]
# TODO:
# - configurable full/incremental
# - configurable retention
recipe = slapos.cookbook:cron.d
cron-entries = ${cron:cron-entries}
name = tidstorage
103
time = {{ dumps(backup_periodicity) }}
104 105
command = ${tidstorage:repozo-wrapper}

106 107 108 109
# Used for ERP5 resiliency or (more probably)
# webrunner resiliency with erp5 inside.
[{{ section("resiliency-exclude-file") }}]
# Generate rdiff exclude file
110 111
recipe = slapos.recipe.template:jinja2
mode = 644
112
template = {{ 'inline:{{ "${directory:zodb}/**\\n${directory:log}\\n" }}' }}
113
rendered = ${directory:srv}/exporter.exclude
114

115 116 117 118 119 120 121 122 123
[{{ section("resiliency-identity-signature-script")}}]
# Generate identity script used by webrunner to check data integrity
# It excludes repozo files as they already include a hash function
# used to check backups when rebuilding the datafs
recipe = slapos.cookbook:wrapper
command-line = {{ bin_directory }}/backup-identity-script-excluding-path --exclude-path "srv/backup/logrotate" --exclude-path "srv/backup/zodb"
wrapper-path = ${directory:srv}/.backup_identity_script
mode = 770

124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155
[{{ section("resiliency-after-import-script") }}]
# Generate after import script used by importer instance of webrunner
recipe = collective.recipe.template
input = inline: #!/bin/sh
  # DO NOT RUN THIS SCRIPT ON PRODUCTION INSTANCE
  # OR ZODB DATA WILL BE ERASED.

  # This script will restore the repozo backup to the real
  # zodb location. It is launched by the clone (importer) instance of webrunner
  # in the end of the import script.

  # Depending on the output, it will create a file containing
  # the status of the restoration (success or failure).
  zodb_directory="${directory:zodb}"
  zodb_backup_directory="{{ default_backup_path }}"
  repozo="${tidstorage:repozo-binary}"
  EXIT_CODE=0
  {% for family, zodb in zodb_dict.iteritems() -%}
  {%   for name, zodb in zodb -%}
  {%   set zeo_section_name = 'zeo-' ~ family %}
  storage_name="{{ name }}"
  zodb_path="$storage_name.fs"
  pid_file={{ "${" ~ zeo_section_name ~ ":pid-path}" }}
  if [ -e "$pid_file" ]; then
    pid=$(cat $pid_file) > /dev/null 2>&1
    if kill -0 "$pid"; then
      echo "Zeo is already running with pid $pid. Aborting."
      exit 1
    fi
  fi
  echo "Removing $zodb_path..."
  echo "Restoring $storage_name into $zodb_path..."
156 157 158 159
  $repozo --verify --repository="$zodb_backup_directory/$storage_name"
  CURRENT_EXIT_CODE=$?
  if [ ! "$CURRENT_EXIT_CODE"="0" ]; then
    echo "$storage_name Backup verification failed. Backup data is inconsistent."
160
    exit "$CURRENT_EXIT_CODE"
161
  fi
162 163 164 165 166 167 168 169 170 171 172 173
  $repozo --recover --output="$zodb_directory/$zodb_path" --repository="$zodb_backup_directory/$storage_name"
  CURRENT_EXIT_CODE=$?
  if [ ! "$CURRENT_EXIT_CODE"="0" ]; then
    EXIT_CODE="$CURRENT_EXIT_CODE"
    echo "$storage_name Backup restoration failed."
  fi
  {%   endfor -%}
  {% endfor -%}
  exit $EXIT_CODE
output = ${directory:srv}/runner-import-restore
mode = 755

174 175
{% endif -%}

176 177 178 179 180
[{{ section('promise-check-computer-memory') }}]
recipe = slapos.cookbook:wrapper
wrapper-path = ${directory:promises}/check-computer-memory
command-line = "{{ parameter_dict["check-computer-memory-binary"] }}" -db ${monitor-instance-parameter:collector-db} --threshold "{{ slapparameter_dict["computer-memory-percent-threshold"] }}" --unit percent

181 182 183 184 185 186 187 188 189 190 191 192
[publish]
recipe = slapos.cookbook:publish.serialised
storage-dict = {{ dumps(storage_dict) }}
{% if len(known_tid_storage_identifier_dict) > 1 -%}
tidstorage-ip = ${tidstorage:ip}
tidstorage-port = ${tidstorage:port}
{% else -%}
tidstorage-ip =
tidstorage-port =
{% endif -%}
monitor-base-url = ${monitor-publish-parameters:monitor-base-url}

193 194
[directory]
recipe = slapos.cookbook:mkdirectory
195
bin = ${buildout:directory}/bin
196 197 198
etc = ${buildout:directory}/etc
services = ${:etc}/run
promises = ${:etc}/promise
199
srv = ${buildout:directory}/srv
200 201 202
var = ${buildout:directory}/var
log = ${:var}/log
run = ${:var}/run
203
backup-zodb = {{ default_backup_path }}
204
zodb = {{ default_zodb_path }}
205
tidstorage = {{ tidstorage_repozo_path }}
206

207 208
[monitor-instance-parameter]
monitor-httpd-ipv6 = {{ (ipv6_set | list)[0] }}
209
monitor-httpd-port = {{ next_port() }}
210
monitor-title = {{ slapparameter_dict['name'] }}
211
password = {{ slapparameter_dict['monitor-passwd'] }}
212

213
[buildout]
214 215 216
extends =
  {{ logrotate_cfg }}
  {{ parameter_dict['template-monitor'] }}
217 218 219
parts +=
  {{ part_list | join('\n  ') }}
  publish