instance-zope.cfg.in 15.9 KB
Newer Older
1
{% set use_ipv6 = slapparameter_dict.get('use-ipv6', False) -%}
2
{% set next_port = itertools.count(slapparameter_dict['port-base']).next -%}
3
{% set site_id = slapparameter_dict['site-id'] -%}
4
{% set zodb_dict = slapparameter_dict['zodb-dict'] -%}
5
{% set instance_index_list = range(slapparameter_dict['instance-count']) -%}
6
{% set node_id_base = slapparameter_dict['name'] -%}
7
{% set node_id_index_format = '-%%0%ii' % (len(str(instance_index_list[-1])), ) -%}
8 9 10 11 12 13 14 15 16 17 18
{% set part_list = [] -%}
{% set publish_list = [] -%}
{% set longrequest_logger_base_path = buildout_directory ~ '/var/log/longrequest_logger_' -%}
{% macro section(name) %}{% do part_list.append(name) %}{{ name }}{% endmacro -%}
{% set bin_directory = parameter_dict['buildout-bin-directory'] -%}
{#
XXX: This template only supports exactly one IPv4 and one IPv6 per
partition. No more (undefined result), no less (IndexError).
-#}
{% set ipv4 = (ipv4_set | list)[0] -%}

19 20 21 22
[jinja2-template-base]
recipe = slapos.recipe.template:jinja2
mode = 644

23 24
{% if slapparameter_dict['mysql-test-url-list'] -%}
[{{ section('test-runner') }}]
25 26 27
recipe = slapos.cookbook:erp5.test
certificate-authority-path = ${test-certificate-authority:ca-dir}
mysql-url-list = {{ dumps(slapparameter_dict['mysql-test-url-list']) }}
28 29 30
kumofs-url = {{ dumps(slapparameter_dict['kumofs-url']) }}
memcached-url = {{ dumps(slapparameter_dict['memcached-url']) }}
cloudooo-url = {{ dumps(slapparameter_dict['cloudooo-url']) }}
31 32
test-instance-path = ${directory:unit-test-path}
prepend-path = ${buildout:bin-directory}
33 34
run-unit-test = ${buildout:bin-directory}/runUnitTest.real
run-test-suite = ${buildout:bin-directory}/runTestSuite.real
35 36 37 38
openssl-binary = ${test-certificate-authority:openssl-binary}
run-unit-test-binary = {{ parameter_dict['bin-directory'] }}/runUnitTest
run-test-suite-binary = {{ parameter_dict['bin-directory'] }}/runTestSuite

39 40 41 42 43 44 45 46 47 48
[{{ section('run-unit-test-userhosts-wrapper') }}]
<= userhosts-wrapper-base
wrapped-command-line = ${test-runner:run-unit-test}
wrapper-path = ${buildout:bin-directory}/runUnitTest

[{{ section('run-test-suite-userhosts-wrapper') }}]
<= userhosts-wrapper-base
wrapped-command-line = ${test-runner:run-test-suite}
wrapper-path = ${buildout:bin-directory}/runTestSuite

49 50
[test-certificate-authority]
recipe = slapos.cookbook:certificate_authority
51
openssl-binary = ${binary-link:target-directory}/openssl
52
ca-dir = ${directory:test-ca-dir}
53
requests-directory = ${directory:test-ca-requests}
54
wrapper = ${directory:services}/test-ca
55 56 57 58
ca-private = ${directory:test-ca-private}
ca-certs = ${directory:test-ca-certs}
ca-newcerts = ${directory:test-ca-newcerts}
ca-crl = ${directory:test-ca-crl}
59
{%- endif %}
60

61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78
[directory]
recipe = slapos.cookbook:mkdirectory
bin = ${buildout:directory}/bin
etc = ${buildout:directory}/etc
instance = ${:srv}/erp5shared
instance-constraint = ${:instance}/Constraint
instance-document = ${:instance}/Document
instance-etc = ${:instance}/etc
instance-etc-package-include = ${:instance}/etc/package-include
instance-extensions = ${:instance}/Extensions
instance-import = ${:instance}/import
instance-lib = ${:instance}/lib
instance-products = ${:instance}/Products
instance-propertysheet = ${:instance}/PropertySheet
instance-tests = ${:instance}/tests
log = ${:var}/log
run = ${:var}/run
services = ${:etc}/run
79
service-on-watch = ${:etc}/service
80 81 82 83
srv = ${buildout:directory}/srv
tmp = ${buildout:directory}/tmp
var = ${buildout:directory}/var
promises = ${:etc}/promise
84 85
unit-test-path = ${:srv}/test-instance/unit_test
test-ca-dir = ${:srv}/test-ca
86 87 88 89 90 91 92 93 94 95 96
test-ca-requests = ${:test-ca-dir}/requests
test-ca-private = ${:test-ca-dir}/private
test-ca-certs = ${:test-ca-dir}/certs
test-ca-newcerts = ${:test-ca-dir}/newcerts
test-ca-crl = ${:test-ca-dir}/crl
ca-dir = ${:srv}/ca
ca-requests = ${:ca-dir}/requests
ca-private = ${:ca-dir}/private
ca-certs = ${:ca-dir}/certs
ca-newcerts = ${:ca-dir}/newcerts
ca-crl = ${:ca-dir}/crl
97 98 99 100

[binary-link]
recipe = slapos.cookbook:symbolic.link
target-directory = ${directory:bin}
101
link-binary = {{ dumps(parameter_dict['link-binary']) }}
102 103

[certificate-authority-common]
104
requests-directory = ${directory:ca-requests}
105
ca-dir = ${directory:ca-dir}
106 107 108 109
ca-private = ${directory:ca-private}
ca-certs = ${directory:ca-certs}
ca-newcerts = ${directory:ca-newcerts}
ca-crl = ${directory:ca-crl}
110

111
[{{ section('certificate-authority') }}]
112 113
< = certificate-authority-common
recipe = slapos.cookbook:certificate_authority
114
openssl-binary = ${binary-link:target-directory}/openssl
115 116 117 118 119 120 121 122 123 124 125 126 127 128
wrapper = ${directory:services}/ca

{% if use_ipv6 -%}
{%   set ipv6 = (ipv6_set | list)[0] -%}

[ipv6toipv4-base]
recipe = slapos.cookbook:ipv6toipv4
runner-path = ${directory:services}/${:base-name}
6tunnel-path = {{ parameter_dict['6tunnel'] }}/bin/6tunnel
shell-path = {{ parameter_dict['dash'] }}/bin/dash
ipv4 = {{ ipv4 }}
ipv6 = {{ ipv6 }}
{% endif -%}

129 130 131 132 133
{% set hosts_dict = {} -%}
{% for alias, url in (
    ('erp5-memcached-volatile', slapparameter_dict['memcached-url']),
    ('erp5-memcached-persistent', slapparameter_dict['kumofs-url']),
    ('erp5-cloudooo', slapparameter_dict['cloudooo-url']),
134
    ('erp5-smtp', slapparameter_dict['smtp-url']),
135 136 137 138 139 140
  ) -%}
{%   do hosts_dict.__setitem__(
       alias,
       urlparse.urlparse(url).hostname,
     )  -%}
{%- endfor %}
141
{% for i, url in enumerate(slapparameter_dict['mysql-url-list']) -%}
142
{%   do hosts_dict.__setitem__(
143
       'erp5-catalog-' ~ i,
144 145 146 147
       urlparse.urlparse(url).hostname,
     ) -%}
{%- endfor %}
{% do hosts_dict.update(slapparameter_dict['hosts-dict']) -%}
148 149
[host-common]

150
[hosts-parameter]
151
# Used for both hosts and hostaliases sections.
152
host-dict = {{ dumps(hosts_dict) }}
153 154 155 156 157 158 159 160 161
hostalias-dict = {{ dumps(slapparameter_dict['hostalias-dict']) }}

# Note: there is a subtle difference between hosts and hostaliases files:
# - hosts files start with resolved, followed by alias(es) (only one alias per
#   line in this case)
# - hostaliases start with alias, followed by resolved
# ...so it's not possible to merge these templates (not a big deal anyway).

[hostaliases]
162
< = jinja2-template-base
163 164 165 166 167 168 169
template = inline: {{ '
  {% for alias, aliased in host_dict.items() -%}
  {{ alias }} {{ aliased }}
  {% endfor %}
' }}
rendered = ${directory:etc}/hostaliases
context = key host_dict hosts-parameter:hostalias-dict
170 171

[hosts]
172
< = jinja2-template-base
173 174 175 176 177 178 179 180
template = inline: {{ '
  {% for alias, aliased in host_dict.items() -%}
  {{ aliased }} {{ alias }}
  {% endfor %}
' }}
rendered = ${directory:etc}/hosts
context = key host_dict hosts-parameter:host-dict

181 182 183 184 185 186 187 188
[userhosts-wrapper-base]
recipe = slapos.cookbook:wrapper
environment =
  HOSTALIASES=${hostaliases:rendered}
  HOSTS=${hosts:rendered}
parameters-extra = true
command-line = '{{ parameter_dict['userhosts'] }}' ${:wrapped-command-line}

189 190 191 192 193 194 195 196 197
{# Hack to deploy SSL certs via instance parameters -#}
{% for zodb in zodb_dict.itervalues() -%}
{%   set storage_dict = zodb.setdefault('storage-dict', {}) -%}
{%   if zodb['type'] == 'neo' and storage_dict.get('ssl', 1) -%}
{%     for k, v in (('_ca', 'ca.crt'),
                    ('_cert', 'neo.crt'),
                    ('_key', 'neo.key')) -%}
{%       if k in storage_dict -%}
[{{ section('neo-ssl-' + k[1:]) }}]
198
< = jinja2-template-base
199 200 201 202 203 204 205 206 207 208 209
rendered = ${directory:etc}/{{v}}
template = inline:{{'{{'}}pem}}
context = key pem :pem
pem = {{dumps(storage_dict.pop(k))}}

{%       endif -%}
{%     endfor -%}
{%   endif -%}
{% endfor -%}
{# endhack -#}

210
[runzope-base]
211 212
<= userhosts-wrapper-base
environment +=
213 214 215 216 217 218
  TMP=${directory:tmp}
  TMPDIR=${directory:tmp}
  HOME=${buildout:directory}
  PATH=${directory:bin}:{{ parameter_dict['coreutils'] }}/bin
  TZ={{ slapparameter_dict['timezone'] }}
  INSTANCE_HOME=${directory:instance}
219
  MATPLOTLIBRC={{ parameter_dict['matplotlibrc'] }}
220 221 222
{% if slapparameter_dict.get('wendelin-core-zblk-fmt') %}
  WENDELIN_CORE_ZBLK_FMT={{ slapparameter_dict['wendelin-core-zblk-fmt'] }}
{% endif %}
223
wrapped-command-line = '{{ bin_directory }}/runzope' -C '${:configuration-file}'
224
private-dev-shm = {{ slapparameter_dict['private-dev-shm'] }}
225 226

[{{ section('zcml') }}]
227
recipe = slapos.cookbook:copyfilelist
228
target-directory = ${directory:instance-etc}
229
file-list = {{ parameter_dict['site-zcml'] }}
230 231

[{{ section('zope-inituser') }}]
232
< = jinja2-template-base
233 234 235 236
rendered = ${directory:instance}/inituser
template = inline:{{ slapparameter_dict['inituser-login'] }}:{SHA}{{ hashlib.sha1(slapparameter_dict['inituser-password']).digest().encode('base64').rstrip() }}
mode = 600
once = ${:rendered}_done
237 238 239 240

[zope-conf-parameter-base]
ip = {{ ipv4 }}
site-id = {{ site_id }}
241 242 243 244 245 246 247 248
{% if site_id -%}
mysql-url = {{ slapparameter_dict['mysql-url-list'][0] }}
inituser = {{ slapparameter_dict['inituser-login'] }}
{%  set mysql = urlparse.urlsplit(slapparameter_dict['mysql-url-list'][0]) -%}
sql-connection-string = {{ '%s@erp5-catalog-0:%s %s %s' % (
    mysql.path.split('/')[1], mysql.port, mysql.username, mysql.password) }}
bt5 = {{ slapparameter_dict['bt5'] }}
bt5-repository-url = {{ slapparameter_dict['bt5-repository-url'] }}
249
id-store-interval = {{ dumps(slapparameter_dict['id-store-interval']) }}
250 251 252 253 254 255 256 257 258
home = ${buildout:directory}
# We only want to change the hostname to 'erp5-cloudooo' if we use the internal
# cloudooo. We plan to remove the ability to have an internal one, so this
# heuristic is enough.
{%  set cloudooo = urlparse.urlsplit(slapparameter_dict['cloudooo-url']) -%}
cloudooo-url = {{ (cloudooo if cloudooo.port == None else
  cloudooo._replace(netloc='erp5-cloudooo:%s' % cloudooo.port)).geturl() }}

{% endif -%}
259
{% set zeo_dict = slapparameter_dict.get('zodb-zeo', {}) -%}
260
{% for name, zodb in zodb_dict.iteritems() -%}
261 262 263 264 265 266 267 268 269 270 271 272 273 274 275
{%   set storage_dict = zodb.setdefault('storage-dict', {}) -%}
{%   if zodb['type'] == 'zeo' -%}
{%     do storage_dict.update(zeo_dict.get(name, ())) -%}
{%   else -%}
{%     if name == slapparameter_dict.get('neo-name') -%}
{%       do storage_dict.update(master_nodes=slapparameter_dict['neo-masters'],
                                name=slapparameter_dict['neo-cluster']) -%}
{%     endif -%}
{{     assert(storage_dict['master_nodes'], name) }}
{%     if storage_dict.pop('ssl', 1) -%}
{%       do storage_dict.update(ca='~/etc/ca.crt',
                                cert='~/etc/neo.crt',
                                key='~/etc/neo.key') -%}
{%     endif -%}
{%   endif -%}
276
{% endfor -%}
277
developer-list = {{ dumps(slapparameter_dict['developer-list']) }}
278 279 280 281
instance = ${directory:instance}
instance-products = ${directory:instance-products}
deadlock-path = /manage_debug_threads
deadlock-debugger-password = {{ dumps(slapparameter_dict['deadlock-debugger-password']) }}
282 283 284 285
{% if slapparameter_dict.get('tidstorage-ip') -%}
tidstorage-ip = {{ dumps(slapparameter_dict['tidstorage-ip']) }}
tidstorage-port = {{ dumps(slapparameter_dict['tidstorage-port']) }}
{% endif -%}
286
{% set thread_amount = slapparameter_dict['thread-amount'] -%}
287
{% set large_file_threshold = slapparameter_dict['large-file-threshold']  -%}
288
thread-amount = {{ thread_amount }}
289 290 291 292 293 294 295 296
{% set webdav = slapparameter_dict['webdav'] -%}
webdav = {{ dumps(webdav) }}
{% if webdav -%}
{%   set timerserver_interval = 0 -%}
{% else -%}
{%   set timerserver_interval = slapparameter_dict['timerserver-interval'] -%}
{%- endif %}
timerserver-interval = {{ dumps(timerserver_interval) }}
297 298

[zope-conf-base]
299
< = jinja2-template-base
300 301 302
template = {{ parameter_dict['zope-conf-template'] }}

{% macro zope(
303
  index,
304 305 306 307
  port,
  longrequest_logger_timeout,
  longrequest_logger_interval
) -%}
308
{% set name = 'zope-' ~ index -%}
309 310 311 312 313 314 315 316 317 318 319
{% set conf_name = name ~ '-conf' -%}
{% set conf_parameter_name = conf_name ~ '-param' -%}
{% set zope_tunnel_section_name = name ~ '-ipv6toipv4' -%}
{% set zope_tunnel_base_name = zope_tunnel_section_name -%}
[{{ conf_parameter_name }}]
< = zope-conf-parameter-base
pid-file = ${directory:run}/{{ name }}.pid
lock-file = ${directory:run}/{{ name }}.lock
port = {{ port }}
event-log = ${directory:log}/{{ name }}-event.log
z2-log = ${directory:log}/{{ name }}-Z2.log
320
node-id = {{ dumps(node_id_base ~ (node_id_index_format % index)) }}
321
{% set log_list = [] -%}
322 323
{% set import_set = set() -%}
{% for db_name, zodb in zodb_dict.iteritems() -%}
324
{%   do zodb.setdefault('pool-size', thread_amount) -%}
325 326
{%   if zodb['type'] == 'neo' -%}
{%     do import_set.add('neo.client') -%}
327 328 329
{%     set log = name ~ '-neo-' ~ db_name ~ '.log' -%}
{%     do log_list.append('${directory:log}/' + log) -%}
{%     do zodb['storage-dict'].update(logfile='~/var/log/'+log) -%}
330 331
{%   endif -%}
{% endfor -%}
332 333
import-list = {{ dumps(list(import_set)) }}
zodb-dict = {{ dumps(zodb_dict) }}
334
large-file-threshold = {{ large_file_threshold }} 
335 336 337 338 339 340 341
{% if longrequest_logger_interval > 0 -%}
longrequest-logger-file = {{ longrequest_logger_base_path ~ name ~ ".log" }}
longrequest-logger-timeout = {{ longrequest_logger_timeout }}
longrequest-logger-interval = {{ longrequest_logger_interval }}
{% else -%}
longrequest-logger-file =
{% endif -%}
342 343 344 345

[{{ conf_name }}]
< = zope-conf-base
rendered = ${directory:etc}/{{ name }}.conf
346
extensions = jinja2.ext.do
347
context =
348
  section parameter_dict {{ conf_parameter_name }}
349
  import os os
350

351
[{{ section(name) }}]
352 353
< = runzope-base
wrapper-path = ${directory:service-on-watch}/{{ name }}
354 355 356 357
configuration-file = {{ '${' ~ conf_name ~ ':rendered}' }}

[{{ section("promise-" ~ name) }}]
recipe = slapos.cookbook:check_port_listening
358 359
hostname = {{ ipv4 }}
port = {{ port }}
360 361
path = ${directory:promises}/{{ name }}

362 363 364 365 366 367 368 369 370
{% set extra_path_list = [] -%}
{% set shell_escaped_extra_path_list = [] -%}
{% for line in parameter_dict['extra-path-list'].splitlines() -%}
{%   set line = line.strip() -%}
{%   do extra_path_list.append(line) -%}
{%   do shell_escaped_extra_path_list.append(line.replace("\x27", "\x27\\\x27\x27")) -%}
{% endfor -%}
[{{ section("promise-" ~ name ~ "-is-running-actual-product") }}]
recipe = slapos.cookbook:wrapper
371
command-line = '{{ parameter_dict['bin-directory'] }}/is-process-older-than-dependency-set' '{{ "${" ~ conf_parameter_name ~ ":pid-file}" }}' {{ " ".join(shell_escaped_extra_path_list) }}
372 373 374
wrapper-path = ${directory:promises}/{{ name }}-is-running-actual-product
parameters-extra = true

375 376 377 378 379 380
{% if use_ipv6 -%}
[{{ zope_tunnel_section_name }}]
< = ipv6toipv4-base
base-name = {{ zope_tunnel_base_name }}
ipv6-port = {{ port }}
ipv4-port = {{ port }}
381
{%   do publish_list.append(("[" ~ ipv6 ~ "]:" ~ port, thread_amount, webdav)) -%}
382 383 384 385 386 387 388

[{{ section("promise-tunnel-" ~ name) }}]
recipe = slapos.cookbook:check_port_listening
hostname = {{ '${' ~ zope_tunnel_section_name ~ ':ipv6}' }}
port = {{ '${' ~ zope_tunnel_section_name ~ ':ipv6-port}' }}
path = ${directory:promises}/{{ zope_tunnel_base_name }}
{% else -%}
389
{%   do publish_list.append((ipv4 ~ ":" ~ port, thread_amount, webdav)) -%}
390 391 392 393 394
{% endif -%}

[{{ section('logrotate-entry-' ~ name) }}]
< = logrotate-entry-base
name = {{ name }}
395
log = {{ '${' ~ conf_parameter_name ~ ':event-log}' }} {{ '${' ~ conf_parameter_name ~ ':z2-log}' }} {{ '${' ~ conf_parameter_name ~ ':longrequest-logger-file}' }} {{ ' '.join(log_list) }}
396
post = test ! -s {{ '${' ~ conf_parameter_name ~ ':pid-file}' }} || {{ bin_directory }}/slapos-kill --pidfile {{ '${' ~ conf_parameter_name ~ ':pid-file}' }} -s USR2
397 398
{% endmacro -%}

399
{% for i in instance_index_list -%}
400 401
{{   zope(
       i,
402
       next_port(),
403 404 405
       slapparameter_dict['longrequest-logger-timeout'],
       slapparameter_dict['longrequest-logger-interval'],
     ) }}
406 407
{% endfor -%}

408
[publish]
409 410
recipe = slapos.cookbook:publish.serialised
zope-address-list = {{ dumps(publish_list) }}
411 412 413 414 415 416 417 418
{#
Note: hosts_dist is generated at zope level rather than at erp5 (root partition)
level, as it is easier: we can access urls as python values trivially here.
This has the downside of making each zope partition publish the (hopefuly) same
dict toward erp5 partition, violating the DRY principle and making the intent
hard to guess.
-#}
hosts-dict = {{ dumps(hosts_dict) }}
419
monitor-base-url = ${monitor-publish-parameters:monitor-base-url}
420

421 422
[monitor-instance-parameter]
monitor-httpd-ipv6 = {{ (ipv6_set | list)[0] }}
423
monitor-httpd-port = {{ next_port() }}
424
monitor-title = {{ slapparameter_dict['name'] }}
425
password = {{ slapparameter_dict['monitor-passwd'] }}
426

427
[buildout]
428 429 430
extends =
  {{ logrotate_cfg }}
  {{ parameter_dict['template-monitor'] }}
431 432
parts +=
  {{ part_list | join('\n  ') }}
433
  publish