instance-mariadb.cfg.in 14.7 KB
Newer Older
1 2 3 4
{% set part_list = [] -%}
{% macro section(name) %}{% do part_list.append(name) %}{{ name }}{% endmacro -%}
{% set use_ipv6 = slapparameter_dict.get('use-ipv6', False) -%}
{% set database_list = slapparameter_dict.get('database-list', [{'name': 'erp5', 'user': 'user', 'password': 'insecure'}]) -%}
5
{% set test_database_list = [] %}
6
{% for database_count in range(slapparameter_dict.get('test-database-amount', 1)) -%}
7
{%   do test_database_list.append({'name': 'erp5_test_' ~ database_count, 'user': 'testuser_' ~ database_count, 'password': 'testpassword' ~ database_count}) -%}
8 9
{% endfor -%}
{% set catalog_backup = slapparameter_dict.get('catalog-backup', {}) -%}
10
{% set backup_periodicity = slapparameter_dict.get('backup-periodicity', 'daily') -%}
11 12
{% set full_backup_retention_days = catalog_backup.get('full-retention-days', 7) -%}
{% set incremental_backup_retention_days = catalog_backup.get('incremental-retention-days', full_backup_retention_days) -%}
13
{% set port = slapparameter_dict['tcpv4-port'] %}
14
{% if use_ipv6 -%}
15
{%   set ip = (ipv6_set | list)[0] -%}
16
{% else -%}
17
{%   set ip = (ipv4_set | list)[0] -%}
18
{% endif -%}
19
{% set dash = parameter_dict['dash-location'] ~ '/bin/dash' %}
20

21
[{{ section('publish') }}]
22
recipe = slapos.cookbook:publish.serialised
23
-extends = publish-early
24 25 26
{% macro render_database_list(database_list) -%}
{% set publish_database_list = [] -%}
{% for database in database_list -%}
27 28
{%   if database.get('user') -%}
{%     do publish_database_list.append("mysql://" ~ database['user'] ~ ":" ~ database['password'] ~ "@" ~ ip ~ ":" ~ port ~ "/" ~ database['name']) -%}
29
{%   else -%}
30
{%     do publish_database_list.append("mysql://" ~ ip ~ ":" ~ port ~ "/" ~ database['name']) -%}
31
{%   endif -%}
32 33 34 35 36
{% endfor -%}
{{ dumps(publish_database_list) }}
{% endmacro -%}
database-list = {{ render_database_list(database_list) }}
test-database-list = {{ render_database_list(test_database_list) }}
37
monitor-base-url = ${monitor-publish-parameters:monitor-base-url}
38

39 40 41 42 43 44 45 46 47 48 49 50 51 52
[publish-early]
recipe = slapos.cookbook:publish-early
-init =
  server-id gen-server-id:value
{%- set server_id = slapparameter_dict.get('server-id') %}
{%- if server_id %}
server-id = {{ dumps(server_id) }}
{%- endif %}

[gen-server-id]
recipe = slapos.cookbook:random.integer
minimum = {{ dumps(1) }}
maximum = {{ dumps(2**32 - 1) }}

53
[jinja2-template-base]
54
recipe = slapos.recipe.template:jinja2
55 56 57 58 59 60 61 62
mode = 644

[jinja2-template-executable]
< = jinja2-template-base
mode = 755

[simplefile]
< = jinja2-template-base
63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92
template = inline:{{ '{{ content }}' }}

{% macro simplefile(section_name, file_path, content, mode='') -%}
{%   set content_section_name = section_name ~ '-content' -%}
[{{  content_section_name }}]
content = {{ dumps(content) }}

[{{  section(section_name) }}]
< = simplefile
rendered = {{ file_path }}
context = key content {{content_section_name}}:content
mode = {{ mode }}
{%- endmacro %}
{% set ssl_dict = {} -%}
{% macro sslfile(key, content, mode='644') -%}
{%   set path = '${directory:mariadb-ssl}/' ~ key ~ '.pem' -%}
{%   do ssl_dict.__setitem__(key, path) -%}
{{   simplefile('ssl-file-' ~ key, path, content, mode) }}
{%- endmacro %}
{% set ssl_parameter_dict = slapparameter_dict.get('ssl') -%}
{% if ssl_parameter_dict -%}
{%   set base_directory = '${directory:mariadb-ssl}/' -%}
{# Note: The key content will be stored in .installed.cfg, and this template's
rendering, so the only point of mode is to avoid risking mariadb complaining
about laxist file mode. -#}
{{   sslfile('key', ssl_parameter_dict['key'], mode='600') }}
{{   sslfile('crt', ssl_parameter_dict['crt']) }}
{%   if 'ca-crt' in ssl_parameter_dict -%}
{{     sslfile('ca-crt', ssl_parameter_dict['ca-crt']) }}
{%   endif -%}
93 94 95
{%   if 'crl' in ssl_parameter_dict -%}
{{     sslfile('crl', ssl_parameter_dict['crl']) }}
{%   endif -%}
96 97
{%- endif %}

98 99
{% if full_backup_retention_days > -1 -%}
[{{ section('cron-entry-mariadb-backup') }}]
100
recipe = slapos.cookbook:cron.d
101
cron-entries = ${cron:cron-entries}
102
name = mariadb-backup
103
time = {{ dumps(backup_periodicity) }}
104 105 106 107 108 109
{# When binlogs are enabled:
# flush-logs: used so no manipulation on binlogs is needed to restore from
#   full + binlogs. The first binlog after a dump starts from dump snapshot and
#   can be fully restored.
# master-data: use value "2" as we are not in a replication case
#}
110
command = "${binary-wrap-mysqldump:wrapper-path}" -u root --all-databases --flush-privileges --single-transaction --max-allowed-packet=128M {% if incremental_backup_retention_days > -1 %}--flush-logs --master-data=2 {% endif %}| {{ parameter_dict['gzip-location'] }}/bin/gzip > "${directory:mariadb-backup-full}/$({{ parameter_dict['coreutils-location'] }}/bin/date "+%Y%m%d%H%M%S").sql.gz"
111 112 113 114 115 116 117
{# KEEP GLOB PATTERN IN SYNC with generated filenames above
#           YYYYmmddHHMMSS -#}
file-glob = ??????????????.sql.gz

{% if full_backup_retention_days > 0 -%}
[{{ section("cron-entry-mariadb-backup-expire") }}]
recipe = slapos.cookbook:cron.d
118
cron-entries = ${cron:cron-entries}
119
name = mariadb-backup-expire
120
time = {{ dumps(backup_periodicity) }}
121 122 123 124 125
command = {{ parameter_dict['findutils-location'] }}/bin/find "${directory:mariadb-backup-full}" -maxdepth 1 -name "${cron-entry-mariadb-backup:file-glob}" -daystart -mtime +{{ full_backup_retention_days }} -delete
{%- endif %}
{%- endif %}

[my-cnf-parameters]
126 127
ip = {{ ip }}
port = {{ port }}
128 129
socket = ${directory:run}/mariadb.sock
data-directory = ${directory:mariadb-data}
130
tmp-directory = ${directory:tmp}
131
etc-directory = ${directory:etc}
132 133 134
pid-file = ${directory:run}/mariadb.pid
error-log = ${directory:log}/mariadb_error.log
slow-query-log = ${directory:log}/mariadb_slowquery.log
135
long-query-time = {{ dumps(slapparameter_dict.get('long-query-time', 1)) }}
136
innodb-buffer-pool-size = {{ dumps(slapparameter_dict.get('innodb-buffer-pool-size', 0)) }}
137
innodb-buffer-pool-instances = {{ dumps(slapparameter_dict.get('innodb-buffer-pool-instances', 0)) }}
138
innodb-log-file-size = {{ dumps(slapparameter_dict.get('innodb-log-file-size', 0)) }}
139
innodb-file-per-table = {{ dumps(slapparameter_dict.get('innodb-file-per-table', 0)) }}
140
innodb-log-buffer-size = {{ dumps(slapparameter_dict.get('innodb-log-buffer-size', 0)) }}
141
relaxed-writes = {{ dumps(slapparameter_dict.get('relaxed-writes', False)) }}
142 143 144
{% if incremental_backup_retention_days > -1 -%}
binlog-path = ${directory:mariadb-backup-incremental}/binlog
# XXX: binlog rotation happens along with other log's rotation
145
binlog-expire-days = {{ dumps(incremental_backup_retention_days) }}
146
server-id = ${publish-early:server-id}
147 148
{% else %}
binlog-path =
149 150 151 152
{%- endif %}
{%- for key, value in ssl_dict.items() -%}
ssl-{{ key }} = {{ value }}
{% endfor %}
153 154

[my-cnf]
155
< = jinja2-template-base
156 157 158 159 160
rendered = ${directory:etc}/mariadb.cnf
template = {{ parameter_dict['template-my-cnf'] }}
context = section parameter_dict my-cnf-parameters

[init-script-parameters]
161
database-list = {{ dumps(database_list + test_database_list) }}
162 163

[init-script]
164
< = jinja2-template-executable
165 166 167 168 169
# XXX: is there a better location ?
rendered = ${directory:etc}/mariadb_initial_setup.sql
template = {{ parameter_dict['template-mariadb-initial-setup'] }}
context = section parameter_dict init-script-parameters

170
[{{ section('update-mysql') }}]
171 172 173 174 175
recipe = slapos.cookbook:generic.mysql.wrap_update_mysql
output = ${directory:services}/mariadb_update
binary = ${binary-wrap-mysql_upgrade:wrapper-path}
mysql = ${binary-wrap-mysql:wrapper-path}
init-script = ${init-script:rendered}
176
mysql_tzinfo_to_sql = ${binary-wrap-mysql_tzinfo_to_sql:wrapper-path}
177

178 179 180 181 182 183
[{{ section('mysqld') }}]
< = jinja2-template-executable
# Note: all rendering is done when this file is rendered, not when the mysqld
# section is installed - so I only use jinja2 as a fancy way to write an
# executable file with partition-dependent but instance-parameters independent
# content.
184
template = inline:#!{{ dash }}
185 186 187 188 189
  '{{ parameter_dict['mariadb-location'] }}/scripts/mysql_install_db' \
    --defaults-file='${my-cnf:rendered}' \
    --skip-name-resolve \
    --datadir='${my-cnf-parameters:data-directory}' \
    --basedir='{{ parameter_dict['mariadb-location'] }}' \
190 191 192 193 194 195
    && ODBCSYSINI='${my-cnf-parameters:etc-directory}' \
    LD_LIBRARY_PATH=$LD_LIBRARY_PATH:{{ parameter_dict['unixodbc-location'] }}/lib \
    {% for variable in slapparameter_dict.get('environment-variables', ()) %}
    {{ variable }} \
    {% endfor %}
    exec '{{ parameter_dict['mariadb-location'] }}/bin/mysqld' \
196 197 198
    --defaults-file='${my-cnf:rendered}' \
    "$@"
rendered = ${directory:services}/mariadb
199

200 201 202 203 204 205 206 207 208 209 210
[odbc-ini-text]
text = {{ dumps(slapparameter_dict.get('odbc-ini', '').encode('base64')) }}

[{{ section('odbc-ini') }}]
< = jinja2-template-base
rendered = ${directory:etc}/odbc.ini
template = inline:{% raw -%}
  {{ parameter_dict['text'].decode('base64') }}
  {%- endraw %}
context = section parameter_dict odbc-ini-text

211
[{{ section('logrotate-entry-mariadb') }}]
212
< = logrotate-entry-base
213
name = mariadb
214
log = ${my-cnf-parameters:error-log} ${my-cnf-parameters:slow-query-log}
215
post = "${binary-wrap-mysql:wrapper-path}" -B -u root -e "FLUSH LOGS"
216

217
[{{ section('binary-link') }}]
218
recipe = slapos.cookbook:symbolic.link
219
target-directory = ${directory:bin}
220
link-binary = {{ dumps(parameter_dict['link-binary']) }}
221

222 223 224 225
[{{ section("binary-link-mysqlbinlog") }}]
< = binary-link
link-binary = {{ parameter_dict['mariadb-location'] }}/bin/mysqlbinlog

226 227 228 229 230 231 232 233 234 235 236 237 238 239 240
[binary-wrap-base]
recipe = slapos.cookbook:wrapper
# Note: --defaults-file must be the first argument, otherwise wrapped binary
# will reject it.
command-line = "{{ parameter_dict['mariadb-location'] }}/bin/${:command}" --defaults-file="${my-cnf:rendered}"
wrapper-path = ${directory:bin}/${:command}

[binary-wrap-mysql]
< = binary-wrap-base
command = mysql

[binary-wrap-mysqldump]
< = binary-wrap-base
command = mysqldump

241
[binary-wrap-mysql_upgrade]
242
< = binary-wrap-base
243 244 245 246
command = mysql_upgrade

[binary-wrap-mysql_tzinfo_to_sql]
< = binary-wrap-base
247
command-line = "{{ parameter_dict['mariadb-location'] }}/bin/${:command}"
248
command = mysql_tzinfo_to_sql
249

250 251 252 253 254 255
[binary-wrap-pt-digest]
<= binary-wrap-base
command-line = "{{ parameter_dict['percona-tools-location'] }}/bin/${:command}"
command = pt-query-digest
 

256 257
[directory]
recipe = slapos.cookbook:mkdirectory
258 259 260 261 262
bin = ${buildout:directory}/bin
etc = ${buildout:directory}/etc
services = ${:etc}/run
promise = ${:etc}/promise
srv = ${buildout:directory}/srv
263
tmp = ${buildout:directory}/tmp
264 265 266 267
backup = ${:srv}/backup
mariadb-backup-full = ${:backup}/mariadb-full
mariadb-backup-incremental = ${:backup}/mariadb-incremental
mariadb-data = ${:srv}/mariadb
268
mariadb-ssl = ${:etc}/mariadb-ssl
269 270 271
var = ${buildout:directory}/var
log = ${:var}/log
run = ${:var}/run
272
slowquery = ${monitor-directory:private}/slowquery_digest
273

274
[{{ section('resiliency-exclude-file') }}]
275
# Generate rdiff exclude file in case of resiliency
276
< = jinja2-template-base
277
template = {{ 'inline:{{ "${directory:mariadb-data}/**\\n${directory:mariadb-backup-incremental}/**\\n${directory:log}/**\\n${directory:tmp}/**\\n" }}' }}
278
rendered = ${directory:srv}/exporter.exclude
279

280 281 282
[{{ section("resiliency-identity-signature-script")}}]
# Generate identity script used by webrunner to check data integrity
recipe = slapos.cookbook:wrapper
283
command-line = {{ bin_directory }}/backup-identity-script-excluding-path --exclude-path "srv/backup/logrotate/**"
284 285 286
wrapper-path = ${directory:srv}/.backup_identity_script
mode = 770

287 288 289
[dash]
dash = {{ dumps(dash) }}

290 291 292 293 294 295 296 297 298 299 300 301 302
[{{ section('start-clone-from-backup') }}]
< = jinja2-template-executable
template = {{ parameter_dict['mariadb-start-clone-from-backup'] }}
rendered = ${directory:bin}/start-clone-from-backup
context =
  key dash dash:dash
  key client binary-wrap-mysql:wrapper-path
  key data_directory directory:mariadb-data
  key pid_file my-cnf-parameters:pid-file
  key server mysqld:rendered
  key update update-mysql:output
  key socket my-cnf-parameters:socket

303
[{{ section('resiliency-after-import-script') }}]
304
# Generate after import script used by importer instance of webrunner
305 306
< = jinja2-template-executable
template = {{ parameter_dict['mariadb-resiliency-after-import-script'] }}
307
rendered = ${directory:bin}/restore-from-backup
308 309 310 311 312 313 314 315 316
context =
  key dash dash:dash
  key mysql_executable binary-wrap-mysql:wrapper-path
  key mariadb_data_directory directory:mariadb-data
  key mariadb_backup_directory directory:mariadb-backup-full
  key pid_file my-cnf-parameters:pid-file
  key binlog_path my-cnf-parameters:binlog-path
  key server_executable mysqld:rendered
  key update_executable update-mysql:output
317

318
[{{ section('monitor-generate-mariadb-slow-query-report') }}]
319 320 321 322 323 324 325 326 327
recipe = slapos.cookbook:cron.d
cron-entries = ${cron:cron-entries}
name = generate-mariadb-slow-query-report
# The goal is to be executed before logrotate log rotation.
# Here, logrotate-entry-base:frequency = daily, so we run at 23 o'clock every day.
frequency = 0 23 * * *
command = ${monitor-generate-mariadb-slow-query-report-wrapper:rendered}

[monitor-generate-mariadb-slow-query-report-wrapper]
328 329
recipe = slapos.recipe.template:jinja2
template = {{ parameter_dict['mariadb-slow-query-report-script'] }}
330 331
rendered = ${directory:bin}/${:filename}
filename = generate-mariadb-slow-query-report
332 333 334 335 336
mode = 755
context = 
  raw slow_query_path ${directory:srv}/backup/logrotate/mariadb_slowquery.log
  raw pt_query_exec ${binary-wrap-pt-digest:wrapper-path}
  raw dash {{ parameter_dict['dash-location'] }}/bin/dash
337
  key output_folder directory:slowquery
338

339 340 341
[slow-query-digest-parameters]
max_queries_threshold = {{ slapparameter_dict['max-slowqueries-threshold'] }}
slowest_queries_threshold = {{ slapparameter_dict['slowest-query-threshold'] }}
342 343

[{{ section('monitor-promise-slowquery-result') }}]
344 345
recipe = slapos.cookbook:wrapper
wrapper-path = ${directory:promise}/check-slow-query-pt-digest-result
346
command-line = "{{ parameter_dict['promise-check-slow-queries-digest-result'] }}" --ptdigest_path "${directory:slowquery}" --status_file ${monitor-directory:private}/mariadb_slow_query.report.json --max_queries_threshold "${slow-query-digest-parameters:max_queries_threshold}" --slowest_query_threshold "${slow-query-digest-parameters:slowest_queries_threshold}"
347

348 349 350 351
[{{ section('promise-check-computer-memory') }}]
recipe = slapos.cookbook:wrapper
wrapper-path = ${directory:promise}/check-computer-memory
command-line = "{{ parameter_dict["check-computer-memory-binary"] }}" -db ${monitor-instance-parameter:collector-db} --threshold "{{ slapparameter_dict["computer-memory-percent-threshold"] }}" --unit percent
352

353
[{{ section('promise') }}]
354 355 356
recipe = slapos.cookbook:wrapper
command-line = "{{ parameter_dict['bin-directory'] }}/is-local-tcp-port-opened" "${my-cnf-parameters:ip}" "${my-cnf-parameters:port}"
wrapper-path = ${directory:promise}/mariadb
357

358 359 360
[monitor-instance-parameter]
monitor-httpd-ipv6 = {{ (ipv6_set | list)[0] }}
monitor-httpd-port = {{ port + 1 }}
361
monitor-title = {{ slapparameter_dict['name'] }}
362
password = {{ slapparameter_dict['monitor-passwd'] }}
363

364
[buildout]
365
extends =
366
  {{ template_monitor }}
367
parts +=
368
  {{ part_list | join('\n  ') }}