instance-mariadb.cfg.in 14.7 KB
Newer Older
1 2 3
{% set part_list = [] -%}
{% macro section(name) %}{% do part_list.append(name) %}{{ name }}{% endmacro -%}
{% set use_ipv6 = slapparameter_dict.get('use-ipv6', False) -%}
4
{% set database_list = slapparameter_dict.get('database-list', [{'name': 'erp5', 'user': 'user', 'password': 'insecure', 'with-process-privilege': True}]) -%}
5
{% set test_database_list = [] %}
6
{% for database_count in range(slapparameter_dict.get('test-database-amount', 1)) -%}
7
{%   do test_database_list.append({'name': 'erp5_test_' ~ database_count, 'user': 'testuser_' ~ database_count, 'password': 'testpassword' ~ database_count}) -%}
8 9
{% endfor -%}
{% set catalog_backup = slapparameter_dict.get('catalog-backup', {}) -%}
10
{% set backup_periodicity = slapparameter_dict.get('backup-periodicity', 'daily') -%}
11 12
{% set full_backup_retention_days = catalog_backup.get('full-retention-days', 7) -%}
{% set incremental_backup_retention_days = catalog_backup.get('incremental-retention-days', full_backup_retention_days) -%}
13
{% set port = slapparameter_dict['tcpv4-port'] %}
14
{% if use_ipv6 -%}
15
{%   set ip = (ipv6_set | list)[0] -%}
16
{% else -%}
17
{%   set ip = (ipv4_set | list)[0] -%}
18
{% endif -%}
19
{% set dash = parameter_dict['dash-location'] ~ '/bin/dash' %}
20

21
[{{ section('publish') }}]
22
recipe = slapos.cookbook:publish.serialised
23
-extends = publish-early
24 25 26
{% macro render_database_list(database_list) -%}
{% set publish_database_list = [] -%}
{% for database in database_list -%}
27 28
{%   if database.get('user') -%}
{%     do publish_database_list.append("mysql://" ~ database['user'] ~ ":" ~ database['password'] ~ "@" ~ ip ~ ":" ~ port ~ "/" ~ database['name']) -%}
29
{%   else -%}
30
{%     do publish_database_list.append("mysql://" ~ ip ~ ":" ~ port ~ "/" ~ database['name']) -%}
31
{%   endif -%}
32 33 34 35 36
{% endfor -%}
{{ dumps(publish_database_list) }}
{% endmacro -%}
database-list = {{ render_database_list(database_list) }}
test-database-list = {{ render_database_list(test_database_list) }}
37
monitor-base-url = ${monitor-publish-parameters:monitor-base-url}
38

39 40 41 42 43 44 45 46 47 48 49 50 51 52
[publish-early]
recipe = slapos.cookbook:publish-early
-init =
  server-id gen-server-id:value
{%- set server_id = slapparameter_dict.get('server-id') %}
{%- if server_id %}
server-id = {{ dumps(server_id) }}
{%- endif %}

[gen-server-id]
recipe = slapos.cookbook:random.integer
minimum = {{ dumps(1) }}
maximum = {{ dumps(2**32 - 1) }}

53
[jinja2-template-base]
54
recipe = slapos.recipe.template:jinja2
55 56 57 58 59 60 61 62
mode = 644

[jinja2-template-executable]
< = jinja2-template-base
mode = 755

[simplefile]
< = jinja2-template-base
63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92
template = inline:{{ '{{ content }}' }}

{% macro simplefile(section_name, file_path, content, mode='') -%}
{%   set content_section_name = section_name ~ '-content' -%}
[{{  content_section_name }}]
content = {{ dumps(content) }}

[{{  section(section_name) }}]
< = simplefile
rendered = {{ file_path }}
context = key content {{content_section_name}}:content
mode = {{ mode }}
{%- endmacro %}
{% set ssl_dict = {} -%}
{% macro sslfile(key, content, mode='644') -%}
{%   set path = '${directory:mariadb-ssl}/' ~ key ~ '.pem' -%}
{%   do ssl_dict.__setitem__(key, path) -%}
{{   simplefile('ssl-file-' ~ key, path, content, mode) }}
{%- endmacro %}
{% set ssl_parameter_dict = slapparameter_dict.get('ssl') -%}
{% if ssl_parameter_dict -%}
{%   set base_directory = '${directory:mariadb-ssl}/' -%}
{# Note: The key content will be stored in .installed.cfg, and this template's
rendering, so the only point of mode is to avoid risking mariadb complaining
about laxist file mode. -#}
{{   sslfile('key', ssl_parameter_dict['key'], mode='600') }}
{{   sslfile('crt', ssl_parameter_dict['crt']) }}
{%   if 'ca-crt' in ssl_parameter_dict -%}
{{     sslfile('ca-crt', ssl_parameter_dict['ca-crt']) }}
{%   endif -%}
93 94 95
{%   if 'crl' in ssl_parameter_dict -%}
{{     sslfile('crl', ssl_parameter_dict['crl']) }}
{%   endif -%}
96 97
{%- endif %}

98 99
{% if full_backup_retention_days > -1 -%}
[{{ section('cron-entry-mariadb-backup') }}]
100
recipe = slapos.cookbook:cron.d
101
cron-entries = ${cron:cron-entries}
102
name = mariadb-backup
103
time = {{ dumps(backup_periodicity) }}
104 105 106 107 108 109
{# When binlogs are enabled:
# flush-logs: used so no manipulation on binlogs is needed to restore from
#   full + binlogs. The first binlog after a dump starts from dump snapshot and
#   can be fully restored.
# master-data: use value "2" as we are not in a replication case
#}
110
command = "${binary-wrap-mysqldump:wrapper-path}" --all-databases --flush-privileges --single-transaction --max-allowed-packet=128M {% if incremental_backup_retention_days > -1 %}--flush-logs --master-data=2 {% endif %}| {{ parameter_dict['gzip-location'] }}/bin/gzip > "${directory:mariadb-backup-full}/$({{ parameter_dict['coreutils-location'] }}/bin/date "+%Y%m%d%H%M%S").sql.gz"
111 112 113 114 115 116 117
{# KEEP GLOB PATTERN IN SYNC with generated filenames above
#           YYYYmmddHHMMSS -#}
file-glob = ??????????????.sql.gz

{% if full_backup_retention_days > 0 -%}
[{{ section("cron-entry-mariadb-backup-expire") }}]
recipe = slapos.cookbook:cron.d
118
cron-entries = ${cron:cron-entries}
119
name = mariadb-backup-expire
120
time = {{ dumps(backup_periodicity) }}
121 122 123 124 125
command = {{ parameter_dict['findutils-location'] }}/bin/find "${directory:mariadb-backup-full}" -maxdepth 1 -name "${cron-entry-mariadb-backup:file-glob}" -daystart -mtime +{{ full_backup_retention_days }} -delete
{%- endif %}
{%- endif %}

[my-cnf-parameters]
126 127
ip = {{ ip }}
port = {{ port }}
128
socket = ${directory:run}/mariadb.sock
129
data-directory = ${directory:srv}/mariadb
130
tmp-directory = ${directory:tmp}
131
etc-directory = ${directory:etc}
132 133
plugin-directory = {{ dumps(parameter_dict['mroonga-mariadb-plugin-dir']) }}
groonga-plugins-path = {{ parameter_dict['groonga-plugins-path'] }}
134 135 136
pid-file = ${directory:run}/mariadb.pid
error-log = ${directory:log}/mariadb_error.log
slow-query-log = ${directory:log}/mariadb_slowquery.log
137
long-query-time = {{ dumps(slapparameter_dict.get('long-query-time', 1)) }}
138
max-connection-count = {{ dumps(slapparameter_dict.get('max-connection-count', 1000)) }}
139
innodb-buffer-pool-size = {{ dumps(slapparameter_dict.get('innodb-buffer-pool-size', 0)) }}
140
innodb-buffer-pool-instances = {{ dumps(slapparameter_dict.get('innodb-buffer-pool-instances', 0)) }}
141
innodb-log-file-size = {{ dumps(slapparameter_dict.get('innodb-log-file-size', 0)) }}
142
innodb-file-per-table = {{ dumps(slapparameter_dict.get('innodb-file-per-table', 0)) }}
143
innodb-log-buffer-size = {{ dumps(slapparameter_dict.get('innodb-log-buffer-size', 0)) }}
144
relaxed-writes = {{ dumps(slapparameter_dict.get('relaxed-writes', False)) }}
145 146 147
{% if incremental_backup_retention_days > -1 -%}
binlog-path = ${directory:mariadb-backup-incremental}/binlog
# XXX: binlog rotation happens along with other log's rotation
148
binlog-expire-days = {{ dumps(incremental_backup_retention_days) }}
149
server-id = ${publish-early:server-id}
150 151
{% else %}
binlog-path =
152 153 154 155
{%- endif %}
{%- for key, value in ssl_dict.items() -%}
ssl-{{ key }} = {{ value }}
{% endfor %}
156 157

[my-cnf]
158
< = jinja2-template-base
159 160 161 162 163
rendered = ${directory:etc}/mariadb.cnf
template = {{ parameter_dict['template-my-cnf'] }}
context = section parameter_dict my-cnf-parameters

[init-script-parameters]
164
database-list = {{ dumps(database_list + test_database_list) }}
165
mroonga-mariadb-install-sql = {{ dumps(parameter_dict['mroonga-mariadb-install-sql']) }}
166 167

[init-script]
168
< = jinja2-template-executable
169 170 171 172 173
# XXX: is there a better location ?
rendered = ${directory:etc}/mariadb_initial_setup.sql
template = {{ parameter_dict['template-mariadb-initial-setup'] }}
context = section parameter_dict init-script-parameters

174
[{{ section('update-mysql') }}]
175 176 177 178 179
recipe = slapos.cookbook:generic.mysql.wrap_update_mysql
output = ${directory:services}/mariadb_update
binary = ${binary-wrap-mysql_upgrade:wrapper-path}
mysql = ${binary-wrap-mysql:wrapper-path}
init-script = ${init-script:rendered}
180
mysql_tzinfo_to_sql = ${binary-wrap-mysql_tzinfo_to_sql:wrapper-path}
181

182 183 184
[{{ section('mysqld') }}]
< = jinja2-template-executable
rendered = ${directory:services}/mariadb
185 186 187 188 189 190
template = {{ parameter_dict['template-mysqld-wrapper'] }}
context =
  key defaults_file my-cnf:rendered
  key datadir my-cnf-parameters:data-directory
  key environ :environ
environ =
191
  GRN_PLUGINS_PATH='${my-cnf-parameters:groonga-plugins-path}'
192 193
  ODBCSYSINI='${my-cnf-parameters:etc-directory}'
  LD_LIBRARY_PATH=$${LD_LIBRARY_PATH:+$LD_LIBRARY_PATH:}'{{ parameter_dict['unixodbc-location'] }}/lib'
194
  {%- for variable in slapparameter_dict.get('environment-variables', ()) %}
195
  {{ variable }}
196
  {%- endfor %}
197

198 199 200 201 202

[{{ section('odbc-ini') }}]
< = jinja2-template-base
rendered = ${directory:etc}/odbc.ini
template = inline:{% raw -%}
203
  {{ base64.b64decode(text).decode('utf-8') }}
204
  {%- endraw %}
205 206 207 208
text = {{ dumps(base64.b64encode(slapparameter_dict.get('odbc-ini', '').encode('utf-8'))) }}
context =
  key text :text
  import base64 base64
209

210
[{{ section('logrotate-entry-mariadb') }}]
211
< = logrotate-entry-base
212
name = mariadb
213
log = ${my-cnf-parameters:error-log} ${my-cnf-parameters:slow-query-log}
214
post = "${binary-wrap-mysql:wrapper-path}" -B -e "FLUSH LOGS"
215

216
[{{ section('binary-link') }}]
217
recipe = slapos.cookbook:symbolic.link
218
target-directory = ${directory:bin}
219
link-binary = {{ dumps(parameter_dict['link-binary']) }}
220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235

[binary-wrap-base]
recipe = slapos.cookbook:wrapper
# Note: --defaults-file must be the first argument, otherwise wrapped binary
# will reject it.
command-line = "{{ parameter_dict['mariadb-location'] }}/bin/${:command}" --defaults-file="${my-cnf:rendered}"
wrapper-path = ${directory:bin}/${:command}

[binary-wrap-mysql]
< = binary-wrap-base
command = mysql

[binary-wrap-mysqldump]
< = binary-wrap-base
command = mysqldump

236
[binary-wrap-mysql_upgrade]
237
< = binary-wrap-base
238 239 240 241
command = mysql_upgrade

[binary-wrap-mysql_tzinfo_to_sql]
< = binary-wrap-base
242
command-line = "{{ parameter_dict['mariadb-location'] }}/bin/${:command}"
243
command = mysql_tzinfo_to_sql
244

245 246 247 248
[binary-wrap-pt-digest]
<= binary-wrap-base
command-line = "{{ parameter_dict['percona-tools-location'] }}/bin/${:command}"
command = pt-query-digest
249

250

251 252
[directory]
recipe = slapos.cookbook:mkdirectory
253 254 255
bin = ${buildout:directory}/bin
etc = ${buildout:directory}/etc
services = ${:etc}/run
256
plugin = ${:etc}/plugin
257
srv = ${buildout:directory}/srv
258
tmp = ${buildout:directory}/tmp
259 260 261
backup = ${:srv}/backup
mariadb-backup-full = ${:backup}/mariadb-full
mariadb-backup-incremental = ${:backup}/mariadb-incremental
262
mariadb-ssl = ${:etc}/mariadb-ssl
263 264 265
var = ${buildout:directory}/var
log = ${:var}/log
run = ${:var}/run
266
slowquery = ${monitor-directory:private}/slowquery_digest
267

268
[{{ section('resiliency-exclude-file') }}]
269
# Generate rdiff exclude file in case of resiliency
270
< = jinja2-template-base
271
template = {{ 'inline:{{ "${my-cnf-parameters:data-directory}/**\\n${directory:mariadb-backup-incremental}/**\\n${directory:log}/**\\n${directory:tmp}/**\\n" }}' }}
272
rendered = ${directory:srv}/exporter.exclude
273

274 275 276
[{{ section("resiliency-identity-signature-script")}}]
# Generate identity script used by webrunner to check data integrity
recipe = slapos.cookbook:wrapper
277
command-line = {{ bin_directory }}/backup-identity-script-excluding-path --exclude-path "srv/backup/logrotate/**"
278 279 280
wrapper-path = ${directory:srv}/.backup_identity_script
mode = 770

281 282 283
[dash]
dash = {{ dumps(dash) }}

284 285 286 287 288 289 290
[{{ section('start-clone-from-backup') }}]
< = jinja2-template-executable
template = {{ parameter_dict['mariadb-start-clone-from-backup'] }}
rendered = ${directory:bin}/start-clone-from-backup
context =
  key dash dash:dash
  key client binary-wrap-mysql:wrapper-path
291
  key data_directory my-cnf-parameters:data-directory
292 293 294 295 296
  key pid_file my-cnf-parameters:pid-file
  key server mysqld:rendered
  key update update-mysql:output
  key socket my-cnf-parameters:socket

297
[{{ section('resiliency-after-import-script') }}]
298
# Generate after import script used by importer instance of webrunner
299 300
< = jinja2-template-executable
template = {{ parameter_dict['mariadb-resiliency-after-import-script'] }}
301
rendered = ${directory:bin}/restore-from-backup
302 303 304
context =
  key dash dash:dash
  key mysql_executable binary-wrap-mysql:wrapper-path
305
  key mariadb_data_directory my-cnf-parameters:data-directory
306 307 308 309
  key mariadb_backup_directory directory:mariadb-backup-full
  key pid_file my-cnf-parameters:pid-file
  key binlog_path my-cnf-parameters:binlog-path
  key server_executable mysqld:rendered
310

311
[{{ section('monitor-generate-mariadb-slow-query-report') }}]
312 313 314 315 316 317 318 319 320
recipe = slapos.cookbook:cron.d
cron-entries = ${cron:cron-entries}
name = generate-mariadb-slow-query-report
# The goal is to be executed before logrotate log rotation.
# Here, logrotate-entry-base:frequency = daily, so we run at 23 o'clock every day.
frequency = 0 23 * * *
command = ${monitor-generate-mariadb-slow-query-report-wrapper:rendered}

[monitor-generate-mariadb-slow-query-report-wrapper]
321 322
recipe = slapos.recipe.template:jinja2
template = {{ parameter_dict['mariadb-slow-query-report-script'] }}
323 324
rendered = ${directory:bin}/${:filename}
filename = generate-mariadb-slow-query-report
325
mode = 755
326
context =
327 328 329
  raw slow_query_path ${directory:srv}/backup/logrotate/mariadb_slowquery.log
  raw pt_query_exec ${binary-wrap-pt-digest:wrapper-path}
  raw dash {{ parameter_dict['dash-location'] }}/bin/dash
330
  raw xz {{ parameter_dict['xz-utils-location'] }}/bin/xz
331
  key output_folder directory:slowquery
332

333
{%if slapparameter_dict.get('max-slowqueries-threshold') and slapparameter_dict.get('slowest-query-threshold') %}
334
[{{ section('monitor-promise-slowquery-result') }}]
335
<= monitor-promise-base
336
promise = check_command_execute
337
name = check-slow-query-pt-digest-result.py
338 339 340 341
config-command = "{{ parameter_dict['promise-check-slow-queries-digest-result'] }}" --ptdigest_path "${directory:slowquery}" --status_file ${monitor-directory:private}/mariadb_slow_query.report.json --max_queries_threshold "${:max_queries_threshold}" --slowest_query_threshold "${:slowest_queries_threshold}"
max_queries_threshold = {{ slapparameter_dict['max-slowqueries-threshold'] }}
slowest_queries_threshold = {{ slapparameter_dict['slowest-query-threshold'] }}
{%-endif%}
342

343
[{{ section('promise-check-computer-memory') }}]
344
<= monitor-promise-base
345
promise = check_command_execute
346 347
name = check-computer-memory.py
config-command = "{{ parameter_dict["check-computer-memory-binary"] }}" -db ${monitor-instance-parameter:collector-db} --threshold "{{ slapparameter_dict["computer-memory-percent-threshold"] }}" --unit percent
348

349
[{{ section('promise') }}]
350
<= monitor-promise-base
351
promise = check_command_execute
352
name = mariadb.py
353
config-command = "${binary-wrap-mysql:wrapper-path}"  --execute ';' {% if database_list and database_list[0].get('user') %} --host="${my-cnf-parameters:ip}" --port="${my-cnf-parameters:port}" --user="{{ database_list[0]['user'] }}" --password="{{ database_list[0]['password'] }}" {% endif %}
354

355 356 357
[monitor-instance-parameter]
monitor-httpd-ipv6 = {{ (ipv6_set | list)[0] }}
monitor-httpd-port = {{ port + 1 }}
358
monitor-title = {{ slapparameter_dict['name'] }}
359
password = {{ slapparameter_dict['monitor-passwd'] }}
360

361
[buildout]
362
extends =
363
  {{ template_monitor }}
364
parts +=
365
  {{ part_list | join('\n  ') }}