instance-mariadb.cfg.in 14.3 KB
Newer Older
1 2 3 4
{% set part_list = [] -%}
{% macro section(name) %}{% do part_list.append(name) %}{{ name }}{% endmacro -%}
{% set use_ipv6 = slapparameter_dict.get('use-ipv6', False) -%}
{% set database_list = slapparameter_dict.get('database-list', [{'name': 'erp5', 'user': 'user', 'password': 'insecure'}]) -%}
5
{% set test_database_list = [] %}
6
{% for database_count in range(slapparameter_dict.get('test-database-amount', 1)) -%}
7
{%   do test_database_list.append({'name': 'erp5_test_' ~ database_count, 'user': 'testuser_' ~ database_count, 'password': 'testpassword' ~ database_count}) -%}
8 9
{% endfor -%}
{% set catalog_backup = slapparameter_dict.get('catalog-backup', {}) -%}
10
{% set backup_periodicity = slapparameter_dict.get('backup-periodicity', 'daily') -%}
11 12
{% set full_backup_retention_days = catalog_backup.get('full-retention-days', 7) -%}
{% set incremental_backup_retention_days = catalog_backup.get('incremental-retention-days', full_backup_retention_days) -%}
13
{% set port = slapparameter_dict['tcpv4-port'] %}
14
{% if use_ipv6 -%}
15
{%   set ip = (ipv6_set | list)[0] -%}
16
{% else -%}
17
{%   set ip = (ipv4_set | list)[0] -%}
18
{% endif -%}
19
{% set dash = parameter_dict['dash-location'] ~ '/bin/dash' %}
20

21
[{{ section('publish') }}]
22
recipe = slapos.cookbook:publish.serialised
23
-extends = publish-early
24 25 26
{% macro render_database_list(database_list) -%}
{% set publish_database_list = [] -%}
{% for database in database_list -%}
27 28
{%   if database.get('user') -%}
{%     do publish_database_list.append("mysql://" ~ database['user'] ~ ":" ~ database['password'] ~ "@" ~ ip ~ ":" ~ port ~ "/" ~ database['name']) -%}
29
{%   else -%}
30
{%     do publish_database_list.append("mysql://" ~ ip ~ ":" ~ port ~ "/" ~ database['name']) -%}
31
{%   endif -%}
32 33 34 35 36
{% endfor -%}
{{ dumps(publish_database_list) }}
{% endmacro -%}
database-list = {{ render_database_list(database_list) }}
test-database-list = {{ render_database_list(test_database_list) }}
37
monitor-base-url = ${monitor-publish-parameters:monitor-base-url}
38

39 40 41 42 43 44 45 46 47 48 49 50 51 52
[publish-early]
recipe = slapos.cookbook:publish-early
-init =
  server-id gen-server-id:value
{%- set server_id = slapparameter_dict.get('server-id') %}
{%- if server_id %}
server-id = {{ dumps(server_id) }}
{%- endif %}

[gen-server-id]
recipe = slapos.cookbook:random.integer
minimum = {{ dumps(1) }}
maximum = {{ dumps(2**32 - 1) }}

53
[jinja2-template-base]
54
recipe = slapos.recipe.template:jinja2
55 56 57 58 59 60 61 62
mode = 644

[jinja2-template-executable]
< = jinja2-template-base
mode = 755

[simplefile]
< = jinja2-template-base
63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92
template = inline:{{ '{{ content }}' }}

{% macro simplefile(section_name, file_path, content, mode='') -%}
{%   set content_section_name = section_name ~ '-content' -%}
[{{  content_section_name }}]
content = {{ dumps(content) }}

[{{  section(section_name) }}]
< = simplefile
rendered = {{ file_path }}
context = key content {{content_section_name}}:content
mode = {{ mode }}
{%- endmacro %}
{% set ssl_dict = {} -%}
{% macro sslfile(key, content, mode='644') -%}
{%   set path = '${directory:mariadb-ssl}/' ~ key ~ '.pem' -%}
{%   do ssl_dict.__setitem__(key, path) -%}
{{   simplefile('ssl-file-' ~ key, path, content, mode) }}
{%- endmacro %}
{% set ssl_parameter_dict = slapparameter_dict.get('ssl') -%}
{% if ssl_parameter_dict -%}
{%   set base_directory = '${directory:mariadb-ssl}/' -%}
{# Note: The key content will be stored in .installed.cfg, and this template's
rendering, so the only point of mode is to avoid risking mariadb complaining
about laxist file mode. -#}
{{   sslfile('key', ssl_parameter_dict['key'], mode='600') }}
{{   sslfile('crt', ssl_parameter_dict['crt']) }}
{%   if 'ca-crt' in ssl_parameter_dict -%}
{{     sslfile('ca-crt', ssl_parameter_dict['ca-crt']) }}
{%   endif -%}
93 94 95
{%   if 'crl' in ssl_parameter_dict -%}
{{     sslfile('crl', ssl_parameter_dict['crl']) }}
{%   endif -%}
96 97
{%- endif %}

98 99
{% if full_backup_retention_days > -1 -%}
[{{ section('cron-entry-mariadb-backup') }}]
100
recipe = slapos.cookbook:cron.d
101
cron-entries = ${cron:cron-entries}
102
name = mariadb-backup
103
time = {{ dumps(backup_periodicity) }}
104 105 106 107 108 109
{# When binlogs are enabled:
# flush-logs: used so no manipulation on binlogs is needed to restore from
#   full + binlogs. The first binlog after a dump starts from dump snapshot and
#   can be fully restored.
# master-data: use value "2" as we are not in a replication case
#}
110
command = "${binary-wrap-mysqldump:wrapper-path}" -u root --all-databases --flush-privileges --single-transaction --max-allowed-packet=128M {% if incremental_backup_retention_days > -1 %}--flush-logs --master-data=2 {% endif %}| {{ parameter_dict['gzip-location'] }}/bin/gzip > "${directory:mariadb-backup-full}/$({{ parameter_dict['coreutils-location'] }}/bin/date "+%Y%m%d%H%M%S").sql.gz"
111 112 113 114 115 116 117
{# KEEP GLOB PATTERN IN SYNC with generated filenames above
#           YYYYmmddHHMMSS -#}
file-glob = ??????????????.sql.gz

{% if full_backup_retention_days > 0 -%}
[{{ section("cron-entry-mariadb-backup-expire") }}]
recipe = slapos.cookbook:cron.d
118
cron-entries = ${cron:cron-entries}
119
name = mariadb-backup-expire
120
time = {{ dumps(backup_periodicity) }}
121 122 123 124 125
command = {{ parameter_dict['findutils-location'] }}/bin/find "${directory:mariadb-backup-full}" -maxdepth 1 -name "${cron-entry-mariadb-backup:file-glob}" -daystart -mtime +{{ full_backup_retention_days }} -delete
{%- endif %}
{%- endif %}

[my-cnf-parameters]
126 127
ip = {{ ip }}
port = {{ port }}
128
socket = ${directory:run}/mariadb.sock
129
data-directory = ${directory:srv}/mariadb
130
tmp-directory = ${directory:tmp}
131
etc-directory = ${directory:etc}
132 133 134
pid-file = ${directory:run}/mariadb.pid
error-log = ${directory:log}/mariadb_error.log
slow-query-log = ${directory:log}/mariadb_slowquery.log
135
long-query-time = {{ dumps(slapparameter_dict.get('long-query-time', 1)) }}
136
max-connection-count = {{ dumps(slapparameter_dict.get('max-connection-count', 1000)) }}
137
innodb-buffer-pool-size = {{ dumps(slapparameter_dict.get('innodb-buffer-pool-size', 0)) }}
138
innodb-buffer-pool-instances = {{ dumps(slapparameter_dict.get('innodb-buffer-pool-instances', 0)) }}
139
innodb-log-file-size = {{ dumps(slapparameter_dict.get('innodb-log-file-size', 0)) }}
140
innodb-file-per-table = {{ dumps(slapparameter_dict.get('innodb-file-per-table', 0)) }}
141
innodb-log-buffer-size = {{ dumps(slapparameter_dict.get('innodb-log-buffer-size', 0)) }}
142
relaxed-writes = {{ dumps(slapparameter_dict.get('relaxed-writes', False)) }}
143 144 145
{% if incremental_backup_retention_days > -1 -%}
binlog-path = ${directory:mariadb-backup-incremental}/binlog
# XXX: binlog rotation happens along with other log's rotation
146
binlog-expire-days = {{ dumps(incremental_backup_retention_days) }}
147
server-id = ${publish-early:server-id}
148 149
{% else %}
binlog-path =
150 151 152 153
{%- endif %}
{%- for key, value in ssl_dict.items() -%}
ssl-{{ key }} = {{ value }}
{% endfor %}
154 155

[my-cnf]
156
< = jinja2-template-base
157 158 159 160 161
rendered = ${directory:etc}/mariadb.cnf
template = {{ parameter_dict['template-my-cnf'] }}
context = section parameter_dict my-cnf-parameters

[init-script-parameters]
162
database-list = {{ dumps(database_list + test_database_list) }}
163 164

[init-script]
165
< = jinja2-template-executable
166 167 168 169 170
# XXX: is there a better location ?
rendered = ${directory:etc}/mariadb_initial_setup.sql
template = {{ parameter_dict['template-mariadb-initial-setup'] }}
context = section parameter_dict init-script-parameters

171
[{{ section('update-mysql') }}]
172 173 174 175 176
recipe = slapos.cookbook:generic.mysql.wrap_update_mysql
output = ${directory:services}/mariadb_update
binary = ${binary-wrap-mysql_upgrade:wrapper-path}
mysql = ${binary-wrap-mysql:wrapper-path}
init-script = ${init-script:rendered}
177
mysql_tzinfo_to_sql = ${binary-wrap-mysql_tzinfo_to_sql:wrapper-path}
178

179 180 181
[{{ section('mysqld') }}]
< = jinja2-template-executable
rendered = ${directory:services}/mariadb
182 183 184 185 186 187 188 189
template = {{ parameter_dict['template-mysqld-wrapper'] }}
context =
  key defaults_file my-cnf:rendered
  key datadir my-cnf-parameters:data-directory
  key environ :environ
environ =
  ODBCSYSINI='${my-cnf-parameters:etc-directory}'
  LD_LIBRARY_PATH=$${LD_LIBRARY_PATH:+$LD_LIBRARY_PATH:}'{{ parameter_dict['unixodbc-location'] }}/lib'
190
  {%- for variable in slapparameter_dict.get('environment-variables', ()) %}
191
  {{ variable }}
192
  {%- endfor %}
193

194 195 196 197 198 199 200 201 202 203 204
[odbc-ini-text]
text = {{ dumps(slapparameter_dict.get('odbc-ini', '').encode('base64')) }}

[{{ section('odbc-ini') }}]
< = jinja2-template-base
rendered = ${directory:etc}/odbc.ini
template = inline:{% raw -%}
  {{ parameter_dict['text'].decode('base64') }}
  {%- endraw %}
context = section parameter_dict odbc-ini-text

205
[{{ section('logrotate-entry-mariadb') }}]
206
< = logrotate-entry-base
207
name = mariadb
208
log = ${my-cnf-parameters:error-log} ${my-cnf-parameters:slow-query-log}
209
post = "${binary-wrap-mysql:wrapper-path}" -B -u root -e "FLUSH LOGS"
210

211
[{{ section('binary-link') }}]
212
recipe = slapos.cookbook:symbolic.link
213
target-directory = ${directory:bin}
214
link-binary = {{ dumps(parameter_dict['link-binary']) }}
215

216 217 218 219
[{{ section("binary-link-mysqlbinlog") }}]
< = binary-link
link-binary = {{ parameter_dict['mariadb-location'] }}/bin/mysqlbinlog

220 221 222 223 224 225 226 227 228 229 230 231 232 233 234
[binary-wrap-base]
recipe = slapos.cookbook:wrapper
# Note: --defaults-file must be the first argument, otherwise wrapped binary
# will reject it.
command-line = "{{ parameter_dict['mariadb-location'] }}/bin/${:command}" --defaults-file="${my-cnf:rendered}"
wrapper-path = ${directory:bin}/${:command}

[binary-wrap-mysql]
< = binary-wrap-base
command = mysql

[binary-wrap-mysqldump]
< = binary-wrap-base
command = mysqldump

235
[binary-wrap-mysql_upgrade]
236
< = binary-wrap-base
237 238 239 240
command = mysql_upgrade

[binary-wrap-mysql_tzinfo_to_sql]
< = binary-wrap-base
241
command-line = "{{ parameter_dict['mariadb-location'] }}/bin/${:command}"
242
command = mysql_tzinfo_to_sql
243

244 245 246 247 248 249
[binary-wrap-pt-digest]
<= binary-wrap-base
command-line = "{{ parameter_dict['percona-tools-location'] }}/bin/${:command}"
command = pt-query-digest
 

250 251
[directory]
recipe = slapos.cookbook:mkdirectory
252 253 254
bin = ${buildout:directory}/bin
etc = ${buildout:directory}/etc
services = ${:etc}/run
255
plugin = ${:etc}/plugin
256
srv = ${buildout:directory}/srv
257
tmp = ${buildout:directory}/tmp
258 259 260
backup = ${:srv}/backup
mariadb-backup-full = ${:backup}/mariadb-full
mariadb-backup-incremental = ${:backup}/mariadb-incremental
261
mariadb-ssl = ${:etc}/mariadb-ssl
262 263 264
var = ${buildout:directory}/var
log = ${:var}/log
run = ${:var}/run
265
slowquery = ${monitor-directory:private}/slowquery_digest
266

267
[{{ section('resiliency-exclude-file') }}]
268
# Generate rdiff exclude file in case of resiliency
269
< = jinja2-template-base
270
template = {{ 'inline:{{ "${my-cnf-parameters:data-directory}/**\\n${directory:mariadb-backup-incremental}/**\\n${directory:log}/**\\n${directory:tmp}/**\\n" }}' }}
271
rendered = ${directory:srv}/exporter.exclude
272

273 274 275
[{{ section("resiliency-identity-signature-script")}}]
# Generate identity script used by webrunner to check data integrity
recipe = slapos.cookbook:wrapper
276
command-line = {{ bin_directory }}/backup-identity-script-excluding-path --exclude-path "srv/backup/logrotate/**"
277 278 279
wrapper-path = ${directory:srv}/.backup_identity_script
mode = 770

280 281 282
[dash]
dash = {{ dumps(dash) }}

283 284 285 286 287 288 289
[{{ section('start-clone-from-backup') }}]
< = jinja2-template-executable
template = {{ parameter_dict['mariadb-start-clone-from-backup'] }}
rendered = ${directory:bin}/start-clone-from-backup
context =
  key dash dash:dash
  key client binary-wrap-mysql:wrapper-path
290
  key data_directory my-cnf-parameters:data-directory
291 292 293 294 295
  key pid_file my-cnf-parameters:pid-file
  key server mysqld:rendered
  key update update-mysql:output
  key socket my-cnf-parameters:socket

296
[{{ section('resiliency-after-import-script') }}]
297
# Generate after import script used by importer instance of webrunner
298 299
< = jinja2-template-executable
template = {{ parameter_dict['mariadb-resiliency-after-import-script'] }}
300
rendered = ${directory:bin}/restore-from-backup
301 302 303
context =
  key dash dash:dash
  key mysql_executable binary-wrap-mysql:wrapper-path
304
  key mariadb_data_directory my-cnf-parameters:data-directory
305 306 307 308
  key mariadb_backup_directory directory:mariadb-backup-full
  key pid_file my-cnf-parameters:pid-file
  key binlog_path my-cnf-parameters:binlog-path
  key server_executable mysqld:rendered
309

310
[{{ section('monitor-generate-mariadb-slow-query-report') }}]
311 312 313 314 315 316 317 318 319
recipe = slapos.cookbook:cron.d
cron-entries = ${cron:cron-entries}
name = generate-mariadb-slow-query-report
# The goal is to be executed before logrotate log rotation.
# Here, logrotate-entry-base:frequency = daily, so we run at 23 o'clock every day.
frequency = 0 23 * * *
command = ${monitor-generate-mariadb-slow-query-report-wrapper:rendered}

[monitor-generate-mariadb-slow-query-report-wrapper]
320 321
recipe = slapos.recipe.template:jinja2
template = {{ parameter_dict['mariadb-slow-query-report-script'] }}
322 323
rendered = ${directory:bin}/${:filename}
filename = generate-mariadb-slow-query-report
324 325 326 327 328
mode = 755
context = 
  raw slow_query_path ${directory:srv}/backup/logrotate/mariadb_slowquery.log
  raw pt_query_exec ${binary-wrap-pt-digest:wrapper-path}
  raw dash {{ parameter_dict['dash-location'] }}/bin/dash
329
  key output_folder directory:slowquery
330

331 332 333
[slow-query-digest-parameters]
max_queries_threshold = {{ slapparameter_dict['max-slowqueries-threshold'] }}
slowest_queries_threshold = {{ slapparameter_dict['slowest-query-threshold'] }}
334 335

[{{ section('monitor-promise-slowquery-result') }}]
336 337 338 339
<= monitor-promise-base
module = check_command_execute
name = check-slow-query-pt-digest-result.py
config-command = "{{ parameter_dict['promise-check-slow-queries-digest-result'] }}" --ptdigest_path "${directory:slowquery}" --status_file ${monitor-directory:private}/mariadb_slow_query.report.json --max_queries_threshold "${slow-query-digest-parameters:max_queries_threshold}" --slowest_query_threshold "${slow-query-digest-parameters:slowest_queries_threshold}"
340

341
[{{ section('promise-check-computer-memory') }}]
342 343 344 345
<= monitor-promise-base
module = check_command_execute
name = check-computer-memory.py
config-command = "{{ parameter_dict["check-computer-memory-binary"] }}" -db ${monitor-instance-parameter:collector-db} --threshold "{{ slapparameter_dict["computer-memory-percent-threshold"] }}" --unit percent
346

347
[{{ section('promise') }}]
348 349 350 351
<= monitor-promise-base
module = check_command_execute
name = mariadb.py
config-command = "{{ parameter_dict['bin-directory'] }}/is-local-tcp-port-opened" "${my-cnf-parameters:ip}" "${my-cnf-parameters:port}"
352

353 354 355
[monitor-instance-parameter]
monitor-httpd-ipv6 = {{ (ipv6_set | list)[0] }}
monitor-httpd-port = {{ port + 1 }}
356
monitor-title = {{ slapparameter_dict['name'] }}
357
password = {{ slapparameter_dict['monitor-passwd'] }}
358

359
[buildout]
360
extends =
361
  {{ template_monitor }}
362
parts +=
363
  {{ part_list | join('\n  ') }}