Commit 2ac1416e authored by Julien Muchembled's avatar Julien Muchembled

NEO: review schema and add support for SQLite configuration

See neoppod@71564067
parent df6b58d2
...@@ -14,7 +14,7 @@ ...@@ -14,7 +14,7 @@
# not need these here). # not need these here).
[instance-common] [instance-common]
filename = instance-common.cfg.in filename = instance-common.cfg.in
md5sum = e000e7134113b9d1c63d40861eaf0489 md5sum = ecc98da90cd446ea224ddeece1374190
[root-common] [root-common]
filename = root-common.cfg.in filename = root-common.cfg.in
...@@ -30,7 +30,7 @@ md5sum = 9f27195d770b2f57461c60a82c851ab9 ...@@ -30,7 +30,7 @@ md5sum = 9f27195d770b2f57461c60a82c851ab9
[instance-neo] [instance-neo]
filename = instance-neo.cfg.in filename = instance-neo.cfg.in
md5sum = fda911d5ef9efee365f1b0ff9843a50b md5sum = c3fdf8c6aae89acd42d9c1ea052dfb7d
[template-neo-my-cnf] [template-neo-my-cnf]
filename = my.cnf.in filename = my.cnf.in
......
...@@ -41,6 +41,7 @@ url = {{ neo_master }} ...@@ -41,6 +41,7 @@ url = {{ neo_master }}
<= jinja2-template-base <= jinja2-template-base
url = {{ neo }} url = {{ neo }}
extra-context = extra-context =
import urllib urllib
key master_cfg neo-master:output key master_cfg neo-master:output
key admin_cfg neo-admin:output key admin_cfg neo-admin:output
{%- if mariadb_location is defined %} {%- if mariadb_location is defined %}
......
...@@ -110,14 +110,6 @@ ...@@ -110,14 +110,6 @@
"default": false, "default": false,
"type": "boolean" "type": "boolean"
}, },
"storage-type": {
"description": "Storage type. Defaults to MySQL if available, else SQLite.",
"enum": [
"MySQL",
"SQLite"
],
"type": "string"
},
"private-tmpfs": { "private-tmpfs": {
"description": "Size of private tmpfs mount to store the database. See filesystems/tmpfs.txt in Linux documentation. Use only for testing.", "description": "Size of private tmpfs mount to store the database. See filesystems/tmpfs.txt in Linux documentation. Use only for testing.",
"type": "string" "type": "string"
...@@ -126,27 +118,58 @@ ...@@ -126,27 +118,58 @@
"description": "List of bindings to test when running the test suite.", "description": "List of bindings to test when running the test suite.",
"type": "array" "type": "array"
}, },
"sqlite": {
"description": "Storage backend configuration. This selects the SQLite backend.",
"properties": {
"relaxed-writes": {
"description": "When enabled, sets synchronous = OFF and journal_mode = MEMORY - RTFM, those options are dangerous.",
"type": "boolean"
}
},
"additionalProperties": {
"description": "See NEO documentation for the list of supported settings.",
"type": ["number", "string"]
},
"type": "object"
},
"mysql": { "mysql": {
"description": "Dictionary containing parameters for MySQL.", "description": "MariaDB server configuration. This selects the MySQL backend.",
"default": {},
"properties": { "properties": {
"relaxed-writes": { "relaxed-writes": {
"description": "When enabled, sets innodb_flush_log_at_trx_commit = 0, innodb_flush_method = nosync, innodb_doublewrite = 0 and sync_frm = 0 - RTFM, those options are dangerous", "description": "When enabled, sets innodb_flush_log_at_trx_commit = 0, innodb_flush_method = nosync, innodb_doublewrite = 0 and sync_frm = 0 - RTFM, those options are dangerous.",
"default": false,
"type": "boolean" "type": "boolean"
} }
}, },
"additionalProperties": { "additionalProperties": {
"description": "To configure important parameters like innodb_buffer_pool_size, rocksdb_block_cache_size, etc.", "description": "To configure parameters like innodb_buffer_pool_size, rocksdb_block_cache_size, etc.",
"type": "string" "type": "string"
}, },
"type": "object" "type": "object"
}, },
"engine": { "engine": {
"description": "Configures storage engine, currently only InnoDB and RocksDB are supported. Defaults to NEO's default.", "description": "For NEO, this is a creation-time parameter and it defaults to NEO's default. For mysqld, this sets plugins to load and it defaults to load all.",
"enum": [
"InnoDB",
"RocksDB"
],
"type": "string" "type": "string"
} }
}, },
"oneOf": [
{
"description": "default (MariaDB if built else SQLite) or SQLite",
"properties": {
"mysql": false,
"engine": false
}
},
{
"required": ["mysql"],
"properties": {
"sqlite": false
}
}
],
"type": "object" "type": "object"
}, },
"node!": { "node!": {
......
{% macro section(name) %}{% do part_list.append(name) %}{{ name }}{% endmacro -%} {% macro section(name) %}{% do part_list.append(name) %}{{ name }}{% endmacro -%}
{% set part_list = [] -%} {% set part_list = [] -%}
{% set init_list = [] -%} {% set init_list = [] -%}
{% set directory_dict = {} -%}
{% set private_tmpfs = slapparameter_dict.get('private-tmpfs') -%} {% set private_tmpfs = slapparameter_dict.get('private-tmpfs') -%}
{% set storage_count = slapparameter_dict.get('storage-count', 1) -%} {% set storage_count = slapparameter_dict.get('storage-count', 1) -%}
{% set storage_type = slapparameter_dict.get('storage-type') or ( {# BBB: storage-type -#}
'MySQL' if mariadb_location is defined else 'SQLite') -%} {% if 'mysql' in slapparameter_dict -%}
{% set mysql = storage_count and storage_type != 'SQLite' -%} {% do assert('sqlite' not in slapparameter_dict) -%}
{% set storage_type = 'MySQL' -%}
{% set extra_dict = slapparameter_dict.pop('mysql') -%}
{% elif 'sqlite' in slapparameter_dict -%}
{% set storage_type = 'SQLite' -%}
{% set extra_dict = slapparameter_dict.pop('sqlite') -%}
{% elif storage_count -%}
{% set storage_type = slapparameter_dict.get('storage-type') or (
'MySQL' if mariadb_location is defined else 'SQLite') -%}
{% set extra_dict = {} -%}
{% else -%}
{% set storage_type = '' -%}
{% endif -%}
{% do assert(storage_count or not storage_type) -%}
{% do assert(slapparameter_dict.get('storage-type', storage_type) == storage_type) -%}
{% set mysql = storage_type == 'MySQL' -%}
{% if mysql -%} {% if mysql -%}
[{{ section('mysqld') }}] [{{ section('mysqld') }}]
...@@ -37,7 +54,7 @@ tmp-directory = ${directory:tmp} ...@@ -37,7 +54,7 @@ tmp-directory = ${directory:tmp}
pid-file = ${directory:var_run}/mariadb.pid pid-file = ${directory:var_run}/mariadb.pid
error-log = ${directory:log}/mariadb_error.log error-log = ${directory:log}/mariadb_error.log
slow-query-log = ${directory:log}/mariadb_slowquery.log slow-query-log = ${directory:log}/mariadb_slowquery.log
extra-dict = {{ dumps(slapparameter_dict.get('mysql', {})) }} extra-dict = {{ dumps(extra_dict) }}
init-file = ${init-script:output} init-file = ${init-script:output}
engine = {{ slapparameter_dict.get('engine', '') }} engine = {{ slapparameter_dict.get('engine', '') }}
...@@ -54,6 +71,14 @@ command-line = '{{ mariadb_location }}/bin/${:command}' --defaults-file="${my-cn ...@@ -54,6 +71,14 @@ command-line = '{{ mariadb_location }}/bin/${:command}' --defaults-file="${my-cn
wrapper-path = ${directory:bin}/${:command} wrapper-path = ${directory:bin}/${:command}
command = mysql command = mysql
{% elif storage_type == 'SQLite' -%}
{% if extra_dict.pop('relaxed-writes', False) -%}
{% do extra_dict.setdefault('synchronous', 'OFF') -%}
{% do extra_dict.setdefault('journal_mode', 'MEMORY') -%}
{% endif -%}
{% set query_string = urllib.urlencode(extra_dict) -%}
{% endif -%} {% endif -%}
[{{ section('binary-neolog') }}] [{{ section('binary-neolog') }}]
...@@ -123,14 +148,19 @@ logfile = ${directory:log}/{{ 'neostorage-' ~ i }}.log ...@@ -123,14 +148,19 @@ logfile = ${directory:log}/{{ 'neostorage-' ~ i }}.log
{%- if mysql %} {%- if mysql %}
{%- do init_list.append('CREATE DATABASE IF NOT EXISTS neo' ~ i ~ ';') %} {%- do init_list.append('CREATE DATABASE IF NOT EXISTS neo' ~ i ~ ';') %}
database-parameters = root@neo{{ i }}${my-cnf-parameters:socket} database-parameters = root@neo{{ i }}${my-cnf-parameters:socket}
{%- elif private_tmpfs %}
private-tmpfs = {{ private_tmpfs }} ${directory:tmp}
database-parameters = ${directory:tmp}/db.sqlite
{%- else %} {%- else %}
database-parameters = ${directory:db-{{i}}}/db.sqlite {%- if private_tmpfs %}
private-tmpfs = {{ private_tmpfs }} ${directory:tmp}
[directory] {%- set path = '${directory:tmp}/db.sqlite' %}
db-{{i}} = ${:srv}/{{ storage_id }} {%- else %}
{%- set path = '${directory:db-' ~ i ~ '}/db.sqlite' %}
{%- do directory_dict.__setitem__('db-' ~ i, '${:srv}/' + storage_id) %}
{%- endif %}
{%- if query_string %}
database-parameters = file:{{ path }}?{{ query_string }}
{%- else %}
database-parameters = {{ path }}
{%- endif -%}
{%- endif %} {%- endif %}
[{{ section('logrotate-storage-' ~ i) }}] [{{ section('logrotate-storage-' ~ i) }}]
...@@ -141,17 +171,6 @@ post = {{ bin_directory }}/slapos-kill -s RTMIN+1 -- {{ bin_directory }}/neostor ...@@ -141,17 +171,6 @@ post = {{ bin_directory }}/slapos-kill -s RTMIN+1 -- {{ bin_directory }}/neostor
{% endfor -%} {% endfor -%}
[directory]
recipe = slapos.cookbook:mkdirectory
bin = ${buildout:directory}/bin
etc = ${buildout:directory}/etc
var = ${buildout:directory}/var
etc_run = ${:etc}/run
var_run = ${:var}/run
log = ${buildout:directory}/var/log
tmp = ${buildout:directory}/tmp
srv = ${buildout:directory}/srv
{% if mysql -%} {% if mysql -%}
[init-script] [init-script]
recipe = slapos.recipe.template recipe = slapos.recipe.template
...@@ -196,13 +215,26 @@ context = ...@@ -196,13 +215,26 @@ context =
key datadir my-cnf-parameters:data-directory key datadir my-cnf-parameters:data-directory
key results_directory directory:results key results_directory directory:results
[directory] {%- do directory_dict.__setitem__('results', '${directory:srv}/tests') %}
results = ${directory:srv}/tests
{%- endif %} {%- endif %}
{%- endif %} {%- endif %}
{%- endif %} {%- endif %}
[directory]
recipe = slapos.cookbook:mkdirectory
bin = ${buildout:directory}/bin
etc = ${buildout:directory}/etc
var = ${buildout:directory}/var
etc_run = ${:etc}/run
var_run = ${:var}/run
log = ${buildout:directory}/var/log
tmp = ${buildout:directory}/tmp
srv = ${buildout:directory}/srv
{%- for k, v in directory_dict.iteritems() %}
{{ k }} = {{ v }}
{%- endfor %}
[buildout] [buildout]
extends = extends =
{{ logrotate_cfg }} {{ logrotate_cfg }}
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment