NEO: add support for SQLite backend

See sqlite-mixin.cfg to skip the building of MariaDB.

slapos/recipe/mkdirectory.py

@@ -39,4 +39,7 @@ class Recipe(GenericBaseRecipe):
     for path in sorted(self.directory.itervalues()):
       if path and not os.path.isdir(path):
         os.makedirs(path, self.mode)
+    # WARNING: This recipe is currently used to create directories that will
+    #          contain user data  (e.g. NEO db). Such directories must never
+    #          be purged by the uninstallation of a recipe.
     return []

software/neoppod/instance-common.cfg.in

@@ -33,11 +33,13 @@ template = {{ neo_admin }}
 <= jinja2-template-base
 template = {{ neo_master }}
 
-[neo-storage-mysql]
+[neo]
 <= jinja2-template-base
-template = {{ neo_storage_mysql }}
+template = {{ neo }}
 extra-context =
     key master_cfg neo-master:rendered
     key admin_cfg neo-admin:rendered
+{%- if mariadb_location is defined %}
     raw mariadb_location {{ mariadb_location }}
     raw template_neo_my_cnf {{ template_neo_my_cnf }}
+{%- endif %}

software/neoppod/instance-neo-input-schema.json

@@ -84,6 +84,11 @@
             "default": false,
             "type": "boolean"
           },
+          "storage-type": {
+              "description": "Storage type. Defaults to MySQL if available, else SQLite.",
+              "enum": ["MySQL", "SQLite"],
+              "type": "string"
+          },
           "mysql": {
             "description": "Dictionary containing parameters for MySQL.",
             "default": {},

software/neoppod/instance-neo-storage-mysql.cfg.in → software/neoppod/instance-neo.cfg.in

@@ -2,6 +2,11 @@
 {% set part_list = [] -%}
 {% set init_list = [] -%}
 
+{% set storage_type = slapparameter_dict.get('storage-type') or (
+     'MySQL' if mariadb_location is defined else 'SQLite') -%}
+{% set mysql = storage_type == 'MySQL' -%}
+{% if mysql -%}
+
 [mysqld]
 recipe = slapos.cookbook:generic.mysql.wrap_mysqld
 output = ${directory:etc_run}/mariadb
@@ -34,6 +39,8 @@ command-line = ${mysqld:mysql-base-directory}/bin/${:command} --defaults-file=${
 wrapper-path = ${directory:bin}/${:command}
 command = mysql
 
+{% endif -%}
+
 [{{ section('binary-neolog') }}]
 recipe = slapos.cookbook:symbolic.link
 target-directory = ${directory:bin}
@@ -95,7 +102,7 @@ ip = ${publish:ip}
 ssl = {{ dumps(bool(slapparameter_dict['ssl'])) }}
 cluster = {{ dumps(slapparameter_dict['cluster']) }}
 masters = ${publish:masters}
-database-adapter = MySQL
+database-adapter = {{ storage_type }}
 wait-database = -1
 engine = {{ slapparameter_dict.get('engine', '') }}
 dedup = {{ dumps(bool(slapparameter_dict.get('data-deduplication'))) }}
@@ -107,8 +114,15 @@ disable-drop-partitions = {{ dumps(bool(slapparameter_dict.get('disable-drop-par
 < = neo-storage
 wrapper = ${directory:etc_run}/{{ 'neostorage-' ~ i }}
 logfile = ${directory:log}/{{ 'neostorage-' ~ i }}.log
-{%   do init_list.append('CREATE DATABASE IF NOT EXISTS neo' ~  i ~ ';') -%}
+{%- if mysql %}
+{%-   do init_list.append('CREATE DATABASE IF NOT EXISTS neo' ~  i ~ ';') %}
 database-parameters = root@neo{{ i }}${my-cnf-parameters:socket}
+{%- else %}
+database-parameters = ${directory:db-{{i}}}/db.sqlite
+
+[directory]
+db-{{i}} = ${buildout:directory}/srv/{{ storage_id }}
+{%- endif %}
 
 [{{ section('logrotate-storage-' ~ i) }}]
 < = logrotate-entry-base
@@ -117,12 +131,6 @@ log = {{ '${' +  storage_id + ':logfile}' }}
 post = {{ bin_directory }}/slapos-kill -s RTMIN+1 -- {{ bin_directory }}/neostorage -l ${:log}
 
 {% endfor -%}
-[init-script]
-recipe = slapos.recipe.template:jinja2
-# XXX: is there a better location ?
-rendered = ${directory:etc}/mariadb_initial_setup.sql
-template = inline:
-  {{ init_list | join('\n\t') }}
 
 [directory]
 recipe = slapos.cookbook:mkdirectory
@@ -132,11 +140,19 @@ etc = ${buildout:directory}/etc
 var = ${buildout:directory}/var
 etc_run =  ${:etc}/run
 var_run =  ${:var}/run
-srv_mariadb = ${buildout:directory}/srv/mariadb
 log = ${buildout:directory}/var/log
 tmp = ${buildout:directory}/tmp
+{% if mysql -%}
+srv_mariadb = ${buildout:directory}/srv/mariadb
+
+[init-script]
+recipe = slapos.recipe.template:jinja2
+# XXX: is there a better location ?
+rendered = ${directory:etc}/mariadb_initial_setup.sql
+template = inline:
+  {{ init_list | join('\n\t') }}
 
-[logrotate-mysql]
+[{{ section('logrotate-mysql') }}]
 < = logrotate-entry-base
 name = mariadb
 log = ${my-cnf-parameters:error-log} ${my-cnf-parameters:slow-query-log}
@@ -154,7 +170,9 @@ context =
     section my_cnf_parameters my-cnf-parameters
     raw     bin_directory     {{ bin_directory }}
     raw     prepend_path      ${mysqld:mysql-base-directory}/bin
-{% endif -%}
+{%- endif %}
+
+{%- endif %}
 
 [buildout]
 extends =
@@ -167,4 +185,3 @@ extends =
 {%- endif %}
 parts +=
   {{ '\n  '.join(part_list) }}
-  logrotate-mysql

software/neoppod/instance.cfg.in

@@ -12,14 +12,16 @@ extra-context =
 import-list =
     rawfile root_common {{ root_common }}
 
-[neo-storage-mysql]
+{% if mariadb_location is defined -%}
+[neo]
 extra-context +=
     raw runTestSuite_in {{ runTestSuite_in }}
 
+{% endif -%}
 [switch-softwaretype]
 recipe = slapos.cookbook:switch-softwaretype
 override = {{ dumps(override_switch_softwaretype |default) }}
 default = neo-cluster:rendered
 # BBB
 RootSoftwareInstance = ${:default}
-neo = neo-storage-mysql:rendered
+neo = neo:rendered

software/neoppod/software-common.cfg

@@ -41,10 +41,10 @@ environment = neoppod-setup-env
 
 [neoppod]
 recipe = zc.recipe.egg
-eggs = neoppod[admin, ctl, master, storage-mysqldb]
+eggs = neoppod[admin, ctl, master]
   ${cython-zstd:egg}
   ${msgpack-python:egg}
-  ${python-mysqlclient:egg}
+  ${:adapter-egg}
   psutil
   BTrees
   ZODB
@@ -52,6 +52,7 @@ eggs = neoppod[admin, ctl, master, storage-mysqldb]
   zodbtools
   coverage
   setproctitle
+adapter-egg = ${python-mysqlclient:egg}
 patch-binary = ${patch:location}/bin/patch
 ZEO-patch-options = -p1
 ZEO-patches =
@@ -81,16 +82,18 @@ mode = 644
 recipe = slapos.recipe.template:jinja2
 template = ${:_profile_base_location_}/${:_buildout_section_name_}.cfg.in
 rendered = ${buildout:directory}/${:_buildout_section_name_}.cfg
-md5sum = e8f5a83580e9791a32c66f5935cb6840
+md5sum = b867ba222a436807954f732642fb116d
 context =
     key bin_directory buildout:bin-directory
     key develop_eggs_directory buildout:develop-eggs-directory
     key eggs_directory buildout:eggs-directory
-    key mariadb_location mariadb:location
     key neo_admin instance-neo-admin:target
     key neo_master instance-neo-master:target
-    key neo_storage_mysql instance-neo-storage-mysql:target
+    key neo instance-neo:target
     key template_logrotate_base template-logrotate-base:rendered
+    ${:adapter-context}
+adapter-context =
+    key mariadb_location mariadb:location
     key template_neo_my_cnf template-neo-my-cnf:target
 
 [root-common]
@@ -105,9 +108,9 @@ md5sum = 4d1ae570b4458e7725454857aabb37f6
 <= download-base-neo
 md5sum = 1fee10f02c2fa2a581e21878ca0fd704
 
-[instance-neo-storage-mysql]
+[instance-neo]
 <= download-base-neo
-md5sum = 16024af1cde9c0fcc7e8e05ec0854b30
+md5sum = a9e5cad9cdb5fd3f1ae7b1d534f967ae
 
 [template-neo-my-cnf]
 <= download-base-neo

software/neoppod/software.cfg

@@ -10,7 +10,7 @@ parts +=
 [template]
 recipe = slapos.recipe.template:jinja2
 template = ${:_profile_base_location_}/instance.cfg.in
-md5sum = aaf5da66d45d4c08cadb0cd1c5342c54
+md5sum = 83dc9faca482b2ddbd3d5fa968af7c33
 # XXX: "template.cfg" is hardcoded in instanciation recipe
 rendered = ${buildout:directory}/template.cfg
 context =
@@ -18,6 +18,7 @@ context =
     key instance_common_cfg instance-common:rendered
     key root_common root-common:target
     key runTestSuite_in runTestSuite.in:target
+    ${instance-common:adapter-context}
 
 [cluster]
 <= download-base-neo

software/neoppod/sqlite-mixin.cfg

+# To be extended after software-common.cfg if only SQLite backend is wanted.
+
+[neoppod]
+adapter-egg =
+
+[instance-common]
+adapter-context =