wip

software/neoppod/buildout.hash.cfg

@@ -14,11 +14,11 @@
 # not need these here).
 [instance-common]
 filename = instance-common.cfg.in
-md5sum = b4baf7f21f450fa522c2a69f5a4aedf7
+md5sum = 0f729dadc9bda3aa105b56f864926518
 
 [root-common]
 filename = root-common.cfg.in
-md5sum = 102a7f1c1bc46a9b3fa5bd9b9a628e1d
+md5sum = 67ebd1b9520d6af2f071cff6c0f722ed
 
 [instance-neo-admin]
 filename = instance-neo-admin.cfg.in
@@ -30,7 +30,7 @@ md5sum = 9f27195d770b2f57461c60a82c851ab9
 
 [instance-neo]
 filename = instance-neo.cfg.in
-md5sum = 53b2c73a0c5c7cb5f6b841ba6892fa46
+md5sum = 0eaaa46684e86c665ea85179e389b995
 
 [template-neo-my-cnf]
 filename = my.cnf.in

software/neoppod/instance-common.cfg.in

@@ -44,6 +44,7 @@ extra-context =
     import urllib urllib
     key master_cfg neo-master:output
     key admin_cfg neo-admin:output
+    raw neoppod_location {{ neoppod_location }}
     raw sqlite3_location {{ sqlite3_location }}
 {%- if mariadb_location is defined %}
     raw mariadb_location {{ mariadb_location }}

software/neoppod/instance-neo-input-schema.json

@@ -28,6 +28,49 @@
           "description": "Master nodes in the cluster to backup.",
           "type": "string"
         },
+        "reflink": {
+          "description": "Track references between OIDs of an external ZODB.",
+          "additionalProperties": false,
+          "required": [
+            "zurl"
+          ],
+          "properties": {
+            "zurl": {
+              "description": "ZODB to track.",
+              "type": "string"
+            },
+            "_ca": {
+              "type": "string"
+            },
+            "_cert": {
+              "type": "string"
+            },
+            "_key": {
+              "type": "string"
+            },
+            "max-txn-size": {
+              "description": "Maximum number of OIDs to delete per transaction.",
+              "default": 2097151,
+              "type": "integer"
+            },
+            "commit-interval": {
+              "description": "Commit every SECONDS of work.",
+              "default": 10,
+              "type": "integer"
+            },
+            "period": {
+              "description": "Age of the historical revision at which a GC is performed (this can be seen as a grace period). See --period option.",
+              "default": 86400,
+              "type": "integer"
+            },
+            "no-gc": {
+              "description": "Only track references.",
+              "default": false,
+              "type": "boolean"
+            },
+          },
+          "type": "object"
+        },
         "monitor": {
           "description": "Parameters for monitoring.",
           "properties": {
@@ -124,7 +167,7 @@
               "type": "array"
             },
             "storage-type": {
-              "description": "Storage type. Required when several types are configured and you select which one to use via 'node!' parameter. Defaults to whatever is configured ('sqlite' or 'mysql'), else MySQL if available, else SQLite.",
+              "description": "Storage type. Required when several types are configured and you select which one to use via 'node!' parameter. Defaults to whatever is configured ('sqlite' or 'mysql'), else MySQL if available, else SQLite. SQLite is recommended for a reflink DB.",
               "enum": [
                 "MySQL",
                 "SQLite"

software/neoppod/instance-neo.cfg.in

@@ -194,6 +194,44 @@ post = {{ bin_directory }}/slapos-kill -s RTMIN+1 -- {{ bin_directory }}/neostor
 
 {% endfor -%}
 
+{% set reflink = slapparameter_dict.get('reflink') -%}
+{% if reflink -%}
+{%   set zurl = 'neo://%s@%s' % (slapparameter_dict['cluster'], slapparameter_dict['masters'].replace(' ', ',')) -%}
+{%   if slapparameter_dict['ssl'] -%}
+{%     set zurl = zurl + '?ca=${ca.crt:output}&cert=${neo.crt:output}&key=${neo.key:output}' -%}
+{%   endif -%}
+[{{ section('reflink') }}]
+recipe = slapos.cookbook:wrapper
+wrapper-path = ${directory:etc_run}/reflink
+{%   set args = [] -%}
+{%   if 'commit-interval' in reflink -%}
+{%     do args.extends(('-i', reflink['commit-interval'])) -%}
+{%   endif -%}
+{%   if 'period' in reflink -%}
+{%     do args.extends(('-p', reflink['period'])) -%}
+{%   endif -%}
+{%   if 'no-gc' in reflink -%}
+{%     do args.append('--no-gc') -%}
+{%   endif -%}
+command-line = {{ bin_directory }}/neopy {{ neoppod_location }}/tools/reflink -v {{ zurl }} run {{ ' '.join(args) }} {{ reflink['zurl'] }}
+{%   if reflink['zurl'].startswith('neos://') -%}
+environment =
+  NEO_CA=${reflink-ca:output}
+  NEO_CERT=${reflink-cert:output}
+  NEO_KEY=${reflink-key:output}
+
+{%     for x in 'ca', 'cert', 'key' -%}
+[reflink-{{x}}]
+recipe = slapos.recipe.template:jinja2
+output = ${directory:etc}/${:_buildout_section_name_}.pem
+inline =
+  {{'{{'}}pem}}
+context = key pem :pem
+pem = {{dumps(reflink['_'+x])}}
+{%     endfor -%}
+{%   endif -%}
+{% endif -%}
+
 {% if mysql -%}
 [init-script]
 recipe = slapos.recipe.template

software/neoppod/root-common.cfg.in

@@ -132,6 +132,9 @@ config-monitor-port = {{ dumps(port) }}
 {%-     endif %}
 config-monitor = {{ dumps(parameter_dict.get('monitor', {})) }}
 {%-   endif %}
+{%- if 'reflink' in parameter_dict %}
+config-reflink = {{ dumps(parameter_dict.pop('reflink')) }}
+{%- endif %}
 {%-   for k, v in six.iteritems(node) %}
 config-{{ k }} = {{ dumps(v) }}
 {%-   endfor %}

software/neoppod/software-common.cfg

@@ -65,6 +65,7 @@ eggs = neoppod[admin, ctl, master]
 adapter-egg =
   ${python-mysqlclient:egg}
   PyMySQL
+interpreter = neopy
 
 [download-base-neo]
 recipe = slapos.recipe.build:download
@@ -79,6 +80,7 @@ context =
     key bin_directory buildout:bin-directory
     key develop_eggs_directory buildout:develop-eggs-directory
     key eggs_directory buildout:eggs-directory
+    key neoppod_location neoppod-develop:setup
     key neo_admin instance-neo-admin:target
     key neo_master instance-neo-master:target
     key neo instance-neo:target

stack/erp5-zope2/buildout.hash.cfg

@@ -74,7 +74,7 @@ md5sum = 3f7b28085ceff321a3cb785db60f7c3e
 
 [template-erp5]
 filename = instance-erp5.cfg.in
-md5sum = b056ec4fec956c2a07befd2fec116f04
+md5sum = bc7d1e5f9cab67d42e7c1dfb5810f521
 
 [template-zeo]
 filename = instance-zeo.cfg.in

stack/erp5-zope2/instance-erp5.cfg.in

@@ -186,6 +186,7 @@ connection-url = smtp://127.0.0.2:0/
 {%     set ((name, server_dict),) = server_dict.items() -%}
 {%     do neo.append(server_dict.get('cluster')) -%}
 {%     do server_dict.update(cluster='${publish-early:neo-cluster}') -%}
+{{     assert('reflink' not in server_dict, 'reflink option is meaningless in ERP5 SR') }}
 {{     root_common.request_neo(server_dict, 'zodb-neo', 'neo-', monitor_base_url_dict) }}
 {%     set client_dict = zodb_dict[name].setdefault('storage-dict', {}) -%}
 {%     for k in 'ssl', '_ca', '_cert', '_key' -%}

stack/erp5/buildout.hash.cfg

@@ -74,7 +74,7 @@ md5sum = ca0cb83950dd9079cc289891cce08e76
 
 [template-erp5]
 filename = instance-erp5.cfg.in
-md5sum = d6f7d2fa1bde019892897c767f93e089
+md5sum = d636781ffe57cb5321223fc4e3a1d003
 
 [template-zeo]
 filename = instance-zeo.cfg.in

stack/erp5/instance-erp5.cfg.in

@@ -183,6 +183,7 @@ connection-url = smtp://127.0.0.2:0/
 {%     set ((name, server_dict),) = server_dict.items() -%}
 {%     do neo.append(server_dict.get('cluster')) -%}
 {%     do server_dict.update(cluster='${publish-early:neo-cluster}') -%}
+{{     assert('reflink' not in server_dict, 'reflink option is meaningless in ERP5 SR') }}
 {{     root_common.request_neo(server_dict, 'zodb-neo', 'neo-', monitor_base_url_dict) }}
 {%     set client_dict = zodb_dict[name].setdefault('storage-dict', {}) -%}
 {%     for k in 'ssl', '_ca', '_cert', '_key' -%}