Make ERP5 instantiable with an external NEO storage

In order not to conflict with a future integration of NEO in ERP5:
- the input schema has a new parameter for external storages.
- zodb-software-type & zodb is used only for internal storages
  and only ZEO is supported.

NEO logging is also enabled for clients.

software/erp5/instance-erp5-input-schema.json

@@ -33,7 +33,7 @@
       "description": "List of logins which should get the Developper role (required to modify portal_components' content)",
       "default": "inituser-login's value",
       "items": {
-        "pattern": "/^\S+$/",
+        "pattern": "/^\\S+$/",
         "type": "string"
       },
       "uniqueItems": true,
@@ -190,16 +190,54 @@
     },
     "zodb-software-type": {
       "description": "Zope Object DataBase implementation",
-      "default": "zeo",
-      "enum": ["zeo", "neo"],
+      "enum": ["zeo"],
       "type": "string"
     },
     "zodb": {
       "description": "Zope Object DataBase service",
       "oneOf": [
-        {"$ref": "./instance-neo-input-schema.json#properties"},
         {"$ref": "./instance-zeo-input-schema.json#properties"}
       ]
+    },
+    "zodb-extern": {
+      "description": "External ZODB Mountpoints. See https://github.com/zopefoundation/ZODB/blob/3.10/src/ZODB/component.xml for extra options.",
+      "minItems": 0,
+      "items": {
+        "required": ["storage_dict"],
+        "properties": {
+          "name": {
+            "description": "Database name",
+            "default": "main",
+            "type": "string"
+          },
+          "mount_point": {
+            "description": "Mount point",
+            "default": "/",
+            "type": "string"
+          },
+          "storage-type": {
+            "description": "Storage type",
+            "default": "NEOStorage",
+            "type": "string"
+          },
+          "storage-dict": {
+            "description": "Storage configuration. For NEOStorage, 'logfile' is automatically set (see http://git.erp5.org/gitweb/neoppod.git/blob/HEAD:/neo/client/component.xml for other settings).",
+            "patternProperties": {
+              ".": {
+                "type": "string"
+              }
+            },
+            "type": "object"
+          },
+          "patternProperties": {
+            ".": {
+              "type": "string"
+            }
+          }
+        },
+        "type": "object"
+      },
+      "type": "array"
     }
   }
 }

software/erp5/instance-neo-input-schema.json

-{
-  "$schema": "http://json-schema.org/draft-04/schema#",
-  "extends": "./schema-definitions.json#",
-  "required": ["zodb-dict"],
-  "properties": {
-    "zodb-dict": {
-      "description": "Maps an export identifier to its settings",
-      "patternProperties": {
-        ".*": {
-          "properties": {
-            "mount-point": {
-              "description": "Traversal path export should be mounted at (opaque to this software type)",
-              "default": "/",
-              "type": "string"
-            },
-            "cache-size": {
-              "description": "Client-side cache size in object count (opaque to this software type)",
-              "default": -1,
-              "type": "integer"
-            },
-            "storage-dict": {
-              "default": {},
-              "required": ["name", "master_nodes"],
-              "properties": {
-                "name": {
-                  "description": "NEO cluster's name",
-                  "type": "string"
-                },
-                "master_nodes": {
-                  "description": "Client-side settings for this mountpoint",
-                  "minItems": 1,
-                  "items": {
-                    "type": "string"
-                  },
-                  "type": "array"
-                }
-              },
-              "type": "object"
-            }
-          },
-          "type": "object"
-        }
-      },
-      "type": "object"
-    }
-  }
-}

software/erp5/instance-neo-output-schema.json

-{
-  "$schema": "http://json-schema.org/draft-04/schema#",
-  "description": "Values returned by ZEO instanciation",
-  "properties": {
-    "zodb-storage-type": {
-      "description": "zope.cfg-friendly storage type",
-      "enum": ["NEOStorage"],
-      "type": "string"
-    },
-    "zodb-dict": {
-      "description": "Maps an export identifier to values received as parameters",
-      "patternProperties": {
-        ".*": {
-          "description": "Contains the mount-point, cache-size and client-dict received as parameters for current export identifier (client-dict's 'storage' and 'server' entries are overwritten with actual values)",
-          "type": "array"
-        }
-      },
-      "type": "object"
-    },
-  },
-  "type": "object"
-}

stack/erp5/buildout.cfg

@@ -183,7 +183,7 @@ md5sum = 838e56f46ba749dfc53b8e69e21be008
 [template-zope-conf]
 < = download-base
 filename = zope.conf.in
-md5sum = 8fe36c41ab784f547b968dc6edd0ca29
+md5sum = 9e68f6a221c16e654efb05632fed5aee
 
 [template-runzope-userhosts-preloaded]
 < = download-base
@@ -215,7 +215,7 @@ md5sum = e40e2e39f4941c6372f4357e8589a5cf
 # XXX: "template.cfg" is hardcoded in instanciation recipe
 filename = template.cfg
 template = ${:_profile_base_location_}/instance.cfg.in
-md5sum = 5e303e59c6674c98944120d6f1d01c28
+md5sum = 99159fc1e32a0651b35f824709d561d0
 extra-context =
     key mariadb_link_binary template-mariadb:link-binary
     key zope_link_binary template-zope:link-binary
@@ -270,7 +270,6 @@ extra-context =
     key template_mariadb template-mariadb:target
     key template_mariadb_initial_setup template-mariadb-initial-setup:target
     key template_my_cnf template-my-cnf:target
-    key template_neo template-neo:target
     key template_runzope_userhosts_preloaded template-runzope-userhosts-preloaded:target
     key template_zeo template-zeo:target
     key template_zope template-zope:target
@@ -282,12 +281,7 @@ extra-context =
 [template-erp5]
 < = download-base
 filename = instance-erp5.cfg.in
-md5sum = ef382581a0b8f2ad01ffc523a59bfe66
-
-[template-neo]
-< = download-base
-filename = instance-neo.cfg.in
-md5sum = 7dcac2eae332e2def17579888c045320
+md5sum = 2ac3bbeb031eb74fbd6f9d033c71b677
 
 [template-zeo]
 < = download-base
@@ -297,7 +291,7 @@ md5sum = 651fa2ec2feb768e339936b7855774ca
 [template-zope]
 < = download-base
 filename = instance-zope.cfg.in
-md5sum = e128a47f37ba78dc7154daa80c2dd272
+md5sum = 7f185192e0536ecacc403cf6b1736d2d
 link-binary =
   ${aspell:location}/bin/aspell
   ${coreutils:location}/bin/basename
@@ -494,6 +488,7 @@ eggs =
 # Needed for checking ZODB Components source code
   pylint
 
+  neoppod[client]
 # Zope
   ZODB3
   Zope2
@@ -539,6 +534,9 @@ eggs =
 # installation of python, which we don't want on an instance
 interpreter = ${buildout:python}
 scripts =
+  neoctl
+  neolog
+  neomigrate
   repozo
   runzope
   runzeo

stack/erp5/instance-erp5.cfg.in

@@ -43,8 +43,13 @@ config-{{ option }} = {{ dumps(value) }}
 {{ request('cloudooo', 'cloudooo', 'cloudooo', {'tcpv4-port': 2020}) }}
 {{ request('mariadb', 'mariadb', 'mariadb', {'tcpv4-port': 2099}, {'database-list': True, 'test-database-list': True}) }}
 {# Fail early if an unexpected value is provided -#}
-{% set possible_software_type_dict = {'zeo': 'zeo', 'neo': 'neo'} -%}
-{{ request('zodb', 'zodb-' ~ possible_software_type_dict[slapparameter_dict.get('zodb-software-type', 'zeo')], 'zodb', {'tcpv4-port': 2100, 'zodb-dict': {'root': {}}}, {'zodb-storage-type': False, 'zodb-dict': False, 'tidstorage-ip': False, 'tidstorage-port': False}) }}
+{% set zodb_type = slapparameter_dict.get('zodb-software-type') -%}
+{% set zodb_extern = slapparameter_dict.get('zodb-extern') -%}
+{% if {'zeo': 1, None: 0}[zodb_type] -%}
+{{   request('zodb', 'zodb-' ~ zodb_type, 'zodb', {'tcpv4-port': 2100, 'zodb-dict': {'root': {}}}, {'zodb-storage-type': False, 'zodb-dict': False, 'tidstorage-ip': False, 'tidstorage-port': False}) }}
+{% else -%}
+{% do zodb_extern[0] %}
+{% endif -%}
 
 [inituser-password]
 {% set inituser_password = slapparameter_dict.get('inituser-password') -%}
@@ -67,7 +72,7 @@ recipe = slapos.cookbook:generate.password
 return =
   zope-address-list
   hosts-dict
-extra-config =
+{% set extra_config = '
   bt5
   bt5-repository-url
   cloudooo-url
@@ -88,13 +93,9 @@ extra-config =
   site-id
   smtp-url
   thread-amount
-  tidstorage-ip
-  tidstorage-port
   timerserver-interval
   timezone
-  webdav
-  zodb-dict
-  zodb-storage-type
+  webdav' -%}
 config-bt5 = {{ dumps(slapparameter_dict.get('bt5', 'erp5_full_text_myisam_catalog erp5_configurator_standard erp5_configurator_maxma_demo erp5_configurator_ung erp5_configurator_run_my_doc')) }}
 config-bt5-repository-url = {{ dumps(slapparameter_dict.get('bt5-repository-url', local_bt5_repository)) }}
 config-cloudooo-url = ${request-cloudooo:connection-url}
@@ -109,12 +110,25 @@ config-mysql-test-url-list = ${request-mariadb:connection-test-database-list}
 config-mysql-url-list = ${request-mariadb:connection-database-list}
 config-site-id = {{ dumps(site_id) }}
 config-smtp-url = {{ dumps(slapparameter_dict.get('smtp-url', 'smtp://localhost:25/')) }}
+config-timezone = {{ dumps(slapparameter_dict.get('timezone', 'UTC')) }}
+{% if zodb_type -%}
 config-tidstorage-ip = ${request-zodb:connection-tidstorage-ip}
 config-tidstorage-port = ${request-zodb:connection-tidstorage-port}
-config-timezone = {{ dumps(slapparameter_dict.get('timezone', 'UTC')) }}
 config-zodb-dict = ${request-zodb:connection-zodb-dict}
 config-zodb-storage-type = ${request-zodb:connection-zodb-storage-type}
+{% set extra_config = extra_config ~ '
+  tidstorage-ip
+  tidstorage-port
+  zodb-dict
+  zodb-storage-type' -%}
+{% endif -%}
+{% if zodb_extern -%}
+config-zodb-extern = {{ dumps(zodb_extern) }}
+{% set extra_config = extra_config ~ '
+  zodb-extern' -%}
+{% endif -%}
 software-type = zope
+extra-config = {{ extra_config }}
 
 {% set zope_family_dict = {} -%}
 {% for custom_name, zope_parameter_dict in slapparameter_dict.get('zope-partition-dict', {'1': {}}).items() -%}

stack/erp5/instance-neo.cfg.in

-{% set zodb_dict = {} -%}
-{% for export_id, mountpoint_dict in slapparameter_dict['zodb-dict'].items() -%}
-{%   do zodb_dict.__setitem__(export_id, [
-       mountpoint_dict.get('mount-point', '/'),
-       mountpoint_dict.get('cache-size', -1),
-       mountpoint_dict.get('storage-dict', {}),
-     ]) -%}
-{% endfor -%}
-[publish]
-recipe = slapos.cookbook:publish.serialised
-zodb-storage-type = NEOStorage
-zodb-dict = {{ dumps(zodb_dict) }}
-
-[buildout]
-parts +=
-  publish

stack/erp5/instance-zope.cfg.in

@@ -5,7 +5,7 @@
 {% set node_id_base = slapparameter_dict['name'] -%}
 {% set part_list = [] -%}
 {% set publish_list = [] -%}
-{% set zodb_dict = slapparameter_dict['zodb-dict'] -%}
+{% set zodb_dict = slapparameter_dict.get('zodb-dict', {}) -%}
 {% set longrequest_logger_base_path = buildout_directory ~ '/var/log/longrequest_logger_' -%}
 {% macro section(name) %}{% do part_list.append(name) %}{{ name }}{% endmacro -%}
 {% set bin_directory = parameter_dict['buildout-bin-directory'] -%}
@@ -225,11 +225,13 @@ bt5-repository =
 [zope-conf-parameter-base]
 ip = {{ ipv4 }}
 site-id = {{ site_id }}
-{% set zodb_list = [] -%}
+{% set zodb_list = slapparameter_dict.get('zodb-extern', []) -%}
 {% for key, (mount_point, cache_size, storage_dict) in zodb_dict.items() -%}
-{%   do zodb_list.append([key, mount_point, cache_size, storage_type, storage_dict]) -%}
+{%   do zodb_list.append({'name': key, 'mount-point': mount_point, 'storage-type': storage_type, 'storage-dict': storage_dict}) -%}
+{%   if cache_size >= 0 -%}
+{%     do zodb_list[-1].__setitem__('cache-size', cache_size) -%}
+{%   endif -%}
 {% endfor -%}
-zodb-list = {{ dumps(zodb_list) }}
 developer-list = {{ dumps(slapparameter_dict['developer-list']) }}
 instance = ${directory:instance}
 instance-products = ${directory:instance-products}
@@ -277,6 +279,18 @@ port = {{ port }}
 event-log = ${directory:log}/{{ name }}-event.log
 z2-log = ${directory:log}/{{ name }}-Z2.log
 node-id = {{ dumps(node_id_base ~ '-' ~ index) }}
+{% set log_list = [] -%}
+{% set import_set = {} -%}
+{% for zodb in zodb_list -%}
+{%   if zodb.setdefault('storage-type', 'NEOStorage') == 'NEOStorage' -%}
+{%     do import_set.__setitem__('neo.client', None) -%}
+{%     set log = buildout_directory ~ '/var/log/' ~ name ~ '-neo-' ~ zodb.get('name', 'main') ~ '.log' -%}
+{%     do log_list.append(log) -%}
+{%     do zodb.setdefault('storage-dict', {}).__setitem__('logfile', log) -%}
+{%   endif -%}
+{% endfor -%}
+import-list = {{ dumps(import_set.keys()) }}
+zodb-list = {{ dumps(zodb_list) }}
 
 [{{ conf_name }}]
 < = zope-conf-base
@@ -324,7 +338,7 @@ path = ${directory:promises}/{{ zope_tunnel_base_name }}
 [{{ section('logrotate-entry-' ~ name) }}]
 < = logrotate-entry-base
 name = {{ name }}
-log = {{ '${' ~ conf_parameter_name ~ ':event-log}' }} {{ '${' ~ conf_parameter_name ~ ':z2-log}' }} {{ '${' ~ name ~ ':longrequest-logger-file}' }}
+log = {{ '${' ~ conf_parameter_name ~ ':event-log}' }} {{ '${' ~ conf_parameter_name ~ ':z2-log}' }} {{ '${' ~ name ~ ':longrequest-logger-file}' }} {{ ' '.join(log_list) }}
 post = {{ bin_directory }}/slapos-kill --pidfile {{ '${' ~ conf_parameter_name ~ ':pid-file}' }} -s USR2
 {% endmacro -%}
 

stack/erp5/instance.cfg.in

@@ -85,12 +85,6 @@ filename = instance-balancer.cfg
 extra-context =
     section parameter_dict dynamic-template-balancer-parameters
 
-[dynamic-template-neo]
-< = jinja2-template-base
-template = {{ template_neo }}
-filename = instance-neo.cfg
-extensions = jinja2.ext.do
-
 [dynamic-template-zeo-parameters]
 buildout-bin-directory = {{ buildout_bin_directory }}
 6tunnel = {{ sixtunnel_location }}

stack/erp5/zope.conf.in

@@ -100,13 +100,18 @@ products {{ parameter_dict['instance-products'] }}
   developers {{ developer_list | join(' ') }}
 </ERP5Type>
 
-{%- endif %}
-{% for db_name, mount_point, cache_size, storage_type, storage_dict in parameter_dict['zodb-list'] -%}
-<zodb_db {{ db_name }}>
-{%-   if cache_size >= 0 %}
-  cache-size {{ cache_size }}
-{%-   endif %}
-  mount-point {{ mount_point }}
+{% endif -%}
+{% for m in parameter_dict['import-list'] -%}
+%import {{ m }}
+{% endfor -%}
+{% for zodb_dict in parameter_dict['zodb-list'] %}
+<zodb_db {{ zodb_dict.pop('name', 'main') }}>
+{%- set storage_type = zodb_dict.pop('storage-type', 'NEOStorage') %}
+{%- set storage_dict = zodb_dict.pop('storage-dict', {}) %}
+  mount-point {{ zodb_dict.pop('mount-point', '/') }}
+{%- for key, value in zodb_dict.iteritems() %}
+  {{ key }} {{ value }}
+{%-   endfor %}
   <{{ storage_type }}>
 {%-   for key, value in storage_dict.iteritems() %}
     {{ key }} {{ value }}