X stack/erp5 += WCFS service (draft)

This complements 7f877621 (Move wendelin.core from Wendelin to ERP5) for
wendelin.core 2 which introduces new filesystem server component that serves
in-ZODB arrays data as OS files for Zopes to memory-map.

- WCFS is by default disabled in ERP5 - we do not want to load every
  ERP5 user with it in the early days of wendelin.core 2.
- WCFS is by default enabled in Wendelin and SlapOS Master.
- There is a duplication for instance-erp5.cfg.in in between stack/erp5/
  and software/slapos-master/ . It was negotiated with VIFIB developers
  to go this way instead of first deduplicating instance-erp5.cfg.in
  copying that was started in 87d13789 (slapos-master: Copy files from
  erp5 stack for local changes).

software/slapos-master/buildout.hash.cfg

@@ -14,7 +14,7 @@
 # not need these here).
 [template-erp5]
 filename = instance-erp5.cfg.in
-md5sum = f078d72db690805eab90a4577066b920
+md5sum = 0d0281b3e197107f40dca0b73e0ef21f
 
 [template-balancer]
 filename = instance-balancer.cfg.in

software/slapos-master/instance-erp5.cfg.in

@@ -9,6 +9,8 @@
 {% set jupyter_dict = slapparameter_dict.get('jupyter', {}) -%}
 {% set has_jupyter = jupyter_dict.get('enable', jupyter_enable_default.lower() in ('true', 'yes')) -%}
 {% set jupyter_zope_family = jupyter_dict.get('zope-family', '') -%}
+{% set wcfs_dict = slapparameter_dict.get('wcfs', {}) -%}
+{% set wcfs_enable = wcfs_dict.get('enable', wcfs_enable_default.lower() in ('true', 'yes')) -%}
 {% set test_runner_enabled = slapparameter_dict.get('test-runner', {}).get('enabled', True) -%}
 {% set test_runner_node_count = slapparameter_dict.get('test-runner', {}).get('node-count', 3) -%}
 {% set test_runner_extra_database_count = slapparameter_dict.get('test-runner', {}).get('extra-database-count', 3) -%}
@@ -247,6 +249,7 @@ config-tidstorage-ip = ${request-zodb:connection-tidstorage-ip}
 config-tidstorage-port = ${request-zodb:connection-tidstorage-port}
 {%   endif -%}
 {% endfor -%}
+config-wcfs_enable = {{ wcfs_enable }}
 software-type = zope
 
 {% set global_publisher_timeout = slapparameter_dict.get('publisher-timeout') -%}
@@ -348,6 +351,28 @@ config-url = ${request-jupyter:connection-url}
 {%   endif -%}
 {%- endif %}
 
+{% if wcfs_enable -%}
+{# request WCFS connected to ZODB -#}
+{%   do root_common.section('request-wcfs') -%}
+{{   request('wcfs', 'wcfs', 'wcfs', {}, {}) }}
+
+[request-wcfs]
+# XXX dup wrt request-zope-base
+config-zodb-dict = {{ dumps(zodb_dict) }}
+{% for server_type, server_dict in storage_dict.iteritems() -%}
+{%   if server_type == 'neo' -%}
+config-neo-cluster = ${publish-early:neo-cluster}
+config-neo-name = {{ server_dict.keys()[0] }}
+config-neo-masters = ${publish-early:neo-masters}
+{%   else -%}
+config-zodb-zeo = ${request-zodb:connection-storage-dict}
+config-tidstorage-ip = ${request-zodb:connection-tidstorage-ip}
+config-tidstorage-port = ${request-zodb:connection-tidstorage-port}
+{%   endif -%}
+{% endfor -%}
+
+{%- endif %}
+
 [directory]
 recipe = slapos.cookbook:mkdirectory
 {% if slapparameter_dict.get('shared-certificate-authority-path', '') -%}

software/slapos-master/software.cfg

@@ -47,6 +47,11 @@ extra-paths +=
   ${slapos-bin:location}
 
 
+# WCFS is enabled by default.
+[erp5-defaults]
+wcfs-enable-default = true
+
+
 ### Overwrite recipes to introduce customized changes
 [download-base-part]
 recipe = slapos.recipe.build:download

software/wendelin/software.cfg

@@ -62,9 +62,10 @@ repository_id_list += wendelin
 [local-bt5-repository]
 list += ${wendelin:location}/bt5
 
-# Jupyter is by default enabled in Wendelin
+# Jupyter and WCFS are by default enabled in Wendelin
 [erp5-defaults]
 jupyter-enable-default = true
+wcfs-enable-default = true
 
 [wendelin]
 recipe = slapos.recipe.build:gitclone

stack/erp5/buildout.cfg

@@ -187,6 +187,7 @@ context =
     key instance_common_cfg instance-common:rendered
     key jsl_location jsl:location
     key jupyter_enable_default erp5-defaults:jupyter-enable-default
+    key wcfs_enable_default erp5-defaults:wcfs-enable-default
     key kumo_location kumo:location
     key local_bt5_repository local-bt5-repository:list
     key logrotate_location logrotate:location
@@ -224,6 +225,7 @@ context =
     key template_postfix_aliases template-postfix-aliases:target
     key template_postfix_main_cf template-postfix-main-cf:target
     key template_postfix_master_cf template-postfix-master-cf:target
+    key instance_wcfs_cfg_in instance-wcfs.cfg.in:target
     key template_zeo template-zeo:target
     key template_zope template-zope:target
     key template_zope_conf template-zope-conf:target
@@ -282,6 +284,9 @@ fontconfig-includes =
 [template-rsyslogd-cfg]
 <= download-base
 
+[instance-wcfs.cfg.in]
+<= download-base
+
 [erp5-bin]
 <= erp5
 repository = https://lab.nexedi.com/nexedi/erp5-bin.git
@@ -321,6 +326,8 @@ repository_id_list = erp5 erp5-bin erp5-doc
 cloudooo-connection-url = https://cloudooo.erp5.net/
 # Jupyter is by default disabled in ERP5
 jupyter-enable-default = false
+# WCFS is by default disabled in ERP5
+wcfs-enable-default = false
 
 [erp5]
 recipe = slapos.recipe.build:gitclone
@@ -559,6 +566,7 @@ scripts =
   runzeo
   tidstoraged
   tidstorage_repozo
+  wcfs
   web_checker_utility
 
 extra-paths =

stack/erp5/buildout.hash.cfg

@@ -70,11 +70,11 @@ md5sum = 7a14019abf48ca100eb94d9add20f5ae
 
 [template]
 filename = instance.cfg.in
-md5sum = b5ac16fdeed8863e465e955ba6d1e12a
+md5sum = 95c4800c964649b8f1f454fc544dff95
 
 [template-erp5]
 filename = instance-erp5.cfg.in
-md5sum = ac155f28a096747fc267f32a1cec46e1
+md5sum = 4f5db6b5a91ad50c5e037980295d5fb2
 
 [template-zeo]
 filename = instance-zeo.cfg.in
@@ -82,7 +82,7 @@ md5sum = 0648e38bd5d3a15bb9f93264932740b9
 
 [template-zope]
 filename = instance-zope.cfg.in
-md5sum = 9fa66b93fbf6a40aa8136c651ad9f539
+md5sum = ccdb32b8f0069e19fda76c9a914e6ed1
 
 [template-balancer]
 filename = instance-balancer.cfg.in
@@ -95,3 +95,7 @@ md5sum = 452c502fabd5a6066c9dee533dfb1c77
 [template-rsyslogd-cfg]
 filename = rsyslogd.cfg.in
 md5sum = 7030e42b50e03f24e036b7785bd6159f
+
+[instance-wcfs.cfg.in]
+filename = instance-wcfs.cfg.in
+md5sum = f4541f45d15009d840ff0537fdcc4c14

stack/erp5/instance-erp5.cfg.in

@@ -9,6 +9,8 @@
 {% set jupyter_dict = slapparameter_dict.get('jupyter', {}) -%}
 {% set has_jupyter = jupyter_dict.get('enable', jupyter_enable_default.lower() in ('true', 'yes')) -%}
 {% set jupyter_zope_family = jupyter_dict.get('zope-family', '') -%}
+{% set wcfs_dict = slapparameter_dict.get('wcfs', {}) -%}
+{% set wcfs_enable = wcfs_dict.get('enable', wcfs_enable_default.lower() in ('true', 'yes')) -%}
 {% set test_runner_enabled = slapparameter_dict.get('test-runner', {}).get('enabled', True) -%}
 {% set test_runner_node_count = slapparameter_dict.get('test-runner', {}).get('node-count', 3) -%}
 {% set test_runner_extra_database_count = slapparameter_dict.get('test-runner', {}).get('extra-database-count', 3) -%}
@@ -252,6 +254,7 @@ config-tidstorage-ip = ${request-zodb:connection-tidstorage-ip}
 config-tidstorage-port = ${request-zodb:connection-tidstorage-port}
 {%   endif -%}
 {% endfor -%}
+config-wcfs_enable = {{ wcfs_enable }}
 software-type = zope
 
 {% set global_publisher_timeout = slapparameter_dict.get('publisher-timeout') -%}
@@ -353,6 +356,28 @@ config-url = ${request-jupyter:connection-url}
 {%   endif -%}
 {%- endif %}
 
+{% if wcfs_enable -%}
+{# request WCFS connected to ZODB -#}
+{%   do root_common.section('request-wcfs') -%}
+{{   request('wcfs', 'wcfs', 'wcfs', {}, {}) }}
+
+[request-wcfs]
+# XXX dup wrt request-zope-base
+config-zodb-dict = {{ dumps(zodb_dict) }}
+{% for server_type, server_dict in storage_dict.iteritems() -%}
+{%   if server_type == 'neo' -%}
+config-neo-cluster = ${publish-early:neo-cluster}
+config-neo-name = {{ server_dict.keys()[0] }}
+config-neo-masters = ${publish-early:neo-masters}
+{%   else -%}
+config-zodb-zeo = ${request-zodb:connection-storage-dict}
+config-tidstorage-ip = ${request-zodb:connection-tidstorage-ip}
+config-tidstorage-port = ${request-zodb:connection-tidstorage-port}
+{%   endif -%}
+{% endfor -%}
+
+{%- endif %}
+
 {% set balancer_ret_dict = {'monitor-base-url': False} -%}
 {% for family in zope_family_dict -%}
 {%   do balancer_ret_dict.__setitem__(family, False) -%}

stack/erp5/instance-wcfs.cfg.in

+{# instance that runs WCFS service associated with ZODB storage  #}
+
+{% set zodb_dict = slapparameter_dict['zodb-dict'] -%}
+
+{# XXX dup from instance-zope.cfg.in #}
+{% set zeo_dict = slapparameter_dict.get('zodb-zeo', {}) -%}
+{% for name, zodb in zodb_dict.iteritems() -%}
+{%   set storage_dict = zodb.setdefault('storage-dict', {}) -%}
+{%   if zodb['type'] == 'zeo' -%}
+{%     do storage_dict.update(zeo_dict.get(name, ())) -%}
+{%   else -%}
+{%     if name == slapparameter_dict.get('neo-name') -%}
+{%       do storage_dict.update(master_nodes=slapparameter_dict['neo-masters'],
+                                name=slapparameter_dict['neo-cluster']) -%}
+{%     endif -%}
+{{     assert(storage_dict['master_nodes'], name) }}
+{%     if storage_dict.pop('ssl', 1) -%}
+{%       do storage_dict.update(ca='~/etc/ca.crt',
+                                cert='~/etc/neo.crt',
+                                key='~/etc/neo.key') -%}
+{%     endif -%}
+{%   endif -%}
+{% endfor -%}
+
+
+{# build zurl to connect to configured ZODB #}
+{% if len(zodb_dict) != 1 -%}
+{%   do assert(False, ("WCFS supports only single ZODB storage", zodb_dict)) -%}
+{% endif -%}
+
+{% set db_name, zodb = zodb_dict.popitem() -%}
+{% set z = zodb['storage-dict'] -%}
+{% if zodb['type'] == 'zeo' -%}
+{%   set zurl = ('zeo://%s?storage=%s' % (z['server'], z['storage'])) -%}
+{% elif zodb['type'] == 'neo' -%}
+{%   set zurl = ('neo://%s@%s' % (z.pop('name'), z.pop('master_nodes'))) -%}
+{%   set argv = [] -%}
+{%   set i = 0 -%}
+{%   for k,v in z|dictsort -%}
+{%     do argv.append('%s=%s' % (k,v)) -%}
+{%   endfor -%}
+{%   if len(argv) > 0 -%}
+{%     set zurl = zurl + '&' + '?'.join(argv) -%}
+{%   endif -%}
+{% else -%}
+{%   do assert(False, ("unsupported ZODB type", zodb)) -%}
+{% endif -%}
+
+
+[buildout]
+extends = {{ template_monitor }}
+parts +=
+  wcfs
+  wcfs-promise
+  publish
+
+
+[directory]
+recipe = slapos.cookbook:mkdirectory
+etc = ${buildout:directory}/etc
+log = ${:var}/log
+run = ${:var}/run
+services = ${:etc}/run
+service-on-watch = ${:etc}/service
+srv = ${buildout:directory}/srv
+tmp = ${buildout:directory}/tmp
+var = ${buildout:directory}/var
+
+[wcfs]
+recipe = slapos.cookbook:wrapper
+command-line = {{ bin_directory }}/wcfs serve -log_dir=${directory:log} {{ zurl }}
+wrapper-path = ${directory:service-on-watch}/wcfs
+
+[wcfs-promise]
+<= monitor-promise-base
+module = check_command_execute
+name = ${:_buildout_section_name_}.py
+config-command = {{ bin_directory }}/wcfs status {{ zurl }}
+
+[publish]
+recipe = slapos.cookbook:publish
+serving-zurl = {{ zurl }}

stack/erp5/instance-zope.cfg.in

@@ -63,6 +63,12 @@ environment +=
   FONTCONFIG_FILE=${fontconfig-conf:rendered}
 {% if slapparameter_dict.get('wendelin-core-zblk-fmt') %}
   WENDELIN_CORE_ZBLK_FMT={{ slapparameter_dict['wendelin-core-zblk-fmt'] }}
+{% endif %}
+  WENDELIN_CORE_WCFS_AUTOSTART=no
+{% if slapparameter_dict['wcfs_enable'] %}
+  WENDELIN_CORE_VIRTMEM=r:wcfs+w:uvmm
+{% else %}
+  WENDELIN_CORE_VIRTMEM=rw:uvmm
 {% endif %}
   ${:environment-extra}
 

stack/erp5/instance.cfg.in

@@ -31,6 +31,7 @@ extra-context =
 [dynamic-template-erp5-parameters]
 default-cloudooo-url =  {{ dumps(default_cloudooo_url) }}
 jupyter-enable-default = {{ jupyter_enable_default }}
+wcfs-enable-default = {{ wcfs_enable_default }}
 local-bt5-repository = {{ ' '.join(local_bt5_repository.split()) }}
 
 [context]
@@ -44,6 +45,7 @@ filename = instance-erp5.cfg
 extra-context =
     key default_cloudooo_url dynamic-template-erp5-parameters:default-cloudooo-url
     key jupyter_enable_default dynamic-template-erp5-parameters:jupyter-enable-default
+    key wcfs_enable_default dynamic-template-erp5-parameters:wcfs-enable-default
     key local_bt5_repository dynamic-template-erp5-parameters:local-bt5-repository
     key openssl_location :openssl-location
     import re re
@@ -190,6 +192,13 @@ mode = 644
 [dynamic-template-jupyter]
 rendered = {{ template_jupyter_cfg }}
 
+[dynamic-template-wcfs]
+<= jinja2-template-base
+template = {{ instance_wcfs_cfg_in }}
+filename = instance_wcfs.cfg
+extra-context =
+    section parameter_dict dynamic-template-zope-parameters
+
 [switch-softwaretype]
 recipe = slapos.cookbook:switch-softwaretype
 override = {{ dumps(override_switch_softwaretype |default) }}
@@ -208,6 +217,7 @@ zodb-zeo = dynamic-template-zeo:rendered
 zodb-neo = neo:rendered
 zope = dynamic-template-zope:rendered
 jupyter = dynamic-template-jupyter:rendered
+wcfs = dynamic-template-wcfs:rendered
 
 # Keep cloudooo backward compatibility
 cloudooo = dynamic-template-legacy:output