Skip to content

S
slapos
Commit 03e4ce33 authored by Kirill Smelkov's avatar Kirill Smelkov
Browse files
Options

stack/erp5 += WCFS service (technology preview) 

This complements 7f877621 (Move wendelin.core from Wendelin to ERP5) and
introduces new WCFS service into ERP5 software-release to support
upcoming wendelin.core 2. Wendelin.core 2 will introduce new WCFS
filesystem server component that serves in-ZODB arrays data as OS files
for Zopes to memory-map. Please see the following links for overview of
what WCFS is and how it interoperates with and serves Python clients:

https://lab.nexedi.com/kirr/wendelin.core/blob/b5fc98bb/wcfs/wcfs.go#L20-247
https://lab.nexedi.com/kirr/wendelin.core/blob/b5fc98bb/wcfs/client/wcfs.h#L20-96

Implementation notes:

- WCFS is by default disabled in ERP5 - we do not want to load every
  ERP5 user with it in the early days of wendelin.core 2. WCFS service
  will also not work when ERP5 is built with wendelin.core 1, so keeping
  it disabled by default is reasonable for now.

- There is a duplication for instance-erp5.cfg.in in between stack/erp5/
  and software/slapos-master/ . It was negotiated with VIFIB developers
  to go this way instead of first deduplicating instance-erp5.cfg.in
  copying that was started in 87d13789 (slapos-master: Copy files from
  erp5 stack for local changes).

To enable wcfs one can either pass `wcfs` configuration parameter, or
add the following to a software-release that uses stack/erp5:

  [erp5-defaults]
  wcfs-enable-default = true

/cc @rafael, @tomo
parent 0803b816
Show whitespace changes
Inline Side-by-side
Showing with 267 additions and 47 deletions
software/erp5/test/test/test_wcfs.py 0 → 100644
View file @ 03e4ce33
# Copyright (C) 2021 Nexedi SA and Contributors.
#
# This program is free software: you can Use, Study, Modify and Redistribute
# it under the terms of the GNU General Public License version 3, or (at your
# option) any later version, as published by the Free Software Foundation.
#
# You can also Link and Combine this program with other software covered by
# the terms of any of the Free Software licenses or any of the Open Source
# Initiative approved licenses and Convey the resulting work. Corresponding
# source of such a combination shall include the source code for all other
# software used.
#
# This program is distributed WITHOUT ANY WARRANTY; without even the implied
# warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
#
# See COPYING file for full licensing terms.
# See https://www.nexedi.com/licensing for rationale and options.
import json
import os.path
import unittest
from slapos.grid.utils import md5digest
from . import ERP5InstanceTestCase, setUpModule as _setUpModule
from .test_erp5 import TestPublishedURLIsReachableMixin
# skip tests when software release is built with wendelin.core 1.
def setUpModule():
_setUpModule()
cls = ERP5InstanceTestCase
if not os.path.exists(
os.path.join(
cls.slap.software_directory,
md5digest(cls.getSoftwareURL()),
'bin', 'wcfs')):
raise unittest.SkipTest("built with wendelin.core 1")
class TestWCFS(ERP5InstanceTestCase, TestPublishedURLIsReachableMixin):
"""Test Wendelin Core File System
"""
__partition_reference__ = 'wcfs'
@classmethod
def getInstanceParameterDict(cls):
return {'_': json.dumps({'wcfs': {'enable': True}})}
def test_wcfs_accessible(self):
"""Verify that wcfs filesystem is basically accessible.
- we can read .wcfs/zurl
- its content is equal to published `serving-zurl`
"""
zurl = json.loads(
self.getComputerPartition('wcfs').getConnectionParameter('_')
)['serving-zurl']
mntpt = lookupMount(zurl)
zurl_ = readfile("%s/.wcfs/zurl" % mntpt)
self.assertEqual(zurl_, zurl)
# lookupMount returns /proc/mount entry for wcfs mounted to serve zurl.
def lookupMount(zurl):
for line in readfile('/proc/mounts').splitlines():
# <zurl> <mountpoint> fuse.wcfs ...
zurl_, mntpt, typ, _ = line.split(None, 3)
if typ != 'fuse.wcfs':
continue
if zurl_ == zurl:
return mntpt
raise KeyError("lookup mount %s: no /proc/mounts entry" % zurl)
# readfile returns content of file @path.
def readfile(path):
with open(path, 'r') as f:
return f.read()
software/slapos-master/buildout.hash.cfg
View file @ 03e4ce33
......@@ -14,7 +14,7 @@
# not need these here).
[template-erp5]
filename = instance-erp5.cfg.in
md5sum = f078d72db690805eab90a4577066b920
md5sum = eba73f868a5bbfb942323eadcd88cb16
[template-balancer]
filename = instance-balancer.cfg.in
......
software/slapos-master/instance-erp5.cfg.in
View file @ 03e4ce33
......@@ -9,6 +9,8 @@
{% set jupyter_dict = slapparameter_dict.get('jupyter', {}) -%}
{% set has_jupyter = jupyter_dict.get('enable', jupyter_enable_default.lower() in ('true', 'yes')) -%}
{% set jupyter_zope_family = jupyter_dict.get('zope-family', '') -%}
{% set wcfs_dict = slapparameter_dict.get('wcfs', {}) -%}
{% set wcfs_enable = wcfs_dict.get('enable', wcfs_enable_default.lower() in ('true', 'yes')) -%}
{% set test_runner_enabled = slapparameter_dict.get('test-runner', {}).get('enabled', True) -%}
{% set test_runner_node_count = slapparameter_dict.get('test-runner', {}).get('node-count', 3) -%}
{% set test_runner_extra_database_count = slapparameter_dict.get('test-runner', {}).get('extra-database-count', 3) -%}
......@@ -189,12 +191,27 @@ connection-url = smtp://127.0.0.2:0/
{% endif -%}
{% endfor -%}
[request-zodb-base]
config-zodb-dict = {{ dumps(zodb_dict) }}
{% for server_type, server_dict in storage_dict.iteritems() -%}
{% if server_type == 'neo' -%}
config-neo-cluster = ${publish-early:neo-cluster}
config-neo-name = {{ server_dict.keys()[0] }}
config-neo-masters = ${publish-early:neo-masters}
{% else -%}
config-zodb-zeo = ${request-zodb:connection-storage-dict}
config-tidstorage-ip = ${request-zodb:connection-tidstorage-ip}
config-tidstorage-port = ${request-zodb:connection-tidstorage-port}
{% endif -%}
{% endfor -%}
{% set zope_address_list_id_dict = {} -%}
{% if zope_partition_dict -%}
[request-zope-base]
<= request-common
request-zodb-base
return =
zope-address-list
hosts-dict
......@@ -233,20 +250,9 @@ config-timezone = {{ dumps(slapparameter_dict.get('timezone', 'UTC')) }}
config-cloudooo-retry-count = {{ slapparameter_dict.get('cloudooo-retry-count', 2) }}
config-wendelin-core-zblk-fmt = {{ dumps(slapparameter_dict.get('wendelin-core-zblk-fmt', '')) }}
config-wsgi = {{ dumps(slapparameter_dict.get('wsgi', True)) }}
config-zodb-dict = {{ dumps(zodb_dict) }}
config-test-runner-enabled = {{ dumps(test_runner_enabled) }}
config-test-runner-node-count = {{ dumps(test_runner_node_count) }}
{% for server_type, server_dict in storage_dict.iteritems() -%}
{% if server_type == 'neo' -%}
config-neo-cluster = ${publish-early:neo-cluster}
config-neo-name = {{ server_dict.keys()[0] }}
config-neo-masters = ${publish-early:neo-masters}
{% else -%}
config-zodb-zeo = ${request-zodb:connection-storage-dict}
config-tidstorage-ip = ${request-zodb:connection-tidstorage-ip}
config-tidstorage-port = ${request-zodb:connection-tidstorage-port}
{% endif -%}
{% endfor -%}
config-wcfs_enable = {{ wcfs_enable }}
software-type = zope
{% set global_publisher_timeout = slapparameter_dict.get('publisher-timeout') -%}
......@@ -348,6 +354,17 @@ config-url = ${request-jupyter:connection-url}
{% endif -%}
{%- endif %}
{% if wcfs_enable -%}
{# request WCFS connected to ZODB -#}
{% do root_common.section('request-wcfs') -%}
{{ request('wcfs', 'wcfs', 'wcfs', {}, {}) }}
[request-wcfs]
<= request-common
request-zodb-base
{%- endif %}
[directory]
recipe = slapos.cookbook:mkdirectory
{% if slapparameter_dict.get('shared-certificate-authority-path', '') -%}
......
stack/erp5/buildout.cfg
View file @ 03e4ce33
......@@ -187,6 +187,7 @@ context =
key instance_common_cfg instance-common:rendered
key jsl_location jsl:location
key jupyter_enable_default erp5-defaults:jupyter-enable-default
key wcfs_enable_default erp5-defaults:wcfs-enable-default
key kumo_location kumo:location
key local_bt5_repository local-bt5-repository:list
key logrotate_location logrotate:location
......@@ -224,7 +225,9 @@ context =
key template_postfix_aliases template-postfix-aliases:target
key template_postfix_main_cf template-postfix-main-cf:target
key template_postfix_master_cf template-postfix-master-cf:target
key instance_wcfs_cfg_in instance-wcfs.cfg.in:target
key template_zeo template-zeo:target
key template_zodb_base template-zodb-base:target
key template_zope template-zope:target
key template_zope_conf template-zope-conf:target
key template_fonts_conf template-fonts-conf:output
......@@ -239,6 +242,9 @@ context =
[template-zeo]
<= download-base
[template-zodb-base]
<= download-base
[template-zope]
<= download-base
link-binary =
......@@ -282,6 +288,9 @@ fontconfig-includes =
[template-rsyslogd-cfg]
<= download-base
[instance-wcfs.cfg.in]
<= download-base
[erp5-bin]
<= erp5
repository = https://lab.nexedi.com/nexedi/erp5-bin.git
......@@ -321,6 +330,8 @@ repository_id_list = erp5 erp5-bin erp5-doc
cloudooo-connection-url = https://cloudooo.erp5.net/
# Jupyter is by default disabled in ERP5
jupyter-enable-default = false
# WCFS is by default disabled in ERP5
wcfs-enable-default = false
[erp5]
recipe = slapos.recipe.build:gitclone
......@@ -559,6 +570,7 @@ scripts =
runzeo
tidstoraged
tidstorage_repozo
wcfs
web_checker_utility
extra-paths =
......
stack/erp5/buildout.hash.cfg
View file @ 03e4ce33
......@@ -70,19 +70,23 @@ md5sum = 7a14019abf48ca100eb94d9add20f5ae
[template]
filename = instance.cfg.in
md5sum = b5ac16fdeed8863e465e955ba6d1e12a
md5sum = 8bd7f89b7c1e453ecc952659a01347e6
[template-erp5]
filename = instance-erp5.cfg.in
md5sum = ac155f28a096747fc267f32a1cec46e1
md5sum = 79f092c54998918423257f4b676425ea
[template-zeo]
filename = instance-zeo.cfg.in
md5sum = 0648e38bd5d3a15bb9f93264932740b9
[template-zodb-base]
filename = instance-zodb-base.cfg.in
md5sum = bc821f9f9696953b10a03ad7b59a1936
[template-zope]
filename = instance-zope.cfg.in
md5sum = 9fa66b93fbf6a40aa8136c651ad9f539
md5sum = e495f7225a49f61c501ccc496a976d63
[template-balancer]
filename = instance-balancer.cfg.in
......@@ -95,3 +99,7 @@ md5sum = 452c502fabd5a6066c9dee533dfb1c77
[template-rsyslogd-cfg]
filename = rsyslogd.cfg.in
md5sum = 7030e42b50e03f24e036b7785bd6159f
[instance-wcfs.cfg.in]
filename = instance-wcfs.cfg.in
md5sum = 945e8e4552a6bdf228b9609567b09399
stack/erp5/instance-erp5.cfg.in
View file @ 03e4ce33
......@@ -9,6 +9,8 @@
{% set jupyter_dict = slapparameter_dict.get('jupyter', {}) -%}
{% set has_jupyter = jupyter_dict.get('enable', jupyter_enable_default.lower() in ('true', 'yes')) -%}
{% set jupyter_zope_family = jupyter_dict.get('zope-family', '') -%}
{% set wcfs_dict = slapparameter_dict.get('wcfs', {}) -%}
{% set wcfs_enable = wcfs_dict.get('enable', wcfs_enable_default.lower() in ('true', 'yes')) -%}
{% set test_runner_enabled = slapparameter_dict.get('test-runner', {}).get('enabled', True) -%}
{% set test_runner_node_count = slapparameter_dict.get('test-runner', {}).get('node-count', 3) -%}
{% set test_runner_extra_database_count = slapparameter_dict.get('test-runner', {}).get('extra-database-count', 3) -%}
......@@ -189,12 +191,27 @@ connection-url = smtp://127.0.0.2:0/
{% endif -%}
{% endfor -%}
[request-zodb-base]
config-zodb-dict = {{ dumps(zodb_dict) }}
{% for server_type, server_dict in storage_dict.iteritems() -%}
{% if server_type == 'neo' -%}
config-neo-cluster = ${publish-early:neo-cluster}
config-neo-name = {{ server_dict.keys()[0] }}
config-neo-masters = ${publish-early:neo-masters}
{% else -%}
config-zodb-zeo = ${request-zodb:connection-storage-dict}
config-tidstorage-ip = ${request-zodb:connection-tidstorage-ip}
config-tidstorage-port = ${request-zodb:connection-tidstorage-port}
{% endif -%}
{% endfor -%}
{% set zope_address_list_id_dict = {} -%}
{% if zope_partition_dict -%}
[request-zope-base]
<= request-common
request-zodb-base
return =
zope-address-list
hosts-dict
......@@ -238,20 +255,9 @@ config-timezone = {{ dumps(slapparameter_dict.get('timezone', 'UTC')) }}
config-cloudooo-retry-count = {{ slapparameter_dict.get('cloudooo-retry-count', 2) }}
config-wendelin-core-zblk-fmt = {{ dumps(slapparameter_dict.get('wendelin-core-zblk-fmt', '')) }}
config-wsgi = {{ dumps(slapparameter_dict.get('wsgi', True)) }}
config-zodb-dict = {{ dumps(zodb_dict) }}
config-test-runner-enabled = {{ dumps(test_runner_enabled) }}
config-test-runner-node-count = {{ dumps(test_runner_node_count) }}
{% for server_type, server_dict in storage_dict.iteritems() -%}
{% if server_type == 'neo' -%}
config-neo-cluster = ${publish-early:neo-cluster}
config-neo-name = {{ server_dict.keys()[0] }}
config-neo-masters = ${publish-early:neo-masters}
{% else -%}
config-zodb-zeo = ${request-zodb:connection-storage-dict}
config-tidstorage-ip = ${request-zodb:connection-tidstorage-ip}
config-tidstorage-port = ${request-zodb:connection-tidstorage-port}
{% endif -%}
{% endfor -%}
config-wcfs_enable = {{ wcfs_enable }}
software-type = zope
{% set global_publisher_timeout = slapparameter_dict.get('publisher-timeout') -%}
......@@ -353,6 +359,17 @@ config-url = ${request-jupyter:connection-url}
{% endif -%}
{%- endif %}
{% if wcfs_enable -%}
{# request WCFS connected to ZODB -#}
{% do root_common.section('request-wcfs') -%}
{{ request('wcfs', 'wcfs', 'wcfs', {}, {}) }}
[request-wcfs]
<= request-common
request-zodb-base
{%- endif %}
{% set balancer_ret_dict = {'monitor-base-url': False} -%}
{% for family in zope_family_dict -%}
{% do balancer_ret_dict.__setitem__(family, False) -%}
......
stack/erp5/instance-wcfs.cfg.in 0 → 100644
View file @ 03e4ce33
{# instance that runs WCFS service associated with ZODB storage #}
{% from "instance_zodb_base" import zodb_dict with context %}
{# build zurl to connect to configured ZODB #}
{% if len(zodb_dict) != 1 -%}
{% do assert(False, ("WCFS supports only single ZODB storage", zodb_dict)) -%}
{% endif -%}
{% set db_name, zodb = zodb_dict.popitem() -%}
{% set z = zodb['storage-dict'] -%}
{% if zodb['type'] == 'zeo' -%}
{% set zurl = ('zeo://%s?storage=%s' % (z['server'], z['storage'])) -%}
{% elif zodb['type'] == 'neo' -%}
{% set zurl = ('neo://%s@%s' % (z.pop('name'), z.pop('master_nodes'))) -%}
{% set argv = [] -%}
{% set i = 0 -%}
{% for k,v in z|dictsort -%}
{% do argv.append('%s=%s' % (k,v)) -%}
{% endfor -%}
{% if len(argv) > 0 -%}
{% set zurl = zurl + '&' + '?'.join(argv) -%}
{% endif -%}
{% else -%}
{% do assert(False, ("unsupported ZODB type", zodb)) -%}
{% endif -%}
[buildout]
extends = {{ template_monitor }}
parts +=
wcfs
wcfs-promise
publish
[directory]
recipe = slapos.cookbook:mkdirectory
etc = ${buildout:directory}/etc
log = ${:var}/log
run = ${:var}/run
services = ${:etc}/run
service-on-watch = ${:etc}/service
srv = ${buildout:directory}/srv
tmp = ${buildout:directory}/tmp
var = ${buildout:directory}/var
[wcfs]
recipe = slapos.cookbook:wrapper
command-line = {{ bin_directory }}/wcfs serve -log_dir=${directory:log} {{ zurl }}
wrapper-path = ${directory:service-on-watch}/wcfs
[wcfs-promise]
<= monitor-promise-base
module = check_command_execute
name = ${:_buildout_section_name_}.py
config-command = {{ bin_directory }}/wcfs status {{ zurl }}
[publish]
recipe = slapos.cookbook:publish.serialised
serving-zurl = {{ zurl }}
stack/erp5/instance-zodb-base.cfg.in 0 → 100644
View file @ 03e4ce33
{# base for instances that need to access ZODB storage #}
{# provides zodb_dict #}
{% set zodb_dict = slapparameter_dict['zodb-dict'] -%}
{% set zeo_dict = slapparameter_dict.get('zodb-zeo', {}) -%}
{% for name, zodb in zodb_dict.iteritems() -%}
{% set storage_dict = zodb.setdefault('storage-dict', {}) -%}
{% if zodb['type'] == 'zeo' -%}
{% do storage_dict.update(zeo_dict.get(name, ())) -%}
{% else -%}
{% if name == slapparameter_dict.get('neo-name') -%}
{% do storage_dict.update(master_nodes=slapparameter_dict['neo-masters'],
name=slapparameter_dict['neo-cluster']) -%}
{% endif -%}
{{ assert(storage_dict['master_nodes'], name) }}
{% if storage_dict.pop('ssl', 1) -%}
{% do storage_dict.update(ca='~/etc/ca.crt',
cert='~/etc/neo.crt',
key='~/etc/neo.key') -%}
{% endif -%}
{% endif -%}
{% endfor -%}
stack/erp5/instance-zope.cfg.in
View file @ 03e4ce33
{% from "instance_zodb_base" import zodb_dict with context %}
{% set wsgi = slapparameter_dict['wsgi'] -%}
{% set webdav = slapparameter_dict['webdav'] -%}
{% set use_ipv6 = slapparameter_dict.get('use-ipv6', False) -%}
{% set next_port = itertools.count(slapparameter_dict['port-base']).next -%}
{% set site_id = slapparameter_dict['site-id'] -%}
{% set zodb_dict = slapparameter_dict['zodb-dict'] -%}
{% set instance_index_list = range(slapparameter_dict['instance-count']) -%}
{% set node_id_base = slapparameter_dict['name'] -%}
{% set selenium_server_configuration_dict = slapparameter_dict.get('selenium-server-configuration-dict', None) -%}
......@@ -63,6 +63,12 @@ environment +=
FONTCONFIG_FILE=${fontconfig-conf:rendered}
{% if slapparameter_dict.get('wendelin-core-zblk-fmt') %}
WENDELIN_CORE_ZBLK_FMT={{ slapparameter_dict['wendelin-core-zblk-fmt'] }}
{% endif %}
WENDELIN_CORE_WCFS_AUTOSTART=no
{% if slapparameter_dict['wcfs_enable'] %}
WENDELIN_CORE_VIRTMEM=r:wcfs+w:uvmm
{% else %}
WENDELIN_CORE_VIRTMEM=rw:uvmm
{% endif %}
${:environment-extra}
......@@ -257,24 +263,6 @@ cloudooo-url = {{ (cloudooo if cloudooo.port == None else
cloudooo._replace(netloc='erp5-cloudooo:%s' % cloudooo.port)).geturl() }}
{% endif -%}
{% set zeo_dict = slapparameter_dict.get('zodb-zeo', {}) -%}
{% for name, zodb in zodb_dict.iteritems() -%}
{% set storage_dict = zodb.setdefault('storage-dict', {}) -%}
{% if zodb['type'] == 'zeo' -%}
{% do storage_dict.update(zeo_dict.get(name, ())) -%}
{% else -%}
{% if name == slapparameter_dict.get('neo-name') -%}
{% do storage_dict.update(master_nodes=slapparameter_dict['neo-masters'],
name=slapparameter_dict['neo-cluster']) -%}
{% endif -%}
{{ assert(storage_dict['master_nodes'], name) }}
{% if storage_dict.pop('ssl', 1) -%}
{% do storage_dict.update(ca='~/etc/ca.crt',
cert='~/etc/neo.crt',
key='~/etc/neo.key') -%}
{% endif -%}
{% endif -%}
{% endfor -%}
developer-list = {{ dumps(slapparameter_dict['developer-list']) }}
publisher-timeout = {{ dumps(slapparameter_dict['publisher-timeout']) }}
activity-timeout = {{ dumps(slapparameter_dict['activity-timeout']) }}
......
stack/erp5/instance.cfg.in
View file @ 03e4ce33
......@@ -31,11 +31,13 @@ extra-context =
[dynamic-template-erp5-parameters]
default-cloudooo-url = {{ dumps(default_cloudooo_url) }}
jupyter-enable-default = {{ jupyter_enable_default }}
wcfs-enable-default = {{ wcfs_enable_default }}
local-bt5-repository = {{ ' '.join(local_bt5_repository.split()) }}
[context]
root-common = {{ root_common }}
caucase-jinja2-library = {{ caucase_jinja2_library }}
template-zodb-base = {{ template_zodb_base }}
[dynamic-template-erp5]
<= jinja2-template-base
......@@ -44,6 +46,7 @@ filename = instance-erp5.cfg
extra-context =
key default_cloudooo_url dynamic-template-erp5-parameters:default-cloudooo-url
key jupyter_enable_default dynamic-template-erp5-parameters:jupyter-enable-default
key wcfs_enable_default dynamic-template-erp5-parameters:wcfs-enable-default
key local_bt5_repository dynamic-template-erp5-parameters:local-bt5-repository
key openssl_location :openssl-location
import re re
......@@ -125,6 +128,8 @@ extra-context =
import urlparse urlparse
import hashlib hashlib
import itertools itertools
import-list =
file instance_zodb_base context:template-zodb-base
[dynamic-template-kumofs-parameters]
<= default-dynamic-template-parameters
......@@ -190,6 +195,15 @@ mode = 644
[dynamic-template-jupyter]
rendered = {{ template_jupyter_cfg }}
[dynamic-template-wcfs]
<= jinja2-template-base
template = {{ instance_wcfs_cfg_in }}
filename = instance_wcfs.cfg
extra-context =
section parameter_dict dynamic-template-zope-parameters
import-list =
file instance_zodb_base context:template-zodb-base
[switch-softwaretype]
recipe = slapos.cookbook:switch-softwaretype
override = {{ dumps(override_switch_softwaretype |default) }}
......@@ -208,6 +222,7 @@ zodb-zeo = dynamic-template-zeo:rendered
zodb-neo = neo:rendered
zope = dynamic-template-zope:rendered
jupyter = dynamic-template-jupyter:rendered
wcfs = dynamic-template-wcfs:rendered
# Keep cloudooo backward compatibility
cloudooo = dynamic-template-legacy:output
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment
GitLab Nexedi Edition | About GitLab | About Nexedi | 沪ICP备2021021310号-2 | 沪ICP备2021021310号-7