Commit d4821f87 authored by Julien Muchembled's avatar Julien Muchembled

NEO/ERP5: new schema of parameters, with overriding mechanism

The main change is a new and unique way to override parameters depending on where these parameters are used:
- for NEO, the goal is to avoid a huge `node_list` value (in particular with MyRocks settings), whereas most of the time all values in the list are the same
- for ERP5, we need a way to define different ZODB cache settings depending on the zope id (e.g. huge cache for some specialized zopes, a small cache for the validation node, etc. whereas currently a huge cache for all nodes would waste a lot of RAM)

For any `key=value` parameter that can be overridden, overriding is done as follows:
- `key` defines the default value (not required if the SR already defines one)
- `key!=[[pattern, value],...]`: the SR takes the value of the first 2-tuple whose *some reference* matches the pattern

For NEO:
- new `node_count` and `node` parameters
- `node!` can be used to override `node`: the *reference* is the partition reference (`node-<i>` for standalone NEO, and `neo-<i>` for ERP5)
- `node_list` is deprecated (if still passed, the new parameters must not be present)

For ERP5 ZODB settings, parameters can be overridden at both:
- zodb level (e.g. `pool-size`, `cache-size`... oh, I've just discovered there exists `cache-size-bytes`)
- storage level (e.g. `compress`, `cache-size`, etc. in the case of NEO)

Here, the *reference* is the zope id (e.g. `actitivies-0`)

The `pattern` is a Python regex that must match the whole reference.

Another not-so-small change is that the schema is fixed to only require NEO `cluster` parameter when instanciating a standalone NEO.

And a few fixes.

/reviewed-on nexedi/slapos!661
parents 2ec92336 9c356374
......@@ -257,7 +257,7 @@
"type": "object"
},
"zodb": {
"description": "Zope Object DataBase mountpoints. See https://github.com/zopefoundation/ZODB/blob/3.10/src/ZODB/component.xml for extra options.",
"description": "Zope Object DataBase mountpoints. See https://github.com/zopefoundation/ZODB/blob/4/src/ZODB/component.xml for extra options.",
"items": {
"required": [
"type"
......@@ -282,8 +282,13 @@
"type": "boolean"
}
},
"patternProperties": {
".!$": {
"$ref": "#/properties/zodb/items/patternProperties/.!$"
}
},
"additionalProperties": {
"type": "string"
"$ref": "#/properties/zodb/items/additionalProperties"
},
"type": "object"
},
......@@ -313,14 +318,33 @@
},
"server": {
"description": "Instantiate a server. If missing, 'storage-dict' must contain the necessary properties to mount the ZODB. Partitions references are 'neo-0', 'neo-1', ...",
"$ref": "../neoppod/instance-neo-input-schema.json"
"$ref": "../neoppod/instance-neo-input-schema.json#/definitions/neo-cluster"
}
}
}
],
"additionalProperties": {
"patternProperties": {
".!$": {
"description": "Override with the value of the first item whose zope id matches against the pattern.",
"items": {
"items": [
{
"description": "Override pattern (Python regular expression).",
"type": "string"
},
{
"description": "Override value (parameter for maching nodes).",
"type": ["integer", "string"]
}
],
"type": "array"
},
"type": "array"
}
},
"additionalProperties": {
"type": ["integer", "string"]
},
"type": "object"
},
"type": "array"
......
{
"$schema": "http://json-schema.org/draft-04/schema#",
"required": [
"tcpv4-port"
],
"type": "object",
"properties": {
"tcpv4-port": {
......
......@@ -46,3 +46,13 @@ class ERP5InstanceTestCase(SlapOSInstanceTestCase):
"""Return the output paramters from the root partition"""
return json.loads(
self.computer_partition.getConnectionParameterDict()['_'])
def getComputerPartition(self, partition_reference):
for computer_partition in self.slap.computer.getComputerPartitionList():
if partition_reference == computer_partition.getInstanceParameter(
'instance_title'):
return computer_partition
def getComputerPartitionPath(self, partition_reference):
partition_id = self.getComputerPartition(partition_reference).getId()
return os.path.join(self.slap._instance_root, partition_id)
......@@ -195,8 +195,8 @@ class TestDisableTestRunner(ERP5InstanceTestCase, TestPublishedURLIsReachableMix
"""
# self.computer_partition_root_path is the path of root partition.
# we want to assert that no scripts exist in any partition.
bin_programs = [os.path.basename(path) for path in
glob.glob("{}/../*/bin/*".format(self.computer_partition_root_path))]
bin_programs = map(os.path.basename,
glob.glob(self.computer_partition_root_path + "/../*/bin/*"))
self.assertTrue(bin_programs) # just to check the glob was correct.
self.assertNotIn('runUnitTest', bin_programs)
......@@ -210,8 +210,99 @@ class TestDisableTestRunner(ERP5InstanceTestCase, TestPublishedURLIsReachableMix
apache_process = psutil.Process(process_info['pid'])
self.assertEqual(
sorted([socket.AF_INET, socket.AF_INET6]),
sorted([
sorted(
c.family
for c in apache_process.connections()
if c.status == 'LISTEN'
]))
))
class TestZopeNodeParameterOverride(ERP5InstanceTestCase, TestPublishedURLIsReachableMixin):
"""Test override zope node parameters
"""
__partition_reference__ = 'override'
@classmethod
def getInstanceParameterDict(cls):
# The following example includes the most commonly used options,
# but not necessarily in a meaningful way.
return {'_': json.dumps({
"zodb": [{
"type": "zeo",
"server": {},
"cache-size-bytes": "20MB",
"cache-size-bytes!": [
("bb-0", 1<<20),
("bb-.*", "500MB"),
],
"pool-timeout": "10m",
"storage-dict": {
"cache-size!": [
("a-.*", "50MB"),
],
},
}],
"zope-partition-dict": {
"a": {
"instance-count": 3,
},
"bb": {
"instance-count": 5,
"port-base": 2300,
},
},
})}
def test_zope_conf(self):
zeo_addr = json.loads(
self.getComputerPartition('zodb').getConnectionParameter('_')
)["storage-dict"]["root"]["server"]
def checkParameter(line, kw):
k, v = line.split()
self.assertFalse(k.endswith('!'), k)
try:
expected = kw.pop(k)
except KeyError:
if k == 'server':
return
self.assertIsNotNone(expected)
self.assertEqual(str(expected), v)
def checkConf(zodb, storage):
zodb["mount-point"] = "/"
zodb["pool-size"] = 4
zodb["pool-timeout"] = "10m"
storage["storage"] = "root"
storage["server"] = zeo_addr
with open('%s/etc/zope-%s.conf' % (partition, zope)) as f:
conf = map(str.strip, f.readlines())
i = conf.index("<zodb_db root>") + 1
conf = iter(conf[i:conf.index("</zodb_db>", i)])
for line in conf:
if line == '<zeoclient>':
for line in conf:
if line == '</zeoclient>':
break
checkParameter(line, storage)
for k, v in storage.iteritems():
self.assertIsNone(v, k)
del storage
else:
checkParameter(line, zodb)
for k, v in zodb.iteritems():
self.assertIsNone(v, k)
partition = self.getComputerPartitionPath('zope-a')
for zope in xrange(3):
checkConf({
"cache-size-bytes": "20MB",
}, {
"cache-size": "50MB",
})
partition = self.getComputerPartitionPath('zope-bb')
for zope in xrange(5):
checkConf({
"cache-size-bytes": "500MB" if zope else 1<<20,
}, {
"cache-size": None,
})
......@@ -14,11 +14,11 @@
# not need these here).
[instance-common]
filename = instance-common.cfg.in
md5sum = 0a3a54fcc7be0bbd63cbd64f006ceebc
md5sum = 80599fcc6e5d07270d7900aebfd62139
[root-common]
filename = root-common.cfg.in
md5sum = ccc6e33412259415ec6c3452d37b77cc
md5sum = c03fbfc9df9edc1ef60be970e0627c5e
[instance-neo-admin]
filename = instance-neo-admin.cfg.in
......@@ -38,7 +38,7 @@ md5sum = 9f6f8f2b5f4cb0d97d50ffc1d3837e2f
[template-neo]
filename = instance.cfg.in
md5sum = 83dc9faca482b2ddbd3d5fa968af7c33
md5sum = 9e63e16eda75e73ad4ffb50afde0505d
[cluster]
filename = cluster.cfg.in
......
......@@ -15,7 +15,9 @@ cert = ${slap-connection:cert-file}
recipe = slapos.recipe.template:jinja2
filename = ${:_buildout_section_name_}.cfg
rendered = ${buildout:parts-directory}/${:_buildout_section_name_}/${:filename}
extensions = jinja2.ext.do
extensions =
jinja2.ext.do
jinja2.ext.loopcontrols
extra-context =
context =
key ipv4_set slap-configuration:ipv4
......
{
"$schema": "http://json-schema.org/draft-04/schema#",
"$schema": "http://json-schema.org/draft-07/schema#",
"description": "Parameters to instantiate a NEO cluster. See https://lab.nexedi.com/nexedi/neoppod/blob/master/neo.conf for more information.",
"definitions": {
"neo-cluster": {
"additionalProperties": false,
"require": [
"cluster"
],
"properties": {
"cluster": {
"description": "Cluster unique identifier. Your last line of defense against mixing up NEO clusters and corrupting your data. Choose a unique value for each of your cluster. Space not allowed.",
......@@ -72,10 +71,13 @@
"_key": {
"type": "string"
},
"node-list": {
"description": "List of dictionaries containing parameters for each node.",
"items": {
"description": "Dictionary containing parameters required to configure individual nodes.",
"node-count": {
"description": "Number of nodes to deploy.",
"default": 1,
"type": "integer"
},
"node": {
"description": "Default node parameters.",
"default": {},
"additionalProperties": false,
"properties": {
......@@ -135,8 +137,41 @@
},
"type": "object"
},
"node!": {
"description": "Node parameters are overridden by the value of the first item whose partition reference matches against the pattern.",
"items": {
"items": [
{
"description": "Override pattern (Python regular expression).",
"type": "string"
},
{
"allOf": [
{
"$ref": "#/definitions/neo-cluster/properties/node"
},
{
"description": "Override value (parameters for maching nodes)."
}
]
}
],
"type": "array"
},
"type": "array"
}
},
"type": "object"
}
},
"allOf": [
{
"$ref": "#/definitions/neo-cluster"
},
{
"required": [
"cluster"
]
}
]
}
......@@ -8,6 +8,7 @@ rendered = ${buildout:parts-directory}/${:_buildout_section_name_}.cfg
<= jinja2-template-base
template = {{ cluster }}
extra-context =
import re re
import urlparse urlparse
import-list =
rawfile root_common {{ root_common }}
......
......@@ -16,6 +16,20 @@ sla-computer_guid = ${slap-connection:computer-id}
{% endif -%}
{% endmacro -%}
{% macro apply_overrides(dict_, reference) -%}
{% for key in list(dict_) -%}
{% if key.endswith('!') -%}
{% for pattern, value in dict_.pop(key, ()) -%}
{% set m = re.match(pattern, reference) -%}
{% if m and m.group() == reference %}{# PY3: fullmatch -#}
{% do dict_.__setitem__(key[:-1], value) -%}
{% break -%}
{% endif -%}
{% endfor -%}
{% endif -%}
{% endfor -%}
{% endmacro -%}
{% macro common_section() -%}
[request-common-base]
recipe = slapos.cookbook:request.serialised
......@@ -52,7 +66,22 @@ config-ssl = {{ dumps((
config-upstream-cluster = {{ dumps(parameter_dict.get('upstream-cluster', '')) }}
config-upstream-masters = {{ dumps(parameter_dict.get('upstream-masters', '')) }}
software-type = {{ software_type }}
{% set node_list = parameter_dict.get('node-list', ({},)) -%}
{% set node_list = parameter_dict.get('node-list') -%}
{% if node_list == None -%}
{% set node_list = [] -%}
{% for i in range(parameter_dict.get('node-count', 1)) -%}
{% set x = parameter_dict.copy() -%}
{% do apply_overrides(x, prefix ~ i) -%}
{% do node_list.append(x.get('node', {})) -%}
{% endfor -%}
{% else %}{# BBB -#}
{% do assert('node-count' not in parameter_dict) -%}
{% do assert('node' not in parameter_dict) -%}
{% do assert('node!' not in parameter_dict) -%}
{% endif -%}
{% do assert(node_list) -%}
{% set storage_count = [] -%}
{% for node in node_list -%}
{% do storage_count.append(node.get('storage-count', 1)) -%}
......
......@@ -34,7 +34,7 @@ md5sum = e91c0fbd0df441884f7422fa7976053c
[template-zope-conf]
filename = zope.conf.in
md5sum = 114e0ac43281b943931754ed317ebc36
md5sum = 762897486b1e7e28b614224a9a577125
[site-zcml]
filename = site.zcml
......@@ -70,7 +70,7 @@ md5sum = cc19560b9400cecbd23064d55c501eec
[template]
filename = instance.cfg.in
md5sum = f6c6820f9b3653d0d5c29708606fc591
md5sum = 520b6bf3461dddc9c8b862e50b14465d
[monitor-template-dummy]
filename = dummy.cfg
......@@ -86,7 +86,7 @@ md5sum = 0648e38bd5d3a15bb9f93264932740b9
[template-zope]
filename = instance-zope.cfg.in
md5sum = b1685783f4c93da918ccc83702559e6f
md5sum = 8b4a15dca7e30ba5a792f1a9622216b0
[template-balancer]
filename = instance-balancer.cfg.in
......
......@@ -267,6 +267,11 @@ timerserver-interval = {{ dumps(timerserver_interval) }}
[zope-conf-base]
< = jinja2-template-base
template = {{ parameter_dict['zope-conf-template'] }}
extensions =
jinja2.ext.do
jinja2.ext.loopcontrols
import-list =
rawfile root_common {{ root_common }}
{% macro zope(
index,
......@@ -312,10 +317,10 @@ longrequest-logger-file =
[{{ conf_name }}]
< = zope-conf-base
rendered = ${directory:etc}/{{ name }}.conf
extensions = jinja2.ext.do
context =
section parameter_dict {{ conf_parameter_name }}
import os os
import re re
[{{ section(name) }}]
< = runzope-base
......
......@@ -46,6 +46,7 @@ extra-context =
key jupyter_enable_default dynamic-template-erp5-parameters:jupyter-enable-default
key local_bt5_repository dynamic-template-erp5-parameters:local-bt5-repository
key openssl_location :openssl-location
import re re
import urlparse urlparse
import-list =
file root_common context:root-common
......@@ -111,6 +112,7 @@ template = {{ template_zope }}
filename = instance-zope.cfg
extra-context =
key buildout_directory buildout:directory
key root_common context:root-common
section parameter_dict dynamic-template-zope-parameters
import urlparse urlparse
import hashlib hashlib
......
{% set slapparameter_dict = {} %}{# dummy -#}
{% import "root_common" as root_common with context -%}
{% set node_id = parameter_dict['node-id'] -%}
# Note: Environment is setup in running wrapper script, as zope.conf is read
# too late for some components.
%define INSTANCE {{ parameter_dict['instance'] }}
......@@ -65,7 +68,7 @@ large-file-threshold {{ parameter_dict['large-file-threshold'] }}
{% endif -%}
<product-config CMFActivity>
node-id {{ parameter_dict['node-id'] }}
node-id {{ node_id }}
</product-config>
{% set timerserver_interval = parameter_dict['timerserver-interval'] -%}
......@@ -145,10 +148,12 @@ large-file-threshold {{ parameter_dict['large-file-threshold'] }}
<zodb_db {{ name }}>
{%- set storage_type = type_dict[zodb_dict.pop('type')] %}
{%- set storage_dict = zodb_dict.pop('storage-dict') %}
{%- do root_common.apply_overrides(zodb_dict, node_id) %}
{%- for key, value in zodb_dict.iteritems() %}
{{ key }} {{ value }}
{%- endfor %}
<{{ storage_type }}>
{%- do root_common.apply_overrides(storage_dict, node_id) %}
{%- for key, value in storage_dict.iteritems() %}
{{ key }} {{ value }}
{%- endfor %}
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment