From 32db684635336230351396902899fce16b3d3fc7 Mon Sep 17 00:00:00 2001 From: Rafael Monnerat <rafael@nexedi.com> Date: Tue, 13 Aug 2019 16:33:40 -0300 Subject: [PATCH] slapos-master: Update buildout profiles to follow up changes on ERP5 --- software/slapos-master/buildout.hash.cfg | 4 +- .../slapos-master/instance-balancer.cfg.in | 11 +++-- software/slapos-master/instance-erp5.cfg.in | 46 +++++++++++++++---- 3 files changed, 45 insertions(+), 16 deletions(-) diff --git a/software/slapos-master/buildout.hash.cfg b/software/slapos-master/buildout.hash.cfg index be858682e..7a3f23604 100644 --- a/software/slapos-master/buildout.hash.cfg +++ b/software/slapos-master/buildout.hash.cfg @@ -14,11 +14,11 @@ # not need these here). [template-erp5] filename = instance-erp5.cfg.in -md5sum = f3cc69df4d34d780af012260db9a4cf5 +md5sum = 1c8cf3013bd8a4b05e62a8742089c59b [template-balancer] filename = instance-balancer.cfg.in -md5sum = b7504fcbd8eaecb91709abbcb5bcabe8 +md5sum = e8033d4fd7b6348b525a6148762ccdb4 [template-apache-backend-conf] filename = apache-backend.conf.in diff --git a/software/slapos-master/instance-balancer.cfg.in b/software/slapos-master/instance-balancer.cfg.in index 97dedc273..38cfef0d6 100644 --- a/software/slapos-master/instance-balancer.cfg.in +++ b/software/slapos-master/instance-balancer.cfg.in @@ -56,13 +56,15 @@ mode = 644 {% endif -%} {% set zope_effective_address = zope_address -%} {% do zope_family_address_list.append((zope_effective_address, maxconn, webdav)) -%} +{% endfor -%} -{# # Generate entries with rewrite rule for test runnners #} +{# # Generate entries with rewrite rule for test runnners #} +{% set test_runner_address_list = slapparameter_dict.get(parameter_id ~ '-test-runner-address-list', []) %} +{% if test_runner_address_list -%} {% set test_runner_backend_mapping = {} %} {% set test_runner_apache_url_list = [] %} {% set test_runner_external_port = next_port() %} -{% for i, (test_runner_internal_ip, test_runner_internal_port) in - enumerate(slapparameter_dict.get(parameter_id ~ '-test-runner-address-list', [])) %} +{% for i, (test_runner_internal_ip, test_runner_internal_port) in enumerate(test_runner_address_list) %} {% do test_runner_backend_mapping.__setitem__( 'unit_test_' ~ i, 'http://' ~ test_runner_internal_ip ~ ':' ~ test_runner_internal_port ) %} @@ -73,8 +75,7 @@ mode = 644 (ipv4, test_runner_external_port), ( ssl_authentication, test_runner_backend_mapping ) ) -%} {% do test_runner_url_dict.__setitem__(family_name, test_runner_apache_url_list) -%} - -{% endfor -%} +{% endif -%} {% endfor -%} {# Make rendering fail artificially if any family has no known backend. diff --git a/software/slapos-master/instance-erp5.cfg.in b/software/slapos-master/instance-erp5.cfg.in index 43270aab8..405053671 100644 --- a/software/slapos-master/instance-erp5.cfg.in +++ b/software/slapos-master/instance-erp5.cfg.in @@ -22,6 +22,14 @@ {% set monitor_base_url_dict = {} -%} {% set monitor_dict = slapparameter_dict.get('monitor', {}) %} {% set use_ipv6 = slapparameter_dict.get('use-ipv6', False) -%} +{% set partition_thread_count_list = [] -%} +{% set zope_partition_dict = slapparameter_dict.get('zope-partition-dict', {'1': {}}) -%} +{% for zope_parameter_dict in zope_partition_dict.values() -%} +{# Apply some zope_parameter_dict default values, to avoid duplication. -#} +{% do zope_parameter_dict.setdefault('thread-amount', 4) -%} +{% do zope_parameter_dict.setdefault('instance-count', 1) -%} +{% do partition_thread_count_list.append(zope_parameter_dict['thread-amount'] * zope_parameter_dict['instance-count']) -%} +{% endfor -%} [request-common] <= request-common-base config-use-ipv6 = {{ dumps(slapparameter_dict.get('use-ipv6', False)) }} @@ -94,7 +102,28 @@ backup-caucased = ${:srv}/backup/caucased {{ request('memcached-persistent', 'kumofs', 'kumofs', {'tcpv4-port': 2000}, {'url': True, 'monitor-base-url': False}, key_config={'monitor-passwd': 'monitor-htpasswd:passwd'}) }} {{ request('memcached-volatile', 'kumofs', 'memcached', {'tcpv4-port': 2010, 'ram-storage-size': 64}, {'url': True, 'monitor-base-url': False}, key_config={'monitor-passwd': 'monitor-htpasswd:passwd'}) }} -{{ request('mariadb', 'mariadb', 'mariadb', {'tcpv4-port': 2099, 'max-slowqueries-threshold': monitor_dict.get('max-slowqueries-threshold', 1000), 'slowest-query-threshold': monitor_dict.get('slowest-query-threshold', ''), 'test-database-amount': test_runner_total_database_count}, {'database-list': True, 'test-database-list': True, 'monitor-base-url': False}, key_config={'monitor-passwd': 'monitor-htpasswd:passwd'}) }} +{# Notes on max-connection-count: On a standard ERP5, each transaction + can have 4 connections to mariadb: activities, catalog, deferred and + transactionless. Count 5 to have some headroom. Multiply by the total + number of zope threads for all processes from all partitions to get the + expected number of connections. Add 50 for have some more zope-independent + headroom (automated probes, replication, ...). +-#} +{{ request('mariadb', 'mariadb', 'mariadb', + { + 'tcpv4-port': 2099, + 'max-slowqueries-threshold': monitor_dict.get('max-slowqueries-threshold', 1000), + 'slowest-query-threshold': monitor_dict.get('slowest-query-threshold', ''), + 'test-database-amount': test_runner_total_database_count, + 'max-connection-count': sum(partition_thread_count_list) * 5 + 50, + }, + { + 'database-list': True, + 'test-database-list': True, + 'monitor-base-url': False, + }, + key_config={'monitor-passwd': 'monitor-htpasswd:passwd'}, +) }} {% if has_posftix -%} {{ request('smtp', 'postfix', 'smtp', {'tcpv4-port': 2025, 'smtpd-sasl-user': 'erp5@nowhere'}, key_config={'smtpd-sasl-password': 'publish-early:smtpd-sasl-password', 'monitor-passwd': 'monitor-htpasswd:passwd'}) }} {%- else %} @@ -136,7 +165,6 @@ connection-url = smtp://127.0.0.2:0/ {% endfor -%} -{% set zope_partition_dict = slapparameter_dict.get('zope-partition-dict', {'1': {}}) -%} {% set zope_address_list_id_dict = {} -%} {% if zope_partition_dict -%} @@ -149,7 +177,7 @@ return = {%- if test_runner_enabled %} test-runner-address-list {% endif %} -{% set bt5_default_list = 'erp5_full_text_myisam_catalog slapos_configurator' -%} +{% set bt5_default_list = 'erp5_full_text_mroonga_catalog slapos_configurator' -%} {% if has_jupyter -%} {% set bt5_default_list = bt5_default_list + ' erp5_data_notebook' -%} {% endif -%} @@ -219,9 +247,9 @@ name = {{ partition_name }} {% do monitor_base_url_dict.__setitem__(section_name, '${' ~ section_name ~ ':connection-monitor-base-url}') -%} {{ root_common.sla(partition_name) }} config-name = {{ dumps(custom_name) }} -config-instance-count = {{ dumps(zope_parameter_dict.get('instance-count', 1)) }} +config-instance-count = {{ dumps(zope_parameter_dict['instance-count']) }} config-private-dev-shm = {{ zope_parameter_dict.get('private-dev-shm', '') }} -config-thread-amount = {{ dumps(zope_parameter_dict.get('thread-amount', 4)) }} +config-thread-amount = {{ dumps(zope_parameter_dict['thread-amount']) }} config-timerserver-interval = {{ dumps(zope_parameter_dict.get('timerserver-interval', 5)) }} config-longrequest-logger-interval = {{ dumps(zope_parameter_dict.get('longrequest-logger-interval', -1)) }} config-longrequest-logger-timeout = {{ dumps(zope_parameter_dict.get('longrequest-logger-timeout', 1)) }} @@ -412,11 +440,11 @@ recipe = slapos.cookbook:publish-early {%- if has_posftix %} smtpd-sasl-password gen-smtpd-sasl-password:passwd {%- endif %} -{% if test_runner_enabled -%} -{% for zope_family_name in zope_family_name_list %} +{%- if test_runner_enabled %} +{%- for zope_family_name in zope_family_name_list %} {{ zope_family_name }}-test-runner-url-list default-balancer-test-runner-url-list:default -{% endfor -%} -{% endif -%} +{%- endfor %} +{%- endif %} {%- if neo %} neo-cluster gen-neo-cluster:name {%- if neo[0] %} -- 2.30.9