software/erp5: Expose mariadb max-connections.

Auto-compute a value suitable for the number of requested Zope processes
and threads for default ERP5 needs.

software/erp5/instance-mariadb-schema.json

@@ -97,6 +97,11 @@
       "default": 1,
       "type": "number"
     },
+    "max-connection-count": {
+      "description": "See MariaDB documentation on max_connections. If not provided, a value suitable for the number of request Zope processes is chosen.",
+      "minimum": 0,
+      "type": "integer"
+    },
     "relaxed-writes": {
       "description": "When enabled, sets innodb_flush_log_at_trx_commit = 0, innodb_flush_method = nosync, innodb_doublewrite = 0 and sync_frm = 0 - RTFM, those options are dangerous",
       "default": false,

stack/erp5/buildout.hash.cfg

@@ -26,7 +26,7 @@ md5sum = d95e8500bdc72d1f40b97cc414656e7e
 
 [template-mariadb]
 filename = instance-mariadb.cfg.in
-md5sum = 14ae385a0fc5c0b4e03466eb786a451a
+md5sum = 9df786692c61bd8d3a6f4e7ed15f272f
 
 [template-kumofs]
 filename = instance-kumofs.cfg.in
@@ -42,7 +42,7 @@ md5sum = d32417746fcf671d4e86a70379815039
 
 [template-my-cnf]
 filename = my.cnf.in
-md5sum = 7a882ff275f723fdf30869cb7f1b90d3
+md5sum = 5a6f337117ba8b72d7fe3b7a9f26f5f6
 
 [template-mariadb-initial-setup]
 filename = mariadb_initial_setup.sql.in
@@ -78,7 +78,7 @@ md5sum = d41d8cd98f00b204e9800998ecf8427e
 
 [template-erp5]
 filename = instance-erp5.cfg.in
-md5sum = 36f00362c6703fc0a5519f90f733a2fd
+md5sum = 7dd00dedef4cc4320ec6977a7e2dc110
 
 [template-zeo]
 filename = instance-zeo.cfg.in

stack/erp5/instance-erp5.cfg.in

@@ -22,6 +22,14 @@
 {% set monitor_base_url_dict = {} -%}
 {% set monitor_dict = slapparameter_dict.get('monitor', {}) %}
 {% set use_ipv6 = slapparameter_dict.get('use-ipv6', False) -%}
+{% set partition_thread_count_list = [] -%}
+{% set zope_partition_dict = slapparameter_dict.get('zope-partition-dict', {'1': {}}) -%}
+{% for zope_parameter_dict in zope_partition_dict.values() -%}
+{#   Apply some zope_parameter_dict default values, to avoid duplication. -#}
+{%   do zope_parameter_dict.setdefault('thread-amount', 4) -%}
+{%   do zope_parameter_dict.setdefault('instance-count', 1) -%}
+{%   do partition_thread_count_list.append(zope_parameter_dict['thread-amount'] * zope_parameter_dict['instance-count']) -%}
+{% endfor -%}
 [request-common]
 <= request-common-base
 config-use-ipv6 = {{ dumps(slapparameter_dict.get('use-ipv6', False)) }}
@@ -94,12 +102,20 @@ backup-caucased = ${:srv}/backup/caucased
 
 {{ request('memcached-persistent', 'kumofs', 'kumofs', {'tcpv4-port': 2000}, {'url': True, 'monitor-base-url': False}, key_config={'monitor-passwd': 'monitor-htpasswd:passwd'}) }}
 {{ request('memcached-volatile', 'kumofs', 'memcached', {'tcpv4-port': 2010, 'ram-storage-size': 64}, {'url': True, 'monitor-base-url': False}, key_config={'monitor-passwd': 'monitor-htpasswd:passwd'}) }}
+{# Notes on max-connection-count: On a standard ERP5, each transaction
+   can have 4 connections to mariadb: activities, catalog, deferred and
+   transactionless. Count 5 to have some headroom. Multiply by the total
+   number of zope threads for all processes from all partitions to get the
+   expected number of connections. Add 50 for have some more zope-independent
+   headroom (automated probes, replication, ...).
+-#}
 {{ request('mariadb', 'mariadb', 'mariadb',
   {
     'tcpv4-port': 2099,
     'max-slowqueries-threshold': monitor_dict.get('max-slowqueries-threshold', 1000),
     'slowest-query-threshold': monitor_dict.get('slowest-query-threshold', ''),
     'test-database-amount': test_runner_total_database_count,
+    'max-connection-count': sum(partition_thread_count_list) * 5 + 50,
   },
   {
     'database-list': True,
@@ -149,7 +165,6 @@ connection-url = smtp://127.0.0.2:0/
 {% endfor -%}
 
 
-{% set zope_partition_dict = slapparameter_dict.get('zope-partition-dict', {'1': {}}) -%}
 {% set zope_address_list_id_dict = {} -%}
 {% if zope_partition_dict -%}
 
@@ -231,9 +246,9 @@ name = {{ partition_name }}
 {% do monitor_base_url_dict.__setitem__(section_name, '${' ~ section_name ~ ':connection-monitor-base-url}') -%}
 {{ root_common.sla(partition_name) }}
 config-name = {{ dumps(custom_name) }}
-config-instance-count = {{ dumps(zope_parameter_dict.get('instance-count', 1)) }}
+config-instance-count = {{ dumps(zope_parameter_dict['instance-count']) }}
 config-private-dev-shm = {{ zope_parameter_dict.get('private-dev-shm', '') }}
-config-thread-amount = {{ dumps(zope_parameter_dict.get('thread-amount', 4)) }}
+config-thread-amount = {{ dumps(zope_parameter_dict['thread-amount']) }}
 config-timerserver-interval = {{ dumps(zope_parameter_dict.get('timerserver-interval', 5)) }}
 config-longrequest-logger-interval = {{ dumps(zope_parameter_dict.get('longrequest-logger-interval', -1)) }}
 config-longrequest-logger-timeout = {{ dumps(zope_parameter_dict.get('longrequest-logger-timeout', 1)) }}

stack/erp5/instance-mariadb.cfg.in

@@ -133,6 +133,7 @@ pid-file = ${directory:run}/mariadb.pid
 error-log = ${directory:log}/mariadb_error.log
 slow-query-log = ${directory:log}/mariadb_slowquery.log
 long-query-time = {{ dumps(slapparameter_dict.get('long-query-time', 1)) }}
+max-connection-count = {{ dumps(slapparameter_dict.get('max-connection-count', 1000)) }}
 innodb-buffer-pool-size = {{ dumps(slapparameter_dict.get('innodb-buffer-pool-size', 0)) }}
 innodb-buffer-pool-instances = {{ dumps(slapparameter_dict.get('innodb-buffer-pool-instances', 0)) }}
 innodb-log-file-size = {{ dumps(slapparameter_dict.get('innodb-log-file-size', 0)) }}

stack/erp5/my.cnf.in

@@ -31,9 +31,7 @@ innodb_file_per_table = {{ parameter_dict['innodb-file-per-table'] }}
 
 plugin_load = ha_mroonga
 
-# By default only 100 connections are allowed, when using zeo
-# we may have much more connections
-max_connections = 1000
+max_connections = {{ parameter_dict['max-connection-count'] }}
 
 {% set innodb_buffer_pool_size = parameter_dict['innodb-buffer-pool-size'] -%}
 {% if innodb_buffer_pool_size %}innodb_buffer_pool_size = {{ innodb_buffer_pool_size }}{% endif %}