diff --git a/software/slapos-master/apache-backend.conf.in b/software/slapos-master/apache-backend.conf.in index 98d1c3e06c966227390d30dc875d43b6b735b3cd..68f1977d7d9358292a8f2abf26cfa485ad81c31b 100644 --- a/software/slapos-master/apache-backend.conf.in +++ b/software/slapos-master/apache-backend.conf.in @@ -46,17 +46,42 @@ # # # The list of backends which apache should redirect to. # "backend-list": [ - # # (port, unused, internal_scheme) - # (8000, _, "http://10.0.0.10:8001"), - # (8002, _, "http://10.0.0.10:8003"), + # # (port, unused, internal_scheme, enable_authentication) + # (8000, _, "http://10.0.0.10:8001", True), + # (8002, _, "http://10.0.0.10:8003", False), # ], + # + # # The mapping of zope paths this apache should redirect to. + # # This is a Zope specific feature. + # # `enable_authentication` has same meaning as for `backend-list`. + # "zope-virtualhost-monster-backend-dict": { + # # {(ip, port): ( enable_authentication, {frontend_path: ( internal_scheme ) }, ) } + # ('[::1]', 8004): ( + # True, { + # 'zope-1': 'http://10.0.0.10:8001', + # 'zope-2': 'http://10.0.0.10:8002', + # }, + # ), + # }, # } # - # This sample of `parameter_dict` will make apache listening to : - # - 0.0.0.0:8000 redirecting internaly to http://10.0.0.10:8001 - # - [::1]:8000 redirecting internaly to http://10.0.0.10:8001 - # - 0.0.0.0:8002 redirecting internaly to http://10.0.0.10:8003 - # - [::1]:8002 redirecting internaly to http://10.0.0.10:8003 + # This sample of `parameter_dict` will make apache listening to : + # From to `backend-list`: + # - 0.0.0.0:8000 redirecting internaly to http://10.0.0.10:8001 and + # - [::1]:8000 redirecting internaly to http://10.0.0.10:8001 + # only accepting requests from clients who provide a valid SSL certificate trusted in `ca-cert`. + # - 0.0.0.0:8002 redirecting internaly to http://10.0.0.10:8003 + # - [::1]:8002 redirecting internaly to http://10.0.0.10:8003 + # accepting requests from any client. + # + # From zope-virtualhost-monster-backend-dict`: + # - [::1]:8004 with some path based rewrite-rules redirecting to: + # * http://10.0.0.10/8001 when path matches /zope-1(.*) + # * http://10.0.0.10/8002 when path matches /zope-2(.*) + # with some VirtualHostMonster rewrite rules so zope writes URLs with + # [::1]:8004 as server name. + # For more details, refer to + # https://docs.zope.org/zope2/zope2book/VirtualHosting.html#using-virtualhostroot-and-virtualhostbase-together -#} LoadModule unixd_module modules/mod_unixd.so LoadModule access_compat_module modules/mod_access_compat.so diff --git a/software/slapos-master/buildout.hash.cfg b/software/slapos-master/buildout.hash.cfg index 16260affcdb73a6f779a8a0f87cfd356ded99b55..48bdc6727f35e87f828c3b8fecb9e1194eb9302b 100644 --- a/software/slapos-master/buildout.hash.cfg +++ b/software/slapos-master/buildout.hash.cfg @@ -14,12 +14,12 @@ # not need these here). [template-erp5] filename = instance-erp5.cfg.in -md5sum = 92f1e522d8e7d0e9a9f7b4db63f092e4 +md5sum = 6dfda47ff95a3fa768382d8002c8a780 [template-balancer] filename = instance-balancer.cfg.in -md5sum = 697e43a020af5edee2151987abc8e7b4 +md5sum = 3c7afbf160b943d2a10f527722792a23 [template-apache-backend-conf] filename = apache-backend.conf.in -md5sum = 516143f5e8a3032a7b7b82741d3a46b7 +md5sum = e0a7b027cb52e5fa21ab64cfa7298f35 diff --git a/software/slapos-master/instance-balancer.cfg.in b/software/slapos-master/instance-balancer.cfg.in index 4fe2b19d862f05e4172139e9c07947ff6d572adf..3686c549b831b3b850d71c759fd0af578bf4ab83 100644 --- a/software/slapos-master/instance-balancer.cfg.in +++ b/software/slapos-master/instance-balancer.cfg.in @@ -98,6 +98,8 @@ ipv4 = {{ ipv4 }} {% endif -%} {% set haproxy_dict = {} -%} {% set apache_dict = {} -%} +{% set zope_virtualhost_monster_backend_dict = {} %} +{% set test_runner_url_dict = {} %} {# family_name => list of apache URLs #} {% set next_port = itertools.count(slapparameter_dict['tcpv4-port']).next -%} {% for family_name, parameter_id_list in sorted( slapparameter_dict['zope-family-dict'].iteritems()) -%} @@ -122,8 +124,27 @@ ipv6 = {{ zope_address.split(']:')[0][1:] }} {% set zope_effective_address = zope_address -%} {% endif -%} {% do zope_family_address_list.append((zope_effective_address, maxconn, webdav)) -%} + +{# # Generate entries with rewrite rule for test runnners #} +{% set test_runner_backend_mapping = {} %} +{% set test_runner_apache_url_list = [] %} +{% set test_runner_external_port = next_port() %} +{% for i, (test_runner_internal_ip, test_runner_internal_port) in + enumerate(slapparameter_dict[parameter_id ~ '-test-runner-address-list']) %} +{% do test_runner_backend_mapping.__setitem__( + 'unit_test_' ~ i, + 'http://' ~ test_runner_internal_ip ~ ':' ~ test_runner_internal_port ) %} +{% do test_runner_apache_url_list.append( + 'https://' ~ ipv4 ~ ':' ~ test_runner_external_port ~ '/unit_test_' ~ i ~ '/' ) %} +{% endfor %} +{% do zope_virtualhost_monster_backend_dict.__setitem__( + (ipv4, test_runner_external_port), + ( ssl_authentication, test_runner_backend_mapping ) ) -%} +{% do test_runner_url_dict.__setitem__(family_name, test_runner_apache_url_list) -%} + {% endfor -%} {% endfor -%} + {# Make rendering fail artificially if any family has no known backend. # This is useful as haproxy's hot-reconfiguration mechanism is # supervisord-incompatible. @@ -162,6 +183,7 @@ extensions = jinja2.ext.do recipe = slapos.cookbook:wrapper wrapper-path = ${directory:services}/haproxy command-line = "{{ parameter_dict['haproxy'] }}/sbin/haproxy" -f "${haproxy-cfg:rendered}" +hash-files = ${haproxy-cfg:rendered} {# TODO: build socat and wrap it as "${directory:bin}/haproxy-ctl" to connect to "${haproxy-cfg-parameter-dict:socket-path}" #} @@ -173,6 +195,7 @@ crl = ${directory:apache-conf}/crl.pem [apache-conf-parameter-dict] backend-list = {{ dumps(apache_dict.values()) }} +zope-virtualhost-monster-backend-dict = {{ dumps(zope_virtualhost_monster_backend_dict) }} ip-list = {{ dumps(apache_ip_list) }} pid-file = ${directory:run}/apache.pid log-dir = ${directory:log} @@ -230,6 +253,10 @@ recipe = slapos.cookbook:publish.serialised {{ family_name ~ '-v6' }} = {% if ipv6_set %}{{ scheme ~ '://[' ~ ipv6 ~ ']:' ~ apache_port }}{% endif %} {{ family_name }} = {{ scheme ~ '://' ~ ipv4 ~ ':' ~ apache_port }} {% endfor -%} +{% for family_name, test_runner_url_list in test_runner_url_dict.items() -%} +{{ family_name ~ '-test-runner-url-list' }} = {{ dumps(test_runner_url_list) }} +{% endfor -%} + monitor-base-url = ${monitor-publish-parameters:monitor-base-url} [apache-ssl] @@ -304,10 +331,19 @@ crl = ${:ca-dir}/crl apachedex = ${monitor-directory:private}/apachedex [{{ section('monitor-generate-apachedex-report') }}] +recipe = slapos.cookbook:cron.d +cron-entries = ${cron:cron-entries} +name = generate-apachedex-report +# The goal is to be executed before logrotate log rotation. +# Here, logrotate-entry-base:frequency = daily, so we run at 23 o'clock every day. +frequency = 0 23 * * * +command = ${monitor-generate-apachedex-report-wrapper:wrapper-path} + +[monitor-generate-apachedex-report-wrapper] recipe = slapos.cookbook:wrapper -wrapper-path = ${monitor-directory:reports}/${:command} +wrapper-path = ${directory:bin}/${:command} command-line = "{{ parameter_dict['run-apachedex-location'] }}" "{{ parameter_dict['apachedex-location'] }}" "${directory:apachedex}" ${monitor-publish-parameters:monitor-base-url}/private/apachedex --apache-log-list "${apachedex-parameters:apache-log-list}" --configuration "${apachedex-parameters:configuration}" -command = apachedex_every_23_hour +command = generate-apachedex-report [apachedex-parameters] # XXX - Sample log file with curent date: apache_access.log-%(date)s.gz @@ -321,6 +357,11 @@ recipe = slapos.cookbook:wrapper wrapper-path = ${directory:promise}/check-apachedex-result command-line = "{{ parameter_dict['promise-check-apachedex-result'] }}" --apachedex_path "${directory:apachedex}" --status_file ${monitor-directory:private}/apachedex.report.json --threshold "${apachedex-parameters:promise-threshold}" +[{{ section('promise-check-computer-memory') }}] +recipe = slapos.cookbook:wrapper +wrapper-path = ${directory:promise}/check-computer-memory +command-line = "{{ parameter_dict["check-computer-memory-binary"] }}" -db ${monitor-instance-parameter:collector-db} --threshold "{{ slapparameter_dict["computer-memory-percent-threshold"] }}" --unit percent + [monitor-instance-parameter] monitor-httpd-ipv6 = {{ (ipv6_set | list)[0] }} monitor-httpd-port = {{ next_port() }} diff --git a/software/slapos-master/instance-erp5.cfg.in b/software/slapos-master/instance-erp5.cfg.in index c5b3e73c09bae2db07aca1b9fd824bc7297eea2a..2350cf65b1934ddb4e5452836da23d92b9188c73 100644 --- a/software/slapos-master/instance-erp5.cfg.in +++ b/software/slapos-master/instance-erp5.cfg.in @@ -8,6 +8,16 @@ {% set jupyter_dict = slapparameter_dict.get('jupyter', {}) -%} {% set has_jupyter = jupyter_dict.get('enable', jupyter_enable_default.lower() in ('true', 'yes')) -%} {% set jupyter_zope_family = jupyter_dict.get('zope-family', '') -%} +{% set test_runner_enabled = slapparameter_dict.get('test-runner', {}).get('enabled', True) -%} +{% set test_runner_node_count = slapparameter_dict.get('test-runner', {}).get('node-count', 3) -%} +{% set test_runner_extra_database_count = slapparameter_dict.get('test-runner', {}).get('extra-database-count', 3) -%} +{% set test_runner_total_database_count = test_runner_node_count * (1 + test_runner_extra_database_count) -%} +{# Backward compatibility for mariadb.test-database-amount #} +{% set mariadb_test_database_amount = slapparameter_dict.get('mariadb', {}).get('test-database-amount') -%} +{% if mariadb_test_database_amount is not none -%} +{% set test_runner_total_database_count = mariadb_test_database_amount %} +{% set test_runner_enabled = mariadb_test_database_amount > 0 %} +{% endif -%} {% set monitor_base_url_dict = {} -%} {% set caucase_url = slapparameter_dict.get('caucase', {}).pop('url', '') -%} {% set monitor_dict = slapparameter_dict.get('monitor', {}) %} @@ -15,6 +25,7 @@ [request-common] <= request-common-base config-use-ipv6 = {{ dumps(slapparameter_dict.get('use-ipv6', False)) }} +config-computer-memory-percent-threshold = {{ dumps(monitor_dict.get('computer-memory-percent-threshold', 80)) }} {% macro request(name, software_type, config_key, config, ret={'url': True}, key_config={}) -%} {% do config.update(slapparameter_dict.get(config_key, {})) -%} @@ -44,7 +55,7 @@ config-name = {{ name }} {{ request('memcached-persistent', 'kumofs', 'kumofs', {'tcpv4-port': 2000}, {'url': True, 'monitor-base-url': False}, key_config={'monitor-passwd': 'monitor-htpasswd:passwd'}) }} {{ request('memcached-volatile', 'kumofs', 'memcached', {'tcpv4-port': 2010, 'ram-storage-size': 64}, {'url': True, 'monitor-base-url': False}, key_config={'monitor-passwd': 'monitor-htpasswd:passwd'}) }} -{{ request('mariadb', 'mariadb', 'mariadb', {'tcpv4-port': 2099, 'max-slowqueries-threshold': monitor_dict.get('max-slowqueries-threshold', 1000), 'slowest-query-threshold': monitor_dict.get('slowest-query-threshold', '') }, {'database-list': True, 'test-database-list': True, 'monitor-base-url': False}, key_config={'monitor-passwd': 'monitor-htpasswd:passwd'}) }} +{{ request('mariadb', 'mariadb', 'mariadb', {'tcpv4-port': 2099, 'max-slowqueries-threshold': monitor_dict.get('max-slowqueries-threshold', 1000), 'slowest-query-threshold': monitor_dict.get('slowest-query-threshold', ''), 'test-database-amount': test_runner_total_database_count}, {'database-list': True, 'test-database-list': True, 'monitor-base-url': False}, key_config={'monitor-passwd': 'monitor-htpasswd:passwd'}) }} {% if has_posftix -%} {{ request('smtp', 'postfix', 'smtp', {'tcpv4-port': 2025, 'smtpd-sasl-user': 'erp5@nowhere'}, key_config={'smtpd-sasl-password': 'publish-early:smtpd-sasl-password'}) }} {%- else %} @@ -93,45 +104,7 @@ connection-http-url = {{ caucase_url }} {% endif -%} {% endfor -%} -[publish-early] -recipe = slapos.cookbook:publish-early --init = - inituser-password gen-password:passwd - deadlock-debugger-password gen-deadlock-debugger-password:passwd -{%- if has_posftix %} - smtpd-sasl-password gen-smtpd-sasl-password:passwd -{%- endif %} -{%- if neo %} - neo-cluster gen-neo-cluster:name -{%- if neo[0] %} -neo-cluster = {{ dumps(neo[0]) }} -{%- endif %} -{%- endif %} -{%- set inituser_password = slapparameter_dict.get('inituser-password') %} -{%- if inituser_password %} -inituser-password = {{ dumps(inituser_password) }} -{%- endif %} -{%- set deadlock_debugger_password = slapparameter_dict.get('deadlock-debugger-password') -%} -{%- if deadlock_debugger_password %} -deadlock-debugger-password = {{ dumps(deadlock_debugger_password) }} -{%- endif %} - -[gen-password] -recipe = slapos.cookbook:generate.password -storage-path = - -[gen-deadlock-debugger-password] -<= gen-password - -[gen-neo-cluster-base] -<= gen-password - -[gen-neo-cluster] -name = neo-${gen-neo-cluster-base:passwd} - -[gen-smtpd-sasl-password] -< = gen-password - +{% set zope_partition_dict = slapparameter_dict.get('zope-partition-dict', {'1': {}}) -%} {% set zope_partition_dict = slapparameter_dict.get('zope-partition-dict', {'1': {}}) -%} {% set zope_address_list_id_dict = {} -%} {% if zope_partition_dict -%} @@ -142,6 +115,9 @@ return = zope-address-list hosts-dict monitor-base-url +{%- if test_runner_enabled %} + test-runner-address-list +{% endif %} {% set bt5_default_list = 'erp5_full_text_myisam_catalog slapos_configurator' -%} {% if has_jupyter -%} {% set bt5_default_list = bt5_default_list + ' erp5_data_notebook' -%} @@ -152,6 +128,7 @@ config-caucase-url = ${request-caucase:connection-http-url} config-cloudooo-url = {{ dumps(slapparameter_dict.get('cloudooo-url', default_cloudooo_url)) }} config-deadlock-debugger-password = ${publish-early:deadlock-debugger-password} config-developer-list = {{ dumps(slapparameter_dict.get('developer-list', [inituser_login])) }} +config-saucelabs-dict = {{ dumps(slapparameter_dict.get('saucelabs-dict', {})) }} config-hosts-dict = {{ dumps(slapparameter_dict.get('hosts-dict', {})) }} config-hostalias-dict = {{ dumps(slapparameter_dict.get('hostalias-dict', {})) }} config-id-store-interval = {{ dumps(slapparameter_dict.get('id-store-interval')) }} @@ -169,6 +146,8 @@ config-cloudooo-retry-count = {{ slapparameter_dict.get('cloudooo-retry-count', config-wendelin-core-zblk-fmt = {{ dumps(slapparameter_dict.get('wendelin-core-zblk-fmt', '')) }} config-ca-path = ${directory:ca-dir} config-zodb-dict = {{ dumps(zodb_dict) }} +config-test-runner-enabled = {{ dumps(test_runner_enabled) }} +config-test-runner-node-count = {{ dumps(test_runner_node_count) }} {% for server_type, server_dict in storage_dict.iteritems() -%} {% if server_type == 'neo' -%} config-neo-cluster = ${publish-early:neo-cluster} @@ -183,6 +162,7 @@ config-tidstorage-port = ${request-zodb:connection-tidstorage-port} software-type = zope {% set zope_family_dict = {} -%} +{% set zope_family_name_list = [] -%} {% set zope_backend_path_dict = {} -%} {% set ssl_authentication_dict = {} -%} {% set jupyter_zope_family_default = [] -%} @@ -190,6 +170,7 @@ software-type = zope {% set partition_name = 'zope-' ~ custom_name -%} {% set section_name = 'request-' ~ partition_name -%} {% set zope_family = zope_parameter_dict.get('family', 'default') -%} +{% do zope_family_name_list.append(zope_family) %} {% set backend_path = zope_parameter_dict.get('backend-path', '/') % {'site-id': site_id} %} {# # default jupyter zope family is first zope family. -#} {# # use list.append() to update it, because in jinja2 set changes only local scope. -#} @@ -214,6 +195,7 @@ config-longrequest-logger-timeout = {{ dumps(zope_parameter_dict.get('longreques config-large-file-threshold = {{ dumps(zope_parameter_dict.get('large-file-threshold', "10MB")) }} config-port-base = {{ dumps(zope_parameter_dict.get('port-base', 2200)) }} config-webdav = {{ dumps(zope_parameter_dict.get('webdav', False)) }} +config-test-runner-apache-url-list = ${publish-early:{{ zope_family }}-test-runner-url-list} {% endfor -%} {# if not explicitly configured, connect jupyter to first zope family, which -#} @@ -309,6 +291,9 @@ return = {%- for family in zope_family_dict %} {{ family }} {{ family }}-v6 + {% if test_runner_enabled %} + {{ family }}-test-runner-url-list + {% endif %} {% endfor -%} {% do monitor_base_url_dict.__setitem__('request-balancer', '${' ~ 'request-balancer' ~ ':connection-monitor-base-url}') -%} @@ -316,6 +301,9 @@ config-zope-family-dict = {{ dumps(zope_family_parameter_dict) }} config-tcpv4-port = {{ dumps(balancer_dict.get('tcpv4-port', 2150)) }} {% for zope_section_id, name in zope_address_list_id_dict.items() -%} config-{{ name }} = {{ ' ${' ~ zope_section_id ~ ':connection-zope-address-list}' }} +{% if test_runner_enabled -%} +config-{{ name }}-test-runner-address-list = {{ ' ${' ~ zope_section_id ~ ':connection-test-runner-address-list}' }} +{% endif -%} {% endfor -%} # XXX: should those really be same for all families ? config-haproxy-server-check-path = {{ dumps(balancer_dict.get('haproxy-server-check-path', '/') % {'site-id': site_id}) }} @@ -377,6 +365,57 @@ hosts-dict = {{ '${' ~ zope_address_list_id_dict.keys()[0] ~ ':connection-hosts- {% for name, value in publish_dict.items() -%} {{ name }} = {{ value }} {% endfor -%} +{% for zope_family_name in zope_family_name_list -%} +{{ zope_family_name }}-test-runner-url-list = ${request-balancer:connection-{{ zope_family_name }}-test-runner-url-list} +{% endfor -%} + + +[publish-early] +recipe = slapos.cookbook:publish-early +-init = + inituser-password gen-password:passwd + deadlock-debugger-password gen-deadlock-debugger-password:passwd +{%- if has_posftix %} + smtpd-sasl-password gen-smtpd-sasl-password:passwd +{%- endif %} +{% for zope_family_name in zope_family_name_list %} + {{ zope_family_name }}-test-runner-url-list default-balancer-test-runner-url-list:default +{% endfor -%} +{%- if neo %} + neo-cluster gen-neo-cluster:name +{%- if neo[0] %} +neo-cluster = {{ dumps(neo[0]) }} +{%- endif %} +{%- endif %} +{%- set inituser_password = slapparameter_dict.get('inituser-password') %} +{%- if inituser_password %} +inituser-password = {{ dumps(inituser_password) }} +{%- endif %} +{%- set deadlock_debugger_password = slapparameter_dict.get('deadlock-debugger-password') -%} +{%- if deadlock_debugger_password %} +deadlock-debugger-password = {{ dumps(deadlock_debugger_password) }} +{%- endif %} + + +[default-balancer-test-runner-url-list] +recipe = +default = not-ready + +[gen-password] +recipe = slapos.cookbook:generate.password +storage-path = + +[gen-deadlock-debugger-password] +<= gen-password + +[gen-neo-cluster-base] +<= gen-password + +[gen-neo-cluster] +name = neo-${gen-neo-cluster-base:passwd} + +[gen-smtpd-sasl-password] +< = gen-password [monitor-instance-parameter] monitor-httpd-port = 8386