caddy-frontend: Provide backend-haproxy statistic

Each node allows for global statistic access for full backend-haproxy, which
is exposed using special frontend, and then transferred back to the master
partition, so that the administrator can access it.

software/caddy-frontend/buildout.hash.cfg

@@ -14,7 +14,7 @@
 # not need these here).
 [template]
 filename = instance.cfg.in
-md5sum = 7b16383a41d403f6c21e684afffcb4b7
+md5sum = bfb647325103640c19c38e7b7f6e2833
 
 [template-common]
 filename = instance-common.cfg.in
@@ -22,15 +22,15 @@ md5sum = 5784bea3bd608913769ff9a8afcccb68
 
 [template-apache-frontend]
 filename = instance-apache-frontend.cfg.in
-md5sum = b908d09e07b479cf09f1c813522e5cac
+md5sum = 02aac183352a6fd6ddd336d2e3757405
 
 [template-caddy-replicate]
 filename = instance-apache-replicate.cfg.in
-md5sum = b684aacbb7b2a6eec3ca3ab06c61a92c
+md5sum = 54c0648c8593699dae0c565bc7dd8629
 
 [template-slave-list]
 _update_hash_filename_ = templates/apache-custom-slave-list.cfg.in
-md5sum = e9c5fe132a26e960df0acf3f5d8f877e
+md5sum = 2690fb2b5b6cefb7de53f35c214bdd52
 
 [template-replicate-publish-slave-information]
 _update_hash_filename_ = templates/replicate-publish-slave-information.cfg.in
@@ -54,7 +54,7 @@ md5sum = 266f175dbdfc588af7a86b0b1884fe73
 
 [template-backend-haproxy-configuration]
 _update_hash_filename_ = templates/backend-haproxy.cfg.in
-md5sum = eaf7491f628e72c877c31588e62eeb89
+md5sum = 80081a7ded0029bb24b4fe2d06b6ae95
 
 [template-log-access]
 _update_hash_filename_ = templates/template-log-access.conf.in

software/caddy-frontend/instance-apache-frontend.cfg.in

@@ -738,6 +738,13 @@ certificate = ${backend-client-login-config:certificate}
 cas-ca-certificate = ${backend-client-login-config:cas-ca-certificate}
 csr = ${backend-client-caucase-updater-csr:csr}
 crl = ${backend-client-login-config:crl}
+# the statistic page
+statistic-certificate = ${self-signed-ip-access:certificate}
+statistic-port = ${configuration:backend-haproxy-statistic-port}
+statistic-username = ${monitor-instance-parameter:username}
+statistic-password = ${monitor-htpasswd:passwd}
+statistic-identification = {{ slapparameter_dict['cluster-identification'] }}
+statistic-frontend-secure_access = ${backend-haproxy-statistic-frontend:connection-secure_access}
 
 [backend-haproxy]
 recipe = slapos.cookbook:wrapper
@@ -930,6 +937,16 @@ config-url = https://[${slap-network-information:global-ipv6}]:{{ instance_param
 config-https-only = true
 return = domain secure_access
 
+[backend-haproxy-statistic-frontend]
+<= slap-connection
+recipe = slapos.cookbook:requestoptional
+name = Backend Haproxy Statistic Frontend {{ instance_parameter['configuration.frontend-name'] }}
+software-url = http://git.erp5.org/gitweb/slapos.git/blob_plain/HEAD:/software/apache-frontend/software.cfg
+slave = true
+config-url = https://[${slap-network-information:global-ipv6}]:{{ instance_parameter['configuration.backend-haproxy-statistic-port'] }}/
+config-https-only = true
+return = domain secure_access
+
 [slave-introspection-configuration-state]
 <= jinja2-template-base
 template = {{ parameter_dict['template_configuration_state_script'] }}

software/caddy-frontend/instance-apache-replicate.cfg.in

@@ -215,7 +215,7 @@ config-monitor-username = ${monitor-instance-parameter:username}
 config-monitor-password = ${monitor-htpasswd:passwd}
 
 software-type = {{frontend_type}}
-return = private-ipv4 public-ipv4 slave-instance-information-list monitor-base-url backend-client-csr_id-url csr_id-url csr_id-certificate
+return = private-ipv4 public-ipv4 slave-instance-information-list monitor-base-url backend-client-csr_id-url csr_id-url csr_id-certificate backend-haproxy-statistic-url
 
 {% for section, frontend_request in request_dict.iteritems() %}
 {%   set state = frontend_request.get('state', '') %}
@@ -292,6 +292,10 @@ kedifa-csr_id-certificate = ${request-kedifa:connection-csr_id-certificate}
 {{ frontend }}-csr_id-url = {{ section_part }}:connection-csr_id-url}
 {% endfor %}
 {% endif %}
+{% for frontend in frontend_list %}
+  {% set section_part = '${request-' + frontend %}
+{{ frontend }}-backend-haproxy-statistic-url = {{ section_part }}:connection-backend-haproxy-statistic-url}
+{% endfor %}
 {% if not aibcc_enabled %}
 {% for frontend in frontend_list %}
   {% set section_part = '${request-' + frontend %}

software/caddy-frontend/instance.cfg.in

@@ -121,6 +121,7 @@ configuration.backend-connect-timeout = 5
 configuration.backend-connect-retries = 3
 configuration.backend-haproxy-http-port = 21080
 configuration.backend-haproxy-https-port = 21443
+configuration.backend-haproxy-statistic-port = 21444
 configuration.authenticate-to-backend = False
 configuration.rotate-num = 4000
 configuration.slave-introspection-https-port = 22443

software/caddy-frontend/templates/apache-custom-slave-list.cfg.in

@@ -394,6 +394,13 @@ monitor-base-url = {{ monitor_base_url }}
 csr_id-url = https://[${expose-csr_id-configuration:ip}]:${expose-csr_id-configuration:port}/csr_id.txt
 backend-client-csr_id-url = https://[${expose-csr_id-configuration:ip}]:${expose-csr_id-configuration:port}/backend-haproxy-csr_id.txt
 csr_id-certificate = ${get-csr_id-certificate:certificate}
+{%-   set furled = furl_module.furl(backend_haproxy_configuration['statistic-frontend-secure_access']) %}
+{%-   do furled.set(username = backend_haproxy_configuration['statistic-username']) %}
+{%-   do furled.set(password = backend_haproxy_configuration['statistic-password']) %}
+{%-   do furled.set(path = '/') %}
+{#-   We unquote, as furl quotes automatically, but there is buildout value on purpose like ${...:...} in the passwod #}
+{%-   set statistic_url = urlparse_module.unquote(furled.tostr()) %}
+backend-haproxy-statistic-url = {{ statistic_url }}
 
 [kedifa-updater]
 recipe = slapos.cookbook:wrapper
@@ -442,16 +449,14 @@ extra-context =
   section configuration backend-haproxy-config
 
 [backend-haproxy-config]
-file = {{ backend_haproxy_configuration['file'] }}
-pid-file = {{ backend_haproxy_configuration['pid-file'] }}
-log-socket = {{ backend_haproxy_configuration['log-socket'] }}
+{%- for key, value in backend_haproxy_configuration.items() %}
+{{ key }} = {{ value }}
+{%- endfor %}
 local-ipv4 = {{ dumps('' ~ local_ipv4) }}
-http-port = {{ ('' ~ backend_haproxy_configuration['http-port']) }}
-https-port = {{ ('' ~ backend_haproxy_configuration['https-port']) }}
+global-ipv6 = ${slap-network-information:global-ipv6}
 request-timeout = {{ dumps('' ~ configuration['request-timeout']) }}
 backend-connect-timeout = {{ dumps('' ~ configuration['backend-connect-timeout']) }}
 backend-connect-retries =  {{ dumps('' ~ configuration['backend-connect-retries']) }}
-certificate = {{ dumps('' ~ backend_haproxy_configuration['certificate']) }}
 
 [store-backend-haproxy-csr_id]
 recipe = plone.recipe.command

software/caddy-frontend/templates/backend-haproxy.cfg.in

@@ -38,6 +38,15 @@ defaults
 {%-   endif %}
 {%- endmacro %}
 
+# statistic
+frontend statistic
+  bind {{ configuration['global-ipv6']}}:{{ configuration['statistic-port'] }} ssl crt {{ configuration['statistic-certificate'] }}
+  stats enable
+  stats uri /
+  stats show-desc {{ configuration['statistic-identification'] }}
+  stats auth {{ configuration['statistic-username'] }}:{{ configuration['statistic-password'] }}
+  stats realm {{ configuration['statistic-identification'] }}
+
 frontend http-backend
   bind {{ configuration['local-ipv4'] }}:{{ configuration['http-port'] }}
 {%- for slave_instance in backend_slave_list %}

software/caddy-frontend/test/test.py

@@ -726,6 +726,27 @@ class HttpFrontendTestCase(SlapOSInstanceTestCase):
 
     return generate_auth_url, upload_url
 
+  def assertBackendHaproxyStatisticUrl(self, parameter_dict):
+    url_key = 'caddy-frontend-1-backend-haproxy-statistic-url'
+    backend_haproxy_statistic_url_dict = {}
+    for key in parameter_dict.keys():
+      if key.startswith('caddy-frontend') and key.endswith(
+        'backend-haproxy-statistic-url'):
+        backend_haproxy_statistic_url_dict[key] = parameter_dict.pop(key)
+    self.assertEqual(
+      [url_key],
+      backend_haproxy_statistic_url_dict.keys()
+    )
+
+    backend_haproxy_statistic_url = backend_haproxy_statistic_url_dict[url_key]
+    result = requests.get(
+      backend_haproxy_statistic_url,
+      verify=False,
+    )
+    self.assertEqual(httplib.OK, result.status_code)
+    self.assertIn('testing partition 0', result.text)
+    self.assertIn('Statistics Report for HAProxy', result.text)
+
   def assertKeyWithPop(self, key, d):
     self.assertTrue(key in d, 'Key %r is missing in %r' % (key, d))
     d.pop(key)
@@ -1042,6 +1063,7 @@ class TestMasterRequestDomain(HttpFrontendTestCase, TestDataMixin):
   def test(self):
     parameter_dict = self.parseConnectionParameterDict()
     self.assertKeyWithPop('monitor-setup-url', parameter_dict)
+    self.assertBackendHaproxyStatisticUrl(parameter_dict)
     self.assertKedifaKeysWithPop(parameter_dict, 'master-')
     self.assertRejectedSlavePromiseWithPop(parameter_dict)
 
@@ -1072,6 +1094,7 @@ class TestMasterRequest(HttpFrontendTestCase, TestDataMixin):
   def test(self):
     parameter_dict = self.parseConnectionParameterDict()
     self.assertKeyWithPop('monitor-setup-url', parameter_dict)
+    self.assertBackendHaproxyStatisticUrl(parameter_dict)
     self.assertKedifaKeysWithPop(parameter_dict, 'master-')
     self.assertRejectedSlavePromiseWithPop(parameter_dict)
     self.assertEqual(
@@ -1543,6 +1566,7 @@ class TestSlave(SlaveHttpFrontendTestCase, TestDataMixin):
   def test_master_partition_state(self):
     parameter_dict = self.parseConnectionParameterDict()
     self.assertKeyWithPop('monitor-setup-url', parameter_dict)
+    self.assertBackendHaproxyStatisticUrl(parameter_dict)
     self.assertKedifaKeysWithPop(parameter_dict, 'master-')
     self.assertRejectedSlavePromiseWithPop(parameter_dict)
 
@@ -5229,6 +5253,7 @@ class TestSlaveSlapOSMasterCertificateCompatibility(
   def test_master_partition_state(self):
     parameter_dict = self.parseConnectionParameterDict()
     self.assertKeyWithPop('monitor-setup-url', parameter_dict)
+    self.assertBackendHaproxyStatisticUrl(parameter_dict)
     self.assertKedifaKeysWithPop(parameter_dict, 'master-')
     self.assertRejectedSlavePromiseWithPop(parameter_dict)
 
@@ -5905,6 +5930,7 @@ class TestSlaveSlapOSMasterCertificateCompatibilityUpdate(
   def test_master_partition_state(self):
     parameter_dict = self.parseConnectionParameterDict()
     self.assertKeyWithPop('monitor-setup-url', parameter_dict)
+    self.assertBackendHaproxyStatisticUrl(parameter_dict)
     self.assertKedifaKeysWithPop(parameter_dict, 'master-')
     self.assertRejectedSlavePromiseWithPop(parameter_dict)
 
@@ -6011,6 +6037,7 @@ class TestSlaveCiphers(SlaveHttpFrontendTestCase, TestDataMixin):
   def test_master_partition_state(self):
     parameter_dict = self.parseConnectionParameterDict()
     self.assertKeyWithPop('monitor-setup-url', parameter_dict)
+    self.assertBackendHaproxyStatisticUrl(parameter_dict)
     self.assertKedifaKeysWithPop(parameter_dict, 'master-')
     self.assertRejectedSlavePromiseWithPop(parameter_dict)
 
@@ -6206,6 +6233,7 @@ class TestSlaveRejectReportUnsafeDamaged(SlaveHttpFrontendTestCase):
   def test_master_partition_state(self):
     parameter_dict = self.parseConnectionParameterDict()
     self.assertKeyWithPop('monitor-setup-url', parameter_dict)
+    self.assertBackendHaproxyStatisticUrl(parameter_dict)
     self.assertKedifaKeysWithPop(parameter_dict, 'master-')
     self.assertRejectedSlavePromiseWithPop(parameter_dict)