caddy-frontend: Deduplicate local host names

custom_domain and server-alias on given slave do not have to clash, and
can be deduplicated during request parameter analysis.

/reviewed-on nexedi/slapos!444

software/caddy-frontend/buildout.hash.cfg

@@ -26,7 +26,7 @@ md5sum = ab1795f92e32655d05c662c965d2b1f5
 
 [template-apache-replicate]
 filename = instance-apache-replicate.cfg.in
-md5sum = a0110d0ec69875946a16ac3a4b269eb8
+md5sum = 44d50bf8391b5a73b2ab72923efe6437
 
 [template-slave-list]
 filename = templates/apache-custom-slave-list.cfg.in
@@ -62,7 +62,7 @@ md5sum = f20d6c3d2d94fb685f8d26dfca1e822b
 
 [template-default-slave-virtualhost]
 filename = templates/default-virtualhost.conf.in
-md5sum = 9e00b6d981b9f93a486ef06a47345ebd
+md5sum = 669a93b7e21b99066a84494fec47906b
 
 [template-cached-slave-virtualhost]
 filename = templates/cached-virtualhost.conf.in

software/caddy-frontend/instance-apache-replicate.cfg.in

@@ -71,6 +71,7 @@ context =
 {% set unauthorized_message = 'slave not authorized' %}
 {% for slave in slave_instance_list %}
 {%   set slave_error_list = [] %}
+{%   set slave_server_alias_unclashed = [] %}
 {#   BBB: apache_custom_https AND apache_custom_http #}
 {%   set custom_domain = slave.get('custom_domain') %}
 {%   if custom_domain and custom_domain in used_host_list %}
@@ -88,13 +89,17 @@ context =
 {%       if not validators.domain(clean_slave_alias) %}
 {%         do slave_error_list.append('server-alias %r not valid' % (slave_alias,)) %}
 {%       else %}
-{%         if slave_alias in used_host_list %}
+{%         if slave_alias in slave_server_alias_unclashed or slave_alias == custom_domain %}
+{#           optionally do something about reporting back that server-alias has been unclashed #}
+{%         elif slave_alias in used_host_list %}
 {%           do slave_error_list.append('server-alias %r clashes' % (slave_alias,)) %}
 {%         else %}
+{%           do slave_server_alias_unclashed.append(slave_alias) %}
 {%           do used_host_list.append(slave_alias) %}
 {%         endif %}
 {%       endif %}
 {%     endfor %}
+{%     do slave.__setitem__('server-alias', ' '.join(slave_server_alias_unclashed)) %}
 {%   endif %}
 {%   for key in ['caddy_custom_http', 'caddy_custom_https', 'apache_custom_http', 'apache_custom_https'] %}
 {%     if slave.get(key) %}

software/caddy-frontend/templates/default-virtualhost.conf.in

@@ -12,12 +12,10 @@
 {%- set disabled_cookie_list =  slave_parameter.get('disabled-cookie-list', '').split() %}
 {%- set https_only = ('' ~ slave_parameter.get('https-only', '')).lower() in TRUE_VALUES %}
 {%- set slave_type = slave_parameter.get('type', '') %}
-{%- set host_list = [] %}
-{%- for host in [slave_parameter.get('custom_domain')] + server_alias_list %}
-{%-   if host not in host_list %}
-{%-     do host_list.append(host) %}
-{%-   endif %}
-{%- endfor %}
+{%- set host_list = server_alias_list %}
+{%- if slave_parameter.get('custom_domain') not in host_list %}
+{%-   do host_list.append(slave_parameter.get('custom_domain')) %}
+{%- endif %}
 {%- set backend_url = slave_parameter.get('https-url', slave_parameter.get('url', '')) %}
 {%- set http_host_list = [] %}
 {%- set https_host_list = [] %}

software/caddy-frontend/test/test.py

@@ -589,6 +589,15 @@ http://apachecustomhttpsaccepted.example.com:%%(http_port)s {
         'url': cls.backend_url,
         'server-alias': '*.alias1.example.com',
       },
+      'server-alias-duplicated': {
+        'url': cls.backend_url,
+        'server-alias': 'alias3.example.com alias3.example.com',
+      },
+      'server-alias_custom_domain-duplicated': {
+        'url': cls.backend_url,
+        'custom_domain': 'alias4.example.com',
+        'server-alias': 'alias4.example.com alias4.example.com',
+      },
       'ssl-proxy-verify_ssl_proxy_ca_crt': {
         'url': cls.backend_https_url,
         'ssl-proxy-verify': True,
@@ -1124,6 +1133,66 @@ http://apachecustomhttpsaccepted.example.com:%%(http_port)s {
 
     self.assertEqualResultJson(result, 'Path', '/test-path')
 
+  def test_server_alias_duplicated(self):
+    parameter_dict = self.slave_connection_parameter_dict_dict[
+      'server-alias-duplicated']
+    self.assertLogAccessUrlWithPop(parameter_dict, 'server-alias-duplicated')
+    self.assertEqual(
+      {
+        'domain': 'serveraliasduplicated.example.com',
+        'replication_number': '1',
+        'url': 'http://serveraliasduplicated.example.com',
+        'site_url': 'http://serveraliasduplicated.example.com',
+        'secure_access': 'https://serveraliasduplicated.example.com',
+        'public-ipv4': LOCAL_IPV4,
+      },
+      parameter_dict
+    )
+
+    result = self.fakeHTTPSResult(
+      parameter_dict['domain'], parameter_dict['public-ipv4'], 'test-path')
+
+    self.assertEqual(
+      open('wildcard.example.com.crt').read(),
+      der2pem(result.peercert))
+
+    self.assertEqualResultJson(result, 'Path', '/test-path')
+
+    result = self.fakeHTTPSResult(
+      'alias3.example.com', parameter_dict['public-ipv4'], 'test-path')
+
+    self.assertEqual(
+      open('wildcard.example.com.crt').read(),
+      der2pem(result.peercert))
+
+    self.assertEqualResultJson(result, 'Path', '/test-path')
+
+  def test_server_alias_custom_domain_duplicated(self):
+    parameter_dict = self.slave_connection_parameter_dict_dict[
+      'server-alias_custom_domain-duplicated']
+    self.assertLogAccessUrlWithPop(
+      parameter_dict, 'server-alias_custom_domain-duplicated')
+    self.assertEqual(
+      {
+        'domain': 'alias4.example.com',
+        'replication_number': '1',
+        'url': 'http://alias4.example.com',
+        'site_url': 'http://alias4.example.com',
+        'secure_access': 'https://alias4.example.com',
+        'public-ipv4': LOCAL_IPV4,
+      },
+      parameter_dict
+    )
+
+    result = self.fakeHTTPSResult(
+      parameter_dict['domain'], parameter_dict['public-ipv4'], 'test-path')
+
+    self.assertEqual(
+      open('wildcard.example.com.crt').read(),
+      der2pem(result.peercert))
+
+    self.assertEqualResultJson(result, 'Path', '/test-path')
+
   @skip('Feature postponed')
   def test_check_error_log(self):
     # Caddy: Need to implement similar thing like check-error-on-apache-log
@@ -3450,10 +3519,9 @@ class TestDuplicateSiteKeyProtection(SlaveHttpFrontendTestCase, TestDataMixin):
       'rejected-slave-amount': '3',
       'slave-amount': '4',
       'rejected-slave-dict':
-      '{"_site_4": ["custom_domain \'duplicate.example.com\' clashes", '
-      '"server-alias \'duplicate.example.com\' clashes"], "_site_1": '
-      '["custom_domain \'duplicate.example.com\' clashes"], "_site_3": '
-      '["server-alias \'duplicate.example.com\' clashes"]}'
+      '{"_site_4": ["custom_domain \'duplicate.example.com\' clashes"], '
+      '"_site_1": ["custom_domain \'duplicate.example.com\' clashes"], '
+      '"_site_3": ["server-alias \'duplicate.example.com\' clashes"]}'
     }
 
     self.assertEqual(
@@ -3505,8 +3573,7 @@ class TestDuplicateSiteKeyProtection(SlaveHttpFrontendTestCase, TestDataMixin):
     self.assertEqual(
       {
         'request-error-list':
-        '["custom_domain \'duplicate.example.com\' clashes", "server-alias '
-        '\'duplicate.example.com\' clashes"]'
+        '["custom_domain \'duplicate.example.com\' clashes"]'
       },
       parameter_dict
     )

software/caddy-frontend/test/test_data/test.TestSlave.test_file_list_log-CADDY.txt

@@ -39,9 +39,13 @@ TestSlave-1/var/log/httpd/_prefer-gzip-encoding-to-backend_access_log
 TestSlave-1/var/log/httpd/_prefer-gzip-encoding-to-backend_error_log
 TestSlave-1/var/log/httpd/_re6st-optimal-test_access_log
 TestSlave-1/var/log/httpd/_re6st-optimal-test_error_log
+TestSlave-1/var/log/httpd/_server-alias-duplicated_access_log
+TestSlave-1/var/log/httpd/_server-alias-duplicated_error_log
 TestSlave-1/var/log/httpd/_server-alias-wildcard_access_log
 TestSlave-1/var/log/httpd/_server-alias-wildcard_error_log
 TestSlave-1/var/log/httpd/_server-alias_access_log
+TestSlave-1/var/log/httpd/_server-alias_custom_domain-duplicated_access_log
+TestSlave-1/var/log/httpd/_server-alias_custom_domain-duplicated_error_log
 TestSlave-1/var/log/httpd/_server-alias_error_log
 TestSlave-1/var/log/httpd/_ssl-proxy-verify-unverified_access_log
 TestSlave-1/var/log/httpd/_ssl-proxy-verify-unverified_error_log

software/caddy-frontend/test/test_data/test.TestSlave.test_monitor_promise_list-CADDY.txt

@@ -39,10 +39,14 @@ TestSlave-1/etc/monitor-promise/check-_prefer-gzip-encoding-to-backend-error-log
 TestSlave-1/etc/monitor-promise/check-_re6st-optimal-test-error-log-last-day
 TestSlave-1/etc/monitor-promise/check-_re6st-optimal-test-error-log-last-hour
 TestSlave-1/etc/monitor-promise/check-_re6st-optimal-test-re6st-optimal-test
+TestSlave-1/etc/monitor-promise/check-_server-alias-duplicated-error-log-last-day
+TestSlave-1/etc/monitor-promise/check-_server-alias-duplicated-error-log-last-hour
 TestSlave-1/etc/monitor-promise/check-_server-alias-error-log-last-day
 TestSlave-1/etc/monitor-promise/check-_server-alias-error-log-last-hour
 TestSlave-1/etc/monitor-promise/check-_server-alias-wildcard-error-log-last-day
 TestSlave-1/etc/monitor-promise/check-_server-alias-wildcard-error-log-last-hour
+TestSlave-1/etc/monitor-promise/check-_server-alias_custom_domain-duplicated-error-log-last-day
+TestSlave-1/etc/monitor-promise/check-_server-alias_custom_domain-duplicated-error-log-last-hour
 TestSlave-1/etc/monitor-promise/check-_ssl-proxy-verify-unverified-error-log-last-day
 TestSlave-1/etc/monitor-promise/check-_ssl-proxy-verify-unverified-error-log-last-hour
 TestSlave-1/etc/monitor-promise/check-_ssl-proxy-verify_ssl_proxy_ca_crt-error-log-last-day