caddy-frontend: Fix host matching in Haproxy

By using regular expressions, matching exact host names up to the optional
port and putting wildcard matches in the end, the Haproxy acl rules will allow
to direct request to correct backend.

software/caddy-frontend/buildout.hash.cfg

@@ -54,7 +54,7 @@ md5sum = 266f175dbdfc588af7a86b0b1884fe73
 
 [template-backend-haproxy-configuration]
 _update_hash_filename_ = templates/backend-haproxy.cfg.in
-md5sum = 68a7758ca8f8b544ba9bc756824be3d3
+md5sum = eaf7491f628e72c877c31588e62eeb89
 
 [template-log-access]
 _update_hash_filename_ = templates/template-log-access.conf.in

software/caddy-frontend/templates/backend-haproxy.cfg.in

@@ -14,32 +14,46 @@ defaults
   timeout connect {{ configuration['backend-connect-timeout'] }}s
   retries {{ configuration['backend-connect-retries'] }}
 
-{%- macro frontend_entry(slave_instance, scheme) %}
+{%- macro frontend_entry(slave_instance, scheme, wildcard) %}
+{#-   wildcard switch allows to put dangerous entries in the end, as haproxy parses with first match #}
 {%-   set host_list = (slave_instance.get('server-alias') or  '').split() %}
 {%-   if slave_instance.get('custom_domain') not in host_list %}
 {%-     do host_list.append(slave_instance.get('custom_domain')) %}
 {%-   endif %}
+{%-   set matched = {'count': 0} %}
 {%-   for host in host_list %}
-{%-     if host.startswith('*.') %}
-{#-     hdr_sub has to be used, as anyway something.example.com shall match *.example.com[:port], with optional port #}
-  acl is_{{ slave_instance['slave_reference'] }} hdr_sub(host) -i {{ host[2:] }}
-{%-     else %}
-  acl is_{{ slave_instance['slave_reference'] }} hdr_dom(host) -i {{ host }}
+{#-     Match up to the end or optional port (starting with ':') #}
+{#-     Please note that this matching is quite sensitive to changes and hard to test, so avoid needless changes #}
+{%-     if wildcard and host.startswith('*.') %}
+{%-       do matched.__setitem__('count', matched['count'] + 1) %}
+# match wildcard {{ host }}
+  acl is_{{ slave_instance['slave_reference'] }} hdr_reg(host) -i {{ host[2:] }}($|:.*)
+{%-     elif not wildcard and not host.startswith('*.') %}
+{%-       do matched.__setitem__('count', matched['count'] + 1) %}
+  acl is_{{ slave_instance['slave_reference'] }} hdr_reg(host) -i ^{{ host }}($|:.*)
 {%-     endif %}
 {%-   endfor %}
+{%-   if matched['count'] > 0 %}
   use_backend {{ slave_instance['slave_reference'] }}-{{ scheme }} if is_{{ slave_instance['slave_reference'] }}
+{%-   endif %}
 {%- endmacro %}
 
 frontend http-backend
   bind {{ configuration['local-ipv4'] }}:{{ configuration['http-port'] }}
 {%- for slave_instance in backend_slave_list %}
-{{ frontend_entry(slave_instance, 'http') }}
+{{ frontend_entry(slave_instance, 'http', False) }}
+{%- endfor %}
+{%- for slave_instance in backend_slave_list %}
+{{ frontend_entry(slave_instance, 'http', True) }}
 {%- endfor %}
 
 frontend https-backend
   bind {{ configuration['local-ipv4'] }}:{{ configuration['https-port'] }}
 {%- for slave_instance in backend_slave_list %}
-{{ frontend_entry(slave_instance, 'https') }}
+{{ frontend_entry(slave_instance, 'https', False) }}
+{%- endfor %}
+{%- for slave_instance in backend_slave_list %}
+{{ frontend_entry(slave_instance, 'https', True) }}
 {%- endfor %}
 
 {%- for slave_instance in backend_slave_list %}

software/caddy-frontend/test/test.py

@@ -6575,3 +6575,92 @@ class TestSlaveRejectReportUnsafeDamaged(SlaveHttpFrontendTestCase):
       },
       parameter_dict
     )
+
+
+class TestSlaveHostHaproxyClash(SlaveHttpFrontendTestCase, TestDataMixin):
+  @classmethod
+  def getInstanceParameterDict(cls):
+    return {
+      'domain': 'example.com',
+      'public-ipv4': cls._ipv4_address,
+      'port': HTTPS_PORT,
+      'plain_http_port': HTTP_PORT,
+      'kedifa_port': KEDIFA_PORT,
+      'caucase_port': CAUCASE_PORT,
+      'mpm-graceful-shutdown-timeout': 2,
+      'request-timeout': '12',
+    }
+
+  @classmethod
+  def getSlaveParameterDictDict(cls):
+    # Note: The slaves are specifically constructed to have an order which
+    #       is triggering the problem. Slave list is sorted in many places,
+    #       and such slave configuration will result with them begin seen
+    #       by backend haproxy configuration in exactly the way seen below
+    #       Ordering it here will not help at all.
+    return {
+      'wildcard': {
+        'url': cls.backend_url + 'wildcard',
+        'custom_domain': '*.alias1.example.com',
+      },
+      'zspecific': {
+        'url': cls.backend_url + 'zspecific',
+        'custom_domain': 'zspecific.alias1.example.com',
+      },
+    }
+
+  def test(self):
+    parameter_dict_wildcard = self.parseSlaveParameterDict('wildcard')
+    self.assertLogAccessUrlWithPop(parameter_dict_wildcard)
+    self.assertKedifaKeysWithPop(parameter_dict_wildcard, '')
+    hostname = '*.alias1'
+    self.assertEqual(
+      {
+        'domain': '%s.example.com' % (hostname,),
+        'replication_number': '1',
+        'url': 'http://%s.example.com' % (hostname, ),
+        'site_url': 'http://%s.example.com' % (hostname, ),
+        'secure_access': 'https://%s.example.com' % (hostname, ),
+        'public-ipv4': self._ipv4_address,
+        'backend-client-caucase-url': 'http://[%s]:8990' % self._ipv6_address,
+      },
+      parameter_dict_wildcard
+    )
+    parameter_dict_specific = self.parseSlaveParameterDict('zspecific')
+    self.assertLogAccessUrlWithPop(parameter_dict_specific)
+    self.assertKedifaKeysWithPop(parameter_dict_specific, '')
+    hostname = 'zspecific.alias1'
+    self.assertEqual(
+      {
+        'domain': '%s.example.com' % (hostname,),
+        'replication_number': '1',
+        'url': 'http://%s.example.com' % (hostname, ),
+        'site_url': 'http://%s.example.com' % (hostname, ),
+        'secure_access': 'https://%s.example.com' % (hostname, ),
+        'public-ipv4': self._ipv4_address,
+        'backend-client-caucase-url': 'http://[%s]:8990' % self._ipv6_address,
+      },
+      parameter_dict_specific
+    )
+
+    result_wildcard = fakeHTTPSResult(
+      'other.alias1.example.com', parameter_dict_wildcard['public-ipv4'],
+      'test-path',
+      headers={
+        'Timeout': '10',  # more than default backend-connect-timeout == 5
+        'Accept-Encoding': 'gzip',
+      }
+    )
+    self.assertEqual(self.certificate_pem, der2pem(result_wildcard.peercert))
+    self.assertEqualResultJson(result_wildcard, 'Path', '/wildcard/test-path')
+
+    result_specific = fakeHTTPSResult(
+      'zspecific.alias1.example.com', parameter_dict_specific['public-ipv4'],
+      'test-path',
+      headers={
+        'Timeout': '10',  # more than default backend-connect-timeout == 5
+        'Accept-Encoding': 'gzip',
+      }
+    )
+    self.assertEqual(self.certificate_pem, der2pem(result_specific.peercert))
+    self.assertEqualResultJson(result_specific, 'Path', '/zspecific/test-path')

software/caddy-frontend/test/test_data/test.TestSlaveHostHaproxyClash.test_file_list_etc_cron_d-CADDY.txt

+T-0/etc/cron.d/logrotate
+T-0/etc/cron.d/monitor-configurator
+T-0/etc/cron.d/monitor-globalstate
+T-0/etc/cron.d/monitor_collect
+T-1/etc/cron.d/logrotate
+T-1/etc/cron.d/monitor-configurator
+T-1/etc/cron.d/monitor-globalstate
+T-1/etc/cron.d/monitor_collect
+T-2/etc/cron.d/logrotate
+T-2/etc/cron.d/monitor-configurator
+T-2/etc/cron.d/monitor-globalstate
+T-2/etc/cron.d/monitor_collect
+T-2/etc/cron.d/trafficserver-logrotate

software/caddy-frontend/test/test_data/test.TestSlaveHostHaproxyClash.test_file_list_log-CADDY.txt

+T-0/var/log/monitor-httpd-access.log
+T-0/var/log/monitor-httpd-error.log
+T-0/var/log/slapgrid-T-0-error.log
+T-1/var/log/expose-csr_id.log
+T-1/var/log/kedifa.log
+T-1/var/log/monitor-httpd-access.log
+T-1/var/log/monitor-httpd-error.log
+T-2/var/log/backend-haproxy.log
+T-2/var/log/frontend-access.log
+T-2/var/log/frontend-error.log
+T-2/var/log/httpd-csr_id/expose-csr_id.log
+T-2/var/log/httpd/_wildcard_access_log
+T-2/var/log/httpd/_wildcard_backend_log
+T-2/var/log/httpd/_wildcard_error_log
+T-2/var/log/httpd/_zspecific_access_log
+T-2/var/log/httpd/_zspecific_backend_log
+T-2/var/log/httpd/_zspecific_error_log
+T-2/var/log/monitor-httpd-access.log
+T-2/var/log/monitor-httpd-error.log
+T-2/var/log/slave-introspection-access.log
+T-2/var/log/slave-introspection-error.log
+T-2/var/log/trafficserver/manager.log
+T-2/var/log/trafficserver/traffic.out

software/caddy-frontend/test/test_data/test.TestSlaveHostHaproxyClash.test_file_list_plugin-CADDY.txt

+T-0/etc/plugin/__init__.py
+T-0/etc/plugin/aibcc-user-caucase-updater.py
+T-0/etc/plugin/aikc-user-caucase-updater.py
+T-0/etc/plugin/buildout-T-0-status.py
+T-0/etc/plugin/caucased-backend-client.py
+T-0/etc/plugin/check-free-disk-space.py
+T-0/etc/plugin/monitor-bootstrap-status.py
+T-0/etc/plugin/monitor-http-frontend.py
+T-0/etc/plugin/monitor-httpd-listening-on-tcp.py
+T-0/etc/plugin/rejected-slave-publish-ip-port-listening.py
+T-0/etc/plugin/rejected-slave.py
+T-1/etc/plugin/__init__.py
+T-1/etc/plugin/buildout-T-1-status.py
+T-1/etc/plugin/caucased.py
+T-1/etc/plugin/check-free-disk-space.py
+T-1/etc/plugin/expose-csr_id-ip-port-listening.py
+T-1/etc/plugin/kedifa-http-reply.py
+T-1/etc/plugin/monitor-bootstrap-status.py
+T-1/etc/plugin/monitor-http-frontend.py
+T-1/etc/plugin/monitor-httpd-listening-on-tcp.py
+T-2/etc/plugin/__init__.py
+T-2/etc/plugin/backend-client-caucase-updater.py
+T-2/etc/plugin/backend-haproxy-configuration.py
+T-2/etc/plugin/backend_haproxy_http.py
+T-2/etc/plugin/backend_haproxy_https.py
+T-2/etc/plugin/buildout-T-2-status.py
+T-2/etc/plugin/caddy_frontend_ipv4_http.py
+T-2/etc/plugin/caddy_frontend_ipv4_https.py
+T-2/etc/plugin/caddy_frontend_ipv6_http.py
+T-2/etc/plugin/caddy_frontend_ipv6_https.py
+T-2/etc/plugin/caucase-updater.py
+T-2/etc/plugin/check-free-disk-space.py
+T-2/etc/plugin/expose-csr_id-ip-port-listening.py
+T-2/etc/plugin/frontend-caddy-configuration-promise.py
+T-2/etc/plugin/monitor-bootstrap-status.py
+T-2/etc/plugin/monitor-http-frontend.py
+T-2/etc/plugin/monitor-httpd-listening-on-tcp.py
+T-2/etc/plugin/re6st-connectivity.py
+T-2/etc/plugin/slave-introspection-configuration.py
+T-2/etc/plugin/slave_introspection_https.py
+T-2/etc/plugin/trafficserver-cache-availability.py
+T-2/etc/plugin/trafficserver-port-listening.py

software/caddy-frontend/test/test_data/test.TestSlaveHostHaproxyClash.test_file_list_run-CADDY.txt

+T-0/var/run/monitor-httpd.pid
+T-1/var/run/kedifa.pid
+T-1/var/run/monitor-httpd.pid
+T-2/var/run/backend-haproxy-rsyslogd.pid
+T-2/var/run/backend-haproxy.pid
+T-2/var/run/backend_haproxy_configuration_last_state
+T-2/var/run/backend_haproxy_graceful_configuration_state_signature
+T-2/var/run/bhlog.sck
+T-2/var/run/graceful_configuration_state_signature
+T-2/var/run/httpd.pid
+T-2/var/run/monitor-httpd.pid
+T-2/var/run/slave-introspection.pid
+T-2/var/run/slave_introspection_configuration_last_state
+T-2/var/run/slave_introspection_graceful_configuration_state_signature

software/caddy-frontend/test/test_data/test.TestSlaveHostHaproxyClash.test_supervisor_state-CADDY.txt

+T-0:aibcc-user-caucase-updater-on-watch RUNNING
+T-0:aikc-user-caucase-updater-on-watch RUNNING
+T-0:bootstrap-monitor EXITED
+T-0:caucased-backend-client-{hash-generic}-on-watch RUNNING
+T-0:certificate_authority-{hash-generic}-on-watch RUNNING
+T-0:crond-{hash-generic}-on-watch RUNNING
+T-0:monitor-httpd-{hash-generic}-on-watch RUNNING
+T-0:monitor-httpd-graceful EXITED
+T-0:rejected-slave-publish-{hash-rejected-slave-publish}-on-watch RUNNING
+T-1:bootstrap-monitor EXITED
+T-1:caucase-updater-on-watch RUNNING
+T-1:caucased-{hash-generic}-on-watch RUNNING
+T-1:certificate_authority-{hash-generic}-on-watch RUNNING
+T-1:crond-{hash-generic}-on-watch RUNNING
+T-1:expose-csr_id-{hash-generic}-on-watch RUNNING
+T-1:kedifa-{hash-generic}-on-watch RUNNING
+T-1:kedifa-reloader EXITED
+T-1:monitor-httpd-{hash-generic}-on-watch RUNNING
+T-1:monitor-httpd-graceful EXITED
+T-2:6tunnel-11080-{hash-generic}-on-watch RUNNING
+T-2:6tunnel-11443-{hash-generic}-on-watch RUNNING
+T-2:backend-client-login-certificate-caucase-updater-on-watch RUNNING
+T-2:backend-haproxy-{hash-generic}-on-watch RUNNING
+T-2:backend-haproxy-rsyslogd-{hash-generic}-on-watch RUNNING
+T-2:backend-haproxy-safe-graceful EXITED
+T-2:bootstrap-monitor EXITED
+T-2:certificate_authority-{hash-generic}-on-watch RUNNING
+T-2:crond-{hash-generic}-on-watch RUNNING
+T-2:expose-csr_id-{hash-generic}-on-watch RUNNING
+T-2:frontend-caddy-safe-graceful EXITED
+T-2:frontend_caddy-{hash-caddy-T-2}-on-watch RUNNING
+T-2:kedifa-login-certificate-caucase-updater-on-watch RUNNING
+T-2:kedifa-updater-{hash-generic}-on-watch RUNNING
+T-2:monitor-httpd-{hash-generic}-on-watch RUNNING
+T-2:monitor-httpd-graceful EXITED
+T-2:slave-instrospection-nginx-{hash-generic}-on-watch RUNNING
+T-2:slave-introspection-safe-graceful EXITED
+T-2:trafficserver-{hash-generic}-on-watch RUNNING
+T-2:trafficserver-reload EXITED