XXX caddy-frontend: Switch from Caddy to Haproxy

XXX: checkpoint reached, the frontend-haproxy is able to start, serves nothing

software/caddy-frontend/buildout.hash.cfg

@@ -22,7 +22,7 @@ md5sum = 5784bea3bd608913769ff9a8afcccb68
 
 [profile-caddy-frontend]
 filename = instance-apache-frontend.cfg.in
-md5sum = 8507a2ace2f789b92c522cc62ca5aace
+md5sum = 3142774a70df7a4da7692a478f65ac3b
 
 [profile-caddy-replicate]
 filename = instance-apache-replicate.cfg.in
@@ -30,15 +30,19 @@ md5sum = 8beb438d06bbb0f917d13e182fb12d17
 
 [profile-slave-list]
 _update_hash_filename_ = templates/apache-custom-slave-list.cfg.in
-md5sum = 613f777a08373088cbaf7f51fd18ea70
+md5sum = f0456985cd6b2e7ea3847ec6c84a250b
 
 [profile-replicate-publish-slave-information]
 _update_hash_filename_ = templates/replicate-publish-slave-information.cfg.in
 md5sum = df304a8aee87b6f2425241016a48f7a5
 
-[profile-caddy-frontend-configuration]
-_update_hash_filename_ = templates/Caddyfile.in
-md5sum = fdf46b1dee6ea6b91b9aa9e322a0530d
+[template-frontend-haproxy-configuration]
+_update_hash_filename_ = templates/frontend-haproxy.cfg.in
+md5sum = 6fd120cdbdacecb2b7b93c8d1237e126
+
+[template-frontend-haproxy-crt-list]
+_update_hash_filename_ = templates/frontend-haproxy-crt-list.in
+md5sum = 251c7d8c775db20381c8f0b5bb220f75
 
 [template-not-found-html]
 _update_hash_filename_ = templates/notfound.html
@@ -50,7 +54,7 @@ md5sum = 13cd08d630cc51666a9f7e469fb6ea52
 
 [template-backend-haproxy-configuration]
 _update_hash_filename_ = templates/backend-haproxy.cfg.in
-md5sum = 5e126be0f74d8ae390a5594e1e912a59
+md5sum = db8c3cc4dcb0282c8316742ecd7f4761
 
 [template-empty]
 _update_hash_filename_ = templates/empty.in
@@ -108,6 +112,10 @@ md5sum = 38792c2dceae38ab411592ec36fff6a8
 filename = instance-kedifa.cfg.in
 md5sum = 16901e9eeb0d4f87e708ad91e7756f12
 
+[template-frontend-haproxy-rsyslogd-conf]
+_update_hash_filename_ = templates/frontend-haproxy-rsyslogd.conf.in
+md5sum = 420f66264d4cd24070a5a7b325e09ccd
+
 [template-backend-haproxy-rsyslogd-conf]
 _update_hash_filename_ = templates/backend-haproxy-rsyslogd.conf.in
 md5sum = 3336d554661b138dcef97b1d1866803c

software/caddy-frontend/instance-slave-caddy-input-schema.json

@@ -85,16 +85,6 @@
       "title": "Disabled Cookies",
       "type": "string"
     },
-    "enable-http2": {
-      "default": "true",
-      "description": "Use HTTP2 Protocol for the site",
-      "enum": [
-        "true",
-        "false"
-      ],
-      "title": "Enable HTTP2 Protocol",
-      "type": "string"
-    },
     "https-url": {
       "description": "HTTPS URL of the backend if it is different from url parameter",
       "pattern": "^(http|https|ftp)://",
@@ -210,11 +200,6 @@
       "title": "HTTP Request timeout in seconds",
       "type": "integer"
     },
-    "ciphers": {
-      "description": "List of ciphers. Empty defaults to cluster list of ciphers, which by default are Caddy list of ciphers. See https://caddyserver.com/docs/tls for more information.",
-      "title": "Ordered space separated list of ciphers",
-      "type": "string"
-    },
     "authenticate-to-backend": {
       "description": "If set to true the frontend certificate will be used as authentication certificate to the backend. Note: backend might have to know the frontend CA, available with 'backend-client-caucase-url'.",
       "enum": [

software/caddy-frontend/software.cfg

@@ -3,7 +3,6 @@ extends =
   buildout.hash.cfg
   ../../stack/slapos.cfg
   ../../component/dash/buildout.cfg
-  ../../component/caddy/buildout.cfg
   ../../component/gzip/buildout.cfg
   ../../component/logrotate/buildout.cfg
   ../../component/trafficserver/buildout.cfg
@@ -97,7 +96,9 @@ profile_slave_list = ${profile-slave-list:target}
 # templates
 template_backend_haproxy_configuration = ${template-backend-haproxy-configuration:target}
 template_backend_haproxy_rsyslogd_conf = ${template-backend-haproxy-rsyslogd-conf:target}
-template_caddy_frontend_configuration = ${profile-caddy-frontend-configuration:target}
+template_frontend_haproxy_configuration = ${template-frontend-haproxy-configuration:target}
+template_frontend_haproxy_crt_list = ${template-frontend-haproxy-crt-list:target}
+template_frontend_haproxy_rsyslogd_conf = ${template-frontend-haproxy-rsyslogd-conf:target}
 template_caddy_lazy_script_call = ${template-caddy-lazy-script-call:target}
 template_configuration_state_script = ${template-configuration-state-script:target}
 template_default_slave_virtualhost = ${template-default-slave-virtualhost:target}
@@ -119,7 +120,6 @@ bin_directory = ${buildout:bin-directory}
 sixtunnel = ${6tunnel:location}
 nginx = ${nginx-output:nginx}
 nginx_mime = ${nginx-output:mime}
-caddy = ${caddy:output}
 haproxy_executable = ${haproxy:location}/sbin/haproxy
 rsyslogd_executable = ${rsyslogd:location}/sbin/rsyslogd
 curl = ${curl:location}
@@ -170,7 +170,10 @@ mode = 640
 [profile-replicate-publish-slave-information]
 <=download-template
 
-[profile-caddy-frontend-configuration]
+[template-frontend-haproxy-configuration]
+<=download-template
+
+[template-frontend-haproxy-crt-list]
 <=download-template
 
 [template-not-found-html]
@@ -221,6 +224,9 @@ mode = 0644
 [template-backend-haproxy-rsyslogd-conf]
 <=download-template
 
+[template-frontend-haproxy-rsyslogd-conf]
+<=download-template
+
 [versions]
 # Modern KeDiFa requires zc.lockfile
 zc.lockfile = 1.4

software/caddy-frontend/templates/Caddyfile.in

-# Main caddy configuration file
-
-import {{ slave_configuration_directory }}/*.conf
-
-:{{ https_port }} {
-  tls {{ master_certificate }} {{ master_certificate }}
-  bind {{ local_ipv4 }}
-  status 404 /
-  log / {{ access_log }} "{remote} - {>REMOTE_USER} [{when}] \"{method} {uri} {proto}\" {status} {size} \"{>Referer}\" \"{>User-Agent}\" {latency_ms}" {
-    rotate_size 10000000
-  }
-  errors {{ error_log }} {
-    rotate_size 10000000
-    * {{ not_found_file }}
-  }
-}
-
-:{{ http_port }} {
-  bind {{ local_ipv4 }}
-  status 404 /
-  log / {{ access_log }} "{remote} - {>REMOTE_USER} [{when}] \"{method} {uri} {proto}\" {status} {size} \"{>Referer}\" \"{>User-Agent}\" {latency_ms}" {
-    rotate_size 10000000
-  }
-  errors {{ error_log }} {
-    rotate_size 10000000
-    * {{ not_found_file }}
-  }
-}
-
-# Access to server-status Caddy-style
-https://[{{ global_ipv6 }}]:{{ https_port }}/server-status, https://{{ local_ipv4 }}:{{ https_port }}/server-status {
-  tls {{ frontend_configuration['ip-access-certificate'] }} {{ frontend_configuration['ip-access-certificate'] }}
-  bind {{ local_ipv4 }}
-  basicauth "{{ username }}" {{ password | trim }} {
-    "Server Status"
-    /
-  }
-  expvar
-  pprof
-  log / {{ access_log }} "{remote} - {>REMOTE_USER} [{when}] \"{method} {uri} {proto}\" {status} {size} \"{>Referer}\" \"{>User-Agent}\" {latency_ms}" {
-    rotate_size 10000000
-  }
-  errors {{ error_log }} {
-    rotate_size 10000000
-    * {{ not_found_file }}
-  }
-}

software/caddy-frontend/templates/backend-haproxy.cfg.in

@@ -169,4 +169,4 @@ backend {{ slave_instance['slave_reference'] }}-{{ scheme }}-failover
 {%-       endif %}
 {%-     endif %}
 {%-   endfor %}
-{%- endfor %}
+{% endfor %}

software/caddy-frontend/templates/frontend-haproxy-crt-list.in

+{%- for slave in frontend_slave_list %}
+{%-   set entry_list = [] %}
+{#-   <crtfile> #}
+{%-   do entry_list.append(slave['certificate']) %}
+{#-   <snifilter> #}
+{%-   do entry_list.extend(slave['host_list']) %}
+{{    ' '.join(entry_list) }}
+{% endfor %}
+{#- Fallback to default certificate -#}
+{{ configuration['master-certificate'] }}
+# END OF FILE

software/caddy-frontend/templates/frontend-haproxy-rsyslogd.conf.in

+module(
+  load="imuxsock"
+  SysSock.Name="{{ configuration['log-socket'] }}")
+
+# Just simply output the raw line without any additional information, as
+# haproxy emits enough information by itself
+# Also cut out first empty space in msg, which is related to rsyslogd
+# internal and end up cutting on 8k, as it's default of $MaxMessageSize
+template(name="rawoutput" type="string" string="%msg:2:8192%\n")
+$ActionFileDefaultTemplate rawoutput
+
+$FileCreateMode 0600
+$DirCreateMode 0700
+$Umask 0022
+
+$WorkDirectory {{ configuration['spool-directory'] }}
+
+# Setup logging per slave, by extracting the slave name from the log stream
+{%- set regex = "^\\\\s*(\\\\S.*)-https{0,1} (.*)" %}
+# Extract file name part from 1st match
+template(name="extract_slave_name" type="string" string="%msg:R,ERE,1,FIELD:{{ regex }}--end%")
+set $!slave_name = exec_template("extract_slave_name");
+template(name="slave_output" type="string" string="{{ configuration['slave-log-directory'] }}/%$!slave_name%_access_log")
+# Output only 2nd match, add the newline in the ned
+template(name="haproxy_slave_line" type="string" string="%msg:R,ERE,2,FIELD:{{ regex }}--end%\n")
+# React on match
+if (re_match($msg, '{{ regex }}')) then {
+ action(type="omfile" dynaFile="slave_output" template="haproxy_slave_line")
+ stop
+}
+
+{#- emit all not catched messages to full log file #}
+*.* {{ configuration['log-file'] }}

software/caddy-frontend/templates/frontend-haproxy.cfg.in

+global
+  pidfile {{ configuration['pid-file'] }}
+  # master-worker is compatible with foreground with process management
+  master-worker
+  expose-experimental-directives
+  log {{ configuration['log-socket'] }} local0
+
+defaults
+  mode http
+  log global
+  option httplog
+{#-  timeout queue XXX TODO #}
+{#-  timeout server XXX TODO #}
+{#-  timeout client XXX TODO #}
+{#-  timeout connect XXX TODO #}
+{#-  retries XXX TODO #}
+
+{%- set SCHEME_PREFIX_MAPPING = { 'http': 'backend-http-info', 'https': 'backend-https-info'} %}
+{%- macro frontend_entry(slave_instance, scheme, wildcard) %}
+{#-   wildcard switch allows to put dangerous entries in the end, as haproxy parses with first match #}
+{#-   if slave_instance[SCHEME_PREFIX_MAPPING[scheme]]['hostname'] and slave_instance[SCHEME_PREFIX_MAPPING[scheme]]['port'] #}
+{%-     set host_list = (slave_instance.get('server-alias') or  '').split() %}
+{%-     if slave_instance.get('custom_domain') not in host_list %}
+{%-       do host_list.append(slave_instance.get('custom_domain')) %}
+{%-     endif %}
+{%-     set matched = {'count': 0} %}
+{%-     for host in host_list %}
+{#-       Match up to the end or optional port (starting with ':') #}
+{#-       Please note that this matching is quite sensitive to changes and hard to test, so avoid needless changes #}
+{%-       if wildcard and host.startswith('*.') %}
+{%-         do matched.__setitem__('count', matched['count'] + 1) %}
+# match wildcard {{ host }}
+  acl is_{{ slave_instance['slave_reference'] }} hdr_reg(host) -i {{ host[2:] }}($|:.*)
+{%-       elif not wildcard and not host.startswith('*.') %}
+{%-         do matched.__setitem__('count', matched['count'] + 1) %}
+  acl is_{{ slave_instance['slave_reference'] }} hdr_reg(host) -i ^{{ host }}($|:.*)
+{%-       endif %}
+{%-     endfor %}
+{%-     if matched['count'] > 0 %}
+  use_backend {{ slave_instance['slave_reference'] }}-{{ scheme }} if is_{{ slave_instance['slave_reference'] }}
+{%-     endif %}
+{#-   endif #}
+{%- endmacro %}
+
+{%- macro frontend_common() %}
+  # normalize URIs as it's expected by the backends
+  http-request normalize-uri path-merge-slashes
+  http-request normalize-uri path-strip-dot
+  http-request normalize-uri path-strip-dotdot
+  # setup X-Forwarded-*
+  option forwardfor
+  # Combined Log Format
+  capture request header REMOTE_USER len 255
+  capture request header Referer len 255
+  capture request header User-Agent len 255
+  log-format "%{+E}o %b %ci - %[capture.req.hdr(0)] [%trl] \"%HM %HU %HV\" %ST %B \"%[capture.req.hdr(1)]\" \"%[capture.req.hdr(2)]\" %Ta"
+{%- endmacro %}
+frontend http-frontend
+  bind {{ configuration['local-ipv4'] }}:{{ configuration['http-port'] }}
+{{ frontend_common() }}
+  http-request set-header X-Forwarded-Proto http
+  http-request set-header X-Forwarded-Port {{ configuration['http-port'] }}
+# Here use Host header
+{%- for slave_instance in frontend_slave_list -%}
+{{ frontend_entry(slave_instance, 'http', False) }}
+{%- endfor %}
+{%- for slave_instance in frontend_slave_list -%}
+{{ frontend_entry(slave_instance, 'http', True) }}
+{%- endfor %}
+
+frontend https-frontend
+  bind {{ configuration['local-ipv4'] }}:{{ configuration['https-port'] }} ssl crt-list {{ crt_list }}
+{{ frontend_common() }}
+  http-request set-header X-Forwarded-Proto https
+  http-request set-header X-Forwarded-Port {{ configuration['https-port'] }}
+# Here use ssl_fc_sni and fallback to Host header
+{%- for slave_instance in frontend_slave_list -%}
+{{ frontend_entry(slave_instance, 'https', False) }}
+{%- endfor %}
+{%- for slave_instance in frontend_slave_list -%}
+{{ frontend_entry(slave_instance, 'https', True) }}
+{%- endfor %}
+
+# Backends
+{%- for slave_instance in frontend_slave_list %}
+{%-   for (scheme, prefix) in SCHEME_PREFIX_MAPPING.items() %}
+{%-     set info_dict = slave_instance.get(prefix, slave_instance.get('backend-http-info')) %}
+backend {{ slave_instance['slave_reference'] }}-{{ scheme }}
+{%-     if scheme == 'http' and slave_instance['https-only'] %}
+{#-     Support https-only if connected via http #}
+  redirect scheme https code 302
+{%-     else %}
+{%-       if 'hostname' in info_dict and 'port' in info_dict %}
+  server {{ slave_instance['slave_reference'] }}-backend-{{ scheme }} {{ info_dict['hostname'] }}:{{ info_dict['port'] }}
+{%-         if info_dict['path'] %}
+  http-request set-path {{ info_dict['path'] }}%[path]
+{%-         endif %} {# if info_dict['path'] #}
+{%-       endif %} {# if 'hostname' in info_dict and 'port' in info_dict #}
+{%-     endif %} {# if scheme == 'http' and slave_instance['https-only'] #}
+{%-   endfor %} {# for (scheme, prefix) in SCHEME_PREFIX_MAPPING.items() #}
+{%-   for k, v in slave_instance.items() %}
+{#- #  {{ k }} => {{ v }} #}
+{%-   endfor %}
+{% endfor %}

software/caddy-frontend/test/test.py

@@ -836,14 +836,14 @@ class HttpFrontendTestCase(SlapOSInstanceTestCase):
     )
     self.assertEqual(
       sorted([q['name'] for q in result.json()]),
-      ['access.log', 'backend.log', 'error.log'])
+      ['access.log', 'backend.log'])
     self.assertEqual(
       httplib.OK,
       requests.get(url + 'access.log', verify=False).status_code
     )
     self.assertEqual(
       httplib.OK,
-      requests.get(url + 'error.log', verify=False).status_code
+      requests.get(url + 'backend.log', verify=False).status_code
     )
     # assert only for few tests, as backend log is not available for many of
     # them, as it's created on the fly