caddy-frontend: Stabilise proxy headers

On backend side headers are asserted in tests:

 * X-Forwarded-For
 * X-Forwarded-Proto
 * X-Forwarded-Port
 * Host

In order to pass cleanly X-Forwarded-For from the frontend to the backend,
it's passed as X-Forwarded-For-Real in case of cached slaves.

Noted problem with IPv6 endpoint was used, as in this case 6tunnel IP would
be used.

software/caddy-frontend/README.caddy_frontend.rst

@@ -488,3 +488,8 @@ Each `caddy-frontend-N` partition downloads certificates from the kedifa server.
 Caucase (exposed by ``kedifa-caucase-url`` in master partition parameters) is used to handle certificates for authentication to kedifa server.
 
 If ``automatic-internal-kedifa-caucase-csr`` is enabled (by default it is) there are scripts running on master partition to simulate human to sign certificates for each caddy-frontend-N node.
+
+Support for X-Real-Ip and X-Forwarded-For
+-----------------------------------------
+
+X-Forwarded-For and X-Real-Ip are transmitted to the backend, but only for IPv4 access to the frontend. In of IPv6 access, the provided IP will be wrong, because of using 6tunnel.

software/caddy-frontend/buildout.hash.cfg

@@ -50,11 +50,11 @@ md5sum = f20d6c3d2d94fb685f8d26dfca1e822b
 
 [template-default-slave-virtualhost]
 _update_hash_filename_ = templates/default-virtualhost.conf.in
-md5sum = 7e26935bb6daf00d8fc01d97eebc7abd
+md5sum = cb3f7ace99346f64f2007c3e94b05800
 
 [template-cached-slave-virtualhost]
 _update_hash_filename_ = templates/cached-virtualhost.conf.in
-md5sum = a73839d777fbd548286bbeccf47be335
+md5sum = e839ca3cb308f7fcdfa06c2f1b95e93f
 
 [template-log-access]
 _update_hash_filename_ = templates/template-log-access.conf.in
@@ -70,7 +70,7 @@ md5sum = 8cde04bfd0c0e9bd56744b988275cfd8
 
 [template-trafficserver-records-config]
 _update_hash_filename_ = templates/trafficserver/records.config.jinja2
-md5sum = 3a4e378932ffc7768426bb7a897e2c45
+md5sum = f3f31188de56bb35383335b3219537f4
 
 [template-trafficserver-storage-config]
 _update_hash_filename_ = templates/trafficserver/storage.config.jinja2

software/caddy-frontend/templates/cached-virtualhost.conf.in

@@ -22,7 +22,10 @@
     try_duration {{ slave_parameter['proxy_try_duration'] }}s
     try_interval {{ slave_parameter['proxy_try_interval'] }}ms
 
-    transparent
+    header_upstream Host {host}
+{#    header_upstream -X-Forwarded-For - caddy behaviour while removing and setting header is unstable, so for now original header has to be kept, even if in that case it comes from after ATS caddy itself #}
+    header_upstream X-Forwarded-For {>X-Forwarded-For-Real}
+    header_upstream -X-Forwarded-For-Real
     timeout {{ slave_parameter['request_timeout'] }}s
 {%- if ssl_proxy_verify %}
 {%-   if 'path_to_ssl_proxy_ca_crt' in slave_parameter %}
@@ -47,7 +50,10 @@
   proxy / {{ slave_parameter.get('https_backend_url', '') }} {
     try_duration {{ slave_parameter['proxy_try_duration'] }}s
     try_interval {{ slave_parameter['proxy_try_interval'] }}ms
-    transparent
+    header_upstream Host {host}
+{#    header_upstream -X-Forwarded-For - caddy behaviour while removing and setting header is unstable, so for now original header has to be kept, even if in that case it comes from after ATS caddy itself #}
+    header_upstream X-Forwarded-For {>X-Forwarded-For-Real}
+    header_upstream -X-Forwarded-For-Real
     timeout {{ slave_parameter['request_timeout'] }}s
 {%- if ssl_proxy_verify %}
 {%-   if 'path_to_ssl_proxy_ca_crt' in slave_parameter %}

software/caddy-frontend/templates/default-virtualhost.conf.in

 {%- set TRUE_VALUES = ['y', 'yes', '1', 'true'] %}
+{%- set enable_cache = slave_parameter.get('enable_cache', '').lower() in TRUE_VALUES %}
 {%- set disable_no_cache_header = slave_parameter.get('disable-no-cache-request', '').lower() in TRUE_VALUES %}
 {%- set disable_via_header = slave_parameter.get('disable-via-header', '').lower() in TRUE_VALUES %}
 {%- set prefer_gzip = slave_parameter.get('prefer-gzip-encoding-to-backend', '').lower() in TRUE_VALUES %}
@@ -104,6 +105,12 @@
   proxy /{{ proxy_name }} {{ backend_url }} {
     try_duration {{ slave_parameter['proxy_try_duration'] }}s
     try_interval {{ slave_parameter['proxy_try_interval'] }}ms
+    # force reset of X-Forwarded-For
+    header_upstream X-Forwarded-For {remote}
+{%-     if enable_cache %}
+    # provide a header for other components
+    header_upstream X-Forwarded-For-Real {remote}
+{%-     endif %}
 {%-     if proxy_name == 'prefer-gzip' %}
     without /prefer-gzip
     header_upstream Accept-Encoding gzip
@@ -176,6 +183,12 @@
   proxy / {{ backend_url }} {
     try_duration {{ slave_parameter['proxy_try_duration'] }}s
     try_interval {{ slave_parameter['proxy_try_interval'] }}ms
+    # force reset of X-Forwarded-For
+    header_upstream X-Forwarded-For {remote}
+{%-     if enable_cache %}
+    # provide a header for other components
+    header_upstream X-Forwarded-For-Real {remote}
+{%-     endif %}
     transparent
     insecure_skip_verify
   }
@@ -186,8 +199,13 @@
   proxy /proxy/ {{ backend_url }} {
     try_duration {{ slave_parameter['proxy_try_duration'] }}s
     try_interval {{ slave_parameter['proxy_try_interval'] }}ms
-    header_upstream X-Real-IP {remote}
-    header_upstream Host {host}
+    # force reset of X-Forwarded-For
+    header_upstream X-Forwarded-For {remote}
+{%-     if enable_cache %}
+    # provide a header for other components
+    header_upstream X-Forwarded-For-Real {remote}
+{%-     endif %}
+    transparent
     websocket
     without /proxy/
     insecure_skip_verify
@@ -197,6 +215,12 @@
   proxy / {{ backend_url }} {
     try_duration {{ slave_parameter['proxy_try_duration'] }}s
     try_interval {{ slave_parameter['proxy_try_interval'] }}ms
+    # force reset of X-Forwarded-For
+    header_upstream X-Forwarded-For {remote}
+{%-     if enable_cache %}
+    # provide a header for other components
+    header_upstream X-Forwarded-For-Real {remote}
+{%-     endif %}
 {%-     if websocket_transparent %}
     transparent
 {%-     endif %}
@@ -206,6 +230,12 @@
   proxy /{{ websocket_path }} {{ backend_url }} {
     try_duration {{ slave_parameter['proxy_try_duration'] }}s
     try_interval {{ slave_parameter['proxy_try_interval'] }}ms
+    # force reset of X-Forwarded-For
+    header_upstream X-Forwarded-For {remote}
+{%-     if enable_cache %}
+    # provide a header for other components
+    header_upstream X-Forwarded-For-Real {remote}
+{%-     endif %}
     websocket
 {%-       if websocket_transparent %}
     transparent
@@ -217,6 +247,12 @@
   proxy / {{ backend_url }} {
     try_duration {{ slave_parameter['proxy_try_duration'] }}s
     try_interval {{ slave_parameter['proxy_try_interval'] }}ms
+    # force reset of X-Forwarded-For
+    header_upstream X-Forwarded-For {remote}
+{%-     if enable_cache %}
+    # provide a header for other components
+    header_upstream X-Forwarded-For-Real {remote}
+{%-     endif %}
     websocket
 {%-   if websocket_transparent %}
     transparent
@@ -239,6 +275,12 @@
   proxy /{{ proxy_name }} {{ backend_url }} {
     try_duration {{ slave_parameter['proxy_try_duration'] }}s
     try_interval {{ slave_parameter['proxy_try_interval'] }}ms
+    # force reset of X-Forwarded-For
+    header_upstream X-Forwarded-For {remote}
+{%-     if enable_cache %}
+    # provide a header for other components
+    header_upstream X-Forwarded-For-Real {remote}
+{%-     endif %}
 {%-     if proxy_name == 'prefer-gzip' %}
     without /prefer-gzip
     header_upstream Accept-Encoding gzip

software/caddy-frontend/templates/trafficserver/records.config.jinja2

@@ -27,6 +27,14 @@ CONFIG proxy.config.http.cache.open_write_fail_action INT 2
 CONFIG proxy.config.body_factory.template_sets_dir STRING  {{ ats_configuration['templates-dir'] }}
 # Support stale-if-error by returning cached content on backend 5xx or unavailability
 CONFIG proxy.config.http.negative_revalidating_enabled INT 1
+##############################################################################
+# Proxy users variables. Docs:
+#    https://docs.trafficserver.apache.org/records.config#proxy-user-variables
+##############################################################################
+# Do not modify headers, as it needlessly pollutes information
+CONFIG proxy.config.http.insert_client_ip INT 0
+CONFIG proxy.config.http.insert_squid_x_forwarded_for INT 0
+
 
 ##############################################################################
 # Thread configurations. Docs:
@@ -98,13 +106,6 @@ CONFIG proxy.config.http.down_server.abort_threshold INT 10
 CONFIG proxy.config.http.negative_caching_enabled INT 0
 CONFIG proxy.config.http.negative_caching_lifetime INT 1800
 
-##############################################################################
-# Proxy users variables. Docs:
-#    https://docs.trafficserver.apache.org/records.config#proxy-user-variables
-##############################################################################
-CONFIG proxy.config.http.insert_client_ip INT 1
-CONFIG proxy.config.http.insert_squid_x_forwarded_for INT 1
-
 ##############################################################################
 # Security. Docs:
 #    https://docs.trafficserver.apache.org/records.config#security