caddy-frontend: Workaround lost connections to haproxy

Due to unknown yet reason ATS and Caddy are loosing connections to haproxy,
which results with increased amount of 5xx (about 0.5% more cases).

No known solution was found yet on haproxy side, but this kind of workaround
stabilises frontend and definitely drops amount of in-frontend 5xx. The
observed situation with this workaround is no more 5xx sourcing in frontend
itself.

software/caddy-frontend/buildout.hash.cfg

@@ -46,7 +46,7 @@ md5sum = 88af61e7abbf30dc99a1a2526161128d
 
 [template-default-slave-virtualhost]
 _update_hash_filename_ = templates/default-virtualhost.conf.in
-md5sum = 266f175dbdfc588af7a86b0b1884fe73
+md5sum = bd9e269130bac989faa639e0903814e2
 
 [template-backend-haproxy-configuration]
 _update_hash_filename_ = templates/backend-haproxy.cfg.in
@@ -62,7 +62,7 @@ md5sum = 975177dedf677d24e14cede5d13187ce
 
 [template-trafficserver-records-config]
 _update_hash_filename_ = templates/trafficserver/records.config.jinja2
-md5sum = 18076ae37306aca1f6ef11e2173c887f
+md5sum = a6a6e21036f8c56e69ad7ac8a1dfd656
 
 [template-trafficserver-storage-config]
 _update_hash_filename_ = templates/trafficserver/storage.config.jinja2

software/caddy-frontend/templates/default-virtualhost.conf.in

@@ -43,6 +43,9 @@
     timeout {{ slave_parameter['request-timeout'] }}s
     # force reset of X-Forwarded-For
     header_upstream X-Forwarded-For {remote}
+    # workaround for lost connection to haproxy by reconnecting
+    try_duration 3s
+    try_interval 250ms
 {%- endmacro %} {# proxy_header #}
 
 {%- for tls in [True, False] %}

software/caddy-frontend/templates/trafficserver/records.config.jinja2

@@ -80,10 +80,9 @@ CONFIG proxy.config.net.default_inactivity_timeout INT 86400
 # Origin server connect attempts. Docs:
 #    https://docs.trafficserver.apache.org/records.config#origin-server-connect-attempts
 ##############################################################################
-# Try only once to connect (do not retry)
-CONFIG proxy.config.http.connect_attempts_max_retries INT 0
-# Try only once with server marked dead (do not retry)
-CONFIG proxy.config.http.connect_attempts_max_retries_dead_server INT 0
+# workaround for lost connection to haproxy by reconnecting
+CONFIG proxy.config.http.connect_attempts_max_retries INT 3
+CONFIG proxy.config.http.connect_attempts_max_retries_dead_server INT 1
 CONFIG proxy.config.http.connect_attempts_rr_retries INT 3
 CONFIG proxy.config.http.connect_attempts_timeout INT {{ ats_configuration['request-timeout'] }}
 CONFIG proxy.config.http.post_connect_attempts_timeout INT {{ ats_configuration['request-timeout'] }}