caddy-frontend: Improve and stabilise Via header

Via header for response and request is fully stabilized and present according
to the RFC now.

Important information, like protocol and protocol version are exposed.

Hops which were present in the request are stored and descriptively named.

Versions of components are exposed in hidden way and in the same time the
frontend administrator and frontend user are able to analyze the versions,
and the version history present on the node is transmitted back.

Additionally each node can be identified by frontend admins and users for
request and response headers, allowing more debugging.

Improve the tests by extending assertSlaveBase and reuse it instead of
duplicating the code everywhere.

software/caddy-frontend/buildout.hash.cfg

@@ -22,19 +22,19 @@ md5sum = 5784bea3bd608913769ff9a8afcccb68
 
 [profile-caddy-frontend]
 filename = instance-apache-frontend.cfg.in
-md5sum = 3e3021b86c3cfe93553489441da85496
+md5sum = 04e550480d3057ca65d87c6fadbaed6e
 
 [profile-caddy-replicate]
 filename = instance-apache-replicate.cfg.in
-md5sum = c028f1c5947494e7f25cf8266a3ecd2d
+md5sum = 63b418626ef0f8ac54d6359fb6637371
 
 [profile-slave-list]
 _update_hash_filename_ = templates/apache-custom-slave-list.cfg.in
-md5sum = 6b6ab13d82bf9ecff6a37c3402ddbf95
+md5sum = e3ba0da5d137dcbd56c2604d200ac3b9
 
 [profile-replicate-publish-slave-information]
 _update_hash_filename_ = templates/replicate-publish-slave-information.cfg.in
-md5sum = df304a8aee87b6f2425241016a48f7a5
+md5sum = be54431846fe7f3cee65260eefc83d62
 
 [profile-caddy-frontend-configuration]
 _update_hash_filename_ = templates/Caddyfile.in
@@ -46,11 +46,11 @@ md5sum = 88af61e7abbf30dc99a1a2526161128d
 
 [template-default-slave-virtualhost]
 _update_hash_filename_ = templates/default-virtualhost.conf.in
-md5sum = 37475d79f28c5f126bc1947fdb938fdb
+md5sum = 57c86795293b11300a036f5f8cf2c868
 
 [template-backend-haproxy-configuration]
 _update_hash_filename_ = templates/backend-haproxy.cfg.in
-md5sum = ae4c9ce775ea003aa51eda5ecbbeec73
+md5sum = 6d4ad68ac44ccc72fe9148bd8e05a6f0
 
 [template-empty]
 _update_hash_filename_ = templates/empty.in
@@ -62,7 +62,7 @@ md5sum = 975177dedf677d24e14cede5d13187ce
 
 [template-trafficserver-records-config]
 _update_hash_filename_ = templates/trafficserver/records.config.jinja2
-md5sum = e87238c53d080ef9ef90040e57bc1395
+md5sum = 715baa302d562a7e4eddc3d1bf72f981
 
 [template-trafficserver-storage-config]
 _update_hash_filename_ = templates/trafficserver/storage.config.jinja2
@@ -94,7 +94,7 @@ md5sum = 8c150e1e6c993708d31936742f3a7302
 
 [caddyprofiledeps-setup]
 filename = setup.py
-md5sum = 6aad2b4c271294f524214192ee197c15
+md5sum = f6f72d03af7d9dc29fb4d4fef1062e73
 
 [caddyprofiledeps-dummy]
 filename = caddyprofiledummy.py

software/caddy-frontend/instance-apache-frontend.cfg.in

@@ -63,6 +63,75 @@ parts =
 [caddyprofiledeps]
 recipe = caddyprofiledeps
 
+[frontend-node-id]
+# Store id file in top of hierarchy, so it does not depend on directory creation
+file = ${buildout:directory}/.frontend-node-id.txt
+recipe = slapos.recipe.build
+init =
+  import os
+  import secrets
+  if not os.path.exists(options['file']):
+    with open(options['file'], 'w') as fh:
+      fh.write(secrets.token_urlsafe(4))
+  with open(options['file'], 'r') as fh:
+    options['value'] = fh.read()
+
+[frontend-node-private-salt]
+# Private, not communicated, stable hash, which can be used to salt other
+# hashes, so their values are connected to the node, but practicaly impossible
+# to crack (until the node is hacked itself, but then those values are
+# stolen anyway)
+recipe = slapos.recipe.build
+init =
+  import os
+  import uuid
+  if not os.path.exists(options['file']):
+    with open(options['file'], 'w') as fh:
+      fh.write(uuid.uuid4().hex)
+  with open(options['file'], 'r') as fh:
+    options['value'] = fh.read()
+file = ${buildout:directory}/.frontend-node-private-salt.txt
+
+[version-hash]
+recipe = slapos.recipe.build
+software-release-url = ${slap-connection:software-release-url}
+hash-salt = ${frontend-node-private-salt:value}
+init =
+  import hashlib
+  import base64
+  options['value'] = base64.urlsafe_b64encode(hashlib.md5(''.join([options['software-release-url'].strip(), options['hash-salt']])).digest())
+
+[frontend-node-information]
+recipe = slapos.recipe.build
+file = ${buildout:directory}/.frontend-node-information.json
+node-id = ${frontend-node-id:value}
+current-hash = ${version-hash:value}
+current-software-release-url = ${version-hash:software-release-url}
+init =
+  import json
+  changed = False
+  try:
+    with open(options['file'], 'r') as fh:
+      data = json.load(fh)
+  except Exception:
+    changed = True
+    data = {
+      'node-id': options['node-id'],
+      'version-hash-history': {options['current-hash']: options['current-software-release-url']}
+    }
+  if 'node-id' not in data:
+    data['node-id'] = options['node-id']
+    changed = True
+  if 'version-hash-history' not in data:
+    data['version-hash-history'] = {}
+    changed = True
+  if options['current-hash'] not in data['version-hash-history']:
+    data['version-hash-history'][options['current-hash']] = options['current-software-release-url']
+    changed = True
+  if changed:
+    with open(options['file'], 'w') as fh:
+      json.dump(data, fh)
+  options['value'] = data
 # Create all needed directories
 [directory]
 recipe = slapos.cookbook:mkdirectory
@@ -305,6 +374,10 @@ extra-context =
     key software_type :software_type
     key frontend_lazy_graceful_reload frontend-caddy-lazy-graceful:rendered
     key monitor_base_url monitor-instance-parameter:monitor-base-url 
+    key node_id frontend-node-id:value
+    key version_hash version-hash:value
+    key software_release_url version-hash:software-release-url
+    key node_information frontend-node-information:value
     key custom_ssl_directory caddy-directory:custom-ssl-directory
 # BBB: SlapOS Master non-zero knowledge BEGIN
     key apache_certificate apache-certificate:rendered
@@ -462,6 +535,8 @@ disk-cache-size = ${configuration:disk-cache-size}
 ram-cache-size = ${configuration:ram-cache-size}
 templates-dir = {{ software_parameter_dict['trafficserver'] }}/etc/trafficserver/body_factory
 request-timeout = ${configuration:request-timeout}
+version-hash = ${version-hash:value}
+node-id = ${frontend-node-id:value}
 
 [trafficserver-configuration-directory]
 recipe = plone.recipe.command

software/caddy-frontend/instance-apache-replicate.cfg.in

@@ -286,7 +286,7 @@ config-monitor-username = ${monitor-instance-parameter:username}
 config-monitor-password = ${monitor-htpasswd:passwd}
 
 software-type = {{frontend_type}}
-return = slave-instance-information-list monitor-base-url backend-client-csr-url kedifa-csr-url csr-certificate backend-haproxy-statistic-url
+return = slave-instance-information-list monitor-base-url backend-client-csr-url kedifa-csr-url csr-certificate backend-haproxy-statistic-url node-information-json
 
 {#- Send only needed parameters to frontend nodes #}
 {%- set base_node_configuration_dict = {} %}
@@ -376,6 +376,7 @@ kedifa-csr-certificate = ${request-kedifa:connection-csr-certificate}
 {% for frontend in frontend_list %}
   {% set section_part = '${request-' + frontend %}
 {{ frontend }}-backend-haproxy-statistic-url = {{ section_part }}:connection-backend-haproxy-statistic-url}
+{{ frontend }}-node-information-json = ${frontend-information:{{ frontend }}-node-information-json}
 {% endfor %}
 {% if not aibcc_enabled %}
 {% for frontend in frontend_list %}
@@ -461,6 +462,12 @@ warning-slave-dict = {{ dumps(warning_slave_dict) }}
 {# sort_keys are important in order to avoid shuffling parameters on each run #}
 active-slave-instance-list = {{ json_module.dumps(active_slave_instance_list, sort_keys=True) }}
 
+[frontend-information]
+{% for frontend in frontend_list %}
+  {% set section_part = '${request-' + frontend %}
+{{ frontend }}-node-information-json = {{ section_part }}:connection-node-information-json}
+{% endfor %}
+
 [dynamic-publish-slave-information]
 < = jinja2-template-base
 template = {{ software_parameter_dict['profile_replicate_publish_slave_information'] }}
@@ -468,6 +475,7 @@ filename = dynamic-publish-slave-information.cfg
 extensions = jinja2.ext.do
 extra-context =
     section slave_information slave-information
+    section frontend_information frontend-information
     section rejected_slave_information rejected-slave-information
     section active_slave_instance_dict active-slave-instance
     section warning_slave_information warning-slave-information

software/caddy-frontend/setup.py

@@ -10,6 +10,7 @@ setup(
     'furl',
     'orderedmultidict',
     'caucase',
+    'python2-secrets',
   ],
   entry_points={
     'zc.buildout': [

software/caddy-frontend/software.cfg

@@ -214,6 +214,7 @@ kedifa = 0.0.6
 # Modern KeDiFa requires zc.lockfile
 zc.lockfile = 1.4
 
+python2-secrets = 1.0.5
 validators = 0.12.2
 PyRSS2Gen = 1.1
 cns.recipe.symlink = 0.2.3

software/caddy-frontend/templates/apache-custom-slave-list.cfg.in

@@ -330,6 +330,8 @@ certificate = {{ certificate }}
 https_port = {{ dumps('' ~ configuration['port']) }}
 http_port = {{ dumps('' ~ configuration['plain_http_port']) }}
 local_ipv4 = {{ dumps('' ~ instance_parameter_dict['ipv4-random']) }}
+version-hash = {{ version_hash }}
+node-id = {{ node_id }}
 {%-   for key, value in slave_instance.iteritems() %}
 {%-     if value is not none %}
 {{ key }} = {{ dumps(value) }}
@@ -463,6 +465,8 @@ csr-certificate = ${expose-csr-certificate-get:certificate}
 {#-   We unquote, as furl quotes automatically, but there is buildout value on purpose like ${...:...} in the passwod #}
 {%-   set statistic_url = urlparse_module.unquote(furled.tostr()) %}
 backend-haproxy-statistic-url = {{ statistic_url }}
+{#-   sort_keys are important in order to avoid shuffling parameters on each run #}
+node-information-json = {{ json_module.dumps(node_information, sort_keys=True) }}
 
 [kedifa-updater]
 recipe = slapos.cookbook:wrapper
@@ -513,6 +517,8 @@ global-ipv6 = ${slap-configuration:ipv6-random}
 request-timeout = {{ dumps('' ~ configuration['request-timeout']) }}
 backend-connect-timeout = {{ dumps('' ~ configuration['backend-connect-timeout']) }}
 backend-connect-retries =  {{ dumps('' ~ configuration['backend-connect-retries']) }}
+version-hash = {{ version_hash }}
+node-id = {{ node_id }}
 
 [template-expose-csr-link-csr]
 recipe = plone.recipe.command

software/caddy-frontend/templates/backend-haproxy.cfg.in

@@ -58,6 +58,8 @@ frontend statistic
 
 frontend http-backend
   bind {{ configuration['local-ipv4'] }}:{{ configuration['http-port'] }}
+  http-request add-header Via "%HV rapid-cdn-backend-{{ configuration['node-id'] }}-{{ configuration['version-hash'] }}"
+  http-response add-header Via "%HV rapid-cdn-backend-{{ configuration['node-id'] }}-{{ configuration['version-hash']}}"
 {%- for slave_instance in backend_slave_list -%}
 {{ frontend_entry(slave_instance, 'http', False) }}
 {%- endfor %}

software/caddy-frontend/templates/default-virtualhost.conf.in

@@ -21,6 +21,10 @@
     # workaround for lost connection to haproxy by reconnecting
     try_duration 3s
     try_interval 250ms
+    header_upstream +Via "{proto} rapid-cdn-frontend-{{ slave_parameter['node-id'] }}-{{ slave_parameter['version-hash'] }}"
+{%-   if not slave_parameter['disable-via-header'] %}
+    header_downstream +Via "{proto} rapid-cdn-frontend-{{ slave_parameter['node-id'] }}-{{ slave_parameter['version-hash'] }}"
+{%-   endif %}
 {%- endmacro %} {# proxy_header #}
 
 {%- macro hsts_header(tls) %}

software/caddy-frontend/templates/replicate-publish-slave-information.cfg.in

@@ -72,6 +72,9 @@ log-access-url = {{ dumps(json_module.dumps(log_access_url, sort_keys=True)) }}
 {{ key }} = {{ dumps(value) }}
 {%     endfor %}
 {%   endif %}
+{%   for frontend_key, frontend_value in frontend_information.iteritems() %}
+{{ frontend_key }} = {{ frontend_value }}
+{%   endfor %}
 {% endfor %}
 
 [buildout]

software/caddy-frontend/templates/trafficserver/records.config.jinja2

@@ -18,6 +18,12 @@ LOCAL proxy.local.incoming_ip_to_bind STRING {{ ats_configuration['local-ip'] }}
 CONFIG proxy.config.log.logfile_dir STRING {{ ats_directory['log'] }}
 # Never change Server header
 CONFIG proxy.config.http.response_server_enabled INT 0
+# Handle Via header
+CONFIG proxy.config.http.insert_request_via_str INT 1
+CONFIG proxy.config.http.request_via_str STRING rapid-cdn-cache-{{ ats_configuration['node-id'] }}-{{ ats_configuration['version-hash'] }}
+CONFIG proxy.config.http.insert_response_via_str INT 1
+CONFIG proxy.config.http.response_via_str STRING rapid-cdn-cache-{{ ats_configuration['node-id'] }}-{{ ats_configuration['version-hash'] }}
+
 # Implement RFC 5861 with core
 CONFIG proxy.config.http.cache.open_write_fail_action INT 2
 CONFIG proxy.config.body_factory.template_sets_dir STRING  {{ ats_configuration['templates-dir'] }}
@@ -53,13 +59,6 @@ CONFIG proxy.config.exec_thread.affinity INT 1
 ##############################################################################
 CONFIG proxy.config.http.server_ports STRING {{ ats_configuration['local-ip'] + ':' + ats_configuration['input-port'] }}
 
-##############################################################################
-# Via: headers. Docs:
-#     https://docs.trafficserver.apache.org/records.config#proxy-config-http-insert-response-via-str
-##############################################################################
-CONFIG proxy.config.http.insert_request_via_str INT 1
-CONFIG proxy.config.http.insert_response_via_str INT 0
-
 ##############################################################################
 # Parent proxy configuration, in addition to these settings also see parent.config. Docs:
 #    https://docs.trafficserver.apache.org/records.config#parent-proxy-configuration