Update Release Candidate

component/caddy/buildout.cfg

 [buildout]
 extends =
   ../../component/golang/buildout.cfg
+  gowork.cfg
 
 parts =
+  gowork
   caddy
 
+[gowork.goinstall]
+command = bash -c ". ${gowork:env.sh} && cd ${go_github.com_caddyserver_caddy:location} && GO111MODULE=on go install -v $(echo -n '${gowork:install}' |tr '\n' ' ')"
+
 [gowork]
-# Caddy 1.x+ uses go modules, for which gowork does not work yet
 golang  = ${golang1.12:location}
 install =
-
-[gowork.goinstall]
-command = :
-depends =
-  ${caddy:recipe}
+  github.com/caddyserver/caddy/...
 
 [caddy]
-# revision and repository can be used to control which caddy version is used
-revision = v1.0.3
-repository = github.com/caddyserver/caddy/caddy
-
-recipe  = plone.recipe.command
+recipe = plone.recipe.command
+command = exit 0
 update-command = ${:command}
-stop-on-error = True
-# GO111MODULE=on enables go modules support
-# the chmod is needed as modules are fetched with u-w
-command =
-  . ${gowork:env.sh} &&
-  cd ${gowork:directory} &&
-  export GO111MODULE=on &&
-  go get ${:repository}@${:revision} &&
-  chmod -R u+w .
 output = ${gowork:bin}/caddy
 location = ${:output}

component/caddy/gowork.cfg

+# Code generated by gowork-snapshot; DO NOT EDIT.
+
+# list of go git repositories to fetch
+[gowork.goinstall]
+depends_gitfetch  =
+    ${go_github.com_caddyserver_caddy:recipe}
+
+[go_github.com_caddyserver_caddy]
+<= go-git-package
+go.importpath = github.com/caddyserver/caddy
+repository    = https://lab.nexedi.com/nexedi/caddy.git
+revision      = nxd-v1.0.3-1-g2c11cedc

component/mariadb/buildout.cfg

@@ -125,3 +125,34 @@ environment =
   CPPFLAGS=-I${groonga:location}/include/groonga -I${pcre:location}/include
   LDFLAGS=-L${groonga:location}/lib
   PKG_CONFIG_PATH=${groonga:location}/lib/pkgconfig:${groonga-normalizer-mysql:location}/lib/pkgconfig
+
+
+### XXX keep using mariadb 10.3.22 because 10.4 cause crash
+### (we just override here for easier revert)
+[mariadb]
+version = 10.3.22
+url = https://downloads.mariadb.org/f/mariadb-10.3.22/source/mariadb-10.3.22.tar.gz/from/http%3A//ftp.hosteurope.de/mirror/archive.mariadb.org/?serve
+md5sum = f712a5e6fde038d0c9c6d2a2cd88b84e
+pre-configure =
+  set -e '\bSET(PLUGIN_AUTH_PAM YES)' cmake/build_configurations/mysql_release.cmake
+  grep -q "$@"
+  sed -i "/$1/d" "$2"
+post-install =
+  ldd=`ldd ${:location}/lib/plugin/ha_rocksdb.so`
+  for x in ${lz4:location} ${snappy:location} ${zstd:location}
+  do echo "$ldd" |grep -qF " $x/lib/"
+  done
+  mkdir -p ${:location}/include/wsrep &&
+  cp -p wsrep/wsrep_api.h ${:location}/include/wsrep
+
+[mroonga-mariadb]
+pre-configure =
+  rm -rf fake_mariadb_source
+  mkdir -p fake_mariadb_source
+  cd fake_mariadb_source
+  ln -s ${mariadb:location}/wsrep-lib
+  cp -a ${mariadb:location}/include/mysql/server include
+  cp -a include/private sql
+  chmod -R a+w include sql # so that buildout can delete this compile-dir after install
+  mkdir -p ${:plugin-dir}
+### /XXX keep using mariadb 10.3.22 because 10.4 cause crash

component/openssl/buildout.cfg

@@ -53,6 +53,7 @@ stop-on-error = true
 update-command = ${:command}
 command = ${coreutils-output:test} -x ${:openssl}
 openssl = ${openssl:location}/bin/openssl
+bin = ${openssl:location}
 
 [openssl-1.0]
 recipe = slapos.recipe.cmmi

component/trafficserver/buildout.cfg

 [buildout]
 extends =
+  ../defaults.cfg
   ../libtool/buildout.cfg
   ../libxml2/buildout.cfg
   ../make/buildout.cfg
@@ -14,10 +15,15 @@ extends =
 parts =
   trafficserver
 
+[gcc]
+# For ATS v8.0.0 and later, a compiler with support for C++17 is required. 
+# https://cwiki.apache.org/confluence/display/TS/Building
+min_version = 8
+
 [trafficserver]
 recipe = slapos.recipe.cmmi
-url = http://apache.claz.org/trafficserver/trafficserver-7.1.11.tar.bz2
-md5sum = 74e8c1e2d7cdc0b97420d8b813321258
+url = http://apache.claz.org/trafficserver/trafficserver-8.1.0.tar.bz2
+md5sum = 99bfeb61095e55cb151ef58d884cb3f1
 shared = true
 configure-options =
   --with-openssl=${openssl:location}
@@ -33,6 +39,7 @@ configure-options =
 environment =
   PATH=${libtool:location}/bin:${make:location}/bin:${perl:location}/bin:${pkgconfig:location}/bin:%(PATH)s
   LDFLAGS =-L${openssl:location}/lib -Wl,-rpath=${openssl:location}/lib -L${tcl:location}/lib -Wl,-rpath=${tcl:location}/lib -L${zlib:location}/lib -Wl,-rpath=${zlib:location}/lib
+  
 make-target =
   check
   install

software/caddy-frontend/buildout.hash.cfg

@@ -14,7 +14,7 @@
 # not need these here).
 [template]
 filename = instance.cfg.in
-md5sum = 30eb6db0d2df9ffb3a50fb64e73df5be
+md5sum = 7b16383a41d403f6c21e684afffcb4b7
 
 [template-common]
 filename = instance-common.cfg.in
@@ -22,7 +22,7 @@ md5sum = 5784bea3bd608913769ff9a8afcccb68
 
 [template-apache-frontend]
 filename = instance-apache-frontend.cfg.in
-md5sum = 1709085d62f46b22cbe9369fe324bb49
+md5sum = b908d09e07b479cf09f1c813522e5cac
 
 [template-caddy-replicate]
 filename = instance-apache-replicate.cfg.in
@@ -30,7 +30,7 @@ md5sum = b684aacbb7b2a6eec3ca3ab06c61a92c
 
 [template-slave-list]
 _update_hash_filename_ = templates/apache-custom-slave-list.cfg.in
-md5sum = da4e3de1c1861459273c6e8d32ead21e
+md5sum = e9c5fe132a26e960df0acf3f5d8f877e
 
 [template-replicate-publish-slave-information]
 _update_hash_filename_ = templates/replicate-publish-slave-information.cfg.in
@@ -46,7 +46,7 @@ md5sum = 0979a03476e86bf038516c9565dadc17
 
 [template-not-found-html]
 _update_hash_filename_ = templates/notfound.html
-md5sum = f20d6c3d2d94fb685f8d26dfca1e822b
+md5sum = 88af61e7abbf30dc99a1a2526161128d
 
 [template-default-slave-virtualhost]
 _update_hash_filename_ = templates/default-virtualhost.conf.in
@@ -70,15 +70,15 @@ md5sum = 975177dedf677d24e14cede5d13187ce
 
 [template-trafficserver-records-config]
 _update_hash_filename_ = templates/trafficserver/records.config.jinja2
-md5sum = 56c7e9f5cc92f8c519f48496010e2078
+md5sum = 18076ae37306aca1f6ef11e2173c887f
 
 [template-trafficserver-storage-config]
 _update_hash_filename_ = templates/trafficserver/storage.config.jinja2
-md5sum = baf7b89cc9ab5506100b0c900808c1ea
+md5sum = d022455a8610bac2dd51101edb035987
 
-[template-trafficserver-logging-config]
-_update_hash_filename_ = templates/trafficserver/logging.config.jinja2
-md5sum = 6aed31174dc262ced02f31624321df41
+[template-trafficserver-logging-yaml]
+_update_hash_filename_ = templates/trafficserver/logging.yaml.jinja2
+md5sum = 45f379e887de07d2b86de2f43937f856
 
 [template-nginx-eventsource-slave-virtualhost]
 _update_hash_filename_ = templates/nginx-eventsource-slave.conf.in
@@ -114,7 +114,7 @@ md5sum = 38792c2dceae38ab411592ec36fff6a8
 
 [template-kedifa]
 filename = instance-kedifa.cfg.in
-md5sum = c784ac325fbd383277bf1ae551abcfd0
+md5sum = d76fe7bf062410eda7049446ed06a736
 
 [template-backend-haproxy-rsyslogd-conf]
 _update_hash_filename_ = templates/backend-haproxy-rsyslogd.conf.in

software/caddy-frontend/common.cfg

@@ -131,7 +131,7 @@ template_not_found_html = ${template-not-found-html:target}
 template_slave_list = ${template-slave-list:target}
 template_trafficserver_records_config = ${template-trafficserver-records-config:target}
 template_trafficserver_storage_config = ${template-trafficserver-storage-config:target}
-template_trafficserver_logging_config = ${template-trafficserver-logging-config:target}
+template_trafficserver_logging_yaml = ${template-trafficserver-logging-yaml:target}
 template_wrapper = ${template-wrapper:output}
 template_slave_introspection_httpd_nginx = ${template-slave-introspection-httpd-nginx:target}
 
@@ -218,7 +218,7 @@ mode = 0644
 [template-trafficserver-storage-config]
 <=download-template
 
-[template-trafficserver-logging-config]
+[template-trafficserver-logging-yaml]
 <=download-template
 
 [template-rotate-script]

software/caddy-frontend/instance-apache-frontend.cfg.in

@@ -23,6 +23,7 @@ parts =
   promise-caddy-frontend-v4-http
   promise-caddy-frontend-v6-https
   promise-caddy-frontend-v6-http
+  promise-logrotate-setup
 
   trafficserver-launcher
   trafficserver-reload
@@ -32,7 +33,7 @@ parts =
   trafficserver-plugin-config
   trafficserver-storage-config
   trafficserver-ip-allow-config
-  trafficserver-logging-config
+  trafficserver-logging-yaml
   trafficserver-promise-listen-port
   trafficserver-promise-cache-availability
   cron-entry-logrotate-trafficserver
@@ -459,6 +460,7 @@ rotate-num = ${configuration:rotate-num}
 #       Here we trust that there will be something to be rotated with error
 #       or access log, and that this will trigger postrotate script.
 post = ${frontend-caddy-lazy-graceful:rendered} &
+delaycompress =
 
 #################
 # Trafficserver
@@ -482,8 +484,6 @@ plugin-config =
 ip-allow-config = src_ip=0.0.0.0-255.255.255.255 action=ip_allow
 cache-path = ${trafficserver-directory:cache-path}
 disk-cache-size = ${configuration:disk-cache-size}
-synthetic-port = ${configuration:trafficserver-synthetic-port}
-mgmt-port = ${configuration:trafficserver-mgmt-port}
 ram-cache-size = ${configuration:ram-cache-size}
 templates-dir = {{ parameter_dict['trafficserver'] }}/etc/trafficserver/body_factory
 request-timeout = ${configuration:request-timeout}
@@ -495,7 +495,7 @@ target = ${trafficserver-directory:configuration}
 
 [trafficserver-launcher]
 recipe = slapos.cookbook:wrapper
-command-line = {{ parameter_dict['trafficserver'] }}/bin/traffic_cop
+command-line = {{ parameter_dict['trafficserver'] }}/bin/traffic_manager
 wrapper-path = ${trafficserver-variable:wrapper-path}
 environment = TS_ROOT=${buildout:directory}
 hash-existing-files = ${buildout:directory}/software_release/buildout.cfg
@@ -529,10 +529,10 @@ extra-context =
 template = {{ parameter_dict['template_trafficserver_storage_config'] }}
 filename = storage.config
 
-[trafficserver-logging-config]
+[trafficserver-logging-yaml]
 < = trafficserver-jinja2-template-base
-template = {{ parameter_dict['template_trafficserver_logging_config'] }}
-filename = logging.config
+template = {{ parameter_dict['template_trafficserver_logging_yaml'] }}
+filename = logging.yaml
 
 [trafficserver-remap-config]
 <= trafficserver-jinja2-template-base
@@ -768,6 +768,7 @@ rotate-num = ${configuration:rotate-num}
 #       Here we trust that there will be something to be rotated with error
 #       or access log, and that this will trigger postrotate script.
 post = ${backend-haproxy-rsyslogd-lazy-graceful:rendered} &
+delaycompress =
 
 [backend-haproxy-configuration-state]
 <= jinja2-template-base
@@ -1003,6 +1004,14 @@ name = slave-introspection
 log = ${caddy-configuration:slave-introspection-access-log} ${caddy-configuration:slave-introspection-error-log}
 rotate-num = ${configuration:rotate-num}
 post = kill -USR1 $(cat ${caddy-configuration:slave-introspection-pid-file})
+delaycompress =
+
+[promise-logrotate-setup]
+<= monitor-promise-base
+module = check_command_execute
+name = ${:_buildout_section_name_}.py
+config-command =
+  ${logrotate:wrapper-path} -d
 
 [configuration]
 {%- for key, value in instance_parameter.iteritems() -%}

software/caddy-frontend/instance-kedifa.cfg.in

@@ -19,6 +19,7 @@ parts =
   caucase-updater
   expose-csr_id
   promise-expose-csr_id-ip-port
+  promise-logrotate-setup
 
 [monitor-instance-parameter]
 # Note: Workaround for monitor stack, which uses monitor-httpd-port parameter
@@ -236,6 +237,7 @@ config-ca-cert-file = ${kedifa-config:ca-certificate}
 name = kedifa
 log = ${kedifa-config:logfile}
 rotate-num = {{ instance_parameter['configuration.rotate-num'] | int }}
+delaycompress =
 
 [kedifa]
 recipe = slapos.cookbook:wrapper
@@ -301,4 +303,12 @@ master-key-download-url = https://[${kedifa-config:ip}]:${kedifa-config:port}/${
 csr_id-url = https://[${expose-csr_id-configuration:ip}]:${expose-csr_id-configuration:port}/csr_id.txt
 csr_id-certificate = ${get-csr_id-certificate:certificate}
 monitor-base-url = ${monitor-instance-parameter:monitor-base-url}
+
+[promise-logrotate-setup]
+<= monitor-promise-base
+module = check_command_execute
+name = ${:_buildout_section_name_}.py
+config-command =
+  ${logrotate:wrapper-path} -d
+
 {%- endif -%} {# if slap_software_type in software_type #}

software/caddy-frontend/instance.cfg.in

@@ -110,8 +110,6 @@ configuration.apache-certificate =
 configuration.open-port = 80 443
 configuration.disk-cache-size = 8G
 configuration.ram-cache-size = 1G
-configuration.trafficserver-synthetic-port = 8083
-configuration.trafficserver-mgmt-port = 8084
 configuration.re6st-verification-url = http://[2001:67c:1254:4::1]/index.html
 configuration.enable-http2-by-default = true
 configuration.global-disable-http2 = false

software/caddy-frontend/templates/apache-custom-slave-list.cfg.in

@@ -156,6 +156,8 @@ name = ${:_buildout_section_name_}
 log = {{slave_parameter_dict.get('access_log')}} {{slave_parameter_dict.get('error_log')}} {{slave_parameter_dict.get('backend_log')}}
 backup = {{ slave_log_folder }}
 rotate-num = {{ dumps('' ~ configuration['rotate-num']) }}
+# disable delayed compression, as log filenames shall be stable
+delaycompress =
 
 {#-   integrate current logs inside #}
 
@@ -480,6 +482,7 @@ parts +=
     kedifa-updater
     kedifa-updater-run
     backend-haproxy-configuration
+    promise-logrotate-setup
 {%- for part in part_list %}
 {{ '    %s' % part }}
 {%- endfor %}
@@ -571,4 +574,10 @@ recipe = collective.recipe.shelloutput
 commands =
   certificate = cat ${certificate-csr_id:certificate}
 
+[promise-logrotate-setup]
+<= monitor-promise-base
+module = check_command_execute
+name = ${:_buildout_section_name_}.py
+config-command =
+  ${logrotate:wrapper-path} -d
 {%- endif %} {# if software_type == slap_software_type #}

software/caddy-frontend/templates/notfound.html

@@ -3,7 +3,17 @@
   <title>Instance not found</title>
 </head>
 <body>
-<h1>This instance has not been found.</h1>
-<p>If this error persists, please check your instance URL and status on SlapOS Master.</p>
+<h1>The instance has not been found</h1>
+<p>The reasons of this could be:</p>
+<ul>
+<li>the instance does not exists or the URL is incorrect
+<ul>
+<li>in this case please check the URL
+</ul>
+<li>the instance has been stopped
+<ul>
+<li>in this case please check in the SlapOS Master if the instance is started or wait a bit for it to start
+</ul>
+</ul>
 </body>
 </html>

software/caddy-frontend/templates/trafficserver/logging.config.jinja2

-squid = format {
-   Format = '%<cqtq> %<ttms> %<chi> %<crc>/%<pssc> %<psql> %<cqhm> %<cquc> %<cluc> %<caun> %<phr>/%<pqsn> %<psct>'
-}
-log.ascii {
-   Format = squid,
-   Filename = 'squid',
-   RollingEnabled = 1,
-   RollingIntervalSec = 86400,
-   RollingOffsetHr = 0
-}

software/caddy-frontend/templates/trafficserver/logging.yaml.jinja2

+formats:
+  - name: squid
+    format: '%<cqtq> %<ttms> %<chi> %<crc>/%<pssc> %<psql> %<cqhm> %<cquc> %<cluc> %<caun> %<phr>/%<shn> %<psct>'
+logs:
+  - filename: squid
+    format: squid
+    mode: ascii
+    rolling_enabled: 1
+    rolling_interval_sec: 86400
+    rolling_offset_hr: 0

software/caddy-frontend/templates/trafficserver/records.config.jinja2

@@ -5,7 +5,7 @@
 ##############################################################################
 
 ##############################################################################
-# SlapOS Specific configuration
+# SlapOS Specific configuration not available in default records.config
 ##############################################################################
 CONFIG proxy.config.proxy_name STRING {{ ats_configuration['hostname'] }}
 CONFIG proxy.config.local_state_dir STRING {{ ats_directory['local-state'] }}
@@ -18,8 +18,6 @@ CONFIG proxy.config.output.logfile STRING traffic.out
 CONFIG proxy.config.snapshot_dir STRING snapshots
 CONFIG proxy.config.admin.user_id STRING {{ '#%s' % os_module.geteuid() }}
 CONFIG proxy.config.admin.number_config_bak INT 0
-CONFIG proxy.config.process_manager.mgmt_port INT {{ ats_configuration['mgmt-port'] }}
-CONFIG proxy.config.admin.synthetic_port INT {{ ats_configuration['synthetic-port'] }}
 LOCAL proxy.local.incoming_ip_to_bind STRING {{ ats_configuration['local-ip'] }}
 CONFIG proxy.config.log.logfile_dir STRING {{ ats_directory['log'] }}
 # Implement RFC 5861 with core
@@ -28,14 +26,6 @@ CONFIG proxy.config.body_factory.template_sets_dir STRING  {{ ats_configuration[
 # Support stale-if-error by returning cached content on backend 5xx or unavailability
 CONFIG proxy.config.http.negative_revalidating_enabled INT 1
 CONFIG proxy.config.http.negative_revalidating_lifetime INT 86400
-##############################################################################
-# Proxy users variables. Docs:
-#    https://docs.trafficserver.apache.org/records.config#proxy-user-variables
-##############################################################################
-# Do not modify headers, as it needlessly pollutes information
-CONFIG proxy.config.http.insert_client_ip INT 0
-CONFIG proxy.config.http.insert_squid_x_forwarded_for INT 0
-
 
 ##############################################################################
 # Thread configurations. Docs:
@@ -80,7 +70,7 @@ CONFIG proxy.config.http.uncacheable_requests_bypass_parent INT 1
 CONFIG proxy.config.http.keep_alive_no_activity_timeout_in INT 120
 CONFIG proxy.config.http.keep_alive_no_activity_timeout_out INT 120
 CONFIG proxy.config.http.transaction_no_activity_timeout_in INT {{ ats_configuration['request-timeout'] }}
-CONFIG proxy.config.http.transaction_no_activity_timeout_out INT {{ ats_configuration['request-timeout'] }}
+CONFIG proxy.config.http.transaction_no_activity_timeout_out INT 30 {{ ats_configuration['request-timeout'] }}
 CONFIG proxy.config.http.transaction_active_timeout_in INT 900
 CONFIG proxy.config.http.transaction_active_timeout_out INT 0
 CONFIG proxy.config.http.accept_no_activity_timeout INT 120
@@ -107,6 +97,13 @@ CONFIG proxy.config.http.down_server.abort_threshold INT 10
 CONFIG proxy.config.http.negative_caching_enabled INT 0
 CONFIG proxy.config.http.negative_caching_lifetime INT 1800
 
+##############################################################################
+# Proxy users variables. Docs:
+#    https://docs.trafficserver.apache.org/records.config#proxy-user-variables
+##############################################################################
+CONFIG proxy.config.http.insert_client_ip INT 0
+CONFIG proxy.config.http.insert_squid_x_forwarded_for INT 0
+
 ##############################################################################
 # Security. Docs:
 #    https://docs.trafficserver.apache.org/records.config#security
@@ -125,7 +122,7 @@ CONFIG proxy.config.http.cache.http INT 1
 #    https://docs.trafficserver.apache.org/en/latest/admin-guide/files/cache.config.en.html
 ##############################################################################
 CONFIG proxy.config.http.cache.ignore_client_cc_max_age INT 1
-CONFIG proxy.config.http.normalize_ae_gzip INT 0
+CONFIG proxy.config.http.normalize_ae INT 0
 CONFIG proxy.config.http.cache.cache_responses_to_cookies INT 1
 CONFIG proxy.config.http.cache.cache_urls_that_look_dynamic INT 1
     # https://docs.trafficserver.apache.org/records.config#proxy-config-http-cache-when-to-revalidate
@@ -165,7 +162,7 @@ CONFIG proxy.config.cache.min_average_object_size INT 8000
 ##############################################################################
 # Logging Config. Docs:
 #    https://docs.trafficserver.apache.org/records.config#logging-configuration
-#    https://docs.trafficserver.apache.org/en/latest/admin-guide/files/logging.config.en.html
+#    https://docs.trafficserver.apache.org/en/latest/admin-guide/files/logging.yaml.en.html
 ##############################################################################
 CONFIG proxy.config.log.logging_enabled INT 3
 CONFIG proxy.config.log.max_space_mb_for_logs INT 25000
@@ -194,14 +191,14 @@ CONFIG proxy.config.reverse_proxy.enabled INT 1
 ##############################################################################
 CONFIG proxy.config.ssl.client.verify.server INT 0
 CONFIG proxy.config.ssl.client.CA.cert.filename STRING NULL
-CONFIG proxy.config.ssl.server.cipher_suite STRING ECDHE-ECDSA-AES256-GCM-SHA384:ECDHE-RSA-AES256-GCM-SHA384:ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-AES128-GCM-SHA256:DHE-RSA-AES256-GCM-SHA384:DHE-DSS-AES256-GCM-SHA384:DHE-RSA-AES128-GCM-SHA256:DHE-DSS-AES128-GCM-SHA256:ECDHE-ECDSA-AES256-SHA384:ECDHE-RSA-AES256-SHA384:ECDHE-ECDSA-AES256-SHA:ECDHE-RSA-AES256-SHA:ECDHE-ECDSA-AES128-SHA256:ECDHE-RSA-AES128-SHA256:ECDHE-ECDSA-AES128-SHA:ECDHE-RSA-AES128-SHA:DHE-RSA-AES256-SHA256:DHE-DSS-AES256-SHA256:DHE-RSA-AES128-SHA256:DHE-DSS-AES128-SHA256:DHE-RSA-AES256-SHA:DHE-DSS-AES256-SHA:DHE-RSA-AES128-SHA:DHE-DSS-AES128-SHA:AES256-GCM-SHA384:AES128-GCM-SHA256:AES256-SHA256:AES128-SHA256:AES256-SHA:AES128-SHA:DES-CBC3-SHA:!aNULL:!eNULL:!EXPORT:!DES:!RC4:!MD5:!PSK:!aECDH:!EDH-DSS-DES-CBC3-SHA:!EDH-RSA-DES-CBC3-SHA:!KRB5-DES-CBC3-SHA
+CONFIG proxy.config.ssl.server.cipher_suite STRING ECDHE-ECDSA-AES256-GCM-SHA384:ECDHE-RSA-AES256-GCM-SHA384:ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-AES128-GCM-SHA256:DHE-RSA-AES256-GCM-SHA384:DHE-DSS-AES256-GCM-SHA384:DHE-RSA-AES128-GCM-SHA256:DHE-DSS-AES128-GCM-SHA256:ECDHE-ECDSA-AES256-SHA384:ECDHE-RSA-AES256-SHA384:ECDHE-ECDSA-AES256-SHA:ECDHE-RSA-AES256-SHA:ECDHE-ECDSA-AES128-SHA256:ECDHE-RSA-AES128-SHA256:ECDHE-ECDSA-AES128-SHA:ECDHE-RSA-AES128-SHA:DHE-RSA-AES256-SHA256:DHE-DSS-AES256-SHA256:DHE-RSA-AES128-SHA256:DHE-DSS-AES128-SHA256:DHE-RSA-AES256-SHA:DHE-DSS-AES256-SHA:DHE-RSA-AES128-SHA:DHE-DSS-AES128-SHA:!aNULL:!eNULL:!EXPORT:!DES:!RC4:!MD5:!PSK:!aECDH:!EDH-DSS-DES-CBC3-SHA:!EDH-RSA-DES-CBC3-SHA:!KRB5-DES-CBC3-SHA
 
 ##############################################################################
 # Debugging. Docs:
 #    https://docs.trafficserver.apache.org/records.config#diagnostic-logging-configuration
 ##############################################################################
 CONFIG proxy.config.diags.debug.enabled INT 0
-CONFIG proxy.config.diags.debug.tags STRING http.*|dns.*
+CONFIG proxy.config.diags.debug.tags STRING http|dns
 # ToDo: Undocumented
 CONFIG proxy.config.dump_mem_info_frequency INT 0
 CONFIG proxy.config.http.slow.log.threshold INT 0

software/caddy-frontend/templates/trafficserver/storage.config.jinja2

@@ -12,4 +12,42 @@
 # Using a file for the cache storage
 #
 # <pathname> <size>
+#
+# Where 'pathname' is full path to the directory where you want
+# the cache-file to live and 'size' is size in bytes
+#
+# Example: 128MB cache file(/opt/slapgrid/shared/trafficserver/4e294bc2396af9b22bee270330a1f340/var/trafficserver/cache.db)
+#      /opt/slapgrid/shared/trafficserver/4e294bc2396af9b22bee270330a1f340/var/trafficserver 128M
+#
+# Example: 144MB cache file(/opt/slapgrid/shared/trafficserver/4e294bc2396af9b22bee270330a1f340/var/trafficserver/cache.db)
+#          assuming prefix of '/opt/slapgrid/shared/trafficserver/4e294bc2396af9b22bee270330a1f340'
+#      var/trafficserver 150994944
+#
+# Example: 512MB cache file(/opt/slapgrid/shared/trafficserver/4e294bc2396af9b22bee270330a1f340/var/trafficserver/cache.db)
+#          assuming prefix of '/opt/slapgrid/shared/trafficserver/4e294bc2396af9b22bee270330a1f340'
+#      var/trafficserver 512M
+#
+#
+#############################################################
+##              O_DIRECT Specific Configuration            ##
+#############################################################
+#
+# Examples: Using O_DIRECT on disks (Linux kernel >= 2.6.3,
+# FreeBSD > 5.3)
+#
+#      /dev/disc/by-id/[Insert_ID_Here_12345]         		# Linux
+#      /dev/disc/by-path/[Insert-Path-Here:12:34:56-1.0.0.0]	# Linux
+#
+#      /dev/ada1            					# FreeBSD
+#
+# Note that disks are identified by id or path. This is to prevent changes
+# by the kernel (which could occur if a disk was simply described as /dev/sda, sdb, etc.).
+#
+# Also note that when using these raw devices in O_DIRECT mode, you
+# do not need to specify the partition size. It's automatically
+# detected.
+#
+# A small default cache (256MB). This is set to allow for the regression test to succeed
+# most likely you'll want to use a larger cache. And, we definitely recommend the use
+# of raw devices for production caches.
 {{ ats_configuration.get("cache-path") }} {{ ats_configuration.get("disk-cache-size") }}

software/caddy-frontend/test/test.py

@@ -307,12 +307,13 @@ class TestDataMixin(object):
       'trafficserver/.diags.log.meta',
       'trafficserver/.manager.log.meta',
       'trafficserver/.squid.log.meta',
-      'trafficserver/.traffic.out.meta',
       'trafficserver/diags.log',
       'trafficserver/squid.log',
       # not important, appears sometimes
       'trafficserver/.error.log.meta',
       'trafficserver/error.log',
+      'trafficserver/.traffic.out.meta',
+      'trafficserver/traffic.out',
     ])
 
   def test_file_list_run(self):
@@ -1628,9 +1629,19 @@ class TestSlave(SlaveHttpFrontendTestCase, TestDataMixin):
   <title>Instance not found</title>
 </head>
 <body>
-<h1>This instance has not been found.</h1>
-<p>If this error persists, please check your instance URL and status on S"""
-      """lapOS Master.</p>
+<h1>The instance has not been found</h1>
+<p>The reasons of this could be:</p>
+<ul>
+<li>the instance does not exists or the URL is incorrect
+<ul>
+<li>in this case please check the URL
+</ul>
+<li>the instance has been stopped
+<ul>
+<li>in this case please check in the SlapOS Master if the instance is """
+      """started or wait a bit for it to start
+</ul>
+</ul>
 </body>
 </html>
 """,
@@ -1672,6 +1683,15 @@ class TestSlave(SlaveHttpFrontendTestCase, TestDataMixin):
       proto
     )
 
+  def test_telemetry_disabled(self):
+    # here we trust that telemetry not present in error log means it was
+    # really disabled
+    error_log_file = glob.glob(
+      os.path.join(
+       self.instance_path, '*', 'var', 'log', 'frontend-error.log'))[0]
+    with open(error_log_file) as fh:
+      self.assertNotIn('Sending telemetry', fh.read(), 'Telemetry enabled')
+
   def test_url(self):
     parameter_dict = self.assertSlaveBase('Url')
 
@@ -3676,7 +3696,7 @@ class TestSlave(SlaveHttpFrontendTestCase, TestDataMixin):
     self.assertNotEqual(via, None)
     self.assertRegexpMatches(
       via,
-      r'^http\/1.1 caddy-frontend-1\[.*\] \(ApacheTrafficServer\/7.1.11\)$'
+      r'^http\/1.1 caddy-frontend-1\[.*\] \(ApacheTrafficServer\/8.1.0\)$'
     )
 
   def test_enable_cache_server_alias(self):
@@ -3718,7 +3738,7 @@ class TestSlave(SlaveHttpFrontendTestCase, TestDataMixin):
     self.assertNotEqual(via, None)
     self.assertRegexpMatches(
       via,
-      r'^http\/1.1 caddy-frontend-1\[.*\] \(ApacheTrafficServer\/7.1.11\)$'
+      r'^http\/1.1 caddy-frontend-1\[.*\] \(ApacheTrafficServer\/8.1.0\)$'
     )
 
     result = fakeHTTPResult(
@@ -3780,7 +3800,7 @@ class TestSlave(SlaveHttpFrontendTestCase, TestDataMixin):
     self.assertNotEqual(via, None)
     self.assertRegexpMatches(
       via,
-      r'^http\/1.1 caddy-frontend-1\[.*\] \(ApacheTrafficServer\/7.1.11\)$'
+      r'^http\/1.1 caddy-frontend-1\[.*\] \(ApacheTrafficServer\/8.1.0\)$'
     )
 
     # check stale-if-error support (assumes stale-while-revalidate is same)
@@ -3832,11 +3852,47 @@ class TestSlave(SlaveHttpFrontendTestCase, TestDataMixin):
       self.assertNotEqual(via, None)
       self.assertRegexpMatches(
         via,
-        r'^http\/1.1 caddy-frontend-1\[.*\] \(ApacheTrafficServer\/7.1.11\)$'
+        r'^http\/1.1 caddy-frontend-1\[.*\] \(ApacheTrafficServer\/8.1.0\)$'
       )
     finally:
       self.startServerProcess()
     # END: check stale-if-error support
+    # BEGIN: Check that squid.log is correctly filled in
+    ats_log_file_list = glob.glob(
+      os.path.join(
+        self.instance_path, '*', 'var', 'log', 'trafficserver', 'squid.log'
+      ))
+    if len(ats_log_file_list) == 1:
+      ats_log_file = ats_log_file_list[0]
+    direct_pattern = re.compile(
+      r'.*TCP_MISS/200 .*test-path/deeper.*enablecache.example.com'
+      '.* - DIRECT*')
+    refresh_pattern = re.compile(
+      r'.*TCP_REFRESH_MISS/200 .*test-path/deeper.*enablecache.example.com'
+      '.* - DIRECT*')
+    # ATS needs some time to flush logs
+    timeout = 5
+    b = time.time()
+    while True:
+      direct_pattern_match = 0
+      refresh_pattern_match = 0
+      if (time.time() - b) > timeout:
+        break
+      with open(ats_log_file) as fh:
+        for line in fh.readlines():
+          if direct_pattern.match(line):
+            direct_pattern_match += 1
+          if refresh_pattern.match(line):
+            refresh_pattern_match += 1
+      if direct_pattern_match > 0 and refresh_pattern_match:
+        break
+      time.sleep(0.1)
+
+    with open(ats_log_file) as fh:
+      ats_log = fh.read()
+    self.assertRegexpMatches(ats_log, direct_pattern)
+    self.assertRegexpMatches(ats_log, refresh_pattern)
+    # END: Check that squid.log is correctly filled in
 
   def test_enable_cache_ats_timeout(self):
     parameter_dict = self.assertSlaveBase('enable_cache')
@@ -3957,7 +4013,7 @@ class TestSlave(SlaveHttpFrontendTestCase, TestDataMixin):
     self.assertNotEqual(via, None)
     self.assertRegexpMatches(
       via,
-      r'^http\/1.1 caddy-frontend-1\[.*\] \(ApacheTrafficServer\/7.1.11\)$'
+      r'^http\/1.1 caddy-frontend-1\[.*\] \(ApacheTrafficServer\/8.1.0\)$'
     )
 
     try:
@@ -4004,7 +4060,7 @@ class TestSlave(SlaveHttpFrontendTestCase, TestDataMixin):
     self.assertNotEqual(via, None)
     self.assertRegexpMatches(
       via,
-      r'^http\/1.1 caddy-frontend-1\[.*\] \(ApacheTrafficServer\/7.1.11\)$'
+      r'^http\/1.1 caddy-frontend-1\[.*\] \(ApacheTrafficServer\/8.1.0\)$'
     )
 
   def test_enable_http2_false(self):

software/caddy-frontend/test/test_data/test.TestEnableHttp2ByDefaultDefaultSlave.test_file_list_log-CADDY.txt

@@ -26,4 +26,3 @@ T-2/var/log/monitor-httpd-error.log
 T-2/var/log/slave-introspection-access.log
 T-2/var/log/slave-introspection-error.log
 T-2/var/log/trafficserver/manager.log
-T-2/var/log/trafficserver/traffic.out

software/caddy-frontend/test/test_data/test.TestEnableHttp2ByDefaultDefaultSlave.test_file_list_plugin-CADDY.txt

@@ -18,6 +18,7 @@ T-1/etc/plugin/kedifa-http-reply.py
 T-1/etc/plugin/monitor-bootstrap-status.py
 T-1/etc/plugin/monitor-http-frontend.py
 T-1/etc/plugin/monitor-httpd-listening-on-tcp.py
+T-1/etc/plugin/promise-logrotate-setup.py
 T-2/etc/plugin/__init__.py
 T-2/etc/plugin/backend-client-caucase-updater.py
 T-2/etc/plugin/backend-haproxy-configuration.py
@@ -35,6 +36,7 @@ T-2/etc/plugin/frontend-caddy-configuration-promise.py
 T-2/etc/plugin/monitor-bootstrap-status.py
 T-2/etc/plugin/monitor-http-frontend.py
 T-2/etc/plugin/monitor-httpd-listening-on-tcp.py
+T-2/etc/plugin/promise-logrotate-setup.py
 T-2/etc/plugin/re6st-connectivity.py
 T-2/etc/plugin/slave-introspection-configuration.py
 T-2/etc/plugin/slave_introspection_https.py

software/caddy-frontend/test/test_data/test.TestEnableHttp2ByDefaultDefaultSlaveGlobalDisableHttp2.test_file_list_log-CADDY.txt

@@ -26,4 +26,3 @@ T-2/var/log/monitor-httpd-error.log
 T-2/var/log/slave-introspection-access.log
 T-2/var/log/slave-introspection-error.log
 T-2/var/log/trafficserver/manager.log
-T-2/var/log/trafficserver/traffic.out

software/caddy-frontend/test/test_data/test.TestEnableHttp2ByDefaultDefaultSlaveGlobalDisableHttp2.test_file_list_plugin-CADDY.txt

@@ -18,6 +18,7 @@ T-1/etc/plugin/kedifa-http-reply.py
 T-1/etc/plugin/monitor-bootstrap-status.py
 T-1/etc/plugin/monitor-http-frontend.py
 T-1/etc/plugin/monitor-httpd-listening-on-tcp.py
+T-1/etc/plugin/promise-logrotate-setup.py
 T-2/etc/plugin/__init__.py
 T-2/etc/plugin/backend-client-caucase-updater.py
 T-2/etc/plugin/backend-haproxy-configuration.py
@@ -35,6 +36,7 @@ T-2/etc/plugin/frontend-caddy-configuration-promise.py
 T-2/etc/plugin/monitor-bootstrap-status.py
 T-2/etc/plugin/monitor-http-frontend.py
 T-2/etc/plugin/monitor-httpd-listening-on-tcp.py
+T-2/etc/plugin/promise-logrotate-setup.py
 T-2/etc/plugin/re6st-connectivity.py
 T-2/etc/plugin/slave-introspection-configuration.py
 T-2/etc/plugin/slave_introspection_https.py

software/caddy-frontend/test/test_data/test.TestEnableHttp2ByDefaultFalseSlave.test_file_list_log-CADDY.txt

@@ -26,4 +26,3 @@ T-2/var/log/monitor-httpd-error.log
 T-2/var/log/slave-introspection-access.log
 T-2/var/log/slave-introspection-error.log
 T-2/var/log/trafficserver/manager.log
-T-2/var/log/trafficserver/traffic.out

software/caddy-frontend/test/test_data/test.TestEnableHttp2ByDefaultFalseSlave.test_file_list_plugin-CADDY.txt

@@ -18,6 +18,7 @@ T-1/etc/plugin/kedifa-http-reply.py
 T-1/etc/plugin/monitor-bootstrap-status.py
 T-1/etc/plugin/monitor-http-frontend.py
 T-1/etc/plugin/monitor-httpd-listening-on-tcp.py
+T-1/etc/plugin/promise-logrotate-setup.py
 T-2/etc/plugin/__init__.py
 T-2/etc/plugin/backend-client-caucase-updater.py
 T-2/etc/plugin/backend-haproxy-configuration.py
@@ -35,6 +36,7 @@ T-2/etc/plugin/frontend-caddy-configuration-promise.py
 T-2/etc/plugin/monitor-bootstrap-status.py
 T-2/etc/plugin/monitor-http-frontend.py
 T-2/etc/plugin/monitor-httpd-listening-on-tcp.py
+T-2/etc/plugin/promise-logrotate-setup.py
 T-2/etc/plugin/re6st-connectivity.py
 T-2/etc/plugin/slave-introspection-configuration.py
 T-2/etc/plugin/slave_introspection_https.py

software/caddy-frontend/test/test_data/test.TestEnableHttp2ByDefaultFalseSlaveGlobalDisableHttp2.test_file_list_log-CADDY.txt

@@ -26,4 +26,3 @@ T-2/var/log/monitor-httpd-error.log
 T-2/var/log/slave-introspection-access.log
 T-2/var/log/slave-introspection-error.log
 T-2/var/log/trafficserver/manager.log
-T-2/var/log/trafficserver/traffic.out

software/caddy-frontend/test/test_data/test.TestEnableHttp2ByDefaultFalseSlaveGlobalDisableHttp2.test_file_list_plugin-CADDY.txt

@@ -18,6 +18,7 @@ T-1/etc/plugin/kedifa-http-reply.py
 T-1/etc/plugin/monitor-bootstrap-status.py
 T-1/etc/plugin/monitor-http-frontend.py
 T-1/etc/plugin/monitor-httpd-listening-on-tcp.py
+T-1/etc/plugin/promise-logrotate-setup.py
 T-2/etc/plugin/__init__.py
 T-2/etc/plugin/backend-client-caucase-updater.py
 T-2/etc/plugin/backend-haproxy-configuration.py
@@ -35,6 +36,7 @@ T-2/etc/plugin/frontend-caddy-configuration-promise.py
 T-2/etc/plugin/monitor-bootstrap-status.py
 T-2/etc/plugin/monitor-http-frontend.py
 T-2/etc/plugin/monitor-httpd-listening-on-tcp.py
+T-2/etc/plugin/promise-logrotate-setup.py
 T-2/etc/plugin/re6st-connectivity.py
 T-2/etc/plugin/slave-introspection-configuration.py
 T-2/etc/plugin/slave_introspection_https.py

software/caddy-frontend/test/test_data/test.TestMasterRequest.test_file_list_log-CADDY.txt

@@ -14,4 +14,3 @@ T-2/var/log/monitor-httpd-error.log
 T-2/var/log/slave-introspection-access.log
 T-2/var/log/slave-introspection-error.log
 T-2/var/log/trafficserver/manager.log
-T-2/var/log/trafficserver/traffic.out

software/caddy-frontend/test/test_data/test.TestMasterRequest.test_file_list_plugin-CADDY.txt

@@ -18,6 +18,7 @@ T-1/etc/plugin/kedifa-http-reply.py
 T-1/etc/plugin/monitor-bootstrap-status.py
 T-1/etc/plugin/monitor-http-frontend.py
 T-1/etc/plugin/monitor-httpd-listening-on-tcp.py
+T-1/etc/plugin/promise-logrotate-setup.py
 T-2/etc/plugin/__init__.py
 T-2/etc/plugin/backend-client-caucase-updater.py
 T-2/etc/plugin/backend-haproxy-configuration.py
@@ -35,6 +36,7 @@ T-2/etc/plugin/frontend-caddy-configuration-promise.py
 T-2/etc/plugin/monitor-bootstrap-status.py
 T-2/etc/plugin/monitor-http-frontend.py
 T-2/etc/plugin/monitor-httpd-listening-on-tcp.py
+T-2/etc/plugin/promise-logrotate-setup.py
 T-2/etc/plugin/re6st-connectivity.py
 T-2/etc/plugin/slave-introspection-configuration.py
 T-2/etc/plugin/slave_introspection_https.py

software/caddy-frontend/test/test_data/test.TestMasterRequestDomain.test_file_list_log-CADDY.txt

@@ -14,4 +14,3 @@ T-2/var/log/monitor-httpd-error.log
 T-2/var/log/slave-introspection-access.log
 T-2/var/log/slave-introspection-error.log
 T-2/var/log/trafficserver/manager.log
-T-2/var/log/trafficserver/traffic.out

software/caddy-frontend/test/test_data/test.TestMasterRequestDomain.test_file_list_plugin-CADDY.txt

@@ -18,6 +18,7 @@ T-1/etc/plugin/kedifa-http-reply.py
 T-1/etc/plugin/monitor-bootstrap-status.py
 T-1/etc/plugin/monitor-http-frontend.py
 T-1/etc/plugin/monitor-httpd-listening-on-tcp.py
+T-1/etc/plugin/promise-logrotate-setup.py
 T-2/etc/plugin/__init__.py
 T-2/etc/plugin/backend-client-caucase-updater.py
 T-2/etc/plugin/backend-haproxy-configuration.py
@@ -35,6 +36,7 @@ T-2/etc/plugin/frontend-caddy-configuration-promise.py
 T-2/etc/plugin/monitor-bootstrap-status.py
 T-2/etc/plugin/monitor-http-frontend.py
 T-2/etc/plugin/monitor-httpd-listening-on-tcp.py
+T-2/etc/plugin/promise-logrotate-setup.py
 T-2/etc/plugin/re6st-connectivity.py
 T-2/etc/plugin/slave-introspection-configuration.py
 T-2/etc/plugin/slave_introspection_https.py

software/caddy-frontend/test/test_data/test.TestRe6stVerificationUrlDefaultSlave.test_file_list_log-CADDY.txt

@@ -16,4 +16,3 @@ T-2/var/log/monitor-httpd-error.log
 T-2/var/log/slave-introspection-access.log
 T-2/var/log/slave-introspection-error.log
 T-2/var/log/trafficserver/manager.log
-T-2/var/log/trafficserver/traffic.out

software/caddy-frontend/test/test_data/test.TestRe6stVerificationUrlDefaultSlave.test_file_list_plugin-CADDY.txt

@@ -18,6 +18,7 @@ T-1/etc/plugin/kedifa-http-reply.py
 T-1/etc/plugin/monitor-bootstrap-status.py
 T-1/etc/plugin/monitor-http-frontend.py
 T-1/etc/plugin/monitor-httpd-listening-on-tcp.py
+T-1/etc/plugin/promise-logrotate-setup.py
 T-2/etc/plugin/__init__.py
 T-2/etc/plugin/backend-client-caucase-updater.py
 T-2/etc/plugin/backend-haproxy-configuration.py
@@ -35,6 +36,7 @@ T-2/etc/plugin/frontend-caddy-configuration-promise.py
 T-2/etc/plugin/monitor-bootstrap-status.py
 T-2/etc/plugin/monitor-http-frontend.py
 T-2/etc/plugin/monitor-httpd-listening-on-tcp.py
+T-2/etc/plugin/promise-logrotate-setup.py
 T-2/etc/plugin/re6st-connectivity.py
 T-2/etc/plugin/slave-introspection-configuration.py
 T-2/etc/plugin/slave_introspection_https.py

software/caddy-frontend/test/test_data/test.TestSlave.test_file_list_log-CADDY.txt

@@ -180,4 +180,3 @@ T-2/var/log/monitor-httpd-error.log
 T-2/var/log/slave-introspection-access.log
 T-2/var/log/slave-introspection-error.log
 T-2/var/log/trafficserver/manager.log
-T-2/var/log/trafficserver/traffic.out

software/caddy-frontend/test/test_data/test.TestSlave.test_file_list_plugin-CADDY.txt

@@ -18,6 +18,7 @@ T-1/etc/plugin/kedifa-http-reply.py
 T-1/etc/plugin/monitor-bootstrap-status.py
 T-1/etc/plugin/monitor-http-frontend.py
 T-1/etc/plugin/monitor-httpd-listening-on-tcp.py
+T-1/etc/plugin/promise-logrotate-setup.py
 T-2/etc/plugin/__init__.py
 T-2/etc/plugin/backend-client-caucase-updater.py
 T-2/etc/plugin/backend-haproxy-configuration.py
@@ -37,6 +38,7 @@ T-2/etc/plugin/frontend-caddy-configuration-promise.py
 T-2/etc/plugin/monitor-bootstrap-status.py
 T-2/etc/plugin/monitor-http-frontend.py
 T-2/etc/plugin/monitor-httpd-listening-on-tcp.py
+T-2/etc/plugin/promise-logrotate-setup.py
 T-2/etc/plugin/re6st-connectivity.py
 T-2/etc/plugin/slave-introspection-configuration.py
 T-2/etc/plugin/slave_introspection_https.py

software/caddy-frontend/test/test_data/test.TestSlaveCiphers.test_file_list_log-CADDY.txt

@@ -20,4 +20,3 @@ T-2/var/log/monitor-httpd-error.log
 T-2/var/log/slave-introspection-access.log
 T-2/var/log/slave-introspection-error.log
 T-2/var/log/trafficserver/manager.log
-T-2/var/log/trafficserver/traffic.out

software/caddy-frontend/test/test_data/test.TestSlaveCiphers.test_file_list_plugin-CADDY.txt

@@ -18,6 +18,7 @@ T-1/etc/plugin/kedifa-http-reply.py
 T-1/etc/plugin/monitor-bootstrap-status.py
 T-1/etc/plugin/monitor-http-frontend.py
 T-1/etc/plugin/monitor-httpd-listening-on-tcp.py
+T-1/etc/plugin/promise-logrotate-setup.py
 T-2/etc/plugin/__init__.py
 T-2/etc/plugin/backend-client-caucase-updater.py
 T-2/etc/plugin/backend-haproxy-configuration.py
@@ -35,6 +36,7 @@ T-2/etc/plugin/frontend-caddy-configuration-promise.py
 T-2/etc/plugin/monitor-bootstrap-status.py
 T-2/etc/plugin/monitor-http-frontend.py
 T-2/etc/plugin/monitor-httpd-listening-on-tcp.py
+T-2/etc/plugin/promise-logrotate-setup.py
 T-2/etc/plugin/re6st-connectivity.py
 T-2/etc/plugin/slave-introspection-configuration.py
 T-2/etc/plugin/slave_introspection_https.py

software/caddy-frontend/test/test_data/test.TestSlaveGlobalDisableHttp2.test_file_list_log-CADDY.txt

@@ -180,4 +180,3 @@ T-2/var/log/monitor-httpd-error.log
 T-2/var/log/slave-introspection-access.log
 T-2/var/log/slave-introspection-error.log
 T-2/var/log/trafficserver/manager.log
-T-2/var/log/trafficserver/traffic.out

software/caddy-frontend/test/test_data/test.TestSlaveGlobalDisableHttp2.test_file_list_plugin-CADDY.txt

@@ -18,6 +18,7 @@ T-1/etc/plugin/kedifa-http-reply.py
 T-1/etc/plugin/monitor-bootstrap-status.py
 T-1/etc/plugin/monitor-http-frontend.py
 T-1/etc/plugin/monitor-httpd-listening-on-tcp.py
+T-1/etc/plugin/promise-logrotate-setup.py
 T-2/etc/plugin/__init__.py
 T-2/etc/plugin/backend-client-caucase-updater.py
 T-2/etc/plugin/backend-haproxy-configuration.py
@@ -37,6 +38,7 @@ T-2/etc/plugin/frontend-caddy-configuration-promise.py
 T-2/etc/plugin/monitor-bootstrap-status.py
 T-2/etc/plugin/monitor-http-frontend.py
 T-2/etc/plugin/monitor-httpd-listening-on-tcp.py
+T-2/etc/plugin/promise-logrotate-setup.py
 T-2/etc/plugin/re6st-connectivity.py
 T-2/etc/plugin/slave-introspection-configuration.py
 T-2/etc/plugin/slave_introspection_https.py

software/caddy-frontend/test/test_data/test.TestSlaveHostHaproxyClash.test_file_list_log-CADDY.txt

@@ -20,4 +20,3 @@ T-2/var/log/monitor-httpd-error.log
 T-2/var/log/slave-introspection-access.log
 T-2/var/log/slave-introspection-error.log
 T-2/var/log/trafficserver/manager.log
-T-2/var/log/trafficserver/traffic.out

software/caddy-frontend/test/test_data/test.TestSlaveHostHaproxyClash.test_file_list_plugin-CADDY.txt

@@ -18,6 +18,7 @@ T-1/etc/plugin/kedifa-http-reply.py
 T-1/etc/plugin/monitor-bootstrap-status.py
 T-1/etc/plugin/monitor-http-frontend.py
 T-1/etc/plugin/monitor-httpd-listening-on-tcp.py
+T-1/etc/plugin/promise-logrotate-setup.py
 T-2/etc/plugin/__init__.py
 T-2/etc/plugin/backend-client-caucase-updater.py
 T-2/etc/plugin/backend-haproxy-configuration.py
@@ -35,6 +36,7 @@ T-2/etc/plugin/frontend-caddy-configuration-promise.py
 T-2/etc/plugin/monitor-bootstrap-status.py
 T-2/etc/plugin/monitor-http-frontend.py
 T-2/etc/plugin/monitor-httpd-listening-on-tcp.py
+T-2/etc/plugin/promise-logrotate-setup.py
 T-2/etc/plugin/re6st-connectivity.py
 T-2/etc/plugin/slave-introspection-configuration.py
 T-2/etc/plugin/slave_introspection_https.py

software/caddy-frontend/test/test_data/test.TestSlaveSlapOSMasterCertificateCompatibility.test_file_list_log-CADDY.txt

@@ -50,4 +50,3 @@ T-2/var/log/monitor-httpd-error.log
 T-2/var/log/slave-introspection-access.log
 T-2/var/log/slave-introspection-error.log
 T-2/var/log/trafficserver/manager.log
-T-2/var/log/trafficserver/traffic.out

software/caddy-frontend/test/test_data/test.TestSlaveSlapOSMasterCertificateCompatibility.test_file_list_plugin-CADDY.txt

@@ -18,6 +18,7 @@ T-1/etc/plugin/kedifa-http-reply.py
 T-1/etc/plugin/monitor-bootstrap-status.py
 T-1/etc/plugin/monitor-http-frontend.py
 T-1/etc/plugin/monitor-httpd-listening-on-tcp.py
+T-1/etc/plugin/promise-logrotate-setup.py
 T-2/etc/plugin/__init__.py
 T-2/etc/plugin/backend-client-caucase-updater.py
 T-2/etc/plugin/backend-haproxy-configuration.py
@@ -35,6 +36,7 @@ T-2/etc/plugin/frontend-caddy-configuration-promise.py
 T-2/etc/plugin/monitor-bootstrap-status.py
 T-2/etc/plugin/monitor-http-frontend.py
 T-2/etc/plugin/monitor-httpd-listening-on-tcp.py
+T-2/etc/plugin/promise-logrotate-setup.py
 T-2/etc/plugin/re6st-connectivity.py
 T-2/etc/plugin/slave-introspection-configuration.py
 T-2/etc/plugin/slave_introspection_https.py

software/caddy-frontend/test/test_data/test.TestSlaveSlapOSMasterCertificateCompatibilityOverrideMaster.test_file_list_log-CADDY.txt

@@ -17,4 +17,3 @@ T-2/var/log/monitor-httpd-error.log
 T-2/var/log/slave-introspection-access.log
 T-2/var/log/slave-introspection-error.log
 T-2/var/log/trafficserver/manager.log
-T-2/var/log/trafficserver/traffic.out

software/caddy-frontend/test/test_data/test.TestSlaveSlapOSMasterCertificateCompatibilityOverrideMaster.test_file_list_plugin-CADDY.txt

@@ -18,6 +18,7 @@ T-1/etc/plugin/kedifa-http-reply.py
 T-1/etc/plugin/monitor-bootstrap-status.py
 T-1/etc/plugin/monitor-http-frontend.py
 T-1/etc/plugin/monitor-httpd-listening-on-tcp.py
+T-1/etc/plugin/promise-logrotate-setup.py
 T-2/etc/plugin/__init__.py
 T-2/etc/plugin/backend-client-caucase-updater.py
 T-2/etc/plugin/backend-haproxy-configuration.py
@@ -35,6 +36,7 @@ T-2/etc/plugin/frontend-caddy-configuration-promise.py
 T-2/etc/plugin/monitor-bootstrap-status.py
 T-2/etc/plugin/monitor-http-frontend.py
 T-2/etc/plugin/monitor-httpd-listening-on-tcp.py
+T-2/etc/plugin/promise-logrotate-setup.py
 T-2/etc/plugin/re6st-connectivity.py
 T-2/etc/plugin/slave-introspection-configuration.py
 T-2/etc/plugin/slave_introspection_https.py

software/caddy-frontend/test/test_data/test.TestSlaveSlapOSMasterCertificateCompatibilityUpdate.test_file_list_log-CADDY.txt

@@ -17,4 +17,3 @@ T-2/var/log/monitor-httpd-error.log
 T-2/var/log/slave-introspection-access.log
 T-2/var/log/slave-introspection-error.log
 T-2/var/log/trafficserver/manager.log
-T-2/var/log/trafficserver/traffic.out

software/caddy-frontend/test/test_data/test.TestSlaveSlapOSMasterCertificateCompatibilityUpdate.test_file_list_plugin-CADDY.txt

@@ -18,6 +18,7 @@ T-1/etc/plugin/kedifa-http-reply.py
 T-1/etc/plugin/monitor-bootstrap-status.py
 T-1/etc/plugin/monitor-http-frontend.py
 T-1/etc/plugin/monitor-httpd-listening-on-tcp.py
+T-1/etc/plugin/promise-logrotate-setup.py
 T-2/etc/plugin/__init__.py
 T-2/etc/plugin/backend-client-caucase-updater.py
 T-2/etc/plugin/backend-haproxy-configuration.py
@@ -35,6 +36,7 @@ T-2/etc/plugin/frontend-caddy-configuration-promise.py
 T-2/etc/plugin/monitor-bootstrap-status.py
 T-2/etc/plugin/monitor-http-frontend.py
 T-2/etc/plugin/monitor-httpd-listening-on-tcp.py
+T-2/etc/plugin/promise-logrotate-setup.py
 T-2/etc/plugin/re6st-connectivity.py
 T-2/etc/plugin/slave-introspection-configuration.py
 T-2/etc/plugin/slave_introspection_https.py

software/slapos-master/test/test/test_erp5.py

@@ -31,13 +31,17 @@ import glob
 import urlparse
 import socket
 import time
+import re
+import BaseHTTPServer
+import multiprocessing
+import subprocess
 
 import psutil
 import requests
 
 from . import ERP5InstanceTestCase
 from . import setUpModule
-setUpModule # pyflakes
+setUpModule  # pyflakes
 
 
 class TestPublishedURLIsReachableMixin(object):
@@ -46,15 +50,17 @@ class TestPublishedURLIsReachableMixin(object):
   def _checkERP5IsReachable(self, url):
     # What happens is that instanciation just create the services, but does not
     # wait for ERP5 to be initialized. When this test run ERP5 instance is
-    # instanciated, but zope is still busy creating the site and haproxy replies
-    # with 503 Service Unavailable when zope is not started yet, with 404 when
-    # erp5 site is not created, with 500 when mysql is not yet reachable, so we
-    # retry in a loop until we get a succesful response.
+    # instanciated, but zope is still busy creating the site and haproxy
+    # replies with 503 Service Unavailable when zope is not started yet, with
+    # 404 when erp5 site is not created, with 500 when mysql is not yet
+    # reachable, so we retry in a loop until we get a succesful response.
     for i in range(1, 60):
-      r = requests.get(url, verify=False)  # XXX can we get CA from caucase already ?
+      # XXX can we get CA from caucase already ?
+      r = requests.get(url, verify=False)
       if r.status_code != requests.codes.ok:
         delay = i * 2
-        self.logger.warn("ERP5 was not available, sleeping for %ds and retrying", delay)
+        self.logger.warn(
+          "ERP5 was not available, sleeping for %ds and retrying", delay)
         time.sleep(delay)
         continue
       r.raise_for_status()
@@ -77,7 +83,8 @@ class TestPublishedURLIsReachableMixin(object):
       urlparse.urljoin(param_dict['family-default'], param_dict['site-id']))
 
 
-class TestDefaultParameters(ERP5InstanceTestCase, TestPublishedURLIsReachableMixin):
+class TestDefaultParameters(
+  ERP5InstanceTestCase, TestPublishedURLIsReachableMixin):
   """Test ERP5 can be instanciated with no parameters
   """
   __partition_reference__ = 'defp'
@@ -125,16 +132,19 @@ class TestApacheBalancerPorts(ERP5InstanceTestCase):
     self.assertTrue(parsed.port)
 
   def test_published_family_parameters(self):
-    # when we request two families, we have two published family-{family_name} URLs
+    # when we request two families, we have two published family-{family_name}
+    # URLs
     param_dict = self.getRootPartitionConnectionParameterDict()
     for family_name in ('family1', 'family2'):
       self.checkValidHTTPSURL(
           param_dict['family-{family_name}'.format(family_name=family_name)])
       self.checkValidHTTPSURL(
-          param_dict['family-{family_name}-v6'.format(family_name=family_name)])
+          param_dict['family-{family_name}-v6'.format(
+            family_name=family_name)])
 
   def test_published_test_runner_url(self):
-    # each family's also a list of test test runner URLs, by default 3 per family
+    # each family's also a list of test test runner URLs, by default 3 per
+    # family
     param_dict = self.getRootPartitionConnectionParameterDict()
     for family_name in ('family1', 'family2'):
       family_test_runner_url_list = param_dict[
@@ -144,7 +154,8 @@ class TestApacheBalancerPorts(ERP5InstanceTestCase):
         self.checkValidHTTPSURL(url)
 
   def test_zope_listen(self):
-    # we requested 3 zope in family1 and 5 zopes in family2, we should have 8 zope running.
+    # we requested 3 zope in family1 and 5 zopes in family2, we should have 8
+    # zope running.
     with self.slap.instance_supervisor_rpc as supervisor:
       all_process_info = supervisor.getAllProcessInfo()
     self.assertEqual(
@@ -179,7 +190,8 @@ class TestApacheBalancerPorts(ERP5InstanceTestCase):
     ])
 
 
-class TestDisableTestRunner(ERP5InstanceTestCase, TestPublishedURLIsReachableMixin):
+class TestDisableTestRunner(
+  ERP5InstanceTestCase, TestPublishedURLIsReachableMixin):
   """Test ERP5 can be instanciated without test runner.
   """
   __partition_reference__ = 'distr'
@@ -192,15 +204,17 @@ class TestDisableTestRunner(ERP5InstanceTestCase, TestPublishedURLIsReachableMix
     """
     # self.computer_partition_root_path is the path of root partition.
     # we want to assert that no scripts exist in any partition.
-    bin_programs = map(os.path.basename,
+    bin_programs = map(
+      os.path.basename,
       glob.glob(self.computer_partition_root_path + "/../*/bin/*"))
 
-    self.assertTrue(bin_programs) # just to check the glob was correct.
+    self.assertTrue(bin_programs)  # just to check the glob was correct.
     self.assertNotIn('runUnitTest', bin_programs)
     self.assertNotIn('runTestSuite', bin_programs)
 
   def test_no_apache_testrunner_port(self):
-    # Apache only listen on two ports, there is no apache ports allocated for test runner
+    # Apache only listen on two ports, there is no apache ports allocated for
+    # test runner
     with self.slap.instance_supervisor_rpc as supervisor:
       all_process_info = supervisor.getAllProcessInfo()
     process_info, = [p for p in all_process_info if p['name'] == 'apache']
@@ -213,7 +227,9 @@ class TestDisableTestRunner(ERP5InstanceTestCase, TestPublishedURLIsReachableMix
             if c.status == 'LISTEN'
         ))
 
-class TestZopeNodeParameterOverride(ERP5InstanceTestCase, TestPublishedURLIsReachableMixin):
+
+class TestZopeNodeParameterOverride(
+  ERP5InstanceTestCase, TestPublishedURLIsReachableMixin):
   """Test override zope node parameters
   """
   __partition_reference__ = 'override'
@@ -228,7 +244,7 @@ class TestZopeNodeParameterOverride(ERP5InstanceTestCase, TestPublishedURLIsReac
         "server": {},
         "cache-size-bytes": "20MB",
         "cache-size-bytes!": [
-          ("bb-0", 1<<20),
+          ("bb-0", 1 << 20),
           ("bb-.*", "500MB"),
         ],
         "pool-timeout": "10m",
@@ -299,7 +315,170 @@ class TestZopeNodeParameterOverride(ERP5InstanceTestCase, TestPublishedURLIsReac
     partition = self.getComputerPartitionPath('zope-bb')
     for zope in xrange(5):
       checkConf({
-          "cache-size-bytes": "500MB" if zope else 1<<20,
+          "cache-size-bytes": "500MB" if zope else 1 << 20,
         }, {
           "cache-size": None,
         })
+
+
+def popenCommunicate(command_list, input_=None, **kwargs):
+  kwargs.update(stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
+  popen = subprocess.Popen(command_list, **kwargs)
+  result = popen.communicate(input_)[0]
+  if popen.returncode is None:
+    popen.kill()
+  if popen.returncode != 0:
+    raise ValueError(
+      'Issue during calling %r, result was:\n%s' % (command_list, result))
+  return result
+
+
+class TestHandler(BaseHTTPServer.BaseHTTPRequestHandler):
+  def do_GET(self):
+    self.send_response(200)
+
+    response = {
+      'Path': self.path,
+      'Incoming Headers': self.headers.dict
+    }
+    response = json.dumps(response, indent=2)
+    self.end_headers()
+    self.wfile.write(response)
+
+
+class TestDeploymentScriptInstantiation(ERP5InstanceTestCase):
+  """This check deployment script like instantiation
+
+  Low level assertions are done here in roder to assure that
+  https://lab.nexedi.com/nexedi/slapos.package/blob/master/playbook/
+  slapos-master-standalone.yml
+  works correctly
+  """
+  __partition_reference__ = 'tdsi'
+  # a bit more partition is required
+  partition_count = 20
+
+  @classmethod
+  def getInstanceParameterDict(cls):
+    # As close as possible configuration to deployment script
+    parameter_dict = {
+      "timezone": "UTC",
+      "site-id": "erp5",
+      "bt5": "erp5_full_text_myisam_catalog slapos_configurator",
+      "wsgi": False,
+      "test-runner": {"enabled": False},  # won't work anyway here
+      "zope-partition-dict": {
+        "admin": {
+          "family": "admin",
+          "thread-amount": 4,
+          "port-base": 2220,
+          "instance-count": 1
+        },
+        "activities-node": {
+          "family": "activities",
+          "thread-amount": 4,
+          "instance-count": 1,
+          "timerserver-interval": 1,
+          "port-base": 2230
+        },
+        "distribution-node": {
+          "family": "distribution",
+          "thread-amount": 1,
+          "instance-count": 1,
+          "port-base": 2210,
+          "timerserver-interval": 1
+        },
+        "web-node": {
+          "family": "web",
+          "thread-amount": 2,
+          "instance-count": 1,
+          "port-base": 2240
+        },
+        "service-slapos": {
+          "family": "service",
+          "thread-amount": 2,
+          "instance-count": 1,
+          "port-base": 2250,
+          "ssl-authentication": True,
+          "backend-path": "/%(site-id)s/portal_slap"
+        }
+      }
+    }
+
+    # put shared-certificate-authority-path in controlled location
+    cls.ca_path = os.path.join(cls.slap.instance_directory, 'ca_path')
+    parameter_dict["shared-certificate-authority-path"] = cls.ca_path
+    return {'_': json.dumps(parameter_dict)}
+
+  @classmethod
+  def callSupervisorMethod(cls, method, *args, **kwargs):
+    with cls.slap.instance_supervisor_rpc as instance_supervisor:
+      return getattr(instance_supervisor, method)(*args, **kwargs)
+
+  def test_ssl_auth(self):
+    backend_apache_configuration_list = glob.glob(
+      os.path.join(
+        self.slap.instance_directory, '*', 'etc', 'apache', 'apache.conf'))
+    self.assertEqual(
+      1,
+      len(backend_apache_configuration_list)
+    )
+    backend_apache_configuration = open(
+      backend_apache_configuration_list[0]).read()
+    self.assertIn(
+      'SSLVerifyClient require',
+      backend_apache_configuration
+    )
+    self.assertIn(
+      r'RequestHeader set Remote-User %{SSL_CLIENT_S_DN_CN}s',
+      backend_apache_configuration
+    )
+
+    # stop haproxy, it's going to be hijacked
+    haproxy_name = ':'.join([
+      (q['group'], q['name'])
+      for q in self.callSupervisorMethod('getAllProcessInfo')
+      if 'haproxy' in q['name']][0])
+    self.callSupervisorMethod('stopProcess', haproxy_name)
+
+    # do similar certificate request like CertificateAuthorityTool
+    openssl_config = os.path.join(self.ca_path, 'openssl.cnf')
+    key = os.path.join(self.ca_path, 'private', 'test.key')
+    csr = os.path.join(self.ca_path, 'text.csr')
+    cert = os.path.join(self.ca_path, 'certs', 'test.crt')
+    common_name = 'TEST-SSL-AUTH'
+    popenCommunicate([
+      'openssl', 'req', '-utf8', '-nodes', '-config', openssl_config, '-new',
+      '-keyout', key, '-out', csr, '-days', '3650'], '%s\n' % (common_name,),
+      stdin=subprocess.PIPE)
+    popenCommunicate([
+      'openssl', 'ca', '-utf8', '-days', '3650', '-batch', '-config',
+      openssl_config, '-out', cert, '-infiles', csr])
+    # find IP and port on which hijacked process shall listen
+    portal_slap_line = [
+      q for q in backend_apache_configuration.splitlines()
+      if 'portal_slap' in q][0]
+    ip, port = re.search(
+      r'.*http:\/\/(.*):(\d*)\/.*', portal_slap_line).groups()
+    port = int(port)
+    server = BaseHTTPServer.HTTPServer((ip, port), TestHandler)
+    server_process = multiprocessing.Process(
+      target=server.serve_forever, name='HTTPServer')
+    server_process.start()
+    try:
+      # assert that accessing the service endpoint results with certificate
+      # authentication and proper information extraction
+      result_json = requests.get(
+        self.getRootPartitionConnectionParameterDict()['family-service'],
+        verify=False, cert=(cert, key)).json()
+      self.assertEqual(
+        common_name,
+        result_json['Incoming Headers']['remote-user']
+      )
+      self.assertEqual(
+        '/erp5/portal_slap/',
+        result_json['Path']
+      )
+    finally:
+      server_process.join(10)
+      server_process.terminate()

software/slapos-sr-testing/buildout.hash.cfg

@@ -15,4 +15,4 @@
 
 [template]
 filename = instance.cfg
-md5sum = 14d2f49d20670e44c2a162bcec9e0a8e
+md5sum = 4246cde0a27138e057ba1635cc621edf

software/slapos-sr-testing/instance.cfg

@@ -44,7 +44,7 @@ environment =
   SLAPOS_TEST_WORKING_DIR=${slapos-test-runner-environment:SLAPOS_TEST_WORKING_DIR}
 
 [slapos-test-runner-environment]
-PATH = {{ buildout['bin-directory'] }}:{{ curl_location }}/bin/:{{ faketime_location }}/bin/:/usr/bin/:/bin
+PATH = {{ buildout['bin-directory'] }}:{{ curl_location }}/bin/:{{ faketime_location }}/bin/:{{ openssl_location }}/bin/:/usr/bin/:/bin
 SLAPOS_TEST_IPV4 = ${slap-configuration:ipv4-random}
 SLAPOS_TEST_IPV6 = ${slap-configuration:ipv6-random}
 SLAPOS_TEST_WORKING_DIR = ${directory:working-dir}

software/slapos-sr-testing/software.cfg

@@ -3,6 +3,7 @@
 extends =
   ../../component/bcrypt/buildout.cfg
   ../../component/curl/buildout.cfg
+  ../../component/openssl/buildout.cfg
   ../../component/git/buildout.cfg
   ../../component/faketime/buildout.cfg
   ../../component/pillow/buildout.cfg
@@ -240,6 +241,7 @@ context =
   key slapos_location slapos-repository:location
   key interpreter eggs:interpreter
   key curl_location curl:location
+  key openssl_location openssl-output:bin
   key faketime_location faketime:location
   key tests :tests
 tests =

stack/erp5/buildout.cfg

@@ -408,6 +408,7 @@ eggs = ${neoppod:eggs}
   jsonschema
   mechanize
   mock
+  oauthlib
   objgraph
   paramiko
   ply
@@ -645,6 +646,7 @@ logilab-common = 1.3.0
 munnel = 0.3
 nt-svcutils = 2.13.0
 oauth2client = 4.0.0
+oauthlib = 3.1.0
 objgraph = 3.1.0
 ply = 3.10
 polib = 1.0.8

stack/logrotate/buildout.hash.cfg

@@ -18,8 +18,8 @@ md5sum = 7c41026716d856bba7c1252b72adbf77
 
 [logrotate-entry-template]
 filename = logrotate_entry.in
-md5sum = 117e46af6d9d31c09eeb86089d11407e
+md5sum = ce6ccdd52148770149e6e7525ab71e80
 
 [template-logrotate-base]
 filename = instance-logrotate-base.cfg.in
-md5sum = c2a06234d923c8e589cc0ddb1d6e0880
+md5sum = 8a774b677623c77c6ff0b88852fac643

stack/logrotate/instance-logrotate-base.cfg.in

@@ -58,12 +58,14 @@ context =
   key frequency :frequency
   key rotate_num :rotate-num
   key nocompress :nocompress
+  key delaycompress :delaycompress
 post =
 pre =
 frequency = daily
 rotate-num = 3650
 backup = ${logrotate-directory:logrotate-backup}
 nocompress =
+delaycompress = true
 
 [logrotate]
 recipe = slapos.cookbook:wrapper

stack/logrotate/logrotate_entry.in

@@ -4,7 +4,7 @@
   rotate {{ rotate_num }}
   {% if nocompress %}nocompress
   {% else %}compress
-  delaycompress{% endif %}
+  {% if delaycompress %}delaycompress{% endif %}{% endif %}
   notifempty
   sharedscripts
   create