ERP5: weekly apachedex

See merge request !1716

software/erp5/instance-erp5-input-schema.json

@@ -798,7 +798,7 @@
         },
         "apachedex-promise-threshold": {
           "type": "number",
-          "title": "Promise fails if the overall Apdex score for the previous day is below than this value.",
+          "title": "Promise fails if the overall Apdex score for the previous day is below than this value. A value of 0 disables the promise.",
           "default": 70,
           "minimum": 0,
           "maximum": 100

software/erp5/test/test/test_balancer.py

 import ipaddress
 import json
 import logging
+import lzma
 import os
 import re
 import socket
@@ -104,7 +105,7 @@ class BalancerTestCase(ERP5InstanceTestCase):
             '--js-embed',
             '--quiet',
           ],
-        'apachedex-promise-threshold': 100,
+        'apachedex-promise-threshold': 0,
         'haproxy-server-check-path': '/',
         'zope-family-dict': {
             'default': ['dummy_http_server'],
@@ -264,7 +265,7 @@ class TestLog(BalancerTestCase, CrontabMixin):
     # make a request so that we have something in the logs
     requests.get(self.default_balancer_zope_url, verify=False)
 
-    # crontab for apachedex is executed
+    # crontab for daily apachedex is executed
     self._executeCrontabAtDate('generate-apachedex-report', '23:59')
     # it creates a report for the day
     apachedex_report, = (
@@ -278,6 +279,39 @@ class TestLog(BalancerTestCase, CrontabMixin):
     # having this table means that apachedex could parse some lines.
     self.assertIn('<h2>Hits per status code</h2>', report_text)
 
+    # weekly apachedex uses the logs after rotation, we'll run log rotation
+    # until we have a xz file for two days ago and a non compressed file for
+    # yesterday
+    # run logrotate a first time so that it create state files
+    self._executeCrontabAtDate('logrotate', '2000-01-01')
+    requests.get(urllib.parse.urljoin(self.default_balancer_zope_url, 'error-two-days-ago'), verify=False)
+    self._executeCrontabAtDate('logrotate', 'yesterday 00:00')
+    requests.get(urllib.parse.urljoin(self.default_balancer_zope_url, 'error-yesterday'), verify=False)
+    self._executeCrontabAtDate('logrotate', '00:00')
+
+    # this apachedex command uses compressed files, verify that our test setup
+    # is correct and that the error from two days ago is in the compressed file.
+    two_days_ago_log, = (
+      self.computer_partition_root_path / 'srv' / 'backup'/ 'logrotate'
+    ).glob("apache-access.log-*.xz")
+    with lzma.open(two_days_ago_log) as f:
+      self.assertIn(b'GET /error-two-days-ago', f.read())
+
+    self._executeCrontabAtDate('generate-weekly-apachedex-report', '23:59')
+    # this creates a report for the week
+    apachedex_weekly_report, = (
+      self.computer_partition_root_path
+        / 'srv'
+        / 'monitor'
+        / 'private'
+        / 'apachedex'
+        / 'weekly').glob('*.html')
+    weekly_report_text = apachedex_weekly_report.read_text()
+    self.assertIn('APacheDEX', weekly_report_text)
+    # because we run apachedex with error details, we can see our error requests
+    self.assertIn('error-two-days-ago', weekly_report_text)
+    self.assertIn('error-yesterday', weekly_report_text)
+
   def test_access_log_rotation(self) -> None:
     # run logrotate a first time so that it create state files
     self._executeCrontabAtDate('logrotate', '2000-01-01')

stack/erp5/buildout.cfg

@@ -529,6 +529,7 @@ eggs =
   lock_file
   astor
   APacheDEX
+  ${backports.lzma:egg}
   chardet
   collective.recipe.template
   erp5diff

stack/erp5/buildout.hash.cfg

@@ -94,7 +94,7 @@ md5sum = 9c580be982d8c63ec06fc273ef3cb971
 
 [template-balancer]
 filename = instance-balancer.cfg.in
-md5sum = 409a7505548576ebf0e4d5cc218e0753
+md5sum = e9aa89754085bdc7a6fb9e53c0c97f9d
 
 [template-haproxy-cfg]
 filename = haproxy.cfg.in

stack/erp5/instance-balancer.cfg.in

@@ -400,7 +400,8 @@ command-line = "{{ parameter_dict['socat'] }}/bin/socat" unix-connect:${haproxy-
 
 [rsyslogd-cfg-parameter-dict]
 log-socket = ${directory:run}/log.sock
-access-log-file = ${directory:log}/apache-access.log
+access-log-file-basename = apache-access.log
+access-log-file = ${directory:log}/${:access-log-file-basename}
 error-log-file = ${directory:log}/apache-error.log
 pid-file = ${directory:run}/rsyslogd.pid
 spool-directory = ${directory:rsyslogd-spool}
@@ -483,6 +484,7 @@ backup-caucased-haproxy-certificate = ${:srv}/backup/caucased{{ caucase_haproxy_
 caucase-updater-haproxy-certificate = ${:srv}/caucase-updater-haproxy-certificate
 tmp = ${buildout:directory}/tmp
 apachedex = ${monitor-directory:private}/apachedex
+apachedex-weekly = ${:apachedex}/weekly
 rsyslogd-spool = ${:run}/rsyslogd-spool
 {% if frontend_caucase_url_list -%}
 ca-cert = ${:etc}/ssl.crt
@@ -511,12 +513,38 @@ wrapper-path = ${directory:bin}/${:command}
 command-line = "{{ parameter_dict['run-apachedex-location'] }}" "{{ parameter_dict['apachedex-location'] }}" "${directory:apachedex}" ${monitor-publish-parameters:monitor-base-url}/private/apachedex --apache-log-list "${apachedex-parameters:apache-log-list}" --configuration ${apachedex-parameters:configuration}
 command = generate-apachedex-report
 
+[{{ section('monitor-generate-weekly-apachedex-report') }}]
+recipe = slapos.cookbook:cron.d
+cron-entries = ${cron:cron-entries}
+name = generate-weekly-apachedex-report
+frequency = 1 * * * 1
+command = ${monitor-generate-weekly-apachedex-report-wrapper:output}
+
+[monitor-generate-weekly-apachedex-report-wrapper]
+recipe = slapos.recipe.template
+output = ${buildout:bin-directory}/${:_buildout_section_name_}
+inline =
+  #!/bin/sh
+  # Yesterday's log file is not compressed
+  LOG_FILES=${logrotate-rsyslogd:backup}/${rsyslogd-cfg-parameter-dict:access-log-file-basename}-$(date +"%Y%m%d")
+  # Days before are compressed
+  {% for i in range(1, 6) -%}
+  LOG_FILE=${logrotate-rsyslogd:backup}/${rsyslogd-cfg-parameter-dict:access-log-file-basename}-$(date -d "{{i}} days ago" +"%Y%m%d").xz
+  if [ -f "$${LOG_FILE}" ]; then
+    LOG_FILES="$${LOG_FILES} $${LOG_FILE}"
+  fi
+  {% endfor %}
+  exec {{ parameter_dict['apachedex-location'] }} \
+    -o ${directory:apachedex-weekly}/$(date -d "1 days ago" +"%Y-%m-%d").html \
+    @${apachedex-parameters:configuration} \
+    -- $${LOG_FILES}
+
 [monitor-apachedex-report-config]
 recipe = slapos.recipe.template
 output = ${directory:etc}/${:_buildout_section_name_}
 inline =
   {% for line in slapparameter_dict['apachedex-configuration'] -%}
-    {# apachedex config files use shlex.split, so we need to quote the arguments. #}
+    {# apachedex config files use shlex.split, so we need to quote the arguments. -#}
     {{ six.moves.shlex_quote(line) }}
   {% endfor %}
 
@@ -525,11 +553,13 @@ apache-log-list = ${rsyslogd-cfg-parameter-dict:access-log-file}
 configuration = ${monitor-apachedex-report-config:output}
 promise-threshold = {{ slapparameter_dict['apachedex-promise-threshold'] }}
 
+{%if slapparameter_dict['apachedex-promise-threshold'] %}
 [{{ section('monitor-promise-apachedex-result') }}]
 <= monitor-promise-base
 promise = check_command_execute
 name = check-apachedex-result.py
 config-command = "{{ parameter_dict['promise-check-apachedex-result'] }}" --apachedex_path "${directory:apachedex}" --status_file ${monitor-directory:private}/apachedex.report.json --threshold "${apachedex-parameters:promise-threshold}"
+{% endif %}
 
 [{{ section('promise-check-computer-memory') }}]
 <= monitor-promise-base