Skip to content
Projects
Groups
Snippets
Help
Loading...
Help
Support
Keyboard shortcuts
?
Submit feedback
Contribute to GitLab
Sign in / Register
Toggle navigation
slapos
Project overview
Project overview
Details
Activity
Releases
Repository
Repository
Files
Commits
Branches
Tags
Contributors
Graph
Compare
Labels
Merge Requests
106
Merge Requests
106
CI / CD
CI / CD
Pipelines
Jobs
Schedules
Analytics
Analytics
CI / CD
Repository
Value Stream
Snippets
Snippets
Members
Members
Collapse sidebar
Close sidebar
Activity
Graph
Jobs
Commits
Open sidebar
nexedi
slapos
Commits
4208a277
Commit
4208a277
authored
1 month ago
by
Jérome Perrin
Browse files
Options
Browse Files
Download
Email Patches
Plain Diff
stack/erp5: generate weekly apachedex reports
parent
064b465f
Changes
3
Hide whitespace changes
Inline
Side-by-side
Showing
3 changed files
with
65 additions
and
3 deletions
+65
-3
software/erp5/test/test/test_balancer.py
software/erp5/test/test/test_balancer.py
+35
-1
stack/erp5/buildout.hash.cfg
stack/erp5/buildout.hash.cfg
+1
-1
stack/erp5/instance-balancer.cfg.in
stack/erp5/instance-balancer.cfg.in
+29
-1
No files found.
software/erp5/test/test/test_balancer.py
View file @
4208a277
import
ipaddress
import
ipaddress
import
json
import
json
import
logging
import
logging
import
lzma
import
os
import
os
import
re
import
re
import
socket
import
socket
...
@@ -264,7 +265,7 @@ class TestLog(BalancerTestCase, CrontabMixin):
...
@@ -264,7 +265,7 @@ class TestLog(BalancerTestCase, CrontabMixin):
# make a request so that we have something in the logs
# make a request so that we have something in the logs
requests.get(self.default_balancer_zope_url, verify=False)
requests.get(self.default_balancer_zope_url, verify=False)
# crontab for apachedex is executed
# crontab for
daily
apachedex is executed
self._executeCrontabAtDate('
generate
-
apachedex
-
report
', '
23
:
59
')
self._executeCrontabAtDate('
generate
-
apachedex
-
report
', '
23
:
59
')
# it creates a report for the day
# it creates a report for the day
apachedex_report, = (
apachedex_report, = (
...
@@ -278,6 +279,39 @@ class TestLog(BalancerTestCase, CrontabMixin):
...
@@ -278,6 +279,39 @@ class TestLog(BalancerTestCase, CrontabMixin):
# having this table means that apachedex could parse some lines.
# having this table means that apachedex could parse some lines.
self.assertIn('
<
h2
>
Hits
per
status
code
</
h2
>
', report_text)
self.assertIn('
<
h2
>
Hits
per
status
code
</
h2
>
', report_text)
# weekly apachedex uses the logs after rotation, we'
ll
run
log
rotation
# until we have a xz file for two days ago and a non compressed file for
# yesterday
# run logrotate a first time so that it create state files
self
.
_executeCrontabAtDate
(
'logrotate'
,
'2000-01-01'
)
requests
.
get
(
urllib
.
parse
.
urljoin
(
self
.
default_balancer_zope_url
,
'error-two-days-ago'
),
verify
=
False
)
self
.
_executeCrontabAtDate
(
'logrotate'
,
'yesterday 00:00'
)
requests
.
get
(
urllib
.
parse
.
urljoin
(
self
.
default_balancer_zope_url
,
'error-yesterday'
),
verify
=
False
)
self
.
_executeCrontabAtDate
(
'logrotate'
,
'00:00'
)
# this apachedex command uses compressed files, verify that our test setup
# is correct and that the error from two days ago is in the compressed file.
two_days_ago_log
,
=
(
self
.
computer_partition_root_path
/
'srv'
/
'backup'
/
'logrotate'
).
glob
(
"apache-access.log-*.xz"
)
with
lzma
.
open
(
two_days_ago_log
)
as
f
:
self
.
assertIn
(
b'GET /error-two-days-ago'
,
f
.
read
())
self
.
_executeCrontabAtDate
(
'generate-weekly-apachedex-report'
,
'23:59'
)
# this creates a report for the week
apachedex_weekly_report
,
=
(
self
.
computer_partition_root_path
/
'srv'
/
'monitor'
/
'private'
/
'apachedex'
/
'weekly'
).
glob
(
'*.html'
)
weekly_report_text
=
apachedex_weekly_report
.
read_text
()
self
.
assertIn
(
'APacheDEX'
,
weekly_report_text
)
# because we run apachedex with error details, we can see our error requests
self
.
assertIn
(
'error-two-days-ago'
,
weekly_report_text
)
self
.
assertIn
(
'error-yesterday'
,
weekly_report_text
)
def
test_access_log_rotation
(
self
)
->
None
:
def
test_access_log_rotation
(
self
)
->
None
:
# run logrotate a first time so that it create state files
# run logrotate a first time so that it create state files
self
.
_executeCrontabAtDate
(
'logrotate'
,
'2000-01-01'
)
self
.
_executeCrontabAtDate
(
'logrotate'
,
'2000-01-01'
)
...
...
This diff is collapsed.
Click to expand it.
stack/erp5/buildout.hash.cfg
View file @
4208a277
...
@@ -94,7 +94,7 @@ md5sum = 9c580be982d8c63ec06fc273ef3cb971
...
@@ -94,7 +94,7 @@ md5sum = 9c580be982d8c63ec06fc273ef3cb971
[template-balancer]
[template-balancer]
filename = instance-balancer.cfg.in
filename = instance-balancer.cfg.in
md5sum =
f7a9b3450f9682ef11195fe2a7c3dad6
md5sum =
5d93da87eda0365058d8a14b19789d19
[template-haproxy-cfg]
[template-haproxy-cfg]
filename = haproxy.cfg.in
filename = haproxy.cfg.in
...
...
This diff is collapsed.
Click to expand it.
stack/erp5/instance-balancer.cfg.in
View file @
4208a277
...
@@ -400,7 +400,8 @@ command-line = "{{ parameter_dict['socat'] }}/bin/socat" unix-connect:${haproxy-
...
@@ -400,7 +400,8 @@ command-line = "{{ parameter_dict['socat'] }}/bin/socat" unix-connect:${haproxy-
[rsyslogd-cfg-parameter-dict]
[rsyslogd-cfg-parameter-dict]
log-socket = ${directory:run}/log.sock
log-socket = ${directory:run}/log.sock
access-log-file = ${directory:log}/apache-access.log
access-log-file-basename = apache-access.log
access-log-file = ${directory:log}/${:access-log-file-basename}
error-log-file = ${directory:log}/apache-error.log
error-log-file = ${directory:log}/apache-error.log
pid-file = ${directory:run}/rsyslogd.pid
pid-file = ${directory:run}/rsyslogd.pid
spool-directory = ${directory:rsyslogd-spool}
spool-directory = ${directory:rsyslogd-spool}
...
@@ -483,6 +484,7 @@ backup-caucased-haproxy-certificate = ${:srv}/backup/caucased{{ caucase_haproxy_
...
@@ -483,6 +484,7 @@ backup-caucased-haproxy-certificate = ${:srv}/backup/caucased{{ caucase_haproxy_
caucase-updater-haproxy-certificate = ${:srv}/caucase-updater-haproxy-certificate
caucase-updater-haproxy-certificate = ${:srv}/caucase-updater-haproxy-certificate
tmp = ${buildout:directory}/tmp
tmp = ${buildout:directory}/tmp
apachedex = ${monitor-directory:private}/apachedex
apachedex = ${monitor-directory:private}/apachedex
apachedex-weekly = ${:apachedex}/weekly
rsyslogd-spool = ${:run}/rsyslogd-spool
rsyslogd-spool = ${:run}/rsyslogd-spool
{% if frontend_caucase_url_list -%}
{% if frontend_caucase_url_list -%}
ca-cert = ${:etc}/ssl.crt
ca-cert = ${:etc}/ssl.crt
...
@@ -511,6 +513,32 @@ wrapper-path = ${directory:bin}/${:command}
...
@@ -511,6 +513,32 @@ wrapper-path = ${directory:bin}/${:command}
command-line = "{{ parameter_dict['run-apachedex-location'] }}" "{{ parameter_dict['apachedex-location'] }}" "${directory:apachedex}" ${monitor-publish-parameters:monitor-base-url}/private/apachedex --apache-log-list "${apachedex-parameters:apache-log-list}" --configuration ${apachedex-parameters:configuration}
command-line = "{{ parameter_dict['run-apachedex-location'] }}" "{{ parameter_dict['apachedex-location'] }}" "${directory:apachedex}" ${monitor-publish-parameters:monitor-base-url}/private/apachedex --apache-log-list "${apachedex-parameters:apache-log-list}" --configuration ${apachedex-parameters:configuration}
command = generate-apachedex-report
command = generate-apachedex-report
[{{ section('monitor-generate-weekly-apachedex-report') }}]
recipe = slapos.cookbook:cron.d
cron-entries = ${cron:cron-entries}
name = generate-weekly-apachedex-report
frequency = 1 * * * 1
command = ${monitor-generate-weekly-apachedex-report-wrapper:output}
[monitor-generate-weekly-apachedex-report-wrapper]
recipe = slapos.recipe.template
output = ${buildout:bin-directory}/${:_buildout_section_name_}
inline =
#!/bin/sh
# Yesterday's log file is not compressed
LOG_FILES=${logrotate-rsyslogd:backup}/${rsyslogd-cfg-parameter-dict:access-log-file-basename}-$(date +"%Y%m%d")
# Days before are compressed
{% for i in range(1, 6) -%}
LOG_FILE=${logrotate-rsyslogd:backup}/${rsyslogd-cfg-parameter-dict:access-log-file-basename}-$(date -d "{{i}} days ago" +"%Y%m%d").xz
if [ -f "$${LOG_FILE}" ]; then
LOG_FILES="$${LOG_FILES} $${LOG_FILE}"
fi
{% endfor %}
exec {{ parameter_dict['apachedex-location'] }} \
-o ${directory:apachedex-weekly}/$(date -d "1 days ago" +"%Y-%m-%d").html \
@${apachedex-parameters:configuration} \
-- $${LOG_FILES}
[monitor-apachedex-report-config]
[monitor-apachedex-report-config]
recipe = slapos.recipe.template
recipe = slapos.recipe.template
output = ${directory:etc}/${:_buildout_section_name_}
output = ${directory:etc}/${:_buildout_section_name_}
...
...
This diff is collapsed.
Click to expand it.
Write
Preview
Markdown
is supported
0%
Try again
or
attach a new file
Attach a file
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment