Skip to content
Projects
Groups
Snippets
Help
Loading...
Help
Support
Keyboard shortcuts
?
Submit feedback
Contribute to GitLab
Sign in / Register
Toggle navigation
slapos
Project overview
Project overview
Details
Activity
Releases
Repository
Repository
Files
Commits
Branches
Tags
Contributors
Graph
Compare
Issues
0
Issues
0
List
Boards
Labels
Milestones
Merge Requests
0
Merge Requests
0
Analytics
Analytics
Repository
Value Stream
Wiki
Wiki
Snippets
Snippets
Members
Members
Collapse sidebar
Close sidebar
Activity
Graph
Create a new issue
Commits
Issue Boards
Open sidebar
Romain Courteaud
slapos
Commits
bbd02230
Commit
bbd02230
authored
Aug 07, 2017
by
Hardik Juneja
Committed by
Rafael Monnerat
Aug 18, 2017
Browse files
Options
Browse Files
Download
Email Patches
Plain Diff
stack/erp5: Clean up apachedex and slowquery promises
parent
11ac4e0e
Changes
5
Hide whitespace changes
Inline
Side-by-side
Showing
5 changed files
with
62 additions
and
80 deletions
+62
-80
stack/erp5/buildout.hash.cfg
stack/erp5/buildout.hash.cfg
+4
-4
stack/erp5/instance-balancer-check-apachedex-result.sh.in
stack/erp5/instance-balancer-check-apachedex-result.sh.in
+20
-26
stack/erp5/instance-mariadb-check-slowquery-result.sh.in
stack/erp5/instance-mariadb-check-slowquery-result.sh.in
+30
-42
stack/erp5/instance-mariadb.cfg.in
stack/erp5/instance-mariadb.cfg.in
+6
-7
stack/erp5/mysql-querydigest.sh.in
stack/erp5/mysql-querydigest.sh.in
+2
-1
No files found.
stack/erp5/buildout.hash.cfg
View file @
bbd02230
...
@@ -19,15 +19,15 @@ md5sum = 844d62cd6f9d6e3d1d78d52de2b72a49
...
@@ -19,15 +19,15 @@ md5sum = 844d62cd6f9d6e3d1d78d52de2b72a49
[mariadb-slow-query-report-script]
[mariadb-slow-query-report-script]
filename = mysql-querydigest.sh.in
filename = mysql-querydigest.sh.in
md5sum =
48fb6a0ed34cfc6de6b40acd7088c51c
md5sum =
cfe6ab8ae54a521ecb269e9d9762cbeb
[mariadb-slowquery-check-script]
[mariadb-slowquery-check-script]
filename = instance-mariadb-check-slowquery-result.sh.in
filename = instance-mariadb-check-slowquery-result.sh.in
md5su
n = c9cd394f3d031cef77da50610544d2b
0
md5su
m = 356e0e2db1da0e8b479908fb739e5cc
0
[template-mariadb]
[template-mariadb]
filename = instance-mariadb.cfg.in
filename = instance-mariadb.cfg.in
md5sum =
9c34cba998ce0ed3b96e2604d2ed1cb5
md5sum =
aac99b03aa10b60e2b17d6882513b5bd
[template-kumofs]
[template-kumofs]
filename = instance-kumofs.cfg.in
filename = instance-kumofs.cfg.in
...
@@ -103,7 +103,7 @@ md5sum = 79fc39f7fbf13b1788adb5c33150dd80
...
@@ -103,7 +103,7 @@ md5sum = 79fc39f7fbf13b1788adb5c33150dd80
[apdex-result-check-script]
[apdex-result-check-script]
filename = instance-balancer-check-apachedex-result.sh.in
filename = instance-balancer-check-apachedex-result.sh.in
md5sum =
e1dd2f2884ea0e9f6f230419e032d927
md5sum =
cdc5e22560a26e072bade314bd4c506e
[template-haproxy-cfg]
[template-haproxy-cfg]
filename = haproxy.cfg.in
filename = haproxy.cfg.in
...
...
stack/erp5/instance-balancer-check-apachedex-result.sh.in
View file @
bbd02230
...
@@ -2,31 +2,25 @@
...
@@ -2,31 +2,25 @@
set
-e
set
-e
apachedex_file
=
'{{ apdex_file }}/ApacheDex-'
$(
date
+%Y-%m-%d
)
'.html'
APACHEDEX_FILE
=
'{{ apdex_file }}/ApacheDex-'
$(
date
+%Y-%m-%d
)
'.html'
apachedex_status_file
={{
apdex_status_file
}}
APACHEDEX_REPORT_JSON_FILE
={{
apdex_status_file
}}
desired_threshold
={{
default_threshold
}}
DESIRED_THRESHOLD
={{
user_threshold
}}
user_threshold
=
$(
<
{{
user_threshold
}}
)
if
[
!
-z
"
$user_threshold
"
]
then
desired_threshold
=
$user_threshold
fi
# Check if the file is there
# Check if the file is there
if
[
!
-s
"
$
apachedex_file
"
]
;
then
if
[
!
-s
"
$
APACHEDEX_FILE
"
]
;
then
# If file doesn't exists create one
# If file doesn't exists create one
# If it is empty check for modification time
# If it is empty check for modification time
if
[
!
-f
"
$
apachedex_file
"
]
;
then
if
[
!
-f
"
$
APACHEDEX_FILE
"
]
;
then
touch
$
apachedex_file
touch
$
APACHEDEX_FILE
else
else
modified_date
=
`
stat
-c
'%Z'
$apachedex_file
`
MODIFIED_DATE
=
`
stat
-c
'%Z'
$APACHEDEX_FILE
`
current_date
=
`
date
+%s
`
CURRENT_DATE
=
`
date
+%s
`
if
[[
`
echo
"
$
current_date
-
$modified_date
"
| bc
`
-gt
108000
]]
if
[[
`
echo
"
$
CURRENT_DATE
-
$MODIFIED_DATE
"
| bc
`
-gt
108000
]]
then
then
echo
"File modification date is greater than 30 hours"
echo
"File modification date is greater than 30 hours"
json_content
=
`
cat
$apachedex_status_file
`
JSON_CONTENT
=
`
cat
$APACHEDEX_REPORT_STATUS_FILE
`
message
=
`
echo
$json_content
| python
-c
'import json,sys;obj=json.load(sys.stdin);print obj["message"]'
`
MESSAGE
=
`
echo
$JSON_CONTENT
| python
-c
'import json,sys;obj=json.load(sys.stdin);print obj["message"]'
`
echo
$
message
echo
$
MESSAGE
exit
2
exit
2
else
else
echo
"File is empty for now"
echo
"File is empty for now"
...
@@ -35,17 +29,17 @@ if [ ! -s "$apachedex_file" ]; then
...
@@ -35,17 +29,17 @@ if [ ! -s "$apachedex_file" ]; then
else
else
# Check if the result exists
# Check if the result exists
{
{
regex
=
"apdex threshold<
\/
th><td>(.*)s
<
\/
td>"
REGEX
=
"<th>apdex.*?<tr><td [^<]*>(.*?)%
<
\/
td>"
file_content
=
`
cat
$apachedex_file
`
FILE_CONTENT
=
`
cat
$APACHDEX_FILE
`
if
[[
$
file_content
=
~
$regex
]]
if
[[
$
FILE_CONTENT
=
~
$REGEX
]]
then
then
threshold
=
${
BASH_REMATCH
[1]
}
RESULT
=
${
BASH_REMATCH
[1]
}
threshold
=
${
threshold
:-
0
}
RESULT
=
${
THRESHOLD
:-
0
}
if
[[
`
echo
"
$
threshold
>
$desired_threshold
"
| bc
`
-eq
1
]]
if
[[
`
echo
"
$
RESULT
>
$DESIRED_THRESHOLD
"
| bc
`
-eq
1
]]
then
then
echo
"Your score is
$
threshold
, Thanks for keeping it all clean"
echo
"Your score is
$
RESULT
%
, Thanks for keeping it all clean"
else
else
echo
"Threshold is lower than exptected: Expected was
$
desired_threshold
and we current is
$threshold
"
echo
"Threshold is lower than exptected: Expected was
$
DESIRED_THRESHOLD
% and we current is
$RESULT
%
"
exit
2
exit
2
fi
fi
else
else
...
...
stack/erp5/instance-mariadb-check-slowquery-result.sh.in
View file @
bbd02230
#!{{ bash }}
#!{{ bash }}
set
-e
set
-e
digest_file
=
'{{ slow_query_digest }}/slowquery_digest.txt'
DIGEST_FILE
=
'{{ slow_query_digest }}/slowquery_digest.txt'
slow_query_status_file
=
'{{ slow_query_status }}'
SLOW_QUERY_STATUS_FILE
=
'{{ slow_query_status }}'
desired_max_query_threshold
={{
default_threshold
}}
DESIRED_MAX_QUERY_THRESHOLD
={{
max_queries_threshold
}}
desired_slow_query_threshold
=
0
DESIRED_SLOW_QUERY_THRESHOLD
={{
slowest_queries_threshold
}}
max_queries_threshold
=
'{{ max_queries_threshold }}'
if
[
!
-z
"
$max_queries_threshold
"
]
then
desired_max_query_threshold
=
$max_queries_threshold
fi
slowest_queries_threshold
=
'{{ slowest_queries_threshold }}'
if
[
!
-z
"
$slowest_queries_threshold
"
]
then
desired_slow_query_threshold
=
$slowest_queries_threshold
fi
# Check if the file is there
# Check if the file is there
if
[
!
-s
"
$
digest_file
"
]
;
then
if
[
!
-s
"
$
DIGEST_FILE
"
]
;
then
# If file doesn't exists create one
# If file doesn't exists create one
# If it is empty check for modification time
# If it is empty check for modification time
if
[
!
-f
"
$
digest_file
"
]
;
then
if
[
!
-f
"
$
DIGEST_FILE
"
]
;
then
touch
$
digest_file
touch
$
DIGEST_FILE
else
else
modified_date
=
`
stat
-c
'%Z'
$digest_file
`
MODIFIED_DATE
=
`
stat
-c
'%Z'
$DIGEST_FILE
`
current_date
=
`
date
+%s
`
CURRENT_DATE
=
`
date
+%s
`
if
[[
`
echo
"
$
current_date
-
$modified_date
"
| bc
`
-gt
108000
]]
if
[[
`
echo
"
$
CURRENT_DATE
-
$MODIFIED_DATE
"
| bc
`
-gt
108000
]]
then
then
echo
"File modification date is greater than 30 hours"
echo
"File modification date is greater than 30 hours"
json_content
=
`
cat
$slow_query_status_file
`
JSON_CONTENT
=
`
cat
$SLOW_QUERY_STATUS_FILE
`
message
=
`
echo
$json_content
| python
-c
'import json,sys;obj=json.load(sys.stdin);print obj["message"]'
`
MESSAGE
=
`
echo
$JSON_CONTENT
| python
-c
'import json,sys;obj=json.load(sys.stdin);print obj["message"]'
`
echo
$
message
echo
$
MESSAGE
exit
2
exit
2
else
else
echo
"File is empty for now"
echo
"File is empty for now"
...
@@ -41,35 +29,35 @@ if [ ! -s "$digest_file" ]; then
...
@@ -41,35 +29,35 @@ if [ ! -s "$digest_file" ]; then
else
else
# Check if the result exists
# Check if the result exists
{
{
# get the total number of queries r
u
n and the max time
# get the total number of queries r
a
n and the max time
# TODO: improve regex
# TODO: improve regex
# TODO: improve the parameters (currently we are using threshold on queries and max execute time)
# TODO: improve the parameters (currently we are using threshold on queries and max execute time)
# # Overall: (.*) total,(?:.*\n){4}# Exec time(?: *\d*m?s){2} *(.*?)m?s
# # Overall: (.*) total,(?:.*\n){4}# Exec time(?: *\d*m?s){2} *(.*?)m?s
regex
=
"# Overall: (.*) total,.*# Exec time *[[:digit:]]*m?s *[[:digit:]]*m?s *([[:digit:]]*)m?s"
REGEX
=
"# Overall: (.*) total,.*# Exec time *[[:digit:]]*m?s *[[:digit:]]*m?s *([[:digit:]]*)m?s"
file_content
=
`
cat
$digest_file
`
FILE_CONTENT
=
`
cat
$DIGEST_FILE
`
if
[[
$
file_content
=
~
$regex
]]
if
[[
$
FILE_CONTENT
=
~
$REGEX
]]
then
then
max_query_threshold
=
${
BASH_REMATCH
[1]
}
TOTAL_QUERIES_EXEC
=
${
BASH_REMATCH
[1]
}
slowest_query_threshold
=
${
BASH_REMATCH
[2]
}
SLOWEST_QUERY_TIME
=
${
BASH_REMATCH
[2]
}
hasK
=
"
${
max_query_threshold
:
-1
}
"
HAS_K
=
"
${
TOTAL_QUERIES_EXEC
:
-1
}
"
if
[[
"
$
has
K
"
==
"k"
]]
if
[[
"
$
HAS_
K
"
==
"k"
]]
then
then
pre
=
"
${
max_query_threshold
::-1
}
"
PRE
=
"
${
TOTAL_QUERIES_EXEC
::-1
}
"
max_query_threshold
=
$(
echo
"scale=4;
${
pre
:-
0
}
*1000"
| bc
)
TOTAL_QUERIES_EXEC
=
$(
echo
"scale=4;
${
PRE
:-
0
}
*1000"
| bc
)
else
else
max_query_threshold
=
${
max_query_threshold
:-
0
}
TOTAL_QUERIES_EXEC
=
${
TOTAL_QUERIES_EXEC
:-
0
}
fi
fi
# TODO: support ms
# TODO: support ms
slowest_query_threshold
=
"
${
slowest_query_threshold
:-
0
}
"
SLOWEST_QUERY_TIME
=
"
${
SLOWEST_QUERY_TIME
:-
0
}
"
if
[[
`
echo
"
$
max_query_threshold
<
$desired_max_query_threshold
"
| bc
`
-eq
1
&&
`
echo
"
$slowest_query_threshold
>
$desired_slow_query_threshold
"
| bc
`
-eq
1
]]
if
[[
`
echo
"
$
TOTAL_QUERIES_EXEC
<
$DESIRED_MAX_QUERY_THRESHOLD
"
| bc
`
-eq
1
&&
`
echo
"
$SLOWEST_QUERY_TIME
<
$DESIRED_SLOW_QUERY_THRESHOLD
"
| bc
`
-eq
1
]]
then
then
echo
"Total number of slow queries are:
$
max_query_threshold
"
echo
"Total number of slow queries are:
$
TOTAL_QUERIES_EXEC
"
echo
"Time taken by slowest query is:
$
slowest_query_threshold
"
echo
"Time taken by slowest query is:
$
SLOWEST_QUERY_TIME
"
echo
"Thanks for keeping it all clean"
echo
"Thanks for keeping it all clean"
else
else
echo
"Ops! One of the two expected parameters did not meet"
echo
"Ops! One of the two expected parameters did not meet"
echo
"Time taken by slowest query is:
$
slowest_query_threshold
s and required maximum is
$desired_slow_query_threshold
s"
echo
"Time taken by slowest query is:
$
SLOWEST_QUERY_TIME
s and required maximum is
$DESIRED_SLOW_QUERY_THRESHOLD
s"
echo
"Total slow queries are
$
max_query_threshold
and expected maximum value is
$desired_max_query_threshold
"
echo
"Total slow queries are
$
TOTAL_QUERIES_EXEC
and expected maximum value is
$DESIRED_MAX_QUERY_THRESHOLD
"
exit
2
exit
2
fi
fi
else
else
...
...
stack/erp5/instance-mariadb.cfg.in
View file @
bbd02230
...
@@ -282,14 +282,13 @@ context =
...
@@ -282,14 +282,13 @@ context =
recipe = slapos.recipe.template:jinja2
recipe = slapos.recipe.template:jinja2
template = {{ parameter_dict['mariadb-slow-query-report-script'] }}
template = {{ parameter_dict['mariadb-slow-query-report-script'] }}
rendered = ${monitor-directory:reports}/${:filename}
rendered = ${monitor-directory:reports}/${:filename}
output-folder = ${directory:srv}/monitor/
filename = mariadb_slow_query_every_23_hour
filename = mariadb_slow_query_every_2_minutes
mode = 755
mode = 755
context =
context =
raw slow_query_path ${directory:srv}/backup/logrotate/mariadb_slowquery.log
raw slow_query_path ${directory:srv}/backup/logrotate/mariadb_slowquery.log
raw pt_query_exec ${binary-wrap-pt-digest:wrapper-path}
raw pt_query_exec ${binary-wrap-pt-digest:wrapper-path}
raw dash {{ parameter_dict['dash-location'] }}/bin/dash
raw dash {{ parameter_dict['dash-location'] }}/bin/dash
key output_folder
:output-folder
key output_folder
monitor-directory:private
[slowquery-parameters]
[slowquery-parameters]
max-queries-threshold = ${monitor-directory:etc}/max-slowqueries-threshold
max-queries-threshold = ${monitor-directory:etc}/max-slowqueries-threshold
...
@@ -301,11 +300,11 @@ template = {{ parameter_dict['mariadb-slowquery-check-script'] }}
...
@@ -301,11 +300,11 @@ template = {{ parameter_dict['mariadb-slowquery-check-script'] }}
rendered = ${monitor-directory:promises}/mariadb-slow-queries-result
rendered = ${monitor-directory:promises}/mariadb-slow-queries-result
status-file = ${monitor-directory:private}/mariadb_slow_query.report.json
status-file = ${monitor-directory:private}/mariadb_slow_query.report.json
context =
context =
raw default_threshold
0.7
raw default_threshold
4000
raw bash {{ parameter_dict['bash'] }}/bin/bash
raw bash {{ parameter_dict['bash'] }}/bin/bash
raw slow_query_digest ${directory:srv}/monitor
key slow_query_digest monitor-directory:private
key slow_query_status :status-file
key slow_query_status :status-file
key max_queries_threshold slowquery-parameters:max-queries-th
er
shold
key max_queries_threshold slowquery-parameters:max-queries-th
re
shold
key slowest_queries_threshold slowquery-parameters:slowest-queries-threshold
key slowest_queries_threshold slowquery-parameters:slowest-queries-threshold
[{{ section('promise') }}]
[{{ section('promise') }}]
...
@@ -320,7 +319,7 @@ monitor-httpd-port = {{ port + 1 }}
...
@@ -320,7 +319,7 @@ monitor-httpd-port = {{ port + 1 }}
monitor-title = {{ slapparameter_dict['name'] }}
monitor-title = {{ slapparameter_dict['name'] }}
password = {{ slapparameter_dict['monitor-passwd'] }}
password = {{ slapparameter_dict['monitor-passwd'] }}
instance-configuration =
instance-configuration =
file max-queries-threshold ${slowquery-parameter
e
s:max-queries-threshold}
file max-queries-threshold ${slowquery-parameters:max-queries-threshold}
file slowest-queries-threshold ${slowquery-parameters:slowest-queries-threshold}
file slowest-queries-threshold ${slowquery-parameters:slowest-queries-threshold}
[buildout]
[buildout]
...
...
stack/erp5/mysql-querydigest.sh.in
View file @
bbd02230
...
@@ -21,4 +21,5 @@ if [ ! -f "$SLOW_LOG" ]; then
...
@@ -21,4 +21,5 @@ if [ ! -f "$SLOW_LOG" ]; then
exit 1
exit 1
fi
fi
eval $PT_QUERY_EXEC $SLOW_LOG | tee $OUTPUT_FILE
$PT_QUERY_EXEC $SLOW_LOG > $OUTPUT_FILE
echo "ok"
Write
Preview
Markdown
is supported
0%
Try again
or
attach a new file
Attach a file
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment