stack/erp5: replace bash based promises with python based promises from toolbox

stack/erp5/buildout.cfg

@@ -164,10 +164,6 @@ command = grep parts ${buildout:develop-eggs-directory}/slapos.cookbook.egg-link
 <= download-base
 mode = 755
 
-[mariadb-slowquery-check-script]
-<= download-base
-mode = 755
-
 [mariadb-slow-query-report-script]
 <= download-base
 mode = 755
@@ -223,8 +219,6 @@ context =
     key mariadb_link_binary template-mariadb:link-binary
     key zope_link_binary template-zope:link-binary
     key apache_location apache:location
-    key apdex_result_check_script apdex-result-check-script:target
-    key mariadb_slowquery_check_script mariadb-slowquery-check-script:target 
     key aspell_location aspell:location
     key bin_directory buildout:bin-directory
     key buildout_bin_directory buildout:bin-directory
@@ -342,9 +336,6 @@ link-binary =
 [template-balancer]
 <= download-base
 
-[apdex-result-check-script]
-<= download-base
-
 [template-haproxy-cfg]
 <= download-base
 
@@ -608,6 +599,8 @@ eggs +=
 scripts +=
   is-local-tcp-port-opened
   is-process-older-than-dependency-set
+  check-slow-queries-digest-result
+  check-apachedex-result
   runApacheDex
   zodbpack
 

stack/erp5/buildout.hash.cfg

@@ -21,13 +21,9 @@ md5sum = 844d62cd6f9d6e3d1d78d52de2b72a49
 filename = mysql-querydigest.sh.in
 md5sum = cfe6ab8ae54a521ecb269e9d9762cbeb
 
-[mariadb-slowquery-check-script]
-filename = instance-mariadb-check-slowquery-result.sh.in 
-md5sum = 356e0e2db1da0e8b479908fb739e5cc0
-
 [template-mariadb]
 filename = instance-mariadb.cfg.in
-md5sum = 7ee2e801dda1181d1b42281e6466fc4d
+md5sum = 5c11abdef0c84cdef91f999490786244
 
 [template-kumofs]
 filename = instance-kumofs.cfg.in
@@ -67,7 +63,7 @@ md5sum = 0969fbb25b05c02ef3c2d437b2f4e1a0
 
 [template]
 filename = instance.cfg.in
-md5sum = 25c059b9a04a05be71e43b0b43685217
+md5sum = 9adb0aea4248a21bfe4dd48df82f864d
 
 [monitor-template-dummy]
 filename = dummy.cfg
@@ -87,11 +83,7 @@ md5sum = 27d26c6380883cf3bd7b2f003f7888d8
 
 [template-balancer]
 filename = instance-balancer.cfg.in
-md5sum = d14ee7f13e2bd815cc96e28101e59670
-
-[apdex-result-check-script]
-filename = instance-balancer-check-apachedex-result.sh.in 
-md5sum = 421c68c97cadc49911382cd3185288a1
+md5sum = a93c7fae7402db91142cc73e5af55c6c
 
 [template-haproxy-cfg]
 filename = haproxy.cfg.in

stack/erp5/instance-balancer-check-apachedex-result.sh.in

-#!{{ bash }}
-
-set -e
-
-APACHEDEX_FILE='{{ apdex_file }}/ApacheDex-'$(date +%Y-%m-%d)'.html'
-APACHEDEX_REPORT_JSON_FILE={{ apdex_status_file }} 
-DESIRED_THRESHOLD={{ user_threshold }}
-
-# Check if the file is there
-if [ ! -s "$APACHEDEX_FILE" ]; then 
-  # If file doesn't exists create one
-  # If it is empty check for modification time
-  if [ ! -f "$APACHEDEX_FILE" ]; then
-    touch $APACHEDEX_FILE
-  else
-    MODIFIED_DATE=`stat -c '%Z' $APACHEDEX_FILE`
-    CURRENT_DATE=`date +%s`
-    if [[ `echo "$CURRENT_DATE - $MODIFIED_DATE" | bc` -gt 108000 ]]
-    then 
-      echo "File modification date is greater than 30 hours"
-      JSON_CONTENT=`cat $APACHEDEX_REPORT_STATUS_FILE`
-      MESSAGE=`echo $JSON_CONTENT | python -c 'import json,sys;obj=json.load(sys.stdin);print obj["message"]'`
-      echo $MESSAGE
-      exit 2
-    else
-      echo "File is empty for now"
-    fi
-  fi
-else
-  # Check if the result exists
-  {
-    REGEX="Overall<\/h2><table .*><tr>[[:space:]]<th>apdex<\/th><th>.*?<tr>[[:space:]]<td [^<]*>(.*?)%<\/td>"
-    FILE_CONTENT=`cat $APACHEDEX_FILE`
-    if [[ $FILE_CONTENT =~ $REGEX ]]
-    then
-      RESULT=${BASH_REMATCH[1]}
-      RESULT=${RESULT:-0}
-      if [[ `echo "$RESULT > $DESIRED_THRESHOLD" | bc` -eq 1 ]]
-      then
-        echo "Your score is $RESULT %, Thanks for keeping it all clean"
-      else
-        echo "Threshold is lower than exptected: Expected was $DESIRED_THRESHOLD % and we current is $RESULT %"
-        exit 2
-      fi
-    else
-      echo "No threshold found in the result"
-    fi
-  } || {
-    echo "Cannot parse the apdex result"
-  }
-fi

stack/erp5/instance-balancer.cfg.in

@@ -298,15 +298,9 @@ configuration = {{ slapparameter_dict['apachedex-configuration'] }}
 promise-threshold = {{ slapparameter_dict['apachedex-promise-threshold'] }}
 
 [{{ section('monitor-promise-apachedex-result') }}]
-recipe = slapos.recipe.template:jinja2
-template = {{ parameter_dict['apdex-result-check-script'] }}
-rendered = ${monitor-directory:promises}/check-apachedex-result
-status-file = ${monitor-directory:private}/apachedex.report.json 
-context = 
-  raw bash {{ parameter_dict['bash'] }}/bin/bash
-  raw user_threshold ${apachedex-parameters:promise-threshold}
-  key apdex_file directory:apachedex
-  key apdex_status_file :status-file
+recipe = slapos.cookbook:wrapper
+wrapper-path = ${directory:promise}/check-apachedex-result
+command-line = "{{ parameter_dict['promise-check-apachedex-result'] }}" --apachedex_file "${directory:apachedex}" --status_file ${monitor-directory:private}/apachedex.report.json --threshold "${apachedex-parameters:promise-threshold}"
 
 [monitor-instance-parameter]
 monitor-httpd-ipv6 = {{ (ipv6_set | list)[0] }}

stack/erp5/instance-mariadb-check-slowquery-result.sh.in

-#!{{ bash }}
-set -e
-
-DIGEST_FILE='{{ slow_query_digest }}/slowquery_digest.txt'
-SLOW_QUERY_STATUS_FILE='{{ slow_query_status }}'
-DESIRED_MAX_QUERY_THRESHOLD={{ max_queries_threshold }}
-DESIRED_SLOW_QUERY_THRESHOLD={{ slowest_queries_threshold  }}
-
-# Check if the file is there
-if [ ! -s "$DIGEST_FILE" ]; then 
-  # If file doesn't exists create one
-  # If it is empty check for modification time
-  if [ ! -f "$DIGEST_FILE" ]; then
-    touch $DIGEST_FILE
-  else
-    MODIFIED_DATE=`stat -c '%Z' $DIGEST_FILE`
-    CURRENT_DATE=`date +%s`
-    if [[ `echo "$CURRENT_DATE - $MODIFIED_DATE" | bc` -gt 108000 ]]
-    then 
-      echo "File modification date is greater than 30 hours"
-      JSON_CONTENT=`cat $SLOW_QUERY_STATUS_FILE`
-      MESSAGE=`echo $JSON_CONTENT | python -c 'import json,sys;obj=json.load(sys.stdin);print obj["message"]'`
-      echo $MESSAGE 
-      exit 2
-    else
-      echo "File is empty for now"
-    fi
-  fi
-else
-  # Check if the result exists
-  {
-    # get the total number of queries ran and the max time
-    # TODO: improve regex
-    # TODO: improve the parameters (currently we are using threshold on queries and max execute time)
-    # # Overall: (.*) total,(?:.*\n){4}# Exec time(?: *\d*m?s){2} *(.*?)m?s
-    REGEX="# Overall: (.*) total,.*# Exec time *[[:digit:]]*m?s *[[:digit:]]*m?s *([[:digit:]]*)m?s"
-    FILE_CONTENT=`cat $DIGEST_FILE`
-    if [[ $FILE_CONTENT =~ $REGEX ]]
-    then
-      TOTAL_QUERIES_EXEC=${BASH_REMATCH[1]}
-      SLOWEST_QUERY_TIME=${BASH_REMATCH[2]}
-      HAS_K="${TOTAL_QUERIES_EXEC: -1}"
-      if [[ "$HAS_K" == "k" ]] 
-      then
-        PRE="${TOTAL_QUERIES_EXEC::-1}"
-        TOTAL_QUERIES_EXEC=$(echo "scale=4; ${PRE:-0}*1000" | bc)
-      else
-        TOTAL_QUERIES_EXEC=${TOTAL_QUERIES_EXEC:-0}
-      fi
-      # TODO: support ms
-      SLOWEST_QUERY_TIME="${SLOWEST_QUERY_TIME:-0}"
-      if [[ `echo "$TOTAL_QUERIES_EXEC < $DESIRED_MAX_QUERY_THRESHOLD" | bc` -eq 1 && `echo "$SLOWEST_QUERY_TIME < $DESIRED_SLOW_QUERY_THRESHOLD" | bc` -eq 1 ]]
-      then
-        echo "Total number of slow queries are: $TOTAL_QUERIES_EXEC"
-        echo "Time taken by slowest query is: $SLOWEST_QUERY_TIME"
-        echo "Thanks for keeping it all clean"
-      else
-        echo "Ops! One of the two expected parameters did not meet"
-        echo "Time taken by slowest query is: $SLOWEST_QUERY_TIME s and required maximum is $DESIRED_SLOW_QUERY_THRESHOLD s"
-        echo "Total slow queries are $TOTAL_QUERIES_EXEC and expected maximum value is $DESIRED_MAX_QUERY_THRESHOLD"
-        exit 2
-      fi
-    else
-      echo "No threshold found in the result"
-    fi
-  } || {
-    echo "Cannot parse the result"
-  }
-fi

stack/erp5/instance-mariadb.cfg.in

@@ -295,17 +295,10 @@ max_queries_threshold = {{ slapparameter_dict['max-slowqueries-threshold'] }}
 slowest_queries_threshold = {{ slapparameter_dict['slowest-query-threshold'] }}
 
 [{{ section('monitor-promise-slowquery-result') }}]
-recipe = slapos.recipe.template:jinja2
-template = {{ parameter_dict['mariadb-slowquery-check-script'] }}
-rendered = ${monitor-directory:promises}/mariadb-slow-queries-result
-status-file = ${monitor-directory:private}/mariadb_slow_query.report.json
-context =
-  raw default_threshold 4000
-  raw bash {{ parameter_dict['bash'] }}/bin/bash
-  key slow_query_digest monitor-directory:private
-  key slow_query_status :status-file
-  key max_queries_threshold slow-query-digest-parameters:max_queries_threshold
-  key slowest_queries_threshold slow-query-digest-parameters:slowest_queries_threshold
+recipe = slapos.cookbook:wrapper
+wrapper-path = ${directory:promise}/check-slow-query-pt-digest-result
+command-line = "{{ parameter_dict['promise-check-slow-queries-digest-result'] }}" --ptdigest_file "${monitor-directory:private}/slowquery_digest.txt" --status_file ${monitor-directory:private}/mariadb_slow_query.report.json --max_queries_threshold "${slow-query-digest-parameters:max_queries_threshold}" --slowest_query_threshold "${slow-query-digest-parameters:slowest_queries_threshold}"
+
 
 [{{ section('promise') }}]
 recipe = slapos.cookbook:wrapper

stack/erp5/instance.cfg.in

@@ -49,12 +49,12 @@ openssl-location = {{ openssl_location }}
 
 [dynamic-template-balancer-parameters]
 apache = {{ apache_location }}
-apdex-result-check-script = {{ apdex_result_check_script }}
 openssl = {{ openssl_location }}
 haproxy = {{ haproxy_location }}
 bin-directory = {{ bin_directory }}
 apachedex-location = {{ bin_directory }}/apachedex
 run-apachedex-location = {{ bin_directory }}/runApacheDex
+promise-check-apachedex-result = {{ bin_directory }}/check-apachedex-result
 6tunnel = {{ sixtunnel_location }}
 curl-location = {{ curl_location }}
 dash = {{ dash_location }}
@@ -139,7 +139,7 @@ link-binary = {{ dumps(mariadb_link_binary) }}
 bin-directory = {{ bin_directory }}
 mariadb-resiliency-after-import-script = {{ mariadb_resiliency_after_import_script }}
 mariadb-slow-query-report-script = {{ mariadb_slow_query_report_script }}
-mariadb-slowquery-check-script = {{ mariadb_slowquery_check_script}}
+promise-check-slow-queries-digest-result = {{ bin_directory }}/check-slow-queries-digest-result
 percona-tools-location = {{ percona_toolkit_location }}
 template-monitor = {{ template_monitor }}