erp5: monitor-check-cache-hit script added to stack monitor

stack/erp5/buildout.cfg

@@ -176,9 +176,23 @@ context =
     ${:extra-context}
 
 [template-http-monitor]
-< = download-base
+recipe = slapos.recipe.template
 filename = instance-http-monitor.cfg.in
-md5sum = b57272c05d47ff55d4473dbed323ae12
+url = ${:_profile_base_location_}/${:filename}
+md5sum = 0186cfdf8a64036dff85411df8524138
+output = ${buildout:directory}/template-http-monitor.cfg.in
+
+##################e
+# Monitor element
+#
+
+[template-monitor-check-cache-hit]
+< = download-base
+url = ${:_profile_base_location_}/monitor-templates/monitor-check-cache-hit.in
+download-only = true
+md5sum = 149b3c42eadf084713125e103c1a69d0
+filename = monitor-check-cache-hit.in
+mode = 0644
 
 [template-mariadb]
 < = download-base
@@ -286,7 +300,7 @@ extra-context =
     key template_create_erp5_site_real template-create-erp5-site-real:target
     key template_erp5 template-erp5:target
     key template_haproxy_cfg template-haproxy-cfg:target
-    key template_http_monitor template-http-monitor:target
+    key template_http_monitor template-http-monitor:output
     key template_kumofs template-kumofs:target
     key template_logrotate_base template-logrotate-base:rendered
     key template_mariadb template-mariadb:target
@@ -571,6 +585,11 @@ eggs =
 # Needed for parsing .po files from our Localizer subset
   polib
 
+# XXX missing two eggs to build stack monitor
+
+# needed for monitoring tool
+  requests
+
 # parameterizing the version of the generated python interpreter name by the
 # python section version causes dependency between this egg section and the
 # installation of python, which we don't want on an instance

stack/erp5/instance-http-monitor.cfg.in

@@ -15,9 +15,19 @@ parts =
   cgi-httpd-graceful-wrapper
   monitor-promise
   monitor-instance-log-access
+  monitor-check-cache-hit
 
 extends = ${monitor-template:output}
 
 eggs-directory = {{ eggs_directory }}
 develop-eggs-directory = {{ develop_eggs_directory }}
+
+[monitor-check-cache-hit]
+recipe = slapos.recipe.template:jinja2
+template = ${template-monitor-check-cache-hit:location}/${template-monitor-check-cache-hit:filename}
+rendered = $${monitor-directory:monitor-custom-scripts}/check-cache-hit.py
+mode = 700
+context =
+  raw python_executable ${buildout:bin-directory}/python2.7
+
 {% endif %}

stack/erp5/monitor-templates/monitor-check-cache-hit.in

+#!{{ python_executable }}
+# -*- coding: utf-8 -*-
+
+# dependency: python2-requests
+
+import sys
+import requests
+
+from HTMLParser import HTMLParser
+
+def log(*args):
+  sys.stdout.write(" ".join((str(arg) for arg in args)) + "\n")
+  pass
+def debug(*args):
+  # sys.stdout.write("DEBUG: " + "DEBUG: ".join(" ".join((str(arg) for arg in args)).split("\n")) + "\n")
+  pass
+def info(*args):
+  sys.stdout.write("INFO : " + "INFO : ".join(" ".join((str(arg) for arg in args)).split("\n")) + "\n")
+  pass
+def warn(*args):
+  last_warn_log = "WARN : " + "WARN : ".join(" ".join((str(arg) for arg in args)).split("\n")) + "\n"
+  sys.stderr.write(last_warn_log)
+last_error_log = None
+def error(*args):
+  global last_error_log
+  last_error_log = "ERROR: " + "ERROR: ".join(" ".join((str(arg) for arg in args)).split("\n")) + "\n"
+  sys.stderr.write(last_error_log)
+
+info("Start checking for cache hits")
+
+class MyHTMLParser(HTMLParser):
+  def handle_starttag(self, tag, attrs):
+    if tag == 'img' or tag == 'script': # TODO: CSS & JS
+      debug(tag, attrs)
+      for attr in attrs:
+        if attr[0] == 'src':
+          url = attr[1]
+          if not url.startswith('http'):
+            url = base + url
+          do_request(url)
+
+headers = {
+  "User-Agent": "test",
+  "Accept-Encoding": "gzip, deflate",
+  "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
+  "Accept-Language": "en-US,en;q=0.5",
+ #   "Connection": "keep-alive"
+}
+
+base = u"http://www.erp5.com/"
+
+vary_dict = dict()
+hit_dict = dict()
+status_dict = dict()
+headers_dict = dict()
+
+def do_request(url):
+  if url in hit_dict: return
+  log("Checking cache hit for", url)
+  r = requests.get(url, headers=headers)
+  hit_dict[url] = "HIT" in r.headers.get('x-cache', '')
+  vary_dict[url] = r.headers.get('vary')
+  status_dict[url] = r.status_code
+  headers_dict[url] = r.headers
+  if not hit_dict[url]:
+    error("X-Cache is", repr(r.headers.get('x-cache')))
+
+  if r.headers['content-type'].startswith('text/html'):
+    MyHTMLParser().feed(r.text)
+
+do_request(base)
+
+from pprint import pformat
+
+debug('--------------------------------------------------')
+debug('hit_dict')
+debug(pformat(hit_dict))
+debug('--------------------------------------------------')
+debug('vary_dict')
+debug(pformat(vary_dict))
+debug('--------------------------------------------------')
+debug('status_dict')
+debug(pformat(status_dict))
+debug('--------------------------------------------------')
+
+for url, hit in hit_dict.items():
+  if not hit:
+    log(url)
+    log(status_dict[url], vary_dict[url])
+    log(pformat(headers_dict[url].items()))
+    log()
+
+if last_error_log is not None:
+  sys.exit(1)
+info("OK");