erp5: monitor check cache hit uses pycurl instead of requests egg

stack/erp5/buildout.cfg

@@ -67,6 +67,7 @@ extends =
   ../../component/6tunnel/buildout.cfg
   ../../component/findutils/buildout.cfg
   ../../stack/monitor/buildout.cfg
+  ../../component/pycurl/buildout.cfg
 
 parts =
   rdiff-backup
@@ -190,7 +191,7 @@ output = ${buildout:directory}/template-http-monitor.cfg.in
 < = download-base
 url = ${:_profile_base_location_}/monitor-templates/monitor-check-cache-hit.in
 download-only = true
-md5sum = 149b3c42eadf084713125e103c1a69d0
+md5sum = 4703a0f64c72da35b897d020e022d1b1
 filename = monitor-check-cache-hit.in
 mode = 0644
 
@@ -588,7 +589,7 @@ eggs =
 # XXX missing two eggs to build stack monitor
 
 # needed for monitoring tool
-  requests
+  ${pycurl:egg}
 
 # parameterizing the version of the generated python interpreter name by the
 # python section version causes dependency between this egg section and the

stack/erp5/monitor-templates/monitor-check-cache-hit.in

 #!{{ python_executable }}
 # -*- coding: utf-8 -*-
 
-# dependency: python2-requests
+# dependency: python2-pycurl
 
 import sys
-import requests
+import pycurl
 
+from mimetools import Message
+from cStringIO import StringIO
 from HTMLParser import HTMLParser
 
+def debug(*args):
+  # sys.stdout.write("DEBUG: " + "\n     : ".join(" ".join((str(arg) for arg in args)).split("\n")) + "\n")
+  pass
 def log(*args):
   sys.stdout.write(" ".join((str(arg) for arg in args)) + "\n")
   pass
-def debug(*args):
-  # sys.stdout.write("DEBUG: " + "DEBUG: ".join(" ".join((str(arg) for arg in args)).split("\n")) + "\n")
-  pass
 def info(*args):
-  sys.stdout.write("INFO : " + "INFO : ".join(" ".join((str(arg) for arg in args)).split("\n")) + "\n")
+  sys.stdout.write("INFO : " + "\n     : ".join(" ".join((str(arg) for arg in args)).split("\n")) + "\n")
   pass
 def warn(*args):
-  last_warn_log = "WARN : " + "WARN : ".join(" ".join((str(arg) for arg in args)).split("\n")) + "\n"
+  last_warn_log = "WARN : " + "\n     : ".join(" ".join((str(arg) for arg in args)).split("\n")) + "\n"
   sys.stderr.write(last_warn_log)
 last_error_log = None
 def error(*args):
   global last_error_log
-  last_error_log = "ERROR: " + "ERROR: ".join(" ".join((str(arg) for arg in args)).split("\n")) + "\n"
+  last_error_log = "ERROR: " + "\n     : ".join(" ".join((str(arg) for arg in args)).split("\n")) + "\n"
   sys.stderr.write(last_error_log)
 
-info("Start checking for cache hits")
-
 class MyHTMLParser(HTMLParser):
   def handle_starttag(self, tag, attrs):
     if tag == 'img' or tag == 'script': # TODO: CSS & JS
@@ -47,26 +47,50 @@ headers = {
  #   "Connection": "keep-alive"
 }
 
-base = u"http://www.erp5.com/"
+base = "http://www.erp5.com/"
 
 vary_dict = dict()
 hit_dict = dict()
 status_dict = dict()
 headers_dict = dict()
 
+info("Start checking for cache hits")
+
 def do_request(url):
   if url in hit_dict: return
   log("Checking cache hit for", url)
-  r = requests.get(url, headers=headers)
-  hit_dict[url] = "HIT" in r.headers.get('x-cache', '')
-  vary_dict[url] = r.headers.get('vary')
-  status_dict[url] = r.status_code
-  headers_dict[url] = r.headers
+  c = pycurl.Curl()
+  response_headers = StringIO()
+  output = StringIO()
+  c.setopt(c.URL, url)
+  c.setopt(c.RESOLVE, ["www.erp5.com:80:5.135.149.226"])
+  c.setopt(c.WRITEFUNCTION, output.write)
+  c.setopt(c.HEADERFUNCTION, response_headers.write)
+  c.perform() # perform a request before testing if the cache is hit
+  response_headers.truncate(0)
+  output.truncate(0)
+  c.perform()
+
+  response_headers.seek(0)
+  status = response_headers.readline().split(" ")[1]
+
+  m = Message(response_headers)
+
+  # see http://labs.omniti.com/people/mark/ats_sa/slides.html#slide-18
+  hit_dict[url] = any("[cHs" in header or "[cSs" in header for header in m.getheaders('via'))
+  vary_dict[url] = m.getheader('vary')
+  status_dict[url] = status
+  headers_dict[url] = response_headers.getvalue()
   if not hit_dict[url]:
-    error("X-Cache is", repr(r.headers.get('x-cache')))
+    error("No cache hit found in", m.getheaders('via'))
+  else:
+    debug("Cache hit found in", m.getheaders('via'))
+
+  if m.getheader('content-type', '').startswith('text/html'):
+    MyHTMLParser().feed(output.getvalue())
 
-  if r.headers['content-type'].startswith('text/html'):
-    MyHTMLParser().feed(r.text)
+  response_headers.close()
+  output.close()
 
 do_request(base)
 
@@ -86,8 +110,7 @@ debug('--------------------------------------------------')
 for url, hit in hit_dict.items():
   if not hit:
     log(url)
-    log(status_dict[url], vary_dict[url])
-    log(pformat(headers_dict[url].items()))
+    log(headers_dict[url])
     log()
 
 if last_error_log is not None: