monitoring_tool: Use apachedex to analyse apache logs

stack/monitor/buildout.cfg

@@ -15,6 +15,7 @@ parts =
   monitor-bin
   monitor-template
   rss-bin
+  run-apachedex
 
 [monitor-eggs]
 recipe = zc.recipe.egg
@@ -28,6 +29,7 @@ interpreter = pythonwitheggs
 eggs =
   PyRSS2Gen
   Jinja2
+  APacheDEX
 
 [make-rss-script]
 recipe = slapos.recipe.template
@@ -41,7 +43,7 @@ recipe = slapos.recipe.template
 url = ${:_profile_base_location_}/monitor.cfg.in
 output = ${buildout:directory}/monitor.cfg
 filename = monitor.cfg
-md5sum = 4d0c3b847c18a56068f04dd926487272
+md5sum = fb6fb065e72773122c5f991b52ebddc5
 mode = 0644
 
 [monitor-bin]
@@ -124,6 +126,14 @@ destination = ${buildout:parts-directory}/monitor-template-rss-bin
 filename = status2rss.py
 mode = 0644
 
+[run-apachedex]
+recipe = hexagonit.recipe.download
+url = ${:_profile_base_location_}/${:filename}
+download-only = true
+md5sum = 052b7818d052d115b66ce2b3dab1d7c3
+filename = run-apachedex.py.in
+mode = 0644
+
 [dcron-service]
 recipe = slapos.recipe.template
 url = ${template-dcron-service:output}

stack/monitor/monitor.cfg.in

@@ -50,6 +50,8 @@ public-cgi = $${:cgi-bin}/monitor-public
 monitor-custom-scripts = $${:etc}/monitor
 monitor-result = $${:var}/monitor
 
+apachedex-result = $${:srv}/apachedex
+
 private-directory = $${:srv}/monitor-private
 
 [public-symlink]
@@ -188,6 +190,41 @@ recipe = plone.recipe.command
 command = ln -s $${:source} $${monitor-directory:private-directory}
 source =
   
+[apachedex-entries-base]
+recipe = slapos.recipe.template:jinja2
+template = ${run-apachedex:location}/${run-apachedex:filename}
+rendered = $${monitor-directory:bin}/$${:script-name}
+mode = 0700
+extensions = jinja2.ext.do
+extra-context =
+context = 
+  raw python_executable ${buildout:executable}
+  raw apachedex_executable ${buildout:directory}/bin/apachedex
+  key output_folder monitor-directory:apachedex-result
+  $${:extra-context}
+  
+[apachedex-entries]
+<= apachedex-entries-base
+script-name = apachedex
+extra-context =
+  section parameter_dict apachedex-parameters
+  key name :script-name
+  
+[apachedex-parameters]
+# XXX - Sample log file with curent date: apache_access.log-%(date)s.gz
+# which will be equivalent to apache_access.log-20150112.gz if the date is 2015-01-12
+apache-log-list = 
+base-list = 
+skip-base-list = 
+erp5-base-list = 
+
+[cron-entry-apachedex]
+<= cron
+recipe = slapos.cookbook:cron.d
+name = $${apachedex-entries:script-name}
+frequency = 0 3 * * *
+command = $${apachedex-entries:rendered}
+
 [monitor-instance-log-access]
 recipe = plone.recipe.command
 command = if [ -d $${:source} ]; then ln -s $${:source} $${monitor-directory:private-directory}/instance-logs; fi

stack/monitor/run-apachedex.py.in

+#!{{ python_executable }}
+# BEWARE: This file is operated by slapgrid
+# BEWARE: It will be overwritten automatically
+
+import os
+import subprocess
+from datetime import date
+
+base_name = "{{ name }}".strip()
+apache_log_list = """{{ parameter_dict['apache-log-list'] }}""".split('\n')
+base_list = [base.strip() for base in
+                """{{ parameter_dict['base-list'] }}""".split('\n') if base]
+skip_base_list = [base.strip() for base in
+                """{{ parameter_dict['skip-base-list'] }}""".split('\n')
+                if base]
+erp5_base_list = [base.strip() for base in
+                  """{{ parameter_dict['erp5-base-list'] }}""".split('\n')
+                  if base]
+
+output_folder = "{{ output_folder }}".strip()
+if not len(apache_log_list):
+  exit(1)
+if not os.path.exists(output_folder) or not os.path.isdir(output_folder):
+  print "ERROR: Output folder is not a directory. Exiting..."
+  exit(1)
+today = date.today().strftime("%Y-%m-%d")
+folder_today = os.path.join(output_folder, 'ApacheDex-%s' % today)
+if not os.path.exists(folder_today):
+  os.makedirs(folder_today)
+
+apachedex = "{{ apachedex_executable }}".strip()
+argument_list = [apachedex, '--js-embed', '--out',
+                  os.path.join(folder_today, 'ApacheDex-%s.html' % base_name)]
+
+log_list = []
+for logfile in apache_log_list:
+  if not logfile:
+    continue
+  # Automaticaly replace variable 'date'.
+  apache_log = logfile.strip() % {'date': date.today().strftime("%Y%m%d")}
+  if not os.path.exists(apache_log):
+    print "WARNING: File %s not found..." % apache_log
+    continue
+  log_list.append(apache_log)
+if not log_list:
+  print "WARNING: Log file list to analyse is empty or not provided. Exiting..."
+  exit(1)
+
+if erp5_base_list:
+  argument_list.append('--erp5-base')
+  argument_list += erp5_base_list
+
+if base_list:
+  argument_list.append('--base')
+  argument_list += base_list
+
+if skip_base_list:
+  argument_list.append('--skip-base')
+  argument_list += skip_base_list
+
+argument_list.append('--error-detail')
+argument_list += log_list
+
+subprocess.check_call(argument_list)