Add decimation support.

Because graphs are nicer at higher definition, and tables are nicer with
fewer columns.

apachedex/__init__.py

@@ -161,7 +161,7 @@ class GenericSiteStats(object):
         float(apdex.duration_max) / US_PER_S, apdex.hit) for date, apdex
       in sorted(self.apdex.iteritems(), key=ITEMGETTER0)]
 
-  def asHTML(self):
+  def asHTML(self, stat_filter=lambda x: x):
     self._housekeeping()
     result = []
     append = result.append
@@ -176,8 +176,12 @@ class GenericSiteStats(object):
       float(apdex.duration_max) / US_PER_S,
     ))
     column_set = set()
-    for data_dict in self.status.itervalues():
-      column_set.update(data_dict)
+    filtered_status = defaultdict(partial(defaultdict, int))
+    for status, date_dict in self.status.iteritems():
+      filtered_date_dict = filtered_status[status]
+      for date, value in date_dict.iteritems():
+        filtered_date_dict[stat_filter(date)] += value
+      column_set.update(filtered_date_dict)
     column_list = sorted(column_set)
     append('<h2>Hit per status code</h2><table><tr><th>status</th>'
       '<th>overall</th>')
@@ -186,7 +190,7 @@ class GenericSiteStats(object):
     append('</tr>')
     def hitTd(hit, status):
       return '<td class="%s">%s</td>' % (getClassForStatusHit(hit, status), hit)
-    for status, data_dict in sorted(self.status.iteritems(), key=ITEMGETTER0):
+    for status, data_dict in sorted(filtered_status.iteritems(), key=ITEMGETTER0):
       append('<tr><td>%s</td>' % status)
       append(hitTd(sum(data_dict.itervalues()), status))
       for date in column_list:
@@ -237,14 +241,24 @@ class ERP5SiteStats(GenericSiteStats):
       else:
         self.no_module[date].accumulate(match)
 
-  def asHTML(self):
+  def asHTML(self, stat_filter=lambda x: x):
     result = []
     append = result.append
     append('<h2>Per module</h2><table><tr><th rowspan="2" colspan="2">'
       'module</th><th colspan="4">overall</th>')
-    column_set = set()
-    for data_dict in self.module.itervalues():
-      column_set.update(data_dict)
+    filtered_module = defaultdict(partial(defaultdict, partial(
+      defaultdict, partial(APDEXStats, self.threshold))))
+    filtered_no_module = defaultdict(partial(APDEXStats, self.threshold))
+    for date, value in self.no_module.iteritems():
+      filtered_no_module[stat_filter(date)].accumulateFrom(value)
+    column_set = set(filtered_no_module)
+    for key, is_document_dict in self.module.iteritems():
+      filtered_is_document_dict = filtered_module[key]
+      for key, data_dict in is_document_dict.iteritems():
+        filtered_data_dict = filtered_is_document_dict[key]
+        for date, value in data_dict.iteritems():
+          filtered_data_dict[stat_filter(date)].accumulateFrom(value)
+        column_set.update(filtered_data_dict)
     column_list = sorted(column_set)
     for date in column_list:
       append('<th colspan="4">%s</th>' % date)
@@ -269,7 +283,7 @@ class ERP5SiteStats(GenericSiteStats):
       append(apdexStatsAsHtml(data_total))
       for date in column_list:
         append(apdexStatsAsHtml(data_dict[date]))
-    for module_id, data_dict in sorted(self.module.iteritems(),
+    for module_id, data_dict in sorted(filtered_module.iteritems(),
         key=ITEMGETTER0):
       append('<tr><th rowspan="2">%s</th><th>module</th>' % module_id)
       apdexAsColumns(data_dict[False])
@@ -277,9 +291,9 @@ class ERP5SiteStats(GenericSiteStats):
       apdexAsColumns(data_dict[True])
       append('</tr>')
     append('<tr><th colspan="2">(none)</th>')
-    apdexAsColumns(self.no_module)
+    apdexAsColumns(filtered_no_module)
     append('</tr></table>')
-    append(super(ERP5SiteStats, self).asHTML())
+    append(super(ERP5SiteStats, self).asHTML(stat_filter=stat_filter))
     return '\n'.join(result)
 
 logformat_dict = {
@@ -327,8 +341,8 @@ def _asHourString(timestamp):
   return '%s/%02i/%s %s' % (year, MONTH_VALUE_DICT[month], day, hour)
 
 period_parser = {
-  'day': _asDayString,
-  'hour': _asHourString,
+  'month': (_asDayString, lambda x: '/'.join(x.split('/', 2)[:2])),
+  'day': (_asHourString, lambda x: x.split(' ')[0]),
 }
 
 def main():
@@ -387,7 +401,7 @@ def main():
     assert not key, key
   matchline = re.compile(line_regex).match
   matchrequest = REQUEST_PATTERN.match
-  asDate = period_parser[args.period]
+  asDate, decimator = period_parser[args.period]
   site_list = args.site_list
   default_site = args.default
   if default_site is None:
@@ -444,7 +458,7 @@ def main():
         skipped_lines += 1
         continue
       date = asDate(match.group('timestamp'))
-      hit_per_day[date] += 1
+      hit_per_day[decimator(date)] += 1
       try:
         site_data = per_site[site]
       except KeyError:
@@ -462,7 +476,7 @@ def main():
       out.write('<h1>Site: %s</h1>' % site_id)
       if matplotlib is not None:
         daily_data = data.getApdexData()
-        date_list = [datetime.strptime(x[0], '%Y/%m/%d' + {'hour': ' %H'}.get(
+        date_list = [datetime.strptime(x[0], '%Y/%m/%d' + {'day': ' %H'}.get(
           args.period, '')) for x in daily_data]
         date_range = date_list[-1] - date_list[0]
         if date_range < timedelta(2):
@@ -505,7 +519,7 @@ def main():
         pyplot.clf()
 
         out.write('<img src="' + plot_filename + '" />')
-      out.write(data.asHTML())
+      out.write(data.asHTML(decimator))
     end_stat_time = time.time()
     if args.stats:
       out.write('<h1>Parsing stats</h1><table>')