Reduce memory used by URL error details.

Use a Counter instance from the beginning to factorise identical referrers.
Simplify code used to render error detail to HTML.

apachedex/__init__.py

@@ -299,7 +299,8 @@ class GenericSiteStats(object):
     self.error_detail = error_detail
     self.status = defaultdict(partial(defaultdict, int))
     if error_detail:
-      self.error_url_count = defaultdict(partial(defaultdict, list))
+      # status -> url -> referrer -> count
+      self.error_url_count = defaultdict(partial(defaultdict, Counter))
     self.url_apdex = defaultdict(partial(APDEXStats, threshold, getDuration))
     self.apdex = defaultdict(partial(APDEXStats, threshold, getDuration))
     self.user_agent_counter = Counter()
@@ -327,7 +328,7 @@ class GenericSiteStats(object):
     self.status[status][value_date] += 1
     if self.error_detail and statusIsError(status):
       # XXX: can eat memory if there are many errors on many different urls
-      self.error_url_count[status][url].append(match.group('referer'))
+      self.error_url_count[status][url][match.group('referer')] += 1
     self.user_agent_counter[match.group('agent')] += 1
 
   def getApdexData(self):
@@ -400,8 +401,7 @@ class GenericSiteStats(object):
         return sum(referer_counter.itervalues())
       filtered_status_url = defaultdict(partial(defaultdict, dict))
       for status, url_dict in self.error_url_count.iteritems():
-        filtered_status_url[status] = sorted(
-          ((key, Counter(value)) for key, value in url_dict.iteritems()),
+        filtered_status_url[status] = sorted(url_dict.iteritems(),
           key=lambda x: getHitForUrl(x[1]), reverse=True)[:N_ERROR_URL]
       append('<h3>Error detail</h3><table class="stats"><tr><th>status</th>'
         '<th>hits</th><th>url</th><th>referers</th></tr>')
@@ -420,11 +420,8 @@ class GenericSiteStats(object):
             getHitForUrl(referer_counter),
             unquoteToHtml(url, encoding),
             '<br/>'.join('%i: %s' % (hit, unquoteToHtml(referer, encoding))
-              for referer, hit in sorted(
-                referer_counter.iteritems(),
-                key=ITEMGETTER1,
-                reverse=True,
-            )[:N_REFERRER_PER_ERROR_URL]),
+              for referer, hit in referer_counter.most_common(
+                N_REFERRER_PER_ERROR_URL)),
           ))
           append('</tr>')
       append('</table>')
@@ -437,7 +434,9 @@ class GenericSiteStats(object):
     if error_detail:
       error_url_count = result.error_url_count
       for state_status, state_url_dict in state['error_url_count'].iteritems():
-        error_url_count[state_status].update(state_url_dict)
+        url_dict = error_url_count[state_status]
+        for url, counter in state_url_dict.iteritems():
+          url_dict[url].update(counter)
     for attribute_id in ('url_apdex', 'apdex'):
       attribute = getattr(result, attribute_id)
       for key, apdex_state in state[attribute_id].iteritems():
@@ -465,8 +464,8 @@ class GenericSiteStats(object):
     if self.error_detail:
       for status, other_url_dict in other.error_url_count.iteritems():
         url_dict = self.error_url_count[status]
-        for url, referer_list in other_url_dict.iteritems():
-          url_dict[url].extend(referer_list)
+        for url, referer_counter in other_url_dict.iteritems():
+          url_dict[url].update(referer_counter)
     for attribute_id in ('url_apdex', 'apdex'):
       self_attribute = getattr(self, attribute_id)
       for key, apdex_data in getattr(other, attribute_id).iteritems():