Merge pull request #355 from zopefoundation/fsdump_fsstats#354

fsdump/fsstats improvements

CHANGES.rst

@@ -5,6 +5,12 @@
 5.6.1 (unreleased)
 ==================
 
+- Readd transaction size information to ``fsdump`` output;
+  adapt `fsstats` to ``fsdump``'s exchanged order for ``size`` and ``class``
+  information in data records;
+  (fixes `#354 <https://github.com/zopefoundation/ZODB/issues/354>_).
+  Make ``fsdump`` callable via Python's ``-m`` command line option.
+
 - Fix UnboundLocalError when running fsoids.py script.
   See `issue 285 <https://github.com/zopefoundation/ZODB/issues/285>`_.
 

src/ZODB/FileStorage/fsdump.py

@@ -23,12 +23,14 @@ from ZODB.utils import u64, get_pickle_metadata
 def fsdump(path, file=None, with_offset=1):
     iter = FileIterator(path)
     for i, trans in enumerate(iter):
+        size = trans._tend - trans._tpos
         if with_offset:
-            print(("Trans #%05d tid=%016x time=%s offset=%d" %
-                  (i, u64(trans.tid), TimeStamp(trans.tid), trans._pos)), file=file)
+            print(("Trans #%05d tid=%016x size=%d time=%s offset=%d" %
+                  (i, u64(trans.tid), size,
+                   TimeStamp(trans.tid), trans._pos)), file=file)
         else:
-            print(("Trans #%05d tid=%016x time=%s" %
-                  (i, u64(trans.tid), TimeStamp(trans.tid))), file=file)
+            print(("Trans #%05d tid=%016x size=%d time=%s" %
+                  (i, u64(trans.tid), size, TimeStamp(trans.tid))), file=file)
         print(("    status=%r user=%r description=%r" %
               (trans.status, trans.user, trans.description)), file=file)
 
@@ -122,3 +124,7 @@ class Dumper(object):
 def main():
     import sys
     fsdump(sys.argv[1])
+
+
+if __name__ == "__main__":
+    main()

src/ZODB/scripts/fsstats.py

@@ -7,7 +7,7 @@ import six
 from six.moves import filter
 
 rx_txn = re.compile("tid=([0-9a-f]+).*size=(\d+)")
-rx_data = re.compile("oid=([0-9a-f]+) class=(\S+) size=(\d+)")
+rx_data = re.compile("oid=([0-9a-f]+) size=(\d+) class=(\S+)")
 
 def sort_byhsize(seq, reverse=False):
     L = [(v.size(), k, v) for k, v in seq]
@@ -31,8 +31,7 @@ class Histogram(dict):
     def median(self):
         # close enough?
         n = self.size() / 2
-        L = self.keys()
-        L.sort()
+        L = sorted(self.keys())
         L.reverse()
         while 1:
             k = L.pop()
@@ -50,11 +49,14 @@ class Histogram(dict):
         return mode
 
     def make_bins(self, binsize):
-        maxkey = max(six.iterkeys(self))
+        try:
+            maxkey = max(six.iterkeys(self))
+        except ValueError:
+            maxkey = 0
         self.binsize = binsize
-        self.bins = [0] * (1 + maxkey / binsize)
+        self.bins = [0] * (1 + maxkey // binsize)
         for k, v in six.iteritems(self):
-            b = k / binsize
+            b = k // binsize
             self.bins[b] += v
 
     def report(self, name, binsize=50, usebins=False, gaps=True, skip=True):
@@ -88,7 +90,7 @@ class Histogram(dict):
             cum += n
             pc = 100 * cum / tot
             print("%6d %6d %3d%% %3d%% %s" % (
-                i * binsize, n, p, pc, "*" * (n / dot)))
+                i * binsize, n, p, pc, "*" * (n // dot)))
         print()
 
 def class_detail(class_size):
@@ -104,7 +106,7 @@ def class_detail(class_size):
     # per class details
     for klass, h in sort_byhsize(six.iteritems(class_size), reverse=True):
         h.make_bins(50)
-        if len(filter(None, h.bins)) == 1:
+        if len(tuple(filter(None, h.bins))) == 1:
             continue
         h.report("Object size for %s" % klass, usebins=True)
 
@@ -138,7 +140,7 @@ def main(path=None):
     objects = 0
     tid = None
 
-    f = open(path, "rb")
+    f = open(path, "r")
     for i, line in enumerate(f):
         if MAX and i > MAX:
             break
@@ -146,7 +148,7 @@ def main(path=None):
             m = rx_data.search(line)
             if not m:
                 continue
-            oid, klass, size = m.groups()
+            oid, size, klass = m.groups()
             size = int(size)
 
             obj_size.add(size)
@@ -178,6 +180,8 @@ def main(path=None):
             objects = 0
 
             txn_bytes.add(size)
+    if objects:
+        txn_objects.add(objects)
     f.close()
 
     print("Summary: %d txns, %d objects, %d revisions" % (

src/ZODB/scripts/tests/test_fsdump_fsstats.py

+##############################################################################
+#
+# Copyright (c) 2021 Zope Foundation and Contributors.
+# All Rights Reserved.
+#
+# This software is subject to the provisions of the Zope Public License,
+# Version 2.1 (ZPL).  A copy of the ZPL should accompany this distribution.
+# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
+# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
+# FOR A PARTICULAR PURPOSE.
+#
+##############################################################################
+
+from ZODB import DB
+from ZODB.scripts.fsstats import rx_data
+from ZODB.scripts.fsstats import rx_txn
+from ZODB.tests.util import TestCase
+from ZODB.tests.util import run_module_as_script
+
+
+class FsdumpFsstatsTests(TestCase):
+    def setUp(self):
+        super(FsdumpFsstatsTests, self).setUp()
+        # create (empty) storage ``data.fs``
+        DB("data.fs").close()
+
+    def test_fsdump(self):
+        run_module_as_script("ZODB.FileStorage.fsdump", ["data.fs"])
+        # verify that ``fsstats`` will understand the output
+        with open("stdout") as f:
+            tno = obno = 0
+            for li in f:
+                if li.startswith("  data"):
+                    m = rx_data.search(li)
+                    if m is None:
+                        continue
+                    oid, size, klass = m.groups()
+                    int(size)
+                    obno += 1
+                elif li.startswith("Trans"):
+                    m = rx_txn.search(li)
+                    if not m:
+                        continue
+                    tid, size = m.groups()
+                    size = int(size)
+                    tno += 1
+        self.assertEqual(tno, 1)
+        self.assertEqual(obno, 1)
+
+    def test_fsstats(self):
+        # The ``fsstats`` output is complex
+        # currently, we just check the first (summary) line
+        run_module_as_script("ZODB.FileStorage.fsdump", ["data.fs"],
+                             "data.dmp")
+        run_module_as_script("ZODB.scripts.fsstats", ["data.dmp"])
+        with open("stdout") as f:
+            self.assertEqual(f.readline().strip(),
+                             "Summary: 1 txns, 1 objects, 1 revisions")
+        
+                    
+                    

src/ZODB/tests/util.py

@@ -16,9 +16,12 @@
 from ZODB.MappingStorage import DB
 
 import atexit
+import doctest
 import os
+import pdb
 import persistent
 import re
+import runpy
 import sys
 import tempfile
 import time
@@ -377,3 +380,28 @@ def with_high_concurrency(f):
             restore()
 
     return _
+
+
+def run_module_as_script(mod, args, stdout="stdout", stderr="stderr"):
+    """run module *mod* as script with arguments *arg*.
+
+    stdout and stderr are redirected to files given by the
+    correcponding parameters.
+
+    The function is usually called in a ``setUp/tearDown`` frame
+    which will remove the created files.
+    """
+    sargv, sout, serr = sys.argv, sys.stdout, sys.stderr
+    s_set_trace = pdb.set_trace
+    try:
+        sys.argv = [sargv[0]] + args
+        sys.stdout = open(stdout, "w")
+        sys.stderr = open(stderr, "w")
+        # to allow debugging
+        pdb.set_trace = doctest._OutputRedirectingPdb(sout)
+        runpy.run_module(mod, run_name="__main__", alter_sys=True)
+    finally:
+        sys.stdout.close()
+        sys.stderr.close()
+        pdb.set_trace = s_set_trace
+        sys.argv, sys.stdout, sys.stderr = sargv, sout, serr