Commit 934650ec authored by Jim Fulton's avatar Jim Fulton Committed by GitHub

Merge pull request #46 from NextThought/argparse-cache-utils

Convert cache_stats and cache_simul from getopt to argparse.
parents d06e32a8 834ed7d4
...@@ -12,14 +12,9 @@ ...@@ -12,14 +12,9 @@
# FOR A PARTICULAR PURPOSE # FOR A PARTICULAR PURPOSE
# #
############################################################################## ##############################################################################
"""Cache simulation. """
Cache simulation.
Usage: simul.py [-s size] tracefile
Options:
-s size: cache size in MB (default 20 MB)
-i: summarizing interval in minutes (default 15; max 60)
-r: rearrange factor
Note: Note:
...@@ -27,102 +22,52 @@ Note: ...@@ -27,102 +22,52 @@ Note:
- The simulation will be far off if the trace file - The simulation will be far off if the trace file
was created starting with a non-empty cache was created starting with a non-empty cache
""" """
from __future__ import print_function from __future__ import print_function, absolute_import
from __future__ import print_function
from __future__ import print_function
from __future__ import print_function
from __future__ import print_function
from __future__ import print_function
from __future__ import print_function
from __future__ import print_function
from __future__ import print_function
from __future__ import print_function
from __future__ import print_function
from __future__ import print_function
import bisect import bisect
import getopt
import struct import struct
import re import re
import sys import sys
import ZEO.cache import ZEO.cache
import argparse
from ZODB.utils import z64
from ZODB.utils import z64, u64 from .cache_stats import add_interval_argument
from .cache_stats import add_tracefile_argument
# we assign ctime locally to facilitate test replacement! # we assign ctime locally to facilitate test replacement!
from time import ctime from time import ctime
import six import six
def usage(msg):
print(msg, file=sys.stderr)
print(__doc__, file=sys.stderr)
def main(args=None): def main(args=None):
if args is None: if args is None:
args = sys.argv[1:] args = sys.argv[1:]
# Parse options. # Parse options.
MB = 1<<20 MB = 1<<20
cachelimit = 20*MB parser = argparse.ArgumentParser(description=__doc__)
rearrange = 0.8 parser.add_argument("--size", "-s",
default=20*MB, dest="cachelimit",
type=lambda s: int(float(s)*MB),
help="cache size in MB (default 20MB)")
add_interval_argument(parser)
parser.add_argument("--rearrange", "-r",
default=0.8, type=float,
help="rearrange factor")
add_tracefile_argument(parser)
simclass = CircularCacheSimulation simclass = CircularCacheSimulation
interval_step = 15
try: options = parser.parse_args(args)
opts, args = getopt.getopt(args, "s:i:r:")
except getopt.error as msg: f = options.tracefile
usage(msg) interval_step = options.interval
return 2
for o, a in opts:
if o == '-s':
cachelimit = int(float(a)*MB)
elif o == '-i':
interval_step = int(a)
elif o == '-r':
rearrange = float(a)
else:
assert False, (o, a)
interval_step *= 60
if interval_step <= 0:
interval_step = 60
elif interval_step > 3600:
interval_step = 3600
if len(args) != 1:
usage("exactly one file argument required")
return 2
filename = args[0]
# Open file.
if filename.endswith(".gz"):
# Open gzipped file.
try:
import gzip
except ImportError:
print("can't read gzipped files (no module gzip)", file=sys.stderr)
return 1
try:
f = gzip.open(filename, "rb")
except IOError as msg:
print("can't open %s: %s" % (filename, msg), file=sys.stderr)
return 1
elif filename == "-":
# Read from stdin.
f = sys.stdin
else:
# Open regular file.
try:
f = open(filename, "rb")
except IOError as msg:
print("can't open %s: %s" % (filename, msg), file=sys.stderr)
return 1
# Create simulation object. # Create simulation object.
sim = simclass(cachelimit, rearrange) sim = simclass(options.cachelimit, options.rearrange)
interval_sim = simclass(cachelimit, rearrange) interval_sim = simclass(options.cachelimit, options.rearrange)
# Print output header. # Print output header.
sim.printheader() sim.printheader()
......
...@@ -14,18 +14,7 @@ from __future__ import print_function ...@@ -14,18 +14,7 @@ from __future__ import print_function
############################################################################## ##############################################################################
"""Trace file statistics analyzer. """Trace file statistics analyzer.
Usage: stats.py [-h] [-i interval] [-q] [-s] [-S] [-v] [-X] tracefile File format:
-h: print histogram of object load frequencies
-i: summarizing interval in minutes (default 15; max 60)
-q: quiet; don't print summaries
-s: print histogram of object sizes
-S: don't print statistics
-v: verbose; print each record
-X: enable heuristic checking for misaligned records: oids > 2**32
will be rejected; this requires the tracefile to be seekable
"""
"""File format:
Each record is 26 bytes, plus a variable number of bytes to store an oid, Each record is 26 bytes, plus a variable number of bytes to store an oid,
with the following layout. Numbers are big-endian integers. with the following layout. Numbers are big-endian integers.
...@@ -58,84 +47,80 @@ i.e. the low bit is always zero. ...@@ -58,84 +47,80 @@ i.e. the low bit is always zero.
""" """
import sys import sys
import time import time
import getopt import argparse
import struct import struct
import gzip
# we assign ctime locally to facilitate test replacement! # we assign ctime locally to facilitate test replacement!
from time import ctime from time import ctime
import six import six
def usage(msg): def add_interval_argument(parser):
print(msg, file=sys.stderr) def _interval(a):
print(__doc__, file=sys.stderr)
def main(args=None):
if args is None:
args = sys.argv[1:]
# Parse options
verbose = False
quiet = False
dostats = True
print_size_histogram = False
print_histogram = False
interval = 15*60 # Every 15 minutes
heuristic = False
try:
opts, args = getopt.getopt(args, "hi:qsSvX")
except getopt.error as msg:
usage(msg)
return 2
for o, a in opts:
if o == '-h':
print_histogram = True
elif o == "-i":
interval = int(60 * float(a)) interval = int(60 * float(a))
if interval <= 0: if interval <= 0:
interval = 60 interval = 60
elif interval > 3600: elif interval > 3600:
interval = 3600 interval = 3600
elif o == "-q": return interval
quiet = True parser.add_argument("--interval", "-i",
verbose = False default=15*60, type=_interval,
elif o == "-s": help="summarizing interval in minutes (default 15; max 60)")
print_size_histogram = True
elif o == "-S":
dostats = False
elif o == "-v":
verbose = True
elif o == '-X':
heuristic = True
else:
assert False, (o, opts)
if len(args) != 1: def add_tracefile_argument(parser):
usage("exactly one file argument required")
return 2
filename = args[0]
# Open file class GzipFileType(argparse.FileType):
if filename.endswith(".gz"): def __init__(self):
# Open gzipped file super(GzipFileType, self).__init__(mode='rb')
try:
import gzip def __call__(self, s):
except ImportError: f = super(GzipFileType, self).__call__(s)
print("can't read gzipped files (no module gzip)", file=sys.stderr) if s.endswith(".gz"):
return 1 f = gzip.GzipFile(filename=s, fileobj=f)
try: return f
f = gzip.open(filename, "rb")
except IOError as msg: parser.add_argument("tracefile", type=GzipFileType(),
print("can't open %s: %s" % (filename, msg), file=sys.stderr) help="The trace to read; may be gzipped")
return 1
elif filename == '-': def main(args=None):
# Read from stdin if args is None:
f = sys.stdin args = sys.argv[1:]
else: # Parse options
# Open regular file parser = argparse.ArgumentParser(description="Trace file statistics analyzer",
try: # Our -h, short for --load-histogram
f = open(filename, "rb") # conflicts with default for help, so we handle
except IOError as msg: # manually.
print("can't open %s: %s" % (filename, msg), file=sys.stderr) add_help=False)
return 1 verbose_group = parser.add_mutually_exclusive_group()
verbose_group.add_argument('--verbose', '-v',
default=False, action='store_true',
help="Be verbose; print each record")
verbose_group.add_argument('--quiet', '-q',
default=False, action='store_true',
help="Reduce output; don't print summaries")
parser.add_argument("--sizes", '-s',
default=False, action="store_true", dest="print_size_histogram",
help="print histogram of object sizes")
parser.add_argument("--no-stats", '-S',
default=True, action="store_false", dest="dostats",
help="don't print statistics")
parser.add_argument("--load-histogram", "-h",
default=False, action="store_true", dest="print_histogram",
help="print histogram of object load frequencies")
parser.add_argument("--check", "-X",
default=False, action="store_true", dest="heuristic",
help=" enable heuristic checking for misaligned records: oids > 2**32"
" will be rejected; this requires the tracefile to be seekable")
add_interval_argument(parser)
add_tracefile_argument(parser)
if '--help' in args:
parser.print_help()
sys.exit(2)
options = parser.parse_args(args)
f = options.tracefile
rt0 = time.time() rt0 = time.time()
bycode = {} # map code to count of occurrences bycode = {} # map code to count of occurrences
...@@ -169,7 +154,7 @@ def main(args=None): ...@@ -169,7 +154,7 @@ def main(args=None):
ts, code, oidlen, start_tid, end_tid = unpack(FMT, r) ts, code, oidlen, start_tid, end_tid = unpack(FMT, r)
if ts == 0: if ts == 0:
# Must be a misaligned record caused by a crash. # Must be a misaligned record caused by a crash.
if not quiet: if not options.quiet:
print("Skipping 8 bytes at offset", f.tell() - FMT_SIZE) print("Skipping 8 bytes at offset", f.tell() - FMT_SIZE)
f.seek(f.tell() - FMT_SIZE + 8) f.seek(f.tell() - FMT_SIZE + 8)
continue continue
...@@ -179,14 +164,14 @@ def main(args=None): ...@@ -179,14 +164,14 @@ def main(args=None):
records += 1 records += 1
if t0 is None: if t0 is None:
t0 = ts t0 = ts
thisinterval = t0 // interval thisinterval = t0 // options.interval
h0 = he = ts h0 = he = ts
te = ts te = ts
if ts // interval != thisinterval: if ts // options.interval != thisinterval:
if not quiet: if not options.quiet:
dumpbyinterval(byinterval, h0, he) dumpbyinterval(byinterval, h0, he)
byinterval = {} byinterval = {}
thisinterval = ts // interval thisinterval = ts // options.interval
h0 = ts h0 = ts
he = ts he = ts
dlen, code = (code & 0x7fffff00) >> 8, code & 0xff dlen, code = (code & 0x7fffff00) >> 8, code & 0xff
...@@ -208,7 +193,7 @@ def main(args=None): ...@@ -208,7 +193,7 @@ def main(args=None):
elif code & 0x70 == 0x50: # All stores elif code & 0x70 == 0x50: # All stores
bysizew[dlen] = d = bysizew.get(dlen) or {} bysizew[dlen] = d = bysizew.get(dlen) or {}
d[oid] = d.get(oid, 0) + 1 d[oid] = d.get(oid, 0) + 1
if verbose: if options.verbose:
print("%s %02x %s %016x %016x %c%s" % ( print("%s %02x %s %016x %016x %c%s" % (
ctime(ts)[4:-5], ctime(ts)[4:-5],
code, code,
...@@ -221,12 +206,12 @@ def main(args=None): ...@@ -221,12 +206,12 @@ def main(args=None):
oids[oid] = oids.get(oid, 0) + 1 oids[oid] = oids.get(oid, 0) + 1
total_loads += 1 total_loads += 1
elif code == 0x00: # restart elif code == 0x00: # restart
if not quiet: if not options.quiet:
dumpbyinterval(byinterval, h0, he) dumpbyinterval(byinterval, h0, he)
byinterval = {} byinterval = {}
thisinterval = ts // interval thisinterval = ts // options.interval
h0 = he = ts h0 = he = ts
if not quiet: if not options.quiet:
print(ctime(ts)[4:-5], end=' ') print(ctime(ts)[4:-5], end=' ')
print('='*20, "Restart", '='*20) print('='*20, "Restart", '='*20)
except KeyboardInterrupt: except KeyboardInterrupt:
...@@ -235,7 +220,7 @@ def main(args=None): ...@@ -235,7 +220,7 @@ def main(args=None):
end_pos = f.tell() end_pos = f.tell()
f.close() f.close()
rte = time.time() rte = time.time()
if not quiet: if not options.quiet:
dumpbyinterval(byinterval, h0, he) dumpbyinterval(byinterval, h0, he)
# Error if nothing was read # Error if nothing was read
...@@ -244,7 +229,7 @@ def main(args=None): ...@@ -244,7 +229,7 @@ def main(args=None):
return 1 return 1
# Print statistics # Print statistics
if dostats: if options.dostats:
print() print()
print("Read %s trace records (%s bytes) in %.1f seconds" % ( print("Read %s trace records (%s bytes) in %.1f seconds" % (
addcommas(records), addcommas(end_pos), rte-rt0)) addcommas(records), addcommas(end_pos), rte-rt0))
...@@ -267,7 +252,7 @@ def main(args=None): ...@@ -267,7 +252,7 @@ def main(args=None):
explain.get(code) or "*** unknown code ***")) explain.get(code) or "*** unknown code ***"))
# Print histogram. # Print histogram.
if print_histogram: if options.print_histogram:
print() print()
print("Histogram of object load frequency") print("Histogram of object load frequency")
total = len(oids) total = len(oids)
...@@ -287,7 +272,7 @@ def main(args=None): ...@@ -287,7 +272,7 @@ def main(args=None):
obj_percent, load_percent, cum)) obj_percent, load_percent, cum))
# Print size histogram. # Print size histogram.
if print_size_histogram: if options.print_size_histogram:
print() print()
print("Histograms of object sizes") print("Histograms of object sizes")
print() print()
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment