Commit 834ed7d4 authored by Jason Madden's avatar Jason Madden

Convert cache_stats and cache_simul from getopt to argparse.

parent edf0bf07
...@@ -12,14 +12,9 @@ ...@@ -12,14 +12,9 @@
# FOR A PARTICULAR PURPOSE # FOR A PARTICULAR PURPOSE
# #
############################################################################## ##############################################################################
"""Cache simulation. """
Cache simulation.
Usage: simul.py [-s size] tracefile
Options:
-s size: cache size in MB (default 20 MB)
-i: summarizing interval in minutes (default 15; max 60)
-r: rearrange factor
Note: Note:
...@@ -27,102 +22,52 @@ Note: ...@@ -27,102 +22,52 @@ Note:
- The simulation will be far off if the trace file - The simulation will be far off if the trace file
was created starting with a non-empty cache was created starting with a non-empty cache
""" """
from __future__ import print_function from __future__ import print_function, absolute_import
from __future__ import print_function
from __future__ import print_function
from __future__ import print_function
from __future__ import print_function
from __future__ import print_function
from __future__ import print_function
from __future__ import print_function
from __future__ import print_function
from __future__ import print_function
from __future__ import print_function
from __future__ import print_function
import bisect import bisect
import getopt
import struct import struct
import re import re
import sys import sys
import ZEO.cache import ZEO.cache
import argparse
from ZODB.utils import z64
from ZODB.utils import z64, u64 from .cache_stats import add_interval_argument
from .cache_stats import add_tracefile_argument
# we assign ctime locally to facilitate test replacement! # we assign ctime locally to facilitate test replacement!
from time import ctime from time import ctime
import six import six
def usage(msg):
print(msg, file=sys.stderr)
print(__doc__, file=sys.stderr)
def main(args=None): def main(args=None):
if args is None: if args is None:
args = sys.argv[1:] args = sys.argv[1:]
# Parse options. # Parse options.
MB = 1<<20 MB = 1<<20
cachelimit = 20*MB parser = argparse.ArgumentParser(description=__doc__)
rearrange = 0.8 parser.add_argument("--size", "-s",
default=20*MB, dest="cachelimit",
type=lambda s: int(float(s)*MB),
help="cache size in MB (default 20MB)")
add_interval_argument(parser)
parser.add_argument("--rearrange", "-r",
default=0.8, type=float,
help="rearrange factor")
add_tracefile_argument(parser)
simclass = CircularCacheSimulation simclass = CircularCacheSimulation
interval_step = 15
try: options = parser.parse_args(args)
opts, args = getopt.getopt(args, "s:i:r:")
except getopt.error as msg: f = options.tracefile
usage(msg) interval_step = options.interval
return 2
for o, a in opts:
if o == '-s':
cachelimit = int(float(a)*MB)
elif o == '-i':
interval_step = int(a)
elif o == '-r':
rearrange = float(a)
else:
assert False, (o, a)
interval_step *= 60
if interval_step <= 0:
interval_step = 60
elif interval_step > 3600:
interval_step = 3600
if len(args) != 1:
usage("exactly one file argument required")
return 2
filename = args[0]
# Open file.
if filename.endswith(".gz"):
# Open gzipped file.
try:
import gzip
except ImportError:
print("can't read gzipped files (no module gzip)", file=sys.stderr)
return 1
try:
f = gzip.open(filename, "rb")
except IOError as msg:
print("can't open %s: %s" % (filename, msg), file=sys.stderr)
return 1
elif filename == "-":
# Read from stdin.
f = sys.stdin
else:
# Open regular file.
try:
f = open(filename, "rb")
except IOError as msg:
print("can't open %s: %s" % (filename, msg), file=sys.stderr)
return 1
# Create simulation object. # Create simulation object.
sim = simclass(cachelimit, rearrange) sim = simclass(options.cachelimit, options.rearrange)
interval_sim = simclass(cachelimit, rearrange) interval_sim = simclass(options.cachelimit, options.rearrange)
# Print output header. # Print output header.
sim.printheader() sim.printheader()
......
...@@ -14,18 +14,7 @@ from __future__ import print_function ...@@ -14,18 +14,7 @@ from __future__ import print_function
############################################################################## ##############################################################################
"""Trace file statistics analyzer. """Trace file statistics analyzer.
Usage: stats.py [-h] [-i interval] [-q] [-s] [-S] [-v] [-X] tracefile File format:
-h: print histogram of object load frequencies
-i: summarizing interval in minutes (default 15; max 60)
-q: quiet; don't print summaries
-s: print histogram of object sizes
-S: don't print statistics
-v: verbose; print each record
-X: enable heuristic checking for misaligned records: oids > 2**32
will be rejected; this requires the tracefile to be seekable
"""
"""File format:
Each record is 26 bytes, plus a variable number of bytes to store an oid, Each record is 26 bytes, plus a variable number of bytes to store an oid,
with the following layout. Numbers are big-endian integers. with the following layout. Numbers are big-endian integers.
...@@ -58,84 +47,80 @@ i.e. the low bit is always zero. ...@@ -58,84 +47,80 @@ i.e. the low bit is always zero.
""" """
import sys import sys
import time import time
import getopt import argparse
import struct import struct
import gzip
# we assign ctime locally to facilitate test replacement! # we assign ctime locally to facilitate test replacement!
from time import ctime from time import ctime
import six import six
def usage(msg): def add_interval_argument(parser):
print(msg, file=sys.stderr) def _interval(a):
print(__doc__, file=sys.stderr) interval = int(60 * float(a))
if interval <= 0:
interval = 60
elif interval > 3600:
interval = 3600
return interval
parser.add_argument("--interval", "-i",
default=15*60, type=_interval,
help="summarizing interval in minutes (default 15; max 60)")
def add_tracefile_argument(parser):
class GzipFileType(argparse.FileType):
def __init__(self):
super(GzipFileType, self).__init__(mode='rb')
def __call__(self, s):
f = super(GzipFileType, self).__call__(s)
if s.endswith(".gz"):
f = gzip.GzipFile(filename=s, fileobj=f)
return f
parser.add_argument("tracefile", type=GzipFileType(),
help="The trace to read; may be gzipped")
def main(args=None): def main(args=None):
if args is None: if args is None:
args = sys.argv[1:] args = sys.argv[1:]
# Parse options # Parse options
verbose = False parser = argparse.ArgumentParser(description="Trace file statistics analyzer",
quiet = False # Our -h, short for --load-histogram
dostats = True # conflicts with default for help, so we handle
print_size_histogram = False # manually.
print_histogram = False add_help=False)
interval = 15*60 # Every 15 minutes verbose_group = parser.add_mutually_exclusive_group()
heuristic = False verbose_group.add_argument('--verbose', '-v',
try: default=False, action='store_true',
opts, args = getopt.getopt(args, "hi:qsSvX") help="Be verbose; print each record")
except getopt.error as msg: verbose_group.add_argument('--quiet', '-q',
usage(msg) default=False, action='store_true',
return 2 help="Reduce output; don't print summaries")
for o, a in opts: parser.add_argument("--sizes", '-s',
if o == '-h': default=False, action="store_true", dest="print_size_histogram",
print_histogram = True help="print histogram of object sizes")
elif o == "-i": parser.add_argument("--no-stats", '-S',
interval = int(60 * float(a)) default=True, action="store_false", dest="dostats",
if interval <= 0: help="don't print statistics")
interval = 60 parser.add_argument("--load-histogram", "-h",
elif interval > 3600: default=False, action="store_true", dest="print_histogram",
interval = 3600 help="print histogram of object load frequencies")
elif o == "-q": parser.add_argument("--check", "-X",
quiet = True default=False, action="store_true", dest="heuristic",
verbose = False help=" enable heuristic checking for misaligned records: oids > 2**32"
elif o == "-s": " will be rejected; this requires the tracefile to be seekable")
print_size_histogram = True add_interval_argument(parser)
elif o == "-S": add_tracefile_argument(parser)
dostats = False
elif o == "-v": if '--help' in args:
verbose = True parser.print_help()
elif o == '-X': sys.exit(2)
heuristic = True
else: options = parser.parse_args(args)
assert False, (o, opts)
f = options.tracefile
if len(args) != 1:
usage("exactly one file argument required")
return 2
filename = args[0]
# Open file
if filename.endswith(".gz"):
# Open gzipped file
try:
import gzip
except ImportError:
print("can't read gzipped files (no module gzip)", file=sys.stderr)
return 1
try:
f = gzip.open(filename, "rb")
except IOError as msg:
print("can't open %s: %s" % (filename, msg), file=sys.stderr)
return 1
elif filename == '-':
# Read from stdin
f = sys.stdin
else:
# Open regular file
try:
f = open(filename, "rb")
except IOError as msg:
print("can't open %s: %s" % (filename, msg), file=sys.stderr)
return 1
rt0 = time.time() rt0 = time.time()
bycode = {} # map code to count of occurrences bycode = {} # map code to count of occurrences
...@@ -169,7 +154,7 @@ def main(args=None): ...@@ -169,7 +154,7 @@ def main(args=None):
ts, code, oidlen, start_tid, end_tid = unpack(FMT, r) ts, code, oidlen, start_tid, end_tid = unpack(FMT, r)
if ts == 0: if ts == 0:
# Must be a misaligned record caused by a crash. # Must be a misaligned record caused by a crash.
if not quiet: if not options.quiet:
print("Skipping 8 bytes at offset", f.tell() - FMT_SIZE) print("Skipping 8 bytes at offset", f.tell() - FMT_SIZE)
f.seek(f.tell() - FMT_SIZE + 8) f.seek(f.tell() - FMT_SIZE + 8)
continue continue
...@@ -179,14 +164,14 @@ def main(args=None): ...@@ -179,14 +164,14 @@ def main(args=None):
records += 1 records += 1
if t0 is None: if t0 is None:
t0 = ts t0 = ts
thisinterval = t0 // interval thisinterval = t0 // options.interval
h0 = he = ts h0 = he = ts
te = ts te = ts
if ts // interval != thisinterval: if ts // options.interval != thisinterval:
if not quiet: if not options.quiet:
dumpbyinterval(byinterval, h0, he) dumpbyinterval(byinterval, h0, he)
byinterval = {} byinterval = {}
thisinterval = ts // interval thisinterval = ts // options.interval
h0 = ts h0 = ts
he = ts he = ts
dlen, code = (code & 0x7fffff00) >> 8, code & 0xff dlen, code = (code & 0x7fffff00) >> 8, code & 0xff
...@@ -208,7 +193,7 @@ def main(args=None): ...@@ -208,7 +193,7 @@ def main(args=None):
elif code & 0x70 == 0x50: # All stores elif code & 0x70 == 0x50: # All stores
bysizew[dlen] = d = bysizew.get(dlen) or {} bysizew[dlen] = d = bysizew.get(dlen) or {}
d[oid] = d.get(oid, 0) + 1 d[oid] = d.get(oid, 0) + 1
if verbose: if options.verbose:
print("%s %02x %s %016x %016x %c%s" % ( print("%s %02x %s %016x %016x %c%s" % (
ctime(ts)[4:-5], ctime(ts)[4:-5],
code, code,
...@@ -221,12 +206,12 @@ def main(args=None): ...@@ -221,12 +206,12 @@ def main(args=None):
oids[oid] = oids.get(oid, 0) + 1 oids[oid] = oids.get(oid, 0) + 1
total_loads += 1 total_loads += 1
elif code == 0x00: # restart elif code == 0x00: # restart
if not quiet: if not options.quiet:
dumpbyinterval(byinterval, h0, he) dumpbyinterval(byinterval, h0, he)
byinterval = {} byinterval = {}
thisinterval = ts // interval thisinterval = ts // options.interval
h0 = he = ts h0 = he = ts
if not quiet: if not options.quiet:
print(ctime(ts)[4:-5], end=' ') print(ctime(ts)[4:-5], end=' ')
print('='*20, "Restart", '='*20) print('='*20, "Restart", '='*20)
except KeyboardInterrupt: except KeyboardInterrupt:
...@@ -235,7 +220,7 @@ def main(args=None): ...@@ -235,7 +220,7 @@ def main(args=None):
end_pos = f.tell() end_pos = f.tell()
f.close() f.close()
rte = time.time() rte = time.time()
if not quiet: if not options.quiet:
dumpbyinterval(byinterval, h0, he) dumpbyinterval(byinterval, h0, he)
# Error if nothing was read # Error if nothing was read
...@@ -244,7 +229,7 @@ def main(args=None): ...@@ -244,7 +229,7 @@ def main(args=None):
return 1 return 1
# Print statistics # Print statistics
if dostats: if options.dostats:
print() print()
print("Read %s trace records (%s bytes) in %.1f seconds" % ( print("Read %s trace records (%s bytes) in %.1f seconds" % (
addcommas(records), addcommas(end_pos), rte-rt0)) addcommas(records), addcommas(end_pos), rte-rt0))
...@@ -267,7 +252,7 @@ def main(args=None): ...@@ -267,7 +252,7 @@ def main(args=None):
explain.get(code) or "*** unknown code ***")) explain.get(code) or "*** unknown code ***"))
# Print histogram. # Print histogram.
if print_histogram: if options.print_histogram:
print() print()
print("Histogram of object load frequency") print("Histogram of object load frequency")
total = len(oids) total = len(oids)
...@@ -287,7 +272,7 @@ def main(args=None): ...@@ -287,7 +272,7 @@ def main(args=None):
obj_percent, load_percent, cum)) obj_percent, load_percent, cum))
# Print size histogram. # Print size histogram.
if print_size_histogram: if options.print_size_histogram:
print() print()
print("Histograms of object sizes") print("Histograms of object sizes")
print() print()
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment