Skip to content

Z
zodb
Commit 437ec8a6 authored by Tim Peters's avatar Tim Peters
Browse files
Options

Merge rev 37385 from 3.4 branch. 

Many little bugfixes and improvements in stats.py.
This has survived several 100 MB of trace files
I generated over the last few days, so it's solid
now if not necessarily perfect.

Replaced simul.py with the much broader-ranging code
Jeremy and I were working on a couple years ago,
although it can't work with the current trace file
format (no real loss there -- the simul.py it's
replacing can't work with the current format either).
parent 6b4c92e0
Hide whitespace changes
Inline Side-by-side
Showing with 1037 additions and 86 deletions
doc/ZEO/trace.txt
View file @ 437ec8a6
......@@ -24,7 +24,7 @@ application server) must be restarted.
The trace file can grow pretty quickly; on a moderately loaded server, we
observed it growing by 5 MB per hour. The file consists of binary records,
each 26 bytes long if 8-byte oids are in use; a detailed description of the
each 34 bytes long if 8-byte oids are in use; a detailed description of the
record lay-out is given in stats.py. No sensitive data is logged: data
record sizes and binary object and transaction ids are logged, but no
information about object types or names, user names, version names,
......
src/ZEO/simul.py
View file @ 437ec8a6
#! /usr/bin/env python
##############################################################################
#
# Copyright (c) 2001, 2002 Zope Corporation and Contributors.
# All Rights Reserved.
#
# This software is subject to the provisions of the Zope Public License,
# Version 2.1 (ZPL). A copy of the ZPL should accompany this distribution.
# Version 2.0 (ZPL). A copy of the ZPL should accompany this distribution.
# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
......@@ -13,7 +14,7 @@
##############################################################################
"""Cache simulation.
Usage: simul.py [-bflyz] [-X] [-s size] tracefile
Usage: simul.py [-bflyz] [-s size] tracefile
Use one of -b, -f, -l, -y or -z select the cache simulator:
-b: buddy system allocator
......@@ -24,8 +25,6 @@ Use one of -b, -f, -l, -y or -z select the cache simulator:
Options:
-s size: cache size in MB (default 20 MB)
-X: enable heuristic checking for misaligned records: oids > 2**32
will be rejected; this requires the tracefile to be seekable
Note: the buddy system allocator rounds the cache size up to a power of 2
"""
......@@ -34,6 +33,9 @@ import sys
import time
import getopt
import struct
import math
from sets import Set
def usage(msg):
print >>sys.stderr, msg
......@@ -44,9 +46,9 @@ def main():
MB = 1000*1000
cachelimit = 20*MB
simclass = ZEOCacheSimulation
heuristic = 0
theta = omicron = None
try:
opts, args = getopt.getopt(sys.argv[1:], "bflyzs:X")
opts, args = getopt.getopt(sys.argv[1:], "bflyz2cOaTUs:o:t:")
except getopt.error, msg:
usage(msg)
return 2
......@@ -63,13 +65,32 @@ def main():
simclass = ZEOCacheSimulation
if o == '-s':
cachelimit = int(float(a)*MB)
if o == '-X':
heuristic = 1
if o == '-2':
simclass = TwoQSimluation
if o == '-c':
simclass = CircularCacheSimulation
if o == '-o':
omicron = float(a)
if o == '-t':
theta = float(a)
if o == '-O':
simclass = OracleSimulation
if o == '-a':
simclass = ARCCacheSimulation
if o == '-T':
simclass = ThorSimulation
if o == '-U':
simclass = UnboundedSimulation
if len(args) != 1:
usage("exactly one file argument required")
return 2
filename = args[0]
if omicron is not None and simclass != CircularCacheSimulation:
usage("-o flag only useful with -c (CircularCacheSimulation)")
return 2
# Open file
if filename.endswith(".gz"):
# Open gzipped file
......@@ -95,7 +116,12 @@ def main():
return 1
# Create simulation object
sim = simclass(cachelimit)
if omicron is not None or theta is not None:
sim = simclass(cachelimit, omicron, theta)
elif simclass is OracleSimulation:
sim = simclass(cachelimit, filename)
else:
sim = simclass(cachelimit)
# Print output header
sim.printheader()
......@@ -107,21 +133,21 @@ def main():
struct_unpack = struct.unpack
while 1:
# Read a record and decode it
r = f_read(10)
if len(r) < 10:
r = f_read(8)
if len(r) < 8:
break
offset += 10
ts, code, lenoid = struct_unpack(">iiH", r)
offset += 8
ts, code = struct_unpack(">ii", r)
if ts == 0:
# Must be a misaligned record caused by a crash
##print "Skipping 8 bytes at offset", offset-8
continue
r = f_read(8 + lenoid)
if len(r) < 8 + lenoid:
r = f_read(16)
if len(r) < 16:
break
offset += 8 + lenoid
offset += 16
records += 1
serial, oid = struct_unpack(">8s%ds" % lenoid, r)
oid, serial = struct_unpack(">8s8s", r)
# Decode the code
dlen, version, code, current = (code & 0x7fffff00,
code & 0x80,
......@@ -153,17 +179,20 @@ class Simulation:
# Initialize global statistics
self.epoch = None
self.total_loads = 0
self.total_hits = 0 # Subclass must increment
self.total_invals = 0
self.total_hits = 0 # subclass must increment
self.total_invals = 0 # subclass must increment
self.total_writes = 0
if not hasattr(self, "extras"):
self.extras = (self.extraname,)
self.format = self.format + " %7s" * len(self.extras)
# Reset per-run statistics and set up simulation data
self.restart()
def restart(self):
# Reset per-run statistics
self.loads = 0
self.hits = 0 # Subclass must increment
self.invals = 0
self.hits = 0 # subclass must increment
self.invals = 0 # subclass must increment
self.writes = 0
self.ts0 = None
......@@ -201,12 +230,14 @@ class Simulation:
pass
def load(self, oid, size):
# Must increment .hits and .total_hits as appropriate.
pass
def inval(self, oid):
# Must increment .invals and .total_invals as appropriate.
pass
format = "%12s %9s %8s %8s %6s %6s %6s %6s"
format = "%12s %9s %8s %8s %6s %6s %7s"
# Subclass should override extraname to name known instance variables;
# if extraname is 'foo', both self.foo and self.total_foo must exist:
......@@ -215,34 +246,46 @@ class Simulation:
def printheader(self):
print "%s, cache size %s bytes" % (self.__class__.__name__,
addcommas(self.cachelimit))
print self.format % (
"START TIME", "DURATION", "LOADS", "HITS",
"INVALS", "WRITES", self.extraname.upper(), "HITRATE")
self.extraheader()
extranames = tuple([s.upper() for s in self.extras])
args = ("START TIME", "DURATION", "LOADS", "HITS",
"INVALS", "WRITES", "HITRATE") + extranames
print self.format % args
def extraheader(self):
pass
nreports = 0
def report(self):
def report(self, extratext=''):
if self.loads:
self.nreports += 1
print self.format % (
time.ctime(self.ts0)[4:-8],
duration(self.ts1 - self.ts0),
self.loads, self.hits, self.invals, self.writes,
getattr(self, self.extraname),
hitrate(self.loads, self.hits))
args = (time.ctime(self.ts0)[4:-8],
duration(self.ts1 - self.ts0),
self.loads, self.hits, self.invals, self.writes,
hitrate(self.loads, self.hits))
args += tuple([getattr(self, name) for name in self.extras])
print self.format % args, extratext
def finish(self):
self.report()
if self.nreports > 1:
print (self.format + " OVERALL") % (
# Make sure that the last line of output ends with "OVERALL". This
# makes it much easier for another program parsing the output to
# find summary statistics.
if self.nreports < 2:
self.report('OVERALL')
else:
self.report()
args = (
time.ctime(self.epoch)[4:-8],
duration(self.ts1 - self.epoch),
self.total_loads,
self.total_hits,
self.total_invals,
self.total_writes,
getattr(self, "total_" + self.extraname),
hitrate(self.total_loads, self.total_hits))
args += tuple([getattr(self, "total_" + name)
for name in self.extras])
print (self.format + " OVERALL") % args
class ZEOCacheSimulation(Simulation):
......@@ -433,6 +476,710 @@ class Node:
self.next = next
prev.next = next.prev = self
am = object()
a1in = object()
a1out = object()
class Node2Q(Node):
__slots__ = ["kind", "hits"]
def __init__(self, oid, size, kind=None):
Node.__init__(self, oid, size)
self.kind = kind
self.hits = 0
def linkbefore(self, next):
if next.kind != self.kind:
self.kind = next.kind
Node.linkbefore(self, next)
class TwoQSimluation(Simulation):
# The original 2Q algorithm is page based and the authors offer
# tuning guidlines based on a page-based cache. Our cache is
# object based, so, for example, it's hard to compute the number
# of oids to store in a1out based on the size of a1in.
extras = "evicts", "hothit", "am_add"
NodeClass = Node2Q
def __init__(self, cachelimit, outlen=10000, threshold=0):
Simulation.__init__(self, cachelimit)
# The promotion threshold: If a hit occurs in a1out, it is
# promoted to am if the number of hits on the object while it
# was in a1in is at least threshold. The standard 2Q scheme
# uses a threshold of 0.
self.threshold = threshold
self.am_limit = 3 * self.cachelimit / 4
self.a1in_limit = self.cachelimit / 4
self.cache = {}
self.am_size = 0
self.a1in_size = 0
self.a1out_size = 0
self.total_evicts = 0
self.total_hothit = 0
self.total_am_add = 0
self.a1out_limit = outlen
# An LRU queue of hot objects
self.am = self.NodeClass(None, None, am)
self.am.linkbefore(self.am)
# A FIFO queue of recently referenced objects. It's purpose
# is to absorb references to objects that are accessed a few
# times in short order, then forgotten about.
self.a1in = self.NodeClass(None, None, a1in)
self.a1in.linkbefore(self.a1in)
# A FIFO queue of recently reference oids.
# This queue only stores the oids, not any data. If we get a
# hit in this queue, promote the object to am.
self.a1out = self.NodeClass(None, None, a1out)
self.a1out.linkbefore(self.a1out)
def makespace(self, size):
for space in 0, size:
if self.enoughspace(size):
return
self.evict_a1in(space)
if self.enoughspace(size):
return
self.evict_am(space)
def enoughspace(self, size):
totalsize = self.a1in_size + self.am_size
return totalsize + size < self.cachelimit
def evict_a1in(self, extra):
while self.a1in_size + extra > self.a1in_limit:
if self.a1in.next is self.a1in:
return
assert self.a1in.next is not None
node = self.a1in.next
self.evicts += 1
self.total_evicts += 1
node.linkbefore(self.a1out)
self.a1out_size += 1
self.a1in_size -= node.size
if self.a1out_size > self.a1out_limit:
assert self.a1out.next is not None
node = self.a1out.next
node.unlink()
del self.cache[node.oid]
self.a1out_size -= 1
def evict_am(self, extra):
while self.am_size + extra > self.am_limit:
if self.am.next is self.am:
return
assert self.am.next is not None
node = self.am.next
self.evicts += 1
self.total_evicts += 1
# This node hasn't been accessed in a while, so just
# forget about it.
node.unlink()
del self.cache[node.oid]
self.am_size -= node.size
def write(self, oid, size):
# A write always follows a read (ZODB doesn't allow blind writes).
# So this write must have followed a recent read of the object.
# Don't change it's position, but do update the size.
# XXX For now, don't evict pages if the new version of the object
# is big enough to require eviction.
node = self.cache.get(oid)
if node is None or node.kind is a1out:
return
if node.kind is am:
self.am_size = self.am_size - node.size + size
node.size = size
else:
self.a1in_size = self.a1in_size - node.size + size
node.size = size
def load(self, oid, size):
node = self.cache.get(oid)
if node is not None:
if node.kind is am:
self.hits += 1
self.total_hits += 1
self.hothit += 1
self.total_hothit += 1
node.hits += 1
node.linkbefore(self.am)
elif node.kind is a1in:
self.hits += 1
self.total_hits += 1
node.hits += 1
elif node.kind is a1out:
self.a1out_size -= 1
if node.hits >= self.threshold:
self.makespace(node.size)
self.am_size += node.size
node.linkbefore(self.am)
self.cache[oid] = node
self.am_add += 1
self.total_am_add += 1
else:
node.unlink()
self.insert(oid, size)
else:
self.insert(oid, size)
def insert(self, oid, size):
# New objects enter the cache via a1in. If they
# are frequently used over a long enough time, they
# will be promoted to am -- but only via a1out.
self.makespace(size)
node = self.NodeClass(oid, size, a1in)
node.linkbefore(self.a1in)
self.cache[oid] = node
self.a1in_size += node.size
def inval(self, oid):
# The original 2Q algorithm didn't have to deal with
# invalidations. My own solution: Move it to the head of
# a1out.
node = self.cache.get(oid)
if node is None:
return
self.invals += 1
self.total_invals += 1
# XXX Should an invalidation to a1out count?
if node.kind is a1out:
return
node.linkbefore(self.a1out)
if node.kind is am:
self.am_size -= node.size
else:
self.a1in_size -= node.size
def restart(self):
Simulation.restart(self)
self.evicts = 0
self.hothit = 0
self.am_add = 0
lruT = object()
lruB = object()
fifoT = object()
fifoB = object()
class ARCCacheSimulation(Simulation):
# Based on the paper ARC: A Self-Tuning, Low Overhead Replacement
# Cache by Nimrod Megiddo and Dharmendra S. Modha, USENIX FAST
# 2003. The paper describes a block-based cache. A lot of the
# details need to be fiddled to work with an object-based cache.
# For size issues, the key insight ended up being conditions
# A.1-A.5 rather than the details of the algorithm in Fig. 4.
extras = "lruThits", "evicts", "p"
def __init__(self, cachelimit):
Simulation.__init__(self, cachelimit)
# There are two pairs of linked lists. Each pair has a top and
# bottom half. The bottom half contains metadata, but not actual
# objects.
# LRU list of frequently used objects
self.lruT = Node2Q(None, None, lruT)
self.lruT.linkbefore(self.lruT)
self.lruT_len = 0
self.lruT_size = 0
self.lruB = Node2Q(None, None, lruB)
self.lruB.linkbefore(self.lruB)
self.lruB_len = 0
self.lruB_size = 0
# FIFO list of objects seen once
self.fifoT = Node2Q(None, None, fifoT)
self.fifoT.linkbefore(self.fifoT)
self.fifoT_len = 0
self.fifoT_size = 0
self.fifoB = Node2Q(None, None, fifoB)
self.fifoB.linkbefore(self.fifoB)
self.fifoB_len = 0
self.fifoB_size = 0
# maps oid to node
self.cache = {}
# The paper says that p should be adjust be 1 as the minimum:
# "The compound effect of such small increments and decrements
# to p s quite profound as we will demonstrated in the next
# section." Not really, as far as I can tell. In my traces
# with a very small cache, it was taking far too long to adjust
# towards favoring some FIFO component. I would guess that the
# chief difference is that our caches are much bigger than the
# ones they experimented with. Their biggest cache had 512K
# entries, while our smallest cache will have 40 times that many
# entries.
self.p = 0
# XXX multiply computed adjustments to p by walk_factor
self.walk_factor = 500
# statistics
self.total_hits = 0
self.total_lruThits = 0
self.total_fifoThits = 0
self.total_evicts = 0
def restart(self):
Simulation.restart(self)
self.hits = 0
self.lruThits = 0
self.fifoThits = 0
self.evicts = 0
def write(self, oid, size):
pass
def replace(self, lruB=False):
self.evicts += 1
self.total_evicts += 1
if self.fifoT_size > self.p or (lruB and self.fifoT_size == self.p):
node = self.fifoT.next
if node is self.fifoT:
return 0
assert node is not self.fifoT, self.stats()
node.linkbefore(self.fifoB)
self.fifoT_len -= 1
self.fifoT_size -= node.size
self.fifoB_len += 1
self.fifoB_size += node.size
else:
node = self.lruT.next
if node is self.lruT:
return 0
assert node is not self.lruT, self.stats()
node.linkbefore(self.lruB)
self.lruT_len -= 1
self.lruT_size -= node.size
self.lruB_len += 1
self.lruB_size += node.size
return node.size
def stats(self):
self.totalsize = self.lruT_size + self.fifoT_size
self.allsize = self.totalsize + self.lruB_size + self.fifoB_size
print "cachelimit = %s totalsize=%s allsize=%s" % (
addcommas(self.cachelimit),
addcommas(self.totalsize),
addcommas(self.allsize))
fmt = (
"p=%(p)d\n"
"lruT = %(lruT_len)5d / %(lruT_size)8d / %(lruThits)d\n"
"fifoT = %(fifoT_len)5d / %(fifoT_size)8d / %(fifoThits)d\n"
"lruB = %(lruB_len)5d / %(lruB_size)8d\n"
"fifoB = %(fifoB_len)5d / %(fifoB_size)8d\n"
"loads=%(loads)d hits=%(hits)d evicts=%(evicts)d\n"
)
print fmt % self.__dict__
def report(self):
self.total_p = self.p
Simulation.report(self)
## self.stats()
def load(self, oid, size):
## maybe(self.stats, p=0.002)
node = self.cache.get(oid)
if node is None:
# cache miss: We're going to insert a new object in fifoT.
# If fifo is full, we'll need to evict something to make
# room for it.
prev = need = size
while need > 0:
if size + self.fifoT_size + self.fifoB_size >= self.cachelimit:
if need + self.fifoT_size >= self.cachelimit:
node = self.fifoB.next
assert node is not self.fifoB, self.stats()
node.unlink()
del self.cache[node.oid]
self.fifoB_size -= node.size
self.fifoB_len -= 1
self.evicts += 1
self.total_evicts += 1
else:
node = self.fifoB.next
assert node is not self.fifoB, self.stats()
node.unlink()
del self.cache[node.oid]
self.fifoB_size -= node.size
self.fifoB_len -= 1
if self.fifoT_size + self.lruT_size > self.cachelimit:
need -= self.replace()
else:
incache_size = self.fifoT_size + self.lruT_size + need
total_size = (incache_size + self.fifoB_size
+ self.lruB_size)
if total_size >= self.cachelimit * 2:
node = self.lruB.next
if node is self.lruB:
break
assert node is not self.lruB
node.unlink()
del self.cache[node.oid]
self.lruB_size -= node.size
self.lruB_len -= 1
elif incache_size > self.cachelimit:
need -= self.replace()
else:
break
if need == prev:
# XXX hack, apparently we can't get rid of anything else
break
prev = need
node = Node2Q(oid, size)
node.linkbefore(self.fifoT)
self.fifoT_len += 1
self.fifoT_size += size
self.cache[oid] = node
else:
# a cache hit, but possibly in a bottom list that doesn't
# actually hold the object
if node.kind is lruT:
node.linkbefore(self.lruT)
self.hits += 1
self.total_hits += 1
self.lruThits += 1
self.total_lruThits += 1
elif node.kind is fifoT:
node.linkbefore(self.lruT)
self.fifoT_len -= 1
self.lruT_len += 1
self.fifoT_size -= node.size
self.lruT_size += node.size
self.hits += 1
self.total_hits += 1
self.fifoThits += 1
self.total_fifoThits += 1
elif node.kind is fifoB:
node.linkbefore(self.lruT)
self.fifoB_len -= 1
self.lruT_len += 1
self.fifoB_size -= node.size
self.lruT_size += node.size
# XXX need a better min than 1?
## print "adapt+", max(1, self.lruB_size // self.fifoB_size)
delta = max(1, self.lruB_size / max(1, self.fifoB_size))
self.p += delta * self.walk_factor
if self.p > self.cachelimit:
self.p = self.cachelimit
need = node.size
if self.lruT_size + self.fifoT_size + need > self.cachelimit:
while need > 0:
r = self.replace()
if not r:
break
need -= r
elif node.kind is lruB:
node.linkbefore(self.lruT)
self.lruB_len -= 1
self.lruT_len += 1
self.lruB_size -= node.size
self.lruT_size += node.size
# XXX need a better min than 1?
## print "adapt-", max(1, self.fifoB_size // self.lruB_size)
delta = max(1, self.fifoB_size / max(1, self.lruB_size))
self.p -= delta * self.walk_factor
if self.p < 0:
self.p = 0
need = node.size
if self.lruT_size + self.fifoT_size + need > self.cachelimit:
while need > 0:
r = self.replace(lruB=True)
if not r:
break
need -= r
def inval(self, oid):
pass
def extraheader(self):
pass
class OracleSimulation(LRUCacheSimulation):
# Not sure how to implement this yet. This is a cache where I
# cheat to see how good we could actually do. The cache
# replacement problem for multi-size caches is NP-hard, so we're
# not going to have an optimal solution.
# At the moment, the oracle is mostly blind. It knows which
# objects will be referenced more than once, so that it can
# ignore objects referenced only once. In most traces, these
# objects account for about 20% of references.
def __init__(self, cachelimit, filename):
LRUCacheSimulation.__init__(self, cachelimit)
self.count = {}
self.scan(filename)
def load(self, oid, size):
node = self.cache.get(oid)
if node is not None:
self.hits += 1
self.total_hits += 1
node.linkbefore(self.head)
else:
if oid in self.count:
self.write(oid, size)
def scan(self, filename):
# scan the file in advance to figure out which objects will
# be referenced more than once.
f = open(filename, "rb")
struct_unpack = struct.unpack
f_read = f.read
offset = 0
while 1:
# Read a record and decode it
r = f_read(8)
if len(r) < 8:
break
offset += 8
ts, code = struct_unpack(">ii", r)
if ts == 0:
# Must be a misaligned record caused by a crash
##print "Skipping 8 bytes at offset", offset-8
continue
r = f_read(16)
if len(r) < 16:
break
offset += 16
oid, serial = struct_unpack(">8s8s", r)
if code & 0x70 == 0x20:
# only look at loads
self.count[oid] = self.count.get(oid, 0) + 1
all = len(self.count)
# Now remove everything with count == 1
once = [oid for oid, count in self.count.iteritems()
if count == 1]
for oid in once:
del self.count[oid]
print "Scanned file, %d unique oids, %d repeats" % (
all, len(self.count))
class CircularCacheSimulation(Simulation):
# The cache is managed as a single file with a pointer that
# goes around the file, circularly, forever. New objects
# are written at the current pointer, evicting whatever was
# there previously.
# For each cache hit, there is some distance between the current
# pointer offset and the offset of the cached data record. The
# cache can be configured to copy objects to the current offset
# depending on how far away they are now. The omicron parameter
# specifies a percentage
extras = "evicts", "copies", "inuse", "skips"
def __init__(self, cachelimit, omicron=None, skip=None):
Simulation.__init__(self, cachelimit)
self.omicron = omicron or 0
self.skip = skip or 0
self.total_evicts = 0
self.total_copies = 0
self.total_skips = 0
# Current offset in file
self.offset = 0
# Map offset in file to tuple of size, oid
self.filemap = {0: (self.cachelimit, None)}
# Map oid to offset, node
self.cache = {}
# LRU list of oids
self.head = Node(None, None)
self.head.linkbefore(self.head)
def extraheader(self):
print "omicron = %s, theta = %s" % (self.omicron, self.skip)
def restart(self):
Simulation.restart(self)
self.evicts = 0
self.copies = 0
self.skips = 0
def load(self, oid, size):
p = self.cache.get(oid)
if p is None:
self.add(oid, size)
else:
pos, node = p
self.hits += 1
self.total_hits += 1
node.linkbefore(self.head)
self.copy(oid, size, pos)
def check(self):
d = dict(self.filemap)
done = {}
while d:
pos, (size, oid) = d.popitem()
next = pos + size
if not (next in d or next in done or next == self.cachelimit):
print "check", next, pos, size, repr(oid)
self.dump()
raise RuntimeError
done[pos] = pos
def dump(self):
print len(self.filemap)
L = list(self.filemap)
L.sort()
for k in L:
v = self.filemap[k]
print k, v[0], repr(v[1])
def add(self, oid, size):
avail = self.makeroom(size)
assert oid not in self.cache
self.filemap[self.offset] = size, oid
node = Node(oid, size)
node.linkbefore(self.head)
self.cache[oid] = self.offset, node
self.offset += size
# All the space made available must be accounted for in filemap.
excess = avail - size
if excess:
self.filemap[self.offset] = excess, None
def makeroom(self, need):
if self.offset + need > self.cachelimit:
self.offset = 0
pos = self.offset
# Evict enough objects to make the necessary space available.
self.compute_closeenough()
evicted = False
while need > 0:
if pos == self.cachelimit:
print "wrap makeroom", need
pos = 0
try:
size, oid = self.filemap[pos]
except:
self.dump()
raise
if not evicted and self.skip and oid and self.closeenough(oid):
self.skips += 1
self.total_skips += 1
self.offset += size
pos += size
continue
evicted = True
del self.filemap[pos]
if oid is not None:
self.evicts += 1
self.total_evicts += 1
pos, node = self.cache.pop(oid)
node.unlink()
need -= size
pos += size
return pos - self.offset
def compute_closeenough(self):
self.lru = {}
n = int(len(self.cache) * self.skip)
node = self.head.prev
while n > 0:
self.lru[node.oid] = True
node = node.prev
n -= 1
def closeenough(self, oid):
# If oid is in the top portion of the most recently used
# elements, skip it.
return oid in self.lru
def copy(self, oid, size, pos):
# Copy only if the distance is greater than omicron.
dist = self.offset - pos
if dist < 0:
dist += self.cachelimit
if dist < self.omicron * self.cachelimit:
self.copies += 1
self.total_copies += 1
self.filemap[pos] = size, None
pos, node = self.cache.pop(oid)
node.unlink()
self.add(oid, size)
def inval(self, oid):
p = self.cache.get(oid)
if p is None:
return
pos, node = p
self.invals += 1
self.total_invals += 1
size, _oid = self.filemap[pos]
assert oid == _oid
self.filemap[pos] = size, None
pos, node = self.cache.pop(oid)
node.unlink()
def write(self, oid, size):
p = self.cache.get(oid)
if p is None:
return
pos, node = p
oldsize, _oid = self.filemap[pos]
assert oid == _oid
if size == oldsize:
return
if size < oldsize:
excess = oldsize - size
self.filemap[pos] = size, oid
self.filemap[pos + size] = excess, None
else:
self.filemap[pos] = oldsize, None
pos, node = self.cache.pop(oid)
node.unlink()
self.add(oid, size)
def report(self):
free = used = total = 0
for size, oid in self.filemap.itervalues():
total += size
if oid:
used += size
else:
free += size
self.inuse = round(100.0 * used / total, 1)
self.total_inuse = self.inuse
Simulation.report(self)
class BuddyCacheSimulation(LRUCacheSimulation):
def __init__(self, cachelimit):
......@@ -753,5 +1500,218 @@ def addcommas(n):
i -= 3
return sign + s
import random
def maybe(f, p=0.5):
if random.random() < p:
f()
#############################################################################
# Thor-like eviction scheme.
#
# The cache keeps such a list of all objects, and uses a travelling pointer
# to decay the worth of objects over time.
class ThorNode(Node):
__slots__ = ['worth']
def __init__(self, oid, size, worth=None):
Node.__init__(self, oid, size)
self.worth = worth
class ThorListHead(Node):
def __init__(self):
Node.__init__(self, 0, 0)
self.next = self.prev = self
class ThorSimulation(Simulation):
extras = "evicts", "trips"
def __init__(self, cachelimit):
Simulation.__init__(self, cachelimit)
# Maximum total of object sizes we keep in cache.
self.maxsize = cachelimit
# Current total of object sizes in cache.
self.currentsize = 0
# A worth byte maps to a set of all objects with that worth.
# This is cheap to keep updated, and makes finding low-worth
# objects for eviction trivial (just march over the worthsets
# list, in order).
self.worthsets = [Set() for dummy in range(256)]
# We keep a circular list of all objects in cache. currentobj
# walks around it forever. Each time _tick() is called, the
# worth of currentobj is decreased, basically by shifting
# right 1, and currentobj moves on to the next object. When
# an object is first inserted, it enters the list right before
# currentobj. When an object is accessed, its worth is
# increased by or'ing in 0x80. This scheme comes from the
# Thor system, and is an inexpensive way to account for both
# recency and frequency of access: recency is reflected in
# the leftmost bit set, and frequency by how many bits are
# set.
#
# Note: because evictions are interleaved with ticks,
# unlinking an object is tricky, lest we evict currentobj. The
# class _unlink method takes care of this properly.
self.listhead = ThorListHead()
self.currentobj = self.listhead
# Map an object.oid to its ThorNode.
self.oid2object = {}
self.total_evicts = self.total_trips = 0
# Unlink object from the circular list, taking care not to lose
# track of the current object. Always call this instead of
# invoking obj.unlink() directly.
def _unlink(self, obj):
assert obj is not self.listhead
if obj is self.currentobj:
self.currentobj = obj.next
obj.unlink()
# Change obj.worth to newworth, maintaining invariants.
def _change_worth(self, obj, newworth):
if obj.worth != newworth:
self.worthsets[obj.worth].remove(obj)
obj.worth = newworth
self.worthsets[newworth].add(obj)
def add(self, object):
assert object.oid not in self.oid2object
self.oid2object[object.oid] = object
newsize = self.currentsize + object.size
if newsize > self.maxsize:
self._evictbytes(newsize - self.maxsize)
self.currentsize += object.size
object.linkbefore(self.currentobj)
if object.worth is None:
# Give smaller objects higher initial worth. This favors kicking
# out unreferenced large objects before kicking out unreferenced
# small objects. On real life traces, this is a significant
# win for the hit rate.
object.worth = 32 - int(round(math.log(object.size, 2)))
self.worthsets[object.worth].add(object)
# Decrease the worth of the current object, and advance the
# current object.
def _tick(self):
c = self.currentobj
if c is self.listhead:
c = c.next
if c is self.listhead: # list is empty
return
self.total_trips += 1
self.trips += 1
self._change_worth(c, (c.worth + 1) >> 1)
self.currentobj = c.next
def access(self, oid):
self._tick()
obj = self.oid2object.get(oid)
if obj is None:
return None
self._change_worth(obj, obj.worth | 0x80)
return obj
# Evict objects of least worth first, until at least nbytes bytes
# have been freed.
def _evictbytes(self, nbytes):
for s in self.worthsets:
while s:
if nbytes <= 0:
return
obj = s.pop()
nbytes -= obj.size
self._evictobj(obj)
def _evictobj(self, obj):
self.currentsize -= obj.size
self.worthsets[obj.worth].discard(obj)
del self.oid2object[obj.oid]
self._unlink(obj)
self.evicts += 1
self.total_evicts += 1
def _evict_without_bumping_evict_stats(self, obj):
self._evictobj(obj)
self.evicts -= 1
self.total_evicts -= 1
# Simulator overrides from here on.
def restart(self):
# Reset base class
Simulation.restart(self)
# Reset additional per-run statistics
self.evicts = self.trips = 0
def write(self, oid, size):
obj = self.oid2object.get(oid)
worth = None
if obj is not None:
worth = obj.worth
self._evict_without_bumping_evict_stats(obj)
self.add(ThorNode(oid, size, worth))
def load(self, oid, size):
if self.access(oid) is not None:
self.hits += 1
self.total_hits += 1
else:
self.write(oid, size)
def inval(self, oid):
obj = self.oid2object.get(oid)
if obj is not None:
self.invals += 1
self.total_invals += 1
self._evict_without_bumping_evict_stats(obj)
# Take the "x" off to see additional stats after each restart period.
def xreport(self):
Simulation.report(self)
print 'non-empty worth sets', sum(map(bool, self.worthsets)),
print 'objects', len(self.oid2object),
print 'size', self.currentsize
#############################################################################
# Perfection: What if the cache were unbounded, and never forgot anything?
# This simulator answers that question directly; the cache size parameter
# isn't used.
class UnboundedSimulation(Simulation):
extraname = 'evicts' # for some reason we *have* to define >= 1 extra
def __init__(self, cachelimit):
Simulation.__init__(self, cachelimit)
self.oids = Set()
self.evicts = self.total_evicts = 0
def write(self, oid, size):
self.oids.add(oid)
def load(self, oid, size):
if oid in self.oids:
self.hits += 1
self.total_hits += 1
else:
self.oids.add(oid)
def inval(self, oid):
if oid in self.oids:
self.invals += 1
self.total_invals += 1
self.oids.remove(oid)
if __name__ == "__main__":
sys.exit(main())
src/ZEO/stats.py
View file @ 437ec8a6
......@@ -26,7 +26,7 @@ Usage: stats.py [-h] [-i interval] [-q] [-s] [-S] [-v] [-X] tracefile
"""File format:
Each record is 18 bytes, plus a variable number of bytes to store an oid,
Each record is 26 bytes, plus a variable number of bytes to store an oid,
with the following layout. Numbers are big-endian integers.
Offset Size Contents
......@@ -35,8 +35,9 @@ Offset Size Contents
4 3 data size, in 256-byte increments, rounded up
7 1 code (see below)
8 2 object id length
10 8 serial number
18 variable object id
10 8 start tid
18 8 end tid
26 variable object id
The code at offset 7 packs three fields:
......@@ -131,74 +132,66 @@ def main():
print >> sys.stderr, "can't open %s: %s" % (filename, msg)
return 1
# Read file, gathering statistics, and printing each record if verbose
rt0 = time.time()
# bycode -- map code to count of occurrences
bycode = {}
# records -- number of records
records = 0
# version -- number of records with versions
versions = 0
t0 = te = None
# datarecords -- number of records with dlen set
datarecords = 0
datasize = 0L
# oids -- maps oid to number of times it was loaded
oids = {}
# bysize -- maps data size to number of loads
bysize = {}
# bysize -- maps data size to number of writes
bysizew = {}
bycode = {} # map code to count of occurrences
byinterval = {} # map code to count in current interval
records = 0 # number of trace records read
versions = 0 # number of trace records with versions
datarecords = 0 # number of records with dlen set
datasize = 0L # sum of dlen across records with dlen set
oids = {} # map oid to number of times it was loaded
bysize = {} # map data size to number of loads
bysizew = {} # map data size to number of writes
total_loads = 0
byinterval = {}
thisinterval = None
h0 = he = None
offset = 0
t0 = None # first timestamp seen
te = None # most recent timestamp seen
h0 = None # timestamp at start of current interval
he = None # timestamp at end of current interval
thisinterval = None # generally te//interval
f_read = f.read
struct_unpack = struct.unpack
# Read file, gathering statistics, and printing each record if verbose.
try:
while 1:
r = f_read(8)
r = f_read(8) # timestamp:4 code:4
if len(r) < 8:
break
offset += 8
ts, code = struct_unpack(">ii", r)
if ts == 0:
# Must be a misaligned record caused by a crash
# Must be a misaligned record caused by a crash.
if not quiet:
print "Skipping 8 bytes at offset", offset-8
print "Skipping 8 bytes at offset", f.tell() - 8
continue
r = f_read(18)
if len(r) < 10:
r = f_read(18) # oidlen:2 starttid:8 endtid:8
if len(r) < 18:
break
offset += 10
records += 1
oidlen, start_tid, end_tid = struct_unpack(">H8s8s", r)
oid = f_read(oidlen)
if len(oid) != oidlen:
if len(oid) < oidlen:
break
offset += oidlen
records += 1
if t0 is None:
t0 = ts
thisinterval = t0 / interval
thisinterval = t0 // interval
h0 = he = ts
te = ts
if ts / interval != thisinterval:
if ts // interval != thisinterval:
if not quiet:
dumpbyinterval(byinterval, h0, he)
byinterval = {}
thisinterval = ts / interval
thisinterval = ts // interval
h0 = ts
he = ts
dlen, code = code & 0x7fffff00, code & 0xff
if dlen:
datarecords += 1
datasize += dlen
version = '-'
if code & 0x80:
version = 'V'
versions += 1
code = code & 0x7e
else:
version = '-'
code &= 0x7e
bycode[code] = bycode.get(code, 0) + 1
byinterval[code] = byinterval.get(code, 0) + 1
if dlen:
......@@ -220,11 +213,11 @@ def main():
if code & 0x70 == 0x20:
oids[oid] = oids.get(oid, 0) + 1
total_loads += 1
if code == 0x00:
if code == 0x00: # restart
if not quiet:
dumpbyinterval(byinterval, h0, he)
byinterval = {}
thisinterval = ts / interval
thisinterval = ts // interval
h0 = he = ts
if not quiet:
print time.ctime(ts)[4:-5],
......@@ -232,6 +225,7 @@ def main():
except KeyboardInterrupt:
print "\nInterrupted. Stats so far:\n"
end_pos = f.tell()
f.close()
rte = time.time()
if not quiet:
......@@ -245,8 +239,8 @@ def main():
# Print statistics
if dostats:
print
print "Read %s records (%s bytes) in %.1f seconds" % (
addcommas(records), addcommas(records*24), rte-rt0)
print "Read %s trace records (%s bytes) in %.1f seconds" % (
addcommas(records), addcommas(end_pos), rte-rt0)
print "Versions: %s records used a version" % addcommas(versions)
print "First time: %s" % time.ctime(t0)
print "Last time: %s" % time.ctime(te)
......@@ -309,9 +303,8 @@ def dumpbysize(bysize, how, how2):
loads)
def dumpbyinterval(byinterval, h0, he):
loads = 0
hits = 0
for code in byinterval.keys():
loads = hits = 0
for code in byinterval:
if code & 0x70 == 0x20:
n = byinterval[code]
loads += n
......@@ -328,8 +321,7 @@ def dumpbyinterval(byinterval, h0, he):
addcommas(loads), addcommas(hits), hr)
def hitrate(bycode):
loads = 0
hits = 0
loads = hits = 0
for code in bycode:
if code & 0x70 == 0x20:
n = bycode[code]
......@@ -389,7 +381,6 @@ explain = {
0x50: "store (version)",
0x52: "store (current, non-version)",
0x54: "store (non-current)",
}
if __name__ == "__main__":
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment
GitLab Nexedi Edition | About GitLab | About Nexedi | 沪ICP备2021021310号-2 | 沪ICP备2021021310号-7