Commit a03527ea authored by Tim Peters's avatar Tim Peters

Improve thread-death error reporting in tests.

Reworked the way some of the ZEO tests use threads, so that unittest is
more likely to notice the real cause of a failure (which usually occurs in
a thread), and less likely to latch on to spurious problems resulting from
the real failure.

Detail:  The TestThread class got the (unittest) test case as an argument,
and remembered it.  If the run() method saw the thread die with an
exception, it called testcase.fail() with the formatted exception.  But
this can't work as apparently intended:  unittest's fail() raises an
AssertionError then, and the piece of unittest running the test expects
to catch that exception and append the message to the result object's
list of failures.  But the piece of unittest running the test isn't on
*this* thread's call stack!  It's on the main thread's call stack.
So unittest simply raised an exception that wasn't noticed by anything,
and the thread died then, leaving a traceback somewhere in the middle
of the output.  unittest didn't know anything about that, and sometimes
thought the test passed despite the thread failure.

Alas, when a thread dies like this it's also got a decent chance of
leaving things in a state where the test's final checking code can't
succeed, but the checking code doesn't know the thread died either,
and spurious unittest failures and errors could get reported then.

Reworked things so that only the main thread ever tries to tell
unittest that a test failed.  A TestThread remembers an exception-
death now, but doesn't report it until the main thread tries to join
it.  This way unittest knows the test failed; records and reports
the true cause of failure; and, because tests generally join all their
threads before doing their final checking, unittest stops the test
before final checking if a thread death occurred, and so there's
less chance of seeing reports of spurious errors and failures.
parent c1071c06
...@@ -52,9 +52,14 @@ Test suite ...@@ -52,9 +52,14 @@ Test suite
Collector #1397: testTimeStamp fails on FreeBSD Collector #1397: testTimeStamp fails on FreeBSD
The *BSD distributions are unique in that their mktime() implementation The *BSD distributions are unique in that their mktime()
usually ignores the input tm_isdst value. Test checkFullTimeStamp() implementation usually ignores the input tm_isdst value. Test
was sensitive to this platform quirk. checkFullTimeStamp() was sensitive to this platform quirk.
Reworked the way some of the ZEO tests use threads, so that unittest is
more likely to notice the real cause of a failure (which usually occurs in
a thread), and less likely to latch on to spurious problems resulting from
the real failure.
What's new in ZODB3 3.3 beta 1 What's new in ZODB3 3.3 beta 1
......
...@@ -35,12 +35,12 @@ class WorkerThread(TestThread): ...@@ -35,12 +35,12 @@ class WorkerThread(TestThread):
# run the entire test in a thread so that the blocking call for # run the entire test in a thread so that the blocking call for
# tpc_vote() doesn't hang the test suite. # tpc_vote() doesn't hang the test suite.
def __init__(self, testcase, storage, trans, method="tpc_finish"): def __init__(self, storage, trans, method="tpc_finish"):
self.storage = storage self.storage = storage
self.trans = trans self.trans = trans
self.method = method self.method = method
self.ready = threading.Event() self.ready = threading.Event()
TestThread.__init__(self, testcase) TestThread.__init__(self)
def testrun(self): def testrun(self):
try: try:
...@@ -115,7 +115,7 @@ class CommitLockTests: ...@@ -115,7 +115,7 @@ class CommitLockTests:
txn = transaction.Transaction() txn = transaction.Transaction()
tid = self._get_timestamp() tid = self._get_timestamp()
t = WorkerThread(self, storage, txn) t = WorkerThread(storage, txn)
self._threads.append(t) self._threads.append(t)
t.start() t.start()
t.ready.wait() t.ready.wait()
......
...@@ -49,6 +49,7 @@ class FailableThread(TestThread): ...@@ -49,6 +49,7 @@ class FailableThread(TestThread):
# - self.stop attribute (an event) # - self.stop attribute (an event)
# - self._testrun() method # - self._testrun() method
# TestThread.run() invokes testrun().
def testrun(self): def testrun(self):
try: try:
self._testrun() self._testrun()
...@@ -64,8 +65,7 @@ class StressTask: ...@@ -64,8 +65,7 @@ class StressTask:
# to 'tree'. If sleep is given, sleep # to 'tree'. If sleep is given, sleep
# that long after each append. At the end, instance var .added_keys # that long after each append. At the end, instance var .added_keys
# is a list of the ints the thread believes it added successfully. # is a list of the ints the thread believes it added successfully.
def __init__(self, testcase, db, threadnum, startnum, def __init__(self, db, threadnum, startnum, step=2, sleep=None):
step=2, sleep=None):
self.db = db self.db = db
self.threadnum = threadnum self.threadnum = threadnum
self.startnum = startnum self.startnum = startnum
...@@ -132,9 +132,9 @@ class StressThread(FailableThread): ...@@ -132,9 +132,9 @@ class StressThread(FailableThread):
# to 'tree' until Event stop is set. If sleep is given, sleep # to 'tree' until Event stop is set. If sleep is given, sleep
# that long after each append. At the end, instance var .added_keys # that long after each append. At the end, instance var .added_keys
# is a list of the ints the thread believes it added successfully. # is a list of the ints the thread believes it added successfully.
def __init__(self, testcase, db, stop, threadnum, commitdict, def __init__(self, db, stop, threadnum, commitdict,
startnum, step=2, sleep=None): startnum, step=2, sleep=None):
TestThread.__init__(self, testcase) TestThread.__init__(self)
self.db = db self.db = db
self.stop = stop self.stop = stop
self.threadnum = threadnum self.threadnum = threadnum
...@@ -180,9 +180,9 @@ class LargeUpdatesThread(FailableThread): ...@@ -180,9 +180,9 @@ class LargeUpdatesThread(FailableThread):
# more than 25 objects so that it can test code that runs vote # more than 25 objects so that it can test code that runs vote
# in a separate thread when it modifies more than 25 objects. # in a separate thread when it modifies more than 25 objects.
def __init__(self, testcase, db, stop, threadnum, commitdict, startnum, def __init__(self, db, stop, threadnum, commitdict, startnum,
step=2, sleep=None): step=2, sleep=None):
TestThread.__init__(self, testcase) TestThread.__init__(self)
self.db = db self.db = db
self.stop = stop self.stop = stop
self.threadnum = threadnum self.threadnum = threadnum
...@@ -192,15 +192,6 @@ class LargeUpdatesThread(FailableThread): ...@@ -192,15 +192,6 @@ class LargeUpdatesThread(FailableThread):
self.added_keys = [] self.added_keys = []
self.commitdict = commitdict self.commitdict = commitdict
def testrun(self):
try:
self._testrun()
except:
# Report the failure here to all the other threads, so
# that they stop quickly.
self.stop.set()
raise
def _testrun(self): def _testrun(self):
cn = self.db.open() cn = self.db.open()
while not self.stop.isSet(): while not self.stop.isSet():
...@@ -260,9 +251,9 @@ class LargeUpdatesThread(FailableThread): ...@@ -260,9 +251,9 @@ class LargeUpdatesThread(FailableThread):
class VersionStressThread(FailableThread): class VersionStressThread(FailableThread):
def __init__(self, testcase, db, stop, threadnum, commitdict, startnum, def __init__(self, db, stop, threadnum, commitdict, startnum,
step=2, sleep=None): step=2, sleep=None):
TestThread.__init__(self, testcase) TestThread.__init__(self)
self.db = db self.db = db
self.stop = stop self.stop = stop
self.threadnum = threadnum self.threadnum = threadnum
...@@ -272,15 +263,6 @@ class VersionStressThread(FailableThread): ...@@ -272,15 +263,6 @@ class VersionStressThread(FailableThread):
self.added_keys = [] self.added_keys = []
self.commitdict = commitdict self.commitdict = commitdict
def testrun(self):
try:
self._testrun()
except:
# Report the failure here to all the other threads, so
# that they stop quickly.
self.stop.set()
raise
def _testrun(self): def _testrun(self):
commit = 0 commit = 0
key = self.startnum key = self.startnum
...@@ -416,6 +398,13 @@ class InvalidationTests: ...@@ -416,6 +398,13 @@ class InvalidationTests:
break break
# Some thread still hasn't managed to commit anything. # Some thread still hasn't managed to commit anything.
stop.set() stop.set()
# Give all the threads some time to stop before trying to clean up.
# cleanup() will cause the test to fail if some thread ended with
# an uncaught exception, and unittest will call the base class
# tearDown then immediately, but if other threads are still
# running that can lead to a cascade of spurious exceptions.
for t in threads:
t.join(10)
for t in threads: for t in threads:
t.cleanup() t.cleanup()
...@@ -432,8 +421,8 @@ class InvalidationTests: ...@@ -432,8 +421,8 @@ class InvalidationTests:
time.sleep(0.1) time.sleep(0.1)
# Run two threads that update the BTree # Run two threads that update the BTree
t1 = StressTask(self, db1, 1, 1,) t1 = StressTask(db1, 1, 1,)
t2 = StressTask(self, db2, 2, 2,) t2 = StressTask(db2, 2, 2,)
_runTasks(100, t1, t2) _runTasks(100, t1, t2)
cn.sync() cn.sync()
...@@ -458,8 +447,8 @@ class InvalidationTests: ...@@ -458,8 +447,8 @@ class InvalidationTests:
# Run two threads that update the BTree # Run two threads that update the BTree
cd = {} cd = {}
t1 = self.StressThread(self, db1, stop, 1, cd, 1) t1 = self.StressThread(db1, stop, 1, cd, 1)
t2 = self.StressThread(self, db2, stop, 2, cd, 2) t2 = self.StressThread(db2, stop, 2, cd, 2)
self.go(stop, cd, t1, t2) self.go(stop, cd, t1, t2)
while db1.lastTransaction() != db2.lastTransaction(): while db1.lastTransaction() != db2.lastTransaction():
...@@ -487,8 +476,8 @@ class InvalidationTests: ...@@ -487,8 +476,8 @@ class InvalidationTests:
# Run two threads that update the BTree # Run two threads that update the BTree
cd = {} cd = {}
t1 = self.StressThread(self, db1, stop, 1, cd, 1, sleep=0.01) t1 = self.StressThread(db1, stop, 1, cd, 1, sleep=0.01)
t2 = self.StressThread(self, db1, stop, 2, cd, 2, sleep=0.01) t2 = self.StressThread(db1, stop, 2, cd, 2, sleep=0.01)
self.go(stop, cd, t1, t2) self.go(stop, cd, t1, t2)
cn = db1.open() cn = db1.open()
...@@ -516,9 +505,9 @@ class InvalidationTests: ...@@ -516,9 +505,9 @@ class InvalidationTests:
# at the same time. # at the same time.
cd = {} cd = {}
t1 = self.StressThread(self, db1, stop, 1, cd, 1, 3) t1 = self.StressThread(db1, stop, 1, cd, 1, 3)
t2 = self.StressThread(self, db2, stop, 2, cd, 2, 3, 0.01) t2 = self.StressThread(db2, stop, 2, cd, 2, 3, 0.01)
t3 = self.StressThread(self, db2, stop, 3, cd, 3, 3, 0.01) t3 = self.StressThread(db2, stop, 3, cd, 3, 3, 0.01)
self.go(stop, cd, t1, t2, t3) self.go(stop, cd, t1, t2, t3)
while db1.lastTransaction() != db2.lastTransaction(): while db1.lastTransaction() != db2.lastTransaction():
...@@ -552,9 +541,9 @@ class InvalidationTests: ...@@ -552,9 +541,9 @@ class InvalidationTests:
# at the same time. # at the same time.
cd = {} cd = {}
t1 = VersionStressThread(self, db1, stop, 1, cd, 1, 3) t1 = VersionStressThread(db1, stop, 1, cd, 1, 3)
t2 = VersionStressThread(self, db2, stop, 2, cd, 2, 3, 0.01) t2 = VersionStressThread(db2, stop, 2, cd, 2, 3, 0.01)
t3 = VersionStressThread(self, db2, stop, 3, cd, 3, 3, 0.01) t3 = VersionStressThread(db2, stop, 3, cd, 3, 3, 0.01)
self.go(stop, cd, t1, t2, t3) self.go(stop, cd, t1, t2, t3)
while db1.lastTransaction() != db2.lastTransaction(): while db1.lastTransaction() != db2.lastTransaction():
...@@ -591,9 +580,9 @@ class InvalidationTests: ...@@ -591,9 +580,9 @@ class InvalidationTests:
# at the same time. # at the same time.
cd = {} cd = {}
t1 = LargeUpdatesThread(self, db1, stop, 1, cd, 1, 3, 0.02) t1 = LargeUpdatesThread(db1, stop, 1, cd, 1, 3, 0.02)
t2 = LargeUpdatesThread(self, db2, stop, 2, cd, 2, 3, 0.01) t2 = LargeUpdatesThread(db2, stop, 2, cd, 2, 3, 0.01)
t3 = LargeUpdatesThread(self, db2, stop, 3, cd, 3, 3, 0.01) t3 = LargeUpdatesThread(db2, stop, 3, cd, 3, 3, 0.01)
self.go(stop, cd, t1, t2, t3) self.go(stop, cd, t1, t2, t3)
while db1.lastTransaction() != db2.lastTransaction(): while db1.lastTransaction() != db2.lastTransaction():
......
...@@ -13,30 +13,44 @@ ...@@ -13,30 +13,44 @@
############################################################################## ##############################################################################
"""A Thread base class for use with unittest.""" """A Thread base class for use with unittest."""
from cStringIO import StringIO
import threading import threading
import traceback import sys
class TestThread(threading.Thread): class TestThread(threading.Thread):
__super_init = threading.Thread.__init__ """Base class for defining threads that run from unittest.
__super_run = threading.Thread.run
def __init__(self, testcase, group=None, target=None, name=None, The subclass should define a testrun() method instead of a run()
args=(), kwargs={}, verbose=None): method.
self.__super_init(group, target, name, args, kwargs, verbose)
Call cleanup() when the test is done with the thread, instead of join().
If the thread exits with an uncaught exception, it's captured and
re-raised when cleanup() is called. cleanup() should be called by
the main thread! Trying to tell unittest that a test failed from
another thread creates a nightmare of timing-depending cascading
failures and missed errors (tracebacks that show up on the screen,
but don't cause unittest to believe the test failed).
cleanup() also joins the thread. If the thread ended without raising
an uncaught exception, and the join doesn't succeed in the timeout
period, then the test is made to fail with a "Thread still alive"
message.
"""
def __init__(self):
threading.Thread.__init__(self)
# In case this thread hangs, don't stop Python from exiting.
self.setDaemon(1) self.setDaemon(1)
self._testcase = testcase self._exc_info = None
def run(self): def run(self):
try: try:
self.testrun() self.testrun()
except Exception: except:
s = StringIO() self._exc_info = sys.exc_info()
traceback.print_exc(file=s)
self._testcase.fail("Exception in thread %s:\n%s\n" %
(self, s.getvalue()))
def cleanup(self, timeout=15): def cleanup(self, timeout=15):
self.join(timeout) self.join(timeout)
if self._exc_info:
raise self._exc_info[0], self._exc_info[1], self._exc_info[2]
if self.isAlive(): if self.isAlive():
self._testcase.fail("Thread did not finish: %s" % self) self._testcase.fail("Thread did not finish: %s" % self)
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment