Factorize test runner code with BenchmarkRunner class.

git-svn-id: https://svn.erp5.org/repos/neo/trunk@2444 71dcc9de-d417-0410-9af5-da40c76e7ee4

neo/tests/benchmark.py

+
+import sys
+import email
+import smtplib
+import optparse
+import platform
+import datetime
+from email.MIMEMultipart import MIMEMultipart
+from email.MIMEText import MIMEText
+
+from neo.tests.functional import NEOCluster
+
+MAIL_SERVER = '127.0.0.1:25'
+
+class AttributeDict(dict):
+
+    def __getattr__(self, item):
+        return self.__getitem__(item)
+
+class BenchmarkRunner(object):
+    """
+        Base class for a command-line benchmark test runner.
+    """
+
+    def __init__(self):
+        self._successful = True
+        self._status = []
+        parser = optparse.OptionParser()
+        # register common options
+        parser.add_option('', '--title')
+        parser.add_option('-v', '--verbose', action='store_true')
+        parser.add_option('', '--mail-to', action='append')
+        parser.add_option('', '--mail-from')
+        parser.add_option('', '--mail-server')
+        self.add_options(parser)
+        # check common arguments
+        options, self._args = parser.parse_args()
+        if bool(options.mail_to) ^ bool(options.mail_from):
+            sys.exit('Need a sender and recipients to mail report')
+        mail_server = options.mail_server or MAIL_SERVER
+        # check specifics arguments
+        self._config = AttributeDict()
+        self._config.update(self.load_options(options, self._args))
+        self._config.update(dict(
+            title = options.title or self.__class__.__name__,
+            verbose = options.verbose,
+            mail_from = options.mail_from,
+            mail_to = options.mail_to,
+            mail_server = mail_server.split(':'),
+        ))
+
+    def add_status(self, key, value):
+        self._status.append((key, value))
+
+    def build_report(self, content):
+        fmt = "%-20s : %s"
+        status = "\n".join([fmt % item for item in [
+            ('Title', self._config.title),
+            ('Date', datetime.date.today().isoformat()),
+            ('Node', platform.node()),
+            ('Machine', platform.machine()),
+            ('System', platform.system()),
+            ('Python', platform.python_version()),
+        ]])
+        status += '\n\n'
+        status += "\n".join([fmt % item for item in self._status])
+        return "%s\n\n%s" % (status, content)
+
+    def send_report(self, subject, report):
+        # build report
+        # build email
+        msg = MIMEMultipart()
+        msg['Subject'] = '%s: %s' % (self._config.title, subject)
+        msg['From'] = self._config.mail_from
+        msg['To'] = ', '.join(self._config.mail_to)
+        msg['X-ERP5-Tests'] = 'NEO'
+        if self._successful:
+            msg['X-ERP5-Tests-Status'] = 'OK'
+        msg.epilogue = ''
+        msg.attach(MIMEText(report))
+
+        # send it
+        s = smtplib.SMTP()
+        s.connect(*self._config.mail_server)
+        mail = msg.as_string()
+        for recipient in self._config.mail_to:
+            try:
+                s.sendmail(self._config.mail_from, recipient, mail)
+            except smtplib.SMTPRecipientsRefused:
+                print "Mail for %s fails" % recipient
+        s.close()
+
+    def run(self):
+        subject, report = self.start()
+        report = self.build_report(report)
+        if self._config.mail_to:
+            self.send_report(subject, report)
+        print subject
+        print 
+        print report
+
+    def was_successful(self):
+        return self._successful
+
+    def add_options(self, parser):
+        """ Append options to command line parser """
+        raise NotImplementedError
+
+    def load_options(self, options, args):
+        """ Check options and return a configuration dict """
+        raise NotImplementedError
+
+    def start(self):
+        """ Run the test """
+        raise NotImplementedError
+

tools/matrix

@@ -3,28 +3,80 @@
 import sys
 import os
 import math
-import optparse
 import traceback
 from time import time
 
+from neo.tests.benchmark import BenchmarkRunner
 from neo.tests.functional import NEOCluster
 from ZODB.FileStorage import FileStorage
 
-def run(masters, storages, replicas, partitions, datafs, verbose):
+MIN_STORAGES = 1
+MAX_STORAGES = 2
+MIN_REPLICAS = 0
+MAX_REPLICAS = 1
+
+class MatrixImportBenchmark(BenchmarkRunner):
+
+    def add_options(self, parser):
+        parser.add_option('-d', '--datafs')
+        parser.add_option('', '--min-storages')
+        parser.add_option('', '--max-storages')
+        parser.add_option('', '--min-replicas')
+        parser.add_option('', '--max-replicas')
+
+    def load_options(self, options, args):
+        if not options.datafs or not os.path.exists(options.datafs):
+            sys.exit('Missing or wrong data.fs argument')
+        return dict(
+            datafs = options.datafs,
+            min_s = int(options.min_storages or MIN_STORAGES),
+            max_s = int(options.max_storages or MAX_STORAGES),
+            min_r = int(options.min_replicas or MIN_REPLICAS),
+            max_r = int(options.max_replicas or MAX_REPLICAS),
+        )
+
+    def start(self):
+        # build storage (logarithm) & replicas (linear) lists
+        min_s, max_s = self._config.min_s, self._config.max_s
+        min_r, max_r = self._config.min_r, self._config.max_r
+        min_s2 = int(math.log(min_s, 2))
+        max_s2 = int(math.log(max_s, 2))
+        storages = [2 ** x for x in range(min_s2, max_s2 + 1)]
+        if storages[0] < min_s:
+            storages[0] = min_s
+        if storages[-1] < max_s:
+            storages.append(max_s)
+        replicas = range(min_r, max_r + 1)
+        results = self.runMatrix(storages, replicas)
+        return self.buildReport(storages, replicas, results)
+
+    def runMatrix(self, storages, replicas):
+        stats = {}
+        size = float(os.path.getsize(self._config.datafs))
+        for s in storages:
+            for r in [r for r in replicas if r < s]:
+                stats.setdefault(s, {})
+                result = self.runImport(1, s, r, 100)
+                if result is not None:
+                    result = size / result / 1024
+                stats[s][r] = result
+        return stats
+
+    def runImport(self, masters, storages, replicas, partitions):
         print "Import of %s with m=%s, s=%s, r=%s, p=%s" % (
-            datafs, masters, storages, replicas, partitions)
+                self._config.datafs, masters, storages, replicas, partitions)
         # cluster
         neo = NEOCluster(
-        db_list=['test_import_%d' % i for i in xrange(storages)],
+            db_list=['neot_matrix_%d' % i for i in xrange(storages)],
             clear_databases=True,
             partitions=partitions,
             replicas=replicas,
             master_node_count=masters,
-        verbose=verbose,
+            verbose=self._config.verbose,
         )
         # import
         neo_storage = neo.getZODBStorage()
-    dfs_storage = FileStorage(file_name=datafs)
+        dfs_storage = FileStorage(file_name=self._config.datafs)
         neo.start()
         start = time()
         try:
@@ -37,19 +89,12 @@ def run(masters, storages, replicas, partitions, datafs, verbose):
         finally:
             neo.stop()
 
-def runMatrix(datafs, storages, replicas, verbose):
-    stats = {}
-    size = float(os.path.getsize(datafs))
-    for s in storages:
-        for r in [r for r in replicas if r < s]:
-            stats.setdefault(s, {})
-            result = run(1, s, r, 100, datafs, verbose)
-            if result is not None:
-                result = size / result / 1024
-            stats[s][r] = result
-    return stats
-
-def buildReport(storages, replicas, results):
+    def buildReport(self, storages, replicas, results):
+        config = self._config
+        self.add_status('Min storages', config.min_s)
+        self.add_status('Max storages', config.max_s)
+        self.add_status('Min replicas', config.min_r)
+        self.add_status('Max replicas', config.max_r)
         # draw an array with results
         fmt = '|' + '|'.join(['  %8s  '] * (len(replicas) + 1)) + '|\n'
         sep = '+' + '+'.join(['-' * 12] * (len(replicas) + 1)) + '+\n'
@@ -80,81 +125,6 @@ def buildReport(storages, replicas, results):
         summary = 'Matrix : %s ' % (info, )
         return (summary, report)
 
-def sendReport(sender, recipients, server, summary, report):
-    """ Send a mail with the report summary """
-    # XXX: C/C from perfs bench
-    import smtplib
-    from email.MIMEMultipart import MIMEMultipart
-    from email.MIMEText import MIMEText
-
-    # build the email
-    msg = MIMEMultipart()
-    msg['Subject'] = summary
-    msg['From']    = sender
-    msg['To']      = ', '.join(recipients)
-    msg.epilogue = ''
-    msg.attach(MIMEText(report))
-
-    # Send via smtp server
-    s = smtplib.SMTP()
-    s.connect(*server)
-    mail = msg.as_string()
-    for recipient in recipients:
-        try:
-            s.sendmail(sender, recipient, mail)
-        except smtplib.SMTPRecipientsRefused:
-            print "Mail for %s fails" % recipient
-    s.close()
-
 if __name__ == "__main__":
-
-    # options
-    parser = optparse.OptionParser()
-    parser.add_option('-d', '--datafs')
-    parser.add_option('', '--min-storages')
-    parser.add_option('', '--max-storages')
-    parser.add_option('', '--min-replicas')
-    parser.add_option('', '--max-replicas')
-    parser.add_option('', '--recipient', action='append')
-    parser.add_option('', '--sender')
-    parser.add_option('', '--server')
-    parser.add_option('-v', '--verbose', action='store_true')
-    (options, args) = parser.parse_args()
-
-    # check arguments
-    if not options.datafs or not os.path.exists(options.datafs):
-        sys.exit('Missing or wrong data.fs argument')
-    if bool(options.sender) ^ bool(options.recipient):
-        sys.exit('Need a sender and recipients to mail report')
-
-    # parse args
-    min_s = int(options.min_storages or 1)
-    max_s = int(options.max_storages or 2)
-    min_r = int(options.min_replicas or 0)
-    max_r = int(options.max_replicas or 1)
-    datafs = options.datafs
-    mail_server = options.server or '127.0.0.1:25'
-    mail_server = mail_server.split(':')
-    sender = options.sender
-    recipient = options.recipient
-    verbose = options.verbose or False
-
-    # build storage (logarithm) & replicas (linear) lists
-    min_s2 = int(math.log(min_s, 2))
-    max_s2 = int(math.log(max_s, 2))
-    storages = [2 ** x for x in range(min_s2, max_s2 + 1)]
-    if storages[0] < min_s:
-        storages[0] = min_s
-    if storages[-1] < max_s:
-        storages.append(max_s)
-    replicas = range(min_r, max_r + 1)
-    results = runMatrix(datafs, storages, replicas, verbose)
-    summary, report = buildReport(storages, replicas, results)
-
-    print summary
-    print
-    print report
-
-    if options.sender:
-        sendReport(sender, recipient, mail_server, summary, report)
+    MatrixImportBenchmark().run()
 

tools/perfs

@@ -2,18 +2,57 @@
 
 import os
 import sys
-import optparse
 import platform
 import datetime
 from time import time
+from ZODB.FileStorage import FileStorage
 
+from neo.tests.benchmark import BenchmarkRunner
 from neo.tests.functional import NEOCluster
-from neo.client.Storage import Storage
-from ZODB.FileStorage import FileStorage
 from neo.profiling import PROFILING_ENABLED, profiler_decorator, \
     profiler_report
 
-def runImport(neo, datafs):
+class ImportBenchmark(BenchmarkRunner):
+    """ Test import of a datafs """
+
+    def add_options(self, parser):
+        parser.add_option('-d', '--datafs')
+        parser.add_option('-m', '--masters')
+        parser.add_option('-s', '--storages')
+        parser.add_option('-p', '--partitions')
+        parser.add_option('-r', '--replicas')
+
+    def load_options(self, options, args):
+        if not options.datafs or not os.path.exists(options.datafs):
+            sys.exit('Missing or wrong data.fs argument')
+        return dict(
+            datafs = options.datafs,
+            masters = int(options.masters or 1),
+            storages = int(options.storages or 1),
+            partitions = int(options.partitions or 10),
+            replicas = int(options.replicas or 0),
+        )
+
+    def start(self):
+        config = self._config
+        # start neo
+        neo = NEOCluster(
+            db_list=['neot_perfs_%d' % i for i in xrange(config.storages)],
+            clear_databases=True,
+            partitions=config.partitions,
+            replicas=config.replicas,
+            master_node_count=config.masters,
+            verbose=False,
+        )
+
+        # import datafs
+        neo.start()
+        try:
+            return self.buildReport(*self.runImport(neo))
+        finally:
+            neo.stop()
+
+    def runImport(self, neo):
 
         def counter(wrapped, d):
             @profiler_decorator
@@ -27,6 +66,7 @@ def runImport(neo, datafs):
             return wrapper
 
         # open storages clients
+        datafs = self._config.datafs
         neo_storage = neo.getZODBStorage()
         dfs_storage = FileStorage(file_name=datafs)
         dfs_size = os.path.getsize(datafs)
@@ -48,40 +88,29 @@ def runImport(neo, datafs):
         }
         return (dfs_size, elapsed, stats)
 
-def buildReport(config, dfs_size, elapsed, stats):
+    def buildReport(self, dfs_size, elapsed, stats):
         """ build a report for the given import data """
-
-    pat = '%19s | %8s | %5s | %5s | %5s \n'
-    sep = '%19s+%8s+%5s+%5s+%5s\n'
-    sep %= ('-' * 20, '-' * 10) + ('-' * 7, ) * 3
-
+        config = self._config
         dfs_size /= 1024
         size = dfs_size / 1024
         speed = dfs_size / elapsed
 
-    # system
-    report = ' ' * 20 + ' NEO PERF REPORT\n\n'
-    report += "\tDate        : %s\n" % datetime.date.today().isoformat()
-    report += "\tNode        : %s\n" % platform.node()
-    report += "\tProcessor   : %s (%s)\n" % (platform.processor(),
-            platform.architecture()[0])
-    report += "\tSystem      : %s (%s)\n" % (platform.system(),
-            platform.release())
-    report += '\n'
         # configuration
-    report += "\tMasters     : %s\n" % (config['masters'], )
-    report += "\tStorages    : %s\n" % (config['storages'], )
-    report += "\tReplicas    : %s\n" % (config['replicas'], )
-    report += "\tPartitions  : %s\n" % (config['partitions'], )
-    report += '\n'
+        self.add_status('Masters', config.masters)
+        self.add_status('Storages', config.storages)
+        self.add_status('Replicas', config.replicas)
+        self.add_status('Partitions', config.partitions)
+
         # results
-    report += '\n%19s: %6.1f MB' % ('Input size', size)
-    report += '\n%19s: %6d sec' % ('Import duration', elapsed)
-    report += '\n%19s: %6.1f KB/s\n' % ('Average speed', speed)
-    report += '\n\n'
+        self.add_status('Input size', '%-.1f MB' % size)
+        self.add_status('Import duration', '%-d secs' % elapsed)
+        self.add_status('Average speed', '%-.1f KB/s' % speed)
 
         # stats on objects and transactions
-    report += pat % ('', ' num ', 'min/s', 'avg/s', 'max/s')
+        pat = '%19s | %8s | %5s | %5s | %5s \n'
+        sep = '%19s+%8s+%5s+%5s+%5s\n'
+        sep %= ('-' * 20, '-' * 10) + ('-' * 7, ) * 3
+        report = pat % ('', ' num ', 'min/s', 'avg/s', 'max/s')
         for k, v in stats.items():
             report += sep
             s = sum(v)
@@ -89,92 +118,11 @@ def buildReport(config, dfs_size, elapsed, stats):
         report += sep
 
         # build summary
-    summary = 'Neo : %6.1f KB/s (%6.1f MB)' % (speed, size)
-
+        summary = 'Perf : %.1f KB/s (%.1f MB)' % (speed, size)
         return (summary, report)
 
-def sendReport(sender, recipients, server, summary, report):
-    """ Send a mail with the report summary """
-
-    import smtplib
-    from email.MIMEMultipart import MIMEMultipart
-    from email.MIMEText import MIMEText
-
-    # build the email
-    msg = MIMEMultipart()
-    msg['Subject'] = summary
-    msg['From']    = sender
-    msg['To']      = ', '.join(recipients)
-    msg.epilogue = ''
-    msg.attach(MIMEText(report))
-
-    # Send via smtp server
-    s = smtplib.SMTP()
-    s.connect(*server)
-    mail = msg.as_string()
-    for recipient in recipients:
-        try:
-            s.sendmail(sender, recipient, mail)
-        except smtplib.SMTPRecipientsRefused:
-            print "Mail for %s fails" % recipient
-    s.close()
-
 if __name__ == "__main__":
-
-    # handle command line options
-    parser = optparse.OptionParser()
-    parser.add_option('-d', '--datafs')
-    parser.add_option('-m', '--master-count')
-    parser.add_option('-s', '--storage-count')
-    parser.add_option('-p', '--partition-count')
-    parser.add_option('-r', '--replica-count')
-    parser.add_option('', '--recipient', action='append')
-    parser.add_option('', '--sender')
-    parser.add_option('', '--server')
-    (options, args) = parser.parse_args()
-
-    # check arguments
-    if not options.datafs or not os.path.exists(options.datafs):
-        sys.exit('Missing or wrong data.fs argument')
-    if bool(options.sender) ^ bool(options.recipient):
-        sys.exit('Need a sender and recipients to mail report')
-
-    # load options or defaults
-    config = dict(
-        masters = int(options.master_count or 1),
-        storages = int(options.storage_count or 1),
-        partitions = int(options.partition_count or 10),
-        replicas = int(options.replica_count or 0),
-    )
-
-    datafs = options.datafs
-    mail_server = options.server or '127.0.0.1:25'
-    mail_server = mail_server.split(':')
-    sender = options.sender
-    recipient = options.recipient
-
-    # start neo
-    neo = NEOCluster(
-        db_list=['test_import_%d' % i for i in xrange(config['storages'])],
-        clear_databases=True,
-        partitions=config['partitions'],
-        replicas=config['replicas'],
-        master_node_count=config['masters'],
-        verbose=False,
-    )
-
-    # import datafs
-    neo.start()
-    summary, report = buildReport(config, *runImport(neo, datafs))
-    neo.stop()
-
+    ImportBenchmark().run()
     if PROFILING_ENABLED:
         print profiler_report()
 
-    # display and/or send the report
-    print summary
-    print report
-    if options.sender:
-        sendReport(sender, recipient, mail_server, summary, report)
-
-

tools/runner

@@ -17,7 +17,6 @@
 # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
 
 import traceback
-import optparse
 import unittest
 import tempfile
 import logging
@@ -26,6 +25,8 @@ import sys
 import neo
 import os
 
+from neo.tests.benchmark import BenchmarkRunner
+
 # list of test modules
 # each of them have to import its TestCase classes
 UNIT_TEST_MODULES = [ 
@@ -153,7 +154,7 @@ class NeoTestRunner(unittest.TestResult):
 
     def startTest(self, test):
         unittest.TestResult.startTest(self, test)
-        logging.info(" * TEST %s" % test)
+        logging.info(" * TEST %s", test)
         stats = self._getModuleStats(test)
         stats.run += 1
         self.lastStart = time.time()
@@ -179,33 +180,10 @@ class NeoTestRunner(unittest.TestResult):
         stats.failures += 1
         self._updateTimer(stats)
 
-    def _buildSystemInfo(self):
-        import platform
-        import datetime
+    def _buildSummary(self, add_status):
         success = self.testsRun - len(self.errors) - len(self.failures)
-        s = """
-    Title       : %s
-    Date        : %s
-    Node        : %s
-    Machine     : %s
-    System      : %s (%s)
-    Python      : %s
-    Directory   : %s
-    Status      : %7.3f%%
-        """ % (
-            self._title,
-            datetime.date.today().isoformat(),
-            platform.node(),
-            platform.machine(),
-            platform.system(),
-            platform.release(),
-            platform.python_version(),
-            self.temp_directory,
-            success * 100.0 / self.testsRun,
-        )
-        return s
-
-    def _buildSummary(self):
+        add_status('Directory', self.temp_directory)
+        add_status('Status', '%.3f%%' % (success * 100.0 / self.testsRun))
         # visual
         header       = "%25s |   run   | success |  errors |  fails  |   time   \n" % 'Test Module'
         separator    = "%25s-+---------+---------+---------+---------+----------\n" % ('-' * 25)
@@ -213,8 +191,7 @@ class NeoTestRunner(unittest.TestResult):
         group_f      = "%25s |         |         |         |         |          \n" 
         # header
         s = ' ' * 30 + ' NEO TESTS REPORT'
-        s += '\n\n'
-        s += self._buildSystemInfo()
+        s += '\n'
         s += '\n' + header + separator
         group = None
         t_success = 0
@@ -243,7 +220,7 @@ class NeoTestRunner(unittest.TestResult):
         return s
 
     def _buildErrors(self):
-        s = '\n'
+        s = ''
         test_formatter = lambda t: t.id()
         if len(self.errors):
             s += '\nERRORS:\n'
@@ -272,98 +249,53 @@ class NeoTestRunner(unittest.TestResult):
         s += '\n'
         return s
 
-    def build(self):
+    def buildReport(self, add_status):
         self.time = sum([s.time for s in self.modulesStats.values()])
         self.subject = "%s: %s Tests, %s Errors, %s Failures" % (self._title,
             self.testsRun, len(self.errors), len(self.failures))
-        self._summary = self._buildSummary()
-        self._errors = self._buildErrors()
-        self._warnings = self._buildWarnings()
-
-    def sendReport(self, smtp_server, sender, recipients):
-        """ Send a mail with the report summary """
-
-        import smtplib
-        from email.MIMEMultipart import MIMEMultipart
-        from email.MIMEText import MIMEText
-
-        # build the email
-        msg = MIMEMultipart()
-        msg['Subject'] = self.subject
-        msg['From']    = sender
-        msg['To']      = ', '.join(recipients)
-        #msg.preamble = self.subject
-        msg.epilogue = ''
+        summary = self._buildSummary(add_status)
+        errors = self._buildErrors()
+        warnings = self._buildWarnings()
+        report = '\n'.join([summary, errors, warnings])
+        return (self.subject, report)
 
-        # Add custom headers for client side filtering
-        msg['X-ERP5-Tests'] = 'NEO'
-        if self.wasSuccessful():
-          msg['X-ERP5-Tests-Status'] = 'OK'
+class TestRunner(BenchmarkRunner):
 
-        # write the body
-        body = MIMEText(self._summary + self._warnings + self._errors)
-        msg.attach(body)
-
-        # attach the log file
-        if ATTACH_LOG:
-            log = MIMEText(file(LOG_FILE, 'r').read())
-            log.add_header('Content-Disposition', 'attachment', filename=LOG_FILE)
-            msg.attach(log)
-
-        # Send the email via a smtp server
-        s = smtplib.SMTP()
-        s.connect(*mail_server)
-        mail = msg.as_string()
-        for recipient in recipients:
-            try:
-                s.sendmail(sender, recipient, mail)
-            except smtplib.SMTPRecipientsRefused, e:
-                print "Mail for %s fails : %s" % (recipient, e)
-        s.close()
-
-if __name__ == "__main__":
-
-    # handle command line options
-    parser = optparse.OptionParser()
+    def add_options(self, parser):
         parser.add_option('-f', '--functional', action='store_true')
         parser.add_option('-u', '--unit', action='store_true')
         parser.add_option('-z', '--zodb', action='store_true')
-    parser.add_option('', '--recipient', action='append')
-    parser.add_option('', '--sender')
-    parser.add_option('', '--server')
-    parser.add_option('', '--title')
-    (options, args) = parser.parse_args()
 
-    # check arguments
-    if bool(options.sender) ^ bool(options.recipient):
-        sys.exit('Need a sender and recipients to mail report')
+    def load_options(self, options, args):
         if not (options.unit or options.functional or options.zodb or args):
             sys.exit('Nothing to run, please give one of -f, -u, -z')
-    mail_server = options.server or '127.0.0.1:25'
-    mail_server = mail_server.split(':')
+        return dict(
+            unit = options.unit,
+            functional = options.functional,
+            zodb = options.zodb,
+        )
 
+    def start(self):
+        config = self._config
         # run requested tests
-    runner = NeoTestRunner(title=options.title or 'Neo')
+        runner = NeoTestRunner(title=config.title or 'Neo')
         try:
-        if options.unit:
+            if config.unit:
                 runner.run('Unit tests', UNIT_TEST_MODULES)
-        if options.functional:
+            if config.functional:
                 runner.run('Functional tests', FUNC_TEST_MODULES)
-        if options.zodb:
+            if config.zodb:
                 runner.run('ZODB tests', ZODB_TEST_MODULES)
         except KeyboardInterrupt:
+            config['mail_to'] = None
             traceback.print_exc()
-        options.sender = False
-
         # build report
-    runner.build()
-    print runner._errors
-    print runner._warnings
-    print runner._summary
+        self._successful = runner.wasSuccessful()
+        return runner.buildReport(self.add_status)
 
-    # send a mail
-    if options.sender:
-        runner.sendReport(mail_server, options.sender, options.recipient)
-    if not runner.wasSuccessful():
+if __name__ == "__main__":
+    runner = TestRunner()
+    runner.run()
+    if not runner.was_successful():
         sys.exit(1)
     sys.exit(0)