runner 12.9 KB
Newer Older
1
#! /usr/bin/python2.4
2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18
#
# Copyright (C) 2009  Nexedi SA
# 
# This program is free software; you can redistribute it and/or
# modify it under the terms of the GNU General Public License
# as published by the Free Software Foundation; either version 2
# of the License, or (at your option) any later version.
# 
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.

19
import optparse
20
import unittest
21
import tempfile
22 23
import logging
import time
Grégory Wisniewski's avatar
Grégory Wisniewski committed
24
import sys
25
import os
26

27 28
# list of test modules
# each of them have to import its TestCase classes
29
UNIT_TEST_MODULES = [ 
30 31 32 33 34 35 36
    # generic parts
    'neo.tests.testBootstrap',
    'neo.tests.testConnection',
    'neo.tests.testEvent',
    'neo.tests.testHandler',
    'neo.tests.testNodes',
    'neo.tests.testProtocol',
37
    'neo.tests.testDispatcher',
38
    'neo.tests.testUtil',
39 40 41 42 43 44
    'neo.tests.testPT',
    # master application
    'neo.tests.master.testClientHandler',
    'neo.tests.master.testElectionHandler',
    'neo.tests.master.testMasterApp',
    'neo.tests.master.testMasterPT',
45
    'neo.tests.master.testRecovery',
46
    'neo.tests.master.testStorageHandler',
47
    'neo.tests.master.testVerification',
48
    'neo.tests.master.testTransactions',
49 50 51 52 53 54 55 56
    # storage application
    'neo.tests.storage.testClientHandler',
    'neo.tests.storage.testInitializationHandler',
    'neo.tests.storage.testMasterHandler',
    'neo.tests.storage.testStorageApp',
    'neo.tests.storage.testStorageHandler',
    'neo.tests.storage.testStorageMySQLdb',
    'neo.tests.storage.testVerificationHandler',
57
    'neo.tests.storage.testTransactions',
58 59
    # client application
    'neo.tests.client.testClientApp',
60 61
    'neo.tests.client.testMasterHandler',
    'neo.tests.client.testStorageHandler',
62
    'neo.tests.client.testConnectionPool',
63 64
]

65
FUNC_TEST_MODULES = [
66
    'neo.tests.functional.testMaster',
67
    'neo.tests.functional.testClient',
68
    'neo.tests.functional.testCluster',
69
    'neo.tests.functional.testStorage',
70 71
]

72
ZODB_TEST_MODULES = [
73 74 75 76 77 78 79 80 81
    ('neo.tests.zodb.testBasic', 'check'),
    ('neo.tests.zodb.testConflict', 'check'),
    ('neo.tests.zodb.testHistory', 'check'),
    ('neo.tests.zodb.testIterator', 'check'),
    ('neo.tests.zodb.testMT', 'check'),
    # ('neo.tests.zodb.testPack', 'check'),
    ('neo.tests.zodb.testPersistent', 'check'),
    ('neo.tests.zodb.testReadOnly', 'check'),
    ('neo.tests.zodb.testRevision', 'check'),
82
    #('neo.tests.zodb.testRecovery', 'check'),
83 84 85
    ('neo.tests.zodb.testSynchronization', 'check'),
    # ('neo.tests.zodb.testVersion', 'check'),
    ('neo.tests.zodb.testUndo', 'check'),
86 87 88
    ('neo.tests.zodb.testZODB', 'check'),
]

89
# configuration 
90 91 92
CONSOLE_LOG = False
ATTACH_LOG = False # for ZODB test, only the client side is logged
LOG_FILE = 'neo.log' 
93 94 95 96 97 98 99 100 101

# override logging configuration to send all messages to a file
logger = logging.getLogger()
logger.setLevel(logging.INFO)
handler = logging.FileHandler(LOG_FILE, 'w+')
format='[%(module)12s:%(levelname)s:%(lineno)3d] %(message)s'
formatter = logging.Formatter(format)
handler.setFormatter(formatter)
logger.addHandler(handler)
102
# enabled console logging if desired
103 104 105 106 107
if CONSOLE_LOG:
    handler = logging.StreamHandler()
    handler.setFormatter(formatter)
    logger.addHandler(handler)

108
class NeoTestRunner(unittest.TestResult):
109 110
    """ Custom result class to build report with statistics per module """

111 112 113
    def __init__(self):
        unittest.TestResult.__init__(self)
        self.modulesStats = {}
114
        self.failedImports = {}
115
        self.lastStart = None
116 117 118
        self.temp_directory = tempfile.mkdtemp(prefix='neo_')
        os.environ['TEMP'] = self.temp_directory
        print "Base directory : %s" % (self.temp_directory, )
119

120
    def run(self, name, modules):
121
        print '\n', name
122 123 124
        suite = unittest.TestSuite()
        loader = unittest.defaultTestLoader
        for test_module in modules:
125 126 127 128 129 130
            # load prefix if supplied
            if isinstance(test_module, tuple):
                test_module, prefix = test_module
                loader.testMethodPrefix = prefix
            else:
                loader.testMethodPrefix = 'test'
131 132 133
            try:
                test_module = __import__(test_module, globals(), locals(), ['*'])
            except ImportError, err:
134
                self.failedImports[test_module] = err
135 136 137 138 139
                print "Import of %s failed : %s" % (test_module, err)
                continue
            suite.addTests(loader.loadTestsFromModule(test_module))
        suite.run(self)

140
    class ModuleStats(object):
141
        run = 0
142 143 144 145 146 147
        errors = 0
        success = 0
        failures = 0
        time = 0.0

    def _getModuleStats(self, test):
148 149
        module = test.__class__.__module__
        module = tuple(module.split('.'))
150 151 152 153 154 155 156 157 158 159
        try:
            return self.modulesStats[module] 
        except KeyError:
            self.modulesStats[module] = self.ModuleStats()
            return self.modulesStats[module]

    def _updateTimer(self, stats):
        stats.time += time.time() - self.lastStart

    def startTest(self, test):
160 161 162
        print ' * %s %s...' % (test.__class__.__module__,
                test._TestCase__testMethodName),
        sys.stdout.flush()
163 164 165 166
        unittest.TestResult.startTest(self, test)
        module = test.__class__.__name__
        method = test._TestCase__testMethodName
        logging.info(" * TEST %s" % test)
167 168
        stats = self._getModuleStats(test)
        stats.run += 1
169 170 171
        self.lastStart = time.time()

    def addSuccess(self, test):
172
        print "OK"
173 174 175 176 177 178
        unittest.TestResult.addSuccess(self, test)
        stats = self._getModuleStats(test)
        stats.success += 1
        self._updateTimer(stats)

    def addError(self, test, err):
179
        print "ERROR"
180 181 182 183 184 185
        unittest.TestResult.addError(self, test, err)
        stats = self._getModuleStats(test)
        stats.errors += 1
        self._updateTimer(stats)

    def addFailure(self, test, err):
186
        print "FAIL"
187 188 189 190 191
        unittest.TestResult.addFailure(self, test, err)
        stats = self._getModuleStats(test)
        stats.failures += 1
        self._updateTimer(stats)

192 193 194
    def _buildSystemInfo(self):
        import platform
        import datetime
195
        success = self.testsRun - len(self.errors) - len(self.failures)
196 197 198 199 200
        s = """
    Date        : %s
    Node        : %s
    Processor   : %s (%s)
    System      : %s (%s)
201
    Directory   : %s
202
    Status      : %7.3f%%
203 204 205 206 207 208 209
        """ % (
            datetime.date.today().isoformat(),
            platform.node(),
            platform.processor(),
            platform.architecture()[0],
            platform.system(),
            platform.release(),
210
            self.temp_directory,
211
            success * 100.0 / self.testsRun,
212 213 214
        )
        return s

215
    def _buildSummary(self):
216 217 218 219 220 221
        # visual 
        header       = "%25s |   run   | success |  errors |  fails  |   time   \n" % 'Test Module'
        separator    = "%25s-+---------+---------+---------+---------+----------\n" % ('-' * 25)
        format       = "%25s |   %3s   |   %3s   |   %3s   |   %3s   | %6.2fs   \n"
        group_f      = "%25s |         |         |         |         |          \n" 
        # header
222
        s = ' ' * 30 + ' NEO TESTS REPORT'
223
        s += '\n\n'
224
        s += self._buildSystemInfo()
225 226 227 228 229 230 231 232 233 234 235 236
        s += '\n' + header + separator
        group = None
        t_success = 0
        # for each test case
        for k, v in sorted(self.modulesStats.items()):
            # display group below its content
            _group = '.'.join(k[:-1])
            if group is None:
                group = _group
            if _group != group:
                s += separator + group_f % group + separator
                group = _group
237 238 239 240 241 242 243
            # test case stats
            t_success += v.success
            run, success = v.run or '.', v.success or '.'
            errors, failures = v.errors or '.', v.failures or '.'
            name = k[-1].lstrip('test')
            args = (name, run, success, errors, failures, v.time)
            s += format % args
244 245 246
        # the last group
        s += separator  + group_f % group + separator
        # the final summary
247
        errors, failures = len(self.errors) or '.', len(self.failures) or '.'
248 249
        args = ("Summary", self.testsRun, t_success, errors, failures, self.time)
        s += format % args + separator + '\n'
250 251 252 253
        return s

    def _buildErrors(self):
        s = '\n'
254 255 256 257 258
        test_formatter = lambda t: '%s.py %s.%s' % (
            t.__class__.__module__.replace('.', '/'),
            t.__class__.__name__,
            t._TestCase__testMethodName,
        )
259 260 261
        if len(self.errors):
            s += '\nERRORS:\n'
            for test, trace in self.errors:
262
                s += "%s\n" % test_formatter(test)
263
                s += "-------------------------------------------------------------\n"
264
                s += trace
265
                s += "-------------------------------------------------------------\n"
266 267 268 269
                s += '\n'
        if len(self.failures):
            s += '\nFAILURES:\n'
            for test, trace in self.failures:
270
                s += "%s\n" % test_formatter(test)
271
                s += "-------------------------------------------------------------\n"
272
                s += trace
273
                s += "-------------------------------------------------------------\n"
274 275 276
                s += '\n'
        return s

277 278 279 280 281 282 283 284 285
    def _buildWarnings(self):
        s = '\n'
        if self.failedImports:
            s += 'Failed imports :\n'
            for module, err in self.failedImports.items():
                s += '%s:\n%s' % (module, err)
        s += '\n'
        return s

286 287 288 289
    def build(self):
        self.time = sum([s.time for s in self.modulesStats.values()])
        args = (self.testsRun, len(self.errors), len(self.failures))
        self.subject = "Neo : %s Tests, %s Errors, %s Failures" % args
290 291 292
        self._summary = self._buildSummary()
        self._errors = self._buildErrors()
        self._warnings = self._buildWarnings()
293

Grégory Wisniewski's avatar
Grégory Wisniewski committed
294
    def sendReport(self, smtp_server, sender, recipients):
295 296 297 298 299 300 301 302 303
        """ Send a mail with the report summary """

        import smtplib
        from email.MIMEMultipart import MIMEMultipart
        from email.MIMEText import MIMEText

        # build the email
        msg = MIMEMultipart()
        msg['Subject'] = self.subject
Grégory Wisniewski's avatar
Grégory Wisniewski committed
304 305
        msg['From']    = sender
        msg['To']      = ', '.join(recipients)
306 307 308 309 310 311 312 313 314
        #msg.preamble = self.subject
        msg.epilogue = ''

        # Add custom headers for client side filtering
        msg['X-ERP5-Tests'] = 'NEO'
        if self.wasSuccessful():
          msg['X-ERP5-Tests-Status'] = 'OK'

        # write the body
315
        body = MIMEText(self._summary + self._warnings + self._errors)
316 317 318 319 320 321 322 323
        msg.attach(body)

        # attach the log file
        if ATTACH_LOG:
            log = MIMEText(file(LOG_FILE, 'r').read())
            log.add_header('Content-Disposition', 'attachment', filename=LOG_FILE)
            msg.attach(log)

Grégory Wisniewski's avatar
Grégory Wisniewski committed
324
        # Send the email via a smtp server
325
        s = smtplib.SMTP()
Grégory Wisniewski's avatar
Grégory Wisniewski committed
326
        s.connect(*mail_server)
327
        mail = msg.as_string()
Grégory Wisniewski's avatar
Grégory Wisniewski committed
328
        for recipient in recipients:
329
            try:
Grégory Wisniewski's avatar
Grégory Wisniewski committed
330
                s.sendmail(sender, recipient, mail)
331 332
            except smtplib.SMTPRecipientsRefused, e:
                print "Mail for %s fails : %s" % (recipient, e)
333
        s.close()
334 335

if __name__ == "__main__":
336

337 338 339 340
    # handle command line options
    parser = optparse.OptionParser()
    parser.add_option('-f', '--functional', action='store_true')
    parser.add_option('-u', '--unit', action='store_true')
341
    parser.add_option('-z', '--zodb', action='store_true')
Grégory Wisniewski's avatar
Grégory Wisniewski committed
342 343 344
    parser.add_option('', '--recipient', action='append')
    parser.add_option('', '--sender')
    parser.add_option('', '--server')
345 346
    (options, args) = parser.parse_args()

Grégory Wisniewski's avatar
Grégory Wisniewski committed
347 348 349
    # check arguments
    if bool(options.sender) ^ bool(options.recipient):
        sys.exit('Need a sender and recipients to mail report')
350
    if not (options.unit or options.functional or options.zodb):
Grégory Wisniewski's avatar
Grégory Wisniewski committed
351 352 353
        sys.exit('Nothing to run, please set -f and/or -u flag')
    mail_server = options.server or '127.0.0.1:25'
    mail_server = mail_server.split(':')
354

Grégory Wisniewski's avatar
Grégory Wisniewski committed
355
    # run requested tests
356
    runner = NeoTestRunner()
Grégory Wisniewski's avatar
Grégory Wisniewski committed
357
    if options.unit:
358
        runner.run('Unit tests', UNIT_TEST_MODULES)
Grégory Wisniewski's avatar
Grégory Wisniewski committed
359
    if options.functional:
360
        runner.run('Functional tests', FUNC_TEST_MODULES)
361 362
    if options.zodb:
        runner.run('ZODB tests', ZODB_TEST_MODULES)
Grégory Wisniewski's avatar
Grégory Wisniewski committed
363 364

    # build report
365
    runner.build()
366 367 368
    print runner._errors
    print runner._warnings
    print runner._summary
Grégory Wisniewski's avatar
Grégory Wisniewski committed
369

370
    # send a mail
Grégory Wisniewski's avatar
Grégory Wisniewski committed
371 372
    if options.sender:
        runner.sendReport(mail_server, options.sender, options.recipient)
373 374 375
    if not runner.wasSuccessful():
        sys.exit(1)
    sys.exit(0)