runtests.py 96.3 KB
Newer Older
1
#!/usr/bin/env python
2

3 4
from __future__ import print_function

5
import atexit
6 7 8
import os
import sys
import re
9
import gc
10
import heapq
11
import locale
12
import shutil
13
import time
14 15
import unittest
import doctest
16
import operator
Robert Bradshaw's avatar
Robert Bradshaw committed
17
import subprocess
18
import tempfile
19
import traceback
Robert Bradshaw's avatar
Robert Bradshaw committed
20
import warnings
21
import zlib
22
import glob
23
from contextlib import contextmanager
Robert Bradshaw's avatar
Robert Bradshaw committed
24

25 26 27 28 29 30 31 32
try:
    import platform
    IS_PYPY = platform.python_implementation() == 'PyPy'
    IS_CPYTHON = platform.python_implementation() == 'CPython'
except (ImportError, AttributeError):
    IS_CPYTHON = True
    IS_PYPY = False

33 34 35 36 37
from io import open as io_open
try:
    from StringIO import StringIO
except ImportError:
    from io import StringIO  # doesn't accept 'str' in Py2
38 39 40 41 42 43

try:
    import cPickle as pickle
except ImportError:
    import pickle

44 45 46 47 48
try:
    import threading
except ImportError: # No threads, no problems
    threading = None

Robert Bradshaw's avatar
Robert Bradshaw committed
49 50 51 52 53 54 55 56 57 58 59 60 61
try:
    from collections import defaultdict
except ImportError:
    class defaultdict(object):
        def __init__(self, default_factory=lambda : None):
            self._dict = {}
            self.default_factory = default_factory
        def __getitem__(self, key):
            if key not in self._dict:
                self._dict[key] = self.default_factory()
            return self._dict[key]
        def __setitem__(self, key, value):
            self._dict[key] = value
Stefan Behnel's avatar
Stefan Behnel committed
62 63
        def __contains__(self, key):
            return key in self._dict
Robert Bradshaw's avatar
Robert Bradshaw committed
64 65
        def __repr__(self):
            return repr(self._dict)
Stefan Behnel's avatar
Stefan Behnel committed
66 67
        def __nonzero__(self):
            return bool(self._dict)
Robert Bradshaw's avatar
Robert Bradshaw committed
68

69 70 71
try:
    from unittest import SkipTest
except ImportError:
72 73
    class SkipTest(Exception):  # don't raise, only provided to allow except-ing it!
        pass
74
    def skip_test(reason):
75
        sys.stderr.write("Skipping test: %s\n" % reason)
76 77 78 79
else:
    def skip_test(reason):
        raise SkipTest(reason)

Stefan Behnel's avatar
Stefan Behnel committed
80 81 82 83 84
try:
    basestring
except NameError:
    basestring = str

85
WITH_CYTHON = True
86
CY3_DIR = None
87 88

from distutils.command.build_ext import build_ext as _build_ext
89
from distutils import sysconfig
90
from distutils import ccompiler
91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108
_to_clean = []

@atexit.register
def _cleanup_files():
    """
    This is only used on Cygwin to clean up shared libraries that are unsafe
    to delete while the test suite is running.
    """

    for filename in _to_clean:
        if os.path.isdir(filename):
            shutil.rmtree(filename, ignore_errors=True)
        else:
            try:
                os.remove(filename)
            except OSError:
                pass

109

110 111 112
def get_distutils_distro(_cache=[]):
    if _cache:
        return _cache[0]
Unknown's avatar
Unknown committed
113
    # late import to accommodate for setuptools override
114 115 116 117 118 119
    from distutils.dist import Distribution
    distutils_distro = Distribution()

    if sys.platform == 'win32':
        # TODO: Figure out why this hackery (see http://thread.gmane.org/gmane.comp.python.cython.devel/8280/).
        config_files = distutils_distro.find_config_files()
120 121 122 123
        try:
            config_files.remove('setup.cfg')
        except ValueError:
            pass
124 125 126
        distutils_distro.parse_config_files(config_files)

        cfgfiles = distutils_distro.find_config_files()
127 128 129 130
        try:
            cfgfiles.remove('setup.cfg')
        except ValueError:
            pass
131 132 133
        distutils_distro.parse_config_files(cfgfiles)
    _cache.append(distutils_distro)
    return distutils_distro
Robert Bradshaw's avatar
Robert Bradshaw committed
134 135


136
EXT_DEP_MODULES = {
137 138
    'tag:numpy':     'numpy',
    'tag:numpy_old': 'numpy',
139
    'tag:pythran':  'pythran',
140
    'tag:setuptools':  'setuptools.sandbox',
141 142 143 144
    'tag:asyncio':  'asyncio',
    'tag:pstats':   'pstats',
    'tag:posix':    'posix',
    'tag:array':    'array',
145
    'tag:coverage': 'Cython.Coverage',
146
    'Coverage':     'Cython.Coverage',
147
    'tag:ipython':  'IPython.testing.globalipapp',
148
    'tag:jedi':     'jedi_BROKEN_AND_DISABLED',
149
    'tag:test.support': 'test.support',  # support module for CPython unit tests
150 151
}

152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168
def patch_inspect_isfunction():
    import inspect
    orig_isfunction = inspect.isfunction
    def isfunction(obj):
        return orig_isfunction(obj) or type(obj).__name__ == 'cython_function_or_method'
    isfunction._orig_isfunction = orig_isfunction
    inspect.isfunction = isfunction

def unpatch_inspect_isfunction():
    import inspect
    try:
        orig_isfunction = inspect.isfunction._orig_isfunction
    except AttributeError:
        pass
    else:
        inspect.isfunction = orig_isfunction

169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218
def def_to_cdef(source):
    '''
    Converts the module-level def methods into cdef methods, i.e.

        @decorator
        def foo([args]):
            """
            [tests]
            """
            [body]

    becomes

        def foo([args]):
            """
            [tests]
            """
            return foo_c([args])

        cdef foo_c([args]):
            [body]
    '''
    output = []
    skip = False
    def_node = re.compile(r'def (\w+)\(([^()*]*)\):').match
    lines = iter(source.split('\n'))
    for line in lines:
        if not line.strip():
            output.append(line)
            continue

        if skip:
            if line[0] != ' ':
                skip = False
            else:
                continue

        if line[0] == '@':
            skip = True
            continue

        m = def_node(line)
        if m:
            name = m.group(1)
            args = m.group(2)
            if args:
                args_no_types = ", ".join(arg.split()[-1] for arg in args.split(','))
            else:
                args_no_types = ""
            output.append("def %s(%s):" % (name, args_no_types))
219
            line = next(lines)
220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239
            if '"""' in line:
                has_docstring = True
                output.append(line)
                for line in lines:
                    output.append(line)
                    if '"""' in line:
                        break
            else:
                has_docstring = False
            output.append("    return %s_c(%s)" % (name, args_no_types))
            output.append('')
            output.append("cdef %s_c(%s):" % (name, args))
            if not has_docstring:
                output.append(line)

        else:
            output.append(line)

    return '\n'.join(output)

240 241 242 243 244 245 246

def exclude_extension_in_pyver(*versions):
    def check(ext):
        return EXCLUDE_EXT if sys.version_info[:2] in versions else ext
    return check


247 248 249 250 251 252
def exclude_extension_on_platform(*platforms):
    def check(ext):
        return EXCLUDE_EXT if sys.platform in platforms else ext
    return check


253 254 255 256
def update_linetrace_extension(ext):
    ext.define_macros.append(('CYTHON_TRACE', 1))
    return ext

257

258 259 260 261 262
def update_old_numpy_extension(ext):
    update_numpy_extension(ext, set_api17_macro=False)


def update_numpy_extension(ext, set_api17_macro=True):
263
    import numpy
264 265
    from numpy.distutils.misc_util import get_info

266
    ext.include_dirs.append(numpy.get_include())
267

268 269 270
    if set_api17_macro:
        ext.define_macros.append(('NPY_NO_DEPRECATED_API', 'NPY_1_7_API_VERSION'))

271 272 273 274 275
    # We need the npymath library for numpy.math.
    # This is typically a static-only library.
    for attr, value in get_info('npymath').items():
        getattr(ext, attr).extend(value)

276

277
def update_openmp_extension(ext):
278
    ext.openmp = True
279 280
    language = ext.language

281 282 283 284
    if sys.platform == 'win32' and sys.version_info[:2] == (3,4):
        # OpenMP tests fail in appveyor in Py3.4 -> just ignore them, EoL of Py3.4 is early 2019...
        return EXCLUDE_EXT

285 286 287 288 289 290 291 292 293 294
    if language == 'cpp':
        flags = OPENMP_CPP_COMPILER_FLAGS
    else:
        flags = OPENMP_C_COMPILER_FLAGS

    if flags:
        compile_flags, link_flags = flags
        ext.extra_compile_args.extend(compile_flags.split())
        ext.extra_link_args.extend(link_flags.split())
        return ext
295 296
    elif sys.platform == 'win32':
        return ext
297 298 299

    return EXCLUDE_EXT

300

301
def update_cpp11_extension(ext):
302
    """
303 304 305
        update cpp11 extensions that will run on versions of gcc >4.8
    """
    gcc_version = get_gcc_version(ext.language)
306
    if gcc_version:
307 308
        compiler_version = gcc_version.group(1)
        if float(compiler_version) > 4.8:
309
            ext.extra_compile_args.append("-std=c++11")
310
        return ext
311

Stefan Behnel's avatar
Stefan Behnel committed
312 313
    clang_version = get_clang_version(ext.language)
    if clang_version:
314
        ext.extra_compile_args.append("-std=c++11")
315 316 317
        if sys.platform == "darwin":
          ext.extra_compile_args.append("-stdlib=libc++")
          ext.extra_compile_args.append("-mmacosx-version-min=10.7")
318 319
        return ext

320
    return EXCLUDE_EXT
321

322

323
def get_cc_version(language):
324 325
    """
        finds gcc version using Popen
326 327 328 329 330
    """
    if language == 'cpp':
        cc = sysconfig.get_config_var('CXX')
    else:
        cc = sysconfig.get_config_var('CC')
331 332
    if not cc:
       cc = ccompiler.get_default_compiler()
333 334

    if not cc:
335
        return ''
336

337 338 339
    # For some reason, cc can be e.g. 'gcc -pthread'
    cc = cc.split()[0]

340 341 342
    # Force english output
    env = os.environ.copy()
    env['LC_MESSAGES'] = 'C'
343
    try:
344
        p = subprocess.Popen([cc, "-v"], stderr=subprocess.PIPE, env=env)
Robert Bradshaw's avatar
Robert Bradshaw committed
345 346 347 348
    except EnvironmentError:
        # Be compatible with Python 3
        warnings.warn("Unable to find the %s compiler: %s: %s" %
                      (language, os.strerror(sys.exc_info()[1].errno), cc))
349
        return ''
Robert Bradshaw's avatar
Robert Bradshaw committed
350
    _, output = p.communicate()
351 352 353 354 355 356 357 358 359
    return output.decode(locale.getpreferredencoding() or 'ASCII', 'replace')


def get_gcc_version(language):
    matcher = re.compile(r"gcc version (\d+\.\d+)").search
    return matcher(get_cc_version(language))


def get_clang_version(language):
360
    matcher = re.compile(r"clang(?:-|\s+version\s+)(\d+\.\d+)").search
361
    return matcher(get_cc_version(language))
362 363 364 365 366 367 368 369 370 371 372


def get_openmp_compiler_flags(language):
    """
    As of gcc 4.2, it supports OpenMP 2.5. Gcc 4.4 implements 3.0. We don't
    (currently) check for other compilers.

    returns a two-tuple of (CFLAGS, LDFLAGS) to build the OpenMP extension
    """
    gcc_version = get_gcc_version(language)

373
    if not gcc_version:
374 375 376 377
        if sys.platform == 'win32':
            return '/openmp', ''
        else:
            return None # not gcc - FIXME: do something about other compilers
378

379 380 381 382
    # gcc defines "__int128_t", assume that at least all 64 bit architectures have it
    global COMPILER_HAS_INT128
    COMPILER_HAS_INT128 = getattr(sys, 'maxsize', getattr(sys, 'maxint', 0)) > 2**60

383
    compiler_version = gcc_version.group(1)
384 385 386
    if compiler_version and compiler_version.split('.') >= ['4', '2']:
        return '-fopenmp', '-fopenmp'

387 388 389 390
try:
    locale.setlocale(locale.LC_ALL, '')
except locale.Error:
    pass
391

392 393
COMPILER = None
COMPILER_HAS_INT128 = False
394 395 396 397 398 399
OPENMP_C_COMPILER_FLAGS = get_openmp_compiler_flags('c')
OPENMP_CPP_COMPILER_FLAGS = get_openmp_compiler_flags('cpp')

# Return this from the EXT_EXTRAS matcher callback to exclude the extension
EXCLUDE_EXT = object()

400 401
EXT_EXTRAS = {
    'tag:numpy' : update_numpy_extension,
402
    'tag:numpy_old' : update_old_numpy_extension,
403
    'tag:openmp': update_openmp_extension,
404
    'tag:cpp11': update_cpp11_extension,
405
    'tag:trace' : update_linetrace_extension,
406
    'tag:bytesformat':  exclude_extension_in_pyver((3, 3), (3, 4)),  # no %-bytes formatting
407
    'tag:no-macos':  exclude_extension_on_platform('darwin'),
408
}
409

410

Robert Bradshaw's avatar
Robert Bradshaw committed
411
# TODO: use tags
412
VER_DEP_MODULES = {
413
    # tests are excluded if 'CurrentPythonVersion OP VersionTuple', i.e.
Stefan Behnel's avatar
Stefan Behnel committed
414
    # (2,4) : (operator.lt, ...) excludes ... when PyVer < 2.4.x
415

416 417 418 419
    # The next line should start (3,); but this is a dictionary, so
    # we can only have one (3,) key.  Since 2.7 is supposed to be the
    # last 2.x release, things would have to change drastically for this
    # to be unsafe...
420 421
    (2,999): (operator.lt, lambda x: x in ['run.special_methods_T561_py3',
                                           'run.test_raisefrom',
422
                                           'run.different_package_names',
423
                                           ]),
424
    (3,): (operator.ge, lambda x: x in ['run.non_future_division',
Stefan Behnel's avatar
Stefan Behnel committed
425
                                        'compile.extsetslice',
426
                                        'compile.extdelslice',
427
                                        'run.special_methods_T561_py2',
428
                                        ]),
429
    (3,3) : (operator.lt, lambda x: x in ['build.package_compilation',
430
                                          'run.yield_from_py33',
431
                                          'pyximport.pyximport_namespace',
432
                                          'run.qualname',
433
                                          ]),
434
    (3,4): (operator.lt, lambda x: x in ['run.py34_signature',
435
                                         'run.test_unicode',  # taken from Py3.7, difficult to backport
436
                                         ]),
437 438
    (3,4,999): (operator.gt, lambda x: x in ['run.initial_file_path',
                                             ]),
439
    (3,5): (operator.lt, lambda x: x in ['run.py35_pep492_interop',
Stefan Behnel's avatar
Stefan Behnel committed
440
                                         'run.py35_asyncio_async_def',
441
                                         'run.mod__spec__',
442
                                         'run.pep526_variable_annotations',  # typing module
443
                                         'run.test_exceptions',  # copied from Py3.7+
444
                                         ]),
445 446
}

447
INCLUDE_DIRS = [ d for d in os.getenv('INCLUDE', '').split(os.pathsep) if d ]
448
CFLAGS = os.getenv('CFLAGS', '').split()
449
CCACHE = os.getenv('CYTHON_RUNTESTS_CCACHE', '').split()
450
TEST_SUPPORT_DIR = 'testsupport'
451

452
BACKENDS = ['c', 'cpp']
453

454 455 456
UTF8_BOM_BYTES = r'\xef\xbb\xbf'.encode('ISO-8859-1').decode('unicode_escape')


457 458 459 460 461 462 463 464 465 466
def memoize(f):
    uncomputed = object()
    f._cache = {}
    def func(*args):
        res = f._cache.get(args, uncomputed)
        if res is uncomputed:
            res = f._cache[args] = f(*args)
        return res
    return func

467

Robert Bradshaw's avatar
Robert Bradshaw committed
468
@memoize
Robert Bradshaw's avatar
Robert Bradshaw committed
469 470
def parse_tags(filepath):
    tags = defaultdict(list)
471
    parse_tag = re.compile(r'#\s*(\w+)\s*:(.*)$').match
472
    with io_open(filepath, encoding='ISO-8859-1', errors='ignore') as f:
473
        for line in f:
474 475
            # ignore BOM-like bytes and whitespace
            line = line.lstrip(UTF8_BOM_BYTES).strip()
476
            if not line:
477 478 479 480
                if tags:
                    break  # assume all tags are in one block
                else:
                    continue
481 482
            if line[0] != '#':
                break
483 484 485 486 487 488 489 490
            parsed = parse_tag(line)
            if parsed:
                tag, values = parsed.groups()
                if tag in ('coding', 'encoding'):
                    continue
                if tag == 'tags':
                    tag = 'tag'
                    print("WARNING: test tags use the 'tag' directive, not 'tags' (%s)" % filepath)
491
                if tag not in ('mode', 'tag', 'ticket', 'cython', 'distutils', 'preparse'):
492 493 494 495 496
                    print("WARNING: unknown test directive '%s' found (%s)" % (tag, filepath))
                values = values.split(',')
                tags[tag].extend(filter(None, [value.strip() for value in values]))
            elif tags:
                break  # assume all tags are in one block
Robert Bradshaw's avatar
Robert Bradshaw committed
497 498
    return tags

499

Robert Bradshaw's avatar
Robert Bradshaw committed
500 501
list_unchanging_dir = memoize(lambda x: os.listdir(x))

Stefan Behnel's avatar
Stefan Behnel committed
502

503 504
@memoize
def _list_pyregr_data_files(test_directory):
505 506 507 508
    is_data_file = re.compile('(?:[.](txt|pem|db|html)|^bad.*[.]py)$').search
    return ['__init__.py'] + [
        filename for filename in list_unchanging_dir(test_directory)
        if is_data_file(filename)]
509 510


511 512 513 514 515 516 517 518 519 520 521 522 523 524
def import_ext(module_name, file_path=None):
    if file_path:
        import imp
        return imp.load_dynamic(module_name, file_path)
    else:
        try:
            from importlib import invalidate_caches
        except ImportError:
            pass
        else:
            invalidate_caches()
        return __import__(module_name, globals(), locals(), ['*'])


525 526
class build_ext(_build_ext):
    def build_extension(self, ext):
527 528 529 530 531 532
        try:
            try: # Py2.7+ & Py3.2+
                compiler_obj = self.compiler_obj
            except AttributeError:
                compiler_obj = self.compiler
            if ext.language == 'c++':
533
                compiler_obj.compiler_so.remove('-Wstrict-prototypes')
534 535
            if CCACHE:
                compiler_obj.compiler_so = CCACHE + compiler_obj.compiler_so
536 537
            if getattr(ext, 'openmp', None) and compiler_obj.compiler_type == 'msvc':
                ext.extra_compile_args.append('/openmp')
538 539
        except Exception:
            pass
540
        _build_ext.build_extension(self, ext)
541

542

543
class ErrorWriter(object):
544
    match_error = re.compile(r'(warning:)?(?:.*:)?\s*([-0-9]+)\s*:\s*([-0-9]+)\s*:\s*(.*)').match
545

546 547 548 549
    def __init__(self):
        self.output = []
        self.write = self.output.append

550
    def _collect(self):
551
        s = ''.join(self.output)
552 553
        results = {'errors': [], 'warnings': []}
        for line in s.splitlines():
554 555
            match = self.match_error(line)
            if match:
556
                is_warning, line, column, message = match.groups()
557 558 559
                results['warnings' if is_warning else 'errors'].append((int(line), int(column), message.strip()))

        return [["%d:%d: %s" % values for values in sorted(results[key])] for key in ('errors', 'warnings')]
560 561

    def geterrors(self):
562
        return self._collect()[0]
563 564

    def getwarnings(self):
565
        return self._collect()[1]
566 567

    def getall(self):
568 569 570 571 572
        return self._collect()

    def close(self):
        pass  # ignore, only to match file-like interface

573

574
class Stats(object):
575 576
    def __init__(self, top_n=8):
        self.top_n = top_n
577 578
        self.test_counts = defaultdict(int)
        self.test_times = defaultdict(float)
579
        self.top_tests = defaultdict(list)
580

581
    def add_time(self, name, language, metric, t):
582 583
        self.test_counts[metric] += 1
        self.test_times[metric] += t
584
        top = self.top_tests[metric]
585 586
        push = heapq.heappushpop if len(top) >= self.top_n else heapq.heappush
        # min-heap => pop smallest/shortest until longest times remain
587
        push(top, (t, name, language))
588 589

    @contextmanager
590
    def time(self, name, language, metric):
591 592 593
        t = time.time()
        yield
        t = time.time() - t
594
        self.add_time(name, language, metric, t)
595

596 597 598 599 600 601 602 603 604 605
    def update(self, stats):
        # type: (Stats) -> None
        for metric, t in stats.test_times.items():
            self.test_times[metric] += t
            self.test_counts[metric] += stats.test_counts[metric]
            top = self.top_tests[metric]
            for entry in stats.top_tests[metric]:
                push = heapq.heappushpop if len(top) >= self.top_n else heapq.heappush
                push(top, entry)

606
    def print_stats(self, out=sys.stderr):
607 608
        if not self.test_times:
            return
609
        lines = ['Times:\n']
610
        for metric, t in sorted(self.test_times.items()):
611
            count = self.test_counts[metric]
612 613 614
            top = self.top_tests[metric]
            lines.append("%-12s: %8.2f sec  (%4d, %6.3f / run) - slowest: %s\n" % (
                metric, t, count, t / count,
615
                ', '.join("'{2}:{1}' ({0:.2f}s)".format(*item) for item in heapq.nlargest(self.top_n, top))))
616 617 618
        out.write(''.join(lines))


619
class TestBuilder(object):
gabrieldemarmiesse's avatar
gabrieldemarmiesse committed
620 621
    def __init__(self, rootdir, workdir, selectors, exclude_selectors, options,
                 with_pyregr, languages, test_bugs, language_level,
622
                 common_utility_dir, pythran_dir=None,
623
                 default_mode='run', stats=None,
gabrieldemarmiesse's avatar
gabrieldemarmiesse committed
624
                 add_embedded_test=False):
625 626
        self.rootdir = rootdir
        self.workdir = workdir
627
        self.selectors = selectors
628
        self.exclude_selectors = exclude_selectors
gabrieldemarmiesse's avatar
gabrieldemarmiesse committed
629 630 631 632
        self.annotate = options.annotate_source
        self.cleanup_workdir = options.cleanup_workdir
        self.cleanup_sharedlibs = options.cleanup_sharedlibs
        self.cleanup_failures = options.cleanup_failures
633
        self.with_pyregr = with_pyregr
gabrieldemarmiesse's avatar
gabrieldemarmiesse committed
634
        self.cython_only = options.cython_only
635
        self.languages = languages
636
        self.test_bugs = test_bugs
gabrieldemarmiesse's avatar
gabrieldemarmiesse committed
637
        self.fork = options.fork
638
        self.language_level = language_level
gabrieldemarmiesse's avatar
gabrieldemarmiesse committed
639
        self.test_determinism = options.test_determinism
640
        self.common_utility_dir = common_utility_dir
641
        self.pythran_dir = pythran_dir
642
        self.default_mode = default_mode
643
        self.stats = stats
644
        self.add_embedded_test = add_embedded_test
645
        self.capture = options.capture
646 647 648

    def build_suite(self):
        suite = unittest.TestSuite()
649 650 651
        filenames = os.listdir(self.rootdir)
        filenames.sort()
        for filename in filenames:
652
            path = os.path.join(self.rootdir, filename)
653
            if os.path.isdir(path) and filename != TEST_SUPPORT_DIR:
654 655
                if filename == 'pyregr' and not self.with_pyregr:
                    continue
656 657
                if filename == 'broken' and not self.test_bugs:
                    continue
658
                suite.addTest(
659
                    self.handle_directory(path, filename))
660
        if sys.platform not in ['win32'] and self.add_embedded_test:
661
            # Non-Windows makefile.
662 663
            if [1 for selector in self.selectors if selector("embedded")] \
                and not [1 for selector in self.exclude_selectors if selector("embedded")]:
664
                suite.addTest(unittest.makeSuite(EmbedTest))
665 666
        return suite

667
    def handle_directory(self, path, context):
668 669 670 671
        workdir = os.path.join(self.workdir, context)
        if not os.path.exists(workdir):
            os.makedirs(workdir)

672
        suite = unittest.TestSuite()
Robert Bradshaw's avatar
Robert Bradshaw committed
673
        filenames = list_unchanging_dir(path)
674 675
        filenames.sort()
        for filename in filenames:
676 677 678
            filepath = os.path.join(path, filename)
            module, ext = os.path.splitext(filename)
            if ext not in ('.py', '.pyx', '.srctree'):
679
                continue
680 681
            if filename.startswith('.'):
                continue # certain emacs backup files
682 683 684 685
            if context == 'pyregr':
                tags = defaultdict(list)
            else:
                tags = parse_tags(filepath)
Robert Bradshaw's avatar
Robert Bradshaw committed
686
            fqmodule = "%s.%s" % (context, module)
687
            if not [ 1 for match in self.selectors
Robert Bradshaw's avatar
Robert Bradshaw committed
688
                     if match(fqmodule, tags) ]:
689
                continue
690
            if self.exclude_selectors:
691
                if [1 for match in self.exclude_selectors
Robert Bradshaw's avatar
Robert Bradshaw committed
692
                        if match(fqmodule, tags)]:
693
                    continue
694

695
            mode = self.default_mode
696 697 698 699 700 701
            if tags['mode']:
                mode = tags['mode'][0]
            elif context == 'pyregr':
                mode = 'pyregr'

            if ext == '.srctree':
702
                if 'cpp' not in tags['tag'] or 'cpp' in self.languages:
703 704 705
                    suite.addTest(EndToEndTest(filepath, workdir,
                             self.cleanup_workdir, stats=self.stats,
                             capture=self.capture))
706 707 708 709 710 711
                continue

            # Choose the test suite.
            if mode == 'pyregr':
                if not filename.startswith('test_'):
                    continue
712
                test_class = CythonPyregrTestCase
713
            elif mode == 'run':
714
                if module.startswith("test_"):
715
                    test_class = CythonUnitTestCase
716
                else:
717
                    test_class = CythonRunTestCase
718
            elif mode in ['compile', 'error']:
719
                test_class = CythonCompileTestCase
720 721
            else:
                raise KeyError('Invalid test mode: ' + mode)
722

723
            for test in self.build_tests(test_class, path, workdir,
Robert Bradshaw's avatar
Robert Bradshaw committed
724
                                         module, mode == 'error', tags):
725
                suite.addTest(test)
726

727
            if mode == 'run' and ext == '.py' and not self.cython_only and not filename.startswith('test_'):
728
                # additionally test file in real Python
729 730 731 732 733 734
                min_py_ver = [
                    (int(pyver.group(1)), int(pyver.group(2)))
                    for pyver in map(re.compile(r'pure([0-9]+)[.]([0-9]+)').match, tags['tag'])
                    if pyver
                ]
                if not min_py_ver or any(sys.version_info >= min_ver for min_ver in min_py_ver):
735
                    suite.addTest(PureDoctestTestCase(module, os.path.join(path, filename), tags, stats=self.stats))
736

737 738
        return suite

Robert Bradshaw's avatar
Robert Bradshaw committed
739
    def build_tests(self, test_class, path, workdir, module, expect_errors, tags):
740 741
        warning_errors = 'werror' in tags['tag']
        expect_warnings = 'warnings' in tags['tag']
Vitja Makarov's avatar
Vitja Makarov committed
742

743
        if expect_errors:
744
            if skip_c(tags) and 'cpp' in self.languages:
Robert Bradshaw's avatar
Robert Bradshaw committed
745 746 747
                languages = ['cpp']
            else:
                languages = self.languages[:1]
748 749
        else:
            languages = self.languages
Robert Bradshaw's avatar
Robert Bradshaw committed
750

751
        if skip_c(tags) and 'c' in languages:
752 753
            languages = list(languages)
            languages.remove('c')
Robert Bradshaw's avatar
Robert Bradshaw committed
754 755 756
        elif 'no-cpp' in tags['tag'] and 'cpp' in self.languages:
            languages = list(languages)
            languages.remove('cpp')
757

758 759 760
        pythran_dir = self.pythran_dir
        if 'pythran' in tags['tag'] and not pythran_dir and 'cpp' in languages:
            import pythran.config
761 762 763 764 765 766
            from pythran import __version__ as pythran_version
            pythran_ext = (
                pythran.config.make_extension(python=True)
                if pythran_version >= '0.9' or pythran_version >= '0.8.7'
                else pythran.config.make_extension()
            )
767 768
            pythran_dir = pythran_ext['include_dirs'][0]

769
        preparse_list = tags.get('preparse', ['id'])
770
        tests = [ self.build_test(test_class, path, workdir, module, tags, language,
771
                                  expect_errors, expect_warnings, warning_errors, preparse,
772
                                  pythran_dir if language == "cpp" else None)
773 774
                  for language in languages
                  for preparse in preparse_list ]
775 776
        return tests

777
    def build_test(self, test_class, path, workdir, module, tags, language,
778
                   expect_errors, expect_warnings, warning_errors, preparse, pythran_dir):
779 780 781 782
        language_workdir = os.path.join(workdir, language)
        if not os.path.exists(language_workdir):
            os.makedirs(language_workdir)
        workdir = os.path.join(language_workdir, module)
783 784
        if preparse != 'id':
            workdir += '_%s' % str(preparse)
785
        return test_class(path, workdir, module, tags,
786
                          language=language,
787
                          preparse=preparse,
788
                          expect_errors=expect_errors,
789
                          expect_warnings=expect_warnings,
790 791 792
                          annotate=self.annotate,
                          cleanup_workdir=self.cleanup_workdir,
                          cleanup_sharedlibs=self.cleanup_sharedlibs,
793
                          cleanup_failures=self.cleanup_failures,
794
                          cython_only=self.cython_only,
795
                          fork=self.fork,
Vitja Makarov's avatar
Vitja Makarov committed
796
                          language_level=self.language_level,
797
                          warning_errors=warning_errors,
798
                          test_determinism=self.test_determinism,
799
                          common_utility_dir=self.common_utility_dir,
800 801
                          pythran_dir=pythran_dir,
                          stats=self.stats)
802

803

804 805 806 807 808 809 810 811 812 813 814 815 816 817 818 819
def skip_c(tags):
    if 'cpp' in tags['tag']:
        return True

    # We don't want to create a distutils key in the
    # dictionary so we check before looping.
    if 'distutils' in tags:
        for option in tags['distutils']:
            splitted = option.split('=')
            if len(splitted) == 2:
                argument, value = splitted
                if argument.strip() == 'language' and value.strip() == 'c++':
                    return True
    return False


820 821 822 823 824 825 826 827 828 829 830 831
def filter_stderr(stderr_bytes):
    """
    Filter annoying warnings from output.
    """
    if b"Command line warning D9025" in stderr_bytes:
        # MSCV: cl : Command line warning D9025 : overriding '/Ox' with '/Od'
        stderr_bytes = b'\n'.join(
            line for line in stderr_bytes.splitlines()
            if b"Command line warning D9025" not in line)
    return stderr_bytes


832
class CythonCompileTestCase(unittest.TestCase):
833
    def __init__(self, test_directory, workdir, module, tags, language='c', preparse='id',
834
                 expect_errors=False, expect_warnings=False, annotate=False, cleanup_workdir=True,
835
                 cleanup_sharedlibs=True, cleanup_failures=True, cython_only=False,
836
                 fork=True, language_level=2, warning_errors=False,
837
                 test_determinism=False,
838
                 common_utility_dir=None, pythran_dir=None, stats=None):
839
        self.test_directory = test_directory
840
        self.tags = tags
841 842
        self.workdir = workdir
        self.module = module
843
        self.language = language
844 845
        self.preparse = preparse
        self.name = module if self.preparse == "id" else "%s_%s" % (module, preparse)
846
        self.expect_errors = expect_errors
847
        self.expect_warnings = expect_warnings
848
        self.annotate = annotate
849
        self.cleanup_workdir = cleanup_workdir
850
        self.cleanup_sharedlibs = cleanup_sharedlibs
851
        self.cleanup_failures = cleanup_failures
852
        self.cython_only = cython_only
853
        self.fork = fork
854
        self.language_level = language_level
Vitja Makarov's avatar
Vitja Makarov committed
855
        self.warning_errors = warning_errors
856
        self.test_determinism = test_determinism
857
        self.common_utility_dir = common_utility_dir
858
        self.pythran_dir = pythran_dir
859
        self.stats = stats
860 861 862
        unittest.TestCase.__init__(self)

    def shortDescription(self):
863
        return "compiling (%s%s) %s" % (self.language, "/pythran" if self.pythran_dir is not None else "", self.name)
864

Stefan Behnel's avatar
Stefan Behnel committed
865
    def setUp(self):
Vitja Makarov's avatar
Vitja Makarov committed
866
        from Cython.Compiler import Options
867 868 869 870
        self._saved_options = [
            (name, getattr(Options, name))
            for name in ('warning_errors', 'clear_to_none', 'error_on_unknown_names', 'error_on_uninitialized')
        ]
871
        self._saved_default_directives = list(Options.get_directive_defaults().items())
Vitja Makarov's avatar
Vitja Makarov committed
872
        Options.warning_errors = self.warning_errors
873
        if sys.version_info >= (3, 4):
874
            Options._directive_defaults['autotestdict'] = False
Vitja Makarov's avatar
Vitja Makarov committed
875

876 877
        if not os.path.exists(self.workdir):
            os.makedirs(self.workdir)
Stefan Behnel's avatar
Stefan Behnel committed
878 879 880
        if self.workdir not in sys.path:
            sys.path.insert(0, self.workdir)

881
    def tearDown(self):
Vitja Makarov's avatar
Vitja Makarov committed
882
        from Cython.Compiler import Options
883 884
        for name, value in self._saved_options:
            setattr(Options, name, value)
885
        Options._directive_defaults = dict(self._saved_default_directives)
886
        unpatch_inspect_isfunction()
Vitja Makarov's avatar
Vitja Makarov committed
887

Stefan Behnel's avatar
Stefan Behnel committed
888 889 890 891 892 893 894 895
        try:
            sys.path.remove(self.workdir)
        except ValueError:
            pass
        try:
            del sys.modules[self.module]
        except KeyError:
            pass
896 897 898
        cleanup = self.cleanup_failures or self.success
        cleanup_c_files = WITH_CYTHON and self.cleanup_workdir and cleanup
        cleanup_lib_files = self.cleanup_sharedlibs and cleanup
899 900
        is_cygwin = sys.platform == 'cygwin'

901
        if os.path.exists(self.workdir):
902
            if cleanup_c_files and cleanup_lib_files and not is_cygwin:
903 904 905 906 907
                shutil.rmtree(self.workdir, ignore_errors=True)
            else:
                for rmfile in os.listdir(self.workdir):
                    if not cleanup_c_files:
                        if (rmfile[-2:] in (".c", ".h") or
908 909
                                rmfile[-4:] == ".cpp" or
                                rmfile.endswith(".html") and rmfile.startswith(self.module)):
910
                            continue
911 912 913 914

                    is_shared_obj = rmfile.endswith(".so") or rmfile.endswith(".dll")

                    if not cleanup_lib_files and is_shared_obj:
915
                        continue
916

917 918 919 920
                    try:
                        rmfile = os.path.join(self.workdir, rmfile)
                        if os.path.isdir(rmfile):
                            shutil.rmtree(rmfile, ignore_errors=True)
921 922 923
                        elif is_cygwin and is_shared_obj:
                            # Delete later
                            _to_clean.append(rmfile)
924 925 926 927
                        else:
                            os.remove(rmfile)
                    except IOError:
                        pass
928

929 930 931 932
                if cleanup_c_files and cleanup_lib_files and is_cygwin:
                    # Finally, remove the work dir itself
                    _to_clean.append(self.workdir)

933 934 935 936
        if cleanup_c_files and os.path.exists(self.workdir + '-again'):
            shutil.rmtree(self.workdir + '-again', ignore_errors=True)


937
    def runTest(self):
938
        self.success = False
939
        self.runCompileTest()
940
        self.success = True
941 942

    def runCompileTest(self):
943 944
        return self.compile(
            self.test_directory, self.module, self.workdir,
945
            self.test_directory, self.expect_errors, self.expect_warnings, self.annotate)
946

947 948 949 950 951
    def find_module_source_file(self, source_file):
        if not os.path.exists(source_file):
            source_file = source_file[:-1]
        return source_file

Stefan Behnel's avatar
Stefan Behnel committed
952 953 954
    def build_target_filename(self, module_name):
        target = '%s.%s' % (module_name, self.language)
        return target
955

Robert Bradshaw's avatar
Robert Bradshaw committed
956
    def related_files(self, test_directory, module_name):
957
        is_related = re.compile('%s_.*[.].*' % module_name).match
Robert Bradshaw's avatar
Robert Bradshaw committed
958
        return [filename for filename in list_unchanging_dir(test_directory)
959
                if is_related(filename)]
Robert Bradshaw's avatar
Robert Bradshaw committed
960 961

    def copy_files(self, test_directory, target_directory, file_list):
962 963 964
        if self.preparse and self.preparse != 'id':
            preparse_func = globals()[self.preparse]
            def copy(src, dest):
965 966 967
                with open(src) as fin:
                    with open(dest, 'w') as fout:
                        fout.write(preparse_func(fin.read()))
968 969 970 971 972 973
        else:
            # use symlink on Unix, copy on Windows
            try:
                copy = os.symlink
            except AttributeError:
                copy = shutil.copy
974 975

        join = os.path.join
Robert Bradshaw's avatar
Robert Bradshaw committed
976
        for filename in file_list:
977
            file_path = join(test_directory, filename)
978
            if os.path.exists(file_path):
979
                copy(file_path, join(target_directory, filename))
980

Robert Bradshaw's avatar
Robert Bradshaw committed
981 982 983
    def source_files(self, workdir, module_name, file_list):
        return ([self.build_target_filename(module_name)] +
            [filename for filename in file_list
984
             if not os.path.isfile(os.path.join(workdir, filename))])
985 986

    def split_source_and_output(self, test_directory, module, workdir):
987
        source_file = self.find_module_source_file(os.path.join(test_directory, module) + '.pyx')
988 989
        with io_open(source_file, 'r', encoding='ISO-8859-1') as source_and_output:
            error_writer = warnings_writer = None
990
            out = io_open(os.path.join(workdir, module + os.path.splitext(source_file)[1]),
991
                          'w', encoding='ISO-8859-1')
992 993 994 995 996 997 998 999 1000 1001 1002 1003
            try:
                for line in source_and_output:
                    if line.startswith("_ERRORS"):
                        out.close()
                        out = error_writer = ErrorWriter()
                    elif line.startswith("_WARNINGS"):
                        out.close()
                        out = warnings_writer = ErrorWriter()
                    else:
                        out.write(line)
            finally:
                out.close()
1004

1005 1006
        return (error_writer.geterrors() if error_writer else [],
                warnings_writer.geterrors() if warnings_writer else [])
1007

1008 1009
    def run_cython(self, test_directory, module, targetdir, incdir, annotate,
                   extra_compile_options=None):
1010
        include_dirs = INCLUDE_DIRS + [os.path.join(test_directory, '..', TEST_SUPPORT_DIR)]
1011 1012
        if incdir:
            include_dirs.append(incdir)
1013

1014 1015 1016 1017 1018 1019
        if self.preparse == 'id':
            source = self.find_module_source_file(
                os.path.join(test_directory, module + '.pyx'))
        else:
            self.copy_files(test_directory, targetdir, [module + '.pyx'])
            source = os.path.join(targetdir, module + '.pyx')
Stefan Behnel's avatar
Stefan Behnel committed
1020
        target = os.path.join(targetdir, self.build_target_filename(module))
1021

1022 1023
        if extra_compile_options is None:
            extra_compile_options = {}
1024

1025 1026 1027 1028
        if 'allow_unknown_names' in self.tags['tag']:
            from Cython.Compiler import Options
            Options.error_on_unknown_names = False

1029 1030 1031 1032 1033 1034
        try:
            CompilationOptions
        except NameError:
            from Cython.Compiler.Main import CompilationOptions
            from Cython.Compiler.Main import compile as cython_compile
            from Cython.Compiler.Main import default_options
1035
        common_utility_include_dir = self.common_utility_dir
1036

1037
        options = CompilationOptions(
1038
            default_options,
1039 1040
            include_path = include_dirs,
            output_file = target,
1041
            annotate = annotate,
1042 1043
            use_listing_file = False,
            cplus = self.language == 'cpp',
1044
            np_pythran = self.pythran_dir is not None,
1045
            language_level = self.language_level,
1046
            generate_pxi = False,
1047
            evaluate_tree_assertions = True,
1048
            common_utility_include_dir = common_utility_include_dir,
1049
            **extra_compile_options
1050
            )
1051 1052 1053
        cython_compile(source, options=options,
                       full_module_name=module)

1054
    def run_distutils(self, test_directory, module, workdir, incdir,
1055
                      extra_extension_args=None):
1056 1057 1058
        cwd = os.getcwd()
        os.chdir(workdir)
        try:
1059
            build_extension = build_ext(get_distutils_distro())
1060 1061 1062 1063
            build_extension.include_dirs = INCLUDE_DIRS[:]
            if incdir:
                build_extension.include_dirs.append(incdir)
            build_extension.finalize_options()
1064 1065
            if COMPILER:
                build_extension.compiler = COMPILER
1066

1067
            ext_compile_flags = CFLAGS[:]
1068

1069 1070
            if  build_extension.compiler == 'mingw32':
                ext_compile_flags.append('-Wno-format')
1071 1072
            if extra_extension_args is None:
                extra_extension_args = {}
1073

Robert Bradshaw's avatar
Robert Bradshaw committed
1074 1075
            related_files = self.related_files(test_directory, module)
            self.copy_files(test_directory, workdir, related_files)
1076 1077

            from distutils.core import Extension
1078 1079
            extension = Extension(
                module,
1080 1081
                sources=self.source_files(workdir, module, related_files),
                extra_compile_args=ext_compile_flags,
1082
                **extra_extension_args
1083
                )
1084 1085 1086 1087 1088

            if self.language == 'cpp':
                # Set the language now as the fixer might need it
                extension.language = 'c++'

1089 1090
            if 'distutils' in self.tags:
                from Cython.Build.Dependencies import DistutilsInfo
1091
                from Cython.Utils import open_source_file
1092
                pyx_path = os.path.join(self.test_directory, self.module + ".pyx")
1093 1094
                with open_source_file(pyx_path) as f:
                    DistutilsInfo(f).apply(extension)
1095

1096 1097 1098 1099
            if self.pythran_dir:
                from Cython.Build.Dependencies import update_pythran_extension
                update_pythran_extension(extension)

Stefan Behnel's avatar
Stefan Behnel committed
1100
            for matcher, fixer in list(EXT_EXTRAS.items()):
1101
                if isinstance(matcher, str):
Stefan Behnel's avatar
Stefan Behnel committed
1102
                    # lazy init
1103 1104 1105
                    del EXT_EXTRAS[matcher]
                    matcher = string_selector(matcher)
                    EXT_EXTRAS[matcher] = fixer
1106
                if matcher(module, self.tags):
1107 1108
                    newext = fixer(extension)
                    if newext is EXCLUDE_EXT:
1109
                        return skip_test("Test '%s' excluded due to tags '%s'" % (
1110
                            self.name, ', '.join(self.tags.get('tag', ''))))
1111
                    extension = newext or extension
1112 1113
            if self.language == 'cpp':
                extension.language = 'c++'
1114 1115 1116 1117 1118 1119
            build_extension.extensions = [extension]
            build_extension.build_temp = workdir
            build_extension.build_lib  = workdir
            build_extension.run()
        finally:
            os.chdir(cwd)
1120

1121 1122 1123 1124 1125 1126 1127 1128 1129 1130 1131 1132 1133 1134 1135 1136 1137
        try:
            get_ext_fullpath = build_extension.get_ext_fullpath
        except AttributeError:
            def get_ext_fullpath(ext_name, self=build_extension):
                # copied from distutils.command.build_ext (missing in Py2.[45])
                fullname = self.get_ext_fullname(ext_name)
                modpath = fullname.split('.')
                filename = self.get_ext_filename(modpath[-1])
                if not self.inplace:
                    filename = os.path.join(*modpath[:-1]+[filename])
                    return os.path.join(self.build_lib, filename)
                package = '.'.join(modpath[0:-1])
                build_py = self.get_finalized_command('build_py')
                package_dir = os.path.abspath(build_py.get_package_dir(package))
                return os.path.join(package_dir, filename)

        return get_ext_fullpath(module)
1138

1139
    def compile(self, test_directory, module, workdir, incdir,
1140 1141 1142 1143
                expect_errors, expect_warnings, annotate):
        expected_errors = expected_warnings = errors = warnings = ()
        if expect_errors or expect_warnings:
            expected_errors, expected_warnings = self.split_source_and_output(
1144 1145
                test_directory, module, workdir)
            test_directory = workdir
1146

1147 1148 1149 1150
        if WITH_CYTHON:
            old_stderr = sys.stderr
            try:
                sys.stderr = ErrorWriter()
1151
                with self.stats.time(self.name, self.language, 'cython'):
1152
                    self.run_cython(test_directory, module, workdir, incdir, annotate)
1153
                errors, warnings = sys.stderr.getall()
1154 1155
            finally:
                sys.stderr = old_stderr
1156 1157 1158 1159 1160 1161 1162 1163 1164 1165 1166 1167 1168 1169 1170
            if self.test_determinism and not expect_errors:
                workdir2 = workdir + '-again'
                os.mkdir(workdir2)
                self.run_cython(test_directory, module, workdir2, incdir, annotate)
                diffs = []
                for file in os.listdir(workdir2):
                    if (open(os.path.join(workdir, file)).read()
                        != open(os.path.join(workdir2, file)).read()):
                        diffs.append(file)
                        os.system('diff -u %s/%s %s/%s > %s/%s.diff' % (
                            workdir, file,
                            workdir2, file,
                            workdir2, file))
                if diffs:
                    self.fail('Nondeterministic file generation: %s' % ', '.join(diffs))
1171

1172
        tostderr = sys.__stderr__.write
1173 1174
        if expected_warnings or (expect_warnings and warnings):
            self._match_output(expected_warnings, warnings, tostderr)
1175
        if 'cerror' in self.tags['tag']:
1176
            if errors:
1177 1178 1179 1180
                tostderr("\n=== Expected C compile error ===\n")
                tostderr("\n=== Got Cython errors: ===\n")
                tostderr('\n'.join(errors))
                tostderr('\n\n')
1181 1182
                raise RuntimeError('should have generated extension code')
        elif errors or expected_errors:
1183
            self._match_output(expected_errors, errors, tostderr)
1184 1185
            return None

1186 1187
        so_path = None
        if not self.cython_only:
1188
            from Cython.Utils import captured_fd, print_bytes
1189
            from distutils.errors import CompileError, LinkError
1190 1191
            show_output = True
            get_stderr = get_stdout = None
1192
            try:
1193 1194
                with captured_fd(1) as get_stdout:
                    with captured_fd(2) as get_stderr:
1195
                        with self.stats.time(self.name, self.language, 'compile-%s' % self.language):
1196
                            so_path = self.run_distutils(test_directory, module, workdir, incdir)
1197
            except Exception as exc:
1198
                if ('cerror' in self.tags['tag'] and
1199
                    ((get_stderr and get_stderr()) or
1200
                     isinstance(exc, (CompileError, LinkError)))):
1201
                    show_output = False  # expected C compiler failure
1202 1203 1204
                else:
                    raise
            else:
1205
                if 'cerror' in self.tags['tag']:
1206
                    raise RuntimeError('should have failed C compile')
1207 1208 1209 1210
            finally:
                if show_output:
                    stdout = get_stdout and get_stdout().strip()
                    if stdout:
1211 1212 1213 1214
                        print_bytes(
                            stdout, header_text="\n=== C/C++ compiler output: =========\n",
                            end=None, file=sys.__stderr__)
                    stderr = get_stderr and filter_stderr(get_stderr()).strip()
1215
                    if stderr:
1216 1217 1218
                        print_bytes(
                            stderr, header_text="\n=== C/C++ compiler error output: ===\n",
                            end=None, file=sys.__stderr__)
1219
                    if stdout or stderr:
1220
                        tostderr("\n====================================\n")
1221
        return so_path
1222

1223 1224 1225
    def _match_output(self, expected_output, actual_output, write):
        try:
            for expected, actual in zip(expected_output, actual_output):
1226
                self.assertEqual(expected, actual)
1227 1228
            if len(actual_output) < len(expected_output):
                expected = expected_output[len(actual_output)]
1229
                self.assertEqual(expected, None)
1230 1231
            elif len(actual_output) > len(expected_output):
                unexpected = actual_output[len(expected_output)]
1232
                self.assertEqual(None, unexpected)
1233 1234 1235 1236 1237 1238 1239 1240
        except AssertionError:
            write("\n=== Expected: ===\n")
            write('\n'.join(expected_output))
            write("\n\n=== Got: ===\n")
            write('\n'.join(actual_output))
            write('\n\n')
            raise

1241

1242
class CythonRunTestCase(CythonCompileTestCase):
1243 1244 1245 1246 1247
    def setUp(self):
        CythonCompileTestCase.setUp(self)
        from Cython.Compiler import Options
        Options.clear_to_none = False

1248
    def shortDescription(self):
Stefan Behnel's avatar
Stefan Behnel committed
1249 1250 1251
        if self.cython_only:
            return CythonCompileTestCase.shortDescription(self)
        else:
1252
            return "compiling (%s%s) and running %s" % (self.language, "/pythran" if self.pythran_dir is not None else "", self.name)
1253 1254

    def run(self, result=None):
1255 1256
        if result is None:
            result = self.defaultTestResult()
Stefan Behnel's avatar
Stefan Behnel committed
1257
        result.startTest(self)
1258
        try:
Stefan Behnel's avatar
Stefan Behnel committed
1259
            self.setUp()
1260
            try:
1261
                self.success = False
1262
                ext_so_path = self.runCompileTest()
1263
                failures, errors, skipped = len(result.failures), len(result.errors), len(result.skipped)
1264
                if not self.cython_only and ext_so_path is not None:
1265
                    self.run_tests(result, ext_so_path)
1266 1267 1268
                if failures == len(result.failures) and errors == len(result.errors):
                    # No new errors...
                    self.success = True
1269 1270
            finally:
                check_thread_termination()
1271 1272 1273
        except SkipTest as exc:
            result.addSkip(self, str(exc))
            result.stopTest(self)
1274 1275 1276
        except Exception:
            result.addError(self, sys.exc_info())
            result.stopTest(self)
1277 1278 1279 1280
        try:
            self.tearDown()
        except Exception:
            pass
1281

1282
    def run_tests(self, result, ext_so_path):
1283
        self.run_doctests(self.module, result, ext_so_path)
1284

Stefan Behnel's avatar
Stefan Behnel committed
1285
    def run_doctests(self, module_or_name, result, ext_so_path):
1286
        def run_test(result):
Stefan Behnel's avatar
Stefan Behnel committed
1287
            if isinstance(module_or_name, basestring):
1288
                with self.stats.time(self.name, self.language, 'import'):
1289
                    module = import_ext(module_or_name, ext_so_path)
Stefan Behnel's avatar
Stefan Behnel committed
1290 1291
            else:
                module = module_or_name
1292
            tests = doctest.DocTestSuite(module)
1293
            with self.stats.time(self.name, self.language, 'run'):
1294
                tests.run(result)
1295 1296
        run_forked_test(result, run_test, self.shortDescription(), self.fork)

1297

1298
def run_forked_test(result, run_func, test_name, fork=True):
Stefan Behnel's avatar
Stefan Behnel committed
1299 1300
    if not fork or sys.version_info[0] >= 3 or not hasattr(os, 'fork'):
        run_func(result)
1301 1302
        sys.stdout.flush()
        sys.stderr.flush()
1303 1304 1305 1306 1307 1308 1309 1310 1311
        gc.collect()
        return

    # fork to make sure we do not keep the tested module loaded
    result_handle, result_file = tempfile.mkstemp()
    os.close(result_handle)
    child_id = os.fork()
    if not child_id:
        result_code = 0
1312
        output = None
1313
        try:
1314
            try:
1315
                tests = partial_result = None
1316
                try:
1317 1318
                    partial_result = PartialTestResult(result)
                    run_func(partial_result)
1319 1320
                    sys.stdout.flush()
                    sys.stderr.flush()
1321 1322 1323
                    gc.collect()
                except Exception:
                    result_code = 1
1324 1325 1326 1327 1328 1329 1330 1331
                    if partial_result is not None:
                        if tests is None:
                            # importing failed, try to fake a test class
                            tests = _FakeClass(
                                failureException=sys.exc_info()[1],
                                _shortDescription=test_name,
                                module_name=None)
                        partial_result.addError(tests, sys.exc_info())
1332 1333 1334 1335
                output = open(result_file, 'wb')
                pickle.dump(partial_result.data(), output)
            except:
                traceback.print_exc()
1336
        finally:
1337
            try: sys.stderr.flush()
1338
            except: pass
1339 1340 1341 1342 1343 1344 1345
            try: sys.stdout.flush()
            except: pass
            try:
                if output is not None:
                    output.close()
            except:
                pass
1346 1347 1348 1349
            os._exit(result_code)

    try:
        cid, result_code = os.waitpid(child_id, 0)
1350
        module_name = test_name.split()[-1]
1351 1352 1353 1354 1355 1356
        # os.waitpid returns the child's result code in the
        # upper byte of result_code, and the signal it was
        # killed by in the lower byte
        if result_code & 255:
            raise Exception("Tests in module '%s' were unexpectedly killed by signal %d"%
                            (module_name, result_code & 255))
1357
        result_code >>= 8
1358 1359 1360 1361 1362 1363 1364 1365 1366 1367
        if result_code in (0,1):
            input = open(result_file, 'rb')
            try:
                PartialTestResult.join_results(result, pickle.load(input))
            finally:
                input.close()
        if result_code:
            raise Exception("Tests in module '%s' exited with status %d" %
                            (module_name, result_code))
    finally:
1368 1369 1370 1371
        try:
            os.unlink(result_file)
        except:
            pass
1372

1373

1374
class PureDoctestTestCase(unittest.TestCase):
1375
    def __init__(self, module_name, module_path, tags, stats=None):
1376
        self.tags = tags
1377
        self.module_name = self.name = module_name
1378
        self.module_path = module_path
1379
        self.stats = stats
1380 1381 1382 1383 1384 1385 1386 1387 1388 1389 1390 1391 1392 1393
        unittest.TestCase.__init__(self, 'run')

    def shortDescription(self):
        return "running pure doctests in %s" % self.module_name

    def run(self, result=None):
        if result is None:
            result = self.defaultTestResult()
        loaded_module_name = 'pure_doctest__' + self.module_name
        result.startTest(self)
        try:
            self.setUp()

            import imp
1394
            with self.stats.time(self.name, 'py', 'pyimport'):
1395
                m = imp.load_source(loaded_module_name, self.module_path)
1396
            try:
1397
                with self.stats.time(self.name, 'py', 'pyrun'):
1398
                    doctest.DocTestSuite(m).run(result)
1399 1400 1401 1402
            finally:
                del m
                if loaded_module_name in sys.modules:
                    del sys.modules[loaded_module_name]
1403
                check_thread_termination()
1404 1405 1406 1407 1408 1409 1410
        except Exception:
            result.addError(self, sys.exc_info())
            result.stopTest(self)
        try:
            self.tearDown()
        except Exception:
            pass
1411

1412 1413 1414 1415 1416 1417
        if 'mypy' in self.tags['tag']:
            try:
                from mypy import api as mypy_api
            except ImportError:
                pass
            else:
1418
                with self.stats.time(self.name, 'py', 'mypy'):
1419 1420 1421 1422 1423 1424 1425
                    mypy_result = mypy_api.run((
                        self.module_path,
                        '--ignore-missing-imports',
                        '--follow-imports', 'skip',
                    ))
                if mypy_result[2]:
                    self.fail(mypy_result[0])
1426 1427


1428 1429 1430 1431
is_private_field = re.compile('^_[^_]').match

class _FakeClass(object):
    def __init__(self, **kwargs):
1432
        self._shortDescription = kwargs.get('module_name')
1433
        self.__dict__.update(kwargs)
1434 1435
    def shortDescription(self):
        return self._shortDescription
1436

1437 1438 1439 1440 1441 1442
try: # Py2.7+ and Py3.2+
    from unittest.runner import _TextTestResult
except ImportError:
    from unittest import _TextTestResult

class PartialTestResult(_TextTestResult):
1443
    def __init__(self, base_result):
1444
        _TextTestResult.__init__(
1445 1446 1447
            self, self._StringIO(), True,
            base_result.dots + base_result.showAll*2)

1448 1449 1450 1451 1452 1453
    def strip_error_results(self, results):
        for test_case, error in results:
            for attr_name in filter(is_private_field, dir(test_case)):
                if attr_name == '_dt_test':
                    test_case._dt_test = _FakeClass(
                        name=test_case._dt_test.name)
Craig Citro's avatar
Craig Citro committed
1454
                elif attr_name != '_shortDescription':
1455 1456
                    setattr(test_case, attr_name, None)

1457
    def data(self):
1458 1459
        self.strip_error_results(self.failures)
        self.strip_error_results(self.errors)
1460
        return (self.failures, self.errors, self.skipped, self.testsRun,
1461 1462 1463 1464 1465 1466
                self.stream.getvalue())

    def join_results(result, data):
        """Static method for merging the result back into the main
        result object.
        """
1467
        failures, errors, skipped, tests_run, output = data
1468 1469 1470
        if output:
            result.stream.write(output)
        result.errors.extend(errors)
1471
        result.skipped.extend(skipped)
1472 1473 1474 1475 1476 1477 1478 1479 1480 1481
        result.failures.extend(failures)
        result.testsRun += tests_run

    join_results = staticmethod(join_results)

    class _StringIO(StringIO):
        def writeln(self, line):
            self.write("%s\n" % line)


1482
class CythonUnitTestCase(CythonRunTestCase):
1483
    def shortDescription(self):
1484
        return "compiling (%s) tests in %s" % (self.language, self.name)
1485

1486
    def run_tests(self, result, ext_so_path):
1487
        with self.stats.time(self.name, self.language, 'import'):
1488
            module = import_ext(self.module, ext_so_path)
1489 1490 1491
        tests = unittest.defaultTestLoader.loadTestsFromModule(module)
        with self.stats.time(self.name, self.language, 'run'):
            tests.run(result)
1492 1493 1494


class CythonPyregrTestCase(CythonRunTestCase):
1495 1496 1497 1498
    def setUp(self):
        CythonRunTestCase.setUp(self)
        from Cython.Compiler import Options
        Options.error_on_unknown_names = False
1499
        Options.error_on_uninitialized = False
1500
        Options._directive_defaults.update(dict(
1501 1502
            binding=True, always_allow_keywords=True,
            set_initial_path="SOURCEFILE"))
1503
        patch_inspect_isfunction()
1504

1505 1506 1507
    def related_files(self, test_directory, module_name):
        return _list_pyregr_data_files(test_directory)

1508 1509 1510 1511 1512 1513 1514 1515 1516 1517 1518 1519 1520 1521
    def _run_unittest(self, result, *classes):
        """Run tests from unittest.TestCase-derived classes."""
        valid_types = (unittest.TestSuite, unittest.TestCase)
        suite = unittest.TestSuite()
        for cls in classes:
            if isinstance(cls, str):
                if cls in sys.modules:
                    suite.addTest(unittest.findTestCases(sys.modules[cls]))
                else:
                    raise ValueError("str arguments must be keys in sys.modules")
            elif isinstance(cls, valid_types):
                suite.addTest(cls)
            else:
                suite.addTest(unittest.makeSuite(cls))
1522
        with self.stats.time(self.name, self.language, 'run'):
1523
            suite.run(result)
1524 1525

    def _run_doctest(self, result, module):
1526
        self.run_doctests(module, result, None)
1527

1528
    def run_tests(self, result, ext_so_path):
1529
        try:
1530
            from test import support
Vitja Makarov's avatar
Vitja Makarov committed
1531 1532
        except ImportError: # Python2.x
            from test import test_support as support
1533

1534 1535 1536 1537 1538
        def run_test(result):
            def run_unittest(*classes):
                return self._run_unittest(result, *classes)
            def run_doctest(module, verbosity=None):
                return self._run_doctest(result, module)
1539

1540
            backup = (support.run_unittest, support.run_doctest)
1541 1542
            support.run_unittest = run_unittest
            support.run_doctest = run_doctest
1543

1544
            try:
1545 1546
                try:
                    sys.stdout.flush() # helps in case of crashes
1547 1548
                    with self.stats.time(self.name, self.language, 'import'):
                        module = import_ext(self.module, ext_so_path)
1549 1550
                    sys.stdout.flush() # helps in case of crashes
                    if hasattr(module, 'test_main'):
1551 1552 1553 1554 1555 1556 1557 1558 1559
                        # help 'doctest.DocFileTest' find the module path through frame inspection
                        fake_caller_module_globals = {
                            'module': module,
                            '__name__': module.__name__,
                        }
                        call_tests = eval(
                            'lambda: module.test_main()',
                            fake_caller_module_globals, fake_caller_module_globals)
                        call_tests()
1560 1561 1562 1563 1564
                        sys.stdout.flush() # helps in case of crashes
                except (unittest.SkipTest, support.ResourceDenied):
                    result.addSkip(self, 'ok')
            finally:
                support.run_unittest, support.run_doctest = backup
1565 1566

        run_forked_test(result, run_test, self.shortDescription(), self.fork)
1567

Stefan Behnel's avatar
Stefan Behnel committed
1568

1569 1570 1571 1572 1573 1574 1575 1576 1577
class TestCodeFormat(unittest.TestCase):

    def __init__(self, cython_dir):
        self.cython_dir = cython_dir
        unittest.TestCase.__init__(self)

    def runTest(self):
        import pycodestyle
        config_file = os.path.join(self.cython_dir, "tox.ini")
1578 1579
        if not os.path.exists(config_file):
            config_file=os.path.join(os.path.dirname(__file__), "tox.ini")
1580 1581 1582 1583 1584 1585 1586
        paths = glob.glob(os.path.join(self.cython_dir, "**/*.py"), recursive=True)
        style = pycodestyle.StyleGuide(config_file=config_file)
        print("")  # Fix the first line of the report.
        result = style.check_files(paths)
        self.assertEqual(result.total_errors, 0, "Found code style errors.")


1587
include_debugger = IS_CPYTHON
1588

Stefan Behnel's avatar
Stefan Behnel committed
1589

1590
def collect_unittests(path, module_prefix, suite, selectors, exclude_selectors):
1591 1592 1593 1594 1595 1596 1597
    def file_matches(filename):
        return filename.startswith("Test") and filename.endswith(".py")

    def package_matches(dirname):
        return dirname == "Tests"

    loader = unittest.TestLoader()
1598

1599 1600 1601
    if include_debugger:
        skipped_dirs = []
    else:
1602
        skipped_dirs = ['Cython' + os.path.sep + 'Debugger' + os.path.sep]
1603

1604
    for dirpath, dirnames, filenames in os.walk(path):
1605 1606 1607 1608 1609 1610 1611 1612 1613
        if dirpath != path and "__init__.py" not in filenames:
            skipped_dirs.append(dirpath + os.path.sep)
            continue
        skip = False
        for dir in skipped_dirs:
            if dirpath.startswith(dir):
                skip = True
        if skip:
            continue
1614 1615 1616 1617 1618
        parentname = os.path.split(dirpath)[-1]
        if package_matches(parentname):
            for f in filenames:
                if file_matches(f):
                    filepath = os.path.join(dirpath, f)[:-len(".py")]
1619
                    modulename = module_prefix + filepath[len(path)+1:].replace(os.path.sep, '.')
Stefan Behnel's avatar
Stefan Behnel committed
1620
                    if not any(1 for match in selectors if match(modulename)):
1621
                        continue
Stefan Behnel's avatar
Stefan Behnel committed
1622
                    if any(1 for match in exclude_selectors if match(modulename)):
1623
                        continue
1624 1625 1626
                    module = __import__(modulename)
                    for x in modulename.split('.')[1:]:
                        module = getattr(module, x)
Robert Bradshaw's avatar
Robert Bradshaw committed
1627
                    suite.addTests([loader.loadTestsFromModule(module)])
1628

1629

1630
def collect_doctests(path, module_prefix, suite, selectors, exclude_selectors):
1631
    def package_matches(dirname):
1632 1633
        if dirname == 'Debugger' and not include_debugger:
            return False
1634
        return dirname not in ("Mac", "Distutils", "Plex", "Tempita")
1635
    def file_matches(filename):
Mark Florisson's avatar
Mark Florisson committed
1636
        filename, ext = os.path.splitext(filename)
1637
        blacklist = ['libcython', 'libpython', 'test_libcython_in_gdb',
1638
                     'TestLibCython']
Mark Florisson's avatar
Mark Florisson committed
1639 1640 1641 1642 1643
        return (ext == '.py' and not
                '~' in filename and not
                '#' in filename and not
                filename.startswith('.') and not
                filename in blacklist)
Stefan Behnel's avatar
Stefan Behnel committed
1644
    import doctest
1645
    for dirpath, dirnames, filenames in os.walk(path):
Robert Bradshaw's avatar
Robert Bradshaw committed
1646 1647 1648 1649 1650 1651 1652 1653 1654 1655 1656 1657
        for dir in list(dirnames):
            if not package_matches(dir):
                dirnames.remove(dir)
        for f in filenames:
            if file_matches(f):
                if not f.endswith('.py'): continue
                filepath = os.path.join(dirpath, f)
                if os.path.getsize(filepath) == 0: continue
                filepath = filepath[:-len(".py")]
                modulename = module_prefix + filepath[len(path)+1:].replace(os.path.sep, '.')
                if not [ 1 for match in selectors if match(modulename) ]:
                    continue
1658 1659
                if [ 1 for match in exclude_selectors if match(modulename) ]:
                    continue
Robert Bradshaw's avatar
Robert Bradshaw committed
1660 1661 1662
                if 'in_gdb' in modulename:
                    # These should only be imported from gdb.
                    continue
Robert Bradshaw's avatar
Robert Bradshaw committed
1663 1664 1665 1666 1667 1668 1669 1670
                module = __import__(modulename)
                for x in modulename.split('.')[1:]:
                    module = getattr(module, x)
                if hasattr(module, "__doc__") or hasattr(module, "__test__"):
                    try:
                        suite.addTest(doctest.DocTestSuite(module))
                    except ValueError: # no tests
                        pass
1671

1672 1673 1674 1675 1676 1677

class EndToEndTest(unittest.TestCase):
    """
    This is a test of build/*.srctree files, where srctree defines a full
    directory structure and its header gives a list of commands to run.
    """
Robert Bradshaw's avatar
Robert Bradshaw committed
1678
    cython_root = os.path.dirname(os.path.abspath(__file__))
1679

1680 1681
    def __init__(self, treefile, workdir, cleanup_workdir=True, stats=None,
                 capture=True):
1682
        self.name = os.path.splitext(os.path.basename(treefile))[0]
1683
        self.treefile = treefile
1684
        self.workdir = os.path.join(workdir, self.name)
1685
        self.cleanup_workdir = cleanup_workdir
1686
        self.stats = stats
1687
        self.capture = capture
1688 1689 1690
        cython_syspath = [self.cython_root]
        for path in sys.path:
            if path.startswith(self.cython_root) and path not in cython_syspath:
1691 1692
                # Py3 installation and refnanny build prepend their
                # fixed paths to sys.path => prefer that over the
1693 1694 1695
                # generic one (cython_root itself goes last)
                cython_syspath.append(path)
        self.cython_syspath = os.pathsep.join(cython_syspath[::-1])
1696 1697 1698
        unittest.TestCase.__init__(self)

    def shortDescription(self):
1699
        return "End-to-end %s" % self.name
1700 1701 1702

    def setUp(self):
        from Cython.TestUtils import unpack_source_tree
1703
        _, self.commands = unpack_source_tree(self.treefile, self.workdir)
1704 1705 1706 1707 1708 1709 1710
        self.old_dir = os.getcwd()
        os.chdir(self.workdir)
        if self.workdir not in sys.path:
            sys.path.insert(0, self.workdir)

    def tearDown(self):
        if self.cleanup_workdir:
1711 1712 1713 1714 1715 1716 1717
            for trial in range(5):
                try:
                    shutil.rmtree(self.workdir)
                except OSError:
                    time.sleep(0.1)
                else:
                    break
1718
        os.chdir(self.old_dir)
1719

1720 1721 1722 1723 1724 1725
    def _try_decode(self, content):
        try:
            return content.decode()
        except UnicodeDecodeError:
            return content.decode('iso-8859-1')

1726
    def runTest(self):
1727
        self.success = False
1728
        commands = (self.commands
Robert Bradshaw's avatar
Robert Bradshaw committed
1729
            .replace("CYTHON", "PYTHON %s" % os.path.join(self.cython_root, 'cython.py'))
1730
            .replace("PYTHON", sys.executable))
1731
        old_path = os.environ.get('PYTHONPATH')
1732
        env = dict(os.environ)
1733 1734 1735 1736
        new_path = self.cython_syspath
        if old_path:
            new_path = new_path + os.pathsep + old_path
        env['PYTHONPATH'] = new_path
1737 1738 1739
        cmd = []
        out = []
        err = []
1740 1741 1742
        for command_no, command in enumerate(filter(None, commands.splitlines()), 1):
            with self.stats.time('%s(%d)' % (self.name, command_no), 'c',
                                 'etoe-build' if ' setup.py ' in command else 'etoe-run'):
1743 1744
                if self.capture:
                    p = subprocess.Popen(command,
1745 1746 1747 1748
                                     stderr=subprocess.PIPE,
                                     stdout=subprocess.PIPE,
                                     shell=True,
                                     env=env)
1749 1750 1751 1752 1753 1754 1755 1756
                    _out, _err = p.communicate()
                    res = p.returncode
                else:
                    p = subprocess.call(command,
                                     shell=True,
                                     env=env)
                    _out, _err = b'', b''
                    res = p
1757 1758 1759
                cmd.append(command)
                out.append(_out)
                err.append(_err)
1760
            if res != 0:
1761 1762 1763
                for c, o, e in zip(cmd, out, err):
                    sys.stderr.write("%s\n%s\n%s\n\n" % (
                        c, self._try_decode(o), self._try_decode(e)))
1764
            self.assertEqual(0, res, "non-zero exit status")
1765
        self.success = True
1766 1767


1768 1769 1770 1771
# TODO: Support cython_freeze needed here as well.
# TODO: Windows support.

class EmbedTest(unittest.TestCase):
1772

1773
    working_dir = "Demos/embed"
1774

1775 1776 1777
    def setUp(self):
        self.old_dir = os.getcwd()
        os.chdir(self.working_dir)
1778
        os.system(
1779
            "make PYTHON='%s' clean > /dev/null" % sys.executable)
1780

1781 1782
    def tearDown(self):
        try:
1783 1784
            os.system(
                "make PYTHON='%s' clean > /dev/null" % sys.executable)
1785 1786 1787
        except:
            pass
        os.chdir(self.old_dir)
1788

1789
    def test_embed(self):
1790
        libname = sysconfig.get_config_var('LIBRARY')
1791
        libdir = sysconfig.get_config_var('LIBDIR')
1792 1793 1794 1795 1796 1797 1798
        if not os.path.isdir(libdir) or libname not in os.listdir(libdir):
            libdir = os.path.join(os.path.dirname(sys.executable), '..', 'lib')
            if not os.path.isdir(libdir) or libname not in os.listdir(libdir):
                libdir = os.path.join(libdir, 'python%d.%d' % sys.version_info[:2], 'config')
                if not os.path.isdir(libdir) or libname not in os.listdir(libdir):
                    # report the error for the original directory
                    libdir = sysconfig.get_config_var('LIBDIR')
1799
        cython = 'cython.py'
Stefan Behnel's avatar
Stefan Behnel committed
1800
        if sys.version_info[0] >=3 and CY3_DIR:
1801 1802
            cython = os.path.join(CY3_DIR, cython)
        cython = os.path.abspath(os.path.join('..', '..', cython))
1803 1804
        self.assertEqual(0, os.system(
            "make PYTHON='%s' CYTHON='%s' LIBDIR1='%s' test > make.output" % (sys.executable, cython, libdir)))
1805 1806 1807 1808
        try:
            os.remove('make.output')
        except OSError:
            pass
1809

1810 1811

class MissingDependencyExcluder(object):
1812
    def __init__(self, deps):
1813
        # deps: { matcher func : module name }
1814
        self.exclude_matchers = []
1815
        for matcher, mod in deps.items():
1816 1817 1818
            try:
                __import__(mod)
            except ImportError:
Robert Bradshaw's avatar
Robert Bradshaw committed
1819
                self.exclude_matchers.append(string_selector(matcher))
1820
        self.tests_missing_deps = []
Robert Bradshaw's avatar
Robert Bradshaw committed
1821
    def __call__(self, testname, tags=None):
1822
        for matcher in self.exclude_matchers:
Robert Bradshaw's avatar
Robert Bradshaw committed
1823
            if matcher(testname, tags):
1824 1825 1826 1827
                self.tests_missing_deps.append(testname)
                return True
        return False

1828 1829

class VersionDependencyExcluder(object):
1830 1831 1832 1833
    def __init__(self, deps):
        # deps: { version : matcher func }
        from sys import version_info
        self.exclude_matchers = []
1834 1835
        for ver, (compare, matcher) in deps.items():
            if compare(version_info, ver):
1836 1837
                self.exclude_matchers.append(matcher)
        self.tests_missing_deps = []
Robert Bradshaw's avatar
Robert Bradshaw committed
1838
    def __call__(self, testname, tags=None):
1839 1840 1841 1842 1843 1844
        for matcher in self.exclude_matchers:
            if matcher(testname):
                self.tests_missing_deps.append(testname)
                return True
        return False

1845

1846
class FileListExcluder(object):
1847 1848
    def __init__(self, list_file, verbose=False):
        self.verbose = verbose
1849
        self.excludes = {}
1850 1851 1852
        self._list_file = os.path.relpath(list_file)
        with open(list_file) as f:
            for line in f:
1853 1854 1855
                line = line.strip()
                if line and line[0] != '#':
                    self.excludes[line.split()[0]] = True
1856

Robert Bradshaw's avatar
Robert Bradshaw committed
1857
    def __call__(self, testname, tags=None):
1858 1859
        exclude = (testname in self.excludes
                   or testname.split('.')[-1] in self.excludes)
1860
        if exclude and self.verbose:
1861 1862 1863
            print("Excluding %s because it's listed in %s"
                  % (testname, self._list_file))
        return exclude
1864

1865

1866
class TagsSelector(object):
Robert Bradshaw's avatar
Robert Bradshaw committed
1867 1868 1869
    def __init__(self, tag, value):
        self.tag = tag
        self.value = value
1870

Robert Bradshaw's avatar
Robert Bradshaw committed
1871 1872 1873 1874 1875 1876
    def __call__(self, testname, tags=None):
        if tags is None:
            return False
        else:
            return self.value in tags[self.tag]

1877

1878
class RegExSelector(object):
Robert Bradshaw's avatar
Robert Bradshaw committed
1879
    def __init__(self, pattern_string):
1880
        try:
1881
            self.regex_matches = re.compile(pattern_string, re.I|re.U).search
1882 1883 1884
        except re.error:
            print('Invalid pattern: %r' % pattern_string)
            raise
Robert Bradshaw's avatar
Robert Bradshaw committed
1885 1886

    def __call__(self, testname, tags=None):
1887
        return self.regex_matches(testname)
Robert Bradshaw's avatar
Robert Bradshaw committed
1888

1889

Robert Bradshaw's avatar
Robert Bradshaw committed
1890
def string_selector(s):
1891 1892
    if ':' in s:
        return TagsSelector(*s.split(':', 1))
Robert Bradshaw's avatar
Robert Bradshaw committed
1893
    else:
1894
        return RegExSelector(s)
1895

1896 1897

class ShardExcludeSelector(object):
1898
    # This is an exclude selector so it can override the (include) selectors.
1899 1900
    # It may not provide uniform distribution (in time or count), but is a
    # determanistic partition of the tests which is important.
1901 1902 1903 1904
    def __init__(self, shard_num, shard_count):
        self.shard_num = shard_num
        self.shard_count = shard_count

1905 1906
    def __call__(self, testname, tags=None, _hash=zlib.crc32, _is_py2=sys.version_info[0] < 3):
        # Cannot use simple hash() here as shard processes might use different hash seeds.
1907 1908
        # CRC32 is fast and simple, but might return negative values in Py2.
        hashval = _hash(testname) & 0x7fffffff if _is_py2 else _hash(testname.encode())
1909
        return hashval % self.shard_count != self.shard_num
1910

Robert Bradshaw's avatar
Robert Bradshaw committed
1911

1912 1913
class PendingThreadsError(RuntimeError):
    pass
1914

1915 1916 1917
threads_seen = []

def check_thread_termination(ignore_seen=True):
1918 1919 1920 1921 1922
    if threading is None: # no threading enabled in CPython
        return
    current = threading.currentThread()
    blocking_threads = []
    for t in threading.enumerate():
1923
        if not t.isAlive() or t == current or t.name == 'time_stamper':
1924 1925 1926
            continue
        t.join(timeout=2)
        if t.isAlive():
1927 1928 1929
            if not ignore_seen:
                blocking_threads.append(t)
                continue
1930 1931 1932 1933 1934 1935
            for seen in threads_seen:
                if t is seen:
                    break
            else:
                threads_seen.append(t)
                blocking_threads.append(t)
1936 1937 1938 1939 1940
    if not blocking_threads:
        return
    sys.stderr.write("warning: left-over threads found after running test:\n")
    for t in blocking_threads:
        sys.stderr.write('...%s\n'  % repr(t))
1941
    raise PendingThreadsError("left-over threads found after running test")
1942

1943 1944
def subprocess_output(cmd):
    try:
Mark Florisson's avatar
Mark Florisson committed
1945 1946
        p = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
        return p.communicate()[0].decode('UTF-8')
1947 1948 1949 1950 1951 1952 1953 1954
    except OSError:
        return ''

def get_version():
    from Cython.Compiler.Version import version as cython_version
    full_version = cython_version
    top = os.path.dirname(os.path.abspath(__file__))
    if os.path.exists(os.path.join(top, '.git')):
Stefan Behnel's avatar
Stefan Behnel committed
1955
        old_dir = os.getcwd()
1956 1957 1958 1959 1960 1961 1962 1963 1964 1965 1966 1967 1968
        try:
            os.chdir(top)
            head_commit = subprocess_output(['git', 'rev-parse', 'HEAD']).strip()
            version_commit = subprocess_output(['git', 'rev-parse', cython_version]).strip()
            diff = subprocess_output(['git', 'diff', '--stat']).strip()
            if head_commit != version_commit:
                full_version += " " + head_commit
            if diff:
                full_version += ' + uncommitted changes'
        finally:
            os.chdir(old_dir)
    return full_version

1969 1970 1971 1972 1973 1974 1975 1976
_orig_stdout, _orig_stderr = sys.stdout, sys.stderr
def flush_and_terminate(status):
    try:
        _orig_stdout.flush()
        _orig_stderr.flush()
    finally:
        os._exit(status)

1977
def main():
1978

1979
    global DISTDIR, WITH_CYTHON
1980 1981
    DISTDIR = os.path.join(os.getcwd(), os.path.dirname(sys.argv[0]))

1982 1983 1984 1985 1986 1987 1988 1989
    from Cython.Compiler import DebugFlags
    args = []
    for arg in sys.argv[1:]:
        if arg.startswith('--debug') and arg[2:].replace('-', '_') in dir(DebugFlags):
            setattr(DebugFlags, arg[2:].replace('-', '_'), True)
        else:
            args.append(arg)

1990 1991 1992
    from optparse import OptionParser
    parser = OptionParser()
    parser.add_option("--no-cleanup", dest="cleanup_workdir",
Stefan Behnel's avatar
Stefan Behnel committed
1993 1994
                      action="store_false", default=True,
                      help="do not delete the generated C files (allows passing --no-cython on next run)")
1995 1996
    parser.add_option("--no-cleanup-sharedlibs", dest="cleanup_sharedlibs",
                      action="store_false", default=True,
Unknown's avatar
Unknown committed
1997
                      help="do not delete the generated shared library files (allows manual module experimentation)")
1998 1999 2000
    parser.add_option("--no-cleanup-failures", dest="cleanup_failures",
                      action="store_false", default=True,
                      help="enable --no-cleanup and --no-cleanup-sharedlibs for failed tests only")
Stefan Behnel's avatar
Stefan Behnel committed
2001 2002 2003
    parser.add_option("--no-cython", dest="with_cython",
                      action="store_false", default=True,
                      help="do not run the Cython compiler, only the C compiler")
2004 2005
    parser.add_option("--compiler", dest="compiler", default=None,
                      help="C compiler type")
2006 2007 2008
    backend_list = ','.join(BACKENDS)
    parser.add_option("--backends", dest="backends", default=backend_list,
                      help="select backends to test (default: %s)" % backend_list)
2009 2010
    parser.add_option("--no-c", dest="use_c",
                      action="store_false", default=True,
2011
                      help="do not test C compilation backend")
2012 2013
    parser.add_option("--no-cpp", dest="use_cpp",
                      action="store_false", default=True,
2014
                      help="do not test C++ compilation backend")
2015 2016 2017
    parser.add_option("--no-unit", dest="unittests",
                      action="store_false", default=True,
                      help="do not run the unit tests")
2018 2019 2020
    parser.add_option("--no-doctest", dest="doctests",
                      action="store_false", default=True,
                      help="do not run the doctests")
2021 2022 2023
    parser.add_option("--no-file", dest="filetests",
                      action="store_false", default=True,
                      help="do not run the file based tests")
2024 2025
    parser.add_option("--no-pyregr", dest="pyregr",
                      action="store_false", default=True,
2026
                      help="do not run the regression tests of CPython in tests/pyregr/")
2027 2028 2029
    parser.add_option("--no-examples", dest="examples",
                      action="store_false", default=True,
                      help="Do not run the documentation tests in the examples directory.")
2030 2031 2032
    parser.add_option("--no-code-style", dest="code_style",
                      action="store_false", default=True,
                      help="Do not run the code style (PEP8) checks.")
2033
    parser.add_option("--cython-only", dest="cython_only",
2034 2035
                      action="store_true", default=False,
                      help="only compile pyx to c, do not run C compiler or run the tests")
2036
    parser.add_option("--no-refnanny", dest="with_refnanny",
2037
                      action="store_false", default=True,
2038
                      help="do not regression test reference counting")
2039 2040 2041
    parser.add_option("--no-fork", dest="fork",
                      action="store_false", default=True,
                      help="do not fork to run tests")
2042 2043 2044
    parser.add_option("--sys-pyregr", dest="system_pyregr",
                      action="store_true", default=False,
                      help="run the regression tests of the CPython installation")
2045 2046 2047
    parser.add_option("-x", "--exclude", dest="exclude",
                      action="append", metavar="PATTERN",
                      help="exclude tests matching the PATTERN")
2048
    parser.add_option("-j", "--shard_count", dest="shard_count", metavar="N",
2049 2050 2051 2052 2053
                      type=int, default=1,
                      help="shard this run into several parallel runs")
    parser.add_option("--shard_num", dest="shard_num", metavar="K",
                      type=int, default=-1,
                      help="test only this single shard")
2054 2055 2056
    parser.add_option("--profile", dest="profile",
                      action="store_true", default=False,
                      help="enable profiling of the tests")
2057
    parser.add_option("-C", "--coverage", dest="coverage",
Stefan Behnel's avatar
Stefan Behnel committed
2058 2059
                      action="store_true", default=False,
                      help="collect source coverage data for the Compiler")
2060 2061 2062
    parser.add_option("--coverage-xml", dest="coverage_xml",
                      action="store_true", default=False,
                      help="collect source coverage data for the Compiler in XML format")
2063 2064 2065
    parser.add_option("--coverage-html", dest="coverage_html",
                      action="store_true", default=False,
                      help="collect source coverage data for the Compiler in HTML format")
Stefan Behnel's avatar
Stefan Behnel committed
2066
    parser.add_option("-A", "--annotate", dest="annotate_source",
2067
                      action="store_true", default=True,
Stefan Behnel's avatar
Stefan Behnel committed
2068
                      help="generate annotated HTML versions of the test source files")
2069 2070 2071
    parser.add_option("--no-annotate", dest="annotate_source",
                      action="store_false",
                      help="do not generate annotated HTML versions of the test source files")
2072
    parser.add_option("-v", "--verbose", dest="verbosity",
Stefan Behnel's avatar
Stefan Behnel committed
2073 2074
                      action="count", default=0,
                      help="display test progress, pass twice to print test names")
2075 2076
    parser.add_option("-T", "--ticket", dest="tickets",
                      action="append",
2077
                      help="a bug ticket number to run the respective test in 'tests/*'")
2078 2079 2080
    parser.add_option("-3", dest="language_level",
                      action="store_const", const=3, default=2,
                      help="set language level to Python 3 (useful for running the CPython regression tests)'")
2081 2082
    parser.add_option("--xml-output", dest="xml_output_dir", metavar="DIR",
                      help="write test results in XML to directory DIR")
2083 2084 2085
    parser.add_option("--exit-ok", dest="exit_ok", default=False,
                      action="store_true",
                      help="exit without error code even on test failures")
2086 2087 2088
    parser.add_option("--failfast", dest="failfast", default=False,
                      action="store_true",
                      help="stop on first failure or error")
2089
    parser.add_option("--root-dir", dest="root_dir", default=os.path.join(DISTDIR, 'tests'),
2090 2091
                      help=("Directory to look for the file based "
                            "tests (the ones which are deactivated with '--no-file'."))
2092 2093
    parser.add_option("--examples-dir", dest="examples_dir",
                      default=os.path.join(DISTDIR, 'docs', 'examples'),
2094
                      help="Directory to look for documentation example tests")
2095
    parser.add_option("--work-dir", dest="work_dir", default=os.path.join(os.getcwd(), 'TEST_TMP'),
2096
                      help="working directory")
2097 2098
    parser.add_option("--cython-dir", dest="cython_dir", default=os.getcwd(),
                      help="Cython installation directory (default: use local source version)")
2099 2100
    parser.add_option("--debug", dest="for_debugging", default=False, action="store_true",
                      help="configure for easier use with a debugger (e.g. gdb)")
2101 2102
    parser.add_option("--pyximport-py", dest="pyximport_py", default=False, action="store_true",
                      help="use pyximport to automatically compile imported .pyx and .py files")
2103 2104
    parser.add_option("--watermark", dest="watermark", default=None,
                      help="deterministic generated by string")
2105
    parser.add_option("--use_common_utility_dir", default=False, action="store_true")
2106
    parser.add_option("--use_formal_grammar", default=False, action="store_true")
2107 2108
    parser.add_option("--test_determinism", default=False, action="store_true",
                      help="test whether Cython's output is deterministic")
2109 2110
    parser.add_option("--pythran-dir", dest="pythran_dir", default=None,
                      help="specify Pythran include directory. This will run the C++ tests using Pythran backend for Numpy")
2111 2112
    parser.add_option("--no-capture", dest="capture", default=True, action="store_false",
                      help="do not capture stdout, stderr in srctree tests. Makes pdb.set_trace interactive")
2113

2114
    options, cmd_args = parser.parse_args(args)
2115

2116 2117
    if options.with_cython and sys.version_info[0] >= 3:
        sys.path.insert(0, options.cython_dir)
2118

2119 2120 2121 2122
    # requires glob with the wildcard.
    if sys.version_info < (3, 5) or cmd_args:
        options.code_style = False

2123
    WITH_CYTHON = options.with_cython
2124

2125
    coverage = None
2126
    if options.coverage or options.coverage_xml or options.coverage_html:
2127 2128 2129 2130 2131
        if not WITH_CYTHON:
            options.coverage = options.coverage_xml = options.coverage_html = False
        elif options.shard_num == -1:
            print("Enabling coverage analysis")
            from coverage import coverage as _coverage
2132
            coverage = _coverage(branch=True)
2133 2134
            coverage.erase()
            coverage.start()
2135

2136 2137 2138
    if options.xml_output_dir:
        shutil.rmtree(options.xml_output_dir, ignore_errors=True)

2139 2140 2141 2142
    if options.capture:
        keep_alive_interval = 10
    else:
        keep_alive_interval = None
2143 2144 2145 2146 2147
    if options.shard_count > 1 and options.shard_num == -1:
        import multiprocessing
        pool = multiprocessing.Pool(options.shard_count)
        tasks = [(options, cmd_args, shard_num) for shard_num in range(options.shard_count)]
        errors = []
2148
        # NOTE: create process pool before time stamper thread to avoid forking issues.
2149
        total_time = time.time()
2150
        stats = Stats()
2151
        with time_stamper_thread(interval=keep_alive_interval):
2152
            for shard_num, shard_stats, return_code in pool.imap_unordered(runtests_callback, tasks):
2153 2154
                if return_code != 0:
                    errors.append(shard_num)
2155 2156
                    sys.stderr.write("FAILED (%s/%s)\n" % (shard_num, options.shard_count))
                sys.stderr.write("ALL DONE (%s/%s)\n" % (shard_num, options.shard_count))
2157
                stats.update(shard_stats)
2158 2159
        pool.close()
        pool.join()
2160
        total_time = time.time() - total_time
2161
        sys.stderr.write("Sharded tests run in %d seconds (%.1f minutes)\n" % (round(total_time), total_time / 60.))
2162
        if errors:
2163
            sys.stderr.write("Errors for shards %s\n" % ", ".join([str(e) for e in errors]))
2164 2165 2166 2167
            return_code = 1
        else:
            return_code = 0
    else:
2168
        with time_stamper_thread(interval=keep_alive_interval):
2169
            _, stats, return_code = runtests(options, cmd_args, coverage)
2170 2171 2172 2173 2174 2175

    if coverage:
        if options.shard_count > 1 and options.shard_num == -1:
            coverage.combine()
        coverage.stop()

2176
    stats.print_stats(sys.stderr)
2177 2178 2179
    if coverage:
        save_coverage(coverage, options)

2180 2181 2182
    sys.stderr.write("ALL DONE\n")
    sys.stderr.flush()

2183 2184 2185 2186 2187
    try:
        check_thread_termination(ignore_seen=False)
    except PendingThreadsError:
        # normal program exit won't kill the threads, do it the hard way here
        flush_and_terminate(return_code)
Stefan Behnel's avatar
Stefan Behnel committed
2188 2189
    else:
        sys.exit(return_code)
2190 2191


2192 2193 2194 2195 2196 2197 2198 2199 2200 2201 2202 2203 2204 2205 2206
@contextmanager
def time_stamper_thread(interval=10):
    """
    Print regular time stamps into the build logs to find slow tests.
    @param interval: time interval in seconds
    """
    try:
        _xrange = xrange
    except NameError:
        _xrange = range

    import threading
    from datetime import datetime
    from time import sleep

2207 2208
    if not interval or interval < 0:
        # Do nothing
2209
        yield
2210 2211 2212 2213 2214 2215 2216 2217 2218 2219 2220 2221 2222 2223 2224 2225 2226 2227 2228 2229 2230 2231
    else:
        interval = _xrange(interval * 4)
        now = datetime.now
        write = sys.__stderr__.write
        stop = False

        def time_stamper():
            while True:
                for _ in interval:
                    if stop:
                        return
                    sleep(1./4)
                write('\n#### %s\n' % now())

        thread = threading.Thread(target=time_stamper, name='time_stamper')
        thread.setDaemon(True)  # Py2 ...
        thread.start()
        try:
            yield
        finally:
            stop = True
            thread.join()
2232 2233


2234 2235 2236 2237
def configure_cython(options):
    global CompilationOptions, pyrex_default_options, cython_compile
    from Cython.Compiler.Main import \
        CompilationOptions, \
2238 2239
        default_options as pyrex_default_options
    from Cython.Compiler.Options import _directive_defaults as directive_defaults
2240 2241 2242 2243 2244 2245 2246
    from Cython.Compiler import Errors
    Errors.LEVEL = 0  # show all warnings
    from Cython.Compiler import Options
    Options.generate_cleanup_code = 3  # complete cleanup code
    from Cython.Compiler import DebugFlags
    DebugFlags.debug_temp_code_comments = 1
    pyrex_default_options['formal_grammar'] = options.use_formal_grammar
2247 2248
    if options.profile:
        directive_defaults['profile'] = True
2249 2250 2251
    if options.watermark:
        import Cython.Compiler.Version
        Cython.Compiler.Version.watermark = options.watermark
2252 2253


2254 2255 2256 2257 2258 2259 2260 2261 2262
def save_coverage(coverage, options):
    if options.coverage:
        coverage.report(show_missing=0)
    if options.coverage_xml:
        coverage.xml_report(outfile="coverage-report.xml")
    if options.coverage_html:
        coverage.html_report(directory="coverage-report-html")


2263 2264 2265 2266 2267
def runtests_callback(args):
    options, cmd_args, shard_num = args
    options.shard_num = shard_num
    return runtests(options, cmd_args)

2268

2269
def runtests(options, cmd_args, coverage=None):
2270

2271 2272 2273 2274
    WITH_CYTHON = options.with_cython
    ROOTDIR = os.path.abspath(options.root_dir)
    WORKDIR = os.path.abspath(options.work_dir)

2275 2276 2277
    if WITH_CYTHON:
        configure_cython(options)

2278
    xml_output_dir = options.xml_output_dir
2279 2280
    if options.shard_num > -1:
        WORKDIR = os.path.join(WORKDIR, str(options.shard_num))
2281 2282
        if xml_output_dir:
            xml_output_dir = os.path.join(xml_output_dir, 'shard-%03d' % options.shard_num)
2283

2284
    # RUN ALL TESTS!
2285
    UNITTEST_MODULE = "Cython"
2286
    UNITTEST_ROOT = os.path.join(os.path.dirname(__file__), UNITTEST_MODULE)
2287 2288
    if WITH_CYTHON:
        if os.path.exists(WORKDIR):
2289
            for path in os.listdir(WORKDIR):
2290
                if path in ("support", "Cy3"): continue
2291
                shutil.rmtree(os.path.join(WORKDIR, path), ignore_errors=True)
2292 2293
    if not os.path.exists(WORKDIR):
        os.makedirs(WORKDIR)
2294

2295 2296 2297 2298 2299 2300 2301
    if options.shard_num <= 0:
        sys.stderr.write("Python %s\n" % sys.version)
        sys.stderr.write("\n")
        if WITH_CYTHON:
            sys.stderr.write("Running tests against Cython %s\n" % get_version())
        else:
            sys.stderr.write("Running tests without Cython.\n")
2302

2303 2304 2305 2306
    if options.for_debugging:
        options.cleanup_workdir = False
        options.cleanup_sharedlibs = False
        options.fork = False
2307
        if WITH_CYTHON and include_debugger:
2308 2309 2310
            from Cython.Compiler.Main import default_options as compiler_default_options
            compiler_default_options['gdb_debug'] = True
            compiler_default_options['output_dir'] = os.getcwd()
2311

2312 2313 2314 2315 2316
    if IS_PYPY:
        if options.with_refnanny:
            sys.stderr.write("Disabling refnanny in PyPy\n")
            options.with_refnanny = False

2317 2318 2319 2320 2321
    if options.with_refnanny:
        from pyximport.pyxbuild import pyx_to_dll
        libpath = pyx_to_dll(os.path.join("Cython", "Runtime", "refnanny.pyx"),
                             build_in_temp=True,
                             pyxbuild_dir=os.path.join(WORKDIR, "support"))
2322
        sys.path.insert(0, os.path.split(libpath)[0])
2323
        CFLAGS.append("-DCYTHON_REFNANNY=1")
2324

2325
    if xml_output_dir and options.fork:
Stefan Behnel's avatar
Stefan Behnel committed
2326 2327 2328 2329
        # doesn't currently work together
        sys.stderr.write("Disabling forked testing to support XML test output\n")
        options.fork = False

2330 2331
    if WITH_CYTHON:
        sys.stderr.write("Using Cython language level %d.\n" % options.language_level)
2332

2333
    test_bugs = False
Stefan Behnel's avatar
Stefan Behnel committed
2334 2335 2336
    if options.tickets:
        for ticket_number in options.tickets:
            test_bugs = True
2337
            cmd_args.append('ticket:%s' % ticket_number)
2338 2339 2340 2341
    if not test_bugs:
        for selector in cmd_args:
            if selector.startswith('bugs'):
                test_bugs = True
2342

Robert Bradshaw's avatar
Robert Bradshaw committed
2343
    selectors = [ string_selector(r) for r in cmd_args ]
2344
    verbose_excludes = selectors or options.verbosity >= 2
2345
    if not selectors:
Robert Bradshaw's avatar
Robert Bradshaw committed
2346
        selectors = [ lambda x, tags=None: True ]
2347

2348
    # Check which external modules are not present and exclude tests
2349 2350
    # which depends on them (by prefix)

2351 2352
    missing_dep_excluder = MissingDependencyExcluder(EXT_DEP_MODULES)
    version_dep_excluder = VersionDependencyExcluder(VER_DEP_MODULES)
Robert Bradshaw's avatar
Robert Bradshaw committed
2353
    exclude_selectors = [missing_dep_excluder, version_dep_excluder] # want to print msg at exit
2354

2355
    try:
2356 2357 2358 2359
        import IPython.core.release
        if list(IPython.core.release._ver) < [1, 0, 0]:
            raise ImportError
    except (ImportError, AttributeError, TypeError):
2360
        exclude_selectors.append(RegExSelector('IPython'))
2361

2362
    try:
2363
        raise ImportError("Jedi typer is currently broken, see GH#1845")
2364
        import jedi
2365
        if not ([0, 9] <= list(map(int, re.findall('[0-9]+', jedi.__version__ or '0')))):
2366
            raise ImportError
Stefan Behnel's avatar
Stefan Behnel committed
2367
    except (ImportError, AttributeError, TypeError):
2368 2369
        exclude_selectors.append(RegExSelector('Jedi'))

2370
    if options.exclude:
Robert Bradshaw's avatar
Robert Bradshaw committed
2371
        exclude_selectors += [ string_selector(r) for r in options.exclude ]
2372

2373 2374 2375
    if not COMPILER_HAS_INT128 or not IS_CPYTHON:
        exclude_selectors += [RegExSelector('int128')]

2376 2377 2378
    if options.shard_num > -1:
        exclude_selectors.append(ShardExcludeSelector(options.shard_num, options.shard_count))

2379
    if not test_bugs:
2380 2381 2382 2383 2384 2385 2386
        bug_files = [
            ('bugs.txt', True),
            ('pypy_bugs.txt', IS_PYPY),
            ('windows_bugs.txt', sys.platform == 'win32'),
            ('cygwin_bugs.txt', sys.platform == 'cygwin')
        ]

2387
        exclude_selectors += [
2388 2389 2390
            FileListExcluder(os.path.join(ROOTDIR, bugs_file_name),
                             verbose=verbose_excludes)
            for bugs_file_name, condition in bug_files if condition
2391
        ]
2392

2393 2394 2395
    global COMPILER
    if options.compiler:
        COMPILER = options.compiler
2396 2397 2398 2399 2400 2401 2402 2403 2404 2405 2406 2407 2408

    selected_backends = [ name.strip() for name in options.backends.split(',') if name.strip() ]
    backends = []
    for backend in selected_backends:
        if backend == 'c' and not options.use_c:
            continue
        elif backend == 'cpp' and not options.use_cpp:
            continue
        elif backend not in BACKENDS:
            sys.stderr.write("Unknown backend requested: '%s' not one of [%s]\n" % (
                backend, ','.join(BACKENDS)))
            sys.exit(1)
        backends.append(backend)
2409 2410
    if options.shard_num <= 0:
        sys.stderr.write("Backends: %s\n" % ','.join(backends))
2411 2412
    languages = backends

2413 2414 2415 2416 2417 2418 2419
    if 'TRAVIS' in os.environ and sys.platform == 'darwin' and 'cpp' in languages:
        bugs_file_name = 'travis_macos_cpp_bugs.txt'
        exclude_selectors += [
            FileListExcluder(os.path.join(ROOTDIR, bugs_file_name),
                             verbose=verbose_excludes)
        ]

2420 2421 2422 2423 2424 2425 2426
    if options.use_common_utility_dir:
        common_utility_dir = os.path.join(WORKDIR, 'utility_code')
        if not os.path.exists(common_utility_dir):
            os.makedirs(common_utility_dir)
    else:
        common_utility_dir = None

2427
    sys.stderr.write("\n")
2428

2429
    test_suite = unittest.TestSuite()
2430
    stats = Stats()
2431 2432

    if options.unittests:
2433
        collect_unittests(UNITTEST_ROOT, UNITTEST_MODULE + ".", test_suite, selectors, exclude_selectors)
2434

2435
    if options.doctests:
2436
        collect_doctests(UNITTEST_ROOT, UNITTEST_MODULE + ".", test_suite, selectors, exclude_selectors)
2437

2438
    if options.filetests and languages:
2439
        filetests = TestBuilder(ROOTDIR, WORKDIR, selectors, exclude_selectors,
gabrieldemarmiesse's avatar
gabrieldemarmiesse committed
2440 2441
                                options, options.pyregr, languages, test_bugs,
                                options.language_level, common_utility_dir,
2442
                                options.pythran_dir, add_embedded_test=True, stats=stats)
2443
        test_suite.addTest(filetests.build_suite())
2444
    if options.examples and languages:
2445 2446 2447 2448 2449
        for subdirectory in glob.glob(os.path.join(options.examples_dir, "*/")):
            filetests = TestBuilder(subdirectory, WORKDIR, selectors, exclude_selectors,
                                    options, options.pyregr, languages, test_bugs,
                                    options.language_level, common_utility_dir,
                                    options.pythran_dir,
2450
                                    default_mode='compile', stats=stats)
2451
            test_suite.addTest(filetests.build_suite())
2452

2453
    if options.system_pyregr and languages:
2454
        sys_pyregr_dir = os.path.join(sys.prefix, 'lib', 'python'+sys.version[:3], 'test')
2455 2456
        if not os.path.isdir(sys_pyregr_dir):
            sys_pyregr_dir = os.path.join(os.path.dirname(sys.executable), 'Lib', 'test')  # source build
2457 2458
        if os.path.isdir(sys_pyregr_dir):
            filetests = TestBuilder(ROOTDIR, WORKDIR, selectors, exclude_selectors,
gabrieldemarmiesse's avatar
gabrieldemarmiesse committed
2459
                                    options, True, languages, test_bugs,
2460
                                    sys.version_info[0], common_utility_dir, stats=stats)
2461 2462
            sys.stderr.write("Including CPython regression tests in %s\n" % sys_pyregr_dir)
            test_suite.addTest(filetests.handle_directory(sys_pyregr_dir, 'pyregr'))
2463

2464
    if options.code_style and options.shard_num <= 0:
2465 2466 2467 2468 2469 2470 2471
        try:
            import pycodestyle
        except ImportError:
            # Hack to make the exclusion visible.
            missing_dep_excluder.tests_missing_deps.append('TestCodeFormat')
        else:
            test_suite.addTest(TestCodeFormat(options.cython_dir))
2472

2473
    if xml_output_dir:
2474
        from Cython.Tests.xmlrunner import XMLTestRunner
2475
        if not os.path.exists(xml_output_dir):
2476 2477 2478 2479
            try:
                os.makedirs(xml_output_dir)
            except OSError:
                pass  # concurrency issue?
2480
        test_runner = XMLTestRunner(output=xml_output_dir,
Stefan Behnel's avatar
Stefan Behnel committed
2481
                                    verbose=options.verbosity > 0)
2482
        if options.failfast:
Stefan Behnel's avatar
Stefan Behnel committed
2483
            sys.stderr.write("--failfast not supported with XML runner\n")
2484
    else:
2485
        text_runner_options = {}
Stefan Behnel's avatar
Stefan Behnel committed
2486 2487 2488 2489 2490
        if options.failfast:
            if sys.version_info < (2, 7):
                sys.stderr.write("--failfast not supported with Python < 2.7\n")
            else:
                text_runner_options['failfast'] = True
2491
        test_runner = unittest.TextTestRunner(verbosity=options.verbosity, **text_runner_options)
2492

2493
    if options.pyximport_py:
2494
        from pyximport import pyximport
2495
        pyximport.install(pyimport=True, build_dir=os.path.join(WORKDIR, '_pyximport'),
2496
                          load_py_module_on_import_failure=True, inplace=True)
2497

2498 2499 2500 2501
    try:
        gc.set_debug(gc.DEBUG_UNCOLLECTABLE)
    except AttributeError:
        pass  # not available on PyPy
2502

2503
    result = test_runner.run(test_suite)
2504

2505 2506 2507
    if common_utility_dir and options.shard_num < 0 and options.cleanup_workdir:
        shutil.rmtree(common_utility_dir)

2508 2509 2510 2511
    if missing_dep_excluder.tests_missing_deps:
        sys.stderr.write("Following tests excluded because of missing dependencies on your system:\n")
        for test in missing_dep_excluder.tests_missing_deps:
            sys.stderr.write("   %s\n" % test)
2512

2513
    if options.with_refnanny:
Dag Sverre Seljebotn's avatar
Dag Sverre Seljebotn committed
2514
        import refnanny
2515
        sys.stderr.write("\n".join([repr(x) for x in refnanny.reflog]))
2516

2517
    if options.exit_ok:
2518
        return options.shard_num, stats, 0
2519
    else:
2520
        return options.shard_num, stats, not result.wasSuccessful()
2521 2522 2523 2524 2525 2526 2527 2528 2529 2530 2531


if __name__ == '__main__':
    try:
        main()
    except Exception:
        traceback.print_exc()
        try:
            check_thread_termination(ignore_seen=False)
        except PendingThreadsError:
            # normal program exit won't kill the threads, do it the hard way here
2532
            flush_and_terminate(1)
2533
        sys.exit(1)