Merge pull request #451 from kmod/doctest

Take another pass over the CPython test suite

from_cpython/Lib/types.py

@@ -64,20 +64,18 @@ ModuleType = type(sys)
 FileType = file
 XRangeType = xrange
 
-# Pyston change: we don't support sys.exc_info yet
-"""
 try:
     raise TypeError
 except TypeError:
     tb = sys.exc_info()[2]
     TracebackType = type(tb)
-    FrameType = type(tb.tb_frame)
+    # Pyston change (we don't support tb_frame yet):
+    FrameType = type(sys._getframe(0))
+    # FrameType = type(tb.tb_frame)
     del tb
-"""
 
 SliceType = slice
-# Pyston change: don't support this yet
-# EllipsisType = type(Ellipsis)
+EllipsisType = type(Ellipsis)
 
 # Pyston change: don't support this yet
 # DictProxyType = type(TypeType.__dict__)

libpypa @ 8f62d5d7

-Subproject commit 98d769dcd3a792f70b5e9c9bb9afa7544a50ae4a
+Subproject commit 8f62d5d7440cff89f98ed5a2e7756321811a0f2d

src/codegen/irgen/hooks.cpp

@@ -380,7 +380,7 @@ Box* eval(Box* boxedCode) {
     // TODO error message if parse fails or if it isn't an expr
     // TODO should have a cleaner interface that can parse the Expression directly
     // TODO this memory leaks
-    RELEASE_ASSERT(boxedCode->cls == str_cls, "");
+    RELEASE_ASSERT(boxedCode->cls == str_cls, "%s", boxedCode->cls->tp_name);
     const char* code = static_cast<BoxedString*>(boxedCode)->s.data();
     AST_Module* parsedModule = parse_string(code);
     RELEASE_ASSERT(parsedModule->body[0]->type == AST_TYPE::Expr, "");

src/runtime/builtin_modules/sys.cpp

@@ -207,7 +207,7 @@ void prependToSysPath(const std::string& path) {
 static BoxedClass* sys_flags_cls;
 class BoxedSysFlags : public Box {
 public:
-    Box* division_warning, *bytes_warning, *no_user_site;
+    Box* division_warning, *bytes_warning, *no_user_site, *optimize;
 
     BoxedSysFlags() {
         auto zero = boxInt(0);
@@ -215,6 +215,7 @@ public:
         division_warning = zero;
         bytes_warning = zero;
         no_user_site = zero;
+        optimize = zero;
     }
 
     DEFAULT_CLASS(sys_flags_cls);
@@ -227,6 +228,7 @@ public:
         v->visit(self->division_warning);
         v->visit(self->bytes_warning);
         v->visit(self->no_user_site);
+        v->visit(self->optimize);
     }
 
     static Box* __new__(Box* cls, Box* args, Box* kwargs) {
@@ -392,6 +394,9 @@ void setupSys() {
     sys_module->giveAttr("stdout", new BoxedFile(stdout, "<stdout>", "w"));
     sys_module->giveAttr("stdin", new BoxedFile(stdin, "<stdin>", "r"));
     sys_module->giveAttr("stderr", new BoxedFile(stderr, "<stderr>", "w"));
+    sys_module->giveAttr("__stdout__", sys_module->getattr("stdout"));
+    sys_module->giveAttr("__stdin__", sys_module->getattr("stdin"));
+    sys_module->giveAttr("__stderr__", sys_module->getattr("stderr"));
 
     sys_module->giveAttr(
         "exc_info", new BoxedBuiltinFunctionOrMethod(boxRTFunction((void*)sysExcInfo, BOXED_TUPLE, 0), "exc_info"));
@@ -458,6 +463,7 @@ void setupSys() {
     ADD(division_warning);
     ADD(bytes_warning);
     ADD(no_user_site);
+    ADD(optimize);
 #undef ADD
 
     sys_flags_cls->tp_mro = BoxedTuple::create({ sys_flags_cls, object_cls });

src/runtime/capi.cpp

@@ -1370,11 +1370,4 @@ void setupCAPI() {
 
 void teardownCAPI() {
 }
-
-void fatalOrError(PyObject* exception, const char* message) noexcept {
-    if (CONTINUE_AFTER_FATAL)
-        PyErr_SetString(exception, message);
-    else
-        Py_FatalError(message);
-}
 }

src/runtime/capi.h

@@ -28,8 +28,13 @@ void throwCAPIException() __attribute__((noreturn));
 struct ExcInfo;
 void setCAPIException(const ExcInfo& e);
 
-// TODO: not sure whether this belongs here
-void fatalOrError(Box* object, const char* message) noexcept;
+#define fatalOrError(exception, message)                                                                               \
+    do {                                                                                                               \
+        if (CONTINUE_AFTER_FATAL)                                                                                      \
+            PyErr_SetString((exception), (message));                                                                   \
+        else                                                                                                           \
+            Py_FatalError((message));                                                                                  \
+    } while (0)
 }
 
 #endif

src/runtime/objmodel.cpp

@@ -1961,7 +1961,7 @@ extern "C" bool nonzero(Box* obj) {
         ASSERT(isUserDefined(obj->cls) || obj->cls == classobj_cls || obj->cls == type_cls
                    || isSubclass(obj->cls, Exception) || obj->cls == file_cls || obj->cls == traceback_cls
                    || obj->cls == instancemethod_cls || obj->cls == module_cls || obj->cls == capifunc_cls
-                   || obj->cls == builtin_function_or_method_cls,
+                   || obj->cls == builtin_function_or_method_cls || obj->cls == method_cls,
                "%s.__nonzero__", getTypeName(obj)); // TODO
 
         // TODO should rewrite these?
@@ -4397,7 +4397,7 @@ extern "C" Box* importStar(Box* _from_module, BoxedModule* to_module) {
     // it looks like mostly a matter of changing the getattr calls to getitem.
     RELEASE_ASSERT(getGlobals() == to_module, "importStar doesn't support custom globals yet");
 
-    assert(_from_module->cls == module_cls);
+    ASSERT(isSubclass(_from_module->cls, module_cls), "%s", _from_module->cls->tp_name);
     BoxedModule* from_module = static_cast<BoxedModule*>(_from_module);
 
     Box* all = from_module->getattr(all_str);

test/cpython/NOTES.org

@@ -38,11 +38,7 @@ disable this, pass --all-cpython-tests to tester.py.
 
 * bugs uncovered
 The CPython tests I've included fail for various reasons. Recurring issues include:
-- Use of sys.flags.optimize, to test whether we kept docstrings around; I
-  commented these out, since we always keep docstrings around.
 - use of compile()
-- `exec' support
-- the doctest module imports bdb, which trips an ASTVisitor assert(0) for visit_extslice
 - missing __hash__ implementations for some builtin types
 - we don't have imp.get_magic()
 - segfaults
@@ -58,41 +54,41 @@ FILE                    REASONS
 test_abstract_numbers   missing .real attribute
 test_augassign          bugs in +=, compile()
 test_bisect             somehow sys.modules['_bisect'] is getting set to 0
-test_builtin            exec
-test_coercion           serialize_ast bug
-test_collections        extslice in bdb in doctest
+test_builtin            execfile scoping issue
+test_coercion           serialize_ast bug (AST node not getting correct lineno or col_offset set)
+test_collections        compile()
 test_compare            segfault
 test_complex            serialize_ast assert
-test_complex_args       exec
+test_complex_args       we apparently don't always unpack args correctly
 test_contains           TypeError not raised
 test_contextlib         file.closed, lock.locked attributes
 test_datetime           kwargs bug in BoxedMethodDescriptor
 test_decimal            float.__getformat__
 test_decorators         compile(), func_name attribute, and another bug in test_eval_order
-test_defaultdict        assert failure in nonzero
+test_defaultdict        "False is not true"
 test_deque              assert in _collectionsmodule.c
-test_descr              pypa assert
-test_descrtut           extslice in bdb in doctest
-test_dict               exec
+test_descr              PySequence_Tuple
+test_descrtut           doctest, compile()
+test_dict               infinite recursion in dict.__repr__
 test_dictcomps          compile()
 test_dictviews          various unique bugs
-test_doctest            extslice in bdb in doctest
-test_doctest2           extslice in bdb in doctest
+test_doctest            code.co_firstlineno, compile()
+test_doctest2           code.co_firstlineno, compile()
 test_enumerate          wrong assert in BoxedEnumerate
-test_exceptions         exec
-test_extcall            extslice in bdb in doctest
+test_exceptions         segfault in dictIterNext
+test_extcall            doctest (compile())
 test_file               segfaults
 test_file2k             "Someone called abort!"
 test_file_eintr         TypeError
-test_filecmp            extslice in filecmp
+test_filecmp            float.__hash__
 test_fileinput          UnboundLocalError (try/finally bug?)
 test_float              capifunc.__name__ in fractions
 test_format             can't float(long)
-test_funcattrs          exec
+test_funcattrs          segfault
 test_functools          segfault
-test_generators         extslice in bdb in doctest
-test_genexps            extslice in bdb in doctest
-test_getopt             extslice in bdb in doctest
+test_generators         doctest (compile())
+test_genexps            doctest (compile())
+test_getopt             doctest (compile())
 test_global             compile()
 test_grammar            bug in our tokenizer
 test_hash               float.__hash__
@@ -100,45 +96,44 @@ test_index              parseSlice assert
 test_int                float.__getformat__
 test_io                 parseSlice assert
 test_isinstance         sys.getrecursionlimit
-test_json               extslice in bdb in doctest
+test_json               doctest (compile())
 test_list               assert in sliceIndex to do with a[1L:2L]
 test_long               float.__getformat__
 test_math               float.__getformat__
-test_module             exec
-test_mutants            segfault
+test_module             exec in globals_
+test_mutants            needs cmp()
 test_opcodes            TypeError: exceptions must be old-style classes or derived from BaseException, not instance
 test_operator           BoxedCApiFunction::__call__: assert(varargs->elts.size() == 1)
 test_optparse           long.cpp: _longNew: assert(r == 0)
 test_pep277             segfaults
 test_pep352             various unique bugs
-test_pkg                exec
+test_pkg                unknown bug
 test_popen              objmodel.cpp: callCLFunc: assert(chosen_cfg->spec->rtn_type->isFitBy(r->cls))
 test_pow                global name `pow' is not defined
-test_property           pypa assert
+test_property           unknown bug
 test_random             floats unhashable
 test_repr               complex.__hash__
-test_richcmp            segfaults
-test_scope              exec
+test_richcmp            some missing C API functions
+test_scope              eval of code object
 test_set                list.__hash__
-test_setcomps           extslice in bdb in doctest
-test_sets               extslice in bdb in doctest
+test_setcomps           doctest (compile())
+test_sets               doctest (compile())
 test_slice              slice.__hash__
 test_sort               listSort(): cmp not supported
 test_stat               16384 != 40960; I guess we implement stat wrong or something?
-test_str                segfault
+test_str                memory leak?
 test_string             infinite loops in test_replace
 test_subprocess         exit code 141, no error message
 test_tuple              parseSlice assert
 test_types              func_name attribute
-test_ucn                can't eval unicode objects
 test_unary              objmodel.cpp: unaryop: Assertion `attr_func' failed: str.__pos__
-test_undocumented_details   segfault
+test_undocumented_details   cmp()
 test_unicode            sys.maxunicode
 test_unicode_file       exit code 139, no error message
 test_unittest           serialize_ast assert
-test_unpack             extslice in bdb in doctest
-test_urllib             'ascii' codec can't encode characters in position 2-3: ordinal not in range(128)
-test_urllib2            extslice in bdb in doctest
+test_unpack             doctest (compile())
+test_urllib             str.join(unicode)
+test_urllib2            doctest (compile())
 test_userdict           segfault
 test_userlist           sliceIndex assert
 test_userstring         std::length_error
@@ -147,4 +142,4 @@ test_weakref            collector.cpp: runCollection: isValidGCObject(head) fail
 test_weakset            set.cpp: setIssubset: assert(container->cls == set_cls)
 test_with               objmodel.cpp: getclsattr: Assertion `gotten' failed: LacksExit:__exit__
 test_wsgiref            ast_interpreter.cpp: createFunction: Assertion `closure' failed.
-test_xrange             exit code 139, no error message
+test_xrange             xrange of long

test/tests/doctest_test.py

+# expected: fail
+# requires:
+# - code.co_firstlineno
+# - sys.dysplayhook
+# - exec compile(foo) in globals_
+
+# This is copied from the Python docs for doctest:
+
+"""
+This is the "example" module.
+
+The example module supplies one function, factorial().  For example,
+
+>>> factorial(5)
+120
+"""
+
+def factorial(n):
+    """Return the factorial of n, an exact integer >= 0.
+
+    If the result is small enough to fit in an int, return an int.
+    Else return a long.
+
+    >>> [factorial(n) for n in range(6)]
+    [1, 1, 2, 6, 24, 120]
+    >>> [factorial(long(n)) for n in range(6)]
+    [1, 1, 2, 6, 24, 120]
+    >>> factorial(30)
+    265252859812191058636308480000000L
+    >>> factorial(30L)
+    265252859812191058636308480000000L
+    >>> factorial(-1)
+    Traceback (most recent call last):
+        ...
+    ValueError: n must be >= 0
+
+    Factorials of floats are OK, but the float must be an exact integer:
+    >>> factorial(30.1)
+    Traceback (most recent call last):
+        ...
+    ValueError: n must be exact integer
+    >>> factorial(30.0)
+    265252859812191058636308480000000L
+
+    It must also not be ridiculously large:
+    >>> factorial(1e100)
+    Traceback (most recent call last):
+        ...
+    OverflowError: n too large
+    """
+
+    import math
+    if not n >= 0:
+        raise ValueError("n must be >= 0")
+    if math.floor(n) != n:
+        raise ValueError("n must be exact integer")
+    if n+1 == n:  # catch a value like 1e300
+        raise OverflowError("n too large")
+    result = 1
+    factor = 2
+    while factor <= n:
+        result *= factor
+        factor += 1
+    return result
+
+
+if __name__ == "__main__":
+    import doctest
+    doctest.testmod()

test/tests/many_attrs_setattr.py

+# skip-if: True
+# - this test just runs until it times out and bloats test time
 # expected: fail
 # - memory explosion
 

test/tests/sys_test.py

@@ -8,3 +8,6 @@ print sys.byteorder
 print sys.getdefaultencoding()
 print sys.getfilesystemencoding()
 print type(sys.maxsize)
+print sys.stdout is sys.__stdout__
+print sys.stderr is sys.__stderr__
+print sys.stdin is sys.__stdin__