Merge pull request #1118 from undingen/parser_module

add parser module

from_cpython/CMakeLists.txt

@@ -138,17 +138,21 @@ add_library(FROM_CPYTHON OBJECT ${STDMODULE_SRCS} ${STDOBJECT_SRCS} ${STDPYTHON_
 add_dependencies(FROM_CPYTHON copy_stdlib)
 
 set(STDMODULES
-    ${CMAKE_BINARY_DIR}/from_cpython/Lib/future_builtins.pyston.so
-    ${CMAKE_BINARY_DIR}/from_cpython/Lib/_multiprocessing.pyston.so
-    ${CMAKE_BINARY_DIR}/from_cpython/Lib/pyexpat.pyston.so
-    ${CMAKE_BINARY_DIR}/from_cpython/Lib/_elementtree.pyston.so
     ${CMAKE_BINARY_DIR}/from_cpython/Lib/bz2.pyston.so
+    ${CMAKE_BINARY_DIR}/from_cpython/Lib/cmath.pyston.so
+    ${CMAKE_BINARY_DIR}/from_cpython/Lib/cPickle.pyston.so
     ${CMAKE_BINARY_DIR}/from_cpython/Lib/_ctypes.pyston.so
+    ${CMAKE_BINARY_DIR}/from_cpython/Lib/_curses.pyston.so
+    ${CMAKE_BINARY_DIR}/from_cpython/Lib/_elementtree.pyston.so
+    ${CMAKE_BINARY_DIR}/from_cpython/Lib/future_builtins.pyston.so
     ${CMAKE_BINARY_DIR}/from_cpython/Lib/grp.pyston.so
+    ${CMAKE_BINARY_DIR}/from_cpython/Lib/_locale.pyston.so
+    ${CMAKE_BINARY_DIR}/from_cpython/Lib/mmap.pyston.so
+    ${CMAKE_BINARY_DIR}/from_cpython/Lib/_multiprocessing.pyston.so
+    ${CMAKE_BINARY_DIR}/from_cpython/Lib/parser.pyston.so
+    ${CMAKE_BINARY_DIR}/from_cpython/Lib/pyexpat.pyston.so
     ${CMAKE_BINARY_DIR}/from_cpython/Lib/readline.pyston.so
     ${CMAKE_BINARY_DIR}/from_cpython/Lib/termios.pyston.so
-    ${CMAKE_BINARY_DIR}/from_cpython/Lib/_curses.pyston.so
-    ${CMAKE_BINARY_DIR}/from_cpython/Lib/mmap.pyston.so
 )
 
 add_custom_command(OUTPUT ${STDMODULES}
@@ -181,6 +185,7 @@ add_custom_command(OUTPUT ${STDMODULES}
 		       Modules/mmapmodule.c
 		       Modules/_localemodule.c
 		       Modules/cPickle.c
+                       Modules/parsermodule.c
                    WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR})
 add_custom_target(sharedmods ALL DEPENDS ${CMAKE_BINARY_DIR}/from_cpython/Lib/_multiprocessing.pyston.so)
 

from_cpython/Include/Python.h

@@ -132,6 +132,7 @@ PyAPI_FUNC(void) PyType_SetDict(PyTypeObject*, PyObject*) PYSTON_NOEXCEPT;
 #include "abstract.h"
 
 #include "compile.h"
+#include "eval.h"
 
 #include "pyctype.h"
 #include "pystrtod.h"

from_cpython/Include/compile.h

@@ -10,7 +10,7 @@ extern "C" {
 
 /* Public interface */
 struct _node; /* Declare the existence of this type */
-PyAPI_FUNC(PyCodeObject *) PyNode_Compile(struct _node *, const char *);
+PyAPI_FUNC(PyCodeObject *) PyNode_Compile(struct _node *, const char *) PYSTON_NOEXCEPT;
 
 /* Future feature support */
 
@@ -30,8 +30,8 @@ typedef struct {
 
 struct _mod; /* Declare the existence of this type */
 PyAPI_FUNC(PyCodeObject *) PyAST_Compile(struct _mod *, const char *,
-					PyCompilerFlags *, PyArena *);
-PyAPI_FUNC(PyFutureFeatures *) PyFuture_FromAST(struct _mod *, const char *);
+                    PyCompilerFlags *, PyArena *) PYSTON_NOEXCEPT;
+PyAPI_FUNC(PyFutureFeatures *) PyFuture_FromAST(struct _mod *, const char *) PYSTON_NOEXCEPT;
 
 
 #ifdef __cplusplus

from_cpython/Include/eval.h

@@ -7,7 +7,7 @@
 extern "C" {
 #endif
 
-PyAPI_FUNC(PyObject *) PyEval_EvalCode(PyCodeObject *, PyObject *, PyObject *);
+PyAPI_FUNC(PyObject *) PyEval_EvalCode(PyCodeObject *, PyObject *, PyObject *) PYSTON_NOEXCEPT;
 
 PyAPI_FUNC(PyObject *) PyEval_EvalCodeEx(PyCodeObject *co,
 					PyObject *globals,
@@ -15,9 +15,9 @@ PyAPI_FUNC(PyObject *) PyEval_EvalCodeEx(PyCodeObject *co,
 					PyObject **args, int argc,
 					PyObject **kwds, int kwdc,
 					PyObject **defs, int defc,
-					PyObject *closure);
+                    PyObject *closure) PYSTON_NOEXCEPT;
 
-PyAPI_FUNC(PyObject *) _PyEval_CallTracing(PyObject *func, PyObject *args);
+PyAPI_FUNC(PyObject *) _PyEval_CallTracing(PyObject *func, PyObject *args) PYSTON_NOEXCEPT;
 
 #ifdef __cplusplus
 }

from_cpython/Lib/test/test_parser.py

-# expected: fail
 import parser
 import unittest
 import sys

from_cpython/Lib/test/test_socketserver.py

-# expected: fail
 """
 Test suite for SocketServer.py.
 """

from_cpython/Lib/test/test_unicode_file.py

-# expected: fail
 # Test some Unicode file name semantics
 # We dont test many operations on files other than
 # that their names can be used with Unicode characters.

from_cpython/Lib/test/test_unicodedata.py

-# expected: fail
 """ Test script for the unicodedata module.
 
     Written by Marc-Andre Lemburg (mal@lemburg.com).

from_cpython/Lib/test/test_univnewlines2k.py

-# expected: fail
 # Tests universal newline support for both reading and parsing files.
 import unittest
 import os

from_cpython/Modules/parsermodule.c

@@ -3453,6 +3453,9 @@ initparser(void)
     if (PyModule_AddObject(module, "ParserError", parser_error) != 0)
         return;
 
+    // Pyston change:
+    PyType_Ready(&PyST_Type);
+
     Py_INCREF(&PyST_Type);
     PyModule_AddObject(module, "ASTType", (PyObject*)&PyST_Type);
     Py_INCREF(&PyST_Type);

from_cpython/setup.py

@@ -159,6 +159,12 @@ def cPickle_ext():
             "Modules/cPickle.c",
             ]))
 
+@unique
+def parser_ext():
+    return Extension("parser", sources = map(relpath, [
+            "Modules/parsermodule.c",
+            ]))
+
 ext_modules = [future_builtins_ext(),
                multiprocessing_ext(),
                pyexpat_ext(),
@@ -173,7 +179,8 @@ ext_modules = [future_builtins_ext(),
                termios_ext(),
                mmap_ext(),
                locale_ext(),
-               cPickle_ext()
+               cPickle_ext(),
+               parser_ext()
                ]
 
 

src/codegen/irgen/hooks.cpp

@@ -380,46 +380,10 @@ static FunctionMetadata* compileForEvalOrExec(AST* source, std::vector<AST_stmt*
     return md;
 }
 
-static AST_Module* parseExec(llvm::StringRef source, FutureFlags future_flags, bool interactive = false) {
-    // TODO error message if parse fails or if it isn't an expr
-    // TODO should have a cleaner interface that can parse the Expression directly
-    // TODO this memory leaks
-    const char* code = source.data();
-    AST_Module* parsedModule = parse_string(code, future_flags);
-
-    if (interactive)
-        makeModuleInteractive(parsedModule);
-
-    return parsedModule;
-}
-
 static FunctionMetadata* compileExec(AST_Module* parsedModule, BoxedString* fn, PyCompilerFlags* flags) {
     return compileForEvalOrExec(parsedModule, parsedModule->body, fn, flags);
 }
 
-static AST_Expression* parseEval(llvm::StringRef source, FutureFlags future_flags) {
-    const char* code = source.data();
-
-    // TODO error message if parse fails or if it isn't an expr
-    // TODO should have a cleaner interface that can parse the Expression directly
-    // TODO this memory leaks
-
-    // Hack: we need to support things like `eval(" 2")`.
-    // This is over-accepting since it will accept things like `eval("\n 2")`
-    while (*code == ' ' || *code == '\t' || *code == '\n' || *code == '\r')
-        code++;
-
-    AST_Module* parsedModule = parse_string(code, future_flags);
-    if (parsedModule->body.size() == 0)
-        raiseSyntaxError("unexpected EOF while parsing", 0, 0, "<string>", "");
-
-    RELEASE_ASSERT(parsedModule->body.size() == 1, "");
-    RELEASE_ASSERT(parsedModule->body[0]->type == AST_TYPE::Expr, "");
-    AST_Expression* parsedExpr = new AST_Expression(std::move(parsedModule->interned_strings));
-    parsedExpr->body = static_cast<AST_Expr*>(parsedModule->body[0])->value;
-    return parsedExpr;
-}
-
 static FunctionMetadata* compileEval(AST_Expression* parsedExpr, BoxedString* fn, PyCompilerFlags* flags) {
     // We need body (list of statements) to compile.
     // Obtain this by simply making a single statement which contains the expression.
@@ -430,140 +394,71 @@ static FunctionMetadata* compileEval(AST_Expression* parsedExpr, BoxedString* fn
     return compileForEvalOrExec(parsedExpr, std::move(body), fn, flags);
 }
 
-Box* compile(Box* source, Box* fn, Box* type, Box** _args) {
-    Box* flags = _args[0];
-
-    RELEASE_ASSERT(PyInt_Check(_args[1]), "");
-    bool dont_inherit = (bool)static_cast<BoxedInt*>(_args[1])->n;
-
-    RELEASE_ASSERT(flags->cls == int_cls, "");
-    int64_t iflags = static_cast<BoxedInt*>(flags)->n;
-
-    // source is allowed to be an AST, unicode, or anything that supports the buffer protocol
-    if (source->cls == unicode_cls) {
-        source = PyUnicode_AsUTF8String(source);
-        if (!source)
-            throwCAPIException();
-        // cf.cf_flags |= PyCF_SOURCE_IS_UTF8
-    }
-
-    if (isSubclass(fn->cls, unicode_cls)) {
-        fn = _PyUnicode_AsDefaultEncodedString(fn, NULL);
-        if (!fn)
-            throwCAPIException();
-    }
-    RELEASE_ASSERT(PyString_Check(fn), "");
+extern "C" PyCodeObject* PyAST_Compile(struct _mod* _mod, const char* filename, PyCompilerFlags* flags,
+                                       PyArena* arena) noexcept {
+    try {
+        mod_ty mod = _mod;
+        AST* parsed = cpythonToPystonAST(mod, filename);
+        FunctionMetadata* md = NULL;
+        switch (mod->kind) {
+            case Module_kind:
+            case Interactive_kind:
+                if (parsed->type != AST_TYPE::Module) {
+                    raiseExcHelper(TypeError, "expected Module node, got %s", AST_TYPE::stringify(parsed->type));
+                }
+                md = compileExec(static_cast<AST_Module*>(parsed), boxString(filename), flags);
+                break;
+            case Expression_kind:
+                if (parsed->type != AST_TYPE::Expression) {
+                    raiseExcHelper(TypeError, "expected Expression node, got %s", AST_TYPE::stringify(parsed->type));
+                }
+                md = compileEval(static_cast<AST_Expression*>(parsed), boxString(filename), flags);
+                break;
+            case Suite_kind:
+                PyErr_SetString(PyExc_SystemError, "suite should not be possible");
+                return NULL;
+            default:
+                PyErr_Format(PyExc_SystemError, "module kind %d should not be possible", mod->kind);
+                return NULL;
+        }
 
-    if (isSubclass(type->cls, unicode_cls)) {
-        type = _PyUnicode_AsDefaultEncodedString(type, NULL);
-        if (!type)
-            throwCAPIException();
+        return (PyCodeObject*)md->getCode();
+    } catch (ExcInfo e) {
+        setCAPIException(e);
+        return NULL;
     }
-    RELEASE_ASSERT(PyString_Check(type), "");
-
-    BoxedString* filename_str = static_cast<BoxedString*>(fn);
-    BoxedString* type_str = static_cast<BoxedString*>(type);
+}
 
-    if (iflags & ~(PyCF_MASK | PyCF_MASK_OBSOLETE | /* PyCF_DONT_IMPLY_DEDENT | */ PyCF_ONLY_AST)) {
-        raiseExcHelper(ValueError, "compile(): unrecognised flags");
-    }
+extern "C" int PyEval_MergeCompilerFlags(PyCompilerFlags* cf) noexcept {
+    int result = cf->cf_flags != 0;
 
-    bool only_ast = (bool)(iflags & PyCF_ONLY_AST);
+    /* Pyston change:
+    PyFrameObject *current_frame = PyEval_GetFrame();
+    int result = cf->cf_flags != 0;
 
-    iflags &= ~PyCF_ONLY_AST;
+    if (current_frame != NULL) {
+        const int codeflags = current_frame->f_code->co_flags;
+    */
 
-    FutureFlags arg_future_flags = iflags & PyCF_MASK;
-    FutureFlags future_flags;
-    if (dont_inherit) {
-        future_flags = arg_future_flags;
-    } else {
-        FunctionMetadata* caller_cl = getTopPythonFunction();
-        assert(caller_cl != NULL);
+    FunctionMetadata* caller_cl = getTopPythonFunction();
+    if (caller_cl != NULL) {
         assert(caller_cl->source != NULL);
         FutureFlags caller_future_flags = caller_cl->source->future_flags;
-        future_flags = arg_future_flags | caller_future_flags;
-    }
-
-    iflags &= !(PyCF_MASK | PyCF_MASK_OBSOLETE);
-    RELEASE_ASSERT(iflags == 0, "");
-
-    AST* parsed;
-    mod_ty mod;
-
-    ArenaWrapper arena;
-
-    if (PyAST_Check(source)) {
-        int mode;
-        if (type_str->s() == "exec")
-            mode = 0;
-        else if (type_str->s() == "eval")
-            mode = 1;
-        else if (type_str->s() == "single")
-            mode = 2;
-        else {
-            raiseExcHelper(ValueError, "compile() arg 3 must be 'exec', 'eval' or 'single'");
-        }
-        if (only_ast) // nothing to do
-            return source;
-
-        mod = PyAST_obj2mod(source, arena, mode);
-        if (PyErr_Occurred())
-            throwCAPIException();
-    } else {
-        RELEASE_ASSERT(PyString_Check(source), "");
-        int mode;
-        if (type_str->s() == "exec")
-            mode = Py_file_input;
-        else if (type_str->s() == "eval")
-            mode = Py_eval_input;
-        else if (type_str->s() == "single")
-            mode = Py_single_input;
-        else {
-            raiseExcHelper(ValueError, "compile() arg 3 must be 'exec', 'eval' or 'single'");
-        }
-
-        PyCompilerFlags cf;
-        cf.cf_flags = future_flags;
-        const char* code = static_cast<BoxedString*>(source)->s().data();
-        assert(arena);
-        const char* fn = filename_str->c_str();
-        mod = PyParser_ASTFromString(code, fn, mode, &cf, arena);
-        if (!mod)
-            throwCAPIException();
-    }
-
-    if (only_ast) {
-        Box* result = PyAST_mod2obj(mod);
-        if (PyErr_Occurred())
-            throwCAPIException();
-
-        return result;
-    }
-
-    // be careful when moving around this function: it does also do some additional syntax checking (=raises exception),
-    // which we should not do when in AST only mode.
-    parsed = cpythonToPystonAST(mod, filename_str->c_str());
 
-    PyCompilerFlags pcf;
-    pcf.cf_flags = future_flags;
-
-    FunctionMetadata* md;
-    if (type_str->s() == "exec" || type_str->s() == "single") {
-        // TODO: CPython parses execs as Modules
-        if (parsed->type != AST_TYPE::Module) {
-            raiseExcHelper(TypeError, "expected Module node, got %s", AST_TYPE::stringify(parsed->type));
+        const int codeflags = caller_future_flags;
+        const int compilerflags = codeflags & PyCF_MASK;
+        if (compilerflags) {
+            result = 1;
+            cf->cf_flags |= compilerflags;
         }
-        md = compileExec(static_cast<AST_Module*>(parsed), filename_str, &pcf);
-    } else if (type_str->s() == "eval") {
-        if (parsed->type != AST_TYPE::Expression) {
-            raiseExcHelper(TypeError, "expected Expression node, got %s", AST_TYPE::stringify(parsed->type));
+#if 0 /* future keyword */
+        if (codeflags & CO_GENERATOR_ALLOWED) {
+            result = 1;
+            cf->cf_flags |= CO_GENERATOR_ALLOWED;
         }
-        md = compileEval(static_cast<AST_Expression*>(parsed), filename_str, &pcf);
-    } else {
-        raiseExcHelper(ValueError, "compile() arg 3 must be 'exec', 'eval' or 'single'");
+#endif
     }
-
-    return (Box*)md->getCode();
+    return result;
 }
 
 static void pickGlobalsAndLocals(Box*& globals, Box*& locals) {
@@ -609,151 +504,111 @@ static void pickGlobalsAndLocals(Box*& globals, Box*& locals) {
     }
 }
 
-static Box* evalMain(Box* boxedCode, Box* globals, Box* locals, PyCompilerFlags* flags) {
-    pickGlobalsAndLocals(globals, locals);
-
-    if (boxedCode->cls == unicode_cls) {
-        boxedCode = PyUnicode_AsUTF8String(boxedCode);
-        if (!boxedCode)
-            throwCAPIException();
-        // cf.cf_flags |= PyCF_SOURCE_IS_UTF8
-    }
-
-    FunctionMetadata* md;
-    if (boxedCode->cls == str_cls) {
-        FunctionMetadata* caller_cl = getTopPythonFunction();
-        assert(caller_cl != NULL);
-        assert(caller_cl->source != NULL);
-
-        AST_Expression* parsed = parseEval(static_cast<BoxedString*>(boxedCode)->s(), caller_cl->source->future_flags);
-        static BoxedString* string_string = internStringImmortal("<string>");
-        md = compileEval(parsed, string_string, flags);
-    } else if (boxedCode->cls == code_cls) {
-        md = metadataFromCode(boxedCode);
-    } else {
-        abort();
+extern "C" PyObject* PyEval_EvalCode(PyCodeObject* co, PyObject* globals, PyObject* locals) noexcept {
+    try {
+        pickGlobalsAndLocals(globals, locals);
+        return evalOrExec(metadataFromCode((Box*)co), globals, locals);
+    } catch (ExcInfo e) {
+        setCAPIException(e);
+        return NULL;
     }
-
-    return evalOrExec(md, globals, locals);
 }
 
-Box* eval(Box* boxedCode, Box* globals, Box* locals) {
-    FunctionMetadata* caller_cl = getTopPythonFunction();
-    assert(caller_cl != NULL);
-    assert(caller_cl->source != NULL);
-    FutureFlags caller_future_flags = caller_cl->source->future_flags;
-    PyCompilerFlags pcf;
-    pcf.cf_flags = caller_future_flags;
-
-    return evalMain(boxedCode, globals, locals, &pcf);
-}
-
-Box* execfile(Box* _fn, Box* globals, Box* locals) {
-    if (!PyString_Check(_fn)) {
-        raiseExcHelper(TypeError, "must be string, not %s", getTypeName(_fn));
-    }
-
-    BoxedString* fn = static_cast<BoxedString*>(_fn);
-
-    pickGlobalsAndLocals(globals, locals);
-
-#if LLVMREV < 217625
-    bool exists;
-    llvm_error_code code = llvm::sys::fs::exists(fn->s, exists);
-
-#if LLVMREV < 210072
-    ASSERT(code == 0, "%s: %s", code.message().c_str(), fn->s.c_str());
-#else
-    assert(!code);
-#endif
-
-#else
-    bool exists = llvm::sys::fs::exists(std::string(fn->s()));
+Box* exec(Box* boxedCode, Box* globals, Box* locals, FutureFlags caller_future_flags) {
+    if (!globals)
+        globals = None;
+    if (!locals)
+        locals = None;
+
+    // this is based on cpythons exec_statement() but (heavily) adopted
+    Box* v = NULL;
+    int plain = 0;
+    int n;
+    PyObject* prog = boxedCode;
+    if (PyTuple_Check(prog) && globals == Py_None && locals == Py_None && ((n = PyTuple_Size(prog)) == 2 || n == 3)) {
+        /* Backward compatibility hack */
+        globals = PyTuple_GetItem(prog, 1);
+        if (n == 3)
+            locals = PyTuple_GetItem(prog, 2);
+        prog = PyTuple_GetItem(prog, 0);
+    }
+    if (globals == Py_None) {
+        globals = PyEval_GetGlobals();
+        if (locals == Py_None) {
+            locals = PyEval_GetLocals();
+            plain = 1;
+        }
+        if (!globals || !locals) {
+            raiseExcHelper(SystemError, "globals and locals cannot be NULL");
+        }
+    } else if (locals == Py_None)
+        locals = globals;
+    if (!PyString_Check(prog) &&
+#ifdef Py_USING_UNICODE
+        !PyUnicode_Check(prog) &&
 #endif
-
-    if (!exists)
-        raiseExcHelper(IOError, "No such file or directory: '%s'", fn->s().data());
-
-    AST_Module* parsed = caching_parse_file(fn->s().data(), /* future_flags = */ 0);
-    assert(parsed);
-
-    FunctionMetadata* caller_cl = getTopPythonFunction();
-    assert(caller_cl != NULL);
-    assert(caller_cl->source != NULL);
-    FutureFlags caller_future_flags = caller_cl->source->future_flags;
-    PyCompilerFlags pcf;
-    pcf.cf_flags = caller_future_flags;
-
-    FunctionMetadata* md = compileForEvalOrExec(parsed, parsed->body, fn, &pcf);
-    assert(md);
-
-    return evalOrExec(md, globals, locals);
-}
-
-Box* execMain(Box* boxedCode, Box* globals, Box* locals, PyCompilerFlags* flags) {
-    if (PyTuple_Check(boxedCode)) {
-        RELEASE_ASSERT(!globals, "");
-        RELEASE_ASSERT(!locals, "");
-
-        BoxedTuple* t = static_cast<BoxedTuple*>(boxedCode);
-        RELEASE_ASSERT(t->size() >= 2 && t->size() <= 3, "%ld", t->size());
-        boxedCode = t->elts[0];
-        globals = t->elts[1];
-        if (t->size() >= 3)
-            locals = t->elts[2];
+        !PyCode_Check(prog) && !PyFile_Check(prog)) {
+        raiseExcHelper(TypeError, "exec: arg 1 must be a string, file, or code object");
     }
 
-    pickGlobalsAndLocals(globals, locals);
-
-    if (boxedCode->cls == unicode_cls) {
-        boxedCode = PyUnicode_AsUTF8String(boxedCode);
-        if (!boxedCode)
-            throwCAPIException();
-        // cf.cf_flags |= PyCF_SOURCE_IS_UTF8
+    if (!PyDict_Check(globals) && globals->cls != attrwrapper_cls) {
+        raiseExcHelper(TypeError, "exec: arg 2 must be a dictionary or None");
     }
+    if (!PyMapping_Check(locals))
+        raiseExcHelper(TypeError, "exec: arg 3 must be a mapping or None");
 
-    FunctionMetadata* md;
-    if (boxedCode->cls == str_cls) {
-        FunctionMetadata* caller_cl = getTopPythonFunction();
-        assert(caller_cl != NULL);
-        assert(caller_cl->source != NULL);
+    if (PyDict_GetItemString(globals, "__builtins__") == NULL)
+        // Pyston change:
+        // PyDict_SetItemString(globals, "__builtins__", f->f_builtins);
+        PyDict_SetItemString(globals, "__builtins__", builtins_module);
 
-        auto parsed = parseExec(static_cast<BoxedString*>(boxedCode)->s(), caller_cl->source->future_flags);
-        static BoxedString* string_string = internStringImmortal("<string>");
-        md = compileExec(parsed, string_string, flags);
-    } else if (boxedCode->cls == code_cls) {
-        md = metadataFromCode(boxedCode);
+    if (PyCode_Check(prog)) {
+        /* Pyston change:
+        if (PyCode_GetNumFree((PyCodeObject *)prog) > 0) {
+            PyErr_SetString(PyExc_TypeError,
+        "code object passed to exec may not contain free variables");
+            return -1;
+        }
+        */
+        v = PyEval_EvalCode((PyCodeObject*)prog, globals, locals);
+    } else if (PyFile_Check(prog)) {
+        FILE* fp = PyFile_AsFile(prog);
+        char* name = PyString_AsString(PyFile_Name(prog));
+        PyCompilerFlags cf;
+        if (name == NULL)
+            throwCAPIException();
+        cf.cf_flags = caller_future_flags & PyCF_MASK;
+        if (cf.cf_flags)
+            v = PyRun_FileFlags(fp, name, Py_file_input, globals, locals, &cf);
+        else
+            v = PyRun_File(fp, name, Py_file_input, globals, locals);
     } else {
-        abort();
-    }
-    assert(md);
-
-    return evalOrExec(md, globals, locals);
-}
-
-Box* exec(Box* boxedCode, Box* globals, Box* locals, FutureFlags caller_future_flags) {
-    PyCompilerFlags pcf;
-    pcf.cf_flags = caller_future_flags;
-    return execMain(boxedCode, globals, locals, &pcf);
-}
-
-extern "C" PyObject* PyRun_StringFlags(const char* str, int start, PyObject* globals, PyObject* locals,
-                                       PyCompilerFlags* flags) noexcept {
-    try {
-        // TODO pass future_flags (the information is in PyCompilerFlags but we need to
-        // unify the format...)
-        if (start == Py_file_input)
-            return execMain(boxString(str), globals, locals, flags);
-        else if (start == Py_eval_input)
-            return evalMain(boxString(str), globals, locals, flags);
-    } catch (ExcInfo e) {
-        setCAPIException(e);
-        return NULL;
+        PyObject* tmp = NULL;
+        char* str;
+        PyCompilerFlags cf;
+        cf.cf_flags = 0;
+#ifdef Py_USING_UNICODE
+        if (PyUnicode_Check(prog)) {
+            tmp = PyUnicode_AsUTF8String(prog);
+            if (tmp == NULL)
+                throwCAPIException();
+            prog = tmp;
+            cf.cf_flags |= PyCF_SOURCE_IS_UTF8;
+        }
+#endif
+        if (PyString_AsStringAndSize(prog, &str, NULL))
+            return 0;
+        cf.cf_flags |= caller_future_flags & PyCF_MASK;
+        if (cf.cf_flags)
+            v = PyRun_StringFlags(str, Py_file_input, globals, locals, &cf);
+        else
+            v = PyRun_String(str, Py_file_input, globals, locals);
+        Py_XDECREF(tmp);
     }
 
-    // Py_single_input is not yet implemented
-    RELEASE_ASSERT(0, "Unimplemented %d", start);
-    return 0;
+    if (!v)
+        throwCAPIException();
+    return v;
 }
 
 // If a function version keeps failing its speculations, kill it (remove it

src/codegen/irgen/hooks.h

@@ -40,9 +40,6 @@ void compileAndRunModule(AST_Module* m, BoxedModule* bm);
 CompiledFunction* cfForMachineFunctionName(const std::string&);
 
 extern "C" Box* exec(Box* boxedCode, Box* globals, Box* locals, FutureFlags caller_future_flags);
-extern "C" Box* execfile(Box* fn, Box* globals, Box* locals);
-extern "C" Box* eval(Box* boxedCode, Box* globals, Box* locals);
-extern "C" Box* compile(Box* source, Box* filename, Box* mode, Box** _args /* flags, dont_inherit */);
 }
 
 #endif

src/runtime/builtin_modules/builtins.cpp

@@ -19,6 +19,9 @@
 
 #include "llvm/Support/FileSystem.h"
 
+#include "Python.h"
+#include "Python-ast.h"
+
 #include "capi/typeobject.h"
 #include "capi/types.h"
 #include "codegen/ast_interpreter.h"
@@ -1604,18 +1607,14 @@ Box* rawInput(Box* prompt) {
 }
 
 Box* input(Box* prompt) {
-    char* str;
-
-    PyObject* line = raw_input(prompt);
-    if (line == NULL)
-        throwCAPIException();
+    PyObject* line = rawInput(prompt);
 
+    char* str = NULL;
     if (!PyArg_Parse(line, "s;embedded '\\0' in input line", &str))
         throwCAPIException();
 
-    // CPython trims the string first, but our eval function takes care of that.
-    // while (*str == ' ' || *str == '\t')
-    //    str++;
+    while (*str == ' ' || *str == '\t')
+        str++;
 
     Box* gbls = globals();
     Box* lcls = locals();
@@ -1628,7 +1627,13 @@ Box* input(Box* prompt) {
             throwCAPIException();
     }
 
-    return eval(line, gbls, lcls);
+    PyCompilerFlags cf;
+    cf.cf_flags = 0;
+    PyEval_MergeCompilerFlags(&cf);
+    Box* res = PyRun_StringFlags(str, Py_eval_input, gbls, lcls, &cf);
+    if (!res)
+        throwCAPIException();
+    return res;
 }
 
 Box* builtinRound(Box* _number, Box* _ndigits) {
@@ -1695,6 +1700,259 @@ Box* builtinFormat(Box* value, Box* format_spec) {
     return res;
 }
 
+static PyObject* builtin_compile(PyObject* self, PyObject* args, PyObject* kwds) noexcept {
+    char* str;
+    char* filename;
+    char* startstr;
+    int mode = -1;
+    int dont_inherit = 0;
+    int supplied_flags = 0;
+    int is_ast;
+    PyCompilerFlags cf;
+    PyObject* result = NULL, *cmd, * tmp = NULL;
+    Py_ssize_t length;
+    static const char* kwlist[] = { "source", "filename", "mode", "flags", "dont_inherit", NULL };
+    int start[] = { Py_file_input, Py_eval_input, Py_single_input };
+
+    if (!PyArg_ParseTupleAndKeywords(args, kwds, "Oss|ii:compile", const_cast<char**>(kwlist), &cmd, &filename,
+                                     &startstr, &supplied_flags, &dont_inherit))
+        return NULL;
+
+    cf.cf_flags = supplied_flags;
+
+    if (supplied_flags & ~(PyCF_MASK | PyCF_MASK_OBSOLETE | PyCF_DONT_IMPLY_DEDENT | PyCF_ONLY_AST)) {
+        PyErr_SetString(PyExc_ValueError, "compile(): unrecognised flags");
+        return NULL;
+    }
+    /* XXX Warn if (supplied_flags & PyCF_MASK_OBSOLETE) != 0? */
+
+    if (!dont_inherit) {
+        PyEval_MergeCompilerFlags(&cf);
+    }
+
+    if (strcmp(startstr, "exec") == 0)
+        mode = 0;
+    else if (strcmp(startstr, "eval") == 0)
+        mode = 1;
+    else if (strcmp(startstr, "single") == 0)
+        mode = 2;
+    else {
+        PyErr_SetString(PyExc_ValueError, "compile() arg 3 must be 'exec', 'eval' or 'single'");
+        return NULL;
+    }
+
+    is_ast = PyAST_Check(cmd);
+    if (is_ast == -1)
+        return NULL;
+    if (is_ast) {
+        if (supplied_flags & PyCF_ONLY_AST) {
+            Py_INCREF(cmd);
+            result = cmd;
+        } else {
+            PyArena* arena;
+            mod_ty mod;
+
+            arena = PyArena_New();
+            if (arena == NULL)
+                return NULL;
+            mod = PyAST_obj2mod(cmd, arena, mode);
+            if (mod == NULL) {
+                PyArena_Free(arena);
+                return NULL;
+            }
+            result = (PyObject*)PyAST_Compile(mod, filename, &cf, arena);
+            PyArena_Free(arena);
+        }
+        return result;
+    }
+
+#ifdef Py_USING_UNICODE
+    if (PyUnicode_Check(cmd)) {
+        tmp = PyUnicode_AsUTF8String(cmd);
+        if (tmp == NULL)
+            return NULL;
+        cmd = tmp;
+        cf.cf_flags |= PyCF_SOURCE_IS_UTF8;
+    }
+#endif
+
+    if (PyObject_AsReadBuffer(cmd, (const void**)&str, &length))
+        goto cleanup;
+    if ((size_t)length != strlen(str)) {
+        PyErr_SetString(PyExc_TypeError, "compile() expected string without null bytes");
+        goto cleanup;
+    }
+    result = Py_CompileStringFlags(str, filename, start[mode], &cf);
+cleanup:
+    Py_XDECREF(tmp);
+    return result;
+}
+
+static PyObject* builtin_eval(PyObject* self, PyObject* args) noexcept {
+    PyObject* cmd, *result, * tmp = NULL;
+    PyObject* globals = Py_None, * locals = Py_None;
+    char* str;
+    PyCompilerFlags cf;
+
+    if (!PyArg_UnpackTuple(args, "eval", 1, 3, &cmd, &globals, &locals))
+        return NULL;
+    if (locals != Py_None && !PyMapping_Check(locals)) {
+        PyErr_SetString(PyExc_TypeError, "locals must be a mapping");
+        return NULL;
+    }
+    // Pyston change:
+    // if (globals != Py_None && !PyDict_Check(globals)) {
+    if (globals != Py_None && !PyDict_Check(globals) && globals->cls != attrwrapper_cls) {
+        PyErr_SetString(PyExc_TypeError, PyMapping_Check(globals)
+                                             ? "globals must be a real dict; try eval(expr, {}, mapping)"
+                                             : "globals must be a dict");
+        return NULL;
+    }
+    if (globals == Py_None) {
+        globals = PyEval_GetGlobals();
+        if (locals == Py_None)
+            locals = PyEval_GetLocals();
+    } else if (locals == Py_None)
+        locals = globals;
+
+    if (globals == NULL || locals == NULL) {
+        PyErr_SetString(PyExc_TypeError, "eval must be given globals and locals "
+                                         "when called without a frame");
+        return NULL;
+    }
+
+    if (PyDict_GetItemString(globals, "__builtins__") == NULL) {
+        if (PyDict_SetItemString(globals, "__builtins__", PyEval_GetBuiltins()) != 0)
+            return NULL;
+    }
+
+    if (PyCode_Check(cmd)) {
+// Pyston change:
+#if 0
+        if (PyCode_GetNumFree((PyCodeObject *)cmd) > 0) {
+            PyErr_SetString(PyExc_TypeError,
+        "code object passed to eval() may not contain free variables");
+            return NULL;
+        }
+#endif
+        return PyEval_EvalCode((PyCodeObject*)cmd, globals, locals);
+    }
+
+    if (!PyString_Check(cmd) && !PyUnicode_Check(cmd)) {
+        PyErr_SetString(PyExc_TypeError, "eval() arg 1 must be a string or code object");
+        return NULL;
+    }
+    cf.cf_flags = 0;
+
+#ifdef Py_USING_UNICODE
+    if (PyUnicode_Check(cmd)) {
+        tmp = PyUnicode_AsUTF8String(cmd);
+        if (tmp == NULL)
+            return NULL;
+        cmd = tmp;
+        cf.cf_flags |= PyCF_SOURCE_IS_UTF8;
+    }
+#endif
+    if (PyString_AsStringAndSize(cmd, &str, NULL)) {
+        Py_XDECREF(tmp);
+        return NULL;
+    }
+    while (*str == ' ' || *str == '\t')
+        str++;
+
+    (void)PyEval_MergeCompilerFlags(&cf);
+    result = PyRun_StringFlags(str, Py_eval_input, globals, locals, &cf);
+    Py_XDECREF(tmp);
+    return result;
+}
+
+static PyObject* builtin_execfile(PyObject* self, PyObject* args) noexcept {
+    char* filename;
+    PyObject* globals = Py_None, * locals = Py_None;
+    PyObject* res;
+    FILE* fp = NULL;
+    PyCompilerFlags cf;
+    int exists;
+
+    if (PyErr_WarnPy3k("execfile() not supported in 3.x; use exec()", 1) < 0)
+        return NULL;
+
+    if (!PyArg_ParseTuple(args, "s|O!O:execfile", &filename, &PyDict_Type, &globals, &locals))
+        return NULL;
+    if (locals != Py_None && !PyMapping_Check(locals)) {
+        PyErr_SetString(PyExc_TypeError, "locals must be a mapping");
+        return NULL;
+    }
+    if (globals == Py_None) {
+        globals = PyEval_GetGlobals();
+        if (locals == Py_None)
+            locals = PyEval_GetLocals();
+    } else if (locals == Py_None)
+        locals = globals;
+
+    if (PyDict_GetItemString(globals, "__builtins__") == NULL) {
+        if (PyDict_SetItemString(globals, "__builtins__", PyEval_GetBuiltins()) != 0)
+            return NULL;
+    }
+
+    exists = 0;
+/* Test for existence or directory. */
+#if defined(PLAN9)
+    {
+        Dir* d;
+
+        if ((d = dirstat(filename)) != nil) {
+            if (d->mode & DMDIR)
+                werrstr("is a directory");
+            else
+                exists = 1;
+            free(d);
+        }
+    }
+#elif defined(RISCOS)
+    if (object_exists(filename)) {
+        if (isdir(filename))
+            errno = EISDIR;
+        else
+            exists = 1;
+    }
+#else /* standard Posix */
+    {
+        struct stat s;
+        if (stat(filename, &s) == 0) {
+            if (S_ISDIR(s.st_mode))
+#if defined(PYOS_OS2) && defined(PYCC_VACPP)
+                errno = EOS2ERR;
+#else
+                errno = EISDIR;
+#endif
+            else
+                exists = 1;
+        }
+    }
+#endif
+
+    if (exists) {
+        Py_BEGIN_ALLOW_THREADS fp = fopen(filename, "r" PY_STDIOTEXTMODE);
+        Py_END_ALLOW_THREADS
+
+            if (fp == NULL) {
+            exists = 0;
+        }
+    }
+
+    if (!exists) {
+        PyErr_SetFromErrnoWithFilename(PyExc_IOError, filename);
+        return NULL;
+    }
+    cf.cf_flags = 0;
+    if (PyEval_MergeCompilerFlags(&cf))
+        res = PyRun_FileExFlags(fp, filename, Py_file_input, globals, locals, 1, &cf);
+    else
+        res = PyRun_FileEx(fp, filename, Py_file_input, globals, locals, 1);
+    return res;
+}
+
 extern "C" {
 BoxedClass* ellipsis_cls;
 }
@@ -2206,16 +2464,6 @@ void setupBuiltins() {
     builtins_module->giveAttr("divmod", new BoxedBuiltinFunctionOrMethod(
                                             FunctionMetadata::create((void*)divmod, UNKNOWN, 2), "divmod", divmod_doc));
 
-    builtins_module->giveAttr("execfile", new BoxedBuiltinFunctionOrMethod(
-                                              FunctionMetadata::create((void*)execfile, UNKNOWN, 3, false, false),
-                                              "execfile", { NULL, NULL }, NULL, execfile_doc));
-
-    FunctionMetadata* compile_func = new FunctionMetadata(
-        5, false, false, ParamNames({ "source", "filename", "mode", "flags", "dont_inherit" }, "", ""));
-    compile_func->addVersion((void*)compile, UNKNOWN, { UNKNOWN, UNKNOWN, UNKNOWN, UNKNOWN, UNKNOWN });
-    builtins_module->giveAttr("compile", new BoxedBuiltinFunctionOrMethod(compile_func, "compile",
-                                                                          { boxInt(0), boxInt(0) }, NULL, compile_doc));
-
     builtins_module->giveAttr("map", new BoxedBuiltinFunctionOrMethod(
                                          FunctionMetadata::create((void*)map, LIST, 1, true, false), "map", map_doc));
     builtins_module->giveAttr(
@@ -2267,9 +2515,6 @@ void setupBuiltins() {
     PyType_Ready(&PyBuffer_Type);
     builtins_module->giveAttr("buffer", &PyBuffer_Type);
 
-    builtins_module->giveAttr(
-        "eval", new BoxedBuiltinFunctionOrMethod(FunctionMetadata::create((void*)eval, UNKNOWN, 3, false, false),
-                                                 "eval", { NULL, NULL }, NULL, eval_doc));
     builtins_module->giveAttr("callable",
                               new BoxedBuiltinFunctionOrMethod(FunctionMetadata::create((void*)callable, UNKNOWN, 1),
                                                                "callable", callable_doc));
@@ -2288,6 +2533,9 @@ void setupBuiltins() {
 
 
     static PyMethodDef builtin_methods[] = {
+        { "compile", (PyCFunction)builtin_compile, METH_VARARGS | METH_KEYWORDS, compile_doc },
+        { "eval", builtin_eval, METH_VARARGS, eval_doc },
+        { "execfile", builtin_execfile, METH_VARARGS, execfile_doc },
         { "print", (PyCFunction)builtin_print, METH_VARARGS | METH_KEYWORDS, print_doc },
         { "reload", builtin_reload, METH_O, reload_doc },
     };

src/runtime/capi.cpp

@@ -1133,6 +1133,11 @@ cleanup:
              : 0)
 #endif
 
+extern "C" void PyParser_SetError(perrdetail* err) noexcept {
+    err_input(err);
+}
+
+
 extern "C" grammar _PyParser_Grammar;
 
 /* Preferred access to parser is through AST. */
@@ -1184,6 +1189,76 @@ extern "C" mod_ty PyParser_ASTFromFile(FILE* fp, const char* filename, int start
     }
 }
 
+extern "C" PyObject* Py_CompileStringFlags(const char* str, const char* filename, int start,
+                                           PyCompilerFlags* flags) noexcept {
+    PyCodeObject* co;
+    mod_ty mod;
+    PyArena* arena = PyArena_New();
+    if (arena == NULL)
+        return NULL;
+
+    mod = PyParser_ASTFromString(str, filename, start, flags, arena);
+    if (mod == NULL) {
+        PyArena_Free(arena);
+        return NULL;
+    }
+    if (flags && (flags->cf_flags & PyCF_ONLY_AST)) {
+        PyObject* result = PyAST_mod2obj(mod);
+        PyArena_Free(arena);
+        return result;
+    }
+    co = PyAST_Compile(mod, filename, flags, arena);
+    PyArena_Free(arena);
+    return (PyObject*)co;
+}
+
+static PyObject* run_mod(mod_ty mod, const char* filename, PyObject* globals, PyObject* locals, PyCompilerFlags* flags,
+                         PyArena* arena) noexcept {
+    PyCodeObject* co;
+    PyObject* v;
+    co = PyAST_Compile(mod, filename, flags, arena);
+    if (co == NULL)
+        return NULL;
+    v = PyEval_EvalCode(co, globals, locals);
+    Py_DECREF(co);
+    return v;
+}
+
+extern "C" PyObject* PyRun_FileExFlags(FILE* fp, const char* filename, int start, PyObject* globals, PyObject* locals,
+                                       int closeit, PyCompilerFlags* flags) noexcept {
+    PyObject* ret;
+    mod_ty mod;
+    PyArena* arena = PyArena_New();
+    if (arena == NULL)
+        return NULL;
+
+    mod = PyParser_ASTFromFile(fp, filename, start, 0, 0, flags, NULL, arena);
+    if (closeit)
+        fclose(fp);
+    if (mod == NULL) {
+        PyArena_Free(arena);
+        return NULL;
+    }
+    ret = run_mod(mod, filename, globals, locals, flags, arena);
+    PyArena_Free(arena);
+    return ret;
+}
+
+extern "C" PyObject* PyRun_StringFlags(const char* str, int start, PyObject* globals, PyObject* locals,
+                                       PyCompilerFlags* flags) noexcept {
+    PyObject* ret = NULL;
+    mod_ty mod;
+    PyArena* arena = PyArena_New();
+    if (arena == NULL)
+        return NULL;
+
+    mod = PyParser_ASTFromString(str, "<string>", start, flags, arena);
+    if (mod != NULL)
+        ret = run_mod(mod, "<string>", globals, locals, flags, arena);
+    PyArena_Free(arena);
+    return ret;
+}
+
 extern "C" int PyRun_InteractiveLoopFlags(FILE* fp, const char* filename, PyCompilerFlags* flags) noexcept {
     PyObject* v;
     int ret;

test/CPYTHON_TEST_NOTES.md

@@ -136,7 +136,6 @@ test_nis                [unknown]
 test_optparse           assertion instead of exceptions for long("invalid number")
 test_ossaudiodev        [unknown]
 test_os                 [unknown]
-test_parser             [unknown]
 test_pdb                [unknown]
 test_peepholer          [unknown]
 test_pep352             various unique bugs
@@ -157,7 +156,6 @@ test_scope              eval of code object from existing function (not currentl
 test_scriptpackages     [unknown]
 test_shelve             [unknown]
 test_site               [unknown]
-test_socketserver       missing imp.lock_held, otherwise works
 test_socket             [unknown]
 test_sort               argument specification issue in listSort?
 test_sqlite             [unknown]
@@ -194,10 +192,7 @@ test_ttk_textonly       [unknown]
 test_types              PyErr_WarnEx
 test_undocumented_details   function.func_closure
 test_unicode            argument passing issue?
-test_unicodedata        [unknown]
-test_unicode_file       exit code 139, no error message
 test_unittest           serialize_ast assert
-test_univnewlines2k     [unknown]
 test_univnewlines       [unknown]
 test_userdict           segfault: repr of recursive dict?
 test_userlist           slice(1L, 1L)

test/extra/cheetah_test.py

@@ -9,5 +9,5 @@ create_virtenv(ENV_NAME, ["cheetah==2.4.4", "Markdown==2.0.1"], force_create = T
 cheetah_exe = os.path.join(ENV_NAME, "bin", "cheetah")
 env = os.environ
 env["PATH"] = os.path.join(ENV_NAME, "bin")
-expected = [{'ran': 2138, 'errors': 4, 'failures': 1}, {'ran': 2138, 'errors': 232, 'failures': 3}]
+expected = [{'ran': 2138, 'errors': 4}, {'ran': 2138, 'errors': 232, 'failures': 2}]
 run_test([cheetah_exe, "test"], cwd=ENV_NAME, expected=expected, env=env)

test/extra/geoip_test.py

@@ -7,7 +7,7 @@ ENV_NAME = "geoip_test_env_" + os.path.basename(sys.executable)
 SRC_DIR = os.path.abspath(os.path.join(ENV_NAME, "src"))
 PYTHON_EXE = os.path.abspath(os.path.join(ENV_NAME, "bin", "python"))
 
-pkg = ["-e", "git+https://github.com/maxmind/geoip-api-python.git@v1.3.2#egg=GeoIP"]
+pkg = ["nose==1.3.7", "-e", "git+http://github.com/maxmind/geoip-api-python.git@v1.3.2#egg=GeoIP"]
 create_virtenv(ENV_NAME, pkg, force_create = True)
 GEOIP_DIR = os.path.abspath(os.path.join(SRC_DIR, "geoip"))
 expected = [{'ran': 10}]

test/tests/compile_test2.py

+# this tests are from cpythons test_compile.py
+import unittest
+from test import test_support
+
+class TestSpecifics(unittest.TestCase):
+    def test_exec_functional_style(self):
+        # Exec'ing a tuple of length 2 works.
+        g = {'b': 2}
+        exec("a = b + 1", g)
+        self.assertEqual(g['a'], 3)
+
+        # As does exec'ing a tuple of length 3.
+        l = {'b': 3}
+        g = {'b': 5, 'c': 7}
+        exec("a = b + c", g, l)
+        self.assertNotIn('a', g)
+        self.assertEqual(l['a'], 10)
+
+        # Tuples not of length 2 or 3 are invalid.
+        with self.assertRaises(TypeError):
+            exec("a = b + 1",)
+
+        with self.assertRaises(TypeError):
+            exec("a = b + 1", {}, {}, {})
+
+        # Can't mix and match the two calling forms.
+        g = {'a': 3, 'b': 4}
+        l = {}
+        with self.assertRaises(TypeError):
+            exec("a = b + 1", g) in g
+        with self.assertRaises(TypeError):
+            exec("a = b + 1", g, l) in g, l
+
+    def test_exec_with_general_mapping_for_locals(self):
+
+        class M:
+            "Test mapping interface versus possible calls from eval()."
+            def __getitem__(self, key):
+                if key == 'a':
+                    return 12
+                raise KeyError
+            def __setitem__(self, key, value):
+                self.results = (key, value)
+            def keys(self):
+                return list('xyz')
+
+        m = M()
+        g = globals()
+        exec 'z = a' in g, m
+        self.assertEqual(m.results, ('z', 12))
+        try:
+            exec 'z = b' in g, m
+        except NameError:
+            pass
+        else:
+            self.fail('Did not detect a KeyError')
+        exec 'z = dir()' in g, m
+        self.assertEqual(m.results, ('z', list('xyz')))
+        exec 'z = globals()' in g, m
+        self.assertEqual(m.results, ('z', g))
+        exec 'z = locals()' in g, m
+        self.assertEqual(m.results, ('z', m))
+        try:
+            exec 'z = b' in m
+        except TypeError:
+            pass
+        else:
+            self.fail('Did not validate globals as a real dict')
+
+        class A:
+            "Non-mapping"
+            pass
+        m = A()
+        try:
+            exec 'z = a' in g, m
+        except TypeError:
+            pass
+        else:
+            self.fail('Did not validate locals as a mapping')
+
+        # Verify that dict subclasses work as well
+        class D(dict):
+            def __getitem__(self, key):
+                if key == 'a':
+                    return 12
+                return dict.__getitem__(self, key)
+        d = D()
+        exec 'z = a' in g, d
+        self.assertEqual(d['z'], 12)
+
+    def test_unicode_encoding(self):
+        code = u"# -*- coding: utf-8 -*-\npass\n"
+        self.assertRaises(SyntaxError, compile, code, "tmp", "exec")
+def test_main():
+    test_support.run_unittest(TestSpecifics)
+
+if __name__ == "__main__":
+    # pyston change: remove duration in test output
+    # test_main()
+    import sys, StringIO, re
+    orig_stdout = sys.stdout
+    out = StringIO.StringIO()
+    sys.stdout = out
+    test_main()
+    sys.stdout = orig_stdout
+    print re.sub(" [.0-9]+s", " TIME", out.getvalue())
+

test/tests/eval_bad_indentation.py

-# expected: fail
-
 try:
     eval("\n 2")
     print "bad, should have thrown an exception"