Merge pull request #1118 from undingen/parser_module

add parser module

from_cpython/CMakeLists.txt

@@ -138,17 +138,21 @@ add_library(FROM_CPYTHON OBJECT ${STDMODULE_SRCS} ${STDOBJECT_SRCS} ${STDPYTHON_
 add_dependencies(FROM_CPYTHON copy_stdlib)
 
 set(STDMODULES
-    ${CMAKE_BINARY_DIR}/from_cpython/Lib/future_builtins.pyston.so
-    ${CMAKE_BINARY_DIR}/from_cpython/Lib/_multiprocessing.pyston.so
-    ${CMAKE_BINARY_DIR}/from_cpython/Lib/pyexpat.pyston.so
-    ${CMAKE_BINARY_DIR}/from_cpython/Lib/_elementtree.pyston.so
     ${CMAKE_BINARY_DIR}/from_cpython/Lib/bz2.pyston.so
+    ${CMAKE_BINARY_DIR}/from_cpython/Lib/cmath.pyston.so
+    ${CMAKE_BINARY_DIR}/from_cpython/Lib/cPickle.pyston.so
     ${CMAKE_BINARY_DIR}/from_cpython/Lib/_ctypes.pyston.so
+    ${CMAKE_BINARY_DIR}/from_cpython/Lib/_curses.pyston.so
+    ${CMAKE_BINARY_DIR}/from_cpython/Lib/_elementtree.pyston.so
+    ${CMAKE_BINARY_DIR}/from_cpython/Lib/future_builtins.pyston.so
     ${CMAKE_BINARY_DIR}/from_cpython/Lib/grp.pyston.so
+    ${CMAKE_BINARY_DIR}/from_cpython/Lib/_locale.pyston.so
+    ${CMAKE_BINARY_DIR}/from_cpython/Lib/mmap.pyston.so
+    ${CMAKE_BINARY_DIR}/from_cpython/Lib/_multiprocessing.pyston.so
+    ${CMAKE_BINARY_DIR}/from_cpython/Lib/parser.pyston.so
+    ${CMAKE_BINARY_DIR}/from_cpython/Lib/pyexpat.pyston.so
     ${CMAKE_BINARY_DIR}/from_cpython/Lib/readline.pyston.so
     ${CMAKE_BINARY_DIR}/from_cpython/Lib/termios.pyston.so
-    ${CMAKE_BINARY_DIR}/from_cpython/Lib/_curses.pyston.so
-    ${CMAKE_BINARY_DIR}/from_cpython/Lib/mmap.pyston.so
 )
 
 add_custom_command(OUTPUT ${STDMODULES}
@@ -181,6 +185,7 @@ add_custom_command(OUTPUT ${STDMODULES}
 		       Modules/mmapmodule.c
 		       Modules/_localemodule.c
 		       Modules/cPickle.c
+                       Modules/parsermodule.c
                    WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR})
 add_custom_target(sharedmods ALL DEPENDS ${CMAKE_BINARY_DIR}/from_cpython/Lib/_multiprocessing.pyston.so)
 

from_cpython/Include/Python.h

@@ -132,6 +132,7 @@ PyAPI_FUNC(void) PyType_SetDict(PyTypeObject*, PyObject*) PYSTON_NOEXCEPT;
 #include "abstract.h"
 
 #include "compile.h"
+#include "eval.h"
 
 #include "pyctype.h"
 #include "pystrtod.h"

from_cpython/Include/compile.h

@@ -10,7 +10,7 @@ extern "C" {
 
 /* Public interface */
 struct _node; /* Declare the existence of this type */
-PyAPI_FUNC(PyCodeObject *) PyNode_Compile(struct _node *, const char *);
+PyAPI_FUNC(PyCodeObject *) PyNode_Compile(struct _node *, const char *) PYSTON_NOEXCEPT;
 
 /* Future feature support */
 
@@ -30,8 +30,8 @@ typedef struct {
 
 struct _mod; /* Declare the existence of this type */
 PyAPI_FUNC(PyCodeObject *) PyAST_Compile(struct _mod *, const char *,
-					PyCompilerFlags *, PyArena *);
-PyAPI_FUNC(PyFutureFeatures *) PyFuture_FromAST(struct _mod *, const char *);
+                    PyCompilerFlags *, PyArena *) PYSTON_NOEXCEPT;
+PyAPI_FUNC(PyFutureFeatures *) PyFuture_FromAST(struct _mod *, const char *) PYSTON_NOEXCEPT;
 
 
 #ifdef __cplusplus

from_cpython/Include/eval.h

@@ -7,7 +7,7 @@
 extern "C" {
 #endif
 
-PyAPI_FUNC(PyObject *) PyEval_EvalCode(PyCodeObject *, PyObject *, PyObject *);
+PyAPI_FUNC(PyObject *) PyEval_EvalCode(PyCodeObject *, PyObject *, PyObject *) PYSTON_NOEXCEPT;
 
 PyAPI_FUNC(PyObject *) PyEval_EvalCodeEx(PyCodeObject *co,
 					PyObject *globals,
@@ -15,9 +15,9 @@ PyAPI_FUNC(PyObject *) PyEval_EvalCodeEx(PyCodeObject *co,
 					PyObject **args, int argc,
 					PyObject **kwds, int kwdc,
 					PyObject **defs, int defc,
-					PyObject *closure);
+                    PyObject *closure) PYSTON_NOEXCEPT;
 
-PyAPI_FUNC(PyObject *) _PyEval_CallTracing(PyObject *func, PyObject *args);
+PyAPI_FUNC(PyObject *) _PyEval_CallTracing(PyObject *func, PyObject *args) PYSTON_NOEXCEPT;
 
 #ifdef __cplusplus
 }

from_cpython/Lib/test/test_parser.py

-# expected: fail
 import parser
 import unittest
 import sys

from_cpython/Lib/test/test_socketserver.py

-# expected: fail
 """
 Test suite for SocketServer.py.
 """

from_cpython/Lib/test/test_unicode_file.py

-# expected: fail
 # Test some Unicode file name semantics
 # We dont test many operations on files other than
 # that their names can be used with Unicode characters.

from_cpython/Lib/test/test_unicodedata.py

-# expected: fail
 """ Test script for the unicodedata module.
 
     Written by Marc-Andre Lemburg (mal@lemburg.com).

from_cpython/Lib/test/test_univnewlines2k.py

-# expected: fail
 # Tests universal newline support for both reading and parsing files.
 import unittest
 import os

from_cpython/Modules/parsermodule.c

@@ -3453,6 +3453,9 @@ initparser(void)
     if (PyModule_AddObject(module, "ParserError", parser_error) != 0)
         return;
 
+    // Pyston change:
+    PyType_Ready(&PyST_Type);
+
     Py_INCREF(&PyST_Type);
     PyModule_AddObject(module, "ASTType", (PyObject*)&PyST_Type);
     Py_INCREF(&PyST_Type);

from_cpython/setup.py

@@ -159,6 +159,12 @@ def cPickle_ext():
             "Modules/cPickle.c",
             ]))
 
+@unique
+def parser_ext():
+    return Extension("parser", sources = map(relpath, [
+            "Modules/parsermodule.c",
+            ]))
+
 ext_modules = [future_builtins_ext(),
                multiprocessing_ext(),
                pyexpat_ext(),
@@ -173,7 +179,8 @@ ext_modules = [future_builtins_ext(),
                termios_ext(),
                mmap_ext(),
                locale_ext(),
-               cPickle_ext()
+               cPickle_ext(),
+               parser_ext()
                ]
 
 

src/codegen/irgen/hooks.h

@@ -40,9 +40,6 @@ void compileAndRunModule(AST_Module* m, BoxedModule* bm);
 CompiledFunction* cfForMachineFunctionName(const std::string&);
 
 extern "C" Box* exec(Box* boxedCode, Box* globals, Box* locals, FutureFlags caller_future_flags);
-extern "C" Box* execfile(Box* fn, Box* globals, Box* locals);
-extern "C" Box* eval(Box* boxedCode, Box* globals, Box* locals);
-extern "C" Box* compile(Box* source, Box* filename, Box* mode, Box** _args /* flags, dont_inherit */);
 }
 
 #endif

src/runtime/capi.cpp

@@ -1133,6 +1133,11 @@ cleanup:
              : 0)
 #endif
 
+extern "C" void PyParser_SetError(perrdetail* err) noexcept {
+    err_input(err);
+}
+
+
 extern "C" grammar _PyParser_Grammar;
 
 /* Preferred access to parser is through AST. */
@@ -1184,6 +1189,76 @@ extern "C" mod_ty PyParser_ASTFromFile(FILE* fp, const char* filename, int start
     }
 }
 
+extern "C" PyObject* Py_CompileStringFlags(const char* str, const char* filename, int start,
+                                           PyCompilerFlags* flags) noexcept {
+    PyCodeObject* co;
+    mod_ty mod;
+    PyArena* arena = PyArena_New();
+    if (arena == NULL)
+        return NULL;
+
+    mod = PyParser_ASTFromString(str, filename, start, flags, arena);
+    if (mod == NULL) {
+        PyArena_Free(arena);
+        return NULL;
+    }
+    if (flags && (flags->cf_flags & PyCF_ONLY_AST)) {
+        PyObject* result = PyAST_mod2obj(mod);
+        PyArena_Free(arena);
+        return result;
+    }
+    co = PyAST_Compile(mod, filename, flags, arena);
+    PyArena_Free(arena);
+    return (PyObject*)co;
+}
+
+static PyObject* run_mod(mod_ty mod, const char* filename, PyObject* globals, PyObject* locals, PyCompilerFlags* flags,
+                         PyArena* arena) noexcept {
+    PyCodeObject* co;
+    PyObject* v;
+    co = PyAST_Compile(mod, filename, flags, arena);
+    if (co == NULL)
+        return NULL;
+    v = PyEval_EvalCode(co, globals, locals);
+    Py_DECREF(co);
+    return v;
+}
+
+extern "C" PyObject* PyRun_FileExFlags(FILE* fp, const char* filename, int start, PyObject* globals, PyObject* locals,
+                                       int closeit, PyCompilerFlags* flags) noexcept {
+    PyObject* ret;
+    mod_ty mod;
+    PyArena* arena = PyArena_New();
+    if (arena == NULL)
+        return NULL;
+
+    mod = PyParser_ASTFromFile(fp, filename, start, 0, 0, flags, NULL, arena);
+    if (closeit)
+        fclose(fp);
+    if (mod == NULL) {
+        PyArena_Free(arena);
+        return NULL;
+    }
+    ret = run_mod(mod, filename, globals, locals, flags, arena);
+    PyArena_Free(arena);
+    return ret;
+}
+
+extern "C" PyObject* PyRun_StringFlags(const char* str, int start, PyObject* globals, PyObject* locals,
+                                       PyCompilerFlags* flags) noexcept {
+    PyObject* ret = NULL;
+    mod_ty mod;
+    PyArena* arena = PyArena_New();
+    if (arena == NULL)
+        return NULL;
+
+    mod = PyParser_ASTFromString(str, "<string>", start, flags, arena);
+    if (mod != NULL)
+        ret = run_mod(mod, "<string>", globals, locals, flags, arena);
+    PyArena_Free(arena);
+    return ret;
+}
+
 extern "C" int PyRun_InteractiveLoopFlags(FILE* fp, const char* filename, PyCompilerFlags* flags) noexcept {
     PyObject* v;
     int ret;

test/CPYTHON_TEST_NOTES.md

@@ -136,7 +136,6 @@ test_nis                [unknown]
 test_optparse           assertion instead of exceptions for long("invalid number")
 test_ossaudiodev        [unknown]
 test_os                 [unknown]
-test_parser             [unknown]
 test_pdb                [unknown]
 test_peepholer          [unknown]
 test_pep352             various unique bugs
@@ -157,7 +156,6 @@ test_scope              eval of code object from existing function (not currentl
 test_scriptpackages     [unknown]
 test_shelve             [unknown]
 test_site               [unknown]
-test_socketserver       missing imp.lock_held, otherwise works
 test_socket             [unknown]
 test_sort               argument specification issue in listSort?
 test_sqlite             [unknown]
@@ -194,10 +192,7 @@ test_ttk_textonly       [unknown]
 test_types              PyErr_WarnEx
 test_undocumented_details   function.func_closure
 test_unicode            argument passing issue?
-test_unicodedata        [unknown]
-test_unicode_file       exit code 139, no error message
 test_unittest           serialize_ast assert
-test_univnewlines2k     [unknown]
 test_univnewlines       [unknown]
 test_userdict           segfault: repr of recursive dict?
 test_userlist           slice(1L, 1L)

test/extra/cheetah_test.py

@@ -9,5 +9,5 @@ create_virtenv(ENV_NAME, ["cheetah==2.4.4", "Markdown==2.0.1"], force_create = T
 cheetah_exe = os.path.join(ENV_NAME, "bin", "cheetah")
 env = os.environ
 env["PATH"] = os.path.join(ENV_NAME, "bin")
-expected = [{'ran': 2138, 'errors': 4, 'failures': 1}, {'ran': 2138, 'errors': 232, 'failures': 3}]
+expected = [{'ran': 2138, 'errors': 4}, {'ran': 2138, 'errors': 232, 'failures': 2}]
 run_test([cheetah_exe, "test"], cwd=ENV_NAME, expected=expected, env=env)

test/extra/geoip_test.py

@@ -7,7 +7,7 @@ ENV_NAME = "geoip_test_env_" + os.path.basename(sys.executable)
 SRC_DIR = os.path.abspath(os.path.join(ENV_NAME, "src"))
 PYTHON_EXE = os.path.abspath(os.path.join(ENV_NAME, "bin", "python"))
 
-pkg = ["-e", "git+https://github.com/maxmind/geoip-api-python.git@v1.3.2#egg=GeoIP"]
+pkg = ["nose==1.3.7", "-e", "git+http://github.com/maxmind/geoip-api-python.git@v1.3.2#egg=GeoIP"]
 create_virtenv(ENV_NAME, pkg, force_create = True)
 GEOIP_DIR = os.path.abspath(os.path.join(SRC_DIR, "geoip"))
 expected = [{'ran': 10}]

test/tests/compile_test2.py

+# this tests are from cpythons test_compile.py
+import unittest
+from test import test_support
+
+class TestSpecifics(unittest.TestCase):
+    def test_exec_functional_style(self):
+        # Exec'ing a tuple of length 2 works.
+        g = {'b': 2}
+        exec("a = b + 1", g)
+        self.assertEqual(g['a'], 3)
+
+        # As does exec'ing a tuple of length 3.
+        l = {'b': 3}
+        g = {'b': 5, 'c': 7}
+        exec("a = b + c", g, l)
+        self.assertNotIn('a', g)
+        self.assertEqual(l['a'], 10)
+
+        # Tuples not of length 2 or 3 are invalid.
+        with self.assertRaises(TypeError):
+            exec("a = b + 1",)
+
+        with self.assertRaises(TypeError):
+            exec("a = b + 1", {}, {}, {})
+
+        # Can't mix and match the two calling forms.
+        g = {'a': 3, 'b': 4}
+        l = {}
+        with self.assertRaises(TypeError):
+            exec("a = b + 1", g) in g
+        with self.assertRaises(TypeError):
+            exec("a = b + 1", g, l) in g, l
+
+    def test_exec_with_general_mapping_for_locals(self):
+
+        class M:
+            "Test mapping interface versus possible calls from eval()."
+            def __getitem__(self, key):
+                if key == 'a':
+                    return 12
+                raise KeyError
+            def __setitem__(self, key, value):
+                self.results = (key, value)
+            def keys(self):
+                return list('xyz')
+
+        m = M()
+        g = globals()
+        exec 'z = a' in g, m
+        self.assertEqual(m.results, ('z', 12))
+        try:
+            exec 'z = b' in g, m
+        except NameError:
+            pass
+        else:
+            self.fail('Did not detect a KeyError')
+        exec 'z = dir()' in g, m
+        self.assertEqual(m.results, ('z', list('xyz')))
+        exec 'z = globals()' in g, m
+        self.assertEqual(m.results, ('z', g))
+        exec 'z = locals()' in g, m
+        self.assertEqual(m.results, ('z', m))
+        try:
+            exec 'z = b' in m
+        except TypeError:
+            pass
+        else:
+            self.fail('Did not validate globals as a real dict')
+
+        class A:
+            "Non-mapping"
+            pass
+        m = A()
+        try:
+            exec 'z = a' in g, m
+        except TypeError:
+            pass
+        else:
+            self.fail('Did not validate locals as a mapping')
+
+        # Verify that dict subclasses work as well
+        class D(dict):
+            def __getitem__(self, key):
+                if key == 'a':
+                    return 12
+                return dict.__getitem__(self, key)
+        d = D()
+        exec 'z = a' in g, d
+        self.assertEqual(d['z'], 12)
+
+    def test_unicode_encoding(self):
+        code = u"# -*- coding: utf-8 -*-\npass\n"
+        self.assertRaises(SyntaxError, compile, code, "tmp", "exec")
+def test_main():
+    test_support.run_unittest(TestSpecifics)
+
+if __name__ == "__main__":
+    # pyston change: remove duration in test output
+    # test_main()
+    import sys, StringIO, re
+    orig_stdout = sys.stdout
+    out = StringIO.StringIO()
+    sys.stdout = out
+    test_main()
+    sys.stdout = orig_stdout
+    print re.sub(" [.0-9]+s", " TIME", out.getvalue())
+

test/tests/eval_bad_indentation.py

-# expected: fail
-
 try:
     eval("\n 2")
     print "bad, should have thrown an exception"