Get the imported files to compile (but not link)

from_cpython/CMakeLists.txt

@@ -100,8 +100,10 @@ file(GLOB_RECURSE STDPYTHON_SRCS Python
     formatter_string.c
     formatter_unicode.c
     getargs.c
+    graminit.c
     marshal.c
     mystrtoul.c
+    pyarena.c
     pyctype.c
     pystrtod.c
     Python-ast.c
@@ -111,7 +113,13 @@ file(GLOB_RECURSE STDPYTHON_SRCS Python
 
 # compile specified files in from_cpython/Python
 file(GLOB_RECURSE STDPARSER_SRCS Parser
+    acceler.c
+    grammar1.c
     myreadline.c
+    node.c
+    parser.c
+    parsetok.c
+    tokenizer.c
     )
 
 set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -Wno-missing-field-initializers -Wno-tautological-compare -Wno-type-limits -Wno-unused-result -Wno-strict-aliasing -DPy_BUILD_CORE")

from_cpython/Include/Python.h

@@ -41,9 +41,6 @@
 
 #include <assert.h>
 
-// CPython doesn't seem to include this but I'm not sure how they get the definition of 'bool':
-#include <stdbool.h>
-
 #include "pyport.h"
 
 #include "pymath.h"
@@ -133,8 +130,11 @@ PyAPI_FUNC(void) PyType_SetDict(PyTypeObject*, PyObject*) PYSTON_NOEXCEPT;
 
 #include "abstract.h"
 
+#include "compile.h"
+
 #include "pyctype.h"
 #include "pystrtod.h"
+#include "pystrcmp.h"
 #include "dtoa.h"
 
 // directly from CPython:
@@ -185,9 +185,6 @@ extern PyTypeObject* Itertool_SafeDealloc_Types[];
 #define PyDoc_STRVAR(name, str) PyDoc_VAR(name) = PyDoc_STR(str)
 #define PyDoc_STR(str) str
 
-// This is in Python-ast.h in CPython, which we don't yet have:
-int PyAST_Check(PyObject* obj) PYSTON_NOEXCEPT;
-
 #ifdef __cplusplus
 #define PyMODINIT_FUNC extern "C" void
 #else

from_cpython/Include/abstract.h

@@ -534,7 +534,7 @@ xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx*/
     /* new buffer API */
 
      // Pyston change: made this a function
-    bool _PyObject_CheckBuffer(PyObject* obj) PYSTON_NOEXCEPT;
+    int _PyObject_CheckBuffer(PyObject* obj) PYSTON_NOEXCEPT;
 #define PyObject_CheckBuffer(obj) _PyObject_CheckBuffer((PyObject*)(obj))
 
     /* Return 1 if the getbuffer function is available, otherwise
@@ -847,7 +847,7 @@ xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx*/
        */
 
      // Pyston change: made this a function:
-    PyAPI_FUNC(bool) _PyIndex_Check(PyObject* o) PYSTON_NOEXCEPT;
+    PyAPI_FUNC(int) _PyIndex_Check(PyObject* o) PYSTON_NOEXCEPT;
 #define PyIndex_Check(obj) _PyIndex_Check((PyObject*)(obj))
 #if 0
 #define PyIndex_Check(obj) \

from_cpython/Include/ast.h

@@ -5,7 +5,7 @@ extern "C" {
 #endif
 
 PyAPI_FUNC(mod_ty) PyAST_FromNode(const node *, PyCompilerFlags *flags,
-				  const char *, PyArena *);
+				  const char *, PyArena *) PYSTON_NOEXCEPT;
 
 #ifdef __cplusplus
 }

from_cpython/Include/fileobject.h

@@ -63,6 +63,9 @@ PyAPI_FUNC(int) PyFile_SoftSpace(PyObject *, int) PYSTON_NOEXCEPT;
 PyAPI_FUNC(int) PyFile_WriteString(const char *, PyObject *) PYSTON_NOEXCEPT;
 PyAPI_FUNC(int) PyObject_AsFileDescriptor(PyObject *) PYSTON_NOEXCEPT;
 
+// Pyston temporary addition:
+PyAPI_FUNC(PyObject *) PyFile_GetEncoding(PyObject *) PYSTON_NOEXCEPT;
+
 /* The default encoding used by the platform file system APIs
    If non-NULL, this is different than the default encoding for strings
 */

from_cpython/Include/object.h

@@ -459,7 +459,7 @@ struct _typeobject {
     char _ics[32];
     void* _gcvisit_func;
     int _attrs_offset;
-    bool _flags[7];
+    char _flags[7]; // These are bools in C++
     void* _tpp_descr_get;
     void* _tpp_hasnext;
     void* _tpp_call_capi;

from_cpython/Include/parsetok.h

@@ -33,30 +33,30 @@ typedef struct {
 
 
 PyAPI_FUNC(node *) PyParser_ParseString(const char *, grammar *, int,
-                                              perrdetail *);
+                                              perrdetail *) PYSTON_NOEXCEPT;
 PyAPI_FUNC(node *) PyParser_ParseFile (FILE *, const char *, grammar *, int,
-                                             char *, char *, perrdetail *);
+                                             char *, char *, perrdetail *) PYSTON_NOEXCEPT;
 
 PyAPI_FUNC(node *) PyParser_ParseStringFlags(const char *, grammar *, int,
-                                              perrdetail *, int);
+                                              perrdetail *, int) PYSTON_NOEXCEPT;
 PyAPI_FUNC(node *) PyParser_ParseFileFlags(FILE *, const char *, grammar *,
 						 int, char *, char *,
-						 perrdetail *, int);
+						 perrdetail *, int) PYSTON_NOEXCEPT;
 PyAPI_FUNC(node *) PyParser_ParseFileFlagsEx(FILE *, const char *, grammar *,
 						 int, char *, char *,
-						 perrdetail *, int *);
+						 perrdetail *, int *) PYSTON_NOEXCEPT;
 
 PyAPI_FUNC(node *) PyParser_ParseStringFlagsFilename(const char *,
 					      const char *,
 					      grammar *, int,
-                                              perrdetail *, int);
+                                              perrdetail *, int) PYSTON_NOEXCEPT;
 PyAPI_FUNC(node *) PyParser_ParseStringFlagsFilenameEx(const char *,
 					      const char *,
 					      grammar *, int,
-                                              perrdetail *, int *);
+                                              perrdetail *, int *) PYSTON_NOEXCEPT;
 
 /* Note that he following function is defined in pythonrun.c not parsetok.c. */
-PyAPI_FUNC(void) PyParser_SetError(perrdetail *);
+PyAPI_FUNC(void) PyParser_SetError(perrdetail *) PYSTON_NOEXCEPT;
 
 #ifdef __cplusplus
 }

from_cpython/Modules/_ctypes/cfield.c

@@ -1293,7 +1293,7 @@ s_get(void *ptr, Py_ssize_t size)
     size = min(size, (Py_ssize_t)slen);
 
     // Pyston change: no ob_refcnt
-    if (false /*result->ob_refcnt == 1*/) {
+    if (0 /*result->ob_refcnt == 1*/) {
         /* shorten the result */
         _PyString_Resize(&result, size);
         return result;

from_cpython/Objects/weakrefobject.c

@@ -951,7 +951,7 @@ PyObject_ClearWeakRefs(PyObject *object)
                 // Pyston change:
                 // current is a stack reference to a GC allocated object.  If it wasn't null when we fetched it from *list, it won't
                 // be collected, and we can trust that it's still valid here.
-                if (true /*current->ob_refcnt > 0*/)
+                if (1 /*current->ob_refcnt > 0*/)
                     handle_callback(current, callback);
                 Py_DECREF(callback);
             }
@@ -973,7 +973,7 @@ PyObject_ClearWeakRefs(PyObject *object)
                 // Pyston change:
                 // current is a stack reference to a GC allocated object.  If it wasn't null when we fetched it from *list, it won't
                 // be collected, and we can trust that it's still valid here.
-                if (true /*current->ob_refcnt > 0*/)
+                if (1 /*current->ob_refcnt > 0*/)
                 {
                     Py_INCREF(current);
                     PyTuple_SET_ITEM(tuple, i * 2, (PyObject *) current);

from_cpython/Parser/acceler.c

@@ -111,7 +111,8 @@ fixstate(grammar *g, state *s)
         k++;
     if (k < nl) {
         int i;
-        s->s_accel = (int *) PyObject_MALLOC((nl-k) * sizeof(int));
+        // Pyston change:
+        s->s_accel = (int *) PyGC_AddRoot(PyObject_MALLOC((nl-k) * sizeof(int)));
         if (s->s_accel == NULL) {
             fprintf(stderr, "no mem to add parser accelerators\n");
             exit(1);

from_cpython/Parser/tokenizer.c

@@ -19,7 +19,7 @@
 #include "pydebug.h"
 #endif /* PGEN */
 
-extern char *PyOS_Readline(FILE *, FILE *, char *);
+extern char *PyOS_Readline(FILE *, FILE *, const char *);
 /* Return malloc'ed string including trailing \n;
    empty malloc'ed string for EOF;
    NULL if interrupted */
@@ -783,7 +783,7 @@ tok_stdin_decode(struct tok_state *tok, char **inp)
     if (sysstdin == NULL || !PyFile_Check(sysstdin))
         return 0;
 
-    enc = ((PyFileObject *)sysstdin)->f_encoding;
+    enc = PyFile_GetEncoding(sysstdin);
     if (enc == NULL || !PyString_Check(enc))
         return 0;
     Py_INCREF(enc);

from_cpython/Python/Python-ast.c

@@ -461,7 +461,7 @@ static PyMethodDef ast_type_methods[] = {
 };
 
 static PyTypeObject AST_type = {
-    PyVarObject_HEAD_INIT(&PyType_Type, 0)
+    PyVarObject_HEAD_INIT(/* Pyston change: &PyType_Type */ NULL, 0)
     "_ast.AST",
     sizeof(PyObject),
     0,
@@ -6779,10 +6779,13 @@ mod_ty PyAST_obj2mod(PyObject* ast, PyArena* arena, int mode)
         return res;
 }
 
+// Pyston temporary change: we're not using this file for the python module, at least not yet.
+#if 0
 int PyAST_Check(PyObject* obj)
 {
     init_types();
     return PyObject_IsInstance(obj, (PyObject*)&AST_type);
 }
+#endif
 
 

src/codegen/cpython_ast.h

@@ -22,8 +22,10 @@
 
 namespace pyston {
 
-AST_Module* cpythonToPystonAst(mod_ty mod);
-
+// Convert a CPython ast object to a Pyston ast object.
+// This will also check for certain kinds of "syntax errors" (ex continue not in loop) and will
+// throw them as C++ exceptions.
+AST_Module* cpythonToPystonAST(mod_ty mod, llvm::StringRef fn);
 }
 
 #endif

src/codegen/irgen/hooks.cpp

@@ -14,6 +14,16 @@
 
 #include "codegen/irgen/hooks.h"
 
+#include "codegen/cpython_ast.h"
+// These #defines in Python-ast.h conflict with llvm:
+#undef Pass
+#undef Module
+#undef alias
+#undef Option
+#undef Name
+#undef Attribute
+#undef Set
+
 #include "llvm/ExecutionEngine/ExecutionEngine.h"
 #include "llvm/Support/raw_ostream.h"
 

src/runtime/capi.cpp

@@ -39,12 +39,12 @@ namespace pyston {
 
 BoxedClass* method_cls;
 
-extern "C" bool _PyIndex_Check(PyObject* obj) noexcept {
+extern "C" int _PyIndex_Check(PyObject* obj) noexcept {
     return (Py_TYPE(obj)->tp_as_number != NULL && PyType_HasFeature(Py_TYPE(obj), Py_TPFLAGS_HAVE_INDEX)
             && Py_TYPE(obj)->tp_as_number->nb_index != NULL);
 }
 
-extern "C" bool _PyObject_CheckBuffer(PyObject* obj) noexcept {
+extern "C" int _PyObject_CheckBuffer(PyObject* obj) noexcept {
     return ((Py_TYPE(obj)->tp_as_buffer != NULL) && (PyType_HasFeature(Py_TYPE(obj), Py_TPFLAGS_HAVE_NEWBUFFER))
             && (Py_TYPE(obj)->tp_as_buffer->bf_getbuffer != NULL));
 }