Add pyexpat support

I disabled the custom traceback stuff for now because we don't support it yet.
It's using the builtin copy of libexpat like cpython does.

Makefile

@@ -1189,11 +1189,17 @@ TEST_EXT_MODULE_NAMES := basic_test descr_test slots_test
 TEST_EXT_MODULE_SRCS := $(TEST_EXT_MODULE_NAMES:%=test/test_extension/%.c)
 TEST_EXT_MODULE_OBJS := $(TEST_EXT_MODULE_NAMES:%=test/test_extension/%.pyston.so)
 
-SHAREDMODS_NAMES := _multiprocessing
+SHAREDMODS_NAMES := _multiprocessing pyexpat
 SHAREDMODS_SRCS := \
 	_multiprocessing/multiprocessing.c \
 	_multiprocessing/semaphore.c \
-	_multiprocessing/socket_connection.c
+	_multiprocessing/socket_connection.c \
+	expat/xmlparse.c \
+	expat/xmlrole.c \
+	expat/xmltok.c \
+	expat/xmltok_impl.c \
+	expat/xmltok_ns.c \
+ 	pyexpat.c
 SHAREDMODS_SRCS := $(SHAREDMODS_SRCS:%=from_cpython/Modules/%)
 SHAREDMODS_OBJS := $(SHAREDMODS_NAMES:%=lib_pyston/%.pyston.so)
 

from_cpython/CMakeLists.txt

@@ -109,7 +109,7 @@ file(GLOB_RECURSE STDPARSER_SRCS Parser
 set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -Wno-missing-field-initializers -Wno-tautological-compare -Wno-type-limits -Wno-unused-result -Wno-strict-aliasing")
 add_library(FROM_CPYTHON OBJECT ${STDMODULE_SRCS} ${STDOBJECT_SRCS} ${STDPYTHON_SRCS} ${STDPARSER_SRCS})
 
-add_custom_command(OUTPUT ${CMAKE_BINARY_DIR}/lib_pyston/_multiprocessing.pyston.so
+add_custom_command(OUTPUT ${CMAKE_BINARY_DIR}/lib_pyston/_multiprocessing.pyston.so ${CMAKE_BINARY_DIR}/lib_pyston/pyexpat.pyston.so
                    COMMAND ${CMAKE_BINARY_DIR}/pyston setup.py build --build-lib ${CMAKE_BINARY_DIR}/lib_pyston
                    DEPENDS
                        pyston
@@ -118,5 +118,11 @@ add_custom_command(OUTPUT ${CMAKE_BINARY_DIR}/lib_pyston/_multiprocessing.pyston
                        Modules/_multiprocessing/multiprocessing.c
                        Modules/_multiprocessing/semaphore.c
                        Modules/_multiprocessing/socket_connection.c
+                       Modules/expat/xmlparse.c
+                       Modules/expat/xmlrole.c
+                       Modules/expat/xmltok.c
+                       Modules/expat/xmltok_impl.c
+                       Modules/expat/xmltok_ns.c
+                       Modules/pyexpat.c
                    WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR})
 add_custom_target(sharedmods DEPENDS ${CMAKE_BINARY_DIR}/lib_pyston/_multiprocessing.pyston.so)

from_cpython/Include/pyexpat.h

+// This file is originally from CPython 2.7, with modifications for Pyston
+
+/* Stuff to export relevant 'expat' entry points from pyexpat to other
+ * parser modules, such as cElementTree. */
+
+/* note: you must import expat.h before importing this module! */
+
+#define PyExpat_CAPI_MAGIC  "pyexpat.expat_CAPI 1.0"
+#define PyExpat_CAPSULE_NAME "pyexpat.expat_CAPI"
+
+struct PyExpat_CAPI
+{
+    char* magic; /* set to PyExpat_CAPI_MAGIC */
+    int size; /* set to sizeof(struct PyExpat_CAPI) */
+    int MAJOR_VERSION;
+    int MINOR_VERSION;
+    int MICRO_VERSION;
+    /* pointers to selected expat functions.  add new functions at
+       the end, if needed */
+    const XML_LChar * (*ErrorString)(enum XML_Error code);
+    enum XML_Error (*GetErrorCode)(XML_Parser parser);
+    XML_Size (*GetErrorColumnNumber)(XML_Parser parser);
+    XML_Size (*GetErrorLineNumber)(XML_Parser parser);
+    enum XML_Status (*Parse)(
+        XML_Parser parser, const char *s, int len, int isFinal);
+    XML_Parser (*ParserCreate_MM)(
+        const XML_Char *encoding, const XML_Memory_Handling_Suite *memsuite,
+        const XML_Char *namespaceSeparator);
+    void (*ParserFree)(XML_Parser parser);
+    void (*SetCharacterDataHandler)(
+        XML_Parser parser, XML_CharacterDataHandler handler);
+    void (*SetCommentHandler)(
+        XML_Parser parser, XML_CommentHandler handler);
+    void (*SetDefaultHandlerExpand)(
+        XML_Parser parser, XML_DefaultHandler handler);
+    void (*SetElementHandler)(
+        XML_Parser parser, XML_StartElementHandler start,
+        XML_EndElementHandler end);
+    void (*SetNamespaceDeclHandler)(
+        XML_Parser parser, XML_StartNamespaceDeclHandler start,
+        XML_EndNamespaceDeclHandler end);
+    void (*SetProcessingInstructionHandler)(
+        XML_Parser parser, XML_ProcessingInstructionHandler handler);
+    void (*SetUnknownEncodingHandler)(
+        XML_Parser parser, XML_UnknownEncodingHandler handler,
+        void *encodingHandlerData);
+    void (*SetUserData)(XML_Parser parser, void *userData);
+    /* always add new stuff to the end! */
+};
+

from_cpython/Modules/pyexpat.c

 #include "Python.h"
 #include <ctype.h>
 
-#include "frameobject.h"
+// Pyston change:
+//#include "frameobject.h"
+#include "code.h" // CPython includes this from compile.h which gets included by Python.h
+
 #include "expat.h"
 
 #include "pyexpat.h"
@@ -29,6 +32,9 @@
 #define FIX_TRACE
 #endif
 
+// Pyston change: Disable tracing for now until we support it
+#undef FIX_TRACE
+
 enum HandlerTypes {
     StartElement,
     EndElement,
@@ -153,7 +159,7 @@ get_handler_name(struct HandlerInfo *hinfo)
 {
     PyObject *name = hinfo->nameobj;
     if (name == NULL) {
-        name = PyString_FromString(hinfo->name);
+        name = PyGC_AddRoot(PyString_FromString(hinfo->name));
         hinfo->nameobj = name;
     }
     Py_XINCREF(name);
@@ -261,11 +267,16 @@ flag_error(xmlparseobject *self)
 static PyCodeObject*
 getcode(enum HandlerTypes slot, char* func_name, int lineno)
 {
+    // Pyston change: Disable this until we support custom code objects
+#if 0
     if (handler_info[slot].tb_code == NULL) {
         handler_info[slot].tb_code =
             PyCode_NewEmpty(__FILE__, func_name, lineno);
     }
     return handler_info[slot].tb_code;
+#else
+    return NULL;
+#endif
 }
 
 #ifdef FIX_TRACE
@@ -336,6 +347,8 @@ static PyObject*
 call_with_frame(PyCodeObject *c, PyObject* func, PyObject* args,
                 xmlparseobject *self)
 {
+// Pyston change: Disable this until we support custom tracebacks
+#if 0
     PyThreadState *tstate = PyThreadState_GET();
     PyFrameObject *f;
     PyObject *res;
@@ -374,6 +387,14 @@ call_with_frame(PyCodeObject *c, PyObject* func, PyObject* args,
     tstate->frame = f->f_back;
     Py_DECREF(f);
     return res;
+#else
+    PyObject *res;
+    res = PyEval_CallObject(func, args);
+    if (res == NULL) {
+        XML_StopParser(self->itself, XML_FALSE);
+    }
+    return res;
+#endif
 }
 
 #ifndef Py_USING_UNICODE
@@ -1122,7 +1143,9 @@ xmlparse_ExternalEntityParserCreate(xmlparseobject *self, PyObject *args)
     for (i = 0; handler_info[i].name != NULL; i++)
         /* do nothing */;
 
-    new_parser->handlers = malloc(sizeof(PyObject *) * i);
+    // Pyston change: use GC alloc routine because those contain Python objects
+    // new_parser->handlers = malloc(sizeof(PyObject *) * i);
+    new_parser->handlers = PyMem_Malloc(sizeof(PyObject *) * i);
     if (!new_parser->handlers) {
         Py_DECREF(new_parser);
         return PyErr_NoMemory();
@@ -1341,7 +1364,9 @@ newxmlparseobject(char *encoding, char *namespace_separator, PyObject *intern)
     for (i = 0; handler_info[i].name != NULL; i++)
         /* do nothing */;
 
-    self->handlers = malloc(sizeof(PyObject *) * i);
+    // Pyston change: use GC alloc routine because those contain Python objects
+    // self->handlers = malloc(sizeof(PyObject *) * i);
+    self->handlers = PyMem_Malloc(sizeof(PyObject *) * i);
     if (!self->handlers) {
         Py_DECREF(self);
         return PyErr_NoMemory();
@@ -1372,7 +1397,8 @@ xmlparse_dealloc(xmlparseobject *self)
             self->handlers[i] = NULL;
             Py_XDECREF(temp);
         }
-        free(self->handlers);
+        // Pyston change: object are allocated using the GC not malloc
+        // free(self->handlers);
         self->handlers = NULL;
     }
     if (self->buffer != NULL) {
@@ -1853,6 +1879,10 @@ MODULE_INITFUNC(void)
 
     Py_TYPE(&Xmlparsetype) = &PyType_Type;
 
+    // Pyston change:
+    if (PyType_Ready(&Xmlparsetype) < 0)
+        return;
+
     /* Create the module and add the functions */
     m = Py_InitModule3(MODULE_NAME, pyexpat_methods,
                        pyexpat_module_documentation);
@@ -1861,8 +1891,8 @@ MODULE_INITFUNC(void)
 
     /* Add some symbolic constants to the module */
     if (ErrorObject == NULL) {
-        ErrorObject = PyErr_NewException("xml.parsers.expat.ExpatError",
-                                         NULL, NULL);
+        ErrorObject = PyGC_AddRoot(PyErr_NewException("xml.parsers.expat.ExpatError",
+                                         NULL, NULL));
         if (ErrorObject == NULL)
             return;
     }

from_cpython/setup.py

@@ -7,13 +7,41 @@ def relpath(fn):
     r =  os.path.join(os.path.dirname(__file__), fn)
     return r
 
+def multiprocessing_ext():
+    return Extension("_multiprocessing", sources = map(relpath, [
+            "Modules/_multiprocessing/multiprocessing.c",
+            "Modules/_multiprocessing/socket_connection.c",
+            "Modules/_multiprocessing/semaphore.c",
+            ]))
+
+def pyexpat_ext():
+    define_macros = [('HAVE_EXPAT_CONFIG_H', '1'),]
+    expat_sources = map(relpath, ['Modules/expat/xmlparse.c',
+                                  'Modules/expat/xmlrole.c',
+                                  'Modules/expat/xmltok.c',
+                                  'Modules/pyexpat.c'])
+
+    expat_depends = map(relpath, ['Modules/expat/ascii.h',
+                                  'Modules/expat/asciitab.h',
+                                  'Modules/expat/expat.h',
+                                  'Modules/expat/expat_config.h',
+                                  'Modules/expat/expat_external.h',
+                                  'Modules/expat/internal.h',
+                                  'Modules/expat/latin1tab.h',
+                                  'Modules/expat/utf8tab.h',
+                                  'Modules/expat/xmlrole.h',
+                                  'Modules/expat/xmltok.h',
+                                  'Modules/expat/xmltok_impl.h'
+                                  ])
+    return Extension('pyexpat',
+                      define_macros = define_macros,
+                      include_dirs = [relpath('Modules/expat')],
+                      sources = expat_sources,
+                      depends = expat_depends,
+                    )
+
 setup(name="Pyston",
         version="1.0",
         description="Pyston shared modules",
-        ext_modules=[Extension("_multiprocessing", sources = map(relpath, [
-                "Modules/_multiprocessing/multiprocessing.c",
-                "Modules/_multiprocessing/socket_connection.c",
-                "Modules/_multiprocessing/semaphore.c",
-            ]),
-        )],
+        ext_modules=[multiprocessing_ext(), pyexpat_ext()]
     )

src/capi/modsupport.cpp

@@ -450,6 +450,15 @@ extern "C" int PyModule_AddIntConstant(PyObject* _m, const char* name, long valu
     return PyModule_AddObject(_m, name, boxInt(value));
 }
 
+extern "C" PyObject* PyModule_New(const char* name) noexcept {
+    BoxedModule* module = new BoxedModule();
+    module->giveAttr("__name__", boxStrConstant(name));
+    module->giveAttr("__doc__", None);
+    module->giveAttr("__package__", None);
+    return module;
+}
+
+
 extern "C" PyObject* PyEval_CallMethod(PyObject* obj, const char* methodname, const char* format, ...) noexcept {
     va_list vargs;
     PyObject* meth;