Merge pull request #612 from toshok/mercurial

add support for the _curses, bz2, and grp modules (mercurial needs them)

.travis.yml

@@ -24,6 +24,7 @@ addons:
     - cmake
     - g++-4.8
     - gdb
+    - libbz2-dev
     - libgmp3-dev
     - liblzma-dev
     - libncurses5-dev

docs/INSTALLING.md

@@ -14,18 +14,18 @@ sudo add-apt-repository --yes ppa:ubuntu-toolchain-r/test
 sudo add-apt-repository --yes ppa:kubuntu-ppa/backports
 sudo apt-get -qq update
 
-sudo apt-get install -yq git cmake ninja-build ccache libncurses5-dev liblzma-dev libreadline-dev libgmp3-dev autoconf libtool python-dev texlive-extra-utils clang-3.4 libstdc++-4.8-dev libssl-dev libsqlite3-dev pkg-config
+sudo apt-get install -yq git cmake ninja-build ccache libncurses5-dev liblzma-dev libreadline-dev libgmp3-dev autoconf libtool python-dev texlive-extra-utils clang-3.4 libstdc++-4.8-dev libssl-dev libsqlite3-dev pkg-config libbz2-dev
 ```
 
 
 **Ubuntu 14.04/14.10/15.04**
 ```
-sudo apt-get install -yq git cmake ninja-build ccache libncurses5-dev liblzma-dev libreadline-dev libgmp3-dev autoconf libtool python-dev texlive-extra-utils clang libssl-dev libsqlite3-dev pkg-config
+sudo apt-get install -yq git cmake ninja-build ccache libncurses5-dev liblzma-dev libreadline-dev libgmp3-dev autoconf libtool python-dev texlive-extra-utils clang libssl-dev libsqlite3-dev pkg-config libbz2-dev
 ```
 
 **Fedora 21**
 ```
-sudo yum install cmake clang gcc gcc-c++ ccache ninja-build xz-devel automake libtool gmp-devel readline-devel openssl-devel sqlite-devel python-devel zlib-devel
+sudo yum install cmake clang gcc gcc-c++ ccache ninja-build xz-devel automake libtool gmp-devel readline-devel openssl-devel sqlite-devel python-devel zlib-devel bzip2-devel ncurses-devel
 ```
 
 ### Building and testing

from_cpython/CMakeLists.txt

@@ -117,6 +117,10 @@ add_custom_command(OUTPUT
                        ${CMAKE_BINARY_DIR}/lib_pyston/_multiprocessing.pyston.so
                        ${CMAKE_BINARY_DIR}/lib_pyston/pyexpat.pyston.so
                        ${CMAKE_BINARY_DIR}/lib_pyston/_elementtree.pyston.so
+                       ${CMAKE_BINARY_DIR}/lib_pyston/bz2.so
+                       ${CMAKE_BINARY_DIR}/lib_pyston/grp.so
+                       ${CMAKE_BINARY_DIR}/lib_pyston/termios.so
+                       ${CMAKE_BINARY_DIR}/lib_pyston/_curses.so
                    COMMAND ${CMAKE_BINARY_DIR}/pyston setup.py build --build-lib ${CMAKE_BINARY_DIR}/lib_pyston
                    DEPENDS
                        pyston
@@ -132,5 +136,9 @@ add_custom_command(OUTPUT
                        Modules/expat/xmltok_ns.c
                        Modules/pyexpat.c
                        Modules/_elementtree.c
+		       Modules/bz2module.c
+		       Modules/grpmodule.c
+		       Modules/termios.c
+		       Modules/_cursesmodule.c
                    WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR})
 add_custom_target(sharedmods DEPENDS ${CMAKE_BINARY_DIR}/lib_pyston/_multiprocessing.pyston.so)

from_cpython/Include/py_curses.h

+
+#ifndef Py_CURSES_H
+#define Py_CURSES_H
+
+#ifdef __APPLE__
+/*
+** On Mac OS X 10.2 [n]curses.h and stdlib.h use different guards
+** against multiple definition of wchar_t.
+*/
+#ifdef	_BSD_WCHAR_T_DEFINED_
+#define _WCHAR_T
+#endif
+
+/* the following define is necessary for OS X 10.6; without it, the
+   Apple-supplied ncurses.h sets NCURSES_OPAQUE to 1, and then Python
+   can't get at the WINDOW flags field. */
+#define NCURSES_OPAQUE 0
+#endif /* __APPLE__ */
+
+#ifdef __FreeBSD__
+/*
+** On FreeBSD, [n]curses.h and stdlib.h/wchar.h use different guards
+** against multiple definition of wchar_t and wint_t.
+*/
+#ifdef	_XOPEN_SOURCE_EXTENDED
+#ifndef __FreeBSD_version
+#include <osreldate.h>
+#endif
+#if __FreeBSD_version >= 500000
+#ifndef __wchar_t
+#define __wchar_t
+#endif
+#ifndef __wint_t
+#define __wint_t
+#endif
+#else
+#ifndef _WCHAR_T
+#define _WCHAR_T
+#endif
+#ifndef _WINT_T
+#define _WINT_T
+#endif
+#endif
+#endif
+#endif
+
+#ifdef HAVE_NCURSES_H
+#include <ncurses.h>
+#else
+#include <curses.h>
+#ifdef HAVE_TERM_H
+/* for tigetstr, which is not declared in SysV curses */
+#include <term.h>
+#endif
+#endif
+
+#ifdef HAVE_NCURSES_H
+/* configure was checking <curses.h>, but we will
+   use <ncurses.h>, which has all these features. */
+#ifndef WINDOW_HAS_FLAGS
+#define WINDOW_HAS_FLAGS 1
+#endif
+#ifndef MVWDELCH_IS_EXPRESSION
+#define MVWDELCH_IS_EXPRESSION 1
+#endif
+#endif
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+#define PyCurses_API_pointers 4
+
+/* Type declarations */
+
+typedef struct {
+	PyObject_HEAD
+	WINDOW *win;
+} PyCursesWindowObject;
+
+#define PyCursesWindow_Check(v)	 (Py_TYPE(v) == &PyCursesWindow_Type)
+
+#define PyCurses_CAPSULE_NAME "_curses._C_API"
+
+
+#ifdef CURSES_MODULE
+/* This section is used when compiling _cursesmodule.c */
+
+#else
+/* This section is used in modules that use the _cursesmodule API */
+
+static void **PyCurses_API;
+
+#define PyCursesWindow_Type (*(PyTypeObject *) PyCurses_API[0])
+#define PyCursesSetupTermCalled  {if (! ((int (*)(void))PyCurses_API[1]) () ) return NULL;}
+#define PyCursesInitialised      {if (! ((int (*)(void))PyCurses_API[2]) () ) return NULL;}
+#define PyCursesInitialisedColor {if (! ((int (*)(void))PyCurses_API[3]) () ) return NULL;}
+
+#define import_curses() \
+    PyCurses_API = (void **)PyCapsule_Import(PyCurses_CAPSULE_NAME, 1);
+
+#endif
+
+/* general error messages */
+static char *catchall_ERR  = "curses function returned ERR";
+static char *catchall_NULL = "curses function returned NULL";
+
+/* Function Prototype Macros - They are ugly but very, very useful. ;-)
+
+   X - function name
+   TYPE - parameter Type
+   ERGSTR - format string for construction of the return value
+   PARSESTR - format string for argument parsing
+   */
+
+#define NoArgNoReturnFunction(X) \
+static PyObject *PyCurses_ ## X (PyObject *self) \
+{ \
+  PyCursesInitialised \
+  return PyCursesCheckERR(X(), # X); }
+
+#define NoArgOrFlagNoReturnFunction(X) \
+static PyObject *PyCurses_ ## X (PyObject *self, PyObject *args) \
+{ \
+  int flag = 0; \
+  PyCursesInitialised \
+  switch(PyTuple_Size(args)) { \
+  case 0: \
+    return PyCursesCheckERR(X(), # X); \
+  case 1: \
+    if (!PyArg_ParseTuple(args, "i;True(1) or False(0)", &flag)) return NULL; \
+    if (flag) return PyCursesCheckERR(X(), # X); \
+    else return PyCursesCheckERR(no ## X (), # X); \
+  default: \
+    PyErr_SetString(PyExc_TypeError, # X " requires 0 or 1 arguments"); \
+    return NULL; } }
+
+#define NoArgReturnIntFunction(X) \
+static PyObject *PyCurses_ ## X (PyObject *self) \
+{ \
+ PyCursesInitialised \
+ return PyInt_FromLong((long) X()); }
+
+
+#define NoArgReturnStringFunction(X) \
+static PyObject *PyCurses_ ## X (PyObject *self) \
+{ \
+  PyCursesInitialised \
+  return PyString_FromString(X()); }
+
+#define NoArgTrueFalseFunction(X) \
+static PyObject *PyCurses_ ## X (PyObject *self) \
+{ \
+  PyCursesInitialised \
+  if (X () == FALSE) { \
+    Py_INCREF(Py_False); \
+    return Py_False; \
+  } \
+  Py_INCREF(Py_True); \
+  return Py_True; }
+
+#define NoArgNoReturnVoidFunction(X) \
+static PyObject *PyCurses_ ## X (PyObject *self) \
+{ \
+  PyCursesInitialised \
+  X(); \
+  Py_INCREF(Py_None); \
+  return Py_None; }
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif /* !defined(Py_CURSES_H) */
+
+

from_cpython/Modules/_cursesmodule.c

@@ -1064,7 +1064,7 @@ PyCursesWindow_InCh(PyCursesWindowObject *self, PyObject *args)
     case 2:
         if (!PyArg_ParseTuple(args,"ii;y,x",&y,&x))
             return NULL;
-        rtn = mvwinch(self->win,y,x);
+        rtn = (int)mvwinch(self->win,y,x);
         break;
     default:
         PyErr_SetString(PyExc_TypeError, "inch requires 0 or 2 arguments");
@@ -2739,6 +2739,9 @@ init_curses(void)
     /* Initialize object type */
     Py_TYPE(&PyCursesWindow_Type) = &PyType_Type;
 
+    /* Pyston change */
+    PyType_Ready(&PyCursesWindow_Type);
+
     /* Initialize the C API pointer array */
     PyCurses_API[0] = (void *)&PyCursesWindow_Type;
     PyCurses_API[1] = (void *)func_PyCursesSetupTermCalled;

from_cpython/setup.py

@@ -14,6 +14,26 @@ def multiprocessing_ext():
             "Modules/_multiprocessing/semaphore.c",
             ]))
 
+def bz2_ext():
+    return Extension("bz2", sources = map(relpath, [
+            "Modules/bz2module.c",
+            ]), libraries = ['bz2'])
+
+def grp_ext():
+    return Extension("grp", sources = map(relpath, [
+            "Modules/grpmodule.c",
+            ]))
+
+def curses_ext():
+    return Extension("_curses", sources = map(relpath, [
+            "Modules/_cursesmodule.c",
+            ]), libraries = ['curses'])
+
+def termios_ext():
+    return Extension("termios", sources = map(relpath, [
+            "Modules/termios.c",
+            ]))
+
 def pyexpat_ext():
     define_macros = [('HAVE_EXPAT_CONFIG_H', '1'),]
     expat_sources = map(relpath, ['Modules/expat/xmlparse.c',
@@ -56,5 +76,5 @@ def elementtree_ext():
 setup(name="Pyston",
         version="1.0",
         description="Pyston shared modules",
-        ext_modules=[multiprocessing_ext(), pyexpat_ext(), elementtree_ext()]
+        ext_modules=[multiprocessing_ext(), pyexpat_ext(), elementtree_ext(), bz2_ext(), grp_ext(), curses_ext(), termios_ext()]
     )

section_ordering.txt

@@ -18,7 +18,6 @@
 .text._ZN6pyston9threading22_allowGLReadPreemptionEv
 .text._ZN6pyston9getOpNameEi
 .text._ZN6pyston8callFuncEPNS_17BoxedFunctionBaseEPNS_15CallRewriteArgsENS_11ArgPassSpecEPNS_3BoxES6_S6_PS6_PKSt6vectorIPKSsSaISA_EE
-.text._ZN6pyston2gc9GCVisitor5visitEPv
 .text.intLtInt
 .text.intModInt
 .text.intAddInt
@@ -33,3 +32,22 @@
 .text.createTuple
 .text._ZN6pyston18rearrangeArgumentsENS_16ParamReceiveSpecEPKNS_10ParamNamesEPKcPPNS_3BoxEPNS_15CallRewriteArgsERbNS_11ArgPassSpecES7_S7_S7_S8_PKSt6vectorIPNS_11BoxedStringESaISF_EERS7_SK_SK_S8_
 .text.sre_match
+.text._ZN6pyston2gc10SmallArena14allocationFromEPv
+.text._ZN6pyston2gc10LargeArena14allocationFromEPv
+.text._ZN6pyston2gc9HugeArena14allocationFromEPv
+.text._ZN6pyston2gc9GCVisitor5visitEPv
+.text._ZN6pyston2gc9markPhaseEv
+.text.boxGCHandler
+.text.typeGCHandler
+.text.closureGCHandler
+.text.propertyGCHandler
+.text.listGCHandler
+.text.functionGCHandler
+.text.tupleGCHandler
+.text.listGCHandler
+.text.setGCHandler
+.text.instancemethodGCHandler
+.text.classmethodGCHandler
+.text.staticmethodGCHandler
+.text.sliceGCHandler
+.text._ZN6pyston2gc13runCollectionEv

src/gc/collector.cpp

@@ -228,15 +228,13 @@ bool GCVisitor::isValid(void* p) {
 }
 
 void GCVisitor::visit(void* p) {
-    if (!p)
+    if ((uintptr_t)p < SMALL_ARENA_START || (uintptr_t)p >= HUGE_ARENA_START + ARENA_SIZE) {
+        ASSERT(isNonheapRoot(p), "%p", p);
         return;
-
-    if (isNonheapRoot(p)) {
-        return;
-    } else {
-        ASSERT(global_heap.getAllocationFromInteriorPointer(p)->user_data == p, "%p", p);
-        stack->push(p);
     }
+
+    ASSERT(global_heap.getAllocationFromInteriorPointer(p)->user_data == p, "%p", p);
+    stack->push(p);
 }
 
 void GCVisitor::visitRange(void* const* start, void* const* end) {

src/runtime/file.cpp

@@ -1127,6 +1127,21 @@ Box* fileIterHasNext(Box* s) {
     return boxBool(!fileEof(self));
 }
 
+extern "C" void PyFile_IncUseCount(PyFileObject* _f) noexcept {
+    BoxedFile* f = reinterpret_cast<BoxedFile*>(_f);
+    assert(f->cls == file_cls);
+    f->unlocked_count++;
+}
+
+extern "C" void PyFile_DecUseCount(PyFileObject* _f) noexcept {
+    BoxedFile* f = reinterpret_cast<BoxedFile*>(_f);
+    assert(f->cls == file_cls);
+    f->unlocked_count--;
+    assert(f->unlocked_count >= 0);
+}
+
+
+
 extern "C" void PyFile_SetFP(PyObject* _f, FILE* fp) noexcept {
     assert(_f->cls == file_cls);
     BoxedFile* f = static_cast<BoxedFile*>(_f);
@@ -1769,6 +1784,8 @@ void setupFile() {
                        new BoxedMemberDescriptor(BoxedMemberDescriptor::INT, offsetof(BoxedFile, f_softspace), false));
     file_cls->giveAttr("name",
                        new BoxedMemberDescriptor(BoxedMemberDescriptor::OBJECT, offsetof(BoxedFile, f_name), true));
+    file_cls->giveAttr("mode",
+                       new BoxedMemberDescriptor(BoxedMemberDescriptor::OBJECT, offsetof(BoxedFile, f_mode), true));
 
     file_cls->giveAttr("__new__", new BoxedFunction(boxRTFunction((void*)fileNew, UNKNOWN, 4, 2, false, false),
                                                     { boxString("r"), boxInt(-1) }));

test/tests/bz2_test.py

+import bz2
+print bz2.decompress(bz2.compress("hello world"))

test/tests/curses_test.py

+import curses, sys
+
+try:
+    try:
+        curses.initscr()
+
+        print 1
+        print curses.version
+        print curses.longname()
+        print curses.baudrate()
+        print curses.can_change_color()
+        curses.start_color()
+        print curses.color_pair(curses.A_BLINK)
+    finally:
+        curses.endwin()
+except:
+    print sys.exc_info()[1]

test/tests/grp_test.py

+import grp
+print grp.getgrnam("root")

test/tests/termios_test.py

+import termios, sys
+fd = sys.stdout.fileno()
+try:
+    print termios.tcgetattr(fd)
+except:
+    print sys.exc_info()[1]