Merge pull request #531 from kmod/multiprocessing

Multiprocessing

CMakeLists.txt

@@ -220,7 +220,7 @@ add_custom_target(check-format ${CMAKE_SOURCE_DIR}/tools/check_format.sh ${LLVM_
 add_custom_target(lint ${PYTHON_EXE} ${CMAKE_SOURCE_DIR}/tools/lint.py WORKING_DIRECTORY ${CMAKE_SOURCE_DIR}/src)
 
 # check
-add_custom_target(check-pyston COMMAND ${CMAKE_CTEST_COMMAND} --output-on-failure DEPENDS pyston copy_stdlib copy_libpyston clang-format ext_cpython ext_pyston unittests) 
+add_custom_target(check-pyston COMMAND ${CMAKE_CTEST_COMMAND} --output-on-failure DEPENDS pyston copy_stdlib copy_libpyston clang-format ext_cpython ext_pyston unittests sharedmods)
 
 # {run,dbg,perf,memcheck,memleaks,cachegrind}_TESTNAME
 file(GLOB RUNTARGETS ${CMAKE_SOURCE_DIR}/test/tests/*.py ${CMAKE_SOURCE_DIR}/microbenchmarks/*.py ${CMAKE_SOURCE_DIR}/minibenchmarks/*.py)

Makefile

@@ -420,16 +420,23 @@ UNITTEST_SRCS := $(wildcard $(UNITTEST_DIR)/*.cpp)
 
 NONSTDLIB_SRCS := $(MAIN_SRCS) $(OPTIONAL_SRCS) $(TOOL_SRCS) $(UNITTEST_SRCS)
 
-.DEFAULT_GOAL := pyston_dbg
-# _ :
-	# $(MAKE) pyston_dbg || (clear; $(MAKE) pyston_dbg -j1 ERROR_LIMIT=1)
+.DEFAULT_GOAL := small_all
+
+RUN_DEPS := ext_pyston
+
+ifneq ($(USE_CMAKE),1)
+	RUN_DEPS := $(RUN_DEPS) sharedmods
+endif
+
+.PHONY: small_all
+small_all: pyston_dbg $(RUN_DEPS)
 
 .PHONY: all _all
 # all: llvm
 	# @# have to do this in a recursive make so that dependency is enforced:
 	# $(MAKE) pyston_all
 # all: pyston_dbg pyston_release pyston_oprof pyston_prof $(OPTIONAL_SRCS:.cpp=.o) ext_python ext_pyston
-all: pyston_dbg pyston_release pyston_prof ext_python ext_pyston unittests
+all: pyston_dbg pyston_release pyston_gcc ext_python ext_pyston unittests sharedmods
 
 ALL_HEADERS := $(wildcard src/*/*.h) $(wildcard src/*/*/*.h) $(wildcard from_cpython/Include/*.h)
 tags: $(SRCS) $(OPTIONAL_SRCS) $(FROM_CPYTHON_SRCS) $(ALL_HEADERS)
@@ -455,10 +462,10 @@ $1_unittest:
 	ln -sf $(HOME)/pyston-build-dbg/$1_unittest .
 endif
 dbg_$1_unittests: $1_unittest
-	zsh -c 'ulimit -v $(MAX_MEM_KB); ulimit -d $(MAX_MEM_KB); time $(GDB) $(GDB_CMDS) --args ./$1_unittest --gtest_break_on_failure $(ARGS)'
+	zsh -c 'ulimit -m $(MAX_MEM_KB); time $(GDB) $(GDB_CMDS) --args ./$1_unittest --gtest_break_on_failure $(ARGS)'
 unittests:: $1_unittest
 run_$1_unittests: $1_unittest
-	zsh -c 'ulimit -v $(MAX_MEM_KB); ulimit -d $(MAX_MEM_KB); time ./$1_unittest $(ARGS)'
+	zsh -c 'ulimit -m $(MAX_MEM_KB); time ./$1_unittest $(ARGS)'
 run_unittests:: run_$1_unittests
 )
 endef
@@ -913,10 +920,10 @@ $(CMAKE_SETUP_RELEASE):
 
 .PHONY: pyston_dbg pyston_release
 pyston_dbg: $(CMAKE_SETUP_DBG)
-	$(NINJA) -C $(HOME)/pyston-build-dbg pyston copy_stdlib copy_libpyston ext_pyston $(NINJAFLAGS)
+	$(NINJA) -C $(HOME)/pyston-build-dbg pyston copy_stdlib copy_libpyston sharedmods ext_pyston $(NINJAFLAGS)
 	ln -sf $(HOME)/pyston-build-dbg/pyston pyston_dbg
 pyston_release: $(CMAKE_SETUP_RELEASE)
-	$(NINJA) -C $(HOME)/pyston-build-release pyston copy_stdlib copy_libpyston ext_pyston $(NINJAFLAGS)
+	$(NINJA) -C $(HOME)/pyston-build-release pyston copy_stdlib copy_libpyston sharedmods ext_pyston $(NINJAFLAGS)
 	ln -sf $(HOME)/pyston-build-release/pyston pyston_release
 endif
 CMAKE_DIR_GCC := $(HOME)/pyston-build-gcc
@@ -927,14 +934,14 @@ $(CMAKE_SETUP_GCC):
 	cd $(CMAKE_DIR_GCC); CC='$(GCC)' CXX='$(GPP)' cmake -GNinja $(HOME)/pyston -DCMAKE_BUILD_TYPE=Debug
 .PHONY: pyston_gcc
 pyston_gcc: $(CMAKE_SETUP_GCC)
-	$(NINJA) -C $(HOME)/pyston-build-gcc pyston copy_stdlib copy_libpyston ext_pyston $(NINJAFLAGS)
+	$(NINJA) -C $(HOME)/pyston-build-gcc pyston copy_stdlib copy_libpyston sharedmods ext_pyston $(NINJAFLAGS)
 	ln -sf $(HOME)/pyston-build-gcc/pyston pyston_gcc
 
 -include $(wildcard src/*.d) $(wildcard src/*/*.d) $(wildcard src/*/*/*.d) $(wildcard $(UNITTEST_DIR)/*.d) $(wildcard from_cpython/*/*.d) $(wildcard from_cpython/*/*/*.d)
 
 .PHONY: clean
 clean:
-	@ find src $(TOOLS_DIR) $(TEST_DIR) ./from_cpython/Modules \( -name '*.o' -o -name '*.d' -o -name '*.py_cache' -o -name '*.bc' -o -name '*.o.ll' -o -name '*.pub.ll' -o -name '*.cache' -o -name 'stdlib*.ll' -o -name '*.pyc' -o -name '*.so' -o -name '*.a' -o -name '*.expected_cache' -o -name '*.pch' \) -print -delete
+	@ find src $(TOOLS_DIR) $(TEST_DIR) ./from_cpython ./lib_pyston \( -name '*.o' -o -name '*.d' -o -name '*.py_cache' -o -name '*.bc' -o -name '*.o.ll' -o -name '*.pub.ll' -o -name '*.cache' -o -name 'stdlib*.ll' -o -name '*.pyc' -o -name '*.so' -o -name '*.a' -o -name '*.expected_cache' -o -name '*.pch' \) -print -delete
 	@ find \( -name 'pyston*' -executable -type f \) -print -delete
 	@ find $(TOOLS_DIR) -maxdepth 0 -executable -type f -print -delete
 	@ rm -rf oprofile_data
@@ -963,8 +970,6 @@ $(patsubst %, $$1: %/nosearch_$$1 ;,$(EXTRA_SEARCH_DIRS))
 )
 endef
 
-RUN_DEPS :=
-
 define make_target
 $(eval \
 .PHONY: test$1 check$1
@@ -980,12 +985,12 @@ check$1 test$1: $(PYTHON_EXE_DEPS) pyston$1 ext_pyston
 run$1: pyston$1 $$(RUN_DEPS)
 	PYTHONPATH=test/test_extension ./pyston$1 $$(ARGS)
 dbg$1: pyston$1 $$(RUN_DEPS)
-	PYTHONPATH=test/test_extension zsh -c 'ulimit -v $$(MAX_DBG_MEM_KB); $$(GDB) $$(GDB_CMDS) --args ./pyston$1 $$(ARGS)'
+	PYTHONPATH=test/test_extension zsh -c 'ulimit -m $$(MAX_DBG_MEM_KB); $$(GDB) $$(GDB_CMDS) --args ./pyston$1 $$(ARGS)'
 nosearch_run$1_%: %.py pyston$1 $$(RUN_DEPS)
-	$(VERB) PYTHONPATH=test/test_extension zsh -c 'ulimit -v $$(MAX_MEM_KB); ulimit -d $$(MAX_MEM_KB); time ./pyston$1 $$(ARGS) $$<'
+	$(VERB) PYTHONPATH=test/test_extension zsh -c 'ulimit -m $$(MAX_MEM_KB); time ./pyston$1 $$(ARGS) $$<'
 $$(call make_search,run$1_%)
 nosearch_dbg$1_%: %.py pyston$1 $$(RUN_DEPS)
-	$(VERB) PYTHONPATH=test/test_extension zsh -c 'ulimit -v $$(MAX_DBG_MEM_KB); $$(GDB) $$(GDB_CMDS) --args ./pyston$1 $$(ARGS) $$<'
+	$(VERB) PYTHONPATH=test/test_extension zsh -c 'ulimit -m $$(MAX_DBG_MEM_KB); $$(GDB) $$(GDB_CMDS) --args ./pyston$1 $$(ARGS) $$<'
 $$(call make_search,dbg$1_%)
 
 ifneq ($$(ENABLE_VALGRIND),0)
@@ -1110,7 +1115,7 @@ opreportcg:
 
 .PHONY: watch_% watch wdbg_%
 watch_%:
-	@ ( ulimit -t 60; ulimit -d $(MAK_MEM_KB); ulimit -v $(MAK_MEM_KB); \
+	@ ( ulimit -t 60; ulimit -m $(MAK_MEM_KB); \
 		TARGET=$(dir $@)$(patsubst watch_%,%,$(notdir $@)); \
 		clear; $(MAKE) $$TARGET $(WATCH_ARGS); true; \
 		while inotifywait -q -e modify -e attrib -e move -e move_self -e create -e delete -e delete_self \
@@ -1145,51 +1150,43 @@ test_cpp_ll:
 .PHONY: bench_exceptions
 bench_exceptions:
 	$(CLANGPP_EXE) $(TEST_DIR)/bench_exceptions.cpp -o bench_exceptions -O3 -std=c++11
-	zsh -c 'ulimit -v $(MAX_MEM_KB); ulimit -d $(MAX_MEM_KB); time ./bench_exceptions'
+	zsh -c 'ulimit -m $(MAX_MEM_KB); time ./bench_exceptions'
 	rm bench_exceptions
 
 TEST_EXT_MODULE_NAMES := basic_test descr_test slots_test
+TEST_EXT_MODULE_SRCS := $(TEST_EXT_MODULE_NAMES:%=test/test_extension/%.c)
+TEST_EXT_MODULE_OBJS := $(TEST_EXT_MODULE_NAMES:%=test/test_extension/%.pyston.so)
 
-# SELF_HOST_EXTENSIONS = SELF_HOST or USE_CMAKE
-# - cmake doesn't support non-self-hosting extensions
-SELF_HOST_EXTENSIONS := $(SELF_HOST)
-ifeq ($(USE_CMAKE),1)
-	SELF_HOST_EXTENSIONS := 1
-endif
+SHAREDMODS_NAMES := _multiprocessing
+SHAREDMODS_SRCS := \
+	_multiprocessing/multiprocessing.c \
+	_multiprocessing/semaphore.c \
+	_multiprocessing/socket_connection.c
+SHAREDMODS_SRCS := $(SHAREDMODS_SRCS:%=from_cpython/Modules/%)
+SHAREDMODS_OBJS := $(SHAREDMODS_NAMES:%=lib_pyston/%.pyston.so)
+
+.PHONY: sharedmods
+sharedmods: $(SHAREDMODS_OBJS)
 
 .PHONY: ext_pyston
-ext_pyston: $(TEST_EXT_MODULE_NAMES:%=$(TEST_DIR)/test_extension/%.pyston.so)
-ifneq ($(SELF_HOST_EXTENSIONS),1)
-$(TEST_DIR)/test_extension/%.pyston.so: $(TEST_DIR)/test_extension/%.o
-	$(CC) -pthread -shared -Wl,-O1 -Wl,-Bsymbolic-functions -Wl,-z,relro $< -o $@ -g
-$(TEST_DIR)/test_extension/%.o: $(TEST_DIR)/test_extension/%.c $(wildcard from_cpython/Include/*.h)
-	$(CC) -pthread $(EXT_CFLAGS) -c $< -o $@
-else
+ext_pyston: $(TEST_EXT_MODULE_OBJS)
+
 # Hax: we want to generate multiple targets from a single rule, and run the rule only if the
 # dependencies have been updated, and only run it once for all the targets.
 # So just tell make to generate the first extension module, and that the non-first ones just
 # depend on the first one.
-$(TEST_DIR)/test_extension/$(firstword $(TEST_EXT_MODULE_NAMES)).pyston.so: $(TEST_EXT_MODULE_NAMES:%=$(TEST_DIR)/test_extension/%.c) | pyston_dbg
-	$(MAKE) ext_pyston_selfhost
-NONFIRST_EXT := $(wordlist 2,9999,$(TEST_EXT_MODULE_NAMES))
-$(NONFIRST_EXT:%=$(TEST_DIR)/test_extension/%.pyston.so): $(TEST_DIR)/test_extension/$(firstword $(TEST_EXT_MODULE_NAMES)).pyston.so
-endif
-
-.PHONY: ext_pyston_selfhost dbg_ext_pyston_selfhost ext_pyston_selfhost_release
-ext_pyston_selfhost: pyston_dbg $(TEST_EXT_MODULE_NAMES:%=$(TEST_DIR)/test_extension/*.c)
-	cd $(TEST_DIR)/test_extension; DISTUTILS_DEBUG=1 time ../../pyston_dbg setup.py build
-	cd $(TEST_DIR)/test_extension; ln -sf $(TEST_EXT_MODULE_NAMES:%=build/lib.linux2-2.7/%.pyston.so) .
-dbg_ext_pyston_selfhost: pyston_dbg $(TEST_EXT_MODULE_NAMES:%=$(TEST_DIR)/test_extension/*.c)
-	cd $(TEST_DIR)/test_extension; DISTUTILS_DEBUG=1 $(GDB) $(GDB_CMDS) --args ../../pyston_dbg setup.py build
-	cd $(TEST_DIR)/test_extension; ln -sf $(TEST_EXT_MODULE_NAMES:%=build/lib.linux2-2.7/%.pyston.so) .
-ext_pyston_selfhost_release: pyston_release $(TEST_EXT_MODULE_NAMES:%=$(TEST_DIR)/test_extension/*.c)
-	cd $(TEST_DIR)/test_extension; DISTUTILS_DEBUG=1 time ../../pyston_release setup.py build
-	cd $(TEST_DIR)/test_extension; ln -sf $(TEST_EXT_MODULE_NAMES:%=build/lib.linux2-2.7/%.pyston.so) .
+$(firstword $(TEST_EXT_MODULE_OBJS)): $(TEST_EXT_MODULE_SRCS) pyston_dbg
+	$(VERB) cd $(TEST_DIR)/test_extension; time ../../pyston_dbg setup.py build
+	$(VERB) cd $(TEST_DIR)/test_extension; ln -sf $(TEST_EXT_MODULE_NAMES:%=build/lib.linux2-2.7/%.pyston.so) .
+$(wordlist 2,9999,$(TEST_EXT_MODULE_OBJS)): $(firstword $(TEST_EXT_MODULE_OBJS))
+$(firstword $(SHAREDMODS_OBJS)): $(SHAREDMODS_SRCS) pyston_dbg
+	$(VERB) cd $(TEST_DIR)/test_extension; time ../../pyston_dbg ../../from_cpython/setup.py build --build-lib ../../lib_pyston
+$(wordlist 2,9999,$(SHAREDMODS_OBJS)): $(firstword $(SHAREDMODS_OBJS))
 
 .PHONY: ext_python ext_pythondbg
-ext_python: $(TEST_EXT_MODULE_NAMES:%=$(TEST_DIR)/test_extension/*.c)
+ext_python: $(TEST_EXT_MODULE_SRCS)
 	cd $(TEST_DIR)/test_extension; python setup.py build
-ext_pythondbg: $(TEST_EXT_MODULE_NAMES:%=$(TEST_DIR)/test_extension/*.c)
+ext_pythondbg: $(TEST_EXT_MODULE_SRCS)
 	cd $(TEST_DIR)/test_extension; python2.7-dbg setup.py build
 
 $(FROM_CPYTHON_SRCS:.c=.o): %.o: %.c $(BUILD_SYSTEM_DEPS)

from_cpython/CMakeLists.txt

@@ -108,3 +108,15 @@ file(GLOB_RECURSE STDPARSER_SRCS Parser
 
 set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -Wno-missing-field-initializers -Wno-tautological-compare -Wno-type-limits -Wno-unused-result -Wno-strict-aliasing")
 add_library(FROM_CPYTHON OBJECT ${STDMODULE_SRCS} ${STDOBJECT_SRCS} ${STDPYTHON_SRCS} ${STDPARSER_SRCS})
+
+add_custom_command(OUTPUT ${CMAKE_BINARY_DIR}/lib_pyston/_multiprocessing.pyston.so
+                   COMMAND ${CMAKE_BINARY_DIR}/pyston setup.py build --build-lib ${CMAKE_BINARY_DIR}/lib_pyston
+                   DEPENDS
+                       pyston
+                       copy_stdlib
+                       copy_libpyston
+                       Modules/_multiprocessing/multiprocessing.c
+                       Modules/_multiprocessing/semaphore.c
+                       Modules/_multiprocessing/socket_connection.c
+                   WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR})
+add_custom_target(sharedmods DEPENDS ${CMAKE_BINARY_DIR}/lib_pyston/_multiprocessing.pyston.so)

from_cpython/Include/pyconfig.h

@@ -62,6 +62,10 @@
 #define HAVE_MKTIME 1
 #define HAVE_PROTOTYPES 1
 #define STDC_HEADERS 1
+#define HAVE_SEM_GETVALUE 1
+#define HAVE_SEM_OPEN 1
+#define HAVE_SEM_TIMEDWAIT 1
+#define HAVE_SEM_UNLINK 1
 
 #define TIME_WITH_SYS_TIME
 #define HAVE_GETTIMEOFDAY 1

from_cpython/Modules/_multiprocessing/connection.h

@@ -358,7 +358,7 @@ connection_poll(ConnectionObject *self, PyObject *args)
     }
 
     Py_BEGIN_ALLOW_THREADS
-    res = conn_poll(self, timeout, _save);
+    res = conn_poll(self, timeout /* Pyston change, don't need this argument: , _save */);
     Py_END_ALLOW_THREADS
 
     switch (res) {

from_cpython/Modules/_multiprocessing/socket_connection.c

@@ -185,7 +185,7 @@ conn_recv_string(ConnectionObject *conn, char *buffer,
  */
 
 static int
-conn_poll(ConnectionObject *conn, double timeout, PyThreadState *_save)
+conn_poll(ConnectionObject *conn, double timeout /* Pyston change: don't need this argument: , PyThreadState *_save */)
 {
 #if defined(HAVE_POLL) && !defined(HAVE_BROKEN_POLL)
     int res;

from_cpython/setup.py

+# CPython has a 2kloc version of this file
+
+from distutils.core import setup, Extension
+import os
+
+def relpath(fn):
+    r =  os.path.join(os.path.dirname(__file__), fn)
+    return r
+
+setup(name="Pyston",
+        version="1.0",
+        description="Pyston shared modules",
+        ext_modules=[Extension("_multiprocessing", sources = map(relpath, [
+                "Modules/_multiprocessing/multiprocessing.c",
+                "Modules/_multiprocessing/socket_connection.c",
+                "Modules/_multiprocessing/semaphore.c",
+            ]),
+        )],
+    )

lib_pyston/CMakeLists.txt

 # Copy any changed lib_pyston sources:
-file(GLOB_RECURSE LIBPYSTON_SRCS . "*")
+file(GLOB_RECURSE LIBPYSTON_SRCS . "*.py")
 set(LIBPYSTON_TARGETS "")
 foreach(STDLIB_FILE ${LIBPYSTON_SRCS})
     file(RELATIVE_PATH FN_REL ${CMAKE_SOURCE_DIR} ${STDLIB_FILE})

src/codegen/ast_interpreter.cpp

@@ -206,6 +206,7 @@ void ASTInterpreter::setFrameInfo(const FrameInfo* frame_info) {
 }
 
 void ASTInterpreter::setGlobals(Box* globals) {
+    assert(gc::isValidGCObject(globals));
     this->globals = globals;
 }
 

src/gc/collector.cpp

@@ -323,7 +323,7 @@ void markPhase() {
 #endif
 }
 
-static void sweepPhase(std::list<Box*, StlCompatAllocator<Box*>>& weakly_referenced) {
+static void sweepPhase(std::vector<Box*>& weakly_referenced) {
     // we need to use the allocator here because these objects are referenced only here, and calling the weakref
     // callbacks could start another gc
     global_heap.freeUnmarked(weakly_referenced);
@@ -341,6 +341,7 @@ void disableGC() {
 }
 
 static int ncollections = 0;
+static bool should_not_reenter_gc = false;
 void runCollection() {
     static StatCounter sc("gc_collections");
     sc.log();
@@ -352,25 +353,62 @@ void runCollection() {
     if (VERBOSITY("gc") >= 2)
         printf("Collection #%d\n", ncollections);
 
+    // The bulk of the GC work is not reentrant-safe.
+    // In theory we should never try to reenter that section, but it's happened due to bugs,
+    // which show up as very-hard-to-understand gc issues.
+    // So keep track if we're in the non-reentrant section and abort if we try to go back in.
+    // We could also just skip the collection if we're currently in the gc, but I think if we
+    // run into this case it's way more likely that it's a bug than something we should ignore.
+    RELEASE_ASSERT(!should_not_reenter_gc, "");
+    should_not_reenter_gc = true; // begin non-reentrant section
+
     Timer _t("collecting", /*min_usec=*/10000);
 
     markPhase();
-    std::list<Box*, StlCompatAllocator<Box*>> weakly_referenced;
+
+    // The sweep phase will not free weakly-referenced objects, so that we can inspect their
+    // weakrefs_list.  We want to defer looking at those lists until the end of the sweep phase,
+    // since the deallocation of other objects (namely, the weakref objects themselves) can affect
+    // those lists, and we want to see the final versions.
+    std::vector<Box*> weakly_referenced;
     sweepPhase(weakly_referenced);
 
+    // Handle weakrefs in two passes:
+    // - first, find all of the weakref objects whose callbacks we need to call.  we need to iterate
+    //   over the garbage-and-corrupt-but-still-alive weakly_referenced list in order to find these objects,
+    //   so the gc is not reentrant during this section.  after this we discard that list.
+    // - then, call all the weakref callbacks we collected from the first pass.
+
+    // Use a StlCompatAllocator to keep the pending weakref objects alive in case we trigger a new collection.
+    // In theory we could push so much onto this list that we would cause a new collection to start:
+    std::list<PyWeakReference*, StlCompatAllocator<PyWeakReference*>> weak_references;
+
     for (auto o : weakly_referenced) {
+        assert(isValidGCObject(o));
         PyWeakReference** list = (PyWeakReference**)PyObject_GET_WEAKREFS_LISTPTR(o);
         while (PyWeakReference* head = *list) {
             assert(isValidGCObject(head));
             if (head->wr_object != Py_None) {
+                assert(head->wr_object == o);
                 _PyWeakref_ClearRef(head);
-                if (head->wr_callback) {
 
-                    runtimeCall(head->wr_callback, ArgPassSpec(1), reinterpret_cast<Box*>(head), NULL, NULL, NULL,
-                                NULL);
-                    head->wr_callback = NULL;
+                if (head->wr_callback)
+                    weak_references.push_back(head);
+            }
         }
+        global_heap.free(GCAllocation::fromUserData(o));
     }
+
+    should_not_reenter_gc = false; // end non-reentrant section
+
+    while (!weak_references.empty()) {
+        PyWeakReference* head = weak_references.front();
+        weak_references.pop_front();
+
+        if (head->wr_callback) {
+
+            runtimeCall(head->wr_callback, ArgPassSpec(1), reinterpret_cast<Box*>(head), NULL, NULL, NULL, NULL);
+            head->wr_callback = NULL;
         }
     }
 

src/gc/heap.cpp

@@ -43,7 +43,7 @@ template <> void return_temporary_buffer<pyston::Box*>(pyston::Box** p) {
 namespace pyston {
 namespace gc {
 
-bool _doFree(GCAllocation* al, std::list<Box*, StlCompatAllocator<Box*>>* weakly_referenced);
+bool _doFree(GCAllocation* al, std::vector<Box*>* weakly_referenced);
 
 // lots of linked lists around here, so let's just use template functions for operations on them.
 template <class ListT> inline void nullNextPrev(ListT* node) {
@@ -86,7 +86,7 @@ template <class ListT, typename Func> inline void forEach(ListT* list, Func func
 }
 
 template <class ListT, typename Free>
-inline void sweepList(ListT* head, std::list<Box*, StlCompatAllocator<Box*>>& weakly_referenced, Free free_func) {
+inline void sweepList(ListT* head, std::vector<Box*>& weakly_referenced, Free free_func) {
     auto cur = head;
     while (cur) {
         GCAllocation* al = cur->data;
@@ -137,7 +137,7 @@ void registerGCManagedBytes(size_t bytes) {
 
 Heap global_heap;
 
-bool _doFree(GCAllocation* al, std::list<Box*, StlCompatAllocator<Box*>>* weakly_referenced) {
+bool _doFree(GCAllocation* al, std::vector<Box*>* weakly_referenced) {
     if (VERBOSITY() >= 4)
         printf("Freeing %p\n", al->user_data);
 
@@ -371,7 +371,7 @@ GCAllocation* SmallArena::allocationFrom(void* ptr) {
     return reinterpret_cast<GCAllocation*>(&b->atoms[atom_idx]);
 }
 
-void SmallArena::freeUnmarked(std::list<Box*, StlCompatAllocator<Box*>>& weakly_referenced) {
+void SmallArena::freeUnmarked(std::vector<Box*>& weakly_referenced) {
     thread_caches.forEachValue([this, &weakly_referenced](ThreadBlockCache* cache) {
         for (int bidx = 0; bidx < NUM_BUCKETS; bidx++) {
             Block* h = cache->cache_free_heads[bidx];
@@ -431,7 +431,7 @@ void SmallArena::getStatistics(HeapStatistics* stats) {
 }
 
 
-SmallArena::Block** SmallArena::_freeChain(Block** head, std::list<Box*, StlCompatAllocator<Box*>>& weakly_referenced) {
+SmallArena::Block** SmallArena::_freeChain(Block** head, std::vector<Box*>& weakly_referenced) {
     while (Block* b = *head) {
         int num_objects = b->numObjects();
         int first_obj = b->minObjIndex();
@@ -449,8 +449,10 @@ SmallArena::Block** SmallArena::_freeChain(Block** head, std::list<Box*, StlComp
             if (isMarked(al)) {
                 clearMark(al);
             } else {
-                if (_doFree(al, &weakly_referenced))
+                if (_doFree(al, &weakly_referenced)) {
                     b->isfree.set(atom_idx);
+                    // memset(al->user_data, 0, b->size - sizeof(GCAllocation));
+                }
             }
         }
 
@@ -656,7 +658,7 @@ GCAllocation* LargeArena::allocationFrom(void* ptr) {
     return NULL;
 }
 
-void LargeArena::freeUnmarked(std::list<Box*, StlCompatAllocator<Box*>>& weakly_referenced) {
+void LargeArena::freeUnmarked(std::vector<Box*>& weakly_referenced) {
     sweepList(head, weakly_referenced, [this](LargeObj* ptr) { _freeLargeObj(ptr); });
 }
 
@@ -848,7 +850,7 @@ GCAllocation* HugeArena::allocationFrom(void* ptr) {
     return NULL;
 }
 
-void HugeArena::freeUnmarked(std::list<Box*, StlCompatAllocator<Box*>>& weakly_referenced) {
+void HugeArena::freeUnmarked(std::vector<Box*>& weakly_referenced) {
     sweepList(head, weakly_referenced, [this](HugeObj* ptr) { _freeHugeObj(ptr); });
 }
 

src/gc/heap.h

@@ -196,7 +196,7 @@ public:
     void free(GCAllocation* al);
 
     GCAllocation* allocationFrom(void* ptr);
-    void freeUnmarked(std::list<Box*, StlCompatAllocator<Box*>>& weakly_referenced);
+    void freeUnmarked(std::vector<Box*>& weakly_referenced);
 
     void getStatistics(HeapStatistics* stats);
 
@@ -328,7 +328,7 @@ private:
     Block* _allocBlock(uint64_t size, Block** prev);
     GCAllocation* _allocFromBlock(Block* b);
     Block* _claimBlock(size_t rounded_size, Block** free_head);
-    Block** _freeChain(Block** head, std::list<Box*, StlCompatAllocator<Box*>>& weakly_referenced);
+    Block** _freeChain(Block** head, std::vector<Box*>& weakly_referenced);
     void _getChainStatistics(HeapStatistics* stats, Block** head);
 
     GCAllocation* __attribute__((__malloc__)) _alloc(size_t bytes, int bucket_idx);
@@ -401,7 +401,7 @@ public:
     void free(GCAllocation* alloc);
 
     GCAllocation* allocationFrom(void* ptr);
-    void freeUnmarked(std::list<Box*, StlCompatAllocator<Box*>>& weakly_referenced);
+    void freeUnmarked(std::vector<Box*>& weakly_referenced);
 
     void getStatistics(HeapStatistics* stats);
 };
@@ -419,7 +419,7 @@ public:
     void free(GCAllocation* alloc);
 
     GCAllocation* allocationFrom(void* ptr);
-    void freeUnmarked(std::list<Box*, StlCompatAllocator<Box*>>& weakly_referenced);
+    void freeUnmarked(std::vector<Box*>& weakly_referenced);
 
     void getStatistics(HeapStatistics* stats);
 
@@ -530,7 +530,7 @@ public:
     }
 
     // not thread safe:
-    void freeUnmarked(std::list<Box*, StlCompatAllocator<Box*>>& weakly_referenced) {
+    void freeUnmarked(std::vector<Box*>& weakly_referenced) {
         small_arena.freeUnmarked(weakly_referenced);
         large_arena.freeUnmarked(weakly_referenced);
         huge_arena.freeUnmarked(weakly_referenced);

src/jit.cpp

@@ -350,7 +350,7 @@ static int main(int argc, char** argv) {
         // encodings module.
         setEncodingAndErrors();
 
-        // end of argument parsing
+        Stats::endOfInit();
 
         _t.split("to run");
         BoxedModule* main_module = NULL;

src/runtime/inline/dict.cpp

@@ -60,6 +60,9 @@ Box* dictIterNext(Box* s) {
     assert(s->cls == dict_iterator_cls);
     BoxedDictIterator* self = static_cast<BoxedDictIterator*>(s);
 
+    if (self->it == self->itEnd)
+        raiseExcHelper(StopIteration, "");
+
     Box* rtn = nullptr;
     if (self->type == BoxedDictIterator::KeyIterator) {
         rtn = self->it->first;

src/runtime/objmodel.cpp

@@ -2439,7 +2439,16 @@ extern "C" void dumpEx(void* p, int levels) {
         }
 
         if (isSubclass(b->cls, list_cls)) {
-            printf("%ld elements\n", static_cast<BoxedList*>(b)->size);
+            auto l = static_cast<BoxedList*>(b);
+            printf("%ld elements\n", l->size);
+
+            if (levels > 0) {
+                int i = 0;
+                for (int i = 0; i < l->size; i++) {
+                    printf("\nElement %d:", i);
+                    dumpEx(l->elts->elts[i], levels - 1);
+                }
+            }
         }
 
         if (isSubclass(b->cls, module_cls)) {
@@ -4732,6 +4741,7 @@ extern "C" void delGlobal(Box* globals, const std::string* name) {
 
 extern "C" Box* getGlobal(Box* globals, const std::string* name) {
     STAT_TIMER(t0, "us_timer_slowpath_getglobal");
+    ASSERT(gc::isValidGCObject(globals), "%p", globals);
 
     static StatCounter slowpath_getglobal("slowpath_getglobal");
     slowpath_getglobal.log();

src/runtime/types.cpp

@@ -1498,6 +1498,21 @@ public:
         return rtn;
     }
 
+    static Box* iterkeys(Box* _self) {
+        Box* r = AttrWrapper::keys(_self);
+        return getiter(r);
+    }
+
+    static Box* itervalues(Box* _self) {
+        Box* r = AttrWrapper::values(_self);
+        return getiter(r);
+    }
+
+    static Box* iteritems(Box* _self) {
+        Box* r = AttrWrapper::items(_self);
+        return getiter(r);
+    }
+
     static Box* copy(Box* _self) {
         RELEASE_ASSERT(_self->cls == attrwrapper_cls, "");
         AttrWrapper* self = static_cast<AttrWrapper*>(_self);
@@ -2496,10 +2511,10 @@ void setupRuntime() {
     attrwrapper_cls->giveAttr("keys", new BoxedFunction(boxRTFunction((void*)AttrWrapper::keys, LIST, 1)));
     attrwrapper_cls->giveAttr("values", new BoxedFunction(boxRTFunction((void*)AttrWrapper::values, LIST, 1)));
     attrwrapper_cls->giveAttr("items", new BoxedFunction(boxRTFunction((void*)AttrWrapper::items, LIST, 1)));
-    // TODO: not quite right
-    attrwrapper_cls->giveAttr("iterkeys", attrwrapper_cls->getattr("keys"));
-    attrwrapper_cls->giveAttr("itervalues", attrwrapper_cls->getattr("values"));
-    attrwrapper_cls->giveAttr("iteritems", attrwrapper_cls->getattr("items"));
+    attrwrapper_cls->giveAttr("iterkeys", new BoxedFunction(boxRTFunction((void*)AttrWrapper::iterkeys, UNKNOWN, 1)));
+    attrwrapper_cls->giveAttr("itervalues",
+                              new BoxedFunction(boxRTFunction((void*)AttrWrapper::itervalues, UNKNOWN, 1)));
+    attrwrapper_cls->giveAttr("iteritems", new BoxedFunction(boxRTFunction((void*)AttrWrapper::iteritems, UNKNOWN, 1)));
     attrwrapper_cls->giveAttr("copy", new BoxedFunction(boxRTFunction((void*)AttrWrapper::copy, UNKNOWN, 1)));
     attrwrapper_cls->giveAttr("__len__", new BoxedFunction(boxRTFunction((void*)AttrWrapper::len, BOXED_INT, 1)));
     attrwrapper_cls->giveAttr("__iter__", new BoxedFunction(boxRTFunction((void*)AttrWrapper::iter, UNKNOWN, 1)));
@@ -2589,8 +2604,6 @@ void setupRuntime() {
 
     setupSysEnd();
 
-    Stats::endOfInit();
-
     TRACK_ALLOCATIONS = true;
 }
 

test/integration/virtualenv_test.py

@@ -21,10 +21,11 @@ set -e
 set -ux
 python -c 'import __future__'
 python -c 'import sys; print sys.executable'
-pip install bcrypt==1.1.0 python-gflags==2.0 sqlalchemy==1.0.0
+pip install bcrypt==1.1.0 python-gflags==2.0 sqlalchemy==1.0.0 Pillow==2.8.1
 python -c 'import bcrypt; assert bcrypt.__version__ == "1.1.0"; assert bcrypt.hashpw("password1", "$2a$12$0123456789012345678901").endswith("I1hdtg4K"); print "bcrypt seems to work"'
 python -c 'import gflags; print "gflags imports"'
 python -c 'import sqlalchemy; print "sqlalchemy imports"'
+python -c 'from PIL import Image; print "Pillow imports"'
 """.strip()
 
 # print sh_script

test/test_extension/CMakeLists.txt

-set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -Wno-missing-field-initializers")
-set(CMAKE_SHARED_LIBRARY_PREFIX "")
-set(CMAKE_SHARED_LIBRARY_SUFFIX ".pyston${CMAKE_SHARED_LIBRARY_SUFFIX}")
-
-add_custom_command(OUTPUT ${CMAKE_CURRENT_SOURCE_DIR}/build/lib.linux-x86_64-2.7/basic_test.so
-                   COMMAND python setup.py build
+add_custom_command(OUTPUT ${CMAKE_CURRENT_BINARY_DIR}/build/lib.linux-x86_64-2.7/basic_test.so
+                   COMMAND python setup.py build --build-lib ${CMAKE_CURRENT_BINARY_DIR}/build/lib.linux-x86_64-2.7
                    DEPENDS basic_test.c descr_test.c slots_test.c
                    WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR})
 
-add_library(basic_test SHARED basic_test.c)
-add_library(descr_test SHARED descr_test.c)
-add_library(slots_test SHARED slots_test.c)
+add_custom_command(OUTPUT ${CMAKE_CURRENT_BINARY_DIR}/basic_test.pyston.so
+                   COMMAND ${CMAKE_BINARY_DIR}/pyston setup.py build --build-lib ${CMAKE_CURRENT_BINARY_DIR}
+                   DEPENDS pyston copy_stdlib copy_libpyston basic_test.c descr_test.c slots_test.c
+                   WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR})
 
-add_custom_target(ext_cpython DEPENDS ${CMAKE_CURRENT_SOURCE_DIR}/build/lib.linux-x86_64-2.7/basic_test.so)
-add_custom_target(ext_pyston DEPENDS basic_test descr_test slots_test)
+add_custom_target(ext_cpython DEPENDS ${CMAKE_CURRENT_BINARY_DIR}/build/lib.linux-x86_64-2.7/basic_test.so)
+add_custom_target(ext_pyston DEPENDS ${CMAKE_CURRENT_BINARY_DIR}/basic_test.pyston.so)

test/tests/30.py

 # run_args: -n
 # statcheck: ("-O" in EXTRA_JIT_ARGS) or (1 <= stats["num_osr_exits"] <= 2)
-# statcheck: stats['slowpath_binop'] <= 10
+# statcheck: noninit_count('slowpath_binop') <= 10
 
 x = 100000
 y = 0

test/tests/47.py

 # run_args: -n
-# statcheck: 1 <= stats['slowpath_binop'] <= 5
+# statcheck: 1 <= noninit_count('slowpath_binop') <= 5
 def i():
     return 0
 

test/tests/call_patches.py

 # run_args: -n
-# statcheck: stats.get('slowpath_callattr', 0) <= 20
+# statcheck: noninit_count('slowpath_callattr') <= 20
 # statcheck: noninit_count('slowpath_runtimecall') <= 20
-# statcheck: stats.get("slowpath_callclfunc", 0) <= 20
+# statcheck: noninit_count("slowpath_callclfunc") <= 20
 # statcheck: noninit_count('rewriter_nopatch') <= 20
 
 def outer():

test/tests/class_clsgetattr_ics.py

 # run_args: -n
-# statcheck: stats['slowpath_getattr'] <= 10
+# statcheck: noninit_count('slowpath_getattr') <= 10
 
 class C(object):
     pass

test/tests/descriptors_getattr_ics.py

 # run_args: -n
-# statcheck: stats['slowpath_getattr'] <= 80
+# statcheck: noninit_count('slowpath_getattr') <= 80
 
 class DataDescriptor(object):
     def __get__(self, obj, typ):

test/tests/descriptors_getset_ics.py

 # run_args: -n
-# statcheck: stats['slowpath_getattr'] <= 10
-# statcheck: stats['slowpath_setattr'] <= 10
+# statcheck: noninit_count('slowpath_getattr') <= 10
+# statcheck: noninit_count('slowpath_setattr') <= 10
 
 class C(object):
     pass

test/tests/descriptors_setattr_ics.py

 # run_args: -n
-# statcheck: stats['slowpath_setattr'] <= 120
+# statcheck: noninit_count('slowpath_setattr') <= 120
 
 class Descriptor(object):
     def __set__(self, obj, value):

test/tests/dict.py

@@ -219,3 +219,14 @@ for i in xrange(6):
     d1[i] = 5 - i
     d2[5 - i] = i
     print d1 == d2, d1 != d2
+
+
+d = dict([(i, i**2) for i in xrange(10)])
+i = d.iteritems()
+l = []
+while True:
+    try:
+        l.append(i.next())
+    except StopIteration:
+        break
+print sorted(l)

test/tests/dir.py

@@ -114,3 +114,12 @@ print sorted([d for d in dir(TestClass3()) if not d.startswith('_')])
 c = C1()
 c.__dict__.update([('a', 1), ('b', 2)])
 print c.a, c.b
+
+i = c.__dict__.iteritems()
+l = []
+while True:
+    try:
+        l.append(i.next())
+    except StopIteration:
+        break
+print sorted(l)

test/tests/getattr_ics.py

 # run_args: -n
-# statcheck: stats['slowpath_getattr'] <= 20
+# statcheck: noninit_count('slowpath_getattr') <= 20
 
 class C(object):
     def f(self):

test/tests/global_ics.py

 # run_args: -n
-# statcheck: stats["slowpath_getglobal"] <= 10
+# statcheck: noninit_count("slowpath_getglobal") <= 10
 
 def f():
     print True

test/tests/icgetattr_invalid.py

 # run_args: -n
-# statcheck: stats['slowpath_getattr'] <= 10
+# statcheck: noninit_count('slowpath_getattr') <= 10
 # Different ways that getattrs can be invalidated
 
 class C(object):

test/tests/multiprocessing_ctypes_test.py

+# expected: fail
+# - relies on ctypes
+
+import multiprocessing
+
+# from https://docs.python.org/2/library/multiprocessing.html
+
+def f(n, a):
+    n.value = 3.1415927
+    for i in range(len(a)):
+        a[i] = -a[i]
+
+if __name__ == '__main__':
+    num = multiprocessing.Value('d', 0.0)
+    arr = multiprocessing.Array('i', range(10))
+
+    p = multiprocessing.Process(target=f, args=(num, arr))
+    p.start()
+    p.join()
+
+    print num.value
+    print arr[:]

test/tests/multiprocessing_test.py

+import multiprocessing
+
+# from https://docs.python.org/2/library/multiprocessing.html
+
+def f(x):
+    return x*x
+
+if __name__ == '__main__':
+    p = multiprocessing.Pool(5)
+    print(p.map(f, [1, 2, 3]))
+
+
+def f(name):
+    print 'hello', name
+
+if __name__ == '__main__':
+    p = multiprocessing.Process(target=f, args=('bob',))
+    p.start()
+    p.join()
+
+
+def f(q):
+    q.put([42, None, 'hello'])
+
+if __name__ == '__main__':
+    q = multiprocessing.Queue()
+    p = multiprocessing.Process(target=f, args=(q,))
+    p.start()
+    print q.get()    # prints "[42, None, 'hello']"
+    p.join()

test/tests/nonzero_patching.py

 # run_args: -n
-# statcheck: stats.get('slowpath_nonzero', 0) <= 10
+# statcheck: noninit_count('slowpath_nonzero') <= 10
 
 def f():
     for i in xrange(-10, 10):

test/tests/polymorphic_ics.py

 # run_args: -n
-# statcheck: stats['slowpath_binop'] < 10
+# statcheck: noninit_count('slowpath_binop') < 10
 
 class O(object):
     def __init__(self, n):

test/tests/xrange.py

 # run_args: -n
 #
 # statcheck: "-O" in EXTRA_JIT_ARGS or 'slowpath_getclsattr' in stats or 'slowpath_callattr' in stats
-# statcheck: stats.get('slowpath_getclsattr', 0) <= 20
-# statcheck: stats.get('slowpath_callattr', 0) <= 22
+# statcheck: noninit_count('slowpath_getclsattr') <= 20
+# statcheck: noninit_count('slowpath_callattr') <= 22
 
 for i in xrange(1000):
     print i

tools/tester.py

@@ -59,7 +59,7 @@ def set_ulimits():
     MAX_MEM_MB = 100
     resource.setrlimit(resource.RLIMIT_RSS, (MAX_MEM_MB * 1024 * 1024, MAX_MEM_MB * 1024 * 1024))
 
-EXTMODULE_DIR = os.path.abspath(os.path.dirname(os.path.realpath(__file__)) + "/../test/test_extension/build/lib.linux-x86_64-2.7/")
+EXTMODULE_DIR = None
 EXTMODULE_DIR_PYSTON = None
 THIS_FILE = os.path.abspath(__file__)
 
@@ -72,6 +72,7 @@ def get_global_mtime():
     # Start off by depending on the tester itself
     rtn = os.stat(THIS_FILE).st_mtime
 
+    assert os.listdir(EXTMODULE_DIR), EXTMODULE_DIR
     for fn in os.listdir(EXTMODULE_DIR):
         if not fn.endswith(".so"):
             continue
@@ -462,6 +463,7 @@ def main(orig_dir):
     global SKIP_FAILING_TESTS
     global VERBOSE
     global EXTMODULE_DIR_PYSTON
+    global EXTMODULE_DIR
 
     run_memcheck = False
 
@@ -479,6 +481,7 @@ def main(orig_dir):
 
     TEST_DIR = os.path.join(orig_dir, opts.test_dir)
     EXTMODULE_DIR_PYSTON = os.path.abspath(os.path.dirname(os.path.realpath(IMAGE)) + "/test/test_extension/")
+    EXTMODULE_DIR = os.path.abspath(os.path.dirname(os.path.realpath(IMAGE)) + "/test/test_extension/build/lib.linux-x86_64-2.7/")
     patterns = opts.pattern
 
     if not patterns and not TESTS_TO_SKIP: