Exceptions part #1: misc support changes

docs/INSTALLING.md

@@ -122,7 +122,7 @@ cd ~/pyston_deps
 cp -rv libunwind-1.1 libunwind-1.1-debug
 mkdir libunwind-1.1-debug-install
 cd libunwind-1.1-debug
-./configure --prefix=$HOME/pyston_deps/libunwind-1.1-debug-install --enable-shared=0 --enable-debug --enable-debug-frame
+CFLAGS="-g -O0" CXXFLAGS="-g -O0" ./configure --prefix=$HOME/pyston_deps/libunwind-1.1-debug-install --enable-shared=0 --enable-debug --enable-debug-frame
 make -j4
 make install
 echo "USE_DEBUG_LIBUNWIND := 1" >> ~/pyston/src/Makefile.local
@@ -151,7 +151,7 @@ make -j4
 ```
 
 ### gdb
-A new version of gdb is highly recommended since debugging a JIT tends to stress GDB:
+A new version of gdb is highly recommended since debugging a JIT tends to use new features of GDB:
 
 ```
 cd ~/pyston_deps
@@ -161,9 +161,14 @@ cd gdb-7.6.2
 ./configure
 make -j4
 cd ~/pyston/src
-echo "GDB := \$(HOME)/pyston_deps/gdb-7.6.2/gdb/gdb" >> Makefile.local
+echo "GDB := \$(DEPS_DIR)/gdb-7.6.2/gdb/gdb --data-directory \$(DEPS_DIR)/gdb-7.6.2/gdb/data-directory" >> Makefile.local
 ```
 
+<!---
+TODO: GDB should be able to determine its data directory.  maybe it should be installed rather that run
+from inside the source dir?
+--->
+
 ### gperftools (-lprofiler)
 ```
 download from http://code.google.com/p/gperftools/downloads/list

microbenchmarks/unwinding.py

+def wrap(f, n):
+    if n:
+        wrap(f, n-1)
+    return f()
+
+def throws():
+    raise AttributeError
+
+for i in xrange(10000):
+    try:
+        wrap(throws, 500)
+    except AttributeError:
+        pass

src/Makefile

@@ -538,6 +538,7 @@ PASS_OBJS := $(PASS_SRCS:.cpp=.standalone.o)
 %.release.h.pch: CXXFLAGS := $(CXXFLAGS_RELEASE)
 %.release.h.pch %.h.pch: %.h $(BUILD_SYSTEM_DEPS)
 	$(ECHO) Compiling $@
+	$(VERB) rm -f $@-*
 	$(VERB) $(CLANG_EXE) $(CXXFLAGS) -MMD -MP -MF $<.d -x c++-header $< -o $@
 
 CODEGEN_SRCS := $(wildcard codegen/*.cpp) $(wildcard codegen/*/*.cpp)
@@ -606,10 +607,10 @@ stdlib.release.unopt.bc: $(STDLIB_SRCS:.cpp=.release.o.pub.bc)
 stdlib.bc: OPT_OPTIONS=-O3
 stdlib.release.bc: OPT_OPTIONS=-O3 -strip-debug
 %.bc: %.unopt.bc
-	$(ECHO) Optimizing $@
+	$(ECHO) Optimizing $< -> $@
 	$(VERB) $(LLVM_BIN)/opt $(OPT_OPTIONS) $< -o $@
 %.stripped.bc: %.bc
-	$(ECHO) Optimizing $@
+	$(ECHO) Stripping $< -> $@
 	$(VERB) $(LLVM_BIN)/opt -strip-debug $< -o $@
 
 # Then do "ld -b binary" to create a .o file for embedding into the executable
@@ -787,7 +788,7 @@ watch_%:
 		TARGET=$(dir $@)$(patsubst watch_%,%,$(notdir $@)); \
 		clear; $(MAKE) $$TARGET; true; \
 		while inotifywait -q -e modify -e attrib -e move -e move_self -e create -e delete -e delete_self \
-		Makefile $$(find \( -name '*.cpp' -o -name '*.h' -o -name '*.py' \) ); do clear; $(MAKE) $$TARGET; done )
+		Makefile $$(find .. \( -name '*.cpp' -o -name '*.h' -o -name '*.py' \) ); do clear; $(MAKE) $$TARGET; done )
 		# Makefile $$(find \( -name '*.cpp' -o -name '*.h' -o -name '*.py' \) -o -type d ); do clear; $(MAKE) $(patsubst watch_%,%,$@); done )
 		# -r . ; do clear; $(MAKE) $(patsubst watch_%,%,$@); done
 wdbg_%:
@@ -795,14 +796,18 @@ wdbg_%:
 
 .PHONY: test_asm test_cpp_asm
 test_asm:
-	$(CLANG_EXE) tests/test.s -c -o test
+	$(CLANG_EXE) ../test/test.s -c -o test
 	objdump -d test | less
 	@ rm test
 test_cpp_asm:
-	$(CLANG_EXE) tests/test.cpp -o test -c -O3
+	$(CLANG_EXE) ../test/test.cpp -o test -c -O3
 	# $(GPP) tests/test.cpp -o test -c -O3
 	objdump -d test | less
 	rm test
+test_cpp_ll:
+	$(CLANG_EXE) ../test/test.cpp -o test.ll -c -O3 -emit-llvm -S
+	less test.ll
+	rm test.ll
 
 .PHONY: ext
 ext: ../test/test_extension/test.so

src/codegen/dis.cpp

@@ -173,7 +173,7 @@ void PystonJITEventListener::NotifyObjectEmitted(const llvm::ObjectImage& Obj) {
 
     for (llvm::object::symbol_iterator I = Obj.begin_symbols(), E = Obj.end_symbols(); I != E;) {
         llvm::StringRef name;
-        uint64_t addr, size, offset=0;
+        uint64_t addr, size, offset = 0;
         code = I->getName(name);
         assert(!code);
         code = I->getAddress(addr);

test/test.cpp

@@ -10,6 +10,14 @@ void set64full(int64_t* ptr) {
     *ptr = 0x1234567890;
 }
 
-void set32(int64_t* ptr) {
-    *(int32_t*)ptr = 0x1234;
+namespace pyston {
+    class Box {};
+
+int throw_catch(Box* b) {
+    try {
+        throw b;
+    } catch (int e) {
+        return e;
+    }
+}
 }

test/tests/exceptions.py

 # expected: fail
 # - exceptions
 
+class TestException(Exception):
+    pass
+
+class ExpectationFailedException(Exception):
+    pass
+
+class ExpectedException(object):
+    def __init__(self, excs):
+        if isinstance(excs, BaseException):
+            excs = (excs,)
+        self.excs = excs
+
+    def __enter__(self):
+        pass
+
+    def __exit__(self, type, val, tback):
+        if not val:
+            raise ExpectationFailedException("Didn't raise any exception")
+        if not isinstance(val, self.excs):
+            raise ExpectationFailedException("Raised %s instead of %s" % (val, self.excs))
+        print "Caught", type.__name__
+        return True
+expected_exception = ExpectedException
+
+# Test the expected_exception manager:
+with expected_exception(Exception):
+    raise Exception()
+
+try:
+    with expected_exception(Exception):
+        pass
+    raise Exception("shouldn't get here")
+except ExpectationFailedException:
+    print "good"
+
+# The inner one will fail, which the outer one should catch:
+with expected_exception(ExpectationFailedException):
+    with expected_exception(Exception):
+        pass
+
+
 def throw(x):
     try:
         raise x
@@ -59,3 +100,223 @@ def f2(throw):
 f2(True)
 f2(False)
 
+def f3():
+    # Finally blocks are actually somewhat complicated, because
+    # you need to catch not just exceptions, but also continue/break/return's
+    print "f3"
+
+    for i in xrange(5):
+        try:
+            if i == 3:
+                break
+            continue
+        finally:
+            print "finally", i
+
+    try:
+        # Looks like this returns from the function, but it needs to go to the finally block
+        return
+    finally:
+        print "in finally"
+f3()
+
+def f4():
+    # Make sure that simply accessing a name can throw exceptions as expected
+
+    print "f4"
+
+    with expected_exception(NameError):
+        print doesnt_exist
+
+    global doesnt_exist2
+    with expected_exception(NameError):
+        print doesnt_exist2
+f4()
+
+def f5():
+    # Make sure that we don't accidentally set 'x' here;
+    # the exception prevents the assignment from happening
+    print "f5"
+
+    try:
+        try:
+            x = doesnt_exist
+        except NameError:
+            print "inner except"
+            print x
+    except NameError:
+        print "outer except"
+
+    # Similar test, except now the reference to 'y' *does* come from
+    # the line that will end up throwing, but from a previous iteration
+    # that didn't throw.
+    def inner(i):
+        if i == 1:
+            raise AttributeError
+    for i in xrange(5):
+        try:
+            y = inner(i)
+        except AttributeError:
+            print y
+f5()
+
+def f6():
+    print "f6"
+
+    a = 0
+
+    with expected_exception(AttributeError):
+        a.x = 0
+
+    with expected_exception(TypeError):
+        a[0] = 0
+
+    # Tricky!
+    # Note: a multiple-assignment statement like this is processed by setting the targets one by one, left-to-right.
+    # So the assignment to "x" should succeed, but then the assignment to x.a will fail.
+    # In the exception handler we should be able to see a value for x, but accessing y should fail.
+    try:
+        x = x.a = y = 1
+        raise Exception("shouldn't get here")
+    except AttributeError:
+        print "caught, as expected"
+        print "x = ", x
+        try:
+            print "y = ", y
+            raise Exception("shouldn't get here")
+        except UnboundLocalError:
+            print "caught, as expected"
+f6()
+
+def f7():
+    print "f7"
+
+    # Make sure that desugaring produces exception handling as expected:
+
+    class NonIterable(object):
+        pass
+
+    with expected_exception(TypeError):
+        for i in NonIterable():
+            pass
+
+    class BadIterable(object):
+        def __iter__(self):
+            return self
+
+        def next(self):
+            raise NotImplementedError()
+
+    with expected_exception(NotImplementedError):
+        for i in BadIterable():
+            print i
+
+    class ExceptionRaiser(object):
+        def __nonzero__(self):
+            raise TestException()
+
+        def __repr__(self):
+            raise TestException()
+
+    with expected_exception(TestException):
+        while ExceptionRaiser():
+            pass
+
+    with expected_exception(TestException):
+        print ExceptionRaiser()
+
+    with expected_exception(TestException):
+        assert ExceptionRaiser()
+
+    with expected_exception(AssertionError):
+        assert 0
+
+    def throw():
+        raise TestException()
+
+    with expected_exception(TestException):
+        def f(x=throw()):
+            pass
+
+    with expected_exception(TestException):
+        class C(object):
+            throw()
+
+    with expected_exception(ImportError):
+        import hopefully_this_package_doesnt_exist
+        hopefully_this_package_doesnt_exist # to silence pyflakes
+
+    with expected_exception(ImportError):
+        from hopefully_this_package_doesnt_exist import a
+        a # to silence pyflakes
+
+    with expected_exception(TestException):
+        print 1 if throw() else 0
+
+    with expected_exception(TestException):
+        print ExceptionRaiser() and 1
+
+    with expected_exception(TestException):
+        if throw():
+            pass
+
+    with expected_exception(TestException):
+        if ExceptionRaiser():
+            pass
+
+    try:
+        # Imports also need to be broken into separate parts:
+        from sys import path, doesnt_exist
+    except ImportError:
+        print type(path)
+        with expected_exception(NameError):
+            print doesnt_exist
+f7()
+
+def f8():
+    print "f8"
+
+    def f(exc):
+        print "evaluating except type;", exc.__name__
+        return exc
+
+    try:
+        raise AttributeError()
+    except f(TypeError):
+        print "shouldn't be here 1"
+    except f(AttributeError):
+        print "should hit this"
+    except f(NotImplementedError):
+        print "shouldn't be here"
+f8()
+
+def f9():
+    print "f9"
+
+    # arithmetic
+
+    with expected_exception(ZeroDivisionError):
+        1/0
+    with expected_exception(ZeroDivisionError):
+        1.0/0
+    with expected_exception(ZeroDivisionError):
+        1/0.0
+    with expected_exception(ZeroDivisionError):
+        1 % 0
+    with expected_exception(ZeroDivisionError):
+        1 % 0.0
+    with expected_exception(ZeroDivisionError):
+        1.0 % 0
+
+    with expected_exception(AttributeError):
+        (1).a
+f9()
+
+def f10():
+    print "f10"
+    try:
+        raise ZeroDivisionError()
+    except:
+        with expected_exception(ZeroDivisionError):
+            raise
+f10()

test/tests/exceptions_basic.py

+# expected: fail
+# - exceptions
+
+def f(x):
+    print x
+    try:
+        if x == 2:
+            raise AttributeError()
+        assert x
+    except AssertionError:
+        print "is assert"
+    except:
+        print "not an assert"
+    else:
+        print "no exception"
+
+f(0)
+f(1)
+f(2)
+
+# You can set attributes on exception objects:
+e = Exception()
+e.n = 1
+print e.n
+
+
+try:
+    1/0
+except ZeroDivisionError, e:
+    print e.message
+    print str(e), repr(e)
+    print e

test/tests/exceptions_nested.py

@@ -158,3 +158,125 @@ def f5():
     except AttributeError:
         print sys.exc_info()[0].__name__
 f5()
+
+def f6():
+    print
+    print "f6"
+
+    # A finally block must somehow track how it was entered, because it's not based
+    # on the value of sys.exc_info at the end of the finally block:
+
+    def inner(nested_throw, reraise):
+        try:
+            pass
+        finally:
+            if nested_throw:
+                try:
+                    raise AttributeError()
+                except:
+                    pass
+
+            print sys.exc_info()
+            if reraise:
+                raise
+
+    inner(False, False) # no exception raised
+    inner(True, False) # no exception raised
+
+    try:
+        inner(True, True)
+        # Shouldn't get here
+        raise Exception()
+    except AttributeError:
+        print "the thrown AttributeError raised as expected"
+    # Have to call this, because the inner throw can reraise the out-of-except
+    # exception from this scope!
+    sys.exc_clear()
+
+    try:
+        inner(False, True)
+        # Shouldn't get here
+        raise Exception()
+    except TypeError:
+        print "Got TypeError as expected, since exc_info was None"
+f6()
+
+def f7():
+    print
+    print "f7"
+
+    # Similar test to f6, but this time with an exception propagating
+    # up through a finally block.
+    # An exception thrown inside that finally shouldn't change the exception
+    # that will end up getting propagated
+
+    def inner():
+        try:
+            raise AttributeError()
+        finally:
+            try:
+                raise NotImplementedError()
+            except:
+                pass
+            print sys.exc_info()[0].__name__
+    try:
+        inner()
+    except:
+        print sys.exc_info()[0].__name__
+f7()
+
+def f8():
+    print
+    print "f8"
+
+    try:
+        raise AttributeError()
+    except:
+        pass
+
+    def reraise():
+        raise
+
+    try:
+        reraise()
+        raise Exception()
+    except AttributeError:
+        print "reraised correctly"
+f8()
+
+def f9():
+    print
+    print "f9"
+
+    # Exceptions thrown inside a catch block should still go through the finally,
+    # but not other catch blocks.
+
+    try:
+        try:
+            raise Exception()
+        except Exception:
+            print "here"
+            raise AttributeError()
+        except AttributeError:
+            print "shouldn't get here"
+        finally:
+            print "in finally"
+    except AttributeError:
+        pass
+f9()
+
+def f10():
+    print "f10"
+
+    x = 1
+    try:
+        try:
+            y = 2
+            raise AttributeError()
+            x = 3
+        except NotImplementedError:
+            print "shouldn't be here"
+    except AttributeError:
+        print x, y
+        print "here"
+f10()

test/tests/exceptions_nonreprable.py

+# expected: fail
+# - inheritance
+
+class BadException(Exception):
+    def __str__(self):
+        print "str"
+        raise NotImplementedError()
+
+try:
+    # This will raise:
+    print BadException()
+    assert 0
+except NotImplementedError:
+    pass
+
+raise BadException()

test/tests/many_attrs_setattr.py

@@ -10,6 +10,8 @@ n = 0
 while n < 100000:
     setattr(c, "a" + str(n), n)
     n = n + 1
+    if n % 1000 == 0:
+        print n
 
 def f(o):
     print o.a1

tools/publicize.cpp

@@ -79,11 +79,20 @@ bool makeVisible(llvm::GlobalValue* gv) {
         changed = true;
     }
 
-    //llvm::GlobalValue::VisibilityTypes visibility = gv->getVisibility();
-    //if (visibility == llvm::GlobalValue::HiddenVisibility) {
-        //gv->setVisibility(llvm::GlobalValue::ProtectedVisibility);
-        //changed = true;
-    //}
+    // Hidden symbols won't end up as globals.
+    // Worse, a hidden symbol, when linked with a default-visibility symbol,
+    // will result in a non-visible symbol.
+    // So it's not enough to just set the visibility here; instead we have to
+    // set it to protected *and* change the name.
+    // The only thing affected by this that I know about is __clang_call_terminate.
+    llvm::GlobalValue::VisibilityTypes visibility = gv->getVisibility();
+    if (visibility == llvm::GlobalValue::HiddenVisibility) {
+        gv->setVisibility(llvm::GlobalValue::ProtectedVisibility);
+        //gv->setDLLStorageClass(llvm::GlobalValue::DLLExportStorageClass);
+        gv->setName(gv->getName() + "_protected");
+
+        changed = true;
+    }
 
     return changed;
 }

tools/tester.py

@@ -37,6 +37,12 @@ FN_JUST_SIZE = 20
 EXTRA_JIT_ARGS = []
 TIME_LIMIT = 2
 
+# For fun, can test pypy.
+# Tough because the tester will check to see if the error messages are exactly the
+# same as the system CPython, but the error messages change over micro CPython versions.
+# Pyston compile-time checks the system CPython version to try to give compatible error messages.
+TEST_PYPY = 0
+
 def set_ulimits():
     # Guard the process from running too long with a hard rlimit.
     # But first try to kill it after a second with a SIGALRM, though that's catchable/clearable by the program:
@@ -77,7 +83,7 @@ def run_test(fn, check_stats, run_memcheck):
     r = fn.rjust(FN_JUST_SIZE)
 
     statchecks = []
-    jit_args = ["-csr"] + EXTRA_JIT_ARGS
+    jit_args = ["-csrq"] + EXTRA_JIT_ARGS
     expected = "success"
     for l in open(fn):
         if not l.startswith("#"):
@@ -93,7 +99,12 @@ def run_test(fn, check_stats, run_memcheck):
 
     assert expected in ("success", "fail", "statfail"), expected
 
-    run_args = ["./%s" % IMAGE] + jit_args + ["-q", fn]
+    if TEST_PYPY:
+        jit_args = []
+        check_stats = False
+        expected = "success"
+
+    run_args = [os.path.abspath(IMAGE)] + jit_args + [fn]
     start = time.time()
     p = subprocess.Popen(run_args, stdout=subprocess.PIPE, stderr=subprocess.PIPE, stdin=open("/dev/null"), preexec_fn=set_ulimits)
     out, err = p.communicate()
@@ -104,7 +115,7 @@ def run_test(fn, check_stats, run_memcheck):
     elapsed = time.time() - start
 
     stats = {}
-    if code == 0:
+    if code == 0 and not TEST_PYPY:
         assert out.count("Stats:") == 1
         out, stats_str = out.split("Stats:")
         for l in stats_str.strip().split('\n'):
@@ -153,7 +164,7 @@ def run_test(fn, check_stats, run_memcheck):
             diff = p.stdout.read()
             assert p.wait() in (0, 1)
             raise Exception("Failed on %s:\n%s" % (fn, diff))
-    elif err != expected_err:
+    elif not TEST_PYPY and err != expected_err:
         if KEEP_GOING:
             r += "    \033[31mFAILED\033[0m (bad stderr)"
             failed.append(fn)
@@ -357,10 +368,14 @@ if __name__ == "__main__":
 
     FN_JUST_SIZE = max(20, 2 + max(map(len, tests)))
 
-    print "Building...",
-    sys.stdout.flush()
-    subprocess.check_call(["make", "-j4", IMAGE], stdout=open("/dev/null", 'w'), stderr=subprocess.PIPE)
-    print "done"
+    if not TEST_PYPY:
+        print "Building...",
+        sys.stdout.flush()
+        subprocess.check_call(["make", "-j4", IMAGE], stdout=open("/dev/null", 'w'), stderr=subprocess.PIPE)
+        print "done"
+
+    if TEST_PYPY:
+        IMAGE = '/usr/local/bin/pypy'
 
     if not patterns:
         tests.sort(key=fileSize)