Merge remote branch 'upstream/master'

Conflicts:
	Cython/Compiler/Nodes.py

Cython/Compiler/ExprNodes.py

@@ -1437,7 +1437,7 @@ class NameNode(AtomicExprNode):
         if self.is_used_as_rvalue:
             entry = self.entry
             if entry.is_builtin:
-                # if not Options.cache_builtins: # cached builtins are ok
+                if not Options.cache_builtins: # cached builtins are ok
                     self.gil_error()
             elif entry.is_pyglobal:
                 self.gil_error()
@@ -2960,7 +2960,14 @@ class SimpleCallNode(CallNode):
         # Coerce arguments
         for i in range(min(max_nargs, actual_nargs)):
             formal_type = func_type.args[i].type
-            self.args[i] = self.args[i].coerce_to(formal_type, env)
+            arg = self.args[i].coerce_to(formal_type, env)
+            if arg.type.is_pyobject and not env.nogil and (arg.is_attribute or not arg.is_simple):
+                # we do not own the argument's reference, but we must
+                # make sure it cannot be collected before we return
+                # from the function, so we create an owned temp
+                # reference to it
+                arg = arg.coerce_to_temp(env)
+            self.args[i] = arg
         for i in range(max_nargs, actual_nargs):
             if self.args[i].type.is_pyobject:
                 error(self.args[i].pos,

Cython/Compiler/Nodes.py

@@ -1459,7 +1459,7 @@ class FuncDefNode(StatNode, BlockNode):
         # Decref any increfed args
         for entry in lenv.arg_entries:
             if entry.type.is_pyobject:
-                if entry.assignments and not entry.in_closure:
+                if (acquire_gil or entry.assignments) and not entry.in_closure:
                     code.put_var_decref(entry)
         if self.needs_closure and not self.is_generator:
             code.put_decref(Naming.cur_scope_cname, lenv.scope_class.type)
@@ -1527,13 +1527,14 @@ class FuncDefNode(StatNode, BlockNode):
         import Buffer
 
         lenv = self.local_scope
+        acquire_gil = self.acquire_gil
 
         self.generate_argument_parsing_code(env, code)
         # If an argument is assigned to in the body, we must
         # incref it to properly keep track of refcounts.
         for entry in lenv.arg_entries:
             if entry.type.is_pyobject:
-                if entry.assignments and not entry.in_closure:
+                if (acquire_gil or entry.assignments) and not entry.in_closure:
                     code.put_var_incref(entry)
         # ----- Initialise local variables
         for entry in lenv.var_entries:
@@ -3479,6 +3480,12 @@ class ExprStatNode(StatNode):
         self.expr.result_is_used = False # hint that .result() may safely be left empty
         self.expr.analyse_expressions(env)
 
+    def nogil_check(self, env):
+        if self.expr.type.is_pyobject and self.expr.is_temp:
+            self.gil_error()
+
+    gil_message = "Discarding owned Python object"
+
     def generate_execution_code(self, code):
         self.expr.generate_evaluation_code(code)
         if not self.expr.is_temp and self.expr.result():

Cython/Distutils/build_ext.py

@@ -226,6 +226,8 @@ class build_ext(_build_ext.build_ext):
         if not self.inplace and (self.pyrex_c_in_temp
                 or getattr(extension, 'pyrex_c_in_temp', 0)):
             target_dir = os.path.join(self.build_temp, "pyrex")
+            for package_name in extension.name.split('.')[:-1]:
+                target_dir = os.path.join(target_dir, package_name)
         else:
             target_dir = None
 

pyximport/pyximport.py

@@ -112,6 +112,7 @@ def handle_special_build(modname, pyxfilename):
     return ext, setup_args
 
 def handle_dependencies(pyxfilename):
+    testing = '_test_files' in globals()
     dependfile = os.path.splitext(pyxfilename)[0] + PYXDEP_EXT
 
     # by default let distutils decide whether to rebuild on its own
@@ -132,6 +133,7 @@ def handle_dependencies(pyxfilename):
             files.extend(glob.glob(fullpath))
 
         # only for unit testing to see we did the right thing
+        if testing:
             _test_files[:] = []  #$pycheck_no
 
         # if any file that the pyxfile depends upon is newer than
@@ -143,6 +145,7 @@ def handle_dependencies(pyxfilename):
                 print("Rebuilding because of ", file)
                 filetime = os.path.getmtime(file)
                 os.utime(pyxfilename, (filetime, filetime))
+                if testing:
                     _test_files.append(file)
 
 def build_module(name, pyxfilename, pyxbuild_dir=None):

pyximport/test/test_pyximport.py

-import pyximport; pyximport.install()
+import pyximport; pyximport.install(reload_support=True)
 import os, sys
 import time, shutil
 import tempfile
@@ -21,6 +21,7 @@ def on_remove_file_error(func, path, excinfo):
     print "You may want to delete this yourself when you get a chance."
 
 def test():
+    pyximport._test_files = []
     tempdir = make_tempdir()
     sys.path.append(tempdir)
     filename = os.path.join(tempdir, "dummy.pyx")

tests/errors/nogil.pyx

@@ -82,20 +82,26 @@ def ticket_338():
         for obj from 0 <= obj < 4:
             pass
 
+def bare_pyvar_name(object x):
+    with nogil:
+        x
+
 # For m(), the important thing is that there are errors on all lines in the range 23-69
-# except these: 34, 44, 56, 58, 60, 62-64
+# except these: 29, 34, 44, 56, 58, 60, 62-64
 
 _ERRORS = u"""
 1:5: Function with Python return type cannot be declared nogil
 4:5: Function declared nogil has Python locals or temporaries
 6:6: Assignment of Python object not allowed without gil
+9:5: Discarding owned Python object not allowed without gil
 11:5: Function with Python return type cannot be declared nogil
 15:5: Calling gil-requiring function not allowed without gil
 24:9: Calling gil-requiring function not allowed without gil
 26:12: Assignment of Python object not allowed without gil
+28:8: Discarding owned Python object not allowed without gil
 28:16: Constructing complex number not allowed without gil
-29:12: Accessing Python global or builtin not allowed without gil
 30:8: Backquote expression not allowed without gil
+30:8: Discarding owned Python object not allowed without gil
 30:9: Operation not allowed without gil
 31:15: Assignment of Python object not allowed without gil
 31:15: Operation not allowed without gil
@@ -105,20 +111,30 @@ _ERRORS = u"""
 32:25: Constructing Python list not allowed without gil
 32:25: Operation not allowed without gil
 33:17: Iterating over Python object not allowed without gil
+35:11: Discarding owned Python object not allowed without gil
 35:11: Indexing Python object not allowed without gil
+36:11: Discarding owned Python object not allowed without gil
 36:11: Slicing Python object not allowed without gil
 37:11: Constructing Python slice object not allowed without gil
+37:11: Discarding owned Python object not allowed without gil
 37:11: Indexing Python object not allowed without gil
 37:13: Converting to Python object not allowed without gil
 37:15: Converting to Python object not allowed without gil
 37:17: Converting to Python object not allowed without gil
 38:11: Accessing Python attribute not allowed without gil
-39: 9: Constructing Python tuple not allowed without gil
-40: 8: Constructing Python list not allowed without gil
-41: 8: Constructing Python dict not allowed without gil
+38:11: Discarding owned Python object not allowed without gil
+39:9: Constructing Python tuple not allowed without gil
+39:9: Discarding owned Python object not allowed without gil
+40:8: Constructing Python list not allowed without gil
+40:8: Discarding owned Python object not allowed without gil
+41:8: Constructing Python dict not allowed without gil
+41:8: Discarding owned Python object not allowed without gil
+42:12: Discarding owned Python object not allowed without gil
 42:12: Truth-testing Python object not allowed without gil
 43:13: Python type test not allowed without gil
+45:10: Discarding owned Python object not allowed without gil
 45:10: Operation not allowed without gil
+46:8: Discarding owned Python object not allowed without gil
 46:8: Operation not allowed without gil
 47:10: Assignment of Python object not allowed without gil
 47:14: Assignment of Python object not allowed without gil

tests/run/owned_arg_refs.pyx

+
+cdef class Owner:
+    cdef object x
+
+cdef call_me_with_owner(Owner owner, x):
+    owner.x = "def" # overwrite external reference
+    return x        # crashes if x is not owned by function or caller
+
+def test_ext_type_attr():
+    """
+    >>> test_ext_type_attr()
+    'abc5'
+    """
+    owner = Owner()
+    owner.x = ''.join("abc%d" % 5) # non-interned object
+    return call_me_with_owner(owner, owner.x)
+
+
+cdef void call_me_without_gil(Owner owner, x) with gil:
+    owner.x = "def" # overwrite external reference
+    print x         # crashes if x is not owned by function or caller
+
+def test_ext_type_attr_nogil():
+    """
+    >>> test_ext_type_attr_nogil()
+    abc5
+    """
+    owner = Owner()
+    owner.x = ''.join("abc%d" % 5) # non-interned object
+    with nogil:
+        call_me_without_gil(owner, owner.x)
+
+
+# the following isn't dangerous as long as index access uses temps
+
+cdef call_me_with_list(list l, x):
+    l[:] = [(1,2), (3,4)] # overwrite external reference
+    return x              # crashes if x is not owned by function or caller
+
+def test_index():
+    """
+    >>> test_index()
+    [3, 4]
+    """
+    l = [[1,2],[3,4]]
+    return call_me_with_list(l, l[1])