Capture and redirect stdout/stderr for %%cython-magic to show C compiler warnings/errors (GH-3872)

Cython/Build/IpythonMagic.py

@@ -82,6 +82,7 @@ from ..Shadow import __version__ as cython_version
 from ..Compiler.Errors import CompileError
 from .Inline import cython_inline
 from .Dependencies import cythonize
+from ..Utils import captured_fd
 
 
 PGO_CONFIG = {
@@ -106,6 +107,37 @@ else:
         return name
 
 
+def get_encoding_candidates():
+    candidates = [sys.getdefaultencoding()]
+    for stream in (sys.stdout, sys.stdin, sys.__stdout__, sys.__stdin__):
+        encoding = getattr(stream, 'encoding', None)
+        # encoding might be None (e.g. somebody redirects stdout):
+        if encoding is not None and encoding not in candidates:
+            candidates.append(encoding)
+    return candidates
+
+
+def prepare_captured(captured):
+    captured_bytes = captured.strip()
+    if not captured_bytes:
+        return None
+    for encoding in get_encoding_candidates():
+        try:
+            return captured_bytes.decode(encoding)
+        except UnicodeDecodeError:
+            pass
+    # last resort: print at least the readable ascii parts correctly.
+    return captured_bytes.decode('latin-1')
+
+
+def print_captured(captured, output, header_line=None):
+    captured = prepare_captured(captured)
+    if captured:
+        if header_line:
+            output.write(header_line)
+        output.write(captured)
+
+
 @magics_class
 class CythonMagics(Magics):
 
@@ -342,13 +374,25 @@ class CythonMagics(Magics):
             if args.pgo:
                 self._profile_pgo_wrapper(extension, lib_dir)
 
+        def print_compiler_output(stdout, stderr, where):
+            # On windows, errors are printed to stdout, we redirect both to sys.stderr.
+            print_captured(stdout, where, u"Content of stdout:\n")
+            print_captured(stderr, where, u"Content of stderr:\n")
+
+        get_stderr = get_stdout = None
         try:
-            self._build_extension(extension, lib_dir, pgo_step_name='use' if args.pgo else None,
-                                  quiet=args.quiet)
-        except distutils.errors.CompileError:
-            # Build failed and printed error message
+            with captured_fd(1) as get_stdout:
+                with captured_fd(2) as get_stderr:
+                    self._build_extension(
+                        extension, lib_dir, pgo_step_name='use' if args.pgo else None, quiet=args.quiet)
+        except (distutils.errors.CompileError, distutils.errors.LinkError):
+            # Build failed, print error message from compiler/linker
+            print_compiler_output(get_stdout(), get_stderr(), sys.stderr)
             return None
 
+        # Build seems ok, but we might still want to show any warnings that occurred
+        print_compiler_output(get_stdout(), get_stderr(), sys.stdout)
+
         module = imp.load_dynamic(module_name, module_path)
         self._import_all(module)
 

Cython/Build/Tests/TestIpythonMagic.py

@@ -6,6 +6,7 @@
 from __future__ import absolute_import
 
 import os
+import io
 import sys
 from contextlib import contextmanager
 from Cython.Build import IpythonMagic
@@ -29,6 +30,26 @@ try:
 except ImportError:
     pass
 
+
+@contextmanager
+def capture_output():
+    backup = sys.stdout, sys.stderr
+    try:
+        replacement = [
+             io.TextIOWrapper(io.BytesIO(), encoding=sys.stdout.encoding),
+             io.TextIOWrapper(io.BytesIO(), encoding=sys.stderr.encoding),
+        ]
+        sys.stdout, sys.stderr = replacement
+        output = []
+        yield output
+    finally:
+        sys.stdout, sys.stderr = backup
+        for wrapper in replacement:
+            wrapper.seek(0)  # rewind
+            output.append(wrapper.read())
+            wrapper.close()
+
+
 code = u"""\
 def f(x):
     return 2*x
@@ -48,6 +69,27 @@ def main():
 main()
 """
 
+compile_error_code = u'''\
+cdef extern from *:
+    """
+    xxx a=1;
+    """
+    int a;
+def doit():
+    return a
+'''
+
+compile_warning_code = u'''\
+cdef extern from *:
+    """
+    #pragma message ( "CWarning" )
+    int a = 42;
+    """
+    int a;
+def doit():
+    return a
+'''
+
 
 if sys.platform == 'win32':
     # not using IPython's decorators here because they depend on "nose"
@@ -143,6 +185,39 @@ class TestIPythonMagic(CythonTest):
         self.assertEqual(ip.user_ns['g'], 2 // 10)
         self.assertEqual(ip.user_ns['h'], 2 // 10)
 
+    def test_cython_compile_error_shown(self):
+        ip = self._ip
+        with capture_output() as out:
+            ip.run_cell_magic('cython', '-3', compile_error_code)
+        captured_out, captured_err = out
+
+        # it could be that c-level output is captured by distutil-extension
+        # (and not by us) and is printed to stdout:
+        captured_all = captured_out + "\n" + captured_err
+        self.assertTrue("error" in captured_all, msg="error in " + captured_all)
+
+    def test_cython_link_error_shown(self):
+        ip = self._ip
+        with capture_output() as out:
+            ip.run_cell_magic('cython', '-3 -l=xxxxxxxx', code)
+        captured_out, captured_err = out
+
+        # it could be that c-level output is captured by distutil-extension
+        # (and not by us) and is printed to stdout:
+        captured_all = captured_out + "\n!" + captured_err
+        self.assertTrue("error" in captured_all, msg="error in " + captured_all)
+
+    def test_cython_warning_shown(self):
+        ip = self._ip
+        with capture_output() as out:
+            # force rebuild, otherwise no warning as after the first success
+            # no build step is performed
+            ip.run_cell_magic('cython', '-3 -f', compile_warning_code)
+        captured_out, captured_err = out
+
+        # check that warning was printed to stdout even if build hasn't failed
+        self.assertTrue("CWarning" in captured_out)
+
     @skip_win32('Skip on Windows')
     def test_cython3_pgo(self):
         # The Cython cell defines the functions f() and call().