Prevent test failures when Python executable path contains whitespace (GH-3372)

Cython/TestUtils.py

@@ -2,6 +2,8 @@ from __future__ import absolute_import
 
 import os
 import unittest
+import shlex
+import sys
 import tempfile
 
 from .Compiler import Errors
@@ -184,34 +186,41 @@ class TreeAssertVisitor(VisitorTransform):
     visit_Node = VisitorTransform.recurse_to_children
 
 
-def unpack_source_tree(tree_file, dir=None):
-    if dir is None:
-        dir = tempfile.mkdtemp()
-    header = []
-    cur_file = None
-    f = open(tree_file)
-    try:
-        lines = f.readlines()
-    finally:
-        f.close()
-    del f
-    try:
-        for line in lines:
-            if line[:5] == '#####':
-                filename = line.strip().strip('#').strip().replace('/', os.path.sep)
-                path = os.path.join(dir, filename)
-                if not os.path.exists(os.path.dirname(path)):
-                    os.makedirs(os.path.dirname(path))
-                if cur_file is not None:
-                    f, cur_file = cur_file, None
-                    f.close()
-                cur_file = open(path, 'w')
-            elif cur_file is not None:
-                cur_file.write(line)
-            elif line.strip() and not line.lstrip().startswith('#'):
-                if line.strip() not in ('"""', "'''"):
-                    header.append(line)
-    finally:
-        if cur_file is not None:
-            cur_file.close()
-    return dir, ''.join(header)
+def unpack_source_tree(tree_file, workdir, cython_root):
+    programs = {
+        'PYTHON': [sys.executable],
+        'CYTHON': [sys.executable, os.path.join(cython_root, 'cython.py')],
+        'CYTHONIZE': [sys.executable, os.path.join(cython_root, 'cythonize.py')]
+    }
+
+    if workdir is None:
+        workdir = tempfile.mkdtemp()
+    header, cur_file = [], None
+    with open(tree_file) as f:
+        try:
+            for line in f:
+                if line.startswith('#####'):
+                    filename = line.strip().strip('#').strip().replace('/', os.path.sep)
+                    path = os.path.join(workdir, filename)
+                    if not os.path.exists(os.path.dirname(path)):
+                        os.makedirs(os.path.dirname(path))
+                    if cur_file is not None:
+                        to_close, cur_file = cur_file, None
+                        to_close.close()
+                    cur_file = open(path, 'w')
+                elif cur_file is not None:
+                    cur_file.write(line)
+                elif line.strip() and not line.lstrip().startswith('#'):
+                    if line.strip() not in ('"""', "'''"):
+                        command = shlex.split(line)
+                        if not command: continue
+                        # In Python 3: prog, *args = command
+                        prog, args = command[0], command[1:]
+                        try:
+                            header.append(programs[prog]+args)
+                        except KeyError:
+                            header.append(command)
+        finally:
+            if cur_file is not None:
+                cur_file.close()
+    return workdir, header

runtests.py

@@ -1728,7 +1728,7 @@ class EndToEndTest(unittest.TestCase):
 
     def setUp(self):
         from Cython.TestUtils import unpack_source_tree
-        _, self.commands = unpack_source_tree(self.treefile, self.workdir)
+        _, self.commands = unpack_source_tree(self.treefile, self.workdir, self.cython_root)
         self.old_dir = os.getcwd()
         os.chdir(self.workdir)
         if self.workdir not in sys.path:
@@ -1753,10 +1753,6 @@ class EndToEndTest(unittest.TestCase):
 
     def runTest(self):
         self.success = False
-        commands = (self.commands
-            .replace("CYTHONIZE", "PYTHON %s" % os.path.join(self.cython_root, 'cythonize.py'))
-            .replace("CYTHON", "PYTHON %s" % os.path.join(self.cython_root, 'cython.py'))
-            .replace("PYTHON", sys.executable))
         old_path = os.environ.get('PYTHONPATH')
         env = dict(os.environ)
         new_path = self.cython_syspath
@@ -1766,21 +1762,15 @@ class EndToEndTest(unittest.TestCase):
         cmd = []
         out = []
         err = []
-        for command_no, command in enumerate(filter(None, commands.splitlines()), 1):
+        for command_no, command in enumerate(self.commands, 1):
             with self.stats.time('%s(%d)' % (self.name, command_no), 'c',
-                                 'etoe-build' if ' setup.py ' in command else 'etoe-run'):
+                                 'etoe-build' if 'setup.py' in command else 'etoe-run'):
                 if self.capture:
-                    p = subprocess.Popen(command,
-                                     stderr=subprocess.PIPE,
-                                     stdout=subprocess.PIPE,
-                                     shell=True,
-                                     env=env)
+                    p = subprocess.Popen(command, stderr=subprocess.PIPE, stdout=subprocess.PIPE, env=env)
                     _out, _err = p.communicate()
                     res = p.returncode
                 else:
-                    p = subprocess.call(command,
-                                     shell=True,
-                                     env=env)
+                    p = subprocess.call(command, env=env)
                     _out, _err = b'', b''
                     res = p
                 cmd.append(command)