Add option to test runner to ensure determinism.

runtests.py

@@ -486,6 +486,7 @@ class TestBuilder(object):
     def __init__(self, rootdir, workdir, selectors, exclude_selectors, annotate,
                  cleanup_workdir, cleanup_sharedlibs, cleanup_failures,
                  with_pyregr, cython_only, languages, test_bugs, fork, language_level,
+                 test_determinism,
                  common_utility_dir):
         self.rootdir = rootdir
         self.workdir = workdir
@@ -501,6 +502,7 @@ class TestBuilder(object):
         self.test_bugs = test_bugs
         self.fork = fork
         self.language_level = language_level
+        self.test_determinism = test_determinism
         self.common_utility_dir = common_utility_dir
 
     def build_suite(self):
@@ -631,6 +633,7 @@ class TestBuilder(object):
                           fork=self.fork,
                           language_level=self.language_level,
                           warning_errors=warning_errors,
+                          test_determinism=self.test_determinism,
                           common_utility_dir=self.common_utility_dir)
 
 
@@ -639,6 +642,7 @@ class CythonCompileTestCase(unittest.TestCase):
                  expect_errors=False, expect_warnings=False, annotate=False, cleanup_workdir=True,
                  cleanup_sharedlibs=True, cleanup_failures=True, cython_only=False,
                  fork=True, language_level=2, warning_errors=False,
+                 test_determinism=False,
                  common_utility_dir=None):
         self.test_directory = test_directory
         self.tags = tags
@@ -657,6 +661,7 @@ class CythonCompileTestCase(unittest.TestCase):
         self.fork = fork
         self.language_level = language_level
         self.warning_errors = warning_errors
+        self.test_determinism = test_determinism
         self.common_utility_dir = common_utility_dir
         unittest.TestCase.__init__(self)
 
@@ -731,6 +736,10 @@ class CythonCompileTestCase(unittest.TestCase):
                     # Finally, remove the work dir itself
                     _to_clean.append(self.workdir)
 
+        if cleanup_c_files and os.path.exists(self.workdir + '-again'):
+            shutil.rmtree(self.workdir + '-again', ignore_errors=True)
+
+
     def runTest(self):
         self.success = False
         self.runCompileTest()
@@ -942,6 +951,21 @@ class CythonCompileTestCase(unittest.TestCase):
                 errors, warnings = sys.stderr.getall()
             finally:
                 sys.stderr = old_stderr
+            if self.test_determinism and not expect_errors:
+                workdir2 = workdir + '-again'
+                os.mkdir(workdir2)
+                self.run_cython(test_directory, module, workdir2, incdir, annotate)
+                diffs = []
+                for file in os.listdir(workdir2):
+                    if (open(os.path.join(workdir, file)).read()
+                        != open(os.path.join(workdir2, file)).read()):
+                        diffs.append(file)
+                        os.system('diff -u %s/%s %s/%s > %s/%s.diff' % (
+                            workdir, file,
+                            workdir2, file,
+                            workdir2, file))
+                if diffs:
+                    self.fail('Nondeterministic file generation: %s' % ', '.join(diffs))
 
         tostderr = sys.__stderr__.write
         if 'cerror' in self.tags['tag']:
@@ -1813,6 +1837,8 @@ def main():
                       help="deterministic generated by string")
     parser.add_option("--use_common_utility_dir", default=False, action="store_true")
     parser.add_option("--use_formal_grammar", default=False, action="store_true")
+    parser.add_option("--test_determinism", default=False, action="store_true",
+                      help="test whether Cython's output is deterministic")
 
     options, cmd_args = parser.parse_args(args)
 
@@ -2068,6 +2094,7 @@ def runtests(options, cmd_args, coverage=None):
                                 options.pyregr,
                                 options.cython_only, languages, test_bugs,
                                 options.fork, options.language_level,
+                                options.test_determinism,
                                 common_utility_dir)
         test_suite.addTest(filetests.build_suite())
 
@@ -2082,6 +2109,7 @@ def runtests(options, cmd_args, coverage=None):
                                     True,
                                     options.cython_only, languages, test_bugs,
                                     options.fork, sys.version_info[0],
+                                    options.test_determinism,
                                     common_utility_dir)
             sys.stderr.write("Including CPython regression tests in %s\n" % sys_pyregr_dir)
             test_suite.addTest(filetests.handle_directory(sys_pyregr_dir, 'pyregr'))