Marking known failures in test_run_core_python_test

test/python_tests.txt

-# Test modules with a failure reason after their name are skipped.
+# Test modules with a failure reason after their name are either skipped
+# or marked as a known failure in pytest.
 #
-# Reason codes are:
+# Following reason codes are skipped:
 # - platform-specific: This is testing something about a particular platform
 #   that isn't relevant here
 # - audioop: Requires the audioop module
@@ -17,11 +18,15 @@
 #   implementation of date/time formatting in strftime and strptime
 # - permissions: Issues with the test writing to the virtual filesystem
 # - locale: Fails due to include locale implementation.
+# - nonsense: This functionality doesn't make sense in this context. Includes
+#   things like `pip`, `distutils`
+#
+# While the below reason codes are marked as a known failure:
 # - crash: The Python interpreter just stopped without a traceback. Will require
 #   further investigation. This usually seems to be caused by calling into a
 #   system function that doesn't behave as one would expect.
-# - nonsense: This functionality doesn't make sense in this context. Includes
-#   things like `pip`, `distutils`
+# - crash_chrome: Same as crash but only affecting Chrome
+# - crash_firefox: Same as crash but only affecting Firefox
 
 test___all__
 test___future__
@@ -101,7 +106,7 @@ test_codeop
 test_collections
 test_colorsys
 test_compare
-test_compile
+test_compile crash-chrome
 test_compileall crash
 test_complex
 test_concurrent_futures

test/test_python.py

@@ -284,12 +284,18 @@ def test_open_url(selenium):
 
 
 @pytest.mark.flaky(reruns=2)
-def test_run_core_python_test(python_test, selenium):
+def test_run_core_python_test(python_test, selenium, request):
     selenium.load_package('test')
+
+    name, error_flags = python_test
+    if error_flags:
+        request.applymarker(pytest.mark.xfail(
+            run=False, reason='known failure with code "{}"'
+                              .format(error_flags)))
     try:
         selenium.run(
             "from test.libregrtest import main\n"
-            "main(['{}'], verbose=True, verbose3=True)".format(python_test))
+            "main(['{}'], verbose=True, verbose3=True)".format(name))
     except selenium.JavascriptException as e:
         assert 'SystemExit: 0' in str(e)
 
@@ -297,17 +303,25 @@ def test_run_core_python_test(python_test, selenium):
 def pytest_generate_tests(metafunc):
     if 'python_test' in metafunc.fixturenames:
         test_modules = []
+        test_modules_ids = []
         if 'CIRCLECI' not in os.environ or True:
             with open(
                     Path(__file__).parent / "python_tests.txt") as fp:
                 for line in fp:
                     line = line.strip()
-                    if line.startswith('#'):
+                    if line.startswith('#') or not line:
                         continue
-                    parts = line.split()
-                    if len(parts) == 1:
-                        test_modules.append(parts[0])
-        metafunc.parametrize("python_test", test_modules)
+                    error_flags = line.split()
+                    name = error_flags.pop(0)
+                    if (not error_flags
+                        or set(error_flags).intersection(
+                                {'crash', 'crash-chrome', 'crash-firefox'})):
+                            test_modules.append((name, error_flags))
+                            # explicitly define test ids to keep
+                            # a human readable test name in pytest
+                            test_modules_ids.append(name)
+        metafunc.parametrize("python_test", test_modules,
+                             ids=test_modules_ids)
 
 
 def test_recursive_repr(selenium):