Merge pull request #117 from rth/mark-known-failures

Mark crashing tests as known failures

.circleci/config.yml

@@ -23,7 +23,7 @@ jobs:
             sudo update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-8 80 --slave /usr/bin/g++ g++ /usr/bin/g++-8
             sudo update-alternatives --set gcc /usr/bin/gcc-8
 
-            sudo pip install pytest-xdist pytest-instafail selenium PyYAML pytest-rerunfailures
+            sudo pip install pytest-xdist pytest-instafail selenium PyYAML
 
             # Get recent version of Firefox and geckodriver
             wget -O firefox.tar.bz2 https://download.mozilla.org/\?product\=firefox-nightly-latest-ssl\&os\=linux64\&lang\=en-US

Makefile

@@ -99,7 +99,7 @@ build/renderedhtml.css: src/renderedhtml.less
 
 
 test: all build/test.html build/test_data.txt
-	py.test test -v --instafail
+	py.test test -v -r sxX --instafail
 
 
 build/test_data.txt: test/data.txt

test/conftest.py

@@ -74,6 +74,9 @@ class SeleniumWrapper:
 
 
 class FirefoxWrapper(SeleniumWrapper):
+
+    browser = 'firefox'
+
     def get_driver(self):
         from selenium.webdriver import Firefox
         from selenium.webdriver.firefox.options import Options
@@ -89,6 +92,9 @@ class FirefoxWrapper(SeleniumWrapper):
 
 
 class ChromeWrapper(SeleniumWrapper):
+
+    browser = 'chrome'
+
     def get_driver(self):
         from selenium.webdriver import Chrome
         from selenium.webdriver.chrome.options import Options

test/python_tests.txt

-# Test modules with a failure reason after their name are skipped.
+# Test modules with a failure reason after their name are either skipped
+# or marked as a known failure in pytest.
 #
-# Reason codes are:
+# Following reason codes are skipped:
 # - platform-specific: This is testing something about a particular platform
 #   that isn't relevant here
 # - audioop: Requires the audioop module
@@ -17,11 +18,15 @@
 #   implementation of date/time formatting in strftime and strptime
 # - permissions: Issues with the test writing to the virtual filesystem
 # - locale: Fails due to include locale implementation.
+# - nonsense: This functionality doesn't make sense in this context. Includes
+#   things like `pip`, `distutils`
+#
+# While the below reason codes are marked as a known failure:
 # - crash: The Python interpreter just stopped without a traceback. Will require
 #   further investigation. This usually seems to be caused by calling into a
 #   system function that doesn't behave as one would expect.
-# - nonsense: This functionality doesn't make sense in this context. Includes
-#   things like `pip`, `distutils`
+# - crash-chrome: Same as crash but only affecting Chrome
+# - crash-firefox: Same as crash but only affecting Firefox
 
 test___all__
 test___future__
@@ -101,8 +106,8 @@ test_codeop
 test_collections
 test_colorsys
 test_compare
-test_compile
-test_compileall crash
+test_compile crash-chrome
+test_compileall
 test_complex
 test_concurrent_futures
 test_configparser
@@ -173,7 +178,7 @@ test_faulthandler
 test_fcntl
 test_file
 test_file_eintr subprocess
-test_filecmp    crash
+test_filecmp
 test_fileinput
 test_fileio
 test_finalization
@@ -201,7 +206,7 @@ test_genexps
 test_getargs2
 test_getopt
 test_getpass  permissions
-test_gettext  crash
+test_gettext
 test_glob     crash
 test_global
 test_grammar
@@ -239,20 +244,20 @@ test_importlib.import_.test_packages
 test_importlib.import_.test_path
 test_importlib.import_.test_relative_imports
 test_importlib.source.test_case_sensitivity
-test_importlib.source.test_file_loader  crash
-test_importlib.source.test_finder       crash
+test_importlib.source.test_file_loader
+test_importlib.source.test_finder
 test_importlib.source.test_path_hook
-test_importlib.source.test_source_encoding  crash
+test_importlib.source.test_source_encoding
 test_importlib.test_abc
-test_importlib.test_api crash
+test_importlib.test_api
 test_importlib.test_lazy
 test_importlib.test_locks
 test_importlib.test_namespace_pkgs
 test_importlib.test_spec
-test_importlib.test_util  crash
+test_importlib.test_util
 test_importlib.test_windows platform-specific
 test_index
-test_inspect  crash
+test_inspect
 test_int
 test_int_literal
 test_io crash
@@ -305,7 +310,7 @@ test_mimetypes
 test_minidom
 test_mmap
 test_module
-test_modulefinder crash
+test_modulefinder
 test_msilib
 test_multibytecodec
 test_multiprocessing_fork
@@ -334,8 +339,8 @@ test_pickle dbm
 test_pickletools  dbm
 test_pipes        platform-specific
 test_pkg
-test_pkgimport  crash
-test_pkgutil    crash
+test_pkgimport
+test_pkgutil
 test_platform   subprocess
 test_plistlib
 test_poll subprocess
@@ -352,7 +357,7 @@ test_pstats
 test_pty
 test_pulldom
 test_pwd  crash
-test_py_compile crash
+test_py_compile
 test_pyclbr
 test_pydoc  crash
 test_pyexpat
@@ -370,7 +375,7 @@ test_resource
 test_richcmp
 test_rlcompleter  crash
 test_robotparser
-test_runpy  crash
+test_runpy
 test_sax
 test_sched
 test_scope
@@ -402,7 +407,7 @@ test_stat
 test_statistics
 test_strftime   strftime
 test_string
-test_string_literals  crash
+test_string_literals
 test_stringprep
 test_strptime strftime
 test_strtod
@@ -502,9 +507,9 @@ test_xml_etree_c
 test_xmlrpc networking
 test_xmlrpc_net
 test_yield_from
-test_zipapp crash
-test_zipfile  crash
+test_zipapp
+test_zipfile
 test_zipfile64
-test_zipimport  crash
-test_zipimport_support  crash
+test_zipimport
+test_zipimport_support
 test_zlib

test/test_common.py

@@ -27,8 +27,8 @@ def registered_packages_meta():
             for name in packages}
 
 
-UNSUPPORTED_PACKAGES = {'ChromeWrapper': ['pandas'],
-                        'FirefoxWrapper': []}
+UNSUPPORTED_PACKAGES = {'chrome': ['pandas'],
+                        'firefox': []}
 
 
 @pytest.mark.parametrize('name', registered_packages())
@@ -36,12 +36,10 @@ def test_import(name, selenium_standalone):
     # check that we can parse the meta.yaml
     meta = common.parse_package(PKG_DIR / name / 'meta.yaml')
 
-    if name in UNSUPPORTED_PACKAGES[selenium_standalone.__class__.__name__]:
+    if name in UNSUPPORTED_PACKAGES[selenium_standalone.browser]:
         pytest.xfail(
-            '{} fails to load and is not supported on {}.'
-            .format(name,
-                    selenium_standalone.__class__.__name__
-                                       .replace('Wrapper', '')))
+                '{} fails to load and is not supported on {}.'
+                .format(name, selenium_standalone.browser))
 
     for import_name in meta.get('test', {}).get('imports', []):
         selenium_standalone.load_package(name)

test/test_pandas.py

 import pytest
 
 
-@pytest.mark.skip
-def test_pandas(selenium):
+def test_pandas(selenium, request):
+    if selenium.browser == 'chrome':
+        request.applymarker(pytest.mark.xfail(
+            run=False, reason='chrome not supported'))
     selenium.load_package("pandas")
     assert len(selenium.run("import pandas\ndir(pandas)")) == 179
 
 
-@pytest.mark.skip
-def test_extra_import(selenium):
+def test_extra_import(selenium, request):
+    if selenium.browser == 'chrome':
+        request.applymarker(pytest.mark.xfail(
+            run=False, reason='chrome not supported'))
+
     selenium.load_package("pandas")
     selenium.run("from pandas import Series, DataFrame, Panel")

test/test_python.py

@@ -283,13 +283,20 @@ def test_open_url(selenium):
         "pyodide.open_url('test_data.txt').read()\n") == 'HELLO\n'
 
 
-@pytest.mark.flaky(reruns=2)
-def test_run_core_python_test(python_test, selenium):
+def test_run_core_python_test(python_test, selenium, request):
+
+    name, error_flags = python_test
+
+    if ('crash' in error_flags or
+            'crash-' + selenium.browser in error_flags):
+        pytest.xfail(reason='known failure with code "{}"'
+                            .format(','.join(error_flags)))
+
     selenium.load_package('test')
     try:
         selenium.run(
             "from test.libregrtest import main\n"
-            "main(['{}'], verbose=True, verbose3=True)".format(python_test))
+            "main(['{}'], verbose=True, verbose3=True)".format(name))
     except selenium.JavascriptException as e:
         assert 'SystemExit: 0' in str(e)
 
@@ -297,17 +304,25 @@ def test_run_core_python_test(python_test, selenium):
 def pytest_generate_tests(metafunc):
     if 'python_test' in metafunc.fixturenames:
         test_modules = []
+        test_modules_ids = []
         if 'CIRCLECI' not in os.environ or True:
             with open(
                     Path(__file__).parent / "python_tests.txt") as fp:
                 for line in fp:
                     line = line.strip()
-                    if line.startswith('#'):
+                    if line.startswith('#') or not line:
                         continue
-                    parts = line.split()
-                    if len(parts) == 1:
-                        test_modules.append(parts[0])
-        metafunc.parametrize("python_test", test_modules)
+                    error_flags = line.split()
+                    name = error_flags.pop(0)
+                    if (not error_flags
+                        or set(error_flags).intersection(
+                                {'crash', 'crash-chrome', 'crash-firefox'})):
+                            test_modules.append((name, error_flags))
+                            # explicitly define test ids to keep
+                            # a human readable test name in pytest
+                            test_modules_ids.append(name)
+        metafunc.parametrize("python_test", test_modules,
+                             ids=test_modules_ids)
 
 
 def test_recursive_repr(selenium):