Mark expected failures under crosshair

Author: Zac-HD

hypothesis-python/tests/common/utils.py

@@ -9,6 +9,7 @@
 # obtain one at https://mozilla.org/MPL/2.0/.
 
 import contextlib
+import enum
 import sys
 from io import StringIO
 from types import SimpleNamespace
@@ -228,3 +229,32 @@ def temp_registered(type_, strat_or_factory):
 # config option, so *linking against* something built this way can break us.
 # Everything is terrible
 PYTHON_FTZ = next_down(sys.float_info.min) == 0.0
+
+
+class Why(enum.Enum):
+    # Use an enum here so it's easier to find and/or exclude some cases later
+    other = "other"
+    flaky_replay = "Inconsistent results from replaying a failing test..."
+    symbolic_outside_context = (
+        "CrosshairInternal error from using value outside context"
+    )
+    undiscovered = "crosshair doesn't find the failing input"
+
+
+def xfail_on_crosshair(why: Why, /):
+    import pytest
+    try:
+        import crosshair.util
+    except ImportError:
+        return pytest.mark.xf_crosshair
+
+    kw = {
+        "strict": True,
+        "reason": f"Expected failure due to: {why.value}",
+        "condition": settings.get_profile(settings._current_profile).backend
+        == "crosshair",
+    }
+    if why is Why.symbolic_outside_context:
+        kw["raises"] = crosshair.util.CrosshairInternal
+
+    return lambda fn: pytest.mark.xf_crosshair(pytest.mark.xfail(**kw)(fn))

hypothesis-python/tests/conftest.py

@@ -46,6 +46,7 @@ def pytest_configure(config):
         "markers",
         "xp_min_version(api_version): run when greater or equal to api_version",
     )
+    config.addinivalue_line("markers", "xf_crosshair: selection for xfailing symbolics")
 
 
 def pytest_addoption(parser):

hypothesis-python/tests/cover/test_testdecorators.py

@@ -30,13 +30,15 @@
 )
 
 from tests.common.utils import (
+    Why,
     assert_falsifying_output,
     capture_out,
     fails,
     fails_with,
     no_shrink,
     raises,
     skipif_emscripten,
+    xfail_on_crosshair,
 )
 
 # This particular test file is run under both pytest and nose, so it can't
@@ -304,6 +306,7 @@ def test_has_ascii(x):
     assert any(c in ascii_characters for c in x)
 
 
+@xfail_on_crosshair(Why.symbolic_outside_context)
 def test_can_derandomize():
     values = []
 
@@ -393,6 +396,7 @@ def test_mixed_text(x):
     assert set(x).issubset(set("abcdefg"))
 
 
+@xfail_on_crosshair(Why.other)  # runs ~five failing examples
 def test_when_set_to_no_simplifies_runs_failing_example_twice():
     failing = []
 
@@ -478,6 +482,7 @@ def test_empty_lists(xs):
     assert xs == []
 
 
+@xfail_on_crosshair(Why.other)  # executes >> 2 tests, but there are only two bools
 def test_given_usable_inline_on_lambdas():
     xs = []
     given(booleans())(lambda x: xs.append(x))()