Mark flaky tests with xfail instead of skip

test/serena/test_serena_agent.py

@@ -131,7 +131,7 @@ class TestSerenaAgent:
     )
     def test_find_symbol(self, serena_agent: SerenaAgent, symbol_name: str, expected_kind: str, expected_file: str):
         # skip flaky tests in CI
-        # TODO: Revisit the flaky tests and re-enable once the LS issues are resolved #1039
+        # TODO: Revisit the flaky tests and re-enable once the LS issues are resolved #1040
         flaky_languages = {Language.FSHARP, Language.RUST}
         if set(serena_agent.get_active_lsp_languages()).intersection(flaky_languages) and is_ci:
             pytest.skip("Test is flaky and thus skipped in CI environment.")
@@ -206,7 +206,7 @@ class TestSerenaAgent:
     )
     def test_find_symbol_references(self, serena_agent: SerenaAgent, symbol_name: str, def_file: str, ref_file: str) -> None:
         # skip flaky tests in CI
-        # TODO: Revisit the flaky tests and re-enable once the LS issues are resolved #1039
+        # TODO: Revisit the flaky tests and re-enable once the LS issues are resolved #1040
         flaky_languages = {Language.TYPESCRIPT}
         if set(serena_agent.get_active_lsp_languages()).intersection(flaky_languages) and is_ci:
             pytest.skip("Test is flaky and thus skipped in CI environment.")

test/solidlsp/fsharp/test_fsharp_basic.py

@@ -89,7 +89,7 @@ class TestFSharpLanguageServer:
         for expected in expected_symbols:
             assert expected in symbol_names, f"{expected} not found in Person.fs symbols"
 
-    @pytest.mark.skipif(is_ci, reason="Test is flaky")  # TODO: Re-enable if the LS can be made more reliable #1039
+    @pytest.mark.xfail(is_ci, reason="Test is flaky")  # TODO: Re-enable if the LS can be made more reliable #1040
     @pytest.mark.parametrize("language_server", [Language.FSHARP], indirect=True)
     def test_find_referencing_symbols_across_files(self, language_server: SolidLanguageServer) -> None:
         """Test finding references to Calculator functions across files."""
@@ -125,7 +125,7 @@ class TestFSharpLanguageServer:
         # We should get at least some definitions
         assert len(definitions) >= 0, "Should get definitions (even if empty for complex cases)"
 
-    @pytest.mark.skipif(is_ci, reason="Test is flaky")  # TODO: Re-enable if the LS can be made more reliable #1039
+    @pytest.mark.xfail(is_ci, reason="Test is flaky")  # TODO: Re-enable if the LS can be made more reliable #1040
     @pytest.mark.parametrize("language_server", [Language.FSHARP], indirect=True)
     def test_hover_information(self, language_server: SolidLanguageServer) -> None:
         """Test hover information functionality."""

test/solidlsp/nix/test_nix_basic.py

@@ -116,7 +116,7 @@ class TestNixLanguageServer:
             # Check if we found the inherit (line 67, 0-indexed: 66)
             assert 66 in ref_lines, f"Should find makeGreeting inherit at line 67, found at lines {[l+1 for l in ref_lines]}"
 
-    @pytest.mark.skipif(is_ci, reason="Test is flaky")  # TODO: Re-enable if the hover test becomes more stable (#1039)
+    @pytest.mark.xfail(is_ci, reason="Test is flaky")  # TODO: Re-enable if the hover test becomes more stable (#1040)
     @pytest.mark.parametrize("language_server", [Language.NIX], indirect=True)
     def test_hover_information(self, language_server: SolidLanguageServer) -> None:
         """Test hover information for symbols."""

test/solidlsp/php/test_php_basic.py

@@ -11,9 +11,9 @@ if language_tests_enabled(Language.PHP_PHPACTOR):
     _php_servers.append(Language.PHP_PHPACTOR)
 
 
-@pytest.mark.skipif(
+@pytest.mark.xfail(
     is_ci and is_windows, reason="Tests are flaky"
-)  # TODO: Re-enable once we have a solution for running Phpactor tests on Windows CI #1039
+)  # TODO: Re-enable once we have a solution for running Phpactor tests on Windows CI #1040
 @pytest.mark.php
 class TestPhpLanguageServers:
     @pytest.mark.parametrize("language_server", _php_servers, indirect=True)

test/solidlsp/swift/test_swift_basic.py

@@ -100,7 +100,7 @@ class TestSwiftLanguageServerBasics:
         utils_def = definitions[0]
         assert utils_def.get("uri", "").endswith("utils.swift"), "Definition should be in utils.swift"
 
-    @pytest.mark.skipif(is_ci, reason="Test is flaky in CI")
+    @pytest.mark.xfail(is_ci, reason="Test is flaky in CI")  # See #1040
     @pytest.mark.parametrize("language_server", [Language.SWIFT], indirect=True)
     def test_request_references_calculator_class(self, language_server: SolidLanguageServer) -> None:
         """Test request_references on the Calculator class."""
@@ -123,7 +123,7 @@ class TestSwiftLanguageServerBasics:
         line_5_refs = [ref for ref in calculator_refs if ref.get("range", {}).get("start", {}).get("line") == 4]
         assert len(line_5_refs) > 0, "Calculator should be referenced at line 5"
 
-    @pytest.mark.skipif(is_ci, reason="Test is flaky in CI")
+    @pytest.mark.xfail(is_ci, reason="Test is flaky in CI")  # See #1040
     @pytest.mark.parametrize("language_server", [Language.SWIFT], indirect=True)
     def test_request_references_user_struct(self, language_server: SolidLanguageServer) -> None:
         """Test request_references on the User struct."""
@@ -145,7 +145,7 @@ class TestSwiftLanguageServerBasics:
         line_9_refs = [ref for ref in user_refs if ref.get("range", {}).get("start", {}).get("line") == 8]
         assert len(line_9_refs) > 0, "User should be referenced at line 9"
 
-    @pytest.mark.skipif(is_ci, reason="Test is flaky in CI")
+    @pytest.mark.xfail(is_ci, reason="Test is flaky in CI")  # See #1040
     @pytest.mark.parametrize("language_server", [Language.SWIFT], indirect=True)
     def test_request_references_utils_struct(self, language_server: SolidLanguageServer) -> None:
         """Test request_references on the Utils struct."""