Merge pull request #1041 from oraios/skip-flaky-tests

Skip flaky tests in CI

test/conftest.py

@@ -1,5 +1,6 @@
 import logging
 import os
+import platform
 import shutil as _sh
 from collections.abc import Iterator
 from contextlib import contextmanager
@@ -209,6 +210,8 @@ is_ci = os.getenv("CI") == "true" or os.getenv("GITHUB_ACTIONS") == "true"
 Flag indicating whether the tests are running in the GitHub CI environment.
 """
 
+is_windows = platform.system() == "Windows"
+
 
 def _determine_disabled_languages() -> list[Language]:
     """

test/serena/test_serena_agent.py

@@ -15,7 +15,7 @@ from serena.project import Project
 from serena.tools import SUCCESS_RESULT, FindReferencingSymbolsTool, FindSymbolTool, ReplaceContentTool, ReplaceSymbolBodyTool
 from solidlsp.ls_config import Language
 from solidlsp.ls_types import SymbolKind
-from test.conftest import get_repo_path, language_tests_enabled
+from test.conftest import get_repo_path, is_ci, language_tests_enabled
 from test.solidlsp import clojure as clj
 
 
@@ -124,6 +124,12 @@ class TestSerenaAgent:
         indirect=["serena_agent"],
     )
     def test_find_symbol(self, serena_agent: SerenaAgent, symbol_name: str, expected_kind: str, expected_file: str):
+        # skip flaky tests in CI
+        # TODO: Revisit the flaky tests and re-enable once the LS issues are resolved #1039
+        flaky_languages = {Language.FSHARP}
+        if set(serena_agent.get_active_lsp_languages()).intersection(flaky_languages) and is_ci:
+            pytest.skip("Test is flaky and thus skipped in CI environment.")
+
         agent = serena_agent
         find_symbol_tool = agent.get_tool(FindSymbolTool)
         result = find_symbol_tool.apply_ex(name_path_pattern=symbol_name, include_info=True)

test/solidlsp/fsharp/test_fsharp_basic.py

@@ -10,6 +10,7 @@ from solidlsp import SolidLanguageServer
 from solidlsp.language_servers.fsharp_language_server import FSharpLanguageServer
 from solidlsp.ls_config import Language
 from solidlsp.ls_utils import SymbolUtils
+from test.conftest import is_ci
 
 
 @pytest.mark.fsharp
@@ -87,6 +88,7 @@ class TestFSharpLanguageServer:
         for expected in expected_symbols:
             assert expected in symbol_names, f"{expected} not found in Person.fs symbols"
 
+    @pytest.mark.skipif(is_ci, reason="Test is flaky")  # TODO: Re-enable if the LS can be made more reliable #1039
     @pytest.mark.parametrize("language_server", [Language.FSHARP], indirect=True)
     def test_find_referencing_symbols_across_files(self, language_server: SolidLanguageServer) -> None:
         """Test finding references to Calculator functions across files."""

test/solidlsp/nix/test_nix_basic.py

@@ -10,6 +10,7 @@ import pytest
 
 from solidlsp import SolidLanguageServer
 from solidlsp.ls_config import Language
+from test.conftest import is_ci
 
 # Skip all Nix tests on Windows as Nix doesn't support Windows
 pytestmark = pytest.mark.skipif(platform.system() == "Windows", reason="Nix and nil are not available on Windows")
@@ -115,6 +116,7 @@ class TestNixLanguageServer:
             # Check if we found the inherit (line 67, 0-indexed: 66)
             assert 66 in ref_lines, f"Should find makeGreeting inherit at line 67, found at lines {[l+1 for l in ref_lines]}"
 
+    @pytest.mark.skipif(is_ci, reason="Test is flaky")  # TODO: Re-enable if the hover test becomes more stable (#1039)
     @pytest.mark.parametrize("language_server", [Language.NIX], indirect=True)
     def test_hover_information(self, language_server: SolidLanguageServer) -> None:
         """Test hover information for symbols."""

test/solidlsp/php/test_php_basic.py

@@ -1,17 +1,19 @@
-import platform
 from pathlib import Path
 
 import pytest
 
 from solidlsp import SolidLanguageServer
 from solidlsp.ls_config import Language
-from test.conftest import language_tests_enabled
+from test.conftest import is_ci, is_windows, language_tests_enabled
 
 _php_servers: list[Language] = [Language.PHP]
 if language_tests_enabled(Language.PHP_PHPACTOR):
     _php_servers.append(Language.PHP_PHPACTOR)
 
 
+@pytest.mark.skipif(
+    is_ci and is_windows, reason="Tests are flaky"
+)  # TODO: Re-enable once we have a solution for running Phpactor tests on Windows CI #1039
 @pytest.mark.php
 class TestPhpLanguageServers:
     @pytest.mark.parametrize("language_server", _php_servers, indirect=True)
@@ -99,9 +101,6 @@ class TestPhpLanguageServers:
     @pytest.mark.parametrize("language_server", _php_servers, indirect=True)
     @pytest.mark.parametrize("repo_path", [Language.PHP], indirect=True)
     def test_find_references_within_file(self, language_server: SolidLanguageServer, repo_path: Path) -> None:
-        if language_server.language == Language.PHP_PHPACTOR and platform.system() == "Windows":
-            pytest.skip("Phpactor references test not working on Windows, skipping for now")
-
         index_php_path = str(repo_path / "index.php")
 
         # In index.php (0-indexed lines):
@@ -150,9 +149,6 @@ class TestPhpLanguageServers:
     @pytest.mark.parametrize("language_server", _php_servers, indirect=True)
     @pytest.mark.parametrize("repo_path", [Language.PHP], indirect=True)
     def test_find_references_across_files(self, language_server: SolidLanguageServer, repo_path: Path) -> None:
-        if language_server.language_server.language == Language.PHP_PHPACTOR and platform.system() == "Windows":
-            pytest.skip("Phpactor references test not working on Windows, skipping for now")
-
         helper_php_path = str(repo_path / "helper.php")
         # In index.php (0-indexed lines):
         # Line 13: helperFunction(); // Usage of helperFunction