From eb78921c0899af26c6eb51508b7c96d5e49045cf Mon Sep 17 00:00:00 2001 From: Dominik Jain Date: Mon, 2 Mar 2026 19:12:10 +0100 Subject: [PATCH] Split test_find_symbol_references and test_find_symbol into stable and unstable tests --- test/serena/test_serena_agent.py | 172 ++++++++++++++++++------------- 1 file changed, 103 insertions(+), 69 deletions(-) diff --git a/test/serena/test_serena_agent.py b/test/serena/test_serena_agent.py index 3dbef5d8..b04dc37f 100644 --- a/test/serena/test_serena_agent.py +++ b/test/serena/test_serena_agent.py @@ -117,37 +117,7 @@ class TestSerenaAgent: serena_config = SerenaConfig(gui_log_window=False, web_dashboard=False) SerenaAgent(project=project, serena_config=serena_config) - @pytest.mark.parametrize( - "serena_agent,symbol_name,expected_kind,expected_file", - [ - pytest.param(Language.PYTHON, "User", "Class", "models.py", marks=pytest.mark.python), - pytest.param(Language.GO, "Helper", "Function", "main.go", marks=pytest.mark.go), - pytest.param(Language.JAVA, "Model", "Class", "Model.java", marks=pytest.mark.java), - pytest.param( - Language.KOTLIN, - "Model", - "Struct", - "Model.kt", - marks=[pytest.mark.kotlin] + ([pytest.mark.skip(reason="Kotlin LSP JVM crashes on restart in CI")] if is_ci else []), - ), - pytest.param(Language.RUST, "add", "Function", "lib.rs", marks=pytest.mark.rust), - pytest.param(Language.TYPESCRIPT, "DemoClass", "Class", "index.ts", marks=pytest.mark.typescript), - pytest.param(Language.PHP, "helperFunction", "Function", "helper.php", marks=pytest.mark.php), - pytest.param(Language.CLOJURE, "greet", "Function", clj.CORE_PATH, marks=pytest.mark.clojure), - pytest.param(Language.CSHARP, "Calculator", "Class", "Program.cs", marks=pytest.mark.csharp), - pytest.param(Language.FSHARP, "Calculator", "Module", "Calculator.fs", marks=pytest.mark.fsharp), - pytest.param(Language.POWERSHELL, "function Greet-User ()", "Function", "main.ps1", marks=pytest.mark.powershell), - pytest.param(Language.CPP_CCLS, "add", "Function", "b.cpp", marks=pytest.mark.cpp), - ], - indirect=["serena_agent"], - ) - def test_find_symbol(self, serena_agent: SerenaAgent, symbol_name: str, expected_kind: str, expected_file: str): - # skip flaky tests in CI - # TODO: Revisit the flaky tests and re-enable once the LS issues are resolved #1040 - flaky_languages = {Language.FSHARP, Language.RUST} - if set(serena_agent.get_active_lsp_languages()).intersection(flaky_languages) and is_ci: - pytest.skip("Test is flaky and thus skipped in CI environment.") - + def _assert_find_symbol(self, serena_agent: SerenaAgent, symbol_name: str, expected_kind: str, expected_file: str) -> None: agent = serena_agent find_symbol_tool = agent.get_tool(FindSymbolTool) result = find_symbol_tool.apply(name_path_pattern=symbol_name, include_info=True) @@ -175,54 +145,53 @@ class TestSerenaAgent: assert "A simple model class" in symbol_info, f"Java class docstring not found in symbol info: {s}" @pytest.mark.parametrize( - "serena_agent,symbol_name,def_file,ref_file", + "serena_agent,symbol_name,expected_kind,expected_file", [ - pytest.param( - Language.PYTHON, - "User", - os.path.join("test_repo", "models.py"), - os.path.join("test_repo", "services.py"), - marks=pytest.mark.python, - ), - pytest.param(Language.GO, "Helper", "main.go", "main.go", marks=pytest.mark.go), - pytest.param( - Language.JAVA, - "Model", - os.path.join("src", "main", "java", "test_repo", "Model.java"), - os.path.join("src", "main", "java", "test_repo", "Main.java"), - marks=pytest.mark.java, - ), + pytest.param(Language.PYTHON, "User", "Class", "models.py", marks=pytest.mark.python), + pytest.param(Language.GO, "Helper", "Function", "main.go", marks=pytest.mark.go), + pytest.param(Language.JAVA, "Model", "Class", "Model.java", marks=pytest.mark.java), pytest.param( Language.KOTLIN, "Model", - os.path.join("src", "main", "kotlin", "test_repo", "Model.kt"), - os.path.join("src", "main", "kotlin", "test_repo", "Main.kt"), + "Struct", + "Model.kt", marks=[pytest.mark.kotlin] + ([pytest.mark.skip(reason="Kotlin LSP JVM crashes on restart in CI")] if is_ci else []), ), - pytest.param(Language.RUST, "add", os.path.join("src", "lib.rs"), os.path.join("src", "main.rs"), marks=pytest.mark.rust), - pytest.param(Language.TYPESCRIPT, "helperFunction", "index.ts", "use_helper.ts", marks=pytest.mark.typescript), - pytest.param(Language.PHP, "helperFunction", "helper.php", "index.php", marks=pytest.mark.php), - pytest.param( - Language.CLOJURE, - "multiply", - clj.CORE_PATH, - clj.UTILS_PATH, - marks=pytest.mark.clojure, - ), - pytest.param(Language.CSHARP, "Calculator", "Program.cs", "Program.cs", marks=pytest.mark.csharp), - pytest.param(Language.FSHARP, "add", "Calculator.fs", "Program.fs", marks=pytest.mark.fsharp), - pytest.param(Language.POWERSHELL, "function Greet-User ()", "main.ps1", "main.ps1", marks=pytest.mark.powershell), - pytest.param(Language.CPP_CCLS, "add", "b.cpp", "a.cpp", marks=pytest.mark.cpp), + pytest.param(Language.TYPESCRIPT, "DemoClass", "Class", "index.ts", marks=pytest.mark.typescript), + pytest.param(Language.PHP, "helperFunction", "Function", "helper.php", marks=pytest.mark.php), + pytest.param(Language.CLOJURE, "greet", "Function", clj.CORE_PATH, marks=pytest.mark.clojure), + pytest.param(Language.CSHARP, "Calculator", "Class", "Program.cs", marks=pytest.mark.csharp), + pytest.param(Language.POWERSHELL, "function Greet-User ()", "Function", "main.ps1", marks=pytest.mark.powershell), + pytest.param(Language.CPP_CCLS, "add", "Function", "b.cpp", marks=pytest.mark.cpp), ], indirect=["serena_agent"], ) - def test_find_symbol_references(self, serena_agent: SerenaAgent, symbol_name: str, def_file: str, ref_file: str) -> None: - # skip flaky tests in CI - # TODO: Revisit the flaky tests and re-enable once the LS issues are resolved #1040 - flaky_languages = {Language.TYPESCRIPT} - if set(serena_agent.get_active_lsp_languages()).intersection(flaky_languages) and is_ci: - pytest.skip("Test is flaky and thus skipped in CI environment.") + def test_find_symbol_stable(self, serena_agent: SerenaAgent, symbol_name: str, expected_kind: str, expected_file: str) -> None: + self._assert_find_symbol(serena_agent, symbol_name, expected_kind, expected_file) + @pytest.mark.parametrize( + "serena_agent,symbol_name,expected_kind,expected_file", + [ + pytest.param(Language.FSHARP, "Calculator", "Module", "Calculator.fs", marks=pytest.mark.fsharp), + ], + indirect=["serena_agent"], + ) + @pytest.mark.xfail(reason="F# language server is unreliable") # See issue #1040 + def test_find_symbol_fsharp(self, serena_agent: SerenaAgent, symbol_name: str, expected_kind: str, expected_file: str) -> None: + self._assert_find_symbol(serena_agent, symbol_name, expected_kind, expected_file) + + @pytest.mark.parametrize( + "serena_agent,symbol_name,expected_kind,expected_file", + [ + pytest.param(Language.RUST, "add", "Function", "lib.rs", marks=pytest.mark.rust), + ], + indirect=["serena_agent"], + ) + @pytest.mark.xfail(reason="Rust language server is unreliable") # See issue #1040 + def test_find_symbol_rust(self, serena_agent: SerenaAgent, symbol_name: str, expected_kind: str, expected_file: str) -> None: + self._assert_find_symbol(serena_agent, symbol_name, expected_kind, expected_file) + + def _assert_find_symbol_references(self, serena_agent: SerenaAgent, symbol_name: str, def_file: str, ref_file: str) -> None: agent = serena_agent # Find the symbol location first @@ -257,6 +226,71 @@ class TestSerenaAgent: refs = json.loads(result) assert contains_ref_with_relative_path(refs, ref_file), f"Expected to find reference to {symbol_name} in {ref_file}. refs={refs}" + @pytest.mark.parametrize( + "serena_agent,symbol_name,def_file,ref_file", + [ + pytest.param( + Language.PYTHON, + "User", + os.path.join("test_repo", "models.py"), + os.path.join("test_repo", "services.py"), + marks=pytest.mark.python, + ), + pytest.param(Language.GO, "Helper", "main.go", "main.go", marks=pytest.mark.go), + pytest.param( + Language.JAVA, + "Model", + os.path.join("src", "main", "java", "test_repo", "Model.java"), + os.path.join("src", "main", "java", "test_repo", "Main.java"), + marks=pytest.mark.java, + ), + pytest.param( + Language.KOTLIN, + "Model", + os.path.join("src", "main", "kotlin", "test_repo", "Model.kt"), + os.path.join("src", "main", "kotlin", "test_repo", "Main.kt"), + marks=[pytest.mark.kotlin] + ([pytest.mark.skip(reason="Kotlin LSP JVM crashes on restart in CI")] if is_ci else []), + ), + pytest.param(Language.RUST, "add", os.path.join("src", "lib.rs"), os.path.join("src", "main.rs"), marks=pytest.mark.rust), + pytest.param(Language.PHP, "helperFunction", "helper.php", "index.php", marks=pytest.mark.php), + pytest.param( + Language.CLOJURE, + "multiply", + clj.CORE_PATH, + clj.UTILS_PATH, + marks=pytest.mark.clojure, + ), + pytest.param(Language.CSHARP, "Calculator", "Program.cs", "Program.cs", marks=pytest.mark.csharp), + pytest.param(Language.POWERSHELL, "function Greet-User ()", "main.ps1", "main.ps1", marks=pytest.mark.powershell), + pytest.param(Language.CPP_CCLS, "add", "b.cpp", "a.cpp", marks=pytest.mark.cpp), + ], + indirect=["serena_agent"], + ) + def test_find_symbol_references_stable(self, serena_agent: SerenaAgent, symbol_name: str, def_file: str, ref_file: str) -> None: + self._assert_find_symbol_references(serena_agent, symbol_name, def_file, ref_file) + + @pytest.mark.parametrize( + "serena_agent,symbol_name,def_file,ref_file", + [ + pytest.param(Language.TYPESCRIPT, "helperFunction", "index.ts", "use_helper.ts", marks=pytest.mark.typescript), + ], + indirect=["serena_agent"], + ) + @pytest.mark.xfail(reason="TypeScript language server is unreliable") # See issue #1040 + def test_find_symbol_references_typescript(self, serena_agent: SerenaAgent, symbol_name: str, def_file: str, ref_file: str) -> None: + self._assert_find_symbol_references(serena_agent, symbol_name, def_file, ref_file) + + @pytest.mark.parametrize( + "serena_agent,symbol_name,def_file,ref_file", + [ + pytest.param(Language.FSHARP, "add", "Calculator.fs", "Program.fs", marks=pytest.mark.fsharp), + ], + indirect=["serena_agent"], + ) + @pytest.mark.xfail(reason="F# language server is unreliable") # See issue #1040 + def test_find_symbol_references_fsharp(self, serena_agent: SerenaAgent, symbol_name: str, def_file: str, ref_file: str) -> None: + self._assert_find_symbol_references(serena_agent, symbol_name, def_file, ref_file) + @pytest.mark.parametrize( "serena_agent,name_path,substring_matching,expected_symbol_name,expected_kind,expected_file", [