From 5333ea809f7d1748bdc2fab6f08696ab5a864439 Mon Sep 17 00:00:00 2001 From: Dominik Jain Date: Sun, 1 Mar 2026 01:04:09 +0100 Subject: [PATCH 01/24] Fix active modes not necessarily being set in activate_project_from_path_or_name during init Ref #1107 --- src/serena/agent.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/src/serena/agent.py b/src/serena/agent.py index d48edbdf..32daf0ef 100644 --- a/src/serena/agent.py +++ b/src/serena/agent.py @@ -343,11 +343,10 @@ class SerenaAgent: self._mode_overrides = modes if project is not None: try: - self.activate_project_from_path_or_name(project, update_active_modes=True, update_active_tools=False) + self.activate_project_from_path_or_name(project, update_active_modes=False, update_active_tools=False) except Exception as e: log.error(f"Error activating project '{project}' at startup: {e}", exc_info=e) - else: - self._update_active_modes() + self._update_active_modes() # determine the base toolset defining the set of exposed tools (which e.g. the MCP shall see), self._base_toolset = self._create_base_toolset( From 67d8173f1e71caccdd2dafb5ba4cbdbe4235563f Mon Sep 17 00:00:00 2001 From: Dominik Jain Date: Sun, 1 Mar 2026 01:19:14 +0100 Subject: [PATCH 02/24] Add test for agent instantiation, including regression test for #1107 --- test/serena/test_serena_agent.py | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/test/serena/test_serena_agent.py b/test/serena/test_serena_agent.py index 8d8a5193..3dbef5d8 100644 --- a/test/serena/test_serena_agent.py +++ b/test/serena/test_serena_agent.py @@ -105,6 +105,18 @@ def serena_agent(request: pytest.FixtureRequest, serena_config) -> Iterator[Sere class TestSerenaAgent: + @pytest.mark.parametrize("project", [None, str(get_repo_path(Language.PYTHON)), "non_existent_path"]) + def test_agent_instantiation(self, project: str | None): + """ + Tests agent instantiation for cases where + * no project is specified at startup + * a valid project path is specified at startup + * an invalid project path is specified at startup + All cases must not raise an exception. + """ + serena_config = SerenaConfig(gui_log_window=False, web_dashboard=False) + SerenaAgent(project=project, serena_config=serena_config) + @pytest.mark.parametrize( "serena_agent,symbol_name,expected_kind,expected_file", [ From 8375da7dc9f9dfd8221c68009e0ba79535fc5fdd Mon Sep 17 00:00:00 2001 From: Dominik Jain Date: Sun, 1 Mar 2026 13:05:49 +0100 Subject: [PATCH 03/24] Rename _cache_context_fingerprint -> _document_symbols_cache_fingerprint Also improve docstring to make clear what the purpose is --- src/solidlsp/language_servers/gopls.py | 2 +- src/solidlsp/ls.py | 20 ++++++++++++++------ 2 files changed, 15 insertions(+), 7 deletions(-) diff --git a/src/solidlsp/language_servers/gopls.py b/src/solidlsp/language_servers/gopls.py index 3575a00b..40768842 100644 --- a/src/solidlsp/language_servers/gopls.py +++ b/src/solidlsp/language_servers/gopls.py @@ -165,7 +165,7 @@ class Gopls(SolidLanguageServer): _CACHE_CONTEXT_ENV_KEYS = ("GOFLAGS", "GOOS", "GOARCH", "CGO_ENABLED") @override - def _cache_context_fingerprint(self) -> str | None: + def _document_symbols_cache_fingerprint(self) -> str | None: """ Compute a deterministic fingerprint of the Go build context. diff --git a/src/solidlsp/ls.py b/src/solidlsp/ls.py index 8f5b8841..25208f06 100644 --- a/src/solidlsp/ls.py +++ b/src/solidlsp/ls.py @@ -1987,14 +1987,22 @@ class SolidLanguageServer(ABC): return defining_symbol - def _cache_context_fingerprint(self) -> Hashable | None: + def _document_symbols_cache_fingerprint(self) -> Hashable | None: """ - Return a fingerprint of any language-server-specific context that affects cached results. + Returns a fingerprint of any language server-specific aspects that result in changes + to the high-level document symbol information. - Subclasses may override to provide a deterministic value that changes when cached results - would be invalidated (e.g., build flags, environment variables). + Language servers must implement this method/change the return value + * whenever they change the `request_document_symbols` implementation to modify the returned content + * are reconfigured in a way that affects the returned contents (e.g. context-specific configuration + such as build flags or environment variables); configuration options can, in such cases, be + hashed together to produce a single fingerprint value. + + Whenever the value changes, the document symbols cache will be invalidated and re-populated. The value must be hashable and safe for inclusion in cache version tuples. + E.g. use an integer, a string or a tuple of integers/strings. + Returns None if no context-specific fingerprint is needed. """ return None @@ -2005,7 +2013,7 @@ class SolidLanguageServer(ABC): Incorporates cache context fingerprint if provided by the language server. """ - fingerprint = self._cache_context_fingerprint() + fingerprint = self._document_symbols_cache_fingerprint() if fingerprint is not None: return (self.DOCUMENT_SYMBOL_CACHE_VERSION, fingerprint) return self.DOCUMENT_SYMBOL_CACHE_VERSION @@ -2030,7 +2038,7 @@ class SolidLanguageServer(ABC): def _raw_document_symbols_cache_version(self) -> tuple[Hashable, ...]: base_version: tuple[Hashable, ...] = (self.RAW_DOCUMENT_SYMBOLS_CACHE_VERSION, self._ls_specific_raw_document_symbols_cache_version) - fingerprint = self._cache_context_fingerprint() + fingerprint = self._document_symbols_cache_fingerprint() if fingerprint is not None: return (*base_version, fingerprint) return base_version From 003a9e2a6d4495804a3c48aeab8a2f0241c87067 Mon Sep 17 00:00:00 2001 From: Terrance Orletsky Date: Thu, 26 Feb 2026 10:54:10 -0500 Subject: [PATCH 04/24] Fix markdown heading symbol recognition in Marksman LSP Marksman returns all headings (h1-h6) with SymbolKind.String (15), which causes them to be treated as low-level symbols and filtered out of get_symbols_overview results. Override request_document_symbols in Marksman to remap SymbolKind.String to SymbolKind.Namespace, which is semantically appropriate for headings (named sections containing other content) and ensures they appear in symbol overviews. --- src/solidlsp/language_servers/marksman.py | 32 ++++++++++++++- test/solidlsp/markdown/test_markdown_basic.py | 40 ++++++++++++++++--- 2 files changed, 66 insertions(+), 6 deletions(-) diff --git a/src/solidlsp/language_servers/marksman.py b/src/solidlsp/language_servers/marksman.py index 14805f21..c2288ed0 100644 --- a/src/solidlsp/language_servers/marksman.py +++ b/src/solidlsp/language_servers/marksman.py @@ -9,8 +9,15 @@ import pathlib from overrides import override -from solidlsp.ls import LanguageServerDependencyProvider, LanguageServerDependencyProviderSinglePath, SolidLanguageServer +from solidlsp.ls import ( + DocumentSymbols, + LanguageServerDependencyProvider, + LanguageServerDependencyProviderSinglePath, + LSPFileBuffer, + SolidLanguageServer, +) from solidlsp.ls_config import LanguageServerConfig +from solidlsp.ls_types import SymbolKind, UnifiedSymbolInformation from solidlsp.lsp_protocol_handler.lsp_types import InitializeParams from solidlsp.settings import SolidLSPSettings @@ -106,6 +113,29 @@ class Marksman(SolidLanguageServer): def is_ignored_dirname(self, dirname: str) -> bool: return super().is_ignored_dirname(dirname) or dirname in ["node_modules", ".obsidian", ".vitepress", ".vuepress"] + @override + def request_document_symbols(self, relative_file_path: str, file_buffer: LSPFileBuffer | None = None) -> DocumentSymbols: + """Override to remap Marksman's heading symbol kinds from String to Namespace. + + Marksman LSP returns all markdown headings (h1-h6) with SymbolKind.String (15). + This is problematic because String (15) >= Variable (13), so headings are + classified as "low-level" and filtered out of symbol overviews. + Remapping to Namespace (3) fixes this and is semantically appropriate + (headings are named sections containing other content). + """ + document_symbols = super().request_document_symbols(relative_file_path, file_buffer=file_buffer) + + def remap_heading_kinds(symbol: UnifiedSymbolInformation) -> None: + if symbol["kind"] == SymbolKind.String: + symbol["kind"] = SymbolKind.Namespace + for child in symbol.get("children", []): + remap_heading_kinds(child) + + for sym in document_symbols.root_symbols: + remap_heading_kinds(sym) + + return document_symbols + @staticmethod def _get_initialize_params(repository_absolute_path: str) -> InitializeParams: """ diff --git a/test/solidlsp/markdown/test_markdown_basic.py b/test/solidlsp/markdown/test_markdown_basic.py index 81a988c0..e8f96c5a 100644 --- a/test/solidlsp/markdown/test_markdown_basic.py +++ b/test/solidlsp/markdown/test_markdown_basic.py @@ -7,8 +7,10 @@ like request_document_symbols using the markdown test repository. import pytest +from serena.symbol import LanguageServerSymbol from solidlsp import SolidLanguageServer from solidlsp.ls_config import Language +from solidlsp.ls_types import SymbolKind @pytest.mark.markdown @@ -24,17 +26,19 @@ class TestMarkdownLanguageServerBasics: @pytest.mark.parametrize("language_server", [Language.MARKDOWN], indirect=True) def test_markdown_request_document_symbols(self, language_server: SolidLanguageServer) -> None: """Test request_document_symbols for markdown files.""" - # Test getting symbols from README.md all_symbols, _root_symbols = language_server.request_document_symbols("README.md").get_all_symbols_and_roots() - # Extract heading symbols (LSP Symbol Kind 15 is String, but marksman uses kind 15 for headings) - # Note: Different markdown LSPs may use different symbol kinds for headings - # Marksman typically uses kind 15 (String) for markdown headings heading_names = [symbol["name"] for symbol in all_symbols] # Should detect headings from README.md assert "Test Repository" in heading_names or len(all_symbols) > 0, "Should find at least one heading" + # Verify that markdown headings are remapped from String to Namespace + for symbol in all_symbols: + assert ( + symbol["kind"] == SymbolKind.Namespace + ), f"Heading '{symbol['name']}' should have kind Namespace, got {SymbolKind(symbol['kind']).name}" + @pytest.mark.parametrize("language_server", [Language.MARKDOWN], indirect=True) def test_markdown_request_symbols_from_guide(self, language_server: SolidLanguageServer) -> None: """Test symbol detection in guide.md file.""" @@ -54,7 +58,6 @@ class TestMarkdownLanguageServerBasics: @pytest.mark.parametrize("language_server", [Language.MARKDOWN], indirect=True) def test_markdown_request_document_symbols_with_body(self, language_server: SolidLanguageServer) -> None: """Test request_document_symbols with body extraction.""" - # Test with include_body=True all_symbols, _root_symbols = language_server.request_document_symbols("README.md").get_all_symbols_and_roots() # Should have found some symbols @@ -63,3 +66,30 @@ class TestMarkdownLanguageServerBasics: # Note: Not all markdown LSPs provide body information for symbols # This test is more lenient and just verifies the API works assert all_symbols is not None, "Should return symbols even if body extraction is limited" + + @pytest.mark.parametrize("language_server", [Language.MARKDOWN], indirect=True) + def test_markdown_headings_not_low_level(self, language_server: SolidLanguageServer) -> None: + """Test that markdown headings are not classified as low-level symbols. + + Verifies the fix for the issue where Marksman's SymbolKind.String (15) + caused all headings to be filtered out of get_symbols_overview. + """ + all_symbols, _root_symbols = language_server.request_document_symbols("README.md").get_all_symbols_and_roots() + assert len(all_symbols) > 0, "Should find headings in README.md" + + for symbol in all_symbols: + ls_symbol = LanguageServerSymbol(symbol) + assert ( + not ls_symbol.is_low_level() + ), f"Heading '{symbol['name']}' should not be low-level (kind={SymbolKind(symbol['kind']).name})" + + @pytest.mark.parametrize("language_server", [Language.MARKDOWN], indirect=True) + def test_markdown_nested_headings_remapped(self, language_server: SolidLanguageServer) -> None: + """Test that nested headings (h1-h5) are all remapped from String to Namespace.""" + all_symbols, _root_symbols = language_server.request_document_symbols("api.md").get_all_symbols_and_roots() + + # api.md has deeply nested headings (h1 through h5) + assert len(all_symbols) > 5, "api.md should have many headings" + + for symbol in all_symbols: + assert symbol["kind"] == SymbolKind.Namespace, f"Nested heading '{symbol['name']}' should be remapped to Namespace" From 7ac79bae4159b6fe98f2d5117217bac7897fd9b7 Mon Sep 17 00:00:00 2001 From: Dominik Jain Date: Sun, 1 Mar 2026 14:02:46 +0100 Subject: [PATCH 05/24] Marksman: Set document symbols cache fingerprint to account for override --- src/solidlsp/language_servers/marksman.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/src/solidlsp/language_servers/marksman.py b/src/solidlsp/language_servers/marksman.py index c2288ed0..c5d039e3 100644 --- a/src/solidlsp/language_servers/marksman.py +++ b/src/solidlsp/language_servers/marksman.py @@ -6,6 +6,7 @@ Contains various configurations and settings specific to Markdown. import logging import os import pathlib +from collections.abc import Hashable from overrides import override @@ -113,6 +114,10 @@ class Marksman(SolidLanguageServer): def is_ignored_dirname(self, dirname: str) -> bool: return super().is_ignored_dirname(dirname) or dirname in ["node_modules", ".obsidian", ".vitepress", ".vuepress"] + def _document_symbols_cache_fingerprint(self) -> Hashable | None: + request_document_symbols_override_version = 1 + return request_document_symbols_override_version + @override def request_document_symbols(self, relative_file_path: str, file_buffer: LSPFileBuffer | None = None) -> DocumentSymbols: """Override to remap Marksman's heading symbol kinds from String to Namespace. @@ -125,6 +130,8 @@ class Marksman(SolidLanguageServer): """ document_symbols = super().request_document_symbols(relative_file_path, file_buffer=file_buffer) + # NOTE: When changing this method, also update the cache fingerprint method above + def remap_heading_kinds(symbol: UnifiedSymbolInformation) -> None: if symbol["kind"] == SymbolKind.String: symbol["kind"] = SymbolKind.Namespace From 827512b4d1ed457c70efbb775b5cc928304b70c2 Mon Sep 17 00:00:00 2001 From: Dominik Jain Date: Sun, 1 Mar 2026 18:05:32 +0100 Subject: [PATCH 06/24] Improve error reporting for LSP backend issues (LS initialisation) * Include concrete information on failing language servers * Explain how to access logs to get detailed information Resolves #1112 --- src/serena/agent.py | 43 ++++++++++++++++++++++-------- src/serena/config/serena_config.py | 8 +++++- src/serena/ls_manager.py | 9 +++++-- 3 files changed, 46 insertions(+), 14 deletions(-) diff --git a/src/serena/agent.py b/src/serena/agent.py index 32daf0ef..83f9694b 100644 --- a/src/serena/agent.py +++ b/src/serena/agent.py @@ -12,6 +12,7 @@ from typing import TYPE_CHECKING, Optional, TypeVar from sensai.util import logging from sensai.util.logging import LogTime +from sensai.util.string import TextBuilder from interprompt.jinja_template import JinjaTemplate from serena import serena_version @@ -23,10 +24,11 @@ from serena.config.serena_config import ( NamedToolInclusionDefinition, RegisteredProject, SerenaConfig, + SerenaPaths, ToolInclusionDefinition, ) from serena.dashboard import SerenaDashboardAPI -from serena.ls_manager import LanguageServerManager +from serena.ls_manager import LanguageServerManager, LanguageServerManagerInitialisationError from serena.project import Project from serena.prompt_factory import SerenaPromptFactory from serena.task_executor import TaskExecutor @@ -252,6 +254,7 @@ class SerenaAgent: # project-specific instances, which will be initialized upon project activation self._active_project: Project | None = None + self._lsp_init_error: LanguageServerManagerInitialisationError | None = None # determine registered project to be activated (if any) registered_project_to_activate: RegisteredProject | None = ( @@ -473,13 +476,26 @@ class SerenaAgent: def get_language_server_manager_or_raise(self) -> LanguageServerManager: language_server_manager = self.get_language_server_manager() if language_server_manager is None: - raise Exception( - "The language server manager is not initialized, indicating a problem during project activation. " - "Inform the user, telling them to inspect Serena's logs in order to determine the issue. " - "IMPORTANT: Wait for further instructions before you continue!" + msg = TextBuilder("The language server manager is not initialized, indicating a problem during project initialisation.") + if self._lsp_init_error is not None: + msg.with_text(str(self._lsp_init_error)) + msg.with_text("For details, please check the logs. " + self.get_log_inspection_instructions()) + msg.with_text( + "IMPORTANT: Stop, do not attempt workarounds. Inform the user and wait for further instructions before you continue!" ) + raise Exception(msg.build()) return language_server_manager + def get_log_inspection_instructions(self) -> str: + if self.serena_config.web_dashboard: + return f"Live logs can be inspected via the dashboard at {self.get_dashboard_url()}" + else: + log_path = SerenaPaths().last_returned_log_file_path + if log_path is not None: + return f"Find the current log file here: f{log_path}" + else: + return "Unfortunately, logs are not available. We recommend enabling the web dashboard/logging in general." + def get_context(self) -> SerenaAgentContext: return self._context @@ -816,12 +832,17 @@ class SerenaAgent: ls_timeout = tool_timeout - 5 # the LS timeout is for a single call, it should be smaller than the tool timeout # instantiate and start the necessary language servers - self.get_active_project_or_raise().create_language_server_manager( - log_level=self.serena_config.log_level, - ls_timeout=ls_timeout, - trace_lsp_communication=self.serena_config.trace_lsp_communication, - ls_specific_settings=self.serena_config.ls_specific_settings, - ) + try: + self._lsp_init_error = None + self.get_active_project_or_raise().create_language_server_manager( + log_level=self.serena_config.log_level, + ls_timeout=ls_timeout, + trace_lsp_communication=self.serena_config.trace_lsp_communication, + ls_specific_settings=self.serena_config.ls_specific_settings, + ) + except LanguageServerManagerInitialisationError as e: + self._lsp_init_error = e + raise def add_language(self, language: Language) -> None: """ diff --git a/src/serena/config/serena_config.py b/src/serena/config/serena_config.py index b286c63b..432e8a65 100644 --- a/src/serena/config/serena_config.py +++ b/src/serena/config/serena_config.py @@ -85,6 +85,11 @@ class SerenaPaths: """ file containing the ID of the last read news snippet """ + self.last_returned_log_file_path: str | None = None + """ + the path to the last log file returned by `get_next_log_file_path`. If this is not None, the logs + are currently being written to this file + """ def get_next_log_file_path(self, prefix: str) -> str: """ @@ -93,7 +98,8 @@ class SerenaPaths: """ log_dir = os.path.join(self.serena_user_home_dir, "logs", datetime.now().strftime("%Y-%m-%d")) os.makedirs(log_dir, exist_ok=True) - return os.path.join(log_dir, prefix + "_" + datetime_tag() + ".txt") + self.last_returned_log_file_path = os.path.join(log_dir, prefix + "_" + datetime_tag() + ".txt") + return self.last_returned_log_file_path # TODO: Paths from constants.py should be moved here diff --git a/src/serena/ls_manager.py b/src/serena/ls_manager.py index 1cb8f3b5..dde1f428 100644 --- a/src/serena/ls_manager.py +++ b/src/serena/ls_manager.py @@ -14,6 +14,11 @@ from solidlsp.settings import SolidLSPSettings log = logging.getLogger(__name__) +class LanguageServerManagerInitialisationError(Exception): + def __init__(self, message: str): + super().__init__(message) + + class LanguageServerFactory: def __init__( self, @@ -122,14 +127,14 @@ class LanguageServerManager: # If any server failed to start up, raise an exception and stop all started language servers. # We intentionally fail fast here. The user's intention is to work with all the specified languages, - # so if any of them is not available, it is better to fail immediately and bring the failure to the + # so if any of them is not available, it is better to make symbolic tool calls fail, bringing the issue to the # user's attention instead of silently continuing with a subset of the language servers and potentially # causing suboptimal agent behaviour. if exceptions: for ls in language_servers.values(): ls.stop() failure_messages = "\n".join([f"{lang.value}: {e}" for lang, e in exceptions.items()]) - raise Exception(f"Failed to start language servers:\n{failure_messages}") + raise LanguageServerManagerInitialisationError(f"Failed to start {len(exceptions)} language server(s):\n{failure_messages}") return LanguageServerManager(language_servers, factory) From 9cd33aa503531da4f986ccb56e3c9da069b9e4b6 Mon Sep 17 00:00:00 2001 From: Dominik Jain Date: Mon, 2 Mar 2026 00:23:43 +0100 Subject: [PATCH 07/24] Fully skip Phpactor tests on Windows (can cause hangs) --- test/solidlsp/php/test_php_basic.py | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/test/solidlsp/php/test_php_basic.py b/test/solidlsp/php/test_php_basic.py index 6b3e7882..eea00e83 100644 --- a/test/solidlsp/php/test_php_basic.py +++ b/test/solidlsp/php/test_php_basic.py @@ -8,12 +8,10 @@ from test.conftest import is_ci, is_windows, language_tests_enabled _php_servers: list[Language] = [Language.PHP] if language_tests_enabled(Language.PHP_PHPACTOR): - _php_servers.append(Language.PHP_PHPACTOR) + if not (is_windows and is_ci): # TODO: Phpactor tests are flaky in Windows CI and can even cause hangs #1040 + _php_servers.append(Language.PHP_PHPACTOR) -@pytest.mark.xfail( - is_ci and is_windows, reason="Tests are flaky" -) # TODO: Re-enable once we have a solution for running Phpactor tests on Windows CI #1040 @pytest.mark.php class TestPhpLanguageServers: @pytest.mark.parametrize("language_server", _php_servers, indirect=True) @@ -26,7 +24,6 @@ class TestPhpLanguageServers: @pytest.mark.parametrize("language_server", _php_servers, indirect=True) @pytest.mark.parametrize("repo_path", [Language.PHP], indirect=True) def test_find_definition_within_file(self, language_server: SolidLanguageServer, repo_path: Path) -> None: - # In index.php: # Line 9 (1-indexed): $greeting = greet($userName); # Line 11 (1-indexed): echo $greeting; From 9800d4e98b5977a5d56c90963df65ea9a772799f Mon Sep 17 00:00:00 2001 From: Dominik Jain Date: Mon, 2 Mar 2026 15:43:01 +0100 Subject: [PATCH 08/24] Introduce DotNETUtil * Apply in FSharpLanguageServer and CSharpLanguageServer * CSharpLanguageServer: Disabled auto-install of .NET (does not work reliably) and remove corresponding test * Remove TestFSharpLanguageServerSetup --- src/serena/util/dotnet.py | 156 ++++++++++ src/serena/util/version.py | 2 +- .../csharp_language_server.py | 101 +----- .../fsharp_language_server.py | 17 +- .../csharp/test_csharp_dotnet_install.py | 290 ------------------ test/solidlsp/fsharp/test_fsharp_basic.py | 42 --- 6 files changed, 161 insertions(+), 447 deletions(-) create mode 100644 src/serena/util/dotnet.py delete mode 100644 test/solidlsp/csharp/test_csharp_dotnet_install.py diff --git a/src/serena/util/dotnet.py b/src/serena/util/dotnet.py new file mode 100644 index 00000000..99b34526 --- /dev/null +++ b/src/serena/util/dotnet.py @@ -0,0 +1,156 @@ +import logging +import platform +import re +import shutil +import subprocess +import urllib +from pathlib import Path + +from serena.util.version import Version +from solidlsp.ls_exceptions import SolidLSPException + +log = logging.getLogger(__name__) + + +class DotNETUtil: + def __init__(self, required_version: str, allow_higher_version: bool = True): + """ + :param required_version: the required .NET runtime version specified as a string (e.g. "10.0" for .NET 10.0) + :param allow_higher_version: whether to allow higher versions than the required version + """ + self._system_dotnet = shutil.which("dotnet") + self._required_version_str = required_version + self._required_version_components = [int(c) for c in required_version.split(".")] + self._allow_higher_version = allow_higher_version + self._installed_versions = self._determine_installed_versions() + + def _determine_installed_versions(self) -> list[Version]: + if self._system_dotnet: + try: + result = subprocess.run([self._system_dotnet, "--list-runtimes"], capture_output=True, text=True, check=True) + version_strings = re.findall(r"Microsoft.NETCore.App\s+([^\s]+)", result.stdout) + log.info("Installed .NET runtime versions: %s", version_strings) + return [Version(v) for v in version_strings] + except: + log.warning("Failed to run 'dotnet --list-runtimes' to check .NET version; assuming no installed .NET versions") + return [] + else: + log.info("Found no `dotnet` on system PATH; assuming no installed .NET versions") + return [] + + def is_required_version_available(self) -> bool: + """ + Checks whether the required .NET runtime version is installed and raises an exception if not. + + :param required_version_components: the required .NET runtime version specified as a list of integers representing the version components (e.g., [6, 1] for .NET 6.1) + :param allow_higher_version: whether to allow higher versions than the required version (e.g., if True, .NET 7.0 would satisfy a requirement of .NET 6.1) + """ + required_version_str = ".".join(str(c) for c in self._required_version_components) + for v in self._installed_versions: + if self._allow_higher_version: + if v.is_at_least(*self._required_version_components): + log.info(f"Found installed .NET runtime version {v} which satisfies requirement of {required_version_str} or higher") + return True + else: + if v.is_equal(*self._required_version_components): + log.info(f"Found installed .NET runtime version {v} which satisfies requirement of {required_version_str}") + return True + return False + + def get_dotnet_path_or_raise(self) -> str: + """ + Returns the path to the dotnet executable if the required .NET runtime version is available, otherwise raises an exception. + """ + if not self.is_required_version_available(): + raise SolidLSPException( + f"Required .NET runtime version {self._required_version_str} not found " + f"(installed versions: {self._installed_versions}). " + "Please install the required .NET runtime version from https://dotnet.microsoft.com/en-us/download/dotnet " + "and ensure that `dotnet` is on the system PATH." + ) + assert self._system_dotnet is not None + return self._system_dotnet + + @staticmethod + def install_dotnet_with_script(version: str, base_path: str) -> str: + """ + Install .NET runtime using Microsoft's official installation script. + + NOTE: This method is unreliable and therefore currently unused. It is kept for reference. + + :version: the version to install as a string (e.g. "10.0") + :return: the path to the dotnet executable. + """ + dotnet_dir = Path(base_path) / f"dotnet-runtime-{version}" + + # Determine binary name based on platform + is_windows = platform.system().lower() == "windows" + dotnet_exe = dotnet_dir / ("dotnet.exe" if is_windows else "dotnet") + + if dotnet_exe.exists(): + log.info(f"Using cached .NET {version} runtime from {dotnet_exe}") + return str(dotnet_exe) + + # Download and run install script + log.info(f"Installing .NET {version} runtime using official Microsoft install script...") + dotnet_dir.mkdir(parents=True, exist_ok=True) + + try: + if is_windows: + # PowerShell script for Windows + script_url = "https://dot.net/v1/dotnet-install.ps1" + script_path = dotnet_dir / "dotnet-install.ps1" + urllib.request.urlretrieve(script_url, script_path) + + cmd = [ + "pwsh", + "-NoProfile", + "-ExecutionPolicy", + "Bypass", + "-File", + str(script_path), + "-Version", + version, + "-InstallDir", + str(dotnet_dir), + "-Runtime", + "dotnet", + "-NoPath", + ] + else: + # Bash script for Linux/macOS + script_url = "https://dot.net/v1/dotnet-install.sh" + script_path = dotnet_dir / "dotnet-install.sh" + urllib.request.urlretrieve(script_url, script_path) + script_path.chmod(0o755) + + cmd = [ + "bash", + str(script_path), + "--version", + version, + "--install-dir", + str(dotnet_dir), + "--runtime", + "dotnet", + "--no-path", + ] + + # Run the install script + log.info("Running .NET install script: %s", cmd) + result = subprocess.run(cmd, capture_output=True, text=True, check=True) + log.debug(f"Install script output: {result.stdout}") + + if not dotnet_exe.exists(): + raise SolidLSPException(f"dotnet executable not found at {dotnet_exe} after installation") + + log.info(f"Successfully installed .NET {version} runtime to {dotnet_exe}") + return str(dotnet_exe) + + except subprocess.CalledProcessError as e: + raise SolidLSPException(f"Failed to install .NET {version} runtime using install script: {e.stderr if e.stderr else e}") from e + except Exception as e: + message = f"Failed to install .NET {version} runtime: {e}" + if is_windows and isinstance(e, FileNotFoundError): + message += "; pwsh, i.e. PowerShell 7+, is required to install .NET runtime. Make sure pwsh is available on your system." + raise SolidLSPException(message) from e diff --git a/src/serena/util/version.py b/src/serena/util/version.py index d8d8381f..142d4356 100644 --- a/src/serena/util/version.py +++ b/src/serena/util/version.py @@ -23,7 +23,7 @@ class Version: self.version_string = version_string self.components = self._get_version_components(version_string) - def __str__(self) -> str: + def __repr__(self) -> str: return self.version_string @staticmethod diff --git a/src/solidlsp/language_servers/csharp_language_server.py b/src/solidlsp/language_servers/csharp_language_server.py index ae9d9264..28621bb2 100644 --- a/src/solidlsp/language_servers/csharp_language_server.py +++ b/src/solidlsp/language_servers/csharp_language_server.py @@ -6,7 +6,6 @@ import logging import os import platform import shutil -import subprocess import threading import urllib.request from collections.abc import Iterable @@ -15,6 +14,7 @@ from typing import Any, cast from overrides import override +from serena.util.dotnet import DotNETUtil from solidlsp.ls import DocumentSymbols, LanguageServerDependencyProvider, LSPFileBuffer, SolidLanguageServer from solidlsp.ls_config import LanguageServerConfig from solidlsp.ls_exceptions import SolidLSPException @@ -366,21 +366,7 @@ class CSharpLanguageServer(SolidLanguageServer): def _ensure_dotnet_runtime(self) -> str: """Ensure .NET runtime is available and return the dotnet executable path.""" - # Check if dotnet is already available on the system - system_dotnet = shutil.which("dotnet") - if system_dotnet: - # Check if it's .NET 10 or compatible - try: - result = subprocess.run([system_dotnet, "--list-runtimes"], capture_output=True, text=True, check=True) - # Accept .NET 10 or higher (10.x, 11.x, etc.) - if any(f"Microsoft.NETCore.App {v}." in result.stdout for v in range(10, 20)): - log.info("Found system .NET 10+ runtime") - return system_dotnet - except subprocess.CalledProcessError: - pass - - # Install .NET 10 runtime using Microsoft's install script - return self._install_dotnet_with_script() + return DotNETUtil("10.0", allow_higher_version=True).get_dotnet_path_or_raise() def _ensure_language_server(self, lang_server_dep: RuntimeDependency) -> str: """Ensure language server is available and return the DLL path.""" @@ -475,89 +461,6 @@ class CSharpLanguageServer(SolidLanguageServer): except Exception as e: raise SolidLSPException(f"Failed to download package {package_name} version {package_version} from NuGet.org: {e}") from e - def _install_dotnet_with_script(self, version: str = "10.0") -> str: - """ - Install .NET runtime using Microsoft's official install script. - Returns the path to the dotnet executable. - """ - dotnet_dir = Path(self._ls_resources_dir) / f"dotnet-runtime-{version}" - - # Determine binary name based on platform - is_windows = platform.system().lower() == "windows" - dotnet_exe = dotnet_dir / ("dotnet.exe" if is_windows else "dotnet") - - if dotnet_exe.exists(): - log.info(f"Using cached .NET {version} runtime from {dotnet_exe}") - return str(dotnet_exe) - - # Download and run install script - log.info(f"Installing .NET {version} runtime using official Microsoft install script...") - dotnet_dir.mkdir(parents=True, exist_ok=True) - - try: - if is_windows: - # PowerShell script for Windows - script_url = "https://dot.net/v1/dotnet-install.ps1" - script_path = dotnet_dir / "dotnet-install.ps1" - urllib.request.urlretrieve(script_url, script_path) - - cmd = [ - "pwsh", - "-NoProfile", - "-ExecutionPolicy", - "Bypass", - "-File", - str(script_path), - "-Version", - version, - "-InstallDir", - str(dotnet_dir), - "-Runtime", - "dotnet", - "-NoPath", - ] - else: - # Bash script for Linux/macOS - script_url = "https://dot.net/v1/dotnet-install.sh" - script_path = dotnet_dir / "dotnet-install.sh" - urllib.request.urlretrieve(script_url, script_path) - script_path.chmod(0o755) - - cmd = [ - "bash", - str(script_path), - "--version", - version, - "--install-dir", - str(dotnet_dir), - "--runtime", - "dotnet", - "--no-path", - ] - - # Run the install script - log.info("Running .NET install script: %s", cmd) - result = subprocess.run(cmd, capture_output=True, text=True, check=True) - log.debug(f"Install script output: {result.stdout}") - - if not dotnet_exe.exists(): - raise SolidLSPException(f"dotnet executable not found at {dotnet_exe} after installation") - - log.info(f"Successfully installed .NET {version} runtime to {dotnet_exe}") - return str(dotnet_exe) - - except subprocess.CalledProcessError as e: - raise SolidLSPException( - f"Failed to install .NET {version} runtime using install script: {e.stderr if e.stderr else e}" - ) from e - except Exception as e: - message = f"Failed to install .NET {version} runtime: {e}" - if is_windows and isinstance(e, FileNotFoundError): - message += ( - "; pwsh, i.e. PowerShell 7+, is required to install .NET runtime. Make sure pwsh is available on your system." - ) - raise SolidLSPException(message) from e - def _get_initialize_params(self) -> InitializeParams: """ Returns the initialize params for the Microsoft.CodeAnalysis.LanguageServer. diff --git a/src/solidlsp/language_servers/fsharp_language_server.py b/src/solidlsp/language_servers/fsharp_language_server.py index e4ee3e2c..9d12cf5d 100644 --- a/src/solidlsp/language_servers/fsharp_language_server.py +++ b/src/solidlsp/language_servers/fsharp_language_server.py @@ -11,6 +11,7 @@ from pathlib import Path from overrides import override +from serena.util.dotnet import DotNETUtil from solidlsp.language_servers.common import RuntimeDependency, RuntimeDependencyCollection from solidlsp.ls import SolidLanguageServer from solidlsp.ls_config import LanguageServerConfig @@ -61,21 +62,7 @@ class FSharpLanguageServer(SolidLanguageServer): """ Setup runtime dependencies for F# Language Server and return the command to start the server. """ - # First check if .NET SDK is installed - dotnet_exe = shutil.which("dotnet") - if not dotnet_exe: - raise RuntimeError( - ".NET SDK is not installed or not in PATH. Please install .NET SDK 8.0 or later and ensure 'dotnet' is in your PATH." - ) - - # Verify dotnet version - import subprocess - - try: - result = subprocess.run([dotnet_exe, "--version"], capture_output=True, text=True, check=True) - log.info(f"Found .NET SDK version: {result.stdout.strip()}") - except subprocess.CalledProcessError: - raise RuntimeError("Failed to get .NET SDK version. Please ensure .NET SDK is properly installed.") + dotnet_exe = DotNETUtil("8.0", allow_higher_version=True).get_dotnet_path_or_raise() RuntimeDependencyCollection( [ diff --git a/test/solidlsp/csharp/test_csharp_dotnet_install.py b/test/solidlsp/csharp/test_csharp_dotnet_install.py deleted file mode 100644 index f59db6e5..00000000 --- a/test/solidlsp/csharp/test_csharp_dotnet_install.py +++ /dev/null @@ -1,290 +0,0 @@ -"""Tests for C# language server .NET runtime installation using official install scripts.""" - -import tempfile -from pathlib import Path -from unittest.mock import Mock, patch - -import pytest - -from solidlsp.language_servers.csharp_language_server import CSharpLanguageServer -from solidlsp.settings import SolidLSPSettings - - -@pytest.mark.csharp -class TestDotNetInstallScript: - """Test .NET runtime installation using Microsoft's official install scripts.""" - - @patch("solidlsp.language_servers.csharp_language_server.CSharpLanguageServer.DependencyProvider._ensure_language_server") - @patch("solidlsp.language_servers.common.PlatformUtils.get_platform_id") - @patch("solidlsp.language_servers.csharp_language_server.platform.system") - @patch("solidlsp.language_servers.csharp_language_server.subprocess.run") - @patch("solidlsp.language_servers.csharp_language_server.urllib.request.urlretrieve") - @patch("solidlsp.language_servers.csharp_language_server.shutil.which") - def test_install_dotnet_uses_bash_script_on_linux( - self, mock_which, mock_urlretrieve, mock_subprocess, mock_platform, mock_platform_id, mock_ensure_ls - ): - """Test that Linux uses bash install script.""" - from solidlsp.ls_utils import PlatformId - - mock_platform.return_value = "Linux" - mock_platform_id.return_value = PlatformId.LINUX_x64 - mock_which.return_value = None # No system dotnet - mock_ensure_ls.return_value = "/fake/server.dll" - - with tempfile.TemporaryDirectory() as temp_dir: - - def mock_retrieve(url, path): - # Create the file that urlretrieve would create - Path(path).touch() - # Also create the dotnet executable after script download - if "dotnet-install.sh" in str(path): - dotnet_dir = Path(temp_dir) / "dotnet-runtime-10.0" - dotnet_dir.mkdir(parents=True, exist_ok=True) - dotnet_exe = dotnet_dir / "dotnet" - dotnet_exe.touch() - dotnet_exe.chmod(0o755) - - mock_urlretrieve.side_effect = mock_retrieve - mock_subprocess.return_value = Mock(returncode=0, stdout="", stderr="") - - mock_settings = SolidLSPSettings() - custom_settings = SolidLSPSettings.CustomLSSettings({}) - - _ = CSharpLanguageServer.DependencyProvider( - custom_settings=custom_settings, - ls_resources_dir=temp_dir, - solidlsp_settings=mock_settings, - repository_root_path="/fake/repo", - ) - - # Verify bash script was downloaded - assert mock_urlretrieve.called - script_url = mock_urlretrieve.call_args[0][0] - assert script_url == "https://dot.net/v1/dotnet-install.sh" - - @patch("solidlsp.language_servers.csharp_language_server.CSharpLanguageServer.DependencyProvider._ensure_language_server") - @patch("solidlsp.language_servers.common.PlatformUtils.get_platform_id") - @patch("solidlsp.language_servers.csharp_language_server.platform.system") - @patch("solidlsp.language_servers.csharp_language_server.subprocess.run") - @patch("solidlsp.language_servers.csharp_language_server.urllib.request.urlretrieve") - @patch("solidlsp.language_servers.csharp_language_server.shutil.which") - def test_install_dotnet_uses_powershell_script_on_windows( - self, mock_which, mock_urlretrieve, mock_subprocess, mock_platform, mock_platform_id, mock_ensure_ls - ): - """Test that Windows uses PowerShell install script.""" - from solidlsp.ls_utils import PlatformId - - mock_platform.return_value = "Windows" - mock_platform_id.return_value = PlatformId.WIN_x64 - mock_which.return_value = None # No system dotnet - mock_ensure_ls.return_value = "/fake/server.dll" - - with tempfile.TemporaryDirectory() as temp_dir: - - def mock_retrieve(url, path): - # Create the file that urlretrieve would create - Path(path).touch() - # Also create the dotnet executable after script download - if "dotnet-install.ps1" in str(path): - dotnet_dir = Path(temp_dir) / "dotnet-runtime-10.0" - dotnet_dir.mkdir(parents=True, exist_ok=True) - dotnet_exe = dotnet_dir / "dotnet.exe" - dotnet_exe.touch() - - mock_urlretrieve.side_effect = mock_retrieve - mock_subprocess.return_value = Mock(returncode=0, stdout="", stderr="") - - mock_settings = SolidLSPSettings() - custom_settings = SolidLSPSettings.CustomLSSettings({}) - - _ = CSharpLanguageServer.DependencyProvider( - custom_settings=custom_settings, - ls_resources_dir=temp_dir, - solidlsp_settings=mock_settings, - repository_root_path="/fake/repo", - ) - - # Verify PowerShell script was downloaded - assert mock_urlretrieve.called - script_url = mock_urlretrieve.call_args[0][0] - assert script_url == "https://dot.net/v1/dotnet-install.ps1" - - def test_uses_system_dotnet_10_if_available(self): - """Test that system .NET 10 is used if available instead of downloading.""" - from solidlsp.ls_utils import PlatformId - - with patch("solidlsp.language_servers.csharp_language_server.shutil.which") as mock_which: - with patch("solidlsp.language_servers.csharp_language_server.subprocess.run") as mock_subprocess: - with patch("solidlsp.language_servers.common.PlatformUtils.get_platform_id") as mock_platform_id: - with patch( - "solidlsp.language_servers.csharp_language_server.CSharpLanguageServer.DependencyProvider._ensure_language_server" - ) as mock_ensure_ls: - mock_which.return_value = "/usr/bin/dotnet" - mock_ensure_ls.return_value = "/fake/server.dll" - mock_platform_id.return_value = PlatformId.LINUX_x64 - - # Mock dotnet --list-runtimes output showing .NET 10 - mock_result = Mock() - mock_result.stdout = "Microsoft.NETCore.App 10.0.1 [/usr/share/dotnet/shared/Microsoft.NETCore.App]" - mock_result.returncode = 0 - mock_subprocess.return_value = mock_result - - with tempfile.TemporaryDirectory() as temp_dir: - mock_settings = SolidLSPSettings() - custom_settings = SolidLSPSettings.CustomLSSettings({}) - - dependency_provider = CSharpLanguageServer.DependencyProvider( - custom_settings=custom_settings, - ls_resources_dir=temp_dir, - solidlsp_settings=mock_settings, - repository_root_path="/fake/repo", - ) - - # Should have used system dotnet - assert dependency_provider._dotnet_path == "/usr/bin/dotnet" - - def test_runtime_dependencies_no_longer_include_dotnet_downloads(self): - """Test that _RUNTIME_DEPENDENCIES no longer includes manual .NET download entries.""" - from solidlsp.language_servers.csharp_language_server import _RUNTIME_DEPENDENCIES - - # Check that DotNetRuntime entries don't exist (we use install script instead) - dotnet_deps = [dep for dep in _RUNTIME_DEPENDENCIES if dep.id == "DotNetRuntime"] - - # After our changes, there should be no DotNetRuntime entries - # (we'll use the install script instead) - assert len(dotnet_deps) == 0, "DotNetRuntime dependencies should be removed in favor of install script" - - @patch("solidlsp.language_servers.csharp_language_server.CSharpLanguageServer.DependencyProvider._ensure_language_server") - @patch("solidlsp.language_servers.common.PlatformUtils.get_platform_id") - @patch("solidlsp.language_servers.csharp_language_server.platform.system") - @patch("solidlsp.language_servers.csharp_language_server.subprocess.run") - @patch("solidlsp.language_servers.csharp_language_server.urllib.request.urlretrieve") - @patch("solidlsp.language_servers.csharp_language_server.shutil.which") - def test_cached_dotnet_is_reused(self, mock_which, mock_urlretrieve, mock_subprocess, mock_platform, mock_platform_id, mock_ensure_ls): - """Test that cached .NET installation is reused without re-downloading.""" - from solidlsp.ls_utils import PlatformId - - mock_platform.return_value = "Linux" - mock_platform_id.return_value = PlatformId.LINUX_x64 - mock_which.return_value = None - mock_ensure_ls.return_value = "/fake/server.dll" - - with tempfile.TemporaryDirectory() as temp_dir: - # Pre-create the cached dotnet executable - dotnet_dir = Path(temp_dir) / "dotnet-runtime-10.0" - dotnet_dir.mkdir(parents=True, exist_ok=True) - dotnet_exe = dotnet_dir / "dotnet" - dotnet_exe.touch() - dotnet_exe.chmod(0o755) - - mock_settings = SolidLSPSettings() - custom_settings = SolidLSPSettings.CustomLSSettings({}) - - dependency_provider = CSharpLanguageServer.DependencyProvider( - custom_settings=custom_settings, - ls_resources_dir=temp_dir, - solidlsp_settings=mock_settings, - repository_root_path="/fake/repo", - ) - - # Should use cached dotnet without downloading install script - assert dependency_provider._dotnet_path == str(dotnet_exe) - assert not mock_urlretrieve.called, "Should not download install script when cached dotnet exists" - - def test_rejects_dotnet_9_and_installs_dotnet_10(self): - """Test that .NET 9 is rejected and .NET 10 install is triggered.""" - from solidlsp.ls_utils import PlatformId - - with patch("solidlsp.language_servers.csharp_language_server.shutil.which") as mock_which: - with patch("solidlsp.language_servers.csharp_language_server.subprocess.run") as mock_subprocess: - with patch("solidlsp.language_servers.common.PlatformUtils.get_platform_id") as mock_platform_id: - with patch("solidlsp.language_servers.csharp_language_server.urllib.request.urlretrieve") as mock_urlretrieve: - with patch("solidlsp.language_servers.csharp_language_server.platform.system") as mock_platform: - with patch( - "solidlsp.language_servers.csharp_language_server.CSharpLanguageServer.DependencyProvider._ensure_language_server" - ) as mock_ensure_ls: - mock_which.return_value = "/usr/bin/dotnet" - mock_ensure_ls.return_value = "/fake/server.dll" - mock_platform_id.return_value = PlatformId.LINUX_x64 - mock_platform.return_value = "Linux" - - # Mock dotnet --list-runtimes output showing only .NET 9 - mock_result = Mock() - mock_result.stdout = "Microsoft.NETCore.App 9.0.1 [/usr/share/dotnet/shared/Microsoft.NETCore.App]" - mock_result.returncode = 0 - - call_count = 0 - - def subprocess_side_effect(*args, **kwargs): - nonlocal call_count - call_count += 1 - if call_count == 1: - # First call: dotnet --list-runtimes returns .NET 9 - return mock_result - else: - # Second call: install script execution - return Mock(returncode=0, stdout="", stderr="") - - mock_subprocess.side_effect = subprocess_side_effect - - with tempfile.TemporaryDirectory() as temp_dir: - - def mock_retrieve(url, path): - Path(path).touch() - if "dotnet-install.sh" in str(path): - dotnet_dir = Path(temp_dir) / "dotnet-runtime-10.0" - dotnet_dir.mkdir(parents=True, exist_ok=True) - dotnet_exe = dotnet_dir / "dotnet" - dotnet_exe.touch() - dotnet_exe.chmod(0o755) - - mock_urlretrieve.side_effect = mock_retrieve - - mock_settings = SolidLSPSettings() - custom_settings = SolidLSPSettings.CustomLSSettings({}) - - dependency_provider = CSharpLanguageServer.DependencyProvider( - custom_settings=custom_settings, - ls_resources_dir=temp_dir, - solidlsp_settings=mock_settings, - repository_root_path="/fake/repo", - ) - - # Should have installed .NET 10 (not used system .NET 9) - assert dependency_provider._dotnet_path == str(Path(temp_dir) / "dotnet-runtime-10.0" / "dotnet") - assert mock_urlretrieve.called, "Should download install script when .NET 9 found" - - def test_accepts_dotnet_11_and_higher(self): - """Test that .NET 11+ versions are accepted for forward compatibility.""" - from solidlsp.ls_utils import PlatformId - - for version in ["11.0.1", "12.0.0", "15.3.2"]: - with patch("solidlsp.language_servers.csharp_language_server.shutil.which") as mock_which: - with patch("solidlsp.language_servers.csharp_language_server.subprocess.run") as mock_subprocess: - with patch("solidlsp.language_servers.common.PlatformUtils.get_platform_id") as mock_platform_id: - with patch( - "solidlsp.language_servers.csharp_language_server.CSharpLanguageServer.DependencyProvider._ensure_language_server" - ) as mock_ensure_ls: - mock_which.return_value = "/usr/bin/dotnet" - mock_ensure_ls.return_value = "/fake/server.dll" - mock_platform_id.return_value = PlatformId.LINUX_x64 - - # Mock dotnet --list-runtimes output showing .NET 11+ - mock_result = Mock() - mock_result.stdout = f"Microsoft.NETCore.App {version} [/usr/share/dotnet/shared/Microsoft.NETCore.App]" - mock_result.returncode = 0 - mock_subprocess.return_value = mock_result - - with tempfile.TemporaryDirectory() as temp_dir: - mock_settings = SolidLSPSettings() - custom_settings = SolidLSPSettings.CustomLSSettings({}) - - dependency_provider = CSharpLanguageServer.DependencyProvider( - custom_settings=custom_settings, - ls_resources_dir=temp_dir, - solidlsp_settings=mock_settings, - repository_root_path="/fake/repo", - ) - - # Should use system dotnet for .NET 11+ - assert dependency_provider._dotnet_path == "/usr/bin/dotnet", f"Should accept .NET {version}" diff --git a/test/solidlsp/fsharp/test_fsharp_basic.py b/test/solidlsp/fsharp/test_fsharp_basic.py index 578ae704..4e6d7024 100644 --- a/test/solidlsp/fsharp/test_fsharp_basic.py +++ b/test/solidlsp/fsharp/test_fsharp_basic.py @@ -1,14 +1,10 @@ import os -import tempfile import threading -from pathlib import Path from typing import Any -from unittest.mock import Mock, patch import pytest from solidlsp import SolidLanguageServer -from solidlsp.language_servers.fsharp_language_server import FSharpLanguageServer from solidlsp.ls_config import Language from solidlsp.ls_utils import SymbolUtils from test.conftest import is_ci @@ -183,41 +179,3 @@ class TestFSharpLanguageServer: # This is a successful test - FsAutoComplete is working with F# files assert True, "F# language server can handle files successfully" - - -@pytest.mark.fsharp -class TestFSharpLanguageServerSetup: - """Test F# language server setup and configuration.""" - - def test_runtime_dependency_setup_without_dotnet(self) -> None: - """Test that setup fails gracefully when .NET is not installed.""" - with patch("shutil.which", return_value=None): - with pytest.raises(RuntimeError, match=r"\.NET SDK is not installed"): - FSharpLanguageServer._setup_runtime_dependencies(Mock(), Mock()) - - def test_runtime_dependency_setup_with_dotnet(self) -> None: - """Test that setup works when .NET is available.""" - mock_config = Mock() - mock_settings = Mock() - - # Mock the ls_resources_dir to return a temp directory - with tempfile.TemporaryDirectory() as temp_dir: - with patch("shutil.which", return_value="/usr/bin/dotnet"): - with patch.object(FSharpLanguageServer, "ls_resources_dir", return_value=temp_dir): - with patch("subprocess.run") as mock_run: - # Mock successful dotnet version check - mock_run.return_value.stdout = "8.0.100" - mock_run.return_value.returncode = 0 - - # Create a fake fsautocomplete executable - fsharp_dir = os.path.join(temp_dir, "fsharp-lsp") - os.makedirs(fsharp_dir, exist_ok=True) - # Use .exe extension on Windows, matching production code - exe_name = "fsautocomplete.exe" if os.name == "nt" else "fsautocomplete" - fsautocomplete_path = os.path.join(fsharp_dir, exe_name) - Path(fsautocomplete_path).touch() - - result = FSharpLanguageServer._setup_runtime_dependencies(mock_config, mock_settings) - - assert fsautocomplete_path in result - assert "--adaptive-lsp-server-enabled --project-graph-enabled --use-fcs-transparent-compiler" in result From c4844ec4a22ce606fd0504a3ef814e4c06257175 Mon Sep 17 00:00:00 2001 From: Dominik Jain Date: Mon, 2 Mar 2026 15:55:17 +0100 Subject: [PATCH 09/24] Raise .NET version in CI to 10 --- .github/workflows/pytest.yml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/.github/workflows/pytest.yml b/.github/workflows/pytest.yml index 058ecbea..51577d5c 100644 --- a/.github/workflows/pytest.yml +++ b/.github/workflows/pytest.yml @@ -139,7 +139,10 @@ jobs: - name: Setup .NET SDK (for F# and C# languages) uses: actions/setup-dotnet@v4 with: - dotnet-version: '8.0.x' + dotnet-version: '10.0.x' + - name: List .NET runtimes + shell: bash + run: dotnet --list-runtimes - name: Install Terraform uses: hashicorp/setup-terraform@v3 with: From 9c336cc3390815aa4d14bef7c2b33316086cf586 Mon Sep 17 00:00:00 2001 From: Dominik Jain Date: Mon, 2 Mar 2026 19:24:07 +0100 Subject: [PATCH 10/24] Update language requirements for C# and F# [skip ci] --- docs/01-about/020_programming-languages.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/01-about/020_programming-languages.md b/docs/01-about/020_programming-languages.md index 97c7fc40..728da19f 100644 --- a/docs/01-about/020_programming-languages.md +++ b/docs/01-about/020_programming-languages.md @@ -30,7 +30,7 @@ Some languages require additional installations or setup steps, as noted. * **AL** * **Bash** * **C#** - (by default, uses the Roslyn language server (language `csharp`), requires `pwsh` ([PowerShell 7+](https://learn.microsoft.com/en-us/powershell/scripting/install/install-powershell-on-windows?view=powershell-7.5)) on Windows; + (by default, uses the Roslyn language server (language `csharp`), requiring [.NET v10+](https://dotnet.microsoft.com/en-us/download/dotnet) and, on Windows, `pwsh` ([PowerShell 7+](https://learn.microsoft.com/en-us/powershell/scripting/install/install-powershell-on-windows?view=powershell-7.5)); set language to `csharp_omnisharp` to use OmiSharp instead) * **C/C++** (by default, uses the clangd language server (language `cpp`) but we also support ccls (language `cpp_ccls`); @@ -45,7 +45,7 @@ Some languages require additional installations or setup steps, as noted. * **Erlang** (requires installation of beam and [erlang_ls](https://github.com/erlang-ls/erlang_ls); experimental, might be slow or hang) * **F#** - (requires .NET SDK 8.0+; uses FsAutoComplete/Ionide, which is auto-installed; for Homebrew .NET on macOS, set DOTNET_ROOT in your environment) + (requires [.NET v8.0+](https://dotnet.microsoft.com/en-us/download/dotnet); uses FsAutoComplete/Ionide, which is auto-installed; for Homebrew .NET on macOS, set DOTNET_ROOT in your environment) * **Fortran** (requires installation of fortls: `pip install fortls`) * **Go** From eb78921c0899af26c6eb51508b7c96d5e49045cf Mon Sep 17 00:00:00 2001 From: Dominik Jain Date: Mon, 2 Mar 2026 19:12:10 +0100 Subject: [PATCH 11/24] Split test_find_symbol_references and test_find_symbol into stable and unstable tests --- test/serena/test_serena_agent.py | 172 ++++++++++++++++++------------- 1 file changed, 103 insertions(+), 69 deletions(-) diff --git a/test/serena/test_serena_agent.py b/test/serena/test_serena_agent.py index 3dbef5d8..b04dc37f 100644 --- a/test/serena/test_serena_agent.py +++ b/test/serena/test_serena_agent.py @@ -117,37 +117,7 @@ class TestSerenaAgent: serena_config = SerenaConfig(gui_log_window=False, web_dashboard=False) SerenaAgent(project=project, serena_config=serena_config) - @pytest.mark.parametrize( - "serena_agent,symbol_name,expected_kind,expected_file", - [ - pytest.param(Language.PYTHON, "User", "Class", "models.py", marks=pytest.mark.python), - pytest.param(Language.GO, "Helper", "Function", "main.go", marks=pytest.mark.go), - pytest.param(Language.JAVA, "Model", "Class", "Model.java", marks=pytest.mark.java), - pytest.param( - Language.KOTLIN, - "Model", - "Struct", - "Model.kt", - marks=[pytest.mark.kotlin] + ([pytest.mark.skip(reason="Kotlin LSP JVM crashes on restart in CI")] if is_ci else []), - ), - pytest.param(Language.RUST, "add", "Function", "lib.rs", marks=pytest.mark.rust), - pytest.param(Language.TYPESCRIPT, "DemoClass", "Class", "index.ts", marks=pytest.mark.typescript), - pytest.param(Language.PHP, "helperFunction", "Function", "helper.php", marks=pytest.mark.php), - pytest.param(Language.CLOJURE, "greet", "Function", clj.CORE_PATH, marks=pytest.mark.clojure), - pytest.param(Language.CSHARP, "Calculator", "Class", "Program.cs", marks=pytest.mark.csharp), - pytest.param(Language.FSHARP, "Calculator", "Module", "Calculator.fs", marks=pytest.mark.fsharp), - pytest.param(Language.POWERSHELL, "function Greet-User ()", "Function", "main.ps1", marks=pytest.mark.powershell), - pytest.param(Language.CPP_CCLS, "add", "Function", "b.cpp", marks=pytest.mark.cpp), - ], - indirect=["serena_agent"], - ) - def test_find_symbol(self, serena_agent: SerenaAgent, symbol_name: str, expected_kind: str, expected_file: str): - # skip flaky tests in CI - # TODO: Revisit the flaky tests and re-enable once the LS issues are resolved #1040 - flaky_languages = {Language.FSHARP, Language.RUST} - if set(serena_agent.get_active_lsp_languages()).intersection(flaky_languages) and is_ci: - pytest.skip("Test is flaky and thus skipped in CI environment.") - + def _assert_find_symbol(self, serena_agent: SerenaAgent, symbol_name: str, expected_kind: str, expected_file: str) -> None: agent = serena_agent find_symbol_tool = agent.get_tool(FindSymbolTool) result = find_symbol_tool.apply(name_path_pattern=symbol_name, include_info=True) @@ -175,54 +145,53 @@ class TestSerenaAgent: assert "A simple model class" in symbol_info, f"Java class docstring not found in symbol info: {s}" @pytest.mark.parametrize( - "serena_agent,symbol_name,def_file,ref_file", + "serena_agent,symbol_name,expected_kind,expected_file", [ - pytest.param( - Language.PYTHON, - "User", - os.path.join("test_repo", "models.py"), - os.path.join("test_repo", "services.py"), - marks=pytest.mark.python, - ), - pytest.param(Language.GO, "Helper", "main.go", "main.go", marks=pytest.mark.go), - pytest.param( - Language.JAVA, - "Model", - os.path.join("src", "main", "java", "test_repo", "Model.java"), - os.path.join("src", "main", "java", "test_repo", "Main.java"), - marks=pytest.mark.java, - ), + pytest.param(Language.PYTHON, "User", "Class", "models.py", marks=pytest.mark.python), + pytest.param(Language.GO, "Helper", "Function", "main.go", marks=pytest.mark.go), + pytest.param(Language.JAVA, "Model", "Class", "Model.java", marks=pytest.mark.java), pytest.param( Language.KOTLIN, "Model", - os.path.join("src", "main", "kotlin", "test_repo", "Model.kt"), - os.path.join("src", "main", "kotlin", "test_repo", "Main.kt"), + "Struct", + "Model.kt", marks=[pytest.mark.kotlin] + ([pytest.mark.skip(reason="Kotlin LSP JVM crashes on restart in CI")] if is_ci else []), ), - pytest.param(Language.RUST, "add", os.path.join("src", "lib.rs"), os.path.join("src", "main.rs"), marks=pytest.mark.rust), - pytest.param(Language.TYPESCRIPT, "helperFunction", "index.ts", "use_helper.ts", marks=pytest.mark.typescript), - pytest.param(Language.PHP, "helperFunction", "helper.php", "index.php", marks=pytest.mark.php), - pytest.param( - Language.CLOJURE, - "multiply", - clj.CORE_PATH, - clj.UTILS_PATH, - marks=pytest.mark.clojure, - ), - pytest.param(Language.CSHARP, "Calculator", "Program.cs", "Program.cs", marks=pytest.mark.csharp), - pytest.param(Language.FSHARP, "add", "Calculator.fs", "Program.fs", marks=pytest.mark.fsharp), - pytest.param(Language.POWERSHELL, "function Greet-User ()", "main.ps1", "main.ps1", marks=pytest.mark.powershell), - pytest.param(Language.CPP_CCLS, "add", "b.cpp", "a.cpp", marks=pytest.mark.cpp), + pytest.param(Language.TYPESCRIPT, "DemoClass", "Class", "index.ts", marks=pytest.mark.typescript), + pytest.param(Language.PHP, "helperFunction", "Function", "helper.php", marks=pytest.mark.php), + pytest.param(Language.CLOJURE, "greet", "Function", clj.CORE_PATH, marks=pytest.mark.clojure), + pytest.param(Language.CSHARP, "Calculator", "Class", "Program.cs", marks=pytest.mark.csharp), + pytest.param(Language.POWERSHELL, "function Greet-User ()", "Function", "main.ps1", marks=pytest.mark.powershell), + pytest.param(Language.CPP_CCLS, "add", "Function", "b.cpp", marks=pytest.mark.cpp), ], indirect=["serena_agent"], ) - def test_find_symbol_references(self, serena_agent: SerenaAgent, symbol_name: str, def_file: str, ref_file: str) -> None: - # skip flaky tests in CI - # TODO: Revisit the flaky tests and re-enable once the LS issues are resolved #1040 - flaky_languages = {Language.TYPESCRIPT} - if set(serena_agent.get_active_lsp_languages()).intersection(flaky_languages) and is_ci: - pytest.skip("Test is flaky and thus skipped in CI environment.") + def test_find_symbol_stable(self, serena_agent: SerenaAgent, symbol_name: str, expected_kind: str, expected_file: str) -> None: + self._assert_find_symbol(serena_agent, symbol_name, expected_kind, expected_file) + @pytest.mark.parametrize( + "serena_agent,symbol_name,expected_kind,expected_file", + [ + pytest.param(Language.FSHARP, "Calculator", "Module", "Calculator.fs", marks=pytest.mark.fsharp), + ], + indirect=["serena_agent"], + ) + @pytest.mark.xfail(reason="F# language server is unreliable") # See issue #1040 + def test_find_symbol_fsharp(self, serena_agent: SerenaAgent, symbol_name: str, expected_kind: str, expected_file: str) -> None: + self._assert_find_symbol(serena_agent, symbol_name, expected_kind, expected_file) + + @pytest.mark.parametrize( + "serena_agent,symbol_name,expected_kind,expected_file", + [ + pytest.param(Language.RUST, "add", "Function", "lib.rs", marks=pytest.mark.rust), + ], + indirect=["serena_agent"], + ) + @pytest.mark.xfail(reason="Rust language server is unreliable") # See issue #1040 + def test_find_symbol_rust(self, serena_agent: SerenaAgent, symbol_name: str, expected_kind: str, expected_file: str) -> None: + self._assert_find_symbol(serena_agent, symbol_name, expected_kind, expected_file) + + def _assert_find_symbol_references(self, serena_agent: SerenaAgent, symbol_name: str, def_file: str, ref_file: str) -> None: agent = serena_agent # Find the symbol location first @@ -257,6 +226,71 @@ class TestSerenaAgent: refs = json.loads(result) assert contains_ref_with_relative_path(refs, ref_file), f"Expected to find reference to {symbol_name} in {ref_file}. refs={refs}" + @pytest.mark.parametrize( + "serena_agent,symbol_name,def_file,ref_file", + [ + pytest.param( + Language.PYTHON, + "User", + os.path.join("test_repo", "models.py"), + os.path.join("test_repo", "services.py"), + marks=pytest.mark.python, + ), + pytest.param(Language.GO, "Helper", "main.go", "main.go", marks=pytest.mark.go), + pytest.param( + Language.JAVA, + "Model", + os.path.join("src", "main", "java", "test_repo", "Model.java"), + os.path.join("src", "main", "java", "test_repo", "Main.java"), + marks=pytest.mark.java, + ), + pytest.param( + Language.KOTLIN, + "Model", + os.path.join("src", "main", "kotlin", "test_repo", "Model.kt"), + os.path.join("src", "main", "kotlin", "test_repo", "Main.kt"), + marks=[pytest.mark.kotlin] + ([pytest.mark.skip(reason="Kotlin LSP JVM crashes on restart in CI")] if is_ci else []), + ), + pytest.param(Language.RUST, "add", os.path.join("src", "lib.rs"), os.path.join("src", "main.rs"), marks=pytest.mark.rust), + pytest.param(Language.PHP, "helperFunction", "helper.php", "index.php", marks=pytest.mark.php), + pytest.param( + Language.CLOJURE, + "multiply", + clj.CORE_PATH, + clj.UTILS_PATH, + marks=pytest.mark.clojure, + ), + pytest.param(Language.CSHARP, "Calculator", "Program.cs", "Program.cs", marks=pytest.mark.csharp), + pytest.param(Language.POWERSHELL, "function Greet-User ()", "main.ps1", "main.ps1", marks=pytest.mark.powershell), + pytest.param(Language.CPP_CCLS, "add", "b.cpp", "a.cpp", marks=pytest.mark.cpp), + ], + indirect=["serena_agent"], + ) + def test_find_symbol_references_stable(self, serena_agent: SerenaAgent, symbol_name: str, def_file: str, ref_file: str) -> None: + self._assert_find_symbol_references(serena_agent, symbol_name, def_file, ref_file) + + @pytest.mark.parametrize( + "serena_agent,symbol_name,def_file,ref_file", + [ + pytest.param(Language.TYPESCRIPT, "helperFunction", "index.ts", "use_helper.ts", marks=pytest.mark.typescript), + ], + indirect=["serena_agent"], + ) + @pytest.mark.xfail(reason="TypeScript language server is unreliable") # See issue #1040 + def test_find_symbol_references_typescript(self, serena_agent: SerenaAgent, symbol_name: str, def_file: str, ref_file: str) -> None: + self._assert_find_symbol_references(serena_agent, symbol_name, def_file, ref_file) + + @pytest.mark.parametrize( + "serena_agent,symbol_name,def_file,ref_file", + [ + pytest.param(Language.FSHARP, "add", "Calculator.fs", "Program.fs", marks=pytest.mark.fsharp), + ], + indirect=["serena_agent"], + ) + @pytest.mark.xfail(reason="F# language server is unreliable") # See issue #1040 + def test_find_symbol_references_fsharp(self, serena_agent: SerenaAgent, symbol_name: str, def_file: str, ref_file: str) -> None: + self._assert_find_symbol_references(serena_agent, symbol_name, def_file, ref_file) + @pytest.mark.parametrize( "serena_agent,name_path,substring_matching,expected_symbol_name,expected_kind,expected_file", [ From 90967ad67870e0a6ebe9b93b8df0c7066baa25d4 Mon Sep 17 00:00:00 2001 From: Dominik Jain Date: Mon, 2 Mar 2026 21:13:18 +0100 Subject: [PATCH 12/24] TestFSharpLanguageServer: Mark test_find_referencing_symbols with xfail --- test/solidlsp/fsharp/test_fsharp_basic.py | 1 + 1 file changed, 1 insertion(+) diff --git a/test/solidlsp/fsharp/test_fsharp_basic.py b/test/solidlsp/fsharp/test_fsharp_basic.py index 4e6d7024..26a0e695 100644 --- a/test/solidlsp/fsharp/test_fsharp_basic.py +++ b/test/solidlsp/fsharp/test_fsharp_basic.py @@ -49,6 +49,7 @@ class TestFSharpLanguageServer: for expected in expected_symbols: assert expected in symbol_names, f"{expected} function not found in Calculator.fs symbols" + @pytest.mark.xfail(is_ci, reason="Test is flaky") # TODO: Re-enable if the LS can be made more reliable #1040 @pytest.mark.parametrize("language_server", [Language.FSHARP], indirect=True) def test_find_referencing_symbols(self, language_server: SolidLanguageServer) -> None: """Test finding references using symbol selection range.""" From ccbd15082bec5f2112cd4f5949c6c324fd0a54ef Mon Sep 17 00:00:00 2001 From: Michael Panchenko <35432522+MischaPanch@users.noreply.github.com> Date: Tue, 3 Mar 2026 15:18:41 +0100 Subject: [PATCH 13/24] Feature/global memories (#1091) Support for global (cross-project) memories, adds a news item --------- Co-authored-by: Ariadne[bot] (butterflysky-ai) Co-authored-by: Dominik Jain --- docs/02-usage/040_workflow.md | 18 +-- docs/02-usage/045_memories.md | 84 ++++++++++ src/serena/agent.py | 7 +- src/serena/config/serena_config.py | 10 +- src/serena/dashboard.py | 35 +++- .../generated/generated_prompt_factory.py | 8 +- src/serena/project.py | 149 ++++++++++++++---- .../resources/config/modes/no-memories.yml | 1 + .../config/prompt_templates/system_prompt.yml | 7 +- src/serena/resources/dashboard/dashboard.css | 36 +++++ src/serena/resources/dashboard/dashboard.js | 77 ++++++++- src/serena/resources/dashboard/index.html | 17 +- .../resources/dashboard/news/20260303.html | 12 ++ .../resources/serena_config.template.yml | 6 + src/serena/tools/file_tools.py | 6 +- src/serena/tools/memory_tools.py | 49 ++++-- src/serena/tools/tools_base.py | 3 +- src/serena/tools/workflow_tools.py | 27 ++-- src/serena/util/inspection.py | 18 ++- 19 files changed, 466 insertions(+), 104 deletions(-) create mode 100644 docs/02-usage/045_memories.md create mode 100644 src/serena/resources/dashboard/news/20260303.html diff --git a/docs/02-usage/040_workflow.md b/docs/02-usage/040_workflow.md index c5153401..098ef523 100644 --- a/docs/02-usage/040_workflow.md +++ b/docs/02-usage/040_workflow.md @@ -103,22 +103,12 @@ By default, Serena will perform an **onboarding process** when it is started for the first time for a project. The goal of the onboarding is for Serena to get familiar with the project and to store memories, which it can then draw upon in future interactions. -If an LLM should fail to complete the onboarding and does not actually write the -respective memories to disk, you may need to ask it to do so explicitly. -The onboarding will usually read a lot of content from the project, thus filling -up the context. It can therefore be advisable to switch to another conversation -once the onboarding is complete. -After the onboarding, we recommend that you have a quick look at the memories and, -if necessary, edit them or add additional ones. +In general, **memories** provide a way for Serena to store and retrieve +information about the project, relevant conventions, and other relevant aspects. -**Memories** are files stored in `.serena/memories/` in the project directory, -which the agent can choose to read in subsequent interactions. -Feel free to read and adjust them as needed; you can also add new ones manually. -Every file in the `.serena/memories/` directory is a memory file. -Whenever Serena starts working on a project, the list of memories is -provided, and the agent can decide to read them. -We found that memories can significantly improve the user experience with Serena. +For more information on this, including how to manage +or disable these features, see [Memories & Onboarding](045-memories). ## Preparing Your Project diff --git a/docs/02-usage/045_memories.md b/docs/02-usage/045_memories.md new file mode 100644 index 00000000..e564d3c2 --- /dev/null +++ b/docs/02-usage/045_memories.md @@ -0,0 +1,84 @@ +# Memories & Onboarding + +Serena provides the functionality of a fully featured agent, and a useful aspect of this is Serena's memory system. +Despite its simplicity, we received positive feedback from many users who tend to combine it with their +agent's internal memory management (e.g., `AGENTS.md` files). + +## Memories + +Memories are simple, human-readable Markdown files that both you and +your agent can create, read, and edit. + +Serena differentiates between + * **project-specific memories**, which are stored in the `.serena/memories/` directory within your project folder, and + * **global memories**, which are shared across all projects and, by default, are stored in `~/.serena/memories/global/` + +The LLM is informed about the existence of memories and instructed to read them when appropriate, +inferring appropriateness from the file name. +When the agent starts working on a project, it receives the list of available memories. +The agent should be instructed to update memories by the user when appropriate. + +### Organizing Memories + +Memories can be organized into **topics** by using `/` in the memory name (e.g. `modules/frontend`). +The structure is mapped to the file system, where topics correspond to subdirectories. +The `list_memories` tool can filter by topic, allowing the agent to explore even large numbers of memories in a structured way. + +(global-memories)= +### Global Memories + +Global memories use the top-level topic `global`, i.e. whenever a memory name starts with `global/`, +it is stored in the global memories directory and is shared across all projects. + +By default, deletion and editing of global memories is allowed. +If you want to primarily manage such memories yourself and protect them from accidental modification by the agent, +set `edit_global_memories: False` in Serena's [global configuration](050-configuration). + +Since global memories are not versioned alongside your project files, +it can be helpful to track global memories with git (i.e. to make `~/.serena/memories/` a git repository) +in order to have a history of changes and the possibility to revert them if needed. + +### Manually Editing Memories + +You may edit memories directly in the file system, using your preferred text editor or IDE. +Alternatively, access them via the [Serena Dashboard](060_dashboard), which provides a graphical interface for +viewing, creating, editing, and deleting memories while Serena is running. + +(onboarding)= +## Onboarding + +By default, Serena performs an **onboarding process** when it encounters a project +for the first time (i.e., when no project memories exist yet). +The goal of the onboarding is for Serena to get familiar with the project — +its structure, build system, testing setup, and other essential aspects — +and to store this knowledge as memories for future interactions. + +In further project activations, Serena will check whether onboarding was already +performed by looking for existing project memories and will skip the onboarding +process if memories are found. + +### How Onboarding Works + +1. When a project is activated, Serena checks whether onboarding was already + performed (by checking if any memories exist). +2. If no memories are found, Serena triggers the onboarding process, which + reads key files and directories to understand the project. +3. The gathered information is written into project-specific memory files (see above). + +### Tips for Onboarding + +- **Context usage**: The onboarding process will read a lot of content from the project, + filling up the context window. It is therefore advisable to **switch to a new conversation** + once the onboarding is complete. +- **LLM failures**: If an LLM fails to complete the onboarding and does not actually + write the respective memories to disk, you may need to ask it to do so explicitly. +- **Review the results**: After onboarding, we recommend having a quick look at the + generated memories and editing them or adding new ones as needed. + +## Disabling Memories and Onboarding + +If you do not require the functionality described in this section, you can selectively disable it. + + * To disable all memory related tools (including onboarding), adding `no-memories` to the `base_modes` + in Serena's [global configuration](050-configuration). + * Similarly, to disable only onboarding, add `no-onboarding` to the `base_modes`. diff --git a/src/serena/agent.py b/src/serena/agent.py index 83f9694b..daa87820 100644 --- a/src/serena/agent.py +++ b/src/serena/agent.py @@ -12,7 +12,7 @@ from typing import TYPE_CHECKING, Optional, TypeVar from sensai.util import logging from sensai.util.logging import LogTime -from sensai.util.string import TextBuilder +from sensai.util.string import TextBuilder, list_string from interprompt.jinja_template import JinjaTemplate from serena import serena_version @@ -29,7 +29,7 @@ from serena.config.serena_config import ( ) from serena.dashboard import SerenaDashboardAPI from serena.ls_manager import LanguageServerManager, LanguageServerManagerInitialisationError -from serena.project import Project +from serena.project import MemoriesManager, Project from serena.prompt_factory import SerenaPromptFactory from serena.task_executor import TaskExecutor from serena.tools import ActivateProjectTool, GetCurrentConfigTool, OpenDashboardTool, ReplaceContentTool, Tool, ToolMarker, ToolRegistry @@ -611,12 +611,15 @@ class SerenaAgent: def create_system_prompt(self) -> str: available_tools = self._active_tools available_markers = available_tools.tool_marker_names + global_memory_names = MemoriesManager.list_global_memories() + global_memories_list = list_string(global_memory_names) if global_memory_names else "" log.info("Generating system prompt with available_tools=(see active tools), available_markers=%s", available_markers) system_prompt = self.prompt_factory.create_system_prompt( context_system_prompt=self._format_prompt(self._context.prompt), mode_system_prompts=[self._format_prompt(mode.prompt) for mode in self.get_active_modes()], available_tools=available_tools.tool_names, available_markers=available_markers, + global_memories_list=global_memories_list, ) # If a project is active at startup, append its activation message diff --git a/src/serena/config/serena_config.py b/src/serena/config/serena_config.py index 432e8a65..9273d31f 100644 --- a/src/serena/config/serena_config.py +++ b/src/serena/config/serena_config.py @@ -85,6 +85,12 @@ class SerenaPaths: """ file containing the ID of the last read news snippet """ + global_memories_path = Path(os.path.join(self.serena_user_home_dir, "memories", "global")) + global_memories_path.mkdir(parents=True, exist_ok=True) + self.global_memories_path = global_memories_path + """ + directory where global memories are stored, i.e. memories that are available across all projects + """ self.last_returned_log_file_path: str | None = None """ the path to the last log file returned by `get_next_log_file_path`. If this is not None, the logs @@ -549,6 +555,9 @@ class SerenaConfig(SharedConfig): """List of paths to ignore across all projects. Same syntax as gitignore, so you can use * and **. These patterns are merged additively with each project's own ignored_paths.""" + edit_global_memories: bool = True + """Whether global memories are allowed to be deleted or edited.""" + # settings with overridden defaults default_modes: Sequence[str] | None = ("interactive", "editing") symbol_info_budget: float = 10.0 @@ -559,7 +568,6 @@ class SerenaConfig(SharedConfig): If the budget is exceeded, Serena stops issuing further requests and returns partial info results. 0 disables the budget (no early stopping). Negative values are invalid. """ - # *** fields that are NOT mapped to/from the configuration file *** _loaded_commented_yaml: CommentedMap | None = None diff --git a/src/serena/dashboard.py b/src/serena/dashboard.py index a5c91a04..26ec8f56 100644 --- a/src/serena/dashboard.py +++ b/src/serena/dashboard.py @@ -89,6 +89,11 @@ class RequestDeleteMemory(BaseModel): memory_name: str +class RequestRenameMemory(BaseModel): + old_name: str + new_name: str + + class ResponseGetSerenaConfig(BaseModel): content: str @@ -269,6 +274,18 @@ class SerenaDashboardAPI: except Exception as e: return {"status": "error", "message": str(e)} + @self._app.route("/rename_memory", methods=["POST"]) + def rename_memory() -> dict[str, str]: + request_data = request.get_json() + if not request_data: + return {"status": "error", "message": "No data provided"} + request_rename_memory = RequestRenameMemory.model_validate(request_data) + try: + result_message = self._rename_memory(request_rename_memory) + return {"status": "success", "message": result_message} + except Exception as e: + return {"status": "error", "message": str(e)} + @self._app.route("/get_serena_config", methods=["GET"]) def get_serena_config() -> dict[str, Any]: try: @@ -556,8 +573,7 @@ class SerenaDashboardAPI: project = self._agent.get_active_project() if project is None: raise ValueError("No active project") - - project.memories_manager.save_memory(request_save_memory.memory_name, request_save_memory.content) + project.memories_manager.save_memory(request_save_memory.memory_name, request_save_memory.content, is_tool_context=False) self._agent.execute_task(run, logged=True, name="SaveMemory") @@ -566,11 +582,22 @@ class SerenaDashboardAPI: project = self._agent.get_active_project() if project is None: raise ValueError("No active project") - - project.memories_manager.delete_memory(request_delete_memory.memory_name) + project.memories_manager.delete_memory(request_delete_memory.memory_name, is_tool_context=False) self._agent.execute_task(run, logged=True, name="DeleteMemory") + def _rename_memory(self, request_rename_memory: RequestRenameMemory) -> str: + def run() -> str: + project = self._agent.get_active_project() + if project is None: + raise ValueError("No active project") + + return project.memories_manager.move_memory( + request_rename_memory.old_name, request_rename_memory.new_name, is_tool_context=False + ) + + return self._agent.execute_task(run, logged=True, name="RenameMemory") + def _get_serena_config(self) -> ResponseGetSerenaConfig: config_path = self._agent.serena_config.config_file_path if config_path is None or not os.path.exists(config_path): diff --git a/src/serena/generated/generated_prompt_factory.py b/src/serena/generated/generated_prompt_factory.py index 3a5e1442..9129318c 100644 --- a/src/serena/generated/generated_prompt_factory.py +++ b/src/serena/generated/generated_prompt_factory.py @@ -33,6 +33,12 @@ class PromptFactory(PromptFactoryBase): return self._render_prompt("prepare_for_new_conversation", locals()) def create_system_prompt( - self, *, available_markers: Any, available_tools: Any, context_system_prompt: Any, mode_system_prompts: Any + self, + *, + available_markers: Any, + available_tools: Any, + context_system_prompt: Any, + global_memories_list: Any, + mode_system_prompts: Any, ) -> str: return self._render_prompt("system_prompt", locals()) diff --git a/src/serena/project.py b/src/serena/project.py index 59b89ec4..ed4821cb 100644 --- a/src/serena/project.py +++ b/src/serena/project.py @@ -1,9 +1,10 @@ import json import logging import os +import shutil import threading from pathlib import Path -from typing import TYPE_CHECKING, Any +from typing import TYPE_CHECKING, Any, Literal import pathspec from sensai.util.logging import LogTime @@ -12,12 +13,13 @@ from sensai.util.string import ToStringMixin from serena.config.serena_config import ( DEFAULT_TOOL_TIMEOUT, ProjectConfig, + SerenaPaths, get_serena_managed_in_project_dir, ) from serena.constants import SERENA_FILE_ENCODING, SERENA_MANAGED_DIR_NAME from serena.ls_manager import LanguageServerFactory, LanguageServerManager from serena.util.file_system import GitignoreParser, match_path -from serena.util.text_utils import MatchedConsecutiveLines, search_files +from serena.util.text_utils import ContentReplacer, MatchedConsecutiveLines, search_files from solidlsp import SolidLanguageServer from solidlsp.ls_config import Language from solidlsp.ls_utils import FileUtils @@ -29,26 +31,59 @@ log = logging.getLogger(__name__) class MemoriesManager: - def __init__(self, project_root: str): - self._memory_dir = Path(get_serena_managed_in_project_dir(project_root)) / "memories" - self._memory_dir.mkdir(parents=True, exist_ok=True) + GLOBAL_TOPIC = "global" + _global_memory_dir = SerenaPaths().global_memories_path + + def __init__(self, project_root: str, global_memory_tool_write_access: bool = False): + """ + :param project_root: the project's root directory + :param global_memory_tool_write_access: whether to allow writing global memories in tool execution contexts + """ + self._project_memory_dir = Path(get_serena_managed_in_project_dir(project_root)) / "memories" + self._project_memory_dir.mkdir(parents=True, exist_ok=True) + self._global_memory_tool_write_access = global_memory_tool_write_access self._encoding = SERENA_FILE_ENCODING + def _is_global(self, name: str) -> bool: + return name == self.GLOBAL_TOPIC or name.startswith(self.GLOBAL_TOPIC + "/") + def get_memory_file_path(self, name: str) -> Path: # Strip .md extension if present name = name.replace(".md", "") - # Split by "/" to handle subdirectories + if self._is_global(name): + if name == self.GLOBAL_TOPIC: + raise ValueError( + f'Bare "{self.GLOBAL_TOPIC}" is not a valid memory name. ' + f'Use "{self.GLOBAL_TOPIC}/" to address a global memory.' + ) + # Strip "global/" prefix and resolve against global dir + sub_name = name[len(self.GLOBAL_TOPIC) + 1 :] + parts = sub_name.split("/") + filename = f"{parts[-1]}.md" + if len(parts) > 1: + subdir = self._global_memory_dir / "/".join(parts[:-1]) + subdir.mkdir(parents=True, exist_ok=True) + return subdir / filename + return self._global_memory_dir / filename + + # Project-local memory parts = name.split("/") filename = f"{parts[-1]}.md" if len(parts) > 1: # Create subdirectory path - subdir = self._memory_dir / "/".join(parts[:-1]) + subdir = self._project_memory_dir / "/".join(parts[:-1]) subdir.mkdir(parents=True, exist_ok=True) return subdir / filename - return self._memory_dir / filename + return self._project_memory_dir / filename + + def _check_write_access(self, name: str, is_tool_context: bool) -> None: + # in tool context, global memory write access can be disabled + if is_tool_context: + if self._is_global(name) and not self._global_memory_tool_write_access: + raise PermissionError(f"Writing to global memories is not allowed (attempted to write to '{name}')") def load_memory(self, name: str) -> str: memory_file_path = self.get_memory_file_path(name) @@ -57,47 +92,70 @@ class MemoriesManager: with open(memory_file_path, encoding=self._encoding) as f: return f.read() - def save_memory(self, name: str, content: str) -> str: + def save_memory(self, name: str, content: str, is_tool_context: bool) -> str: + self._check_write_access(name, is_tool_context) memory_file_path = self.get_memory_file_path(name) with open(memory_file_path, "w", encoding=self._encoding) as f: f.write(content) return f"Memory {name} written." + @staticmethod + def _list_memories(search_dir: Path, base_dir: Path, prefix: str = "") -> list[str]: + if not search_dir.exists(): + return [] + results = [] + for md_file in search_dir.rglob("*.md"): + rel = str(md_file.relative_to(base_dir).with_suffix("")).replace(os.sep, "/") + results.append(prefix + rel) + return results + + @classmethod + def list_global_memories(cls, subtopic: str = "") -> list[str]: + dir_path = cls._global_memory_dir + if subtopic: + dir_path = dir_path / subtopic.replace("/", os.sep) + return cls._list_memories(dir_path, cls._global_memory_dir, cls.GLOBAL_TOPIC + "/") + + def list_project_memories(self, topic: str = "") -> list[str]: + dir_path = self._project_memory_dir + if topic: + dir_path = dir_path / topic.replace("/", os.sep) + return self._list_memories(dir_path, self._project_memory_dir) + def list_memories(self, topic: str = "") -> list[str]: """ - List memories, optionally filtered by topic. + Lists all memories, optionally filtered by topic. + If the topic is omitted, both global and project-specific memories are returned. """ - memories = [] + memories: list[str] if topic: - # Only list memories in specified subdirectory - search_dir = self._memory_dir / topic.replace("/", os.sep) - if not search_dir.exists(): - return [] + if self._is_global(topic): + topic_parts = topic.split("/") + subtopic = "/".join(topic_parts[1:]) + memories = self.list_global_memories(subtopic=subtopic) + else: + memories = self.list_project_memories(topic=topic) else: - search_dir = self._memory_dir + memories = self.list_project_memories() + self.list_global_memories() - # Recursively find all .md files - for md_file in search_dir.rglob("*.md"): - # Calculate relative path as memory name - rel_path = md_file.relative_to(self._memory_dir) - name = str(rel_path.with_suffix("")).replace(os.sep, "/") - memories.append(name) - - # Sort alphabetically by name return sorted(memories) - def delete_memory(self, name: str) -> str: + def delete_memory(self, name: str, is_tool_context: bool) -> str: + self._check_write_access(name, is_tool_context) memory_file_path = self.get_memory_file_path(name) if not memory_file_path.exists(): return f"Memory {name} not found." memory_file_path.unlink() return f"Memory {name} deleted." - def rename_memory(self, old_name: str, new_name: str) -> str: + def move_memory(self, old_name: str, new_name: str, is_tool_context: bool) -> str: """ Rename or move a memory file. + Moving between global and project scope (e.g. "global/foo" -> "bar") is supported. """ + self._check_write_access(new_name, is_tool_context) + old_path = self.get_memory_file_path(old_name) new_path = self.get_memory_file_path(new_name) @@ -106,13 +164,35 @@ class MemoriesManager: if new_path.exists(): raise FileExistsError(f"Memory {new_name} already exists.") - # Ensure target directory exists new_path.parent.mkdir(parents=True, exist_ok=True) + shutil.move(old_path, new_path) - # Move/rename the file - old_path.rename(new_path) return f"Memory renamed from {old_name} to {new_name}." + def edit_memory( + self, name: str, needle: str, repl: str, mode: Literal["literal", "regex"], allow_multiple_occurrences: bool, is_tool_context: bool + ) -> str: + """ + Edit a memory by replacing content matching a pattern. + + :param name: the memory name + :param needle: the string or regex to search for + :param repl: the replacement string + :param mode: "literal" or "regex" + :param allow_multiple_occurrences: + """ + self._check_write_access(name, is_tool_context) + memory_file_path = self.get_memory_file_path(name) + if not memory_file_path.exists(): + raise FileNotFoundError(f"Memory {name} not found.") + with open(memory_file_path, encoding=self._encoding) as f: + original_content = f.read() + replacer = ContentReplacer(mode=mode, allow_multiple_occurrences=allow_multiple_occurrences) + updated_content = replacer.replace(original_content, needle, repl) + with open(memory_file_path, "w", encoding=self._encoding) as f: + f.write(updated_content) + return f"Memory {name} edited successfully." + class Project(ToStringMixin): def __init__( @@ -124,7 +204,10 @@ class Project(ToStringMixin): ): self.project_root = project_root self.project_config = project_config - self.memories_manager = MemoriesManager(project_root) + + global_memory_write_access = serena_config.edit_global_memories if serena_config else False + self.memories_manager = MemoriesManager(project_root, global_memory_write_access) + self.language_server_manager: LanguageServerManager | None = None self._is_newly_created = is_newly_created @@ -220,10 +303,10 @@ class Project(ToStringMixin): msg = f"The project with name '{self.project_name}' at {self.project_root} is activated." languages_str = ", ".join([lang.value for lang in self.project_config.languages]) msg += f"\nProgramming languages: {languages_str}; file encoding: {self.project_config.encoding}" - memories = self.memories_manager.list_memories() - if memories: + project_memories = self.memories_manager.list_project_memories() + if project_memories: msg += ( - f"\nAvailable project memories: {json.dumps(memories)}\n" + f"\nAvailable project memories: {json.dumps(project_memories)}\n" + "Use the `read_memory` tool to read these memories later if they are relevant to the task." ) if self.project_config.initial_prompt: diff --git a/src/serena/resources/config/modes/no-memories.yml b/src/serena/resources/config/modes/no-memories.yml index 7a522875..08602141 100644 --- a/src/serena/resources/config/modes/no-memories.yml +++ b/src/serena/resources/config/modes/no-memories.yml @@ -6,6 +6,7 @@ excluded_tools: - read_memory - delete_memory - edit_memory + - rename_memory - list_memories - onboarding - check_onboarding_performed diff --git a/src/serena/resources/config/prompt_templates/system_prompt.yml b/src/serena/resources/config/prompt_templates/system_prompt.yml index 13064c14..125a6818 100644 --- a/src/serena/resources/config/prompt_templates/system_prompt.yml +++ b/src/serena/resources/config/prompt_templates/system_prompt.yml @@ -35,10 +35,13 @@ prompts: You can understand relationships between symbols by using the `find_referencing_symbols` tool. {% endif %} - {% if 'read_memory' in available_tools %} + {% if 'read_memory' in available_tools -%} You generally have access to memories and it may be useful for you to read them. You infer whether memories are relevant based on their names. - {% endif %} + {% if global_memories_list -%} + The following global (not project-specific) memories are available to you: {{ global_memories_list }} + {%- endif -%} + {%- endif %} The context and modes of operation are described below. These determine how to interact with your user and which kinds of interactions are expected of you. diff --git a/src/serena/resources/dashboard/dashboard.css b/src/serena/resources/dashboard/dashboard.css index 662fed57..ec22d61e 100644 --- a/src/serena/resources/dashboard/dashboard.css +++ b/src/serena/resources/dashboard/dashboard.css @@ -1308,6 +1308,42 @@ body { gap: 8px; } +/* Memory Rename Styles */ +.memory-rename-btn { + display: inline-flex; + align-items: center; + justify-content: center; + width: 24px; + height: 24px; + background: var(--bg-secondary); + border: 1px solid var(--border-color); + border-radius: 4px; + cursor: pointer; + color: var(--text-muted); + opacity: 0.5; + transition: opacity 0.2s ease, background-color 0.2s ease; +} + +.memory-rename-btn:hover { + opacity: 1; + background-color: var(--border-color); +} + +.memory-rename-input { + font-size: inherit; + font-weight: inherit; + font-family: inherit; + color: var(--text-primary); + background: transparent; + border: none; + border-bottom: 1px solid var(--btn-primary); + outline: none; + padding: 0; + flex: 1; + min-width: 200px; + max-width: 80%; +} + /* Log Action Buttons (Save, Copy, Clear) */ .log-action-buttons { position: absolute; diff --git a/src/serena/resources/dashboard/dashboard.js b/src/serena/resources/dashboard/dashboard.js index f59dbb6a..5819cec3 100644 --- a/src/serena/resources/dashboard/dashboard.js +++ b/src/serena/resources/dashboard/dashboard.js @@ -307,6 +307,8 @@ class Dashboard { this.$modalCloseRemove = $('.modal-close-remove'); this.$editMemoryModal = $('#edit-memory-modal'); this.$editMemoryName = $('#edit-memory-name'); + this.$editMemoryRenameBtn = $('#edit-memory-rename-btn'); + this.$editMemoryRenameInput = $('#edit-memory-rename-input'); this.$editMemoryContent = $('#edit-memory-content'); this.$editMemorySaveBtn = $('#edit-memory-save-btn'); this.$editMemoryCancelBtn = $('#edit-memory-cancel-btn'); @@ -365,6 +367,19 @@ class Dashboard { this.$editMemoryCancelBtn.click(this.closeEditMemoryModal.bind(this)); this.$modalCloseEditMemory.click(this.closeEditMemoryModal.bind(this)); this.$editMemoryContent.on('input', this.trackMemoryChanges.bind(this)); + this.$editMemoryRenameBtn.click(this.startMemoryRename.bind(this)); + this.$editMemoryRenameInput.keydown(function (e) { + if (e.which === 13) { // Enter key + e.preventDefault(); + self.commitMemoryRename(); + } else if (e.which === 27) { // Escape key + e.preventDefault(); + self.cancelMemoryRename(); + } + }); + this.$editMemoryRenameInput.on('blur', function () { + self.cancelMemoryRename(); + }); this.$deleteMemoryOkBtn.click(this.confirmDeleteMemoryOk.bind(this)); this.$deleteMemoryCancelBtn.click(this.closeDeleteMemoryModal.bind(this)); this.$modalCloseDeleteMemory.click(this.closeDeleteMemoryModal.bind(this)); @@ -1887,6 +1902,62 @@ class Dashboard { }); } + startMemoryRename() { + this.$editMemoryName.hide(); + this.$editMemoryRenameBtn.hide(); + this.$editMemoryRenameInput.val(this.currentMemoryName).show().focus().select(); + } + + cancelMemoryRename() { + this.$editMemoryRenameInput.hide(); + this.$editMemoryName.show(); + this.$editMemoryRenameBtn.show(); + } + + commitMemoryRename() { + const newName = this.$editMemoryRenameInput.val().trim(); + const oldName = this.currentMemoryName; + + // If name unchanged, just cancel + if (!newName || newName === oldName) { + this.cancelMemoryRename(); + return; + } + + // Validate memory name (alphanumeric, underscores, and slashes for subdirectories) + if (!/^[a-zA-Z0-9_]+(?:\/[a-zA-Z0-9_]+)*$/.test(newName)) { + alert('Memory name can only contain letters, numbers, underscores, and "/" for subdirectories (e.g., "topic/memory_name")'); + this.$editMemoryRenameInput.focus(); + return; + } + + const self = this; + this.$editMemoryRenameInput.prop('disabled', true); + + $.ajax({ + url: '/rename_memory', type: 'POST', contentType: 'application/json', data: JSON.stringify({ + old_name: oldName, new_name: newName + }), success: function (response) { + if (response.status === 'success') { + self.currentMemoryName = newName; + self.$editMemoryName.text(newName); + self.cancelMemoryRename(); + // Reload config to reflect the rename in the memory list + self.loadConfigOverview(); + } else { + alert('Error: ' + response.message); + self.$editMemoryRenameInput.focus(); + } + }, error: function (xhr, status, error) { + console.error('Error renaming memory:', error); + alert('Error renaming memory: ' + (xhr.responseJSON ? xhr.responseJSON.message : error)); + self.$editMemoryRenameInput.focus(); + }, complete: function () { + self.$editMemoryRenameInput.prop('disabled', false); + } + }); + } + confirmDeleteMemory(memoryName) { // Set memory name to delete this.memoryToDelete = memoryName; @@ -1960,9 +2031,9 @@ class Dashboard { return; } - // Validate memory name (alphanumeric and underscores only) - if (!/^[a-zA-Z0-9_]+$/.test(memoryName)) { - alert('Memory name can only contain letters, numbers, and underscores'); + // Validate memory name (alphanumeric, underscores, and slashes for subdirectories) + if (!/^[a-zA-Z0-9_]+(?:\/[a-zA-Z0-9_]+)*$/.test(memoryName)) { + alert('Memory name can only contain letters, numbers, underscores, and "/" for subdirectories (e.g., "topic/memory_name")'); return; } diff --git a/src/serena/resources/dashboard/index.html b/src/serena/resources/dashboard/index.html index c8bf75d7..874aecf9 100644 --- a/src/serena/resources/dashboard/index.html +++ b/src/serena/resources/dashboard/index.html @@ -275,7 +275,17 @@