From e0ab4db1891ca5d03a779c1b3ea0ad947655d4d6 Mon Sep 17 00:00:00 2001 From: Dominik Jain Date: Tue, 28 Oct 2025 01:08:20 +0100 Subject: [PATCH 01/10] Add LanguageServerManager, adding multi-language support in a single project --- src/serena/agent.py | 42 ++++------- src/serena/cli.py | 3 +- src/serena/code_editor.py | 22 +++--- src/serena/config/serena_config.py | 31 +++++--- src/serena/dashboard.py | 2 +- src/serena/ls_manager.py | 88 +++++++++++++++++++++++ src/serena/project.py | 90 ++++++++++++++++-------- src/serena/symbol.py | 49 +++++++------ src/serena/tools/symbol_tools.py | 2 +- src/serena/tools/tools_base.py | 28 +++++--- src/solidlsp/ls.py | 1 + src/solidlsp/ls_exceptions.py | 12 ++++ src/solidlsp/ls_handler.py | 20 ++++-- test/serena/config/test_serena_config.py | 8 +-- test/serena/test_serena_agent.py | 4 +- test/serena/test_symbol_editing.py | 2 +- 16 files changed, 277 insertions(+), 127 deletions(-) create mode 100644 src/serena/ls_manager.py diff --git a/src/serena/agent.py b/src/serena/agent.py index 8236db55..2cbb5dfa 100644 --- a/src/serena/agent.py +++ b/src/serena/agent.py @@ -22,12 +22,12 @@ from serena.analytics import RegisteredTokenCountEstimator, ToolUsageStats from serena.config.context_mode import RegisteredContext, SerenaAgentContext, SerenaAgentMode from serena.config.serena_config import SerenaConfig, ToolInclusionDefinition, ToolSet from serena.dashboard import SerenaDashboardAPI +from serena.ls_manager import LanguageServerManager from serena.project import Project from serena.prompt_factory import SerenaPromptFactory from serena.tools import ActivateProjectTool, GetCurrentConfigTool, Tool, ToolMarker, ToolRegistry from serena.util.inspection import iter_subclasses from serena.util.logging import MemoryLogHandler -from solidlsp import SolidLanguageServer if TYPE_CHECKING: from serena.gui_log_viewer import GuiLogViewer @@ -86,7 +86,7 @@ class SerenaAgent: # project-specific instances, which will be initialized upon project activation self._active_project: Project | None = None - self.language_server: SolidLanguageServer | None = None + self.language_server_manager: LanguageServerManager | None = None # adjust log level serena_log_level = self.serena_config.log_level @@ -391,15 +391,14 @@ class SerenaAgent: self._active_project = project self._update_active_tools() - def init_language_server() -> None: + def init_language_server_manager() -> None: # start the language server with LogTime("Language server initialization", logger=log): - self.reset_language_server() - assert self.language_server is not None + self.reset_language_server_manager() # initialize the language server in the background (if in language server mode) if self.is_using_language_server(): - self.issue_task(init_language_server) + self.issue_task(init_language_server_manager) if self._project_activation_callback is not None: self._project_activation_callback() @@ -506,10 +505,7 @@ class SerenaAgent: return result_str - def is_language_server_running(self) -> bool: - return self.language_server is not None and self.language_server.is_running() - - def reset_language_server(self) -> None: + def reset_language_server_manager(self) -> None: """ Starts/resets the language server for the current project """ @@ -522,26 +518,18 @@ class SerenaAgent: ls_timeout = tool_timeout - 5 # the LS timeout is for a single call, it should be smaller than the tool timeout # stop the language server if it is running - if self.is_language_server_running(): - assert self.language_server is not None - log.info(f"Stopping the current language server at {self.language_server.repository_root_path} ...") - self.language_server.stop() - self.language_server = None + if self.language_server_manager is not None: + self.language_server_manager.stop_all() + self.language_server_manager = None - # instantiate and start the language server + # instantiate and start the necessarys language servers assert self._active_project is not None - self.language_server = self._active_project.create_language_server( + self.language_server_manager = self._active_project.create_language_server_manager( log_level=self.serena_config.log_level, ls_timeout=ls_timeout, trace_lsp_communication=self.serena_config.trace_lsp_communication, ls_specific_settings=self.serena_config.ls_specific_settings, ) - log.info(f"Starting the language server for {self._active_project.project_name}") - self.language_server.start() - if not self.language_server.is_running(): - raise RuntimeError( - f"Failed to start the language server for {self._active_project.project_name} at {self._active_project.project_root}" - ) def get_tool(self, tool_class: type[TTool]) -> TTool: return self._all_tools[tool_class] # type: ignore @@ -556,11 +544,9 @@ class SerenaAgent: if not hasattr(self, "_is_initialized"): return log.info("SerenaAgent is shutting down ...") - if self.is_language_server_running(): - log.info("Stopping the language server ...") - assert self.language_server is not None - self.language_server.save_cache() - self.language_server.stop() + if self.language_server_manager is not None: + self.language_server_manager.stop_all(save_cache=True) + self.language_server_manager = None if self._gui_log_viewer: log.info("Stopping the GUI log window ...") self._gui_log_viewer.stop() diff --git a/src/serena/cli.py b/src/serena/cli.py index 0db76046..a5a7e2d6 100644 --- a/src/serena/cli.py +++ b/src/serena/cli.py @@ -429,7 +429,7 @@ class ProjectCommands(AutoRegisteringGroup): all_langs = [l.name.lower() for l in Language.iter_all(include_experimental=True)] raise ValueError(f"Unknown language '{language}'. Supported: {all_langs}") generated_conf = ProjectConfig.autogenerate(project_root=project_path, project_language=lang_inst) - print(f"Generated project.yml with language {generated_conf.language.value} at {yml_path}.") + print(f"Generated project.yml with languages {generated_conf.languages} at {yml_path}.") @staticmethod @click.command("index", help="Index a project by saving symbols to the LSP cache.") @@ -460,6 +460,7 @@ class ProjectCommands(AutoRegisteringGroup): serena_config = SerenaConfig.from_config_file() proj = Project.load(os.path.abspath(project)) click.echo(f"Indexing symbols in project {project}…") + # TODO FIXME ls = proj.create_language_server(log_level=lvl, ls_timeout=timeout, ls_specific_settings=serena_config.ls_specific_settings) log_file = os.path.join(project, ".serena", "logs", "indexing.txt") diff --git a/src/serena/code_editor.py b/src/serena/code_editor.py index 6fb1e668..6e9f65b1 100644 --- a/src/serena/code_editor.py +++ b/src/serena/code_editor.py @@ -231,12 +231,11 @@ class CodeEditor(Generic[TSymbol], ABC): class LanguageServerCodeEditor(CodeEditor[LanguageServerSymbol]): def __init__(self, symbol_retriever: LanguageServerSymbolRetriever, agent: Optional["SerenaAgent"] = None): - super().__init__(project_root=symbol_retriever.get_language_server().repository_root_path, agent=agent) + super().__init__(project_root=symbol_retriever.get_root_path(), agent=agent) self._symbol_retriever = symbol_retriever - @property - def _lang_server(self) -> SolidLanguageServer: - return self._symbol_retriever.get_language_server() + def _get_language_server(self, relative_path: str) -> SolidLanguageServer: + return self._symbol_retriever.get_language_server(relative_path) class EditedFile(CodeEditor.EditedFile): def __init__(self, lang_server: SolidLanguageServer, relative_path: str, file_buffer: LSPFileBuffer): @@ -258,12 +257,14 @@ class LanguageServerCodeEditor(CodeEditor[LanguageServerSymbol]): @contextmanager def _open_file_context(self, relative_path: str) -> Iterator["CodeEditor.EditedFile"]: - with self._lang_server.open_file(relative_path) as file_buffer: - yield self.EditedFile(self._lang_server, relative_path, file_buffer) + lang_server = self._get_language_server(relative_path) + with lang_server.open_file(relative_path) as file_buffer: + yield self.EditedFile(lang_server, relative_path, file_buffer) def _get_code_file_content(self, relative_path: str) -> str: """Get the content of a file using the language server.""" - return self._lang_server.language_server.retrieve_full_file_content(relative_path) + lang_server = self._get_language_server(relative_path) + return lang_server.language_server.retrieve_full_file_content(relative_path) def _find_unique_symbol(self, name_path: str, relative_file_path: str) -> LanguageServerSymbol: symbol_candidates = self._symbol_retriever.find_by_name(name_path, within_relative_path=relative_file_path) @@ -289,7 +290,7 @@ class LanguageServerCodeEditor(CodeEditor[LanguageServerSymbol]): # Handle the 'changes' format (URI -> list of TextEdits) for uri, edits in uri_to_edits.items(): file_path = PathUtils.uri_to_path(uri) - relative_path = os.path.relpath(file_path, self._lang_server.repository_root_path) + relative_path = os.path.relpath(file_path, self._symbol_retriever.get_root_path()) modified_relative_paths.append(relative_path) with self._edited_file_context(relative_path) as edited_file: edited_file = cast(LanguageServerCodeEditor.EditedFile, edited_file) @@ -305,12 +306,13 @@ class LanguageServerCodeEditor(CodeEditor[LanguageServerSymbol]): assert symbol.location.line is not None assert symbol.location.column is not None - rename_result = self._lang_server.request_rename_symbol_edit( + lang_server = self._get_language_server(relative_file_path) + rename_result = lang_server.request_rename_symbol_edit( relative_file_path=relative_file_path, line=symbol.location.line, column=symbol.location.column, new_name=new_name ) if rename_result is None: raise ValueError( - f"Language server for {self._lang_server.language_id} returned no rename edits for symbol '{name_path}'. " + f"Language server for {lang_server.language_id} returned no rename edits for symbol '{name_path}'. " f"The symbol might not support renaming." ) modified_files = self._apply_workspace_edit(rename_result) diff --git a/src/serena/config/serena_config.py b/src/serena/config/serena_config.py index c462d561..930a820b 100644 --- a/src/serena/config/serena_config.py +++ b/src/serena/config/serena_config.py @@ -162,7 +162,7 @@ def is_running_in_docker() -> bool: @dataclass(kw_only=True) class ProjectConfig(ToolInclusionDefinition, ToStringMixin): project_name: str - language: Language + languages: list[Language] ignored_paths: list[str] = field(default_factory=list) read_only: bool = False ignore_all_files_in_gitignore: bool = True @@ -212,7 +212,7 @@ class ProjectConfig(ToolInclusionDefinition, ToStringMixin): dominant_language = project_language.value config_with_comments = load_yaml(PROJECT_TEMPLATE_FILE, preserve_comments=True) config_with_comments["project_name"] = project_name - config_with_comments["language"] = dominant_language + config_with_comments["languages"] = [dominant_language] if save_to_disk: save_yaml(str(project_root / cls.rel_path_to_project_yml()), config_with_comments, preserve_comments=True) return cls._from_dict(config_with_comments) @@ -226,19 +226,28 @@ class ProjectConfig(ToolInclusionDefinition, ToStringMixin): """ Create a ProjectConfig instance from a configuration dictionary """ - language_str = data["language"].lower() project_name = data["project_name"] + language_strings = data.get("languages", []) + # backwards compatibility - if language_str == "javascript": - log.warning(f"Found deprecated project language `javascript` in project {project_name}, please change to `typescript`") - language_str = "typescript" - try: - language = Language(language_str) - except ValueError as e: - raise ValueError(f"Invalid language: {data['language']}.\nValid languages are: {[l.value for l in Language]}") from e + lang_name_mapping = {"javascript": "typescript"} + if len(language_strings) == 0 and "language" in data: + language_strings = [data["language"]] + + languages: list[Language] = [] + for language_str in language_strings: + try: + language_str = language_str.lower() + if language_str in lang_name_mapping: + language_str = lang_name_mapping[language_str] + language = Language(language_str) + languages.append(language) + except ValueError as e: + raise ValueError(f"Invalid language: {data['language']}.\nValid language_strings are: {[l.value for l in Language]}") from e + return cls( project_name=project_name, - language=language, + languages=languages, ignored_paths=data.get("ignored_paths", []), excluded_tools=data.get("excluded_tools", []), included_optional_tools=data.get("included_optional_tools", []), diff --git a/src/serena/dashboard.py b/src/serena/dashboard.py index 0468e0fa..b2a64f5e 100644 --- a/src/serena/dashboard.py +++ b/src/serena/dashboard.py @@ -154,7 +154,7 @@ class SerenaDashboardAPI: active_project_name = project.project_name if project else None project_info = { "name": active_project_name, - "language": project.project_config.language.value if project else None, + "language": ", ".join([l.value for l in project.project_config.languages]) if project else None, } # Get context info diff --git a/src/serena/ls_manager.py b/src/serena/ls_manager.py new file mode 100644 index 00000000..c0d228e5 --- /dev/null +++ b/src/serena/ls_manager.py @@ -0,0 +1,88 @@ +import logging +from collections.abc import Callable, Iterator + +from solidlsp import SolidLanguageServer +from solidlsp.ls_config import Language + +log = logging.getLogger(__name__) + + +class LanguageServerManager: + """ + Manages one or more language servers + """ + + def __init__( + self, + language_servers: dict[Language, SolidLanguageServer], + language_server_factories: dict[Language, Callable[[], SolidLanguageServer]] | None = None, + ) -> None: + """ + :param language_servers: a mapping from language to language server; the servers are assumed to be already started. + The first server in the iteration order is used as the default server. + All servers are assumed to serve the same project root. + :param language_server_factories: factories for the language servers, to be used for restarting them if needed + """ + self._language_servers = language_servers + self._language_server_factories = language_server_factories or {} + self._default_language_server = next(iter(language_servers.values())) + self._root_path = self._default_language_server.repository_root_path + + def get_root_path(self) -> str: + return self._root_path + + def _ensure_functional_ls(self, ls: SolidLanguageServer) -> SolidLanguageServer: + if not ls.is_running(): + log.warning(f"Language server for language {ls.language} is not running; restarting ...") + ls = self.restart_language_server(ls.language) + return ls + + def get_language_server(self, relative_path: str) -> SolidLanguageServer: + ls: SolidLanguageServer | None = None + if len(self._language_servers) > 1: + for candidate in self._language_servers.values(): + if not candidate.is_ignored_path(relative_path, ignore_unsupported_files=True): + ls = candidate + break + if ls is None: + ls = self._default_language_server + return self._ensure_functional_ls(ls) + + def restart_language_server(self, language: Language) -> SolidLanguageServer: + """ + Forces recreation and restart of the language server for the given language. + + :param language: the language + :return: the newly created language server + """ + factory = self._language_server_factories.get(language) + if factory is None: + raise ValueError(f"No language server factory found for language {language}; cannot recreate and restart") + self._language_servers[language] = factory() + self._language_servers[language].start() + return self._language_servers[language] + + def iter_language_servers(self) -> Iterator[SolidLanguageServer]: + for ls in self._language_servers.values(): + yield self._ensure_functional_ls(ls) + + def stop_all(self, save_cache: bool = False) -> None: + """ + Stops all managed language servers. + + :param save_cache: whether to save the cache before stopping + """ + for ls in self.iter_language_servers(): + if ls.is_running(): + if save_cache: + ls.save_cache() + log.info(f"Stopping language server for language {ls.language} ...") + ls.stop() + + def save_all_caches(self) -> None: + """ + Saves the caches of all managed language servers. + """ + for ls in self.iter_language_servers(): + if ls.is_running(): + ls.save_cache() diff --git a/src/serena/project.py b/src/serena/project.py index e96266cf..f36e0760 100644 --- a/src/serena/project.py +++ b/src/serena/project.py @@ -1,6 +1,7 @@ import json import logging import os +from collections.abc import Callable from pathlib import Path from typing import Any @@ -9,6 +10,7 @@ from sensai.util.string import ToStringMixin from serena.config.serena_config import DEFAULT_TOOL_TIMEOUT, ProjectConfig, get_serena_managed_in_project_dir from serena.constants import SERENA_FILE_ENCODING, SERENA_MANAGED_DIR_IN_HOME, SERENA_MANAGED_DIR_NAME +from serena.ls_manager import LanguageServerManager from serena.text_utils import MatchedConsecutiveLines, search_files from serena.util.file_system import GitignoreParser, match_path from solidlsp import SolidLanguageServer @@ -101,10 +103,6 @@ class Project(ToStringMixin): def project_name(self) -> str: return self.project_config.project_name - @property - def language(self) -> Language: - return self.project_config.language - @classmethod def load(cls, project_root: str | Path, autogenerate: bool = True) -> "Project": project_root = Path(project_root).resolve() @@ -127,7 +125,8 @@ class Project(ToStringMixin): msg = f"Created and activated a new project with name '{self.project_name}' at {self.project_root}. " else: msg = f"The project with name '{self.project_name}' at {self.project_root} is activated." - msg += f"\nProgramming language: {self.project_config.language.value}; file encoding: {self.project_config.encoding}" + languages_str = ", ".join([lang.value for lang in self.project_config.languages]) + msg += f"\nProgramming languages: {languages_str}; file encoding: {self.project_config.encoding}" memories = self.memories_manager.list_memories() if memories: msg += ( @@ -179,8 +178,13 @@ class Project(ToStringMixin): # Check file extension if it's a file is_file = os.path.isfile(abs_path) if is_file and ignore_non_source_files: - fn_matcher = self.language.get_source_fn_matcher() - if not fn_matcher.is_relevant_filename(abs_path): + is_file_in_supported_language = False + for language in self.project_config.languages: + fn_matcher = language.get_source_fn_matcher() + if fn_matcher.is_relevant_filename(abs_path): + is_file_in_supported_language = True + break + if not is_file_in_supported_language: return True # Create normalized path for consistent handling @@ -345,13 +349,46 @@ class Project(ToStringMixin): source_file_path=relative_file_path, ) - def create_language_server( + def _create_language_server_factory( + self, + language: Language, + ls_timeout: float | None, + ls_specific_settings: dict | None, + log_level: int, + trace_lsp_communication: bool, + ) -> Callable[[], SolidLanguageServer]: + + def factory() -> SolidLanguageServer: + ls_config = LanguageServerConfig( + code_language=language, + ignored_paths=self._ignored_patterns, + trace_lsp_communication=trace_lsp_communication, + encoding=self.project_config.encoding, + ) + ls_logger = LanguageServerLogger(log_level=log_level) + + log.info(f"Creating language server instance for {self.project_root}, language={language}.") + return SolidLanguageServer.create( + ls_config, + ls_logger, + self.project_root, + timeout=ls_timeout, + solidlsp_settings=SolidLSPSettings( + solidlsp_dir=SERENA_MANAGED_DIR_IN_HOME, + project_data_relative_path=SERENA_MANAGED_DIR_NAME, + ls_specific_settings=ls_specific_settings or {}, + ), + ) + + return factory + + def create_language_server_manager( self, log_level: int = logging.INFO, ls_timeout: float | None = DEFAULT_TOOL_TIMEOUT - 5, trace_lsp_communication: bool = False, ls_specific_settings: dict[Language, Any] | None = None, - ) -> SolidLanguageServer: + ) -> LanguageServerManager: """ Create a language server for a project. Note that you will have to start it before performing any LS operations. @@ -365,23 +402,18 @@ class Project(ToStringMixin): see docstrings in the inits of subclasses of SolidLanguageServer to see what values may be passed. :return: the language server """ - ls_config = LanguageServerConfig( - code_language=self.language, - ignored_paths=self._ignored_patterns, - trace_lsp_communication=trace_lsp_communication, - encoding=self.project_config.encoding, - ) - ls_logger = LanguageServerLogger(log_level=log_level) - - log.info(f"Creating language server instance for {self.project_root}.") - return SolidLanguageServer.create( - ls_config, - ls_logger, - self.project_root, - timeout=ls_timeout, - solidlsp_settings=SolidLSPSettings( - solidlsp_dir=SERENA_MANAGED_DIR_IN_HOME, - project_data_relative_path=SERENA_MANAGED_DIR_NAME, - ls_specific_settings=ls_specific_settings or {}, - ), - ) + log.info(f"Creating language server manager for {self.project_root}") + language_servers: dict[Language, SolidLanguageServer] = {} + language_server_factories: dict[Language, Callable[[], SolidLanguageServer]] = {} + for language in self.project_config.languages: + language_server_factory = self._create_language_server_factory( + language, ls_timeout, ls_specific_settings, log_level, trace_lsp_communication + ) + language_server = language_server_factory() + log.info(f"Starting language server for {language} at root {self.project_root}") + language_server.start() + if not language_server.is_running(): + raise RuntimeError(f"Failed to start the language server for {language} for root {self.project_root}") + language_servers[language] = language_server + language_server_factories[language] = language_server_factory + return LanguageServerManager(language_servers, language_server_factories) diff --git a/src/serena/symbol.py b/src/serena/symbol.py index 134b217e..e97b6c77 100644 --- a/src/serena/symbol.py +++ b/src/serena/symbol.py @@ -12,6 +12,7 @@ from solidlsp import SolidLanguageServer from solidlsp.ls import ReferenceInSymbol as LSPReferenceInSymbol from solidlsp.ls_types import Position, SymbolKind, UnifiedSymbolInformation +from .ls_manager import LanguageServerManager from .project import Project if TYPE_CHECKING: @@ -441,24 +442,25 @@ class ReferenceInLanguageServerSymbol(ToStringMixin): class LanguageServerSymbolRetriever: - def __init__(self, lang_server: SolidLanguageServer, agent: Union["SerenaAgent", None] = None) -> None: + def __init__(self, ls: SolidLanguageServer | LanguageServerManager, agent: Union["SerenaAgent", None] = None) -> None: """ - :param lang_server: the language server to use for symbol retrieval as well as editing operations. + :param ls: the language server or language server manager to use for symbol retrieval and editing operations. :param agent: the agent to use (only needed for marking files as modified). You can pass None if you don't need an agent to be aware of file modifications performed by the symbol manager. """ - self._lang_server = lang_server + if isinstance(ls, SolidLanguageServer): + ls_manager = LanguageServerManager({ls.language: ls}) + else: + ls_manager = ls + assert isinstance(ls_manager, LanguageServerManager) + self._ls_manager: LanguageServerManager = ls_manager self.agent = agent - def set_language_server(self, lang_server: SolidLanguageServer) -> None: - """ - Set the language server to use for symbol retrieval and editing operations. - This is useful if you want to change the language server after initializing the SymbolManager. - """ - self._lang_server = lang_server + def get_root_path(self) -> str: + return self._ls_manager.get_root_path() - def get_language_server(self) -> SolidLanguageServer: - return self._lang_server + def get_language_server(self, relative_path: str) -> SolidLanguageServer: + return self._ls_manager.get_language_server(relative_path) def find_by_name( self, @@ -475,24 +477,27 @@ class LanguageServerSymbolRetriever: to symbols within a specific file or directory. """ symbols: list[LanguageServerSymbol] = [] - symbol_roots = self._lang_server.request_full_symbol_tree(within_relative_path=within_relative_path, include_body=include_body) - for root in symbol_roots: - symbols.extend( - LanguageServerSymbol(root).find( - name_path, include_kinds=include_kinds, exclude_kinds=exclude_kinds, substring_matching=substring_matching + for lang_server in self._ls_manager.iter_language_servers(): + symbol_roots = lang_server.request_full_symbol_tree(within_relative_path=within_relative_path, include_body=include_body) + for root in symbol_roots: + symbols.extend( + LanguageServerSymbol(root).find( + name_path, include_kinds=include_kinds, exclude_kinds=exclude_kinds, substring_matching=substring_matching + ) ) - ) return symbols def get_document_symbols(self, relative_path: str) -> list[LanguageServerSymbol]: - symbol_dicts, _roots = self._lang_server.request_document_symbols(relative_path, include_body=False) + lang_server = self.get_language_server(relative_path) + symbol_dicts, _roots = lang_server.request_document_symbols(relative_path, include_body=False) symbols = [LanguageServerSymbol(s) for s in symbol_dicts] return symbols def find_by_location(self, location: LanguageServerSymbolLocation) -> LanguageServerSymbol | None: if location.relative_path is None: return None - symbol_dicts, _roots = self._lang_server.request_document_symbols(location.relative_path, include_body=False) + lang_server = self.get_language_server(location.relative_path) + symbol_dicts, _roots = lang_server.request_document_symbols(location.relative_path, include_body=False) for symbol_dict in symbol_dicts: symbol = LanguageServerSymbol(symbol_dict) if symbol.location == location: @@ -561,7 +566,8 @@ class LanguageServerSymbolRetriever: assert symbol_location.relative_path is not None assert symbol_location.line is not None assert symbol_location.column is not None - references = self._lang_server.request_referencing_symbols( + lang_server = self.get_language_server(symbol_location.relative_path) + references = lang_server.request_referencing_symbols( relative_file_path=symbol_location.relative_path, line=symbol_location.line, column=symbol_location.column, @@ -589,7 +595,8 @@ class LanguageServerSymbolRetriever: return cls(name_path=symbol.get_name_path(), kind=int(symbol.symbol_kind)) def get_symbol_overview(self, relative_path: str) -> dict[str, list[SymbolOverviewElement]]: - path_to_unified_symbols = self._lang_server.request_overview(relative_path) + lang_server = self.get_language_server(relative_path) + path_to_unified_symbols = lang_server.request_overview(relative_path) result = {} for file_path, unified_symbols in path_to_unified_symbols.items(): # TODO: maybe include not just top-level symbols? We could filter by kind to exclude variables diff --git a/src/serena/tools/symbol_tools.py b/src/serena/tools/symbol_tools.py index 1f8cbf58..bd31fc48 100644 --- a/src/serena/tools/symbol_tools.py +++ b/src/serena/tools/symbol_tools.py @@ -42,7 +42,7 @@ class RestartLanguageServerTool(Tool, ToolMarkerOptional): """Use this tool only on explicit user request or after confirmation. It may be necessary to restart the language server if it hangs. """ - self.agent.reset_language_server() + self.agent.reset_language_server_manager() return SUCCESS_RESULT diff --git a/src/serena/tools/tools_base.py b/src/serena/tools/tools_base.py index 58e70ea2..5fe58675 100644 --- a/src/serena/tools/tools_base.py +++ b/src/serena/tools/tools_base.py @@ -47,9 +47,9 @@ class Component(ABC): def create_language_server_symbol_retriever(self) -> LanguageServerSymbolRetriever: if not self.agent.is_using_language_server(): raise Exception("Cannot create LanguageServerSymbolRetriever; agent is not in language server mode.") - language_server = self.agent.language_server - assert language_server is not None - return LanguageServerSymbolRetriever(language_server, agent=self.agent) + language_server_manager = self.agent.language_server_manager + assert language_server_manager is not None + return LanguageServerSymbolRetriever(language_server_manager, agent=self.agent) @property def project(self) -> Project: @@ -244,18 +244,24 @@ class Tool(Component): "Error: No active project. Ask the user to provide the project path or to select a project from this list of known projects: " + f"{self.agent.serena_config.project_names}" ) - if self.agent.is_using_language_server() and not self.agent.is_language_server_running(): - log.info("Language server is not running. Starting it ...") - self.agent.reset_language_server() # apply the actual tool try: result = apply_fn(**kwargs) except SolidLSPException as e: if e.is_language_server_terminated(): - log.error(f"Language server terminated while executing tool ({e}). Restarting the language server and retrying ...") - self.agent.reset_language_server() - result = apply_fn(**kwargs) + affected_language = e.get_affected_language() + if affected_language is not None and self.agent.language_server_manager is not None: + log.error( + f"Language server terminated while executing tool ({e}). Restarting the language server and retrying ..." + ) + self.agent.language_server_manager.restart_language_server(affected_language) + result = apply_fn(**kwargs) + else: + log.error( + f"Language server terminated while executing tool ({e}), but affected language is unknown. Not retrying." + ) + raise else: raise @@ -273,8 +279,8 @@ class Tool(Component): log.info(f"Result: {result}") try: - if self.agent.language_server is not None: - self.agent.language_server.save_cache() + if self.agent.language_server_manager is not None: + self.agent.language_server_manager.save_all_caches() except Exception as e: log.error(f"Error saving language server cache: {e}") diff --git a/src/solidlsp/ls.py b/src/solidlsp/ls.py index 849b1256..c144c2aa 100644 --- a/src/solidlsp/ls.py +++ b/src/solidlsp/ls.py @@ -214,6 +214,7 @@ class SolidLanguageServer(ABC): ) self.server = SolidLanguageServerHandler( process_launch_info, + language=self.language, logger=logging_fn, start_independent_lsp_process=config.start_independent_lsp_process, ) diff --git a/src/solidlsp/ls_exceptions.py b/src/solidlsp/ls_exceptions.py index 7c03cfe5..61751eff 100644 --- a/src/solidlsp/ls_exceptions.py +++ b/src/solidlsp/ls_exceptions.py @@ -2,6 +2,8 @@ This module contains the exceptions raised by the framework. """ +from solidlsp.ls_config import Language + class SolidLSPException(Exception): def __init__(self, message: str, cause: Exception | None = None): @@ -26,6 +28,16 @@ class SolidLSPException(Exception): return isinstance(self.cause, LanguageServerTerminatedException) + def get_affected_language(self) -> Language | None: + """ + :return: the affected language for the case where the exception is caused by the language server having terminated + """ + from .ls_handler import LanguageServerTerminatedException + + if isinstance(self.cause, LanguageServerTerminatedException): + return self.cause.language + return None + def __str__(self) -> str: """ Returns a string representation of the exception. diff --git a/src/solidlsp/ls_handler.py b/src/solidlsp/ls_handler.py index 02363aaf..9be3f782 100644 --- a/src/solidlsp/ls_handler.py +++ b/src/solidlsp/ls_handler.py @@ -14,6 +14,7 @@ from typing import Any import psutil from sensai.util.string import ToStringMixin +from solidlsp.ls_config import Language from solidlsp.ls_exceptions import SolidLSPException from solidlsp.ls_request import LanguageServerRequest from solidlsp.lsp_protocol_handler.lsp_requests import LspNotification @@ -42,9 +43,10 @@ class LanguageServerTerminatedException(Exception): Exception raised when the language server process has terminated unexpectedly. """ - def __init__(self, message: str, cause: Exception | None = None) -> None: + def __init__(self, message: str, language: Language, cause: Exception | None = None) -> None: super().__init__(message) self.message = message + self.language = language self.cause = cause def __str__(self) -> str: @@ -132,10 +134,12 @@ class SolidLanguageServerHandler: def __init__( self, process_launch_info: ProcessLaunchInfo, + language: Language, logger: Callable[[str, str, StringDict | str], None] | None = None, start_independent_lsp_process=True, request_timeout: float | None = None, ) -> None: + self.language = language self.send = LanguageServerRequest(self) self.notify = LspNotification(self.send_notification) @@ -310,8 +314,7 @@ class SolidLanguageServerHandler: if self.logger is not None: self.logger("client", "logger", message) - @staticmethod - def _read_bytes_from_process(process, stream, num_bytes): + def _read_bytes_from_process(self, process, stream, num_bytes): """Read exactly num_bytes from process stdout""" data = b"" while len(data) < num_bytes: @@ -319,7 +322,8 @@ class SolidLanguageServerHandler: if not chunk: if process.poll() is not None: raise LanguageServerTerminatedException( - f"Process terminated while trying to read response (read {num_bytes} of {len(data)} bytes before termination)" + f"Process terminated while trying to read response (read {num_bytes} of {len(data)} bytes before termination)", + language=self.language, ) # Process still running but no data available yet, retry after a short delay time.sleep(0.01) @@ -356,13 +360,15 @@ class SolidLanguageServerHandler: except LanguageServerTerminatedException as e: exception = e except (BrokenPipeError, ConnectionResetError) as e: - exception = LanguageServerTerminatedException("Language server process terminated while reading stdout", cause=e) + exception = LanguageServerTerminatedException("Language server process terminated while reading stdout", self.language, cause=e) except Exception as e: - exception = LanguageServerTerminatedException("Unexpected error while reading stdout from language server process", cause=e) + exception = LanguageServerTerminatedException( + "Unexpected error while reading stdout from language server process", self.language, cause=e + ) log.info("Language server stdout reader thread has terminated") if not self._is_shutting_down: if exception is None: - exception = LanguageServerTerminatedException("Language server stdout read process terminated unexpectedly") + exception = LanguageServerTerminatedException("Language server stdout read process terminated unexpectedly", self.language) log.error(str(exception)) self._cancel_pending_requests(exception) diff --git a/test/serena/config/test_serena_config.py b/test/serena/config/test_serena_config.py index 14eb3cd9..8c2329fe 100644 --- a/test/serena/config/test_serena_config.py +++ b/test/serena/config/test_serena_config.py @@ -50,7 +50,7 @@ class TestProjectConfigAutogenerate: # Verify the configuration assert config.project_name == self.project_path.name - assert config.language == Language.PYTHON + assert config.languages == [Language.PYTHON] def test_autogenerate_with_multiple_languages(self): """Test autogeneration picks dominant language when multiple are present.""" @@ -62,7 +62,7 @@ class TestProjectConfigAutogenerate: # Run autogenerate - should pick Python as dominant config = ProjectConfig.autogenerate(self.project_path, save_to_disk=False) - assert config.language == Language.PYTHON + assert config.languages == [Language.PYTHON] def test_autogenerate_saves_to_disk(self): """Test that autogenerate can save the configuration to disk.""" @@ -78,7 +78,7 @@ class TestProjectConfigAutogenerate: assert config_path.exists() # Verify the content - assert config.language == Language.GO + assert config.languages == [Language.GO] def test_autogenerate_nonexistent_path(self): """Test that autogenerate raises FileNotFoundError for non-existent path.""" @@ -115,7 +115,7 @@ class TestProjectConfigAutogenerate: config = ProjectConfig.autogenerate(self.project_path, project_name=custom_name, save_to_disk=False) assert config.project_name == custom_name - assert config.language == Language.TYPESCRIPT + assert config.languages == [Language.TYPESCRIPT] def test_autogenerate_error_message_format(self): """Test the specific format of the error message for better user experience.""" diff --git a/test/serena/test_serena_agent.py b/test/serena/test_serena_agent.py index 4f067ffb..882b45be 100644 --- a/test/serena/test_serena_agent.py +++ b/test/serena/test_serena_agent.py @@ -37,7 +37,7 @@ def serena_config(): project_root=str(repo_path), project_config=ProjectConfig( project_name=project_name, - language=language, + languages=[language], ignored_paths=[], excluded_tools=set(), read_only=False, @@ -248,7 +248,7 @@ class TestSerenaAgent: and expected_kind.lower() in s["kind"].lower() and expected_file in s["relative_path"] for s in symbols - ), f"Expected to find {name_path} ({expected_kind}) in {expected_file} for {agent._active_project.language.name}. Symbols: {symbols}" + ), f"Expected to find {name_path} ({expected_kind}) in {expected_file}. Symbols: {symbols}" @pytest.mark.parametrize( "serena_agent,name_path", diff --git a/test/serena/test_symbol_editing.py b/test/serena/test_symbol_editing.py index ee97149a..6a56a30c 100644 --- a/test/serena/test_symbol_editing.py +++ b/test/serena/test_symbol_editing.py @@ -205,7 +205,7 @@ class EditingTest(ABC): log.info(f"Starting language server for {self.language} {self.rel_path}") language_server.start() log.info(f"Language server started for {self.language} {self.rel_path}") - yield LanguageServerSymbolRetriever(lang_server=language_server) + yield LanguageServerSymbolRetriever(ls=language_server) finally: if language_server is not None and language_server.is_running(): log.info(f"Stopping language server for {self.language} {self.rel_path}") From c7a0afb6ce5d69312cc11dfeb1c1e87a913b7cd3 Mon Sep 17 00:00:00 2001 From: Dominik Jain Date: Tue, 28 Oct 2025 14:59:45 +0100 Subject: [PATCH 02/10] Spawn language servers in parallel (using threads) --- src/serena/project.py | 48 ++++++++++++++++++++++++++++++++++--------- src/solidlsp/ls.py | 11 +++++++++- 2 files changed, 48 insertions(+), 11 deletions(-) diff --git a/src/serena/project.py b/src/serena/project.py index f36e0760..0484784f 100644 --- a/src/serena/project.py +++ b/src/serena/project.py @@ -1,11 +1,13 @@ import json import logging import os +import threading from collections.abc import Callable from pathlib import Path from typing import Any import pathspec +from sensai.util.logging import LogTime from sensai.util.string import ToStringMixin from serena.config.serena_config import DEFAULT_TOOL_TIMEOUT, ProjectConfig, get_serena_managed_in_project_dir @@ -405,15 +407,41 @@ class Project(ToStringMixin): log.info(f"Creating language server manager for {self.project_root}") language_servers: dict[Language, SolidLanguageServer] = {} language_server_factories: dict[Language, Callable[[], SolidLanguageServer]] = {} + threads = [] + exceptions = {} + lock = threading.Lock() + + def start_language_server(language: Language, factory: Callable[[], SolidLanguageServer]): + try: + with LogTime(f"Language server startup (language={language.value})"): + language_server = factory() + language_server.start() + if not language_server.is_running(): + raise RuntimeError( + f"Failed to start the language server for language {language.value} with root {self.project_root}" + ) + with lock: + language_servers[language] = language_server + except Exception as e: + log.error(f"Error starting language server for language {language.value}: {e}", exc_info=e) + with lock: + exceptions[language] = e + + # start language servers in parallel threads for language in self.project_config.languages: - language_server_factory = self._create_language_server_factory( - language, ls_timeout, ls_specific_settings, log_level, trace_lsp_communication - ) - language_server = language_server_factory() - log.info(f"Starting language server for {language} at root {self.project_root}") - language_server.start() - if not language_server.is_running(): - raise RuntimeError(f"Failed to start the language server for {language} for root {self.project_root}") - language_servers[language] = language_server - language_server_factories[language] = language_server_factory + factory = self._create_language_server_factory(language, ls_timeout, ls_specific_settings, log_level, trace_lsp_communication) + language_server_factories[language] = factory + thread = threading.Thread(target=start_language_server, args=(language, factory), name="StartLS:" + language.value) + thread.start() + threads.append(thread) + for thread in threads: + thread.join() + + # If any server failed to start up, raise an exception and stop all started language servers + if exceptions: + for ls in language_servers.values(): + ls.stop() + failure_messages = "\n".join([f"{lang.value}: {e}" for lang, e in exceptions.items()]) + raise Exception(f"Failed to start language servers:\n{failure_messages}") + return LanguageServerManager(language_servers, language_server_factories) diff --git a/src/solidlsp/ls.py b/src/solidlsp/ls.py index c144c2aa..93575dbb 100644 --- a/src/solidlsp/ls.py +++ b/src/solidlsp/ls.py @@ -1688,7 +1688,16 @@ class SolidLanguageServer(ABC): return self def stop(self, shutdown_timeout: float = 2.0) -> None: - self._shutdown(timeout=shutdown_timeout) + """ + Stops the language server process. + This function never raises an exception (any exceptions during shutdown are logged). + + :param shutdown_timeout: time, in seconds, to wait for the server to shutdown gracefully before killing it + """ + try: + self._shutdown(timeout=shutdown_timeout) + except Exception as e: + self.logger.log(f"Exception while shutting down language server: {e}", logging.WARNING) @property def language_server(self) -> Self: From 747d3421660034520c03d9ad84a73e9bcd88ecb0 Mon Sep 17 00:00:00 2001 From: Dominik Jain Date: Tue, 28 Oct 2025 15:46:11 +0100 Subject: [PATCH 03/10] Make stream reader thread names indicate the language --- src/solidlsp/ls_handler.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/solidlsp/ls_handler.py b/src/solidlsp/ls_handler.py index 9be3f782..7dd0189c 100644 --- a/src/solidlsp/ls_handler.py +++ b/src/solidlsp/ls_handler.py @@ -215,12 +215,12 @@ class SolidLanguageServerHandler: # start threads to read stdout and stderr of the process threading.Thread( target=self._read_ls_process_stdout, - name="LSP-stdout-reader", + name=f"LSP-stdout-reader:{self.language.value}", daemon=True, ).start() threading.Thread( target=self._read_ls_process_stderr, - name="LSP-stderr-reader", + name=f"LSP-stderr-reader:{self.language.value}", daemon=True, ).start() From 27b933796a90fa21a52f70e9fa19ed501c96fa1b Mon Sep 17 00:00:00 2001 From: Dominik Jain Date: Tue, 28 Oct 2025 16:10:30 +0100 Subject: [PATCH 04/10] Improve LS-related error handling, reporting and propagation --- src/serena/agent.py | 32 ++++++++++++++++++++++---------- src/serena/tools/tools_base.py | 16 ++++++++-------- 2 files changed, 30 insertions(+), 18 deletions(-) diff --git a/src/serena/agent.py b/src/serena/agent.py index 2cbb5dfa..69200043 100644 --- a/src/serena/agent.py +++ b/src/serena/agent.py @@ -86,7 +86,7 @@ class SerenaAgent: # project-specific instances, which will be initialized upon project activation self._active_project: Project | None = None - self.language_server_manager: LanguageServerManager | None = None + self._language_server_manager: LanguageServerManager | None = None # adjust log level serena_log_level = self.serena_config.log_level @@ -192,6 +192,18 @@ class SerenaAgent: process.start() process.join(timeout=1) + def get_language_server_manager(self) -> LanguageServerManager | None: + return self._language_server_manager + + def get_language_server_manager_or_raise(self) -> LanguageServerManager: + if self._language_server_manager is None: + raise Exception( + "The language server manager is not initialized, indicating a problem during project activation. " + "Inform the user, telling them to inspect Serena's logs in order to determine the issue. " + "IMPORTANT: Wait for further instructions before you continue!" + ) + return self._language_server_manager + def get_context(self) -> SerenaAgentContext: return self._context @@ -518,13 +530,13 @@ class SerenaAgent: ls_timeout = tool_timeout - 5 # the LS timeout is for a single call, it should be smaller than the tool timeout # stop the language server if it is running - if self.language_server_manager is not None: - self.language_server_manager.stop_all() - self.language_server_manager = None + if self._language_server_manager is not None: + self._language_server_manager.stop_all() + self._language_server_manager = None - # instantiate and start the necessarys language servers - assert self._active_project is not None - self.language_server_manager = self._active_project.create_language_server_manager( + # instantiate and start the necessary language servers + assert self._active_project is not None, "Active project required" + self._language_server_manager = self._active_project.create_language_server_manager( log_level=self.serena_config.log_level, ls_timeout=ls_timeout, trace_lsp_communication=self.serena_config.trace_lsp_communication, @@ -544,9 +556,9 @@ class SerenaAgent: if not hasattr(self, "_is_initialized"): return log.info("SerenaAgent is shutting down ...") - if self.language_server_manager is not None: - self.language_server_manager.stop_all(save_cache=True) - self.language_server_manager = None + if self._language_server_manager is not None: + self._language_server_manager.stop_all(save_cache=True) + self._language_server_manager = None if self._gui_log_viewer: log.info("Stopping the GUI log window ...") self._gui_log_viewer.stop() diff --git a/src/serena/tools/tools_base.py b/src/serena/tools/tools_base.py index 5fe58675..c4b0f300 100644 --- a/src/serena/tools/tools_base.py +++ b/src/serena/tools/tools_base.py @@ -47,8 +47,7 @@ class Component(ABC): def create_language_server_symbol_retriever(self) -> LanguageServerSymbolRetriever: if not self.agent.is_using_language_server(): raise Exception("Cannot create LanguageServerSymbolRetriever; agent is not in language server mode.") - language_server_manager = self.agent.language_server_manager - assert language_server_manager is not None + language_server_manager = self.agent.get_language_server_manager_or_raise() return LanguageServerSymbolRetriever(language_server_manager, agent=self.agent) @property @@ -239,7 +238,7 @@ class Tool(Component): try: # check whether the tool requires an active project and language server if not isinstance(self, ToolMarkerDoesNotRequireActiveProject): - if self.agent._active_project is None: + if self.agent.get_active_project() is None: return ( "Error: No active project. Ask the user to provide the project path or to select a project from this list of known projects: " + f"{self.agent.serena_config.project_names}" @@ -251,11 +250,11 @@ class Tool(Component): except SolidLSPException as e: if e.is_language_server_terminated(): affected_language = e.get_affected_language() - if affected_language is not None and self.agent.language_server_manager is not None: + if affected_language is not None: log.error( f"Language server terminated while executing tool ({e}). Restarting the language server and retrying ..." ) - self.agent.language_server_manager.restart_language_server(affected_language) + self.agent.get_language_server_manager_or_raise().restart_language_server(affected_language) result = apply_fn(**kwargs) else: log.error( @@ -271,7 +270,7 @@ class Tool(Component): except Exception as e: if not catch_exceptions: raise - msg = f"Error executing tool: {e}" + msg = f"Error executing tool: {e.__class__.__name__} - {e}" log.error(f"Error executing tool: {e}", exc_info=e) result = msg @@ -279,8 +278,9 @@ class Tool(Component): log.info(f"Result: {result}") try: - if self.agent.language_server_manager is not None: - self.agent.language_server_manager.save_all_caches() + ls_manager = self.agent.get_language_server_manager() + if ls_manager is not None: + ls_manager.save_all_caches() except Exception as e: log.error(f"Error saving language server cache: {e}") From 66a72d2b76f4aaa7fe092a61ea45f59f15df8c99 Mon Sep 17 00:00:00 2001 From: Dominik Jain Date: Tue, 28 Oct 2025 18:28:14 +0100 Subject: [PATCH 05/10] Update language settings in project configuration template --- src/serena/resources/project.template.yml | 21 ++++++++++++++++----- 1 file changed, 16 insertions(+), 5 deletions(-) diff --git a/src/serena/resources/project.template.yml b/src/serena/resources/project.template.yml index 0b2b59d9..f003b361 100644 --- a/src/serena/resources/project.template.yml +++ b/src/serena/resources/project.template.yml @@ -1,9 +1,19 @@ -# language of the project (csharp, python, rust, java, typescript, go, cpp, or ruby) -# * For C, use cpp -# * For JavaScript, use typescript +# list of languages for which language servers are started; choose from: +# al bash clojure cpp csharp csharp_omnisharp +# dart elixir elm erlang fortran go +# haskell java julia kotlin lua markdown +# nix perl php python python_jedi r +# rego ruby ruby_solargraph rust scala swift +# terraform typescript typescript_vts zig +# Note: +# - For C, use cpp +# - For JavaScript, use typescript # Special requirements: -# * csharp: Requires the presence of a .sln file in the project folder. -language: python +# - csharp: Requires the presence of a .sln file in the project folder. +# When using multiple languages, the first language server that supports a given file will be used for that file. +# The first language is the default language and the respective language server will be used as a fallback. +# Note that when using the JetBrains backend, language servers are not used and this list is correspondingly ignored. +languages: ["python"] # the encoding used by text files in the project # For a list of possible encodings, see https://docs.python.org/3.11/library/codecs.html#standard-encodings @@ -12,6 +22,7 @@ encoding: "utf-8" # whether to use the project's gitignore file to ignore files # Added on 2025-04-07 ignore_all_files_in_gitignore: true + # list of additional paths to ignore # same syntax as gitignore, so you can use * and ** # Was previously called `ignored_dirs`, please update your config if you are using that. From 31c39947a0824a312d8b074aa53dea2abb19892b Mon Sep 17 00:00:00 2001 From: Dominik Jain Date: Tue, 28 Oct 2025 18:43:47 +0100 Subject: [PATCH 06/10] Adapt CLI tools to use LanguageServerManager --- src/serena/cli.py | 84 +++++++++++++++++++++++++++-------------------- 1 file changed, 48 insertions(+), 36 deletions(-) diff --git a/src/serena/cli.py b/src/serena/cli.py index a5a7e2d6..e0b64d85 100644 --- a/src/serena/cli.py +++ b/src/serena/cli.py @@ -459,34 +459,42 @@ class ProjectCommands(AutoRegisteringGroup): logging.configure(level=lvl) serena_config = SerenaConfig.from_config_file() proj = Project.load(os.path.abspath(project)) - click.echo(f"Indexing symbols in project {project}…") - # TODO FIXME - ls = proj.create_language_server(log_level=lvl, ls_timeout=timeout, ls_specific_settings=serena_config.ls_specific_settings) - log_file = os.path.join(project, ".serena", "logs", "indexing.txt") + click.echo(f"Indexing symbols in project {project} …") + ls_mgr = proj.create_language_server_manager( + log_level=lvl, ls_timeout=timeout, ls_specific_settings=serena_config.ls_specific_settings + ) + try: + log_file = os.path.join(project, ".serena", "logs", "indexing.txt") - collected_exceptions: list[Exception] = [] - files_failed = [] - with ls.start_server(): files = proj.gather_source_files() - for i, f in enumerate(tqdm(files, desc="Indexing")): - try: - ls.request_document_symbols(f, include_body=False) - ls.request_document_symbols(f, include_body=True) - except Exception as e: - log.error(f"Failed to index {f}, continuing.") - collected_exceptions.append(e) - files_failed.append(f) - if (i + 1) % 10 == 0: - ls.save_cache() - ls.save_cache() - click.echo(f"Symbols saved to {ls.cache_path}") - if len(files_failed) > 0: - os.makedirs(os.path.dirname(log_file), exist_ok=True) - with open(log_file, "w") as f: - for file, exception in zip(files_failed, collected_exceptions, strict=True): - f.write(f"{file}\n") - f.write(f"{exception}\n") - click.echo(f"Failed to index {len(files_failed)} files, see:\n{log_file}") + + servers = list(ls_mgr.iter_language_servers()) + for k, ls in enumerate(servers, start=1): + click.echo(f"Indexing for language {ls.language.value} ({k}/{len(servers)}) …") + collected_exceptions: list[Exception] = [] + files_failed = [] + for i, f in enumerate(tqdm(files, desc="Indexing")): + try: + ls.request_document_symbols(f, include_body=False) + ls.request_document_symbols(f, include_body=True) + except Exception as e: + log.error(f"Failed to index {f}, continuing.") + collected_exceptions.append(e) + files_failed.append(f) + if (i + 1) % 10 == 0: + ls.save_cache() + ls.save_cache() + click.echo(f"Symbols saved to {ls.cache_path}") + + if len(files_failed) > 0: + os.makedirs(os.path.dirname(log_file), exist_ok=True) + with open(log_file, "w") as f: + for file, exception in zip(files_failed, collected_exceptions, strict=True): + f.write(f"{file}\n") + f.write(f"{exception}\n") + click.echo(f"Failed to index {len(files_failed)} files, see:\n{log_file}") + finally: + ls_mgr.stop_all() @staticmethod @click.command("is_ignored_path", help="Check if a path is ignored by the project configuration.") @@ -523,16 +531,20 @@ class ProjectCommands(AutoRegisteringGroup): if proj.is_ignored_path(file, ignore_non_source_files=True): click.echo(f"'{file}' is ignored or declared as non-code file by the project configuration, won't index.") exit(1) - ls = proj.create_language_server() - with ls.start_server(): - symbols, _ = ls.request_document_symbols(file, include_body=False) - ls.request_document_symbols(file, include_body=True) - if verbose: - click.echo(f"Symbols in file '{file}':") - for symbol in symbols: - click.echo(f" - {symbol['name']} at line {symbol['selectionRange']['start']['line']} of kind {symbol['kind']}") - ls.save_cache() - click.echo(f"Successfully indexed file '{file}', {len(symbols)} symbols saved to {ls.cache_path}.") + ls_mgr = proj.create_language_server_manager() + try: + for ls in ls_mgr.iter_language_servers(): + click.echo(f"Indexing for language {ls.language.value} …") + symbols, _ = ls.request_document_symbols(file, include_body=False) + ls.request_document_symbols(file, include_body=True) + if verbose: + click.echo(f"Symbols in file '{file}':") + for symbol in symbols: + click.echo(f" - {symbol['name']} at line {symbol['selectionRange']['start']['line']} of kind {symbol['kind']}") + ls.save_cache() + click.echo(f"Successfully indexed file '{file}', {len(symbols)} symbols saved to {ls.cache_path}.") + finally: + ls_mgr.stop_all() @staticmethod @click.command("health-check", help="Perform a comprehensive health check of the project's tools and language server.") From d693580a7df0d1fb6bceaf7ce578663c9633d200 Mon Sep 17 00:00:00 2001 From: Dominik Jain Date: Tue, 28 Oct 2025 23:30:05 +0100 Subject: [PATCH 07/10] Support multiple languages in generate-yml CLI command --- src/serena/cli.py | 21 ++++++++++++--------- src/serena/config/serena_config.py | 11 ++++++----- 2 files changed, 18 insertions(+), 14 deletions(-) diff --git a/src/serena/cli.py b/src/serena/cli.py index e0b64d85..fb78cd3d 100644 --- a/src/serena/cli.py +++ b/src/serena/cli.py @@ -416,19 +416,22 @@ class ProjectCommands(AutoRegisteringGroup): @staticmethod @click.command("generate-yml", help="Generate a project.yml file.") @click.argument("project_path", type=click.Path(exists=True, file_okay=False), default=os.getcwd()) - @click.option("--language", type=str, default=None, help="Programming language; inferred if not specified.") - def generate_yml(project_path: str, language: str | None = None) -> None: + @click.option( + "--language", type=str, multiple=True, help="Programming language(s); inferred if not specified. Can be passed multiple times." + ) + def generate_yml(project_path: str, language: tuple[str, ...]) -> None: yml_path = os.path.join(project_path, ProjectConfig.rel_path_to_project_yml()) if os.path.exists(yml_path): raise FileExistsError(f"Project file {yml_path} already exists.") - lang_inst = None + languages: list[Language] = [] if language: - try: - lang_inst = Language[language.upper()] - except KeyError: - all_langs = [l.name.lower() for l in Language.iter_all(include_experimental=True)] - raise ValueError(f"Unknown language '{language}'. Supported: {all_langs}") - generated_conf = ProjectConfig.autogenerate(project_root=project_path, project_language=lang_inst) + for lang in language: + try: + languages.append(Language(lang.lower())) + except ValueError: + all_langs = [l.value for l in Language] + raise ValueError(f"Unknown language '{lang}'. Supported: {all_langs}") + generated_conf = ProjectConfig.autogenerate(project_root=project_path, languages=languages if languages else None) print(f"Generated project.yml with languages {generated_conf.languages} at {yml_path}.") @staticmethod diff --git a/src/serena/config/serena_config.py b/src/serena/config/serena_config.py index 930a820b..2f17d058 100644 --- a/src/serena/config/serena_config.py +++ b/src/serena/config/serena_config.py @@ -176,7 +176,7 @@ class ProjectConfig(ToolInclusionDefinition, ToStringMixin): @classmethod def autogenerate( - cls, project_root: str | Path, project_name: str | None = None, project_language: Language | None = None, save_to_disk: bool = True + cls, project_root: str | Path, project_name: str | None = None, languages: list[Language] | None = None, save_to_disk: bool = True ) -> Self: """ Autogenerate a project configuration for a given project root. @@ -184,7 +184,7 @@ class ProjectConfig(ToolInclusionDefinition, ToStringMixin): :param project_root: the path to the project root :param project_name: the name of the project; if None, the name of the project will be the name of the directory containing the project - :param project_language: the programming language of the project; if None, it will be determined automatically + :param languages: the languages of the project; if None, they will be determined automatically :param save_to_disk: whether to save the project configuration to disk :return: the project configuration """ @@ -193,7 +193,7 @@ class ProjectConfig(ToolInclusionDefinition, ToStringMixin): raise FileNotFoundError(f"Project root not found: {project_root}") with LogTime("Project configuration auto-generation", logger=log): project_name = project_name or project_root.name - if project_language is None: + if languages is None: language_composition = determine_programming_language_composition(str(project_root)) if len(language_composition) == 0: raise ValueError( @@ -208,11 +208,12 @@ class ProjectConfig(ToolInclusionDefinition, ToStringMixin): ) # find the language with the highest percentage dominant_language = max(language_composition.keys(), key=lambda lang: language_composition[lang]) + languages_to_use = [dominant_language] else: - dominant_language = project_language.value + languages_to_use = [lang.value for lang in languages] config_with_comments = load_yaml(PROJECT_TEMPLATE_FILE, preserve_comments=True) config_with_comments["project_name"] = project_name - config_with_comments["languages"] = [dominant_language] + config_with_comments["languages"] = languages_to_use if save_to_disk: save_yaml(str(project_root / cls.rel_path_to_project_yml()), config_with_comments, preserve_comments=True) return cls._from_dict(config_with_comments) From 653e8c4b9047ded673f55a8f9a24246df06cdd1f Mon Sep 17 00:00:00 2001 From: Dominik Jain Date: Wed, 29 Oct 2025 00:45:20 +0100 Subject: [PATCH 08/10] Support the dynamic addition of languages while a project is already activated Move LanguageServerManager ownership from SerenaAgent to Project Add Project.save_config to support saving updated configurations --- src/serena/agent.py | 41 +++++----- src/serena/config/serena_config.py | 9 +++ src/serena/ls_manager.py | 88 +++++++++++++++++++--- src/serena/project.py | 115 +++++++++++++++++------------ 4 files changed, 178 insertions(+), 75 deletions(-) diff --git a/src/serena/agent.py b/src/serena/agent.py index 69200043..53ed791b 100644 --- a/src/serena/agent.py +++ b/src/serena/agent.py @@ -28,6 +28,7 @@ from serena.prompt_factory import SerenaPromptFactory from serena.tools import ActivateProjectTool, GetCurrentConfigTool, Tool, ToolMarker, ToolRegistry from serena.util.inspection import iter_subclasses from serena.util.logging import MemoryLogHandler +from solidlsp.ls_config import Language if TYPE_CHECKING: from serena.gui_log_viewer import GuiLogViewer @@ -86,7 +87,6 @@ class SerenaAgent: # project-specific instances, which will be initialized upon project activation self._active_project: Project | None = None - self._language_server_manager: LanguageServerManager | None = None # adjust log level serena_log_level = self.serena_config.log_level @@ -193,16 +193,19 @@ class SerenaAgent: process.join(timeout=1) def get_language_server_manager(self) -> LanguageServerManager | None: - return self._language_server_manager + if self._active_project is not None: + return self._active_project.language_server_manager + return None def get_language_server_manager_or_raise(self) -> LanguageServerManager: - if self._language_server_manager is None: + language_server_manager = self.get_language_server_manager() + if language_server_manager is None: raise Exception( "The language server manager is not initialized, indicating a problem during project activation. " "Inform the user, telling them to inspect Serena's logs in order to determine the issue. " "IMPORTANT: Wait for further instructions before you continue!" ) - return self._language_server_manager + return language_server_manager def get_context(self) -> SerenaAgentContext: return self._context @@ -381,7 +384,7 @@ class SerenaAgent: log.info(f"Scheduling {task_name}") return self._task_executor.submit(task_execution_wrapper) - def execute_task(self, task: Callable[[], T]) -> T: + def execute_task(self, task: Callable[[], T], name: str | None = None) -> T: """ Executes the given task synchronously via the agent's task executor. This is useful for tasks that need to be executed immediately and whose results are needed right away. @@ -389,7 +392,7 @@ class SerenaAgent: :param task: the task to execute :return: the result of the task execution """ - future = self.issue_task(task) + future = self.issue_task(task, name=name) return future.result() def is_using_language_server(self) -> bool: @@ -519,7 +522,7 @@ class SerenaAgent: def reset_language_server_manager(self) -> None: """ - Starts/resets the language server for the current project + Starts/resets the language server manager for the current project """ tool_timeout = self.serena_config.tool_timeout if tool_timeout is None or tool_timeout < 0: @@ -529,20 +532,24 @@ class SerenaAgent: raise ValueError(f"Tool timeout must be at least 10 seconds, but is {tool_timeout} seconds") ls_timeout = tool_timeout - 5 # the LS timeout is for a single call, it should be smaller than the tool timeout - # stop the language server if it is running - if self._language_server_manager is not None: - self._language_server_manager.stop_all() - self._language_server_manager = None - # instantiate and start the necessary language servers - assert self._active_project is not None, "Active project required" - self._language_server_manager = self._active_project.create_language_server_manager( + self.get_active_project_or_raise().create_language_server_manager( log_level=self.serena_config.log_level, ls_timeout=ls_timeout, trace_lsp_communication=self.serena_config.trace_lsp_communication, ls_specific_settings=self.serena_config.ls_specific_settings, ) + def add_language(self, language: Language) -> None: + """ + Adds a new language to the active project, spawning the respective language server and updating the project configuration. + The addition is scheduled via the agent's task executor and executed synchronously, i.e. the method returns + when the addition is complete. + + :param language: the language to add + """ + self.execute_task(lambda: self.get_active_project_or_raise().add_language(language), name=f"AddLanguage:{language.value}") + def get_tool(self, tool_class: type[TTool]) -> TTool: return self._all_tools[tool_class] # type: ignore @@ -556,9 +563,9 @@ class SerenaAgent: if not hasattr(self, "_is_initialized"): return log.info("SerenaAgent is shutting down ...") - if self._language_server_manager is not None: - self._language_server_manager.stop_all(save_cache=True) - self._language_server_manager = None + if self._active_project is not None: + self._active_project.shutdown() + self._active_project = None if self._gui_log_viewer: log.info("Stopping the GUI log window ...") self._gui_log_viewer.stop() diff --git a/src/serena/config/serena_config.py b/src/serena/config/serena_config.py index 2f17d058..cbdbf6d4 100644 --- a/src/serena/config/serena_config.py +++ b/src/serena/config/serena_config.py @@ -2,6 +2,7 @@ The Serena Model Context Protocol (MCP) Server """ +import dataclasses import os import shutil from collections.abc import Iterable @@ -258,6 +259,14 @@ class ProjectConfig(ToolInclusionDefinition, ToStringMixin): encoding=data.get("encoding", DEFAULT_SOURCE_FILE_ENCODING), ) + def to_yaml_dict(self) -> dict: + """ + :return: a yaml-serializable dictionary representation of this configuration + """ + d = dataclasses.asdict(self) + d["languages"] = [lang.value for lang in self.languages] + return d + @classmethod def load(cls, project_root: Path | str, autogenerate: bool = False) -> Self: """ diff --git a/src/serena/ls_manager.py b/src/serena/ls_manager.py index c0d228e5..7ebd417a 100644 --- a/src/serena/ls_manager.py +++ b/src/serena/ls_manager.py @@ -1,12 +1,54 @@ import logging -from collections.abc import Callable, Iterator +from collections.abc import Iterator +from dataclasses import dataclass +from serena.constants import SERENA_MANAGED_DIR_IN_HOME, SERENA_MANAGED_DIR_NAME from solidlsp import SolidLanguageServer -from solidlsp.ls_config import Language +from solidlsp.ls_config import Language, LanguageServerConfig +from solidlsp.ls_logger import LanguageServerLogger +from solidlsp.settings import SolidLSPSettings log = logging.getLogger(__name__) +@dataclass +class LanguageServerFactoryParams: + project_root: str + encoding: str + ignored_patterns: list[str] + ls_timeout: float | None = None + ls_specific_settings: dict | None = None + log_level: int = logging.INFO + trace_lsp_communication: bool = False + + +class LanguageServerFactory: + def __init__(self, language: Language): + self.language = language + + def create_language_server(self, params: LanguageServerFactoryParams) -> SolidLanguageServer: + ls_config = LanguageServerConfig( + code_language=self.language, + ignored_paths=params.ignored_patterns, + trace_lsp_communication=params.trace_lsp_communication, + encoding=params.encoding, + ) + ls_logger = LanguageServerLogger(log_level=params.log_level) + + log.info(f"Creating language server instance for {params.project_root}, language={self.language}.") + return SolidLanguageServer.create( + ls_config, + ls_logger, + params.project_root, + timeout=params.ls_timeout, + solidlsp_settings=SolidLSPSettings( + solidlsp_dir=SERENA_MANAGED_DIR_IN_HOME, + project_data_relative_path=SERENA_MANAGED_DIR_NAME, + ls_specific_settings=params.ls_specific_settings or {}, + ), + ) + + class LanguageServerManager: """ Manages one or more language servers @@ -15,16 +57,19 @@ class LanguageServerManager: def __init__( self, language_servers: dict[Language, SolidLanguageServer], - language_server_factories: dict[Language, Callable[[], SolidLanguageServer]] | None = None, + language_server_factories: dict[Language, LanguageServerFactory] | None = None, + language_server_factory_params: LanguageServerFactoryParams | None = None, ) -> None: """ :param language_servers: a mapping from language to language server; the servers are assumed to be already started. The first server in the iteration order is used as the default server. All servers are assumed to serve the same project root. :param language_server_factories: factories for the language servers, to be used for restarting them if needed + :param language_server_factory_params: parameters to be used when (re-)creating language servers via factories """ self._language_servers = language_servers self._language_server_factories = language_server_factories or {} + self._language_server_factory_params = language_server_factory_params self._default_language_server = next(iter(language_servers.values())) self._root_path = self._default_language_server.repository_root_path @@ -48,19 +93,44 @@ class LanguageServerManager: ls = self._default_language_server return self._ensure_functional_ls(ls) + def _create_and_start_language_server(self, language: Language) -> SolidLanguageServer: + factory = self._language_server_factories.get(language) + if factory is None: + raise ValueError(f"No language server factory found for language {language}; cannot create new instance") + if self._language_server_factory_params is None: + raise ValueError("No language server factory parameters provided; cannot instantiate new language server instance") + self._language_servers[language] = factory.create_language_server(self._language_server_factory_params) + self._language_servers[language].start() + return self._language_servers[language] + def restart_language_server(self, language: Language) -> SolidLanguageServer: """ Forces recreation and restart of the language server for the given language. + It is assumed that the language server for the given language is no longer running. :param language: the language :return: the newly created language server """ - factory = self._language_server_factories.get(language) - if factory is None: - raise ValueError(f"No language server factory found for language {language}; cannot recreate and restart") - self._language_servers[language] = factory() - self._language_servers[language].start() - return self._language_servers[language] + return self._create_and_start_language_server(language) + + def add_language_server(self, language: Language) -> SolidLanguageServer: + """ + Dynamically adds a new language server for the given language. + + :param language: the language + :param factory: the factory to create the language server + :return: the newly created language server + """ + if language in self._language_servers: + raise ValueError(f"Language server for language {language} already present") + self._language_server_factories[language] = LanguageServerFactory(language) + try: + return self._create_and_start_language_server(language) + except Exception as e: + del self._language_server_factories[language] + if language in self._language_servers: + del self._language_servers[language] + raise Exception(f"Addition of language server for {language.value} failed") from e def iter_language_servers(self) -> Iterator[SolidLanguageServer]: for ls in self._language_servers.values(): diff --git a/src/serena/project.py b/src/serena/project.py index 0484784f..ced8cbfa 100644 --- a/src/serena/project.py +++ b/src/serena/project.py @@ -2,7 +2,6 @@ import json import logging import os import threading -from collections.abc import Callable from pathlib import Path from typing import Any @@ -11,15 +10,14 @@ from sensai.util.logging import LogTime from sensai.util.string import ToStringMixin from serena.config.serena_config import DEFAULT_TOOL_TIMEOUT, ProjectConfig, get_serena_managed_in_project_dir -from serena.constants import SERENA_FILE_ENCODING, SERENA_MANAGED_DIR_IN_HOME, SERENA_MANAGED_DIR_NAME -from serena.ls_manager import LanguageServerManager +from serena.constants import SERENA_FILE_ENCODING, SERENA_MANAGED_DIR_NAME +from serena.ls_manager import LanguageServerFactory, LanguageServerFactoryParams, LanguageServerManager from serena.text_utils import MatchedConsecutiveLines, search_files from serena.util.file_system import GitignoreParser, match_path +from serena.util.general import load_yaml, save_yaml from solidlsp import SolidLanguageServer -from solidlsp.ls_config import Language, LanguageServerConfig -from solidlsp.ls_logger import LanguageServerLogger +from solidlsp.ls_config import Language from solidlsp.ls_utils import FileUtils -from solidlsp.settings import SolidLSPSettings log = logging.getLogger(__name__) @@ -63,6 +61,7 @@ class Project(ToStringMixin): self.project_root = project_root self.project_config = project_config self.memories_manager = MemoriesManager(project_root) + self.language_server_manager: LanguageServerManager | None = None self._is_newly_created = is_newly_created # create .gitignore file in the project's Serena data folder if not yet present @@ -113,6 +112,16 @@ class Project(ToStringMixin): project_config = ProjectConfig.load(project_root, autogenerate=autogenerate) return Project(project_root=str(project_root), project_config=project_config) + def save_config(self) -> None: + """ + Saves the current project configuration to disk. + """ + config_path = os.path.join(self.project_root, self.project_config.rel_path_to_project_yml()) + log.info("Saving updated project configuration to %s", config_path) + config_with_comments = load_yaml(config_path, preserve_comments=True) + config_with_comments.update(self.project_config.to_yaml_dict()) + save_yaml(config_path, config_with_comments, preserve_comments=True) + def path_to_serena_data_folder(self) -> str: return os.path.join(self.project_root, SERENA_MANAGED_DIR_NAME) @@ -351,39 +360,6 @@ class Project(ToStringMixin): source_file_path=relative_file_path, ) - def _create_language_server_factory( - self, - language: Language, - ls_timeout: float | None, - ls_specific_settings: dict | None, - log_level: int, - trace_lsp_communication: bool, - ) -> Callable[[], SolidLanguageServer]: - - def factory() -> SolidLanguageServer: - ls_config = LanguageServerConfig( - code_language=language, - ignored_paths=self._ignored_patterns, - trace_lsp_communication=trace_lsp_communication, - encoding=self.project_config.encoding, - ) - ls_logger = LanguageServerLogger(log_level=log_level) - - log.info(f"Creating language server instance for {self.project_root}, language={language}.") - return SolidLanguageServer.create( - ls_config, - ls_logger, - self.project_root, - timeout=ls_timeout, - solidlsp_settings=SolidLSPSettings( - solidlsp_dir=SERENA_MANAGED_DIR_IN_HOME, - project_data_relative_path=SERENA_MANAGED_DIR_NAME, - ls_specific_settings=ls_specific_settings or {}, - ), - ) - - return factory - def create_language_server_manager( self, log_level: int = logging.INFO, @@ -392,29 +368,41 @@ class Project(ToStringMixin): ls_specific_settings: dict[Language, Any] | None = None, ) -> LanguageServerManager: """ - Create a language server for a project. Note that you will have to start it - before performing any LS operations. + Creates the language server manager for the project, starting one language server per configured programming language. - :param project: either a path to the project root or a ProjectConfig instance. - If no project.yml is found, the default project configuration will be used. :param log_level: the log level for the language server :param ls_timeout: the timeout for the language server :param trace_lsp_communication: whether to trace LSP communication :param ls_specific_settings: optional LS specific configuration of the language server, see docstrings in the inits of subclasses of SolidLanguageServer to see what values may be passed. - :return: the language server + :return: the language server manager, which is also stored in the project instance """ + # if there is an existing instance, stop its language servers first + if self.language_server_manager is not None: + log.info("Stopping existing language server manager ...") + self.language_server_manager.stop_all() + self.language_server_manager = None + log.info(f"Creating language server manager for {self.project_root}") + language_server_factory_params = LanguageServerFactoryParams( + project_root=self.project_root, + encoding=self.project_config.encoding, + ignored_patterns=self._ignored_patterns, + ls_timeout=ls_timeout, + ls_specific_settings=ls_specific_settings, + log_level=log_level, + trace_lsp_communication=trace_lsp_communication, + ) language_servers: dict[Language, SolidLanguageServer] = {} - language_server_factories: dict[Language, Callable[[], SolidLanguageServer]] = {} + language_server_factories: dict[Language, LanguageServerFactory] = {} threads = [] exceptions = {} lock = threading.Lock() - def start_language_server(language: Language, factory: Callable[[], SolidLanguageServer]): + def start_language_server(language: Language, factory: LanguageServerFactory) -> None: try: with LogTime(f"Language server startup (language={language.value})"): - language_server = factory() + language_server = factory.create_language_server(language_server_factory_params) language_server.start() if not language_server.is_running(): raise RuntimeError( @@ -429,7 +417,7 @@ class Project(ToStringMixin): # start language servers in parallel threads for language in self.project_config.languages: - factory = self._create_language_server_factory(language, ls_timeout, ls_specific_settings, log_level, trace_lsp_communication) + factory = LanguageServerFactory(language) language_server_factories[language] = factory thread = threading.Thread(target=start_language_server, args=(language, factory), name="StartLS:" + language.value) thread.start() @@ -444,4 +432,33 @@ class Project(ToStringMixin): failure_messages = "\n".join([f"{lang.value}: {e}" for lang, e in exceptions.items()]) raise Exception(f"Failed to start language servers:\n{failure_messages}") - return LanguageServerManager(language_servers, language_server_factories) + self.language_server_manager = LanguageServerManager(language_servers, language_server_factories, language_server_factory_params) + return self.language_server_manager + + def add_language(self, language: Language) -> None: + """ + Adds a new programming language to the project configuration, starting the corresponding + language server instance if the LS manager is active. + The project configuration is saved to disk after adding the language. + + :param language: the programming language to add + """ + if language in self.project_config.languages: + log.info(f"Language {language.value} is already present in the project configuration.") + return + + # start the language server (if the LS manager is active) + if self.language_server_manager is None: + log.info("Language server manager is not active; skipping language server startup for the new language.") + else: + log.info("Adding and starting the language server for new language %s ...", language.value) + self.language_server_manager.add_language_server(language) + + # update the project configuration + self.project_config.languages.append(language) + self.save_config() + + def shutdown(self) -> None: + if self.language_server_manager is not None: + self.language_server_manager.stop_all(save_cache=True) + self.language_server_manager = None From 7d60326340ba8f4aef1f648a27cd430352afb0a2 Mon Sep 17 00:00:00 2001 From: Dominik Jain Date: Wed, 29 Oct 2025 01:42:49 +0100 Subject: [PATCH 09/10] Simplify language server factory interfaces (one factory for all fits current requirements best) --- src/serena/ls_manager.py | 88 +++++++++++++++++++--------------------- src/serena/project.py | 15 +++---- 2 files changed, 47 insertions(+), 56 deletions(-) diff --git a/src/serena/ls_manager.py b/src/serena/ls_manager.py index 7ebd417a..eb14719e 100644 --- a/src/serena/ls_manager.py +++ b/src/serena/ls_manager.py @@ -1,6 +1,5 @@ import logging from collections.abc import Iterator -from dataclasses import dataclass from serena.constants import SERENA_MANAGED_DIR_IN_HOME, SERENA_MANAGED_DIR_NAME from solidlsp import SolidLanguageServer @@ -11,65 +10,67 @@ from solidlsp.settings import SolidLSPSettings log = logging.getLogger(__name__) -@dataclass -class LanguageServerFactoryParams: - project_root: str - encoding: str - ignored_patterns: list[str] - ls_timeout: float | None = None - ls_specific_settings: dict | None = None - log_level: int = logging.INFO - trace_lsp_communication: bool = False - - class LanguageServerFactory: - def __init__(self, language: Language): - self.language = language + def __init__( + self, + project_root: str, + encoding: str, + ignored_patterns: list[str], + ls_timeout: float | None = None, + ls_specific_settings: dict | None = None, + log_level: int = logging.INFO, + trace_lsp_communication: bool = False, + ): + self.project_root = project_root + self.encoding = encoding + self.ignored_patterns = ignored_patterns + self.ls_timeout = ls_timeout + self.ls_specific_settings = ls_specific_settings + self.log_level = log_level + self.trace_lsp_communication = trace_lsp_communication - def create_language_server(self, params: LanguageServerFactoryParams) -> SolidLanguageServer: + def create_language_server(self, language: Language) -> SolidLanguageServer: ls_config = LanguageServerConfig( - code_language=self.language, - ignored_paths=params.ignored_patterns, - trace_lsp_communication=params.trace_lsp_communication, - encoding=params.encoding, + code_language=language, + ignored_paths=self.ignored_patterns, + trace_lsp_communication=self.trace_lsp_communication, + encoding=self.encoding, ) - ls_logger = LanguageServerLogger(log_level=params.log_level) + ls_logger = LanguageServerLogger(log_level=self.log_level) - log.info(f"Creating language server instance for {params.project_root}, language={self.language}.") + log.info(f"Creating language server instance for {self.project_root}, language={language}.") return SolidLanguageServer.create( ls_config, ls_logger, - params.project_root, - timeout=params.ls_timeout, + self.project_root, + timeout=self.ls_timeout, solidlsp_settings=SolidLSPSettings( solidlsp_dir=SERENA_MANAGED_DIR_IN_HOME, project_data_relative_path=SERENA_MANAGED_DIR_NAME, - ls_specific_settings=params.ls_specific_settings or {}, + ls_specific_settings=self.ls_specific_settings or {}, ), ) class LanguageServerManager: """ - Manages one or more language servers + Manages one or more language servers for a project. """ def __init__( self, language_servers: dict[Language, SolidLanguageServer], - language_server_factories: dict[Language, LanguageServerFactory] | None = None, - language_server_factory_params: LanguageServerFactoryParams | None = None, + language_server_factory: LanguageServerFactory | None = None, ) -> None: """ :param language_servers: a mapping from language to language server; the servers are assumed to be already started. The first server in the iteration order is used as the default server. All servers are assumed to serve the same project root. - :param language_server_factories: factories for the language servers, to be used for restarting them if needed - :param language_server_factory_params: parameters to be used when (re-)creating language servers via factories + :param language_server_factory: factory for language server creation; if None, dynamic (re)creation of language servers + is not supported """ self._language_servers = language_servers - self._language_server_factories = language_server_factories or {} - self._language_server_factory_params = language_server_factory_params + self._language_server_factory = language_server_factory self._default_language_server = next(iter(language_servers.values())) self._root_path = self._default_language_server.repository_root_path @@ -94,14 +95,12 @@ class LanguageServerManager: return self._ensure_functional_ls(ls) def _create_and_start_language_server(self, language: Language) -> SolidLanguageServer: - factory = self._language_server_factories.get(language) - if factory is None: - raise ValueError(f"No language server factory found for language {language}; cannot create new instance") - if self._language_server_factory_params is None: - raise ValueError("No language server factory parameters provided; cannot instantiate new language server instance") - self._language_servers[language] = factory.create_language_server(self._language_server_factory_params) + if self._language_server_factory is None: + raise ValueError(f"No language server factory available to create language server for {language}") + language_server = self._language_server_factory.create_language_server(language) self._language_servers[language].start() - return self._language_servers[language] + self._language_servers[language] = language_server + return language_server def restart_language_server(self, language: Language) -> SolidLanguageServer: """ @@ -111,6 +110,8 @@ class LanguageServerManager: :param language: the language :return: the newly created language server """ + if language not in self._language_servers: + raise ValueError(f"No language server for language {language.value} present; cannot restart") return self._create_and_start_language_server(language) def add_language_server(self, language: Language) -> SolidLanguageServer: @@ -122,15 +123,8 @@ class LanguageServerManager: :return: the newly created language server """ if language in self._language_servers: - raise ValueError(f"Language server for language {language} already present") - self._language_server_factories[language] = LanguageServerFactory(language) - try: - return self._create_and_start_language_server(language) - except Exception as e: - del self._language_server_factories[language] - if language in self._language_servers: - del self._language_servers[language] - raise Exception(f"Addition of language server for {language.value} failed") from e + raise ValueError(f"Language server for language {language.value} already present") + return self._create_and_start_language_server(language) def iter_language_servers(self) -> Iterator[SolidLanguageServer]: for ls in self._language_servers.values(): diff --git a/src/serena/project.py b/src/serena/project.py index ced8cbfa..216e36e1 100644 --- a/src/serena/project.py +++ b/src/serena/project.py @@ -11,7 +11,7 @@ from sensai.util.string import ToStringMixin from serena.config.serena_config import DEFAULT_TOOL_TIMEOUT, ProjectConfig, get_serena_managed_in_project_dir from serena.constants import SERENA_FILE_ENCODING, SERENA_MANAGED_DIR_NAME -from serena.ls_manager import LanguageServerFactory, LanguageServerFactoryParams, LanguageServerManager +from serena.ls_manager import LanguageServerFactory, LanguageServerManager from serena.text_utils import MatchedConsecutiveLines, search_files from serena.util.file_system import GitignoreParser, match_path from serena.util.general import load_yaml, save_yaml @@ -384,7 +384,7 @@ class Project(ToStringMixin): self.language_server_manager = None log.info(f"Creating language server manager for {self.project_root}") - language_server_factory_params = LanguageServerFactoryParams( + language_server_factory = LanguageServerFactory( project_root=self.project_root, encoding=self.project_config.encoding, ignored_patterns=self._ignored_patterns, @@ -394,15 +394,14 @@ class Project(ToStringMixin): trace_lsp_communication=trace_lsp_communication, ) language_servers: dict[Language, SolidLanguageServer] = {} - language_server_factories: dict[Language, LanguageServerFactory] = {} threads = [] exceptions = {} lock = threading.Lock() - def start_language_server(language: Language, factory: LanguageServerFactory) -> None: + def start_language_server(language: Language) -> None: try: with LogTime(f"Language server startup (language={language.value})"): - language_server = factory.create_language_server(language_server_factory_params) + language_server = language_server_factory.create_language_server(language) language_server.start() if not language_server.is_running(): raise RuntimeError( @@ -417,9 +416,7 @@ class Project(ToStringMixin): # start language servers in parallel threads for language in self.project_config.languages: - factory = LanguageServerFactory(language) - language_server_factories[language] = factory - thread = threading.Thread(target=start_language_server, args=(language, factory), name="StartLS:" + language.value) + thread = threading.Thread(target=start_language_server, args=(language,), name="StartLS:" + language.value) thread.start() threads.append(thread) for thread in threads: @@ -432,7 +429,7 @@ class Project(ToStringMixin): failure_messages = "\n".join([f"{lang.value}: {e}" for lang, e in exceptions.items()]) raise Exception(f"Failed to start language servers:\n{failure_messages}") - self.language_server_manager = LanguageServerManager(language_servers, language_server_factories, language_server_factory_params) + self.language_server_manager = LanguageServerManager(language_servers, language_server_factory) return self.language_server_manager def add_language(self, language: Language) -> None: From c73637a8be0ce74dd855c6c80b1a465f7077ddfa Mon Sep 17 00:00:00 2001 From: Dominik Jain Date: Wed, 29 Oct 2025 12:20:42 +0100 Subject: [PATCH 10/10] Add LanguageServerManager.from_languages (moved from Project) --- src/serena/ls_manager.py | 49 ++++++++++++++++++++++++++++++++++++++++ src/serena/project.py | 42 ++-------------------------------- 2 files changed, 51 insertions(+), 40 deletions(-) diff --git a/src/serena/ls_manager.py b/src/serena/ls_manager.py index eb14719e..c5dcdbd7 100644 --- a/src/serena/ls_manager.py +++ b/src/serena/ls_manager.py @@ -1,6 +1,9 @@ import logging +import threading from collections.abc import Iterator +from sensai.util.logging import LogTime + from serena.constants import SERENA_MANAGED_DIR_IN_HOME, SERENA_MANAGED_DIR_NAME from solidlsp import SolidLanguageServer from solidlsp.ls_config import Language, LanguageServerConfig @@ -74,6 +77,52 @@ class LanguageServerManager: self._default_language_server = next(iter(language_servers.values())) self._root_path = self._default_language_server.repository_root_path + @staticmethod + def from_languages(languages: list[Language], factory: LanguageServerFactory) -> "LanguageServerManager": + """ + Creates a manager with language servers for the given languages using the given factory. + The language servers are started in parallel threads. + + :param languages: the languages for which to spawn language servers + :param factory: the factory for language server creation + :return: the instance + """ + language_servers: dict[Language, SolidLanguageServer] = {} + threads = [] + exceptions = {} + lock = threading.Lock() + + def start_language_server(language: Language) -> None: + try: + with LogTime(f"Language server startup (language={language.value})"): + language_server = factory.create_language_server(language) + language_server.start() + if not language_server.is_running(): + raise RuntimeError(f"Failed to start the language server for language {language.value}") + with lock: + language_servers[language] = language_server + except Exception as e: + log.error(f"Error starting language server for language {language.value}: {e}", exc_info=e) + with lock: + exceptions[language] = e + + # start language servers in parallel threads + for language in languages: + thread = threading.Thread(target=start_language_server, args=(language,), name="StartLS:" + language.value) + thread.start() + threads.append(thread) + for thread in threads: + thread.join() + + # If any server failed to start up, raise an exception and stop all started language servers + if exceptions: + for ls in language_servers.values(): + ls.stop() + failure_messages = "\n".join([f"{lang.value}: {e}" for lang, e in exceptions.items()]) + raise Exception(f"Failed to start language servers:\n{failure_messages}") + + return LanguageServerManager(language_servers, factory) + def get_root_path(self) -> str: return self._root_path diff --git a/src/serena/project.py b/src/serena/project.py index 216e36e1..758b7bf7 100644 --- a/src/serena/project.py +++ b/src/serena/project.py @@ -1,12 +1,10 @@ import json import logging import os -import threading from pathlib import Path from typing import Any import pathspec -from sensai.util.logging import LogTime from sensai.util.string import ToStringMixin from serena.config.serena_config import DEFAULT_TOOL_TIMEOUT, ProjectConfig, get_serena_managed_in_project_dir @@ -384,7 +382,7 @@ class Project(ToStringMixin): self.language_server_manager = None log.info(f"Creating language server manager for {self.project_root}") - language_server_factory = LanguageServerFactory( + factory = LanguageServerFactory( project_root=self.project_root, encoding=self.project_config.encoding, ignored_patterns=self._ignored_patterns, @@ -393,43 +391,7 @@ class Project(ToStringMixin): log_level=log_level, trace_lsp_communication=trace_lsp_communication, ) - language_servers: dict[Language, SolidLanguageServer] = {} - threads = [] - exceptions = {} - lock = threading.Lock() - - def start_language_server(language: Language) -> None: - try: - with LogTime(f"Language server startup (language={language.value})"): - language_server = language_server_factory.create_language_server(language) - language_server.start() - if not language_server.is_running(): - raise RuntimeError( - f"Failed to start the language server for language {language.value} with root {self.project_root}" - ) - with lock: - language_servers[language] = language_server - except Exception as e: - log.error(f"Error starting language server for language {language.value}: {e}", exc_info=e) - with lock: - exceptions[language] = e - - # start language servers in parallel threads - for language in self.project_config.languages: - thread = threading.Thread(target=start_language_server, args=(language,), name="StartLS:" + language.value) - thread.start() - threads.append(thread) - for thread in threads: - thread.join() - - # If any server failed to start up, raise an exception and stop all started language servers - if exceptions: - for ls in language_servers.values(): - ls.stop() - failure_messages = "\n".join([f"{lang.value}: {e}" for lang, e in exceptions.items()]) - raise Exception(f"Failed to start language servers:\n{failure_messages}") - - self.language_server_manager = LanguageServerManager(language_servers, language_server_factory) + self.language_server_manager = LanguageServerManager.from_languages(self.project_config.languages, factory) return self.language_server_manager def add_language(self, language: Language) -> None: