From 1cd05b4a2d27bd13671d39e0f57fe7249af5f804 Mon Sep 17 00:00:00 2001 From: Dominik Jain Date: Fri, 27 Feb 2026 21:06:12 +0100 Subject: [PATCH 1/7] Add QueryProjectTool and ListQueryableProjectsTool These tools allow to query information from other projects (without activation) by allowing tools to be executed in the respective project context. LSP tools are not supported, as this would require language servers to be spawned and managed, but when using the JetBrains backend, symbolic access is fully supported for all projects, for which an IDE is open. This thus implements #1086 for users of the JetBrains language backend. The tools are optional. Mode 'query-projects' enables them. --- src/serena/agent.py | 11 +--- src/serena/code_editor.py | 27 +++------ src/serena/config/serena_config.py | 6 ++ src/serena/dashboard.py | 2 +- .../jetbrains/jetbrains_plugin_client.py | 50 ++++++++++++++++ .../resources/config/modes/query-projects.yml | 8 +++ src/serena/tools/__init__.py | 1 + src/serena/tools/query_project_tools.py | 60 +++++++++++++++++++ src/serena/tools/tools_base.py | 52 +++++++++++++--- test/serena/config/test_serena_config.py | 6 +- 10 files changed, 181 insertions(+), 42 deletions(-) create mode 100644 src/serena/resources/config/modes/query-projects.yml create mode 100644 src/serena/tools/query_project_tools.py diff --git a/src/serena/agent.py b/src/serena/agent.py index d48edbdf..7017518f 100644 --- a/src/serena/agent.py +++ b/src/serena/agent.py @@ -536,15 +536,6 @@ class SerenaAgent: ) return True - def get_project_root(self) -> str: - """ - :return: the root directory of the active project (if any); raises a ValueError if there is no active project - """ - project = self.get_active_project() - if project is None: - raise ValueError("Cannot get project root if no project is active.") - return project.project_root - def get_exposed_tool_instances(self) -> list["Tool"]: """ :return: the tool instances which are exposed (e.g. to the MCP client). @@ -711,7 +702,7 @@ class SerenaAgent: self.reset_language_server_manager() # initialize the language server in the background (if in language server mode) - if self.is_using_language_server(): + if self.get_language_backend().is_lsp(): self.issue_task(init_language_server_manager) if self._project_activation_callback is not None: diff --git a/src/serena/code_editor.py b/src/serena/code_editor.py index b5075c38..ea83da25 100644 --- a/src/serena/code_editor.py +++ b/src/serena/code_editor.py @@ -4,8 +4,9 @@ import os from abc import ABC, abstractmethod from collections.abc import Iterable, Iterator, Reversible from contextlib import contextmanager -from typing import TYPE_CHECKING, Generic, Optional, TypeVar, cast +from typing import Generic, TypeVar, cast +from serena.config.serena_config import ProjectConfig from serena.jetbrains.jetbrains_plugin_client import JetBrainsPluginClient from serena.symbol import JetBrainsSymbol, LanguageServerSymbol, LanguageServerSymbolRetriever, PositionInFile, Symbol from solidlsp import SolidLanguageServer, ls_types @@ -15,26 +16,16 @@ from solidlsp.ls_utils import PathUtils, TextUtils from .constants import DEFAULT_SOURCE_FILE_ENCODING from .project import Project -if TYPE_CHECKING: - from .agent import SerenaAgent - - log = logging.getLogger(__name__) TSymbol = TypeVar("TSymbol", bound=Symbol) class CodeEditor(Generic[TSymbol], ABC): - def __init__(self, project_root: str, agent: Optional["SerenaAgent"] = None) -> None: + def __init__(self, project_root: str, project_config: ProjectConfig | None = None) -> None: self.project_root = project_root - self.agent = agent - # set encoding based on active project, if available - encoding = DEFAULT_SOURCE_FILE_ENCODING - if agent is not None: - project = agent.get_active_project() - if project is not None: - encoding = project.project_config.encoding - self.encoding = encoding + # set encoding based on project configuration, if available + self.encoding = project_config.encoding if project_config is not None else DEFAULT_SOURCE_FILE_ENCODING class EditedFile(ABC): def __init__(self, relative_path: str) -> None: @@ -244,8 +235,8 @@ class CodeEditor(Generic[TSymbol], ABC): class LanguageServerCodeEditor(CodeEditor[LanguageServerSymbol]): - def __init__(self, symbol_retriever: LanguageServerSymbolRetriever, agent: Optional["SerenaAgent"] = None): - super().__init__(project_root=symbol_retriever.get_root_path(), agent=agent) + def __init__(self, symbol_retriever: LanguageServerSymbolRetriever, project_config: ProjectConfig | None = None): + super().__init__(project_root=symbol_retriever.get_root_path(), project_config=project_config) self._symbol_retriever = symbol_retriever def _get_language_server(self, relative_path: str) -> SolidLanguageServer: @@ -379,9 +370,9 @@ class LanguageServerCodeEditor(CodeEditor[LanguageServerSymbol]): class JetBrainsCodeEditor(CodeEditor[JetBrainsSymbol]): - def __init__(self, project: Project, agent: Optional["SerenaAgent"] = None) -> None: + def __init__(self, project: Project) -> None: self._project = project - super().__init__(project_root=project.project_root, agent=agent) + super().__init__(project_root=project.project_root, project_config=project.project_config) class EditedFile(CodeEditor.EditedFile): def __init__(self, relative_path: str, project: Project): diff --git a/src/serena/config/serena_config.py b/src/serena/config/serena_config.py index b286c63b..a8679525 100644 --- a/src/serena/config/serena_config.py +++ b/src/serena/config/serena_config.py @@ -161,6 +161,12 @@ class LanguageBackend(Enum): return backend raise ValueError(f"Unknown language backend '{backend_str}': valid values are {[b.value for b in LanguageBackend]}") + def is_lsp(self) -> bool: + return self == LanguageBackend.LSP + + def is_jetbrains(self) -> bool: + return self == LanguageBackend.JETBRAINS + @dataclass class SharedConfig(ModeSelectionDefinition, ToolInclusionDefinition, ToStringMixin): diff --git a/src/serena/dashboard.py b/src/serena/dashboard.py index a5c91a04..d835861f 100644 --- a/src/serena/dashboard.py +++ b/src/serena/dashboard.py @@ -507,7 +507,7 @@ class SerenaDashboardAPI: available_modes=available_modes, available_contexts=available_contexts, available_memories=available_memories, - jetbrains_mode=not self._agent.is_using_language_server(), + jetbrains_mode=self._agent.get_language_backend().is_jetbrains(), languages=languages, encoding=encoding, current_client=Tool.get_last_tool_call_client_str(), diff --git a/src/serena/jetbrains/jetbrains_plugin_client.py b/src/serena/jetbrains/jetbrains_plugin_client.py index f11dc080..f237a079 100644 --- a/src/serena/jetbrains/jetbrains_plugin_client.py +++ b/src/serena/jetbrains/jetbrains_plugin_client.py @@ -8,6 +8,7 @@ import logging import re import threading from concurrent.futures.thread import ThreadPoolExecutor +from dataclasses import dataclass from pathlib import Path from typing import Any, Literal, Optional, Self, TypeVar, cast @@ -16,6 +17,7 @@ from requests import Response from sensai.util.string import ToStringMixin import serena.jetbrains.jetbrains_types as jb +from serena.config.serena_config import RegisteredProject from serena.jetbrains.jetbrains_types import PluginStatusDTO from serena.project import Project from serena.util.class_decorators import singleton @@ -63,6 +65,12 @@ class ServerNotFoundError(Exception): """Raised when the plugin's service is not found.""" +@dataclass +class MatchedClient: + client: "JetBrainsPluginClient" + registered_project: RegisteredProject + + @singleton class JetBrainsPluginClientManager: """ @@ -73,6 +81,7 @@ class JetBrainsPluginClientManager: def __init__(self) -> None: self._clients: dict[int, "JetBrainsPluginClient"] = {} + self._matched_clients: list[MatchedClient] = [] self._lock = threading.Lock() def _submit_scan(self) -> list[concurrent.futures.Future["JetBrainsPluginClient"]]: @@ -116,6 +125,47 @@ class JetBrainsPluginClientManager: "with the Serena plugin installed and running!" ) + def match_clients(self, registered_projects: list[RegisteredProject]) -> list[MatchedClient]: + """ + Scans for plugin instances and matches them against the given registered projects. + + :param registered_projects: the list of registered projects to match plugin instances against + :return: the list of matched clients with their corresponding registered project + """ + matched_clients = [] + for future in self._submit_scan(): + client = future.result() + if client.project_root is not None: + for rp in registered_projects: + if client.matches(Path(rp.project_root)): + matched_clients.append(MatchedClient(client, rp)) + break + self._matched_clients = matched_clients + return matched_clients + + def get_matched_client( + self, registered_project: RegisteredProject, registered_projects: list[RegisteredProject] + ) -> Optional["JetBrainsPluginClient"]: + """ + Gets the matched client for a given registered project, if any. + + :param registered_project: the registered project to get the matched client for + :param registered_projects: the list of all registered projects (used to perform matching of all clients + if no match is found for the given project) + :return: the matched client or None if no match is found + """ + + def find_match() -> Optional["JetBrainsPluginClient"]: + for matched_client in self._matched_clients: + if matched_client.registered_project.project_root == registered_project.project_root: + return matched_client.client + return None + + match = find_match() + if match is None: + self.match_clients(registered_projects) + return find_match() + class JetBrainsPluginClient(ToStringMixin): """ diff --git a/src/serena/resources/config/modes/query-projects.yml b/src/serena/resources/config/modes/query-projects.yml new file mode 100644 index 00000000..ed140de9 --- /dev/null +++ b/src/serena/resources/config/modes/query-projects.yml @@ -0,0 +1,8 @@ +description: Enables tools that allow inactive projects to be queried +prompt: | + You can use the 'query_project' tool to query Serena projects without activating them. + Use this when a project is related to the active project and you need to query it for information. +excluded_tools: [] +included_optional_tools: + - list_queryable_projects + - query_project diff --git a/src/serena/tools/__init__.py b/src/serena/tools/__init__.py index e6fb4b08..08bb4338 100644 --- a/src/serena/tools/__init__.py +++ b/src/serena/tools/__init__.py @@ -7,3 +7,4 @@ from .cmd_tools import * from .config_tools import * from .workflow_tools import * from .jetbrains_tools import * +from .query_project_tools import * diff --git a/src/serena/tools/query_project_tools.py b/src/serena/tools/query_project_tools.py new file mode 100644 index 00000000..c59cd420 --- /dev/null +++ b/src/serena/tools/query_project_tools.py @@ -0,0 +1,60 @@ +import json + +from serena.jetbrains.jetbrains_plugin_client import JetBrainsPluginClientManager +from serena.tools import Tool, ToolMarkerDoesNotRequireActiveProject, ToolMarkerOptional + + +class ListQueryableProjectsTool(Tool, ToolMarkerDoesNotRequireActiveProject, ToolMarkerOptional): + """ + Tool for listing all projects that can be queried by the QueryProjectTool. + """ + + def apply(self, symbol_access: bool = True) -> str: + """ + Lists available projects that can be queried with `query_project_tool`. + + :param symbol_access: whether to return only projects for which symbol access is available. Default: true + """ + # determine relevant projects + registered_projects = self.agent.serena_config.projects + if symbol_access: + backend = self.agent.get_language_backend() + if backend.is_jetbrains(): + matched_clients = JetBrainsPluginClientManager().match_clients(registered_projects) + relevant_projects = [mc.registered_project for mc in matched_clients] + else: + raise NotImplementedError(f"External symbol access is generally unavailable for language backend {backend.value}") + else: + relevant_projects = registered_projects + + # return project names, excluding the active project (if any) + project_names = [p.project_name for p in relevant_projects] + active_project = self.agent.get_active_project() + if active_project is not None: + project_names = [n for n in project_names if n != active_project.project_name] + return self._to_json(project_names) + + +class QueryProjectTool(Tool, ToolMarkerDoesNotRequireActiveProject, ToolMarkerOptional): + """ + Tool for querying external project information (i.e. information from projects other than the current one), + by executing a read-only tool. + """ + + def apply(self, project_name: str, tool_name: str, tool_params_json: str) -> str: + """ + Queries a project by executing a read-only Serena tool. The tool will be executed in the context of the project. + Use this to query information from projects other than the activated project. + + :param project_name: the name of the project to query + :param tool_name: the name of the tool to execute in the other project. The tool must be read-only. + :param tool_params_json: the parameters to pass to the tool, encoded as a JSON string + """ + tool = self.agent.get_tool_by_name(tool_name) + assert tool.is_active(), f"Tool {tool_name} is not active." + assert tool.is_readonly(), f"Tool {tool_name} is not read-only and cannot be executed in another project." + registered_project = self.agent.serena_config.get_registered_project(project_name) + assert registered_project is not None, f"Project {project_name} is not registered and cannot be queried." + project = registered_project.get_project_instance(self.agent.serena_config) + with tool.project_override_context(project): + return tool.apply(**json.loads(tool_params_json)) # type: ignore diff --git a/src/serena/tools/tools_base.py b/src/serena/tools/tools_base.py index 9fb1a393..426529aa 100644 --- a/src/serena/tools/tools_base.py +++ b/src/serena/tools/tools_base.py @@ -1,7 +1,8 @@ import inspect import json from abc import ABC -from collections.abc import Iterable +from collections.abc import Iterable, Iterator +from contextlib import contextmanager from dataclasses import dataclass from types import TracebackType from typing import TYPE_CHECKING, Any, Protocol, Self, TypeVar, cast @@ -12,6 +13,7 @@ from mcp.server.fastmcp.utilities.func_metadata import FuncMetadata, func_metada from sensai.util import logging from sensai.util.string import dict_string +from serena.config.serena_config import LanguageBackend from serena.project import MemoriesManager, Project from serena.prompt_factory import PromptFactory from serena.util.class_decorators import singleton @@ -31,12 +33,33 @@ SUCCESS_RESULT = "OK" class Component(ABC): def __init__(self, agent: "SerenaAgent"): self.agent = agent + self._project_override: Project | None = None + + @contextmanager + def project_override_context(self, project: Project) -> Iterator[None]: + """ + Context manager for temporarily overriding the active project in a component, + allowing tool executions to use the overridden project instead of the active project. + + :param project: the project to override with + """ + if self._project_override is not None: + raise RuntimeError("Nested project overrides are not supported.") + self._project_override = project + try: + yield + finally: + self._project_override = None + + def _assert_is_not_project_override(self) -> None: + if self._project_override is not None: + raise RuntimeError("This method is only supported for the active project.") def get_project_root(self) -> str: """ :return: the root directory of the active project, raises a ValueError if no active project configuration is set """ - return self.agent.get_project_root() + return self.project.project_root @property def prompt_factory(self) -> PromptFactory: @@ -49,22 +72,28 @@ class Component(ABC): def create_language_server_symbol_retriever(self) -> "LanguageServerSymbolRetriever": from serena.symbol import LanguageServerSymbolRetriever - if not self.agent.is_using_language_server(): - raise Exception("Cannot create LanguageServerSymbolRetriever; agent is not in language server mode.") + self._assert_is_not_project_override() + assert self.agent.get_language_backend().is_lsp(), "Language server symbol retriever can only be created for LSP language backend" language_server_manager = self.agent.get_language_server_manager_or_raise() return LanguageServerSymbolRetriever(language_server_manager, agent=self.agent) @property def project(self) -> Project: - return self.agent.get_active_project_or_raise() + if self._project_override is not None: + return self._project_override + else: + return self.agent.get_active_project_or_raise() def create_code_editor(self) -> "CodeEditor": from ..code_editor import JetBrainsCodeEditor, LanguageServerCodeEditor - if self.agent.is_using_language_server(): - return LanguageServerCodeEditor(self.create_language_server_symbol_retriever(), agent=self.agent) - else: - return JetBrainsCodeEditor(project=self.project, agent=self.agent) + match self.agent.get_language_backend(): + case LanguageBackend.LSP: + return LanguageServerCodeEditor(self.create_language_server_symbol_retriever(), project_config=self.project.project_config) + case LanguageBackend.JETBRAINS: + return JetBrainsCodeEditor(project=self.project) + case _: + raise ValueError class ToolMarker: @@ -234,6 +263,9 @@ class Tool(Component): def is_active(self) -> bool: return self.agent.tool_is_active(self.get_name()) + def is_readonly(self) -> bool: + return not self.can_edit() + def apply_ex(self, log_call: bool = True, catch_exceptions: bool = True, mcp_ctx: Context | None = None, **kwargs) -> str: # type: ignore """ Applies the tool with logging and exception handling, using the given keyword arguments @@ -392,6 +424,8 @@ class ToolRegistry: self._tool_dict[name] = RegisteredTool(tool_class=cls, is_optional=is_optional, tool_name=name) def get_tool_class_by_name(self, tool_name: str) -> type[Tool]: + if tool_name not in self._tool_dict: + raise ValueError(f"Tool named '{tool_name}' not found.") return self._tool_dict[tool_name].tool_class def get_all_tool_classes(self) -> list[type[Tool]]: diff --git a/test/serena/config/test_serena_config.py b/test/serena/config/test_serena_config.py index e4515a0d..87e295b7 100644 --- a/test/serena/config/test_serena_config.py +++ b/test/serena/config/test_serena_config.py @@ -217,8 +217,7 @@ class TestEffectiveLanguageBackend: config, name = _make_config_with_project("test_proj", language_backend=None, global_backend=LanguageBackend.LSP) agent = SerenaAgent(project=name, serena_config=config) try: - assert agent.get_language_backend() == LanguageBackend.LSP - assert agent.is_using_language_server() is True + assert agent.get_language_backend().is_lsp() finally: agent.shutdown(timeout=5) @@ -229,8 +228,7 @@ class TestEffectiveLanguageBackend: ) agent = SerenaAgent(project=name, serena_config=config) try: - assert agent.get_language_backend() == LanguageBackend.JETBRAINS - assert agent.is_using_language_server() is False + assert agent.get_language_backend().is_jetbrains() finally: agent.shutdown(timeout=5) From 59f344cc00461bf2f3627496e3b5d963ab5a4401 Mon Sep 17 00:00:00 2001 From: Dominik Jain Date: Sat, 28 Feb 2026 12:59:05 +0100 Subject: [PATCH 2/7] Add project system prompt --- .serena/project.yml | 17 ++++++++++++++++- 1 file changed, 16 insertions(+), 1 deletion(-) diff --git a/.serena/project.yml b/.serena/project.yml index f416c01c..a9218e45 100644 --- a/.serena/project.yml +++ b/.serena/project.yml @@ -90,7 +90,22 @@ included_optional_tools: [] # This cannot be combined with non-empty excluded_tools or included_optional_tools. # initial prompt for the project. It will always be given to the LLM upon activating the project # (contrary to the memories, which are loaded on demand). -initial_prompt: "" +initial_prompt: | + IMPORTANT: You use an idiomatic, object-oriented style. + In particular, this implies that, for any non-trivial interfaces, you use interfaces that expect explicitly typed abstractions + rather than mere functions (i.e. use the strategy pattern, for example). + You avoid the use of low-level data structures in all cases where an object-oriented abstraction would be more appropriate. + For simple data storage, you use dataclasses instead of dictionaries or tuples. + + You structure function implementations into functional blocks that are separated by blank lines. + Atop each functional block, you write an elliptical phrase (starting with lower-case letter) that describes the purpose of the + block in a concise manner. + + Docstrings: You consistently use reStructuredText. + + Comments: + When describing parameters, methods/functions and classes, you use a precise style, where the initial (elliptical) phrase + clearly defines *what* it is. Any details then follow in subsequent sentences. # the encoding used by text files in the project # For a list of possible encodings, see https://docs.python.org/3.11/library/codecs.html#standard-encodings From 1af824c025c47c398105eb6f31aa6a4c43c5496f Mon Sep 17 00:00:00 2001 From: Dominik Jain Date: Sat, 28 Feb 2026 13:36:17 +0100 Subject: [PATCH 3/7] Project: Make SerenaConfig compulsory, simplifying LanguageServerManager creation --- src/serena/agent.py | 16 +------- src/serena/cli.py | 4 +- src/serena/config/serena_config.py | 2 +- src/serena/project.py | 37 +++++++++---------- test/conftest.py | 5 ++- .../config/test_global_ignored_paths.py | 22 ++--------- test/serena/config/test_serena_config.py | 16 +++++--- test/serena/test_serena_agent.py | 7 ++-- 8 files changed, 41 insertions(+), 68 deletions(-) diff --git a/src/serena/agent.py b/src/serena/agent.py index 7017518f..36ac3fc9 100644 --- a/src/serena/agent.py +++ b/src/serena/agent.py @@ -799,21 +799,7 @@ class SerenaAgent: """ Starts/resets the language server manager for the current project """ - tool_timeout = self.serena_config.tool_timeout - if tool_timeout is None or tool_timeout < 0: - ls_timeout = None - else: - if tool_timeout < 10: - raise ValueError(f"Tool timeout must be at least 10 seconds, but is {tool_timeout} seconds") - ls_timeout = tool_timeout - 5 # the LS timeout is for a single call, it should be smaller than the tool timeout - - # instantiate and start the necessary language servers - self.get_active_project_or_raise().create_language_server_manager( - log_level=self.serena_config.log_level, - ls_timeout=ls_timeout, - trace_lsp_communication=self.serena_config.trace_lsp_communication, - ls_specific_settings=self.serena_config.ls_specific_settings, - ) + self.get_active_project_or_raise().create_language_server_manager() def add_language(self, language: Language) -> None: """ diff --git a/src/serena/cli.py b/src/serena/cli.py index 711e8e9a..0c77310b 100644 --- a/src/serena/cli.py +++ b/src/serena/cli.py @@ -640,9 +640,7 @@ class ProjectCommands(AutoRegisteringGroup): serena_config = SerenaConfig.from_config_file() proj = registered_project.get_project_instance(serena_config=serena_config) click.echo(f"Indexing symbols in {proj} …") - ls_mgr = proj.create_language_server_manager( - log_level=lvl, ls_timeout=timeout, ls_specific_settings=serena_config.ls_specific_settings - ) + ls_mgr = proj.create_language_server_manager() try: log_file = os.path.join(proj.project_root, ".serena", "logs", "indexing.txt") diff --git a/src/serena/config/serena_config.py b/src/serena/config/serena_config.py index a8679525..30a47c30 100644 --- a/src/serena/config/serena_config.py +++ b/src/serena/config/serena_config.py @@ -488,7 +488,7 @@ class RegisteredProject(ToStringMixin): """ return self.project_root.samefile(Path(path).resolve()) - def get_project_instance(self, serena_config: "SerenaConfig | None") -> "Project": + def get_project_instance(self, serena_config: "SerenaConfig") -> "Project": """ Returns the project instance for this registered project, loading it if necessary. """ diff --git a/src/serena/project.py b/src/serena/project.py index 59b89ec4..59cb8098 100644 --- a/src/serena/project.py +++ b/src/serena/project.py @@ -10,7 +10,6 @@ from sensai.util.logging import LogTime from sensai.util.string import ToStringMixin from serena.config.serena_config import ( - DEFAULT_TOOL_TIMEOUT, ProjectConfig, get_serena_managed_in_project_dir, ) @@ -117,13 +116,16 @@ class MemoriesManager: class Project(ToStringMixin): def __init__( self, + *, project_root: str, project_config: ProjectConfig, + serena_config: "SerenaConfig", is_newly_created: bool = False, - serena_config: "SerenaConfig | None" = None, ): + assert serena_config is not None self.project_root = project_root self.project_config = project_config + self._serena_config = serena_config self.memories_manager = MemoriesManager(project_root) self.language_server_manager: LanguageServerManager | None = None self._is_newly_created = is_newly_created @@ -137,7 +139,6 @@ class Project(ToStringMixin): f.write(f"/{SolidLanguageServer.CACHE_FOLDER_NAME}\n") # prepare ignore spec asynchronously, ensuring immediate project activation. - self._serena_config = serena_config self.__ignored_patterns: list[str] self.__ignore_spec: pathspec.PathSpec self._ignore_spec_available = threading.Event() @@ -147,7 +148,7 @@ class Project(ToStringMixin): with LogTime(f"Gathering ignore spec for project {self.project_config.project_name}", logger=log): # gather ignored paths from the global configuration, project configuration, and gitignore files - global_ignored_paths = self._serena_config.ignored_paths if self._serena_config else [] + global_ignored_paths = self._serena_config.ignored_paths ignored_patterns = list(global_ignored_paths) + list(self.project_config.ignored_paths) if len(global_ignored_paths) > 0: log.info(f"Using {len(global_ignored_paths)} ignored paths from the global configuration.") @@ -189,7 +190,7 @@ class Project(ToStringMixin): def load( cls, project_root: str | Path, - serena_config: "SerenaConfig | None", + serena_config: "SerenaConfig", autogenerate: bool = True, ) -> "Project": project_root = Path(project_root).resolve() @@ -464,23 +465,21 @@ class Project(ToStringMixin): source_file_path=relative_file_path, ) - def create_language_server_manager( - self, - log_level: int = logging.INFO, - ls_timeout: float | None = DEFAULT_TOOL_TIMEOUT - 5, - trace_lsp_communication: bool = False, - ls_specific_settings: dict[Language, Any] | None = None, - ) -> LanguageServerManager: + def create_language_server_manager(self) -> LanguageServerManager: """ Creates the language server manager for the project, starting one language server per configured programming language. - :param log_level: the log level for the language server - :param ls_timeout: the timeout for the language server - :param trace_lsp_communication: whether to trace LSP communication - :param ls_specific_settings: optional LS specific configuration of the language server, - see docstrings in the inits of subclasses of SolidLanguageServer to see what values may be passed. :return: the language server manager, which is also stored in the project instance """ + # determine timeout to use for LS calls + tool_timeout = self._serena_config.tool_timeout + if tool_timeout is None or tool_timeout < 0: + ls_timeout = None + else: + if tool_timeout < 10: + raise ValueError(f"Tool timeout must be at least 10 seconds, but is {tool_timeout} seconds") + ls_timeout = tool_timeout - 5 # the LS timeout is for a single call, it should be smaller than the tool timeout + # if there is an existing instance, stop its language servers first if self.language_server_manager is not None: log.info("Stopping existing language server manager ...") @@ -493,8 +492,8 @@ class Project(ToStringMixin): encoding=self.project_config.encoding, ignored_patterns=self._ignored_patterns, ls_timeout=ls_timeout, - ls_specific_settings=ls_specific_settings, - trace_lsp_communication=trace_lsp_communication, + ls_specific_settings=self._serena_config.ls_specific_settings, + trace_lsp_communication=self._serena_config.trace_lsp_communication, ) self.language_server_manager = LanguageServerManager.from_languages(self.project_config.languages, factory) return self.language_server_manager diff --git a/test/conftest.py b/test/conftest.py index 565d40b7..6a93ff30 100644 --- a/test/conftest.py +++ b/test/conftest.py @@ -10,7 +10,7 @@ from typing import Any import pytest from sensai.util.logging import configure -from serena.config.serena_config import SerenaPaths +from serena.config.serena_config import SerenaConfig, SerenaPaths from serena.constants import SERENA_MANAGED_DIR_NAME from serena.project import Project from serena.util.file_system import GitignoreParser @@ -117,7 +117,8 @@ def start_default_ls_context(language: Language) -> Iterator[SolidLanguageServer def _create_default_project(language: Language) -> Project: repo_path = str(get_repo_path(language)) - return Project.load(repo_path, serena_config=None) + serena_config = SerenaConfig() + return Project.load(repo_path, serena_config=serena_config) @pytest.fixture(scope="session") diff --git a/test/serena/config/test_global_ignored_paths.py b/test/serena/config/test_global_ignored_paths.py index 23481373..b804fe8b 100644 --- a/test/serena/config/test_global_ignored_paths.py +++ b/test/serena/config/test_global_ignored_paths.py @@ -20,9 +20,7 @@ def _create_test_project( ignored_paths=project_ignored_paths or [], ignore_all_files_in_gitignore=False, ) - serena_config: SerenaConfig | None = None - if global_ignored_paths: - serena_config = SerenaConfig(gui_log_window=False, web_dashboard=False, ignored_paths=global_ignored_paths) + serena_config = SerenaConfig(gui_log_window=False, web_dashboard=False, ignored_paths=global_ignored_paths) return Project( project_root=str(project_root), project_config=config, @@ -84,21 +82,6 @@ class TestGlobalIgnoredPaths: # Non-ignored files still accessible assert not project.is_ignored_path(str(self.project_path / "node_modules" / "pkg" / "index.js")) - def test_default_global_ignored_paths_backward_compatible(self) -> None: - """Project created without global_ignored_paths parameter works as before.""" - config = ProjectConfig( - project_name="test_project", - languages=[Language.PYTHON], - ignored_paths=["build"], - ignore_all_files_in_gitignore=False, - ) - project = Project( - project_root=str(self.project_path), - project_config=config, - ) - assert project.is_ignored_path(str(self.project_path / "build" / "output.js")) - assert not project.is_ignored_path(str(self.project_path / "node_modules" / "pkg" / "index.js")) - def test_duplicate_patterns_across_global_and_project(self) -> None: """Duplicate patterns across global and project do not cause errors.""" project = _create_test_project( @@ -161,7 +144,8 @@ class TestRegisteredProjectGlobalIgnoredPaths: project_root=str(self.project_path), project_config=config, ) - project = registered.get_project_instance(serena_config=None) + serena_config = SerenaConfig(gui_log_window=False, web_dashboard=False, ignored_paths=[]) + project = registered.get_project_instance(serena_config=serena_config) assert not project.is_ignored_path(str(self.project_path / "node_modules" / "pkg.js")) def test_from_project_root_passes_global_ignored_paths(self) -> None: diff --git a/test/serena/config/test_serena_config.py b/test/serena/config/test_serena_config.py index 87e295b7..63f5b30f 100644 --- a/test/serena/config/test_serena_config.py +++ b/test/serena/config/test_serena_config.py @@ -191,6 +191,12 @@ def _make_config_with_project( global_backend: LanguageBackend = LanguageBackend.LSP, ) -> tuple[SerenaConfig, str]: """Create a SerenaConfig with a single registered project and return (config, project_name).""" + config = SerenaConfig( + gui_log_window=False, + web_dashboard=False, + log_level=logging.ERROR, + language_backend=global_backend, + ) project = Project( project_root=str(Path(__file__).parent.parent / "resources" / "repos" / "python" / "test_repo"), project_config=ProjectConfig( @@ -198,12 +204,7 @@ def _make_config_with_project( languages=[Language.PYTHON], language_backend=language_backend, ), - ) - config = SerenaConfig( - gui_log_window=False, - web_dashboard=False, - log_level=logging.ERROR, - language_backend=global_backend, + serena_config=config, ) config.projects = [RegisteredProject.from_project_instance(project)] return config, project_name @@ -259,6 +260,7 @@ class TestEffectiveLanguageBackend: languages=[Language.PYTHON], language_backend=LanguageBackend.JETBRAINS, ), + serena_config=config, ) config.projects.append(RegisteredProject.from_project_instance(jb_project)) @@ -281,6 +283,7 @@ class TestEffectiveLanguageBackend: languages=[Language.PYTHON], language_backend=LanguageBackend.LSP, ), + serena_config=config, ) config.projects.append(RegisteredProject.from_project_instance(lsp_project2)) @@ -303,6 +306,7 @@ class TestEffectiveLanguageBackend: languages=[Language.PYTHON], language_backend=None, ), + serena_config=config, ) config.projects.append(RegisteredProject.from_project_instance(proj2)) diff --git a/test/serena/test_serena_agent.py b/test/serena/test_serena_agent.py index 8d8a5193..a0f978a6 100644 --- a/test/serena/test_serena_agent.py +++ b/test/serena/test_serena_agent.py @@ -21,7 +21,8 @@ from test.solidlsp import clojure as clj @pytest.fixture def serena_config(): - """Create an in-memory configuration for tests with test repositories pre-registered.""" + config = SerenaConfig(gui_log_window=False, web_dashboard=False, log_level=logging.ERROR) + # Create test projects for all supported languages test_projects = [] for language in [ @@ -47,16 +48,16 @@ def serena_config(): project_name=project_name, languages=[language], ignored_paths=[], - excluded_tools=set(), + excluded_tools=[], read_only=False, ignore_all_files_in_gitignore=True, initial_prompt="", encoding="utf-8", ), + serena_config=config, ) test_projects.append(RegisteredProject.from_project_instance(project)) - config = SerenaConfig(gui_log_window=False, web_dashboard=False, log_level=logging.ERROR) config.projects = test_projects return config From 0f6471d2a851c8b37e29ca34f44d47e3182fb13b Mon Sep 17 00:00:00 2001 From: Dominik Jain Date: Sat, 28 Feb 2026 13:38:38 +0100 Subject: [PATCH 4/7] Introduce ProjectServer to support QueryProjectTool for the LSP backend The server allows arbitrary tools to be called remotely, spawning language servers dynamically for each requested project. This implements #1086 for LSP users. --- src/serena/agent.py | 17 ++- src/serena/cli.py | 55 +++++++++ src/serena/project_server.py | 157 ++++++++++++++++++++++++ src/serena/tools/query_project_tools.py | 36 ++++-- src/serena/tools/tools_base.py | 33 +---- 5 files changed, 261 insertions(+), 37 deletions(-) create mode 100644 src/serena/project_server.py diff --git a/src/serena/agent.py b/src/serena/agent.py index 36ac3fc9..5b779ed4 100644 --- a/src/serena/agent.py +++ b/src/serena/agent.py @@ -6,7 +6,8 @@ import os import platform import subprocess import sys -from collections.abc import Callable, Sequence +from collections.abc import Callable, Iterator, Sequence +from contextlib import contextmanager from logging import Logger from typing import TYPE_CHECKING, Optional, TypeVar @@ -853,3 +854,17 @@ class SerenaAgent: if ls_manager is None: return [] return ls_manager.get_active_languages() + + @contextmanager + def active_project_context(self, project: Project) -> Iterator[None]: + """ + Context manager for temporarily setting/overriding the active project + + :param project: the project to be active + """ + original_project = self._active_project + self._active_project = project + try: + yield + finally: + self._active_project = original_project diff --git a/src/serena/cli.py b/src/serena/cli.py index 0c77310b..9fdfc728 100644 --- a/src/serena/cli.py +++ b/src/serena/cli.py @@ -350,6 +350,61 @@ class TopLevelCommands(AutoRegisteringGroup): else: print(f"{prefix}\n{instr}\n{postfix}") + @staticmethod + @click.command( + "start-project-server", + help="Starts the Serena project server, which exposes project querying capabilities via HTTP.", + context_settings={"max_content_width": _MAX_CONTENT_WIDTH}, + ) + @click.option( + "--host", + type=str, + default="127.0.0.1", + show_default=True, + help="Listen address for the project server.", + ) + @click.option( + "--port", + type=int, + default=None, + help="Listen port for the project server (default: ProjectServer.PORT).", + ) + @click.option( + "--log-level", + type=click.Choice(["DEBUG", "INFO", "WARNING", "ERROR", "CRITICAL"]), + default=None, + help="Override log level in config.", + ) + def start_project_server( + host: str, + port: int | None, + log_level: Literal["DEBUG", "INFO", "WARNING", "ERROR", "CRITICAL"] | None, + ) -> None: + from serena.project_server import ProjectServer + + # initialize logging + Logger.root.setLevel(logging.INFO) + formatter = logging.Formatter(SERENA_LOG_FORMAT) + stderr_handler = logging.StreamHandler(stream=sys.stderr) + stderr_handler.formatter = formatter + Logger.root.addHandler(stderr_handler) + log_path = SerenaPaths().get_next_log_file_path("project-server") + file_handler = logging.FileHandler(log_path, mode="w") + file_handler.formatter = formatter + Logger.root.addHandler(file_handler) + + if log_level is not None: + Logger.root.setLevel(logging.getLevelNamesMapping()[log_level]) + + log.info("Starting Serena project server") + log.info("Storing logs in %s", log_path) + + server = ProjectServer() + run_kwargs: dict[str, Any] = {"host": host} + if port is not None: + run_kwargs["port"] = port + server.run(**run_kwargs) + class ModeCommands(AutoRegisteringGroup): """Group for 'mode' subcommands.""" diff --git a/src/serena/project_server.py b/src/serena/project_server.py new file mode 100644 index 00000000..ef88e883 --- /dev/null +++ b/src/serena/project_server.py @@ -0,0 +1,157 @@ +import json +import logging +from typing import TYPE_CHECKING + +import requests as requests_lib +from flask import Flask, request +from pydantic import BaseModel +from sensai.util.logging import LogTime + +from serena.config.serena_config import LanguageBackend, SerenaConfig +from serena.jetbrains.jetbrains_plugin_client import JetBrainsPluginClient + +if TYPE_CHECKING: + from serena.project import Project + +log = logging.getLogger(__name__) + +# disable Werkzeug's logging to avoid cluttering the output +logging.getLogger("werkzeug").setLevel(logging.WARNING) + + +class QueryProjectRequest(BaseModel): + """ + Request model for the /query_project endpoint, matching the interface of + :class:`~serena.tools.query_project_tools.QueryProjectTool`. + """ + + project_name: str + tool_name: str + tool_params_json: str + + +class ProjectServer: + """ + A lightweight Flask server that exposes a SerenaAgent's project querying + capabilities via HTTP, using the LSP language server backend for symbolic retrieval. + + Projects are loaded on demand when a query is made for them, and cached in memory for subsequent queries. + + The server instantiates a :class:`SerenaAgent` with default options and + provides a ``/query_project`` endpoint whose interface matches + :class:`~serena.tools.query_project_tools.QueryProjectTool`. + """ + + PORT = JetBrainsPluginClient.BASE_PORT - 1 + + def __init__(self) -> None: + from serena.agent import SerenaAgent + + serena_config = SerenaConfig.from_config_file() + serena_config.gui_log_window = False + serena_config.web_dashboard = False + serena_config.language_backend = LanguageBackend.LSP + + self._agent = SerenaAgent(serena_config=serena_config) + self._loaded_projects_by_name: dict[str, "Project"] = {} + + # create the Flask application + self._app = Flask(__name__) + self._setup_routes() + + def _setup_routes(self) -> None: + @self._app.route("/heartbeat", methods=["GET"]) + def heartbeat() -> dict[str, str]: + return {"status": "alive"} + + @self._app.route("/query_project", methods=["POST"]) + def query_project() -> str: + query_request = QueryProjectRequest.model_validate(request.get_json()) + return self._query_project(query_request) + + def _get_project(self, project_name: str) -> "Project": + """Gets the project with the given name, loading it if necessary.""" + if project_name in self._loaded_projects_by_name: + return self._loaded_projects_by_name[project_name] + else: + serena_config = self._agent.serena_config + registered_project = serena_config.get_registered_project(project_name) + if registered_project is None: + raise ValueError(f"Project '{project_name}' is not registered with Serena.") + with LogTime(f"Loading project '{project_name}'"): + project = registered_project.get_project_instance(serena_config) + project.create_language_server_manager() + self._loaded_projects_by_name[project_name] = project + return project + + def _query_project(self, req: QueryProjectRequest) -> str: + """Handle a /query_project request by invoking the agent on the specified project and tool.""" + project = self._get_project(req.project_name) + with self._agent.active_project_context(project): + tool = self._agent.get_tool_by_name(req.tool_name) + params = json.loads(req.tool_params_json) + return tool.apply_ex(**params) + + def run(self, host: str = "127.0.0.1", port: int = PORT) -> int: + """Run the server on the given host and port. + + :param host: the host address to listen on. + :param port: the port to listen on. + :return: the port number the server is running on. + """ + from flask import cli + + # suppress the default Flask startup banner + cli.show_server_banner = lambda *args, **kwargs: None + + self._app.run(host=host, port=port, debug=False, use_reloader=False, threaded=True) + return port + + +class ProjectServerClient: + """Client for interacting with a running :class:`ProjectServer`. + + Upon instantiation, the client verifies that the server is reachable + by sending a heartbeat request. If the server is not running, a + :class:`ConnectionError` is raised. + """ + + def __init__(self, host: str = "127.0.0.1", port: int = ProjectServer.PORT, timeout: int = 300) -> None: + """ + :param host: the host address of the project server. + :param port: the port of the project server. + :raises ConnectionError: if the project server is not reachable. + """ + self._base_url = f"http://{host}:{port}" + self._timeout = timeout + + # verify that the server is running + try: + response = requests_lib.get(f"{self._base_url}/heartbeat", timeout=5) + response.raise_for_status() + except requests_lib.ConnectionError: + raise ConnectionError(f"ProjectServer is not reachable at {self._base_url}. Make sure the server is running.") + except requests_lib.RequestException as e: + raise ConnectionError(f"ProjectServer health check failed: {e}") + + def query_project(self, project_name: str, tool_name: str, tool_params_json: str) -> str: + """ + Query a project by executing a Serena tool in its context. + + The interface matches :meth:`QueryProjectTool.apply + `. + + :param project_name: the name of the project to query. + :param tool_name: the name of the tool to execute. The tool must be read-only. + :param tool_params_json: the parameters to pass to the tool, encoded as a JSON string. + :return: the tool's result as a string. + """ + payload = QueryProjectRequest( + project_name=project_name, + tool_name=tool_name, + tool_params_json=tool_params_json, + ).model_dump() + + response = requests_lib.post(f"{self._base_url}/query_project", json=payload, timeout=self._timeout) + response.raise_for_status() + return response.text diff --git a/src/serena/tools/query_project_tools.py b/src/serena/tools/query_project_tools.py index c59cd420..b675a0aa 100644 --- a/src/serena/tools/query_project_tools.py +++ b/src/serena/tools/query_project_tools.py @@ -1,10 +1,12 @@ import json +from serena.config.serena_config import LanguageBackend from serena.jetbrains.jetbrains_plugin_client import JetBrainsPluginClientManager +from serena.project_server import ProjectServerClient from serena.tools import Tool, ToolMarkerDoesNotRequireActiveProject, ToolMarkerOptional -class ListQueryableProjectsTool(Tool, ToolMarkerDoesNotRequireActiveProject, ToolMarkerOptional): +class ListQueryableProjectsTool(Tool, ToolMarkerOptional, ToolMarkerDoesNotRequireActiveProject): """ Tool for listing all projects that can be queried by the QueryProjectTool. """ @@ -20,10 +22,12 @@ class ListQueryableProjectsTool(Tool, ToolMarkerDoesNotRequireActiveProject, Too if symbol_access: backend = self.agent.get_language_backend() if backend.is_jetbrains(): + # projects with open IDE instances can be queried matched_clients = JetBrainsPluginClientManager().match_clients(registered_projects) relevant_projects = [mc.registered_project for mc in matched_clients] else: - raise NotImplementedError(f"External symbol access is generally unavailable for language backend {backend.value}") + # all projects can be queried via ProjectServer (which instantiates projects dynamically) + relevant_projects = registered_projects else: relevant_projects = registered_projects @@ -35,7 +39,7 @@ class ListQueryableProjectsTool(Tool, ToolMarkerDoesNotRequireActiveProject, Too return self._to_json(project_names) -class QueryProjectTool(Tool, ToolMarkerDoesNotRequireActiveProject, ToolMarkerOptional): +class QueryProjectTool(Tool, ToolMarkerOptional, ToolMarkerDoesNotRequireActiveProject): """ Tool for querying external project information (i.e. information from projects other than the current one), by executing a read-only tool. @@ -53,8 +57,24 @@ class QueryProjectTool(Tool, ToolMarkerDoesNotRequireActiveProject, ToolMarkerOp tool = self.agent.get_tool_by_name(tool_name) assert tool.is_active(), f"Tool {tool_name} is not active." assert tool.is_readonly(), f"Tool {tool_name} is not read-only and cannot be executed in another project." - registered_project = self.agent.serena_config.get_registered_project(project_name) - assert registered_project is not None, f"Project {project_name} is not registered and cannot be queried." - project = registered_project.get_project_instance(self.agent.serena_config) - with tool.project_override_context(project): - return tool.apply(**json.loads(tool_params_json)) # type: ignore + if self._is_project_server_required(tool): + client = ProjectServerClient() + return client.query_project(project_name, tool_name, tool_params_json) + else: + registered_project = self.agent.serena_config.get_registered_project(project_name) + assert registered_project is not None, f"Project {project_name} is not registered and cannot be queried." + project = registered_project.get_project_instance(self.agent.serena_config) + with tool.agent.active_project_context(project): + return tool.apply(**json.loads(tool_params_json)) # type: ignore + + def _is_project_server_required(self, tool: Tool) -> bool: + match self.agent.get_language_backend(): + case LanguageBackend.JETBRAINS: + return False + case LanguageBackend.LSP: + # Note: As long as only read-only tools are considered, only symbolic tools require the project server. + # But if we were to allow non-read-only tools, then tools using a CodeEditor also indirectly require language servers. + assert tool.is_readonly() + return tool.is_symbolic() + case _: + raise NotImplementedError diff --git a/src/serena/tools/tools_base.py b/src/serena/tools/tools_base.py index 426529aa..556321d9 100644 --- a/src/serena/tools/tools_base.py +++ b/src/serena/tools/tools_base.py @@ -1,8 +1,7 @@ import inspect import json from abc import ABC -from collections.abc import Iterable, Iterator -from contextlib import contextmanager +from collections.abc import Iterable from dataclasses import dataclass from types import TracebackType from typing import TYPE_CHECKING, Any, Protocol, Self, TypeVar, cast @@ -33,27 +32,6 @@ SUCCESS_RESULT = "OK" class Component(ABC): def __init__(self, agent: "SerenaAgent"): self.agent = agent - self._project_override: Project | None = None - - @contextmanager - def project_override_context(self, project: Project) -> Iterator[None]: - """ - Context manager for temporarily overriding the active project in a component, - allowing tool executions to use the overridden project instead of the active project. - - :param project: the project to override with - """ - if self._project_override is not None: - raise RuntimeError("Nested project overrides are not supported.") - self._project_override = project - try: - yield - finally: - self._project_override = None - - def _assert_is_not_project_override(self) -> None: - if self._project_override is not None: - raise RuntimeError("This method is only supported for the active project.") def get_project_root(self) -> str: """ @@ -72,17 +50,13 @@ class Component(ABC): def create_language_server_symbol_retriever(self) -> "LanguageServerSymbolRetriever": from serena.symbol import LanguageServerSymbolRetriever - self._assert_is_not_project_override() assert self.agent.get_language_backend().is_lsp(), "Language server symbol retriever can only be created for LSP language backend" language_server_manager = self.agent.get_language_server_manager_or_raise() return LanguageServerSymbolRetriever(language_server_manager, agent=self.agent) @property def project(self) -> Project: - if self._project_override is not None: - return self._project_override - else: - return self.agent.get_active_project_or_raise() + return self.agent.get_active_project_or_raise() def create_code_editor(self) -> "CodeEditor": from ..code_editor import JetBrainsCodeEditor, LanguageServerCodeEditor @@ -266,6 +240,9 @@ class Tool(Component): def is_readonly(self) -> bool: return not self.can_edit() + def is_symbolic(self) -> bool: + return issubclass(self.__class__, ToolMarkerSymbolicRead) or issubclass(self.__class__, ToolMarkerSymbolicEdit) + def apply_ex(self, log_call: bool = True, catch_exceptions: bool = True, mcp_ctx: Context | None = None, **kwargs) -> str: # type: ignore """ Applies the tool with logging and exception handling, using the given keyword arguments From 918a90280e9fc2e4e915432d23589b1259a1c185 Mon Sep 17 00:00:00 2001 From: Dominik Jain Date: Sat, 28 Feb 2026 21:12:37 +0100 Subject: [PATCH 5/7] Add information on project querying to the documentation & changelog --- CHANGELOG.md | 4 +++ docs/02-usage/040_workflow.md | 42 ++++++++++++++++++++++++++++-- docs/02-usage/050_configuration.md | 3 ++- 3 files changed, 46 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index e0d94210..dad0e4c6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,10 @@ Status of the `main` branch. Changes prior to the next official version change w * Project configuration files (`project.yml`) can now define multiple languages. Auto-detection adds only the most prominent language by default. * Additional languages can be conveniently added via the Dashboard while a project is already activated. + * Add support for querying projects other than the currently active one via new tools `QueryProjectTool` and `ListQueryableProjectsTool`. + The `QueryProjectTool` allows Serena tools to be called on other projects. + * For the LSP backend, calling symbolic tools require a project server to be spawned that will launch the respective language servers + * For the JetBrains backend, all projects for which IDE instances are open can directly be queried * The current project can be activated automatically even when the MCP configuration is global (through the --project-from-cwd flag) * Support overloaded symbols in `FindSymbolTool` and related tools * Name paths of overloaded symbols now include an index (e.g., `myOverloadedFunction[2]`) diff --git a/docs/02-usage/040_workflow.md b/docs/02-usage/040_workflow.md index c5153401..e52560ea 100644 --- a/docs/02-usage/040_workflow.md +++ b/docs/02-usage/040_workflow.md @@ -36,7 +36,7 @@ For instance, when using `uvx`, run * For an existing project, the main programming language will be detected automatically, but you can choose to explicitly specify multiple languages by passing the `--language` parameter multiple times (e.g. `--language python --language typescript`). - * You can optionally specify a custom project name with `--name "My Project"`. + * You can optionally specify a custom project name with `--name my-name`. * You can immediately index the project after creation with `--index`. (project-config)= @@ -163,4 +163,42 @@ linting results, and test results to assess the correctness of its actions. Therefore, software that is designed to meaningful interpretable outputs (e.g. log messages) and that has a good test coverage is much easier to work with for Serena. -We generally recommend to start an editing task from a state where all linting checks and tests pass. \ No newline at end of file +We generally recommend to start an editing task from a state where all linting checks and tests pass. + +## Working with Multiple Projects Simultaneously + +There are several ways in which you might want to work with multiple projects simultaneously. + +### Simultaneously Editing in Multiple Projects + +If fulfilling a task requires the agent to edit code in multiple projects, the recommended approach is to create a **monorepo folder**, +i.e. a folder that contains all the projects as sub-folders, and open that monorepo folder as a project in Serena. +You may also use symbolic links to create a monorepo folder if the projects are located in different places on your filesystem. + +If several languages are used across the projects, specify all of them as needed when using the LSP backend; +For JetBrains mode, make sure that your IDE is configured to work with all the languages used across the projects (e.g. by installing the respective language plugins). + +(query-projects)= +### Reading from External Projects + +If, while working on a project, you want Serena to be able to read code or other information from another project (e.g. a library or otherwise related project), +this can be enabled via the `query_project` tool. +Provided that the project you want to query is known to Serena (i.e. you have created it as described above), +the `query_project` tool allows the agent to query files and symbolic information from that project. + +To enable this tool, [activate the mode](modes) `query-projects`. +This also enables a second tool for listing projects that can be queried. + +Depending on the language backend being used, the management of resources for the external projects varies: + +* When using the JetBrains backend, make sure that every project for which you want symbolic queries to work is open in an IDE instance. +* When using the LSP backend, executing symbolic tools via the query tool requires that Serena's **Project Server** be started, + which will automatically spawn the necessary language servers for the projects that are queried. + + To start the server, run + + start-project-server + + where `` is your way of running Serena. For example, when using `uvx`, run + + uvx --from git+https://github.com/oraios/serena serena start-project-server diff --git a/docs/02-usage/050_configuration.md b/docs/02-usage/050_configuration.md index aaf256af..dd0704ae 100644 --- a/docs/02-usage/050_configuration.md +++ b/docs/02-usage/050_configuration.md @@ -113,7 +113,8 @@ Examples of built-in modes include: * `one-shot`: Configures Serena for tasks that should be completed in a single response, often used with `planning` for generating reports or initial plans. * `no-onboarding`: Skips the initial onboarding process if it's not needed for a particular session but retains the memory tools (assuming initial memories were created externally). * `onboarding`: Focuses on the project onboarding process. -* `no-memories`: Disables all memory tools (and tools building on memories such as onboarding tools) +* `no-memories`: Disables all memory tools (and tools building on memories such as onboarding tools) +* `query-projects`: Enables tools for querying other Serena projects (without activating them); see section [Reading from External Projects](query-projects) Find the concrete definitions of these modes [here](https://github.com/oraios/serena/tree/main/src/serena/resources/config/modes). From b9a1ce9c3ee471f5f24024c582f8c254b8dbeff7 Mon Sep 17 00:00:00 2001 From: Dominik Jain Date: Thu, 5 Mar 2026 19:04:29 +0100 Subject: [PATCH 6/7] Adjust exception wording to improve generality [skip ci] --- src/serena/config/serena_config.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/serena/config/serena_config.py b/src/serena/config/serena_config.py index 30a47c30..8ab33baa 100644 --- a/src/serena/config/serena_config.py +++ b/src/serena/config/serena_config.py @@ -765,7 +765,7 @@ class SerenaConfig(SharedConfig): return project_candidates[0] elif len(project_candidates) > 1: raise ValueError( - f"Multiple projects found with name '{project_root_or_name}'. Please activate it by location instead. " + f"Multiple projects found with name '{project_root_or_name}'. Please reference it by location instead. " f"Locations: {[p.project_root for p in project_candidates]}" ) # no project found by name; check if it's a path From 58efe413e35299952ddca9c2554976a3fd5d4f5f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alberto=20Sim=C3=B5es?= <104087721+cx-alberto-simoes@users.noreply.github.com> Date: Thu, 5 Mar 2026 19:44:44 +0000 Subject: [PATCH 7/7] Merge pull request #1131 from cx-alberto-simoes/feature/configurable-serena-folder-location feat: add configurable project_serena_folder_location setting Add a new `project_serena_folder_location` setting to `serena_config.yml` that allows configuring where the per-project `.serena` data folder (memories, caches) is stored, with `$projectDir` and `$projectFolderName` placeholder support (default: `$projectDir/.serena`, preserving current behavior). Resolves #1123 Co-authored-by: Dominik Jain --- docs/02-usage/050_configuration.md | 29 +++ src/serena/cli.py | 13 +- src/serena/config/serena_config.py | 152 +++++++++--- src/serena/ls_manager.py | 5 +- src/serena/project.py | 33 +-- .../resources/serena_config.template.yml | 8 + src/solidlsp/ls.py | 4 +- src/solidlsp/settings.py | 8 +- test/conftest.py | 10 +- .../config/test_global_ignored_paths.py | 1 + test/serena/config/test_serena_config.py | 224 +++++++++++++++++- test/solidlsp/csharp/test_csharp_basic.py | 4 +- test/solidlsp/groovy/test_groovy_basic.py | 3 +- 13 files changed, 418 insertions(+), 76 deletions(-) diff --git a/docs/02-usage/050_configuration.md b/docs/02-usage/050_configuration.md index dd0704ae..67d8fec0 100644 --- a/docs/02-usage/050_configuration.md +++ b/docs/02-usage/050_configuration.md @@ -173,6 +173,35 @@ For advanced users, Serena's configuration can be further customized. The Serena user data directory (where configuration, language server files, logs, etc. are stored) defaults to `~/.serena`. You can change this location by setting the `SERENA_HOME` environment variable to your desired path. +### Per-Project Serena Folder Location + +By default, each project stores its Serena data (memories, caches, etc.) in a `.serena` folder inside the project root. +You can customize this location globally via the `project_serena_folder_location` setting in `serena_config.yml`. + +The setting supports two placeholders: + +| Placeholder | Description | +|----------------------|-------------------------------------------------| +| `$projectDir` | The absolute path to the project root directory | +| `$projectFolderName` | The name of the project folder | + +**Examples:** + +```yaml +# Default: data stored inside the project directory +project_serena_folder_location: "$projectDir/.serena" + +# Central location: all project data under a shared directory +project_serena_folder_location: "/projects-metadata/$projectFolderName/.serena" +``` + +When a project is loaded, Serena uses the following fallback logic: +1. Check if a `.serena` folder exists at the configured path. +2. If not, check if one exists in the project root (default/legacy location). +3. If neither exists, create the folder at the configured path. + +This ensures backward compatibility: existing projects that already have a `.serena` folder in the project root will continue to work, even after changing the `project_serena_folder_location` setting. + (ls-specific-settings)= ### Language Server-Specific Settings diff --git a/src/serena/cli.py b/src/serena/cli.py index ef14d17b..6ac341ce 100644 --- a/src/serena/cli.py +++ b/src/serena/cli.py @@ -599,7 +599,8 @@ class ProjectCommands(AutoRegisteringGroup): :return: the RegisteredProject instance """ project_root = Path(project_path).resolve() - yml_path = ProjectConfig.path_to_project_yml(project_root) + serena_config = SerenaConfig.from_config_file() + yml_path = serena_config.get_project_yml_location(str(project_root)) if os.path.exists(yml_path): raise FileExistsError(f"Project file {yml_path} already exists.") @@ -613,14 +614,14 @@ class ProjectCommands(AutoRegisteringGroup): raise ValueError(f"Unknown language '{lang}'. Supported: {all_langs}") generated_conf = ProjectConfig.autogenerate( - project_root=project_path, project_name=name, languages=languages if languages else None, interactive=True + project_root=project_path, + serena_config=serena_config, + project_name=name, + languages=languages if languages else None, + interactive=True, ) - yml_path = ProjectConfig.path_to_project_yml(project_path) languages_str = ", ".join([lang.value for lang in generated_conf.languages]) if generated_conf.languages else "N/A" click.echo(f"Generated project with languages {{{languages_str}}} at {yml_path}.") - - # add to SerenaConfig's list of registered projects - serena_config = SerenaConfig.from_config_file() registered_project = serena_config.get_registered_project(str(project_root)) if registered_project is None: registered_project = RegisteredProject(str(project_root), generated_conf) diff --git a/src/serena/config/serena_config.py b/src/serena/config/serena_config.py index 83ad206f..c47f52d3 100644 --- a/src/serena/config/serena_config.py +++ b/src/serena/config/serena_config.py @@ -4,6 +4,7 @@ The Serena Model Context Protocol (MCP) Server import dataclasses import os +import re import shutil from collections.abc import Iterator, Sequence from copy import deepcopy @@ -195,8 +196,11 @@ class SerenaConfigError(Exception): pass -def get_serena_managed_in_project_dir(project_root: str | Path) -> str: - return os.path.join(project_root, SERENA_MANAGED_DIR_NAME) +DEFAULT_PROJECT_SERENA_FOLDER_LOCATION = "$projectDir/" + SERENA_MANAGED_DIR_NAME +""" +The default template for the project Serena folder location. +Uses $projectDir and $projectFolderName as placeholders. +""" @dataclass(kw_only=True) @@ -224,6 +228,7 @@ class ProjectConfig(SharedConfig): def autogenerate( cls, project_root: str | Path, + serena_config: "SerenaConfig", project_name: str | None = None, languages: list[Language] | None = None, save_to_disk: bool = True, @@ -233,6 +238,7 @@ class ProjectConfig(SharedConfig): Autogenerate a project configuration for a given project root. :param project_root: the path to the project root + :param serena_config: the global Serena configuration :param project_name: the name of the project; if None, the name of the project will be the name of the directory containing the project :param languages: the languages of the project; if None, they will be determined automatically @@ -245,7 +251,8 @@ class ProjectConfig(SharedConfig): raise FileNotFoundError(f"Project root not found: {project_root}") with LogTime("Project configuration auto-generation", logger=log): log.info("Project root: %s", project_root) - project_name = project_name or project_root.name + project_folder_name = project_root.name + project_name = project_name or project_folder_name if languages is None: # determine languages automatically log.info("Determining programming languages used in the project") @@ -293,18 +300,19 @@ class ProjectConfig(SharedConfig): config_with_comments["project_name"] = project_name config_with_comments["languages"] = languages_to_use if save_to_disk: - project_yml_path = cls.path_to_project_yml(project_root) + project_yml_path = serena_config.get_project_yml_location(str(project_root)) log.info("Saving project configuration to %s", project_yml_path) save_yaml(project_yml_path, config_with_comments) return cls._from_dict(config_with_comments) @classmethod - def path_to_project_yml(cls, project_root: str | Path) -> str: - return os.path.join(project_root, cls.rel_path_to_project_yml()) - - @classmethod - def rel_path_to_project_yml(cls) -> str: - return os.path.join(SERENA_MANAGED_DIR_NAME, cls.SERENA_DEFAULT_PROJECT_FILE) + def default_project_yml_path(cls, project_root: str | Path) -> str: + """ + :return: the default path to the project.yml file (inside ``$projectDir/.serena/``). + This is suitable as a fallback when no ``SerenaConfig`` is available to resolve + a potentially customised location. + """ + return os.path.join(str(project_root), SERENA_MANAGED_DIR_NAME, cls.SERENA_DEFAULT_PROJECT_FILE) @classmethod def _load_yaml( @@ -401,24 +409,29 @@ class ProjectConfig(SharedConfig): return d @classmethod - def load(cls, project_root: Path | str, autogenerate: bool = False) -> Self: + def load(cls, project_root: Path | str, serena_config: "SerenaConfig", autogenerate: bool = False) -> Self: """ Load a ProjectConfig instance from the path to the project root. + + :param project_root: the path to the project root + :param serena_config: the global Serena configuration + :param autogenerate: whether to auto-generate the configuration if it does not exist """ project_root = Path(project_root) - yaml_path = project_root / cls.rel_path_to_project_yml() + project_folder_name = project_root.name + yaml_path = serena_config.get_project_yml_location(project_root) # auto-generate if necessary - if not yaml_path.exists(): + if not os.path.exists(yaml_path): if autogenerate: - return cls.autogenerate(project_root) + return cls.autogenerate(project_root, serena_config) else: raise FileNotFoundError(f"Project configuration file not found: {yaml_path}") # load the configuration dictionary yaml_data, was_complete = cls._load_yaml(str(yaml_path)) if "project_name" not in yaml_data: - yaml_data["project_name"] = project_root.name + yaml_data["project_name"] = project_folder_name # instantiate the ProjectConfig project_config = cls._from_dict(yaml_data) @@ -426,17 +439,17 @@ class ProjectConfig(SharedConfig): # if the configuration was incomplete, re-save it to disk if not was_complete: log.info("Project configuration in %s was incomplete, re-saving with default values for missing fields", yaml_path) - project_config.save(project_root) + project_config.save(str(yaml_path)) return project_config - def save(self, project_root: Path | str) -> None: + def save(self, project_yml_path: str) -> None: """ Saves the project configuration to disk. - :param project_root: the root directory of the project + :param project_yml_path: the path to the project.yml file """ - config_path = self.path_to_project_yml(project_root) + config_path = project_yml_path log.info("Saving updated project configuration to %s", config_path) # load original commented map and update it with current values @@ -484,8 +497,8 @@ class RegisteredProject(ToStringMixin): ) @classmethod - def from_project_root(cls, project_root: str | Path) -> "RegisteredProject": - project_config = ProjectConfig.load(project_root) + def from_project_root(cls, project_root: str | Path, serena_config: "SerenaConfig") -> "RegisteredProject": + project_config = ProjectConfig.load(project_root, serena_config=serena_config) return RegisteredProject( project_root=str(project_root), project_config=project_config, @@ -564,6 +577,17 @@ class SerenaConfig(SharedConfig): edit_global_memories: bool = True """Whether global memories are allowed to be deleted or edited.""" + project_serena_folder_location: str = DEFAULT_PROJECT_SERENA_FOLDER_LOCATION + """ + Template for the location of the per-project .serena data folder (memories, caches, etc.). + Supports the following placeholders: + - $projectDir: the absolute path to the project root directory + - $projectFolderName: the name of the project folder + Examples: + - "$projectDir/.serena" (default, stores data inside the project) + - "/projects-metadata/$projectFolderName/.serena" (stores data in a central location) + """ + # settings with overridden defaults default_modes: Sequence[str] | None = ("interactive", "editing") symbol_info_budget: float = 10.0 @@ -689,15 +713,15 @@ class SerenaConfig(SharedConfig): instance.projects = [] for path in loaded_commented_yaml["projects"]: path = Path(path).resolve() - if not path.exists() or (path.is_dir() and not (path / ProjectConfig.rel_path_to_project_yml()).exists()): - log.warning(f"Project path {path} does not exist or does not contain a project configuration file, skipping.") + if not path.exists() or (path.is_dir() and not os.path.isfile(instance.get_project_yml_location(str(path)))): + log.warning(f"Project path {path} does not exist or no associated project configuration file found, skipping.") continue if path.is_file(): path = cls._migrate_out_of_project_config_file(path) if path is None: continue num_migrations += 1 - project_config = ProjectConfig.load(path) + project_config = ProjectConfig.load(path, serena_config=instance) # instance is sufficiently populated project = RegisteredProject( project_root=str(path), project_config=project_config, @@ -750,7 +774,7 @@ class SerenaConfig(SharedConfig): with open(path, "a", encoding=SERENA_FILE_ENCODING) as f: f.write(f"\nproject_name: {project_name}") project_root = project_config_data["project_root"] - shutil.move(str(path), str(Path(project_root) / ProjectConfig.rel_path_to_project_yml())) + shutil.move(str(path), ProjectConfig.default_project_yml_path(project_root)) return Path(project_root).resolve() except Exception as e: log.error(f"Error migrating configuration file: {e}") @@ -789,9 +813,9 @@ class SerenaConfig(SharedConfig): return project # no registered project found; auto-register if project configuration exists if autoregister: - config_path = ProjectConfig.path_to_project_yml(project_root_or_name) + config_path = self.get_project_yml_location(project_root_or_name) if os.path.isfile(config_path): - registered_project = RegisteredProject.from_project_root(project_root_or_name) + registered_project = RegisteredProject.from_project_root(project_root_or_name, serena_config=self) self.add_registered_project(registered_project) return registered_project # nothing found @@ -834,7 +858,7 @@ class SerenaConfig(SharedConfig): f"Project with path {project_root} was already added with name '{already_registered_project.project_name}'." ) - project_config = ProjectConfig.load(project_root, autogenerate=True) + project_config = ProjectConfig.load(project_root, serena_config=self, autogenerate=True) new_project = Project( project_root=str(project_root), @@ -887,6 +911,78 @@ class SerenaConfig(SharedConfig): save_yaml(self.config_file_path, commented_yaml) + @staticmethod + def _resolve_serena_folder_location(template: str, placeholders: dict[str, str]) -> str: + """ + Resolves a folder location template by replacing known ``$placeholder`` tokens + and raising on any unrecognised ones. + + :param template: the template string (e.g. ``"$projectDir/.serena"``) + :param placeholders: mapping from placeholder name (without ``$``) to replacement value + :return: the resolved absolute path + :raises SerenaConfigError: if the template contains an unknown ``$placeholder`` + """ + + def _replace(match: re.Match[str]) -> str: + name = match.group(1) + if name not in placeholders: + raise SerenaConfigError( + f"Unknown placeholder '${name}' in project_serena_folder_location. " + f"Supported placeholders: {', '.join('$' + k for k in placeholders)}" + ) + return placeholders[name] + + result = re.sub(r"\$([A-Za-z_]\w*)", _replace, template) + return os.path.abspath(result) + + def get_configured_project_serena_folder(self, project_root: str | Path) -> str: + """ + Returns the resolved absolute path to the .serena data folder for a project, + applying placeholder substitution to ``project_serena_folder_location`` + without any fallback logic. + + :param project_root: the absolute path to the project root directory + :return: the resolved absolute path to the project's .serena folder + :raises SerenaConfigError: if the template contains an unknown placeholder + """ + project_folder_name = Path(project_root).name + placeholders = { + "projectDir": str(project_root), + "projectFolderName": project_folder_name, + } + return self._resolve_serena_folder_location(self.project_serena_folder_location, placeholders) + + def get_project_serena_folder(self, project_root: str | Path) -> str: + """ + Resolves the location of the project's .serena data folder using fallback logic: + + 1. If the folder exists at the configured path (``project_serena_folder_location``), use it. + 2. Otherwise, if it exists at the default location inside the project root, use that. + 3. If neither exists, return the configured path (for creation). + + :param project_root: the absolute path to the project root directory + :return: the resolved absolute path to the .serena data folder + :raises SerenaConfigError: if the configured template contains an unknown placeholder + """ + configured_path = self.get_configured_project_serena_folder(project_root) + if os.path.isdir(configured_path): + return configured_path + default_path = os.path.join(str(project_root), SERENA_MANAGED_DIR_NAME) + if configured_path != default_path and os.path.isdir(default_path): + return default_path + return configured_path + + def get_project_yml_location(self, project_root: str | Path) -> str: + """ + Returns the resolved absolute path to the project.yml configuration file, + based on the resolved .serena data folder (with fallback logic). + + :param project_root: the absolute path to the project root directory + :return: the resolved absolute path to the project's project.yml file + """ + serena_folder = self.get_project_serena_folder(project_root) + return os.path.join(serena_folder, ProjectConfig.SERENA_DEFAULT_PROJECT_FILE) + def propagate_settings(self) -> None: """ Propagate settings from this configuration to individual components that are statically configured diff --git a/src/serena/ls_manager.py b/src/serena/ls_manager.py index 1ef862f8..99569095 100644 --- a/src/serena/ls_manager.py +++ b/src/serena/ls_manager.py @@ -6,7 +6,6 @@ from collections.abc import Iterator from sensai.util.logging import LogTime from serena.config.serena_config import SerenaPaths -from serena.constants import SERENA_MANAGED_DIR_NAME from solidlsp import SolidLanguageServer from solidlsp.ls_config import Language, LanguageServerConfig from solidlsp.settings import SolidLSPSettings @@ -23,6 +22,7 @@ class LanguageServerFactory: def __init__( self, project_root: str, + project_data_path: str, encoding: str, ignored_patterns: list[str], ls_timeout: float | None = None, @@ -30,6 +30,7 @@ class LanguageServerFactory: trace_lsp_communication: bool = False, ): self.project_root = project_root + self.project_data_path = project_data_path self.encoding = encoding self.ignored_patterns = ignored_patterns self.ls_timeout = ls_timeout @@ -51,7 +52,7 @@ class LanguageServerFactory: timeout=self.ls_timeout, solidlsp_settings=SolidLSPSettings( solidlsp_dir=SerenaPaths().serena_user_home_dir, - project_data_relative_path=SERENA_MANAGED_DIR_NAME, + project_data_path=self.project_data_path, ls_specific_settings=self.ls_specific_settings or {}, ), ) diff --git a/src/serena/project.py b/src/serena/project.py index dd9a4d3e..bc3f0613 100644 --- a/src/serena/project.py +++ b/src/serena/project.py @@ -4,7 +4,7 @@ import os import shutil import threading from pathlib import Path -from typing import TYPE_CHECKING, Any, Literal +from typing import Any, Literal import pathspec from sensai.util.logging import LogTime @@ -12,10 +12,10 @@ from sensai.util.string import ToStringMixin from serena.config.serena_config import ( ProjectConfig, + SerenaConfig, SerenaPaths, - get_serena_managed_in_project_dir, ) -from serena.constants import SERENA_FILE_ENCODING, SERENA_MANAGED_DIR_NAME +from serena.constants import SERENA_FILE_ENCODING from serena.ls_manager import LanguageServerFactory, LanguageServerManager from serena.util.file_system import GitignoreParser, match_path from serena.util.text_utils import ContentReplacer, MatchedConsecutiveLines, search_files @@ -23,9 +23,6 @@ from solidlsp import SolidLanguageServer from solidlsp.ls_config import Language from solidlsp.ls_utils import FileUtils -if TYPE_CHECKING: - from serena.config.serena_config import SerenaConfig - log = logging.getLogger(__name__) @@ -33,12 +30,12 @@ class MemoriesManager: GLOBAL_TOPIC = "global" _global_memory_dir = SerenaPaths().global_memories_path - def __init__(self, project_root: str, global_memory_tool_write_access: bool = False): + def __init__(self, serena_data_folder: str | Path, global_memory_tool_write_access: bool = False): """ - :param project_root: the project's root directory + :param serena_data_folder: the absolute path to the project's .serena data folder :param global_memory_tool_write_access: whether to allow writing global memories in tool execution contexts """ - self._project_memory_dir = Path(get_serena_managed_in_project_dir(project_root)) / "memories" + self._project_memory_dir = Path(serena_data_folder) / "memories" self._project_memory_dir.mkdir(parents=True, exist_ok=True) self._global_memory_tool_write_access = global_memory_tool_write_access self._encoding = SERENA_FILE_ENCODING @@ -199,22 +196,24 @@ class Project(ToStringMixin): *, project_root: str, project_config: ProjectConfig, - serena_config: "SerenaConfig", + serena_config: SerenaConfig, is_newly_created: bool = False, ): assert serena_config is not None self.project_root = project_root self.project_config = project_config self._serena_config = serena_config + self._serena_data_folder = serena_config.get_project_serena_folder(self.project_root) + log.info("Serena project data folder: %s", self._serena_data_folder) global_memory_write_access = serena_config.edit_global_memories if serena_config else False - self.memories_manager = MemoriesManager(project_root, global_memory_write_access) + self.memories_manager = MemoriesManager(self._serena_data_folder, global_memory_write_access) self.language_server_manager: LanguageServerManager | None = None self._is_newly_created = is_newly_created # create .gitignore file in the project's Serena data folder if not yet present - serena_data_gitignore_path = os.path.join(self.path_to_serena_data_folder(), ".gitignore") + serena_data_gitignore_path = os.path.join(self._serena_data_folder, ".gitignore") if not os.path.exists(serena_data_gitignore_path): os.makedirs(os.path.dirname(serena_data_gitignore_path), exist_ok=True) log.info(f"Creating .gitignore file in {serena_data_gitignore_path}") @@ -276,23 +275,24 @@ class Project(ToStringMixin): serena_config: "SerenaConfig", autogenerate: bool = True, ) -> "Project": + assert serena_config is not None project_root = Path(project_root).resolve() if not project_root.exists(): raise FileNotFoundError(f"Project root not found: {project_root}") - project_config = ProjectConfig.load(project_root, autogenerate=autogenerate) + project_config = ProjectConfig.load(project_root, serena_config=serena_config, autogenerate=autogenerate) return Project(project_root=str(project_root), project_config=project_config, serena_config=serena_config) def save_config(self) -> None: """ Saves the current project configuration to disk. """ - self.project_config.save(self.project_root) + self.project_config.save(self.path_to_project_yml()) def path_to_serena_data_folder(self) -> str: - return os.path.join(self.project_root, SERENA_MANAGED_DIR_NAME) + return self._serena_data_folder def path_to_project_yml(self) -> str: - return os.path.join(self.project_root, self.project_config.rel_path_to_project_yml()) + return os.path.join(self._serena_data_folder, ProjectConfig.SERENA_DEFAULT_PROJECT_FILE) def get_activation_message(self) -> str: """ @@ -572,6 +572,7 @@ class Project(ToStringMixin): log.info(f"Creating language server manager for {self.project_root}") factory = LanguageServerFactory( project_root=self.project_root, + project_data_path=self._serena_data_folder, encoding=self.project_config.encoding, ignored_patterns=self._ignored_patterns, ls_timeout=ls_timeout, diff --git a/src/serena/resources/serena_config.template.yml b/src/serena/resources/serena_config.template.yml index 57ba3820..234928ce 100644 --- a/src/serena/resources/serena_config.template.yml +++ b/src/serena/resources/serena_config.template.yml @@ -126,5 +126,13 @@ token_count_estimator: CHAR_COUNT # Can be overridden in project.yml. symbol_info_budget: 10 +# template for the location of the per-project .serena data folder (memories, caches, etc.). +# Supports the following placeholders: +# $projectDir - the absolute path to the project root directory +# $projectFolderName - the name of the project directory +# Default: "$projectDir/.serena" (data stored inside the project directory) +# Example for a central location: "/projects-metadata/$projectFolderName/.serena" +project_serena_folder_location: "$projectDir/.serena" + # the list of registered project paths (updated automatically). projects: [] diff --git a/src/solidlsp/ls.py b/src/solidlsp/ls.py index 25208f06..02b1ab5f 100644 --- a/src/solidlsp/ls.py +++ b/src/solidlsp/ls.py @@ -477,9 +477,7 @@ class SolidLanguageServer(ABC): self.language = Language(language_id) # initialise symbol caches - self.cache_dir = ( - Path(self.repository_root_path) / self._solidlsp_settings.project_data_relative_path / self.CACHE_FOLDER_NAME / self.language_id - ) + self.cache_dir = Path(self._solidlsp_settings.project_data_path) / self.CACHE_FOLDER_NAME / self.language_id self.cache_dir.mkdir(parents=True, exist_ok=True) # * raw document symbols cache self._ls_specific_raw_document_symbols_cache_version = cache_version_raw_document_symbols diff --git a/src/solidlsp/settings.py b/src/solidlsp/settings.py index ad77eb22..db975945 100644 --- a/src/solidlsp/settings.py +++ b/src/solidlsp/settings.py @@ -22,11 +22,11 @@ class SolidLSPSettings: """ Path to the directory in which to store global Solid-LSP data (which is not project-specific) """ - project_data_relative_path: str = ".solidlsp" + project_data_path: str = "" """ - Relative path within each project directory where Solid-LSP can store project-specific data, e.g. cache files. - For instance, if this is ".solidlsp" and the project is located at "/home/user/myproject", - then Solid-LSP will store project-specific data in "/home/user/myproject/.solidlsp". + Absolute path to a directory where Solid-LSP can store project-specific data, e.g. cache files. + For instance, if this is "/home/user/myproject/.solidlsp", + then Solid-LSP will store project-specific data (e.g. caches) in that directory. """ ls_specific_settings: dict["Language", dict[str, Any]] = field(default_factory=dict) """ diff --git a/test/conftest.py b/test/conftest.py index 6a93ff30..15849156 100644 --- a/test/conftest.py +++ b/test/conftest.py @@ -69,12 +69,13 @@ def _create_ls( trace_lsp_communication=trace_lsp_communication, ) effective_solidlsp_dir = solidlsp_dir if solidlsp_dir is not None else SerenaPaths().serena_user_home_dir + project_data_path = os.path.join(repo_path, SERENA_MANAGED_DIR_NAME) return SolidLanguageServer.create( config, repo_path, solidlsp_settings=SolidLSPSettings( solidlsp_dir=effective_solidlsp_dir, - project_data_relative_path=SERENA_MANAGED_DIR_NAME, + project_data_path=project_data_path, ls_specific_settings=ls_specific_settings or {}, ), ) @@ -115,10 +116,13 @@ def start_default_ls_context(language: Language) -> Iterator[SolidLanguageServer yield ls +def create_default_serena_config(): + return SerenaConfig(gui_log_window=False, web_dashboard=False) + + def _create_default_project(language: Language) -> Project: repo_path = str(get_repo_path(language)) - serena_config = SerenaConfig() - return Project.load(repo_path, serena_config=serena_config) + return Project.load(repo_path, serena_config=create_default_serena_config()) @pytest.fixture(scope="session") diff --git a/test/serena/config/test_global_ignored_paths.py b/test/serena/config/test_global_ignored_paths.py index b804fe8b..af33ceea 100644 --- a/test/serena/config/test_global_ignored_paths.py +++ b/test/serena/config/test_global_ignored_paths.py @@ -159,6 +159,7 @@ class TestRegisteredProjectGlobalIgnoredPaths: serena_config = SerenaConfig(gui_log_window=False, web_dashboard=False, ignored_paths=["node_modules"]) registered = RegisteredProject.from_project_root( str(self.project_path), + serena_config=serena_config, ) project = registered.get_project_instance(serena_config=serena_config) assert project.is_ignored_path(str(self.project_path / "node_modules" / "pkg.js")) diff --git a/test/serena/config/test_serena_config.py b/test/serena/config/test_serena_config.py index 63f5b30f..0e018c50 100644 --- a/test/serena/config/test_serena_config.py +++ b/test/serena/config/test_serena_config.py @@ -1,4 +1,5 @@ import logging +import os import shutil import tempfile from pathlib import Path @@ -6,10 +7,18 @@ from pathlib import Path import pytest from serena.agent import SerenaAgent -from serena.config.serena_config import LanguageBackend, ProjectConfig, RegisteredProject, SerenaConfig -from serena.constants import PROJECT_TEMPLATE_FILE -from serena.project import Project +from serena.config.serena_config import ( + DEFAULT_PROJECT_SERENA_FOLDER_LOCATION, + LanguageBackend, + ProjectConfig, + RegisteredProject, + SerenaConfig, + SerenaConfigError, +) +from serena.constants import PROJECT_TEMPLATE_FILE, SERENA_MANAGED_DIR_NAME +from serena.project import MemoriesManager, Project from solidlsp.ls_config import Language +from test.conftest import create_default_serena_config class TestProjectConfigAutogenerate: @@ -19,6 +28,7 @@ class TestProjectConfigAutogenerate: """Set up test environment before each test method.""" # Create a temporary directory for testing self.test_dir = tempfile.mkdtemp() + self.serena_config = create_default_serena_config() self.project_path = Path(self.test_dir) def teardown_method(self): @@ -29,7 +39,7 @@ class TestProjectConfigAutogenerate: def test_autogenerate_empty_directory(self): """Test that autogenerate raises ValueError with helpful message for empty directory.""" with pytest.raises(ValueError) as exc_info: - ProjectConfig.autogenerate(self.project_path, save_to_disk=False) + ProjectConfig.autogenerate(self.project_path, self.serena_config, save_to_disk=False) error_message = str(exc_info.value) assert "No source files found" in error_message @@ -41,7 +51,7 @@ class TestProjectConfigAutogenerate: python_file.write_text("def hello():\n print('Hello, world!')\n") # Run autogenerate - config = ProjectConfig.autogenerate(self.project_path, save_to_disk=False) + config = ProjectConfig.autogenerate(self.project_path, self.serena_config, save_to_disk=False) # Verify the configuration assert config.project_name == self.project_path.name @@ -53,7 +63,7 @@ class TestProjectConfigAutogenerate: (self.project_path / "small.js").write_text("console.log('JS');") # Run autogenerate - should pick Python as dominant - config = ProjectConfig.autogenerate(self.project_path, save_to_disk=False) + config = ProjectConfig.autogenerate(self.project_path, self.serena_config, save_to_disk=False) assert config.languages == [Language.TYPESCRIPT] @@ -65,7 +75,7 @@ class TestProjectConfigAutogenerate: (self.project_path / "small.js").write_text("console.log('JS');") # Run autogenerate - should pick Python as dominant - config = ProjectConfig.autogenerate(self.project_path, save_to_disk=False) + config = ProjectConfig.autogenerate(self.project_path, self.serena_config, save_to_disk=False) assert config.languages == [Language.PYTHON] @@ -76,7 +86,7 @@ class TestProjectConfigAutogenerate: go_file.write_text("package main\n\nfunc main() {}\n") # Run autogenerate with save_to_disk=True - config = ProjectConfig.autogenerate(self.project_path, save_to_disk=True) + config = ProjectConfig.autogenerate(self.project_path, self.serena_config, save_to_disk=True) # Verify the configuration file was created config_path = self.project_path / ".serena" / "project.yml" @@ -90,7 +100,7 @@ class TestProjectConfigAutogenerate: non_existent = self.project_path / "does_not_exist" with pytest.raises(FileNotFoundError) as exc_info: - ProjectConfig.autogenerate(non_existent, save_to_disk=False) + ProjectConfig.autogenerate(non_existent, self.serena_config, save_to_disk=False) assert "Project root not found" in str(exc_info.value) @@ -105,7 +115,7 @@ class TestProjectConfigAutogenerate: # Should still raise ValueError as no source files are detected with pytest.raises(ValueError) as exc_info: - ProjectConfig.autogenerate(self.project_path, save_to_disk=False) + ProjectConfig.autogenerate(self.project_path, self.serena_config, save_to_disk=False) assert "No source files found" in str(exc_info.value) @@ -117,7 +127,7 @@ class TestProjectConfigAutogenerate: # Run autogenerate with custom name custom_name = "my-custom-project" - config = ProjectConfig.autogenerate(self.project_path, project_name=custom_name, save_to_disk=False) + config = ProjectConfig.autogenerate(self.project_path, self.serena_config, project_name=custom_name, save_to_disk=False) assert config.project_name == custom_name assert config.languages == [Language.TYPESCRIPT] @@ -316,3 +326,195 @@ class TestEffectiveLanguageBackend: agent.activate_project_from_path_or_name("proj2") finally: agent.shutdown(timeout=5) + + +class TestGetConfiguredProjectSerenaFolder: + """Tests for SerenaConfig.get_configured_project_serena_folder (pure template resolution).""" + + def test_default_location(self): + config = SerenaConfig( + gui_log_window=False, + web_dashboard=False, + ) + result = config.get_configured_project_serena_folder("/home/user/myproject") + assert result == os.path.abspath("/home/user/myproject/.serena") + + def test_custom_location_with_project_folder_name(self): + config = SerenaConfig( + gui_log_window=False, + web_dashboard=False, + project_serena_folder_location="/projects-metadata/$projectFolderName/.serena", + ) + result = config.get_configured_project_serena_folder("/home/user/myproject") + assert result == os.path.abspath("/projects-metadata/myproject/.serena") + + def test_custom_location_with_project_dir(self): + config = SerenaConfig( + gui_log_window=False, + web_dashboard=False, + project_serena_folder_location="$projectDir/.custom-serena", + ) + result = config.get_configured_project_serena_folder("/home/user/myproject") + assert result == os.path.abspath("/home/user/myproject/.custom-serena") + + def test_custom_location_with_both_placeholders(self): + config = SerenaConfig( + gui_log_window=False, + web_dashboard=False, + project_serena_folder_location="/data/$projectFolderName/$projectDir/.serena", + ) + result = config.get_configured_project_serena_folder("/home/user/proj") + assert result == os.path.abspath("/data/proj/home/user/proj/.serena") + + def test_default_field_value(self): + config = SerenaConfig( + gui_log_window=False, + web_dashboard=False, + ) + assert config.project_serena_folder_location == DEFAULT_PROJECT_SERENA_FOLDER_LOCATION + + def test_rejects_unknown_placeholder(self): + config = SerenaConfig( + gui_log_window=False, + web_dashboard=False, + project_serena_folder_location="$projectDir/$unknownVar/.serena", + ) + with pytest.raises(SerenaConfigError, match=r"Unknown placeholder '\$unknownVar'"): + config.get_configured_project_serena_folder("/home/user/myproject") + + def test_rejects_typo_projectDirs(self): + """$projectDirs should not be silently treated as $projectDir + 's'.""" + config = SerenaConfig( + gui_log_window=False, + web_dashboard=False, + project_serena_folder_location="$projectDirs/.serena", + ) + with pytest.raises(SerenaConfigError, match=r"Unknown placeholder '\$projectDirs'"): + config.get_configured_project_serena_folder("/home/user/myproject") + + def test_rejects_typo_projectfoldername_lowercase(self): + config = SerenaConfig( + gui_log_window=False, + web_dashboard=False, + project_serena_folder_location="/data/$projectfoldername/.serena", + ) + with pytest.raises(SerenaConfigError, match=r"Unknown placeholder '\$projectfoldername'"): + config.get_configured_project_serena_folder("/home/user/myproject") + + def test_no_placeholders_is_valid(self): + config = SerenaConfig( + gui_log_window=False, + web_dashboard=False, + project_serena_folder_location="/fixed/path/.serena", + ) + result = config.get_configured_project_serena_folder("/home/user/myproject") + assert result == os.path.abspath("/fixed/path/.serena") + + def test_error_message_lists_supported_placeholders(self): + config = SerenaConfig( + gui_log_window=False, + web_dashboard=False, + project_serena_folder_location="$bogus/.serena", + ) + with pytest.raises(SerenaConfigError, match=r"\$projectDir.*\$projectFolderName|\$projectFolderName.*\$projectDir"): + config.get_configured_project_serena_folder("/home/user/myproject") + + +class TestProjectSerenaDataFolder: + """Tests for SerenaConfig.get_project_serena_folder fallback logic (via Project).""" + + def setup_method(self): + self.test_dir = tempfile.mkdtemp() + self.project_path = Path(self.test_dir) / "myproject" + self.project_path.mkdir() + (self.project_path / "main.py").write_text("print('hello')\n") + + def teardown_method(self): + shutil.rmtree(self.test_dir) + + def _make_project(self, serena_config: "SerenaConfig | None" = None) -> Project: + project_config = ProjectConfig( + project_name="myproject", + languages=[Language.PYTHON], + ) + return Project( + project_root=str(self.project_path), + project_config=project_config, + serena_config=serena_config, + ) + + def test_default_config_creates_in_project_dir(self): + config = SerenaConfig(gui_log_window=False, web_dashboard=False) + project = self._make_project(config) + expected = os.path.abspath(str(self.project_path / SERENA_MANAGED_DIR_NAME)) + assert project.path_to_serena_data_folder() == expected + + def test_custom_location_creates_outside_project(self): + custom_base = Path(self.test_dir) / "metadata" + custom_base.mkdir() + config = SerenaConfig( + gui_log_window=False, + web_dashboard=False, + project_serena_folder_location=str(custom_base) + "/$projectFolderName/.serena", + ) + project = self._make_project(config) + expected = os.path.abspath(str(custom_base / "myproject" / ".serena")) + assert project.path_to_serena_data_folder() == expected + + def test_fallback_to_existing_project_dir(self): + """If config points to a non-existent path but .serena exists in the project root, use the existing one.""" + existing_serena = self.project_path / SERENA_MANAGED_DIR_NAME + existing_serena.mkdir() + config = SerenaConfig( + gui_log_window=False, + web_dashboard=False, + project_serena_folder_location="/nonexistent/path/$projectFolderName/.serena", + ) + project = self._make_project(config) + assert project.path_to_serena_data_folder() == str(existing_serena) + + def test_configured_path_takes_precedence_when_exists(self): + """If both config path and project root path exist, use the config path.""" + existing_serena = self.project_path / SERENA_MANAGED_DIR_NAME + existing_serena.mkdir() + + custom_base = Path(self.test_dir) / "metadata" + custom_serena = custom_base / "myproject" / ".serena" + custom_serena.mkdir(parents=True) + + config = SerenaConfig( + gui_log_window=False, + web_dashboard=False, + project_serena_folder_location=str(custom_base) + "/$projectFolderName/.serena", + ) + project = self._make_project(config) + assert project.path_to_serena_data_folder() == str(custom_serena) + + +class TestMemoriesManagerCustomPath: + """Tests for MemoriesManager with a custom serena data folder.""" + + def setup_method(self): + self.test_dir = tempfile.mkdtemp() + self.data_folder = Path(self.test_dir) / "custom_serena" + + def teardown_method(self): + shutil.rmtree(self.test_dir) + + def test_memories_subdir_is_created(self): + assert not self.data_folder.exists() + MemoriesManager(str(self.data_folder)) + assert (self.data_folder / "memories").exists() + + def test_save_and_load_memory(self): + manager = MemoriesManager(str(self.data_folder)) + manager.save_memory("test_topic", "test content", is_tool_context=False) + content = manager.load_memory("test_topic") + assert content == "test content" + + def test_list_memories(self): + manager = MemoriesManager(str(self.data_folder)) + manager.save_memory("topic_a", "content a", is_tool_context=False) + manager.save_memory("topic_b", "content b", is_tool_context=False) + memories = manager.list_project_memories() + assert sorted(memories) == ["topic_a", "topic_b"] diff --git a/test/solidlsp/csharp/test_csharp_basic.py b/test/solidlsp/csharp/test_csharp_basic.py index 51e4f9b1..01e19da6 100644 --- a/test/solidlsp/csharp/test_csharp_basic.py +++ b/test/solidlsp/csharp/test_csharp_basic.py @@ -302,7 +302,7 @@ class TestCSharpSolutionProjectOpening: # Create CSharpLanguageServer instance mock_settings = Mock(spec=SolidLSPSettings) mock_settings.ls_resources_dir = "/tmp/test_ls_resources" - mock_settings.project_data_relative_path = "project_data" + mock_settings.project_data_path = str(temp_path / "project_data") with SuspendedLoggersContext(): logging.getLogger().setLevel(logging.DEBUG) @@ -330,7 +330,7 @@ class TestCSharpSolutionProjectOpening: mock_settings = Mock(spec=SolidLSPSettings) mock_settings.ls_resources_dir = "/tmp/test_ls_resources" - mock_settings.project_data_relative_path = "project_data" + mock_settings.project_data_path = str(temp_path / "project_data") # Create CSharpLanguageServer instance with SuspendedLoggersContext(): diff --git a/test/solidlsp/groovy/test_groovy_basic.py b/test/solidlsp/groovy/test_groovy_basic.py index 8792f1dc..57e0baf3 100644 --- a/test/solidlsp/groovy/test_groovy_basic.py +++ b/test/solidlsp/groovy/test_groovy_basic.py @@ -43,9 +43,10 @@ class TestGroovyLanguageServer: repo_path = str(cls.test_repo_path) config = LanguageServerConfig(code_language=Language.GROOVY, ignored_paths=[], trace_lsp_communication=False) + project_data_path = os.path.join(repo_path, SERENA_MANAGED_DIR_NAME) solidlsp_settings = SolidLSPSettings( solidlsp_dir=str(Path.home() / ".serena"), - project_data_relative_path=SERENA_MANAGED_DIR_NAME, + project_data_path=project_data_path, ls_specific_settings={Language.GROOVY: groovy_settings}, )