Add QueryProjectTool and ListQueryableProjectsTool

These tools allow to query information from other projects (without activation)
by allowing tools to be executed in the respective project context.

LSP tools are not supported, as this would require language servers to be
spawned and managed, but when using the JetBrains backend, symbolic
access is fully supported for all projects, for which an IDE is open.
This thus implements #1086 for users of the JetBrains language backend.

The tools are optional. Mode 'query-projects' enables them.

src/serena/agent.py

@@ -536,15 +536,6 @@ class SerenaAgent:
         )
         return True
 
-    def get_project_root(self) -> str:
-        """
-        :return: the root directory of the active project (if any); raises a ValueError if there is no active project
-        """
-        project = self.get_active_project()
-        if project is None:
-            raise ValueError("Cannot get project root if no project is active.")
-        return project.project_root
-
     def get_exposed_tool_instances(self) -> list["Tool"]:
         """
         :return: the tool instances which are exposed (e.g. to the MCP client).
@@ -711,7 +702,7 @@ class SerenaAgent:
                 self.reset_language_server_manager()
 
         # initialize the language server in the background (if in language server mode)
-        if self.is_using_language_server():
+        if self.get_language_backend().is_lsp():
             self.issue_task(init_language_server_manager)
 
         if self._project_activation_callback is not None:

src/serena/code_editor.py

@@ -4,8 +4,9 @@ import os
 from abc import ABC, abstractmethod
 from collections.abc import Iterable, Iterator, Reversible
 from contextlib import contextmanager
-from typing import TYPE_CHECKING, Generic, Optional, TypeVar, cast
+from typing import Generic, TypeVar, cast
 
+from serena.config.serena_config import ProjectConfig
 from serena.jetbrains.jetbrains_plugin_client import JetBrainsPluginClient
 from serena.symbol import JetBrainsSymbol, LanguageServerSymbol, LanguageServerSymbolRetriever, PositionInFile, Symbol
 from solidlsp import SolidLanguageServer, ls_types
@@ -15,26 +16,16 @@ from solidlsp.ls_utils import PathUtils, TextUtils
 from .constants import DEFAULT_SOURCE_FILE_ENCODING
 from .project import Project
 
-if TYPE_CHECKING:
-    from .agent import SerenaAgent
-
-
 log = logging.getLogger(__name__)
 TSymbol = TypeVar("TSymbol", bound=Symbol)
 
 
 class CodeEditor(Generic[TSymbol], ABC):
-    def __init__(self, project_root: str, agent: Optional["SerenaAgent"] = None) -> None:
+    def __init__(self, project_root: str, project_config: ProjectConfig | None = None) -> None:
         self.project_root = project_root
-        self.agent = agent
 
-        # set encoding based on active project, if available
-        encoding = DEFAULT_SOURCE_FILE_ENCODING
-        if agent is not None:
-            project = agent.get_active_project()
-            if project is not None:
-                encoding = project.project_config.encoding
-        self.encoding = encoding
+        # set encoding based on project configuration, if available
+        self.encoding = project_config.encoding if project_config is not None else DEFAULT_SOURCE_FILE_ENCODING
 
     class EditedFile(ABC):
         def __init__(self, relative_path: str) -> None:
@@ -244,8 +235,8 @@ class CodeEditor(Generic[TSymbol], ABC):
 
 
 class LanguageServerCodeEditor(CodeEditor[LanguageServerSymbol]):
-    def __init__(self, symbol_retriever: LanguageServerSymbolRetriever, agent: Optional["SerenaAgent"] = None):
-        super().__init__(project_root=symbol_retriever.get_root_path(), agent=agent)
+    def __init__(self, symbol_retriever: LanguageServerSymbolRetriever, project_config: ProjectConfig | None = None):
+        super().__init__(project_root=symbol_retriever.get_root_path(), project_config=project_config)
         self._symbol_retriever = symbol_retriever
 
     def _get_language_server(self, relative_path: str) -> SolidLanguageServer:
@@ -379,9 +370,9 @@ class LanguageServerCodeEditor(CodeEditor[LanguageServerSymbol]):
 
 
 class JetBrainsCodeEditor(CodeEditor[JetBrainsSymbol]):
-    def __init__(self, project: Project, agent: Optional["SerenaAgent"] = None) -> None:
+    def __init__(self, project: Project) -> None:
         self._project = project
-        super().__init__(project_root=project.project_root, agent=agent)
+        super().__init__(project_root=project.project_root, project_config=project.project_config)
 
     class EditedFile(CodeEditor.EditedFile):
         def __init__(self, relative_path: str, project: Project):

src/serena/config/serena_config.py

@@ -161,6 +161,12 @@ class LanguageBackend(Enum):
                 return backend
         raise ValueError(f"Unknown language backend '{backend_str}': valid values are {[b.value for b in LanguageBackend]}")
 
+    def is_lsp(self) -> bool:
+        return self == LanguageBackend.LSP
+
+    def is_jetbrains(self) -> bool:
+        return self == LanguageBackend.JETBRAINS
+
 
 @dataclass
 class SharedConfig(ModeSelectionDefinition, ToolInclusionDefinition, ToStringMixin):

src/serena/dashboard.py

@@ -507,7 +507,7 @@ class SerenaDashboardAPI:
             available_modes=available_modes,
             available_contexts=available_contexts,
             available_memories=available_memories,
-            jetbrains_mode=not self._agent.is_using_language_server(),
+            jetbrains_mode=self._agent.get_language_backend().is_jetbrains(),
             languages=languages,
             encoding=encoding,
             current_client=Tool.get_last_tool_call_client_str(),

src/serena/jetbrains/jetbrains_plugin_client.py

@@ -8,6 +8,7 @@ import logging
 import re
 import threading
 from concurrent.futures.thread import ThreadPoolExecutor
+from dataclasses import dataclass
 from pathlib import Path
 from typing import Any, Literal, Optional, Self, TypeVar, cast
 
@@ -16,6 +17,7 @@ from requests import Response
 from sensai.util.string import ToStringMixin
 
 import serena.jetbrains.jetbrains_types as jb
+from serena.config.serena_config import RegisteredProject
 from serena.jetbrains.jetbrains_types import PluginStatusDTO
 from serena.project import Project
 from serena.util.class_decorators import singleton
@@ -63,6 +65,12 @@ class ServerNotFoundError(Exception):
     """Raised when the plugin's service is not found."""
 
 
+@dataclass
+class MatchedClient:
+    client: "JetBrainsPluginClient"
+    registered_project: RegisteredProject
+
+
 @singleton
 class JetBrainsPluginClientManager:
     """
@@ -73,6 +81,7 @@ class JetBrainsPluginClientManager:
 
     def __init__(self) -> None:
         self._clients: dict[int, "JetBrainsPluginClient"] = {}
+        self._matched_clients: list[MatchedClient] = []
         self._lock = threading.Lock()
 
     def _submit_scan(self) -> list[concurrent.futures.Future["JetBrainsPluginClient"]]:
@@ -116,6 +125,47 @@ class JetBrainsPluginClientManager:
             "with the Serena plugin installed and running!"
         )
 
+    def match_clients(self, registered_projects: list[RegisteredProject]) -> list[MatchedClient]:
+        """
+        Scans for plugin instances and matches them against the given registered projects.
+
+        :param registered_projects: the list of registered projects to match plugin instances against
+        :return: the list of matched clients with their corresponding registered project
+        """
+        matched_clients = []
+        for future in self._submit_scan():
+            client = future.result()
+            if client.project_root is not None:
+                for rp in registered_projects:
+                    if client.matches(Path(rp.project_root)):
+                        matched_clients.append(MatchedClient(client, rp))
+                        break
+        self._matched_clients = matched_clients
+        return matched_clients
+
+    def get_matched_client(
+        self, registered_project: RegisteredProject, registered_projects: list[RegisteredProject]
+    ) -> Optional["JetBrainsPluginClient"]:
+        """
+        Gets the matched client for a given registered project, if any.
+
+        :param registered_project: the registered project to get the matched client for
+        :param registered_projects: the list of all registered projects (used to perform matching of all clients
+            if no match is found for the given project)
+        :return: the matched client or None if no match is found
+        """
+
+        def find_match() -> Optional["JetBrainsPluginClient"]:
+            for matched_client in self._matched_clients:
+                if matched_client.registered_project.project_root == registered_project.project_root:
+                    return matched_client.client
+            return None
+
+        match = find_match()
+        if match is None:
+            self.match_clients(registered_projects)
+        return find_match()
+
 
 class JetBrainsPluginClient(ToStringMixin):
     """

src/serena/resources/config/modes/query-projects.yml

@@ -0,0 +1,8 @@
+description: Enables tools that allow inactive projects to be queried
+prompt: |
+  You can use the 'query_project' tool to query Serena projects without activating them.
+  Use this when a project is related to the active project and you need to query it for information.
+excluded_tools: []
+included_optional_tools:
+  - list_queryable_projects
+  - query_project

src/serena/tools/__init__.py

@@ -7,3 +7,4 @@ from .cmd_tools import *
 from .config_tools import *
 from .workflow_tools import *
 from .jetbrains_tools import *
+from .query_project_tools import *

src/serena/tools/query_project_tools.py

@@ -0,0 +1,60 @@
+import json
+
+from serena.jetbrains.jetbrains_plugin_client import JetBrainsPluginClientManager
+from serena.tools import Tool, ToolMarkerDoesNotRequireActiveProject, ToolMarkerOptional
+
+
+class ListQueryableProjectsTool(Tool, ToolMarkerDoesNotRequireActiveProject, ToolMarkerOptional):
+    """
+    Tool for listing all projects that can be queried by the QueryProjectTool.
+    """
+
+    def apply(self, symbol_access: bool = True) -> str:
+        """
+        Lists available projects that can be queried with `query_project_tool`.
+
+        :param symbol_access: whether to return only projects for which symbol access is available. Default: true
+        """
+        # determine relevant projects
+        registered_projects = self.agent.serena_config.projects
+        if symbol_access:
+            backend = self.agent.get_language_backend()
+            if backend.is_jetbrains():
+                matched_clients = JetBrainsPluginClientManager().match_clients(registered_projects)
+                relevant_projects = [mc.registered_project for mc in matched_clients]
+            else:
+                raise NotImplementedError(f"External symbol access is generally unavailable for language backend {backend.value}")
+        else:
+            relevant_projects = registered_projects
+
+        # return project names, excluding the active project (if any)
+        project_names = [p.project_name for p in relevant_projects]
+        active_project = self.agent.get_active_project()
+        if active_project is not None:
+            project_names = [n for n in project_names if n != active_project.project_name]
+        return self._to_json(project_names)
+
+
+class QueryProjectTool(Tool, ToolMarkerDoesNotRequireActiveProject, ToolMarkerOptional):
+    """
+    Tool for querying external project information (i.e. information from projects other than the current one),
+    by executing a read-only tool.
+    """
+
+    def apply(self, project_name: str, tool_name: str, tool_params_json: str) -> str:
+        """
+        Queries a project by executing a read-only Serena tool. The tool will be executed in the context of the project.
+        Use this to query information from projects other than the activated project.
+
+        :param project_name: the name of the project to query
+        :param tool_name: the name of the tool to execute in the other project. The tool must be read-only.
+        :param tool_params_json: the parameters to pass to the tool, encoded as a JSON string
+        """
+        tool = self.agent.get_tool_by_name(tool_name)
+        assert tool.is_active(), f"Tool {tool_name} is not active."
+        assert tool.is_readonly(), f"Tool {tool_name} is not read-only and cannot be executed in another project."
+        registered_project = self.agent.serena_config.get_registered_project(project_name)
+        assert registered_project is not None, f"Project {project_name} is not registered and cannot be queried."
+        project = registered_project.get_project_instance(self.agent.serena_config)
+        with tool.project_override_context(project):
+            return tool.apply(**json.loads(tool_params_json))  # type: ignore

src/serena/tools/tools_base.py

@@ -1,7 +1,8 @@
 import inspect
 import json
 from abc import ABC
-from collections.abc import Iterable
+from collections.abc import Iterable, Iterator
+from contextlib import contextmanager
 from dataclasses import dataclass
 from types import TracebackType
 from typing import TYPE_CHECKING, Any, Protocol, Self, TypeVar, cast
@@ -12,6 +13,7 @@ from mcp.server.fastmcp.utilities.func_metadata import FuncMetadata, func_metada
 from sensai.util import logging
 from sensai.util.string import dict_string
 
+from serena.config.serena_config import LanguageBackend
 from serena.project import MemoriesManager, Project
 from serena.prompt_factory import PromptFactory
 from serena.util.class_decorators import singleton
@@ -31,12 +33,33 @@ SUCCESS_RESULT = "OK"
 class Component(ABC):
     def __init__(self, agent: "SerenaAgent"):
         self.agent = agent
+        self._project_override: Project | None = None
+
+    @contextmanager
+    def project_override_context(self, project: Project) -> Iterator[None]:
+        """
+        Context manager for temporarily overriding the active project in a component,
+        allowing tool executions to use the overridden project instead of the active project.
+
+        :param project: the project to override with
+        """
+        if self._project_override is not None:
+            raise RuntimeError("Nested project overrides are not supported.")
+        self._project_override = project
+        try:
+            yield
+        finally:
+            self._project_override = None
+
+    def _assert_is_not_project_override(self) -> None:
+        if self._project_override is not None:
+            raise RuntimeError("This method is only supported for the active project.")
 
     def get_project_root(self) -> str:
         """
         :return: the root directory of the active project, raises a ValueError if no active project configuration is set
         """
-        return self.agent.get_project_root()
+        return self.project.project_root
 
     @property
     def prompt_factory(self) -> PromptFactory:
@@ -49,22 +72,28 @@ class Component(ABC):
     def create_language_server_symbol_retriever(self) -> "LanguageServerSymbolRetriever":
         from serena.symbol import LanguageServerSymbolRetriever
 
-        if not self.agent.is_using_language_server():
-            raise Exception("Cannot create LanguageServerSymbolRetriever; agent is not in language server mode.")
+        self._assert_is_not_project_override()
+        assert self.agent.get_language_backend().is_lsp(), "Language server symbol retriever can only be created for LSP language backend"
         language_server_manager = self.agent.get_language_server_manager_or_raise()
         return LanguageServerSymbolRetriever(language_server_manager, agent=self.agent)
 
     @property
     def project(self) -> Project:
-        return self.agent.get_active_project_or_raise()
+        if self._project_override is not None:
+            return self._project_override
+        else:
+            return self.agent.get_active_project_or_raise()
 
     def create_code_editor(self) -> "CodeEditor":
         from ..code_editor import JetBrainsCodeEditor, LanguageServerCodeEditor
 
-        if self.agent.is_using_language_server():
-            return LanguageServerCodeEditor(self.create_language_server_symbol_retriever(), agent=self.agent)
-        else:
-            return JetBrainsCodeEditor(project=self.project, agent=self.agent)
+        match self.agent.get_language_backend():
+            case LanguageBackend.LSP:
+                return LanguageServerCodeEditor(self.create_language_server_symbol_retriever(), project_config=self.project.project_config)
+            case LanguageBackend.JETBRAINS:
+                return JetBrainsCodeEditor(project=self.project)
+            case _:
+                raise ValueError
 
 
 class ToolMarker:
@@ -234,6 +263,9 @@ class Tool(Component):
     def is_active(self) -> bool:
         return self.agent.tool_is_active(self.get_name())
 
+    def is_readonly(self) -> bool:
+        return not self.can_edit()
+
     def apply_ex(self, log_call: bool = True, catch_exceptions: bool = True, mcp_ctx: Context | None = None, **kwargs) -> str:  # type: ignore
         """
         Applies the tool with logging and exception handling, using the given keyword arguments
@@ -392,6 +424,8 @@ class ToolRegistry:
             self._tool_dict[name] = RegisteredTool(tool_class=cls, is_optional=is_optional, tool_name=name)
 
     def get_tool_class_by_name(self, tool_name: str) -> type[Tool]:
+        if tool_name not in self._tool_dict:
+            raise ValueError(f"Tool named '{tool_name}' not found.")
         return self._tool_dict[tool_name].tool_class
 
     def get_all_tool_classes(self) -> list[type[Tool]]:

test/serena/config/test_serena_config.py

@@ -217,8 +217,7 @@ class TestEffectiveLanguageBackend:
         config, name = _make_config_with_project("test_proj", language_backend=None, global_backend=LanguageBackend.LSP)
         agent = SerenaAgent(project=name, serena_config=config)
         try:
-            assert agent.get_language_backend() == LanguageBackend.LSP
-            assert agent.is_using_language_server() is True
+            assert agent.get_language_backend().is_lsp()
         finally:
             agent.shutdown(timeout=5)
 
@@ -229,8 +228,7 @@ class TestEffectiveLanguageBackend:
         )
         agent = SerenaAgent(project=name, serena_config=config)
         try:
-            assert agent.get_language_backend() == LanguageBackend.JETBRAINS
-            assert agent.is_using_language_server() is False
+            assert agent.get_language_backend().is_jetbrains()
         finally:
             agent.shutdown(timeout=5)