diff --git a/.serena/project.yml b/.serena/project.yml index f416c01c..a9218e45 100644 --- a/.serena/project.yml +++ b/.serena/project.yml @@ -90,7 +90,22 @@ included_optional_tools: [] # This cannot be combined with non-empty excluded_tools or included_optional_tools. # initial prompt for the project. It will always be given to the LLM upon activating the project # (contrary to the memories, which are loaded on demand). -initial_prompt: "" +initial_prompt: | + IMPORTANT: You use an idiomatic, object-oriented style. + In particular, this implies that, for any non-trivial interfaces, you use interfaces that expect explicitly typed abstractions + rather than mere functions (i.e. use the strategy pattern, for example). + You avoid the use of low-level data structures in all cases where an object-oriented abstraction would be more appropriate. + For simple data storage, you use dataclasses instead of dictionaries or tuples. + + You structure function implementations into functional blocks that are separated by blank lines. + Atop each functional block, you write an elliptical phrase (starting with lower-case letter) that describes the purpose of the + block in a concise manner. + + Docstrings: You consistently use reStructuredText. + + Comments: + When describing parameters, methods/functions and classes, you use a precise style, where the initial (elliptical) phrase + clearly defines *what* it is. Any details then follow in subsequent sentences. # the encoding used by text files in the project # For a list of possible encodings, see https://docs.python.org/3.11/library/codecs.html#standard-encodings diff --git a/CHANGELOG.md b/CHANGELOG.md index 8d3dd8d0..90d7039b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,10 @@ Status of the `main` branch. Changes prior to the next official version change w * Project configuration files (`project.yml`) can now define multiple languages. Auto-detection adds only the most prominent language by default. * Additional languages can be conveniently added via the Dashboard while a project is already activated. + * Add support for querying projects other than the currently active one via new tools `QueryProjectTool` and `ListQueryableProjectsTool`. + The `QueryProjectTool` allows Serena tools to be called on other projects. + * For the LSP backend, calling symbolic tools require a project server to be spawned that will launch the respective language servers + * For the JetBrains backend, all projects for which IDE instances are open can directly be queried * The current project can be activated automatically even when the MCP configuration is global (through the --project-from-cwd flag) * Support overloaded symbols in `FindSymbolTool` and related tools * Name paths of overloaded symbols now include an index (e.g., `myOverloadedFunction[2]`) diff --git a/docs/02-usage/040_workflow.md b/docs/02-usage/040_workflow.md index 53006b18..1b73e622 100644 --- a/docs/02-usage/040_workflow.md +++ b/docs/02-usage/040_workflow.md @@ -36,7 +36,7 @@ For instance, when using `uvx`, run * For an existing project, the main programming language will be detected automatically, but you can choose to explicitly specify multiple languages by passing the `--language` parameter multiple times (e.g. `--language python --language typescript`). - * You can optionally specify a custom project name with `--name "My Project"`. + * You can optionally specify a custom project name with `--name my-name`. * You can immediately index the project after creation with `--index`. (project-config)= @@ -153,4 +153,42 @@ linting results, and test results to assess the correctness of its actions. Therefore, software that is designed to meaningful interpretable outputs (e.g. log messages) and that has a good test coverage is much easier to work with for Serena. -We generally recommend to start an editing task from a state where all linting checks and tests pass. \ No newline at end of file +We generally recommend to start an editing task from a state where all linting checks and tests pass. + +## Working with Multiple Projects Simultaneously + +There are several ways in which you might want to work with multiple projects simultaneously. + +### Simultaneously Editing in Multiple Projects + +If fulfilling a task requires the agent to edit code in multiple projects, the recommended approach is to create a **monorepo folder**, +i.e. a folder that contains all the projects as sub-folders, and open that monorepo folder as a project in Serena. +You may also use symbolic links to create a monorepo folder if the projects are located in different places on your filesystem. + +If several languages are used across the projects, specify all of them as needed when using the LSP backend; +For JetBrains mode, make sure that your IDE is configured to work with all the languages used across the projects (e.g. by installing the respective language plugins). + +(query-projects)= +### Reading from External Projects + +If, while working on a project, you want Serena to be able to read code or other information from another project (e.g. a library or otherwise related project), +this can be enabled via the `query_project` tool. +Provided that the project you want to query is known to Serena (i.e. you have created it as described above), +the `query_project` tool allows the agent to query files and symbolic information from that project. + +To enable this tool, [activate the mode](modes) `query-projects`. +This also enables a second tool for listing projects that can be queried. + +Depending on the language backend being used, the management of resources for the external projects varies: + +* When using the JetBrains backend, make sure that every project for which you want symbolic queries to work is open in an IDE instance. +* When using the LSP backend, executing symbolic tools via the query tool requires that Serena's **Project Server** be started, + which will automatically spawn the necessary language servers for the projects that are queried. + + To start the server, run + + start-project-server + + where `` is your way of running Serena. For example, when using `uvx`, run + + uvx --from git+https://github.com/oraios/serena serena start-project-server diff --git a/docs/02-usage/050_configuration.md b/docs/02-usage/050_configuration.md index aaf256af..dd0704ae 100644 --- a/docs/02-usage/050_configuration.md +++ b/docs/02-usage/050_configuration.md @@ -113,7 +113,8 @@ Examples of built-in modes include: * `one-shot`: Configures Serena for tasks that should be completed in a single response, often used with `planning` for generating reports or initial plans. * `no-onboarding`: Skips the initial onboarding process if it's not needed for a particular session but retains the memory tools (assuming initial memories were created externally). * `onboarding`: Focuses on the project onboarding process. -* `no-memories`: Disables all memory tools (and tools building on memories such as onboarding tools) +* `no-memories`: Disables all memory tools (and tools building on memories such as onboarding tools) +* `query-projects`: Enables tools for querying other Serena projects (without activating them); see section [Reading from External Projects](query-projects) Find the concrete definitions of these modes [here](https://github.com/oraios/serena/tree/main/src/serena/resources/config/modes). diff --git a/src/serena/agent.py b/src/serena/agent.py index daa87820..89a4f838 100644 --- a/src/serena/agent.py +++ b/src/serena/agent.py @@ -6,7 +6,8 @@ import os import platform import subprocess import sys -from collections.abc import Callable, Sequence +from collections.abc import Callable, Iterator, Sequence +from contextlib import contextmanager from logging import Logger from typing import TYPE_CHECKING, Optional, TypeVar @@ -551,15 +552,6 @@ class SerenaAgent: ) return True - def get_project_root(self) -> str: - """ - :return: the root directory of the active project (if any); raises a ValueError if there is no active project - """ - project = self.get_active_project() - if project is None: - raise ValueError("Cannot get project root if no project is active.") - return project.project_root - def get_exposed_tool_instances(self) -> list["Tool"]: """ :return: the tool instances which are exposed (e.g. to the MCP client). @@ -729,7 +721,7 @@ class SerenaAgent: self.reset_language_server_manager() # initialize the language server in the background (if in language server mode) - if self.is_using_language_server(): + if self.get_language_backend().is_lsp(): self.issue_task(init_language_server_manager) if self._project_activation_callback is not None: @@ -826,23 +818,9 @@ class SerenaAgent: """ Starts/resets the language server manager for the current project """ - tool_timeout = self.serena_config.tool_timeout - if tool_timeout is None or tool_timeout < 0: - ls_timeout = None - else: - if tool_timeout < 10: - raise ValueError(f"Tool timeout must be at least 10 seconds, but is {tool_timeout} seconds") - ls_timeout = tool_timeout - 5 # the LS timeout is for a single call, it should be smaller than the tool timeout - - # instantiate and start the necessary language servers try: self._lsp_init_error = None - self.get_active_project_or_raise().create_language_server_manager( - log_level=self.serena_config.log_level, - ls_timeout=ls_timeout, - trace_lsp_communication=self.serena_config.trace_lsp_communication, - ls_specific_settings=self.serena_config.ls_specific_settings, - ) + self.get_active_project_or_raise().create_language_server_manager() except LanguageServerManagerInitialisationError as e: self._lsp_init_error = e raise @@ -899,3 +877,17 @@ class SerenaAgent: if ls_manager is None: return [] return ls_manager.get_active_languages() + + @contextmanager + def active_project_context(self, project: Project) -> Iterator[None]: + """ + Context manager for temporarily setting/overriding the active project + + :param project: the project to be active + """ + original_project = self._active_project + self._active_project = project + try: + yield + finally: + self._active_project = original_project diff --git a/src/serena/cli.py b/src/serena/cli.py index 20139f23..ef14d17b 100644 --- a/src/serena/cli.py +++ b/src/serena/cli.py @@ -350,6 +350,61 @@ class TopLevelCommands(AutoRegisteringGroup): else: print(f"{prefix}\n{instr}\n{postfix}") + @staticmethod + @click.command( + "start-project-server", + help="Starts the Serena project server, which exposes project querying capabilities via HTTP.", + context_settings={"max_content_width": _MAX_CONTENT_WIDTH}, + ) + @click.option( + "--host", + type=str, + default="127.0.0.1", + show_default=True, + help="Listen address for the project server.", + ) + @click.option( + "--port", + type=int, + default=None, + help="Listen port for the project server (default: ProjectServer.PORT).", + ) + @click.option( + "--log-level", + type=click.Choice(["DEBUG", "INFO", "WARNING", "ERROR", "CRITICAL"]), + default=None, + help="Override log level in config.", + ) + def start_project_server( + host: str, + port: int | None, + log_level: Literal["DEBUG", "INFO", "WARNING", "ERROR", "CRITICAL"] | None, + ) -> None: + from serena.project_server import ProjectServer + + # initialize logging + Logger.root.setLevel(logging.INFO) + formatter = logging.Formatter(SERENA_LOG_FORMAT) + stderr_handler = logging.StreamHandler(stream=sys.stderr) + stderr_handler.formatter = formatter + Logger.root.addHandler(stderr_handler) + log_path = SerenaPaths().get_next_log_file_path("project-server") + file_handler = logging.FileHandler(log_path, mode="w") + file_handler.formatter = formatter + Logger.root.addHandler(file_handler) + + if log_level is not None: + Logger.root.setLevel(logging.getLevelNamesMapping()[log_level]) + + log.info("Starting Serena project server") + log.info("Storing logs in %s", log_path) + + server = ProjectServer() + run_kwargs: dict[str, Any] = {"host": host} + if port is not None: + run_kwargs["port"] = port + server.run(**run_kwargs) + class ModeCommands(AutoRegisteringGroup): """Group for 'mode' subcommands.""" @@ -640,9 +695,7 @@ class ProjectCommands(AutoRegisteringGroup): serena_config = SerenaConfig.from_config_file() proj = registered_project.get_project_instance(serena_config=serena_config) click.echo(f"Indexing symbols in {proj} …") - ls_mgr = proj.create_language_server_manager( - log_level=lvl, ls_timeout=timeout, ls_specific_settings=serena_config.ls_specific_settings - ) + ls_mgr = proj.create_language_server_manager() try: log_file = os.path.join(proj.project_root, ".serena", "logs", "indexing.txt") diff --git a/src/serena/code_editor.py b/src/serena/code_editor.py index b5075c38..ea83da25 100644 --- a/src/serena/code_editor.py +++ b/src/serena/code_editor.py @@ -4,8 +4,9 @@ import os from abc import ABC, abstractmethod from collections.abc import Iterable, Iterator, Reversible from contextlib import contextmanager -from typing import TYPE_CHECKING, Generic, Optional, TypeVar, cast +from typing import Generic, TypeVar, cast +from serena.config.serena_config import ProjectConfig from serena.jetbrains.jetbrains_plugin_client import JetBrainsPluginClient from serena.symbol import JetBrainsSymbol, LanguageServerSymbol, LanguageServerSymbolRetriever, PositionInFile, Symbol from solidlsp import SolidLanguageServer, ls_types @@ -15,26 +16,16 @@ from solidlsp.ls_utils import PathUtils, TextUtils from .constants import DEFAULT_SOURCE_FILE_ENCODING from .project import Project -if TYPE_CHECKING: - from .agent import SerenaAgent - - log = logging.getLogger(__name__) TSymbol = TypeVar("TSymbol", bound=Symbol) class CodeEditor(Generic[TSymbol], ABC): - def __init__(self, project_root: str, agent: Optional["SerenaAgent"] = None) -> None: + def __init__(self, project_root: str, project_config: ProjectConfig | None = None) -> None: self.project_root = project_root - self.agent = agent - # set encoding based on active project, if available - encoding = DEFAULT_SOURCE_FILE_ENCODING - if agent is not None: - project = agent.get_active_project() - if project is not None: - encoding = project.project_config.encoding - self.encoding = encoding + # set encoding based on project configuration, if available + self.encoding = project_config.encoding if project_config is not None else DEFAULT_SOURCE_FILE_ENCODING class EditedFile(ABC): def __init__(self, relative_path: str) -> None: @@ -244,8 +235,8 @@ class CodeEditor(Generic[TSymbol], ABC): class LanguageServerCodeEditor(CodeEditor[LanguageServerSymbol]): - def __init__(self, symbol_retriever: LanguageServerSymbolRetriever, agent: Optional["SerenaAgent"] = None): - super().__init__(project_root=symbol_retriever.get_root_path(), agent=agent) + def __init__(self, symbol_retriever: LanguageServerSymbolRetriever, project_config: ProjectConfig | None = None): + super().__init__(project_root=symbol_retriever.get_root_path(), project_config=project_config) self._symbol_retriever = symbol_retriever def _get_language_server(self, relative_path: str) -> SolidLanguageServer: @@ -379,9 +370,9 @@ class LanguageServerCodeEditor(CodeEditor[LanguageServerSymbol]): class JetBrainsCodeEditor(CodeEditor[JetBrainsSymbol]): - def __init__(self, project: Project, agent: Optional["SerenaAgent"] = None) -> None: + def __init__(self, project: Project) -> None: self._project = project - super().__init__(project_root=project.project_root, agent=agent) + super().__init__(project_root=project.project_root, project_config=project.project_config) class EditedFile(CodeEditor.EditedFile): def __init__(self, relative_path: str, project: Project): diff --git a/src/serena/config/serena_config.py b/src/serena/config/serena_config.py index 798ef1af..83ad206f 100644 --- a/src/serena/config/serena_config.py +++ b/src/serena/config/serena_config.py @@ -173,6 +173,12 @@ class LanguageBackend(Enum): return backend raise ValueError(f"Unknown language backend '{backend_str}': valid values are {[b.value for b in LanguageBackend]}") + def is_lsp(self) -> bool: + return self == LanguageBackend.LSP + + def is_jetbrains(self) -> bool: + return self == LanguageBackend.JETBRAINS + @dataclass class SharedConfig(ModeSelectionDefinition, ToolInclusionDefinition, ToStringMixin): @@ -494,7 +500,7 @@ class RegisteredProject(ToStringMixin): """ return self.project_root.samefile(Path(path).resolve()) - def get_project_instance(self, serena_config: "SerenaConfig | None") -> "Project": + def get_project_instance(self, serena_config: "SerenaConfig") -> "Project": """ Returns the project instance for this registered project, loading it if necessary. """ @@ -773,7 +779,7 @@ class SerenaConfig(SharedConfig): return project_candidates[0] elif len(project_candidates) > 1: raise ValueError( - f"Multiple projects found with name '{project_root_or_name}'. Please activate it by location instead. " + f"Multiple projects found with name '{project_root_or_name}'. Please reference it by location instead. " f"Locations: {[p.project_root for p in project_candidates]}" ) # no project found by name; check if it's a path diff --git a/src/serena/dashboard.py b/src/serena/dashboard.py index 26ec8f56..7c944a72 100644 --- a/src/serena/dashboard.py +++ b/src/serena/dashboard.py @@ -524,7 +524,7 @@ class SerenaDashboardAPI: available_modes=available_modes, available_contexts=available_contexts, available_memories=available_memories, - jetbrains_mode=not self._agent.is_using_language_server(), + jetbrains_mode=self._agent.get_language_backend().is_jetbrains(), languages=languages, encoding=encoding, current_client=Tool.get_last_tool_call_client_str(), diff --git a/src/serena/jetbrains/jetbrains_plugin_client.py b/src/serena/jetbrains/jetbrains_plugin_client.py index f11dc080..f237a079 100644 --- a/src/serena/jetbrains/jetbrains_plugin_client.py +++ b/src/serena/jetbrains/jetbrains_plugin_client.py @@ -8,6 +8,7 @@ import logging import re import threading from concurrent.futures.thread import ThreadPoolExecutor +from dataclasses import dataclass from pathlib import Path from typing import Any, Literal, Optional, Self, TypeVar, cast @@ -16,6 +17,7 @@ from requests import Response from sensai.util.string import ToStringMixin import serena.jetbrains.jetbrains_types as jb +from serena.config.serena_config import RegisteredProject from serena.jetbrains.jetbrains_types import PluginStatusDTO from serena.project import Project from serena.util.class_decorators import singleton @@ -63,6 +65,12 @@ class ServerNotFoundError(Exception): """Raised when the plugin's service is not found.""" +@dataclass +class MatchedClient: + client: "JetBrainsPluginClient" + registered_project: RegisteredProject + + @singleton class JetBrainsPluginClientManager: """ @@ -73,6 +81,7 @@ class JetBrainsPluginClientManager: def __init__(self) -> None: self._clients: dict[int, "JetBrainsPluginClient"] = {} + self._matched_clients: list[MatchedClient] = [] self._lock = threading.Lock() def _submit_scan(self) -> list[concurrent.futures.Future["JetBrainsPluginClient"]]: @@ -116,6 +125,47 @@ class JetBrainsPluginClientManager: "with the Serena plugin installed and running!" ) + def match_clients(self, registered_projects: list[RegisteredProject]) -> list[MatchedClient]: + """ + Scans for plugin instances and matches them against the given registered projects. + + :param registered_projects: the list of registered projects to match plugin instances against + :return: the list of matched clients with their corresponding registered project + """ + matched_clients = [] + for future in self._submit_scan(): + client = future.result() + if client.project_root is not None: + for rp in registered_projects: + if client.matches(Path(rp.project_root)): + matched_clients.append(MatchedClient(client, rp)) + break + self._matched_clients = matched_clients + return matched_clients + + def get_matched_client( + self, registered_project: RegisteredProject, registered_projects: list[RegisteredProject] + ) -> Optional["JetBrainsPluginClient"]: + """ + Gets the matched client for a given registered project, if any. + + :param registered_project: the registered project to get the matched client for + :param registered_projects: the list of all registered projects (used to perform matching of all clients + if no match is found for the given project) + :return: the matched client or None if no match is found + """ + + def find_match() -> Optional["JetBrainsPluginClient"]: + for matched_client in self._matched_clients: + if matched_client.registered_project.project_root == registered_project.project_root: + return matched_client.client + return None + + match = find_match() + if match is None: + self.match_clients(registered_projects) + return find_match() + class JetBrainsPluginClient(ToStringMixin): """ diff --git a/src/serena/project.py b/src/serena/project.py index ed4821cb..dd9a4d3e 100644 --- a/src/serena/project.py +++ b/src/serena/project.py @@ -11,7 +11,6 @@ from sensai.util.logging import LogTime from sensai.util.string import ToStringMixin from serena.config.serena_config import ( - DEFAULT_TOOL_TIMEOUT, ProjectConfig, SerenaPaths, get_serena_managed_in_project_dir, @@ -197,13 +196,16 @@ class MemoriesManager: class Project(ToStringMixin): def __init__( self, + *, project_root: str, project_config: ProjectConfig, + serena_config: "SerenaConfig", is_newly_created: bool = False, - serena_config: "SerenaConfig | None" = None, ): + assert serena_config is not None self.project_root = project_root self.project_config = project_config + self._serena_config = serena_config global_memory_write_access = serena_config.edit_global_memories if serena_config else False self.memories_manager = MemoriesManager(project_root, global_memory_write_access) @@ -220,7 +222,6 @@ class Project(ToStringMixin): f.write(f"/{SolidLanguageServer.CACHE_FOLDER_NAME}\n") # prepare ignore spec asynchronously, ensuring immediate project activation. - self._serena_config = serena_config self.__ignored_patterns: list[str] self.__ignore_spec: pathspec.PathSpec self._ignore_spec_available = threading.Event() @@ -230,7 +231,7 @@ class Project(ToStringMixin): with LogTime(f"Gathering ignore spec for project {self.project_config.project_name}", logger=log): # gather ignored paths from the global configuration, project configuration, and gitignore files - global_ignored_paths = self._serena_config.ignored_paths if self._serena_config else [] + global_ignored_paths = self._serena_config.ignored_paths ignored_patterns = list(global_ignored_paths) + list(self.project_config.ignored_paths) if len(global_ignored_paths) > 0: log.info(f"Using {len(global_ignored_paths)} ignored paths from the global configuration.") @@ -272,7 +273,7 @@ class Project(ToStringMixin): def load( cls, project_root: str | Path, - serena_config: "SerenaConfig | None", + serena_config: "SerenaConfig", autogenerate: bool = True, ) -> "Project": project_root = Path(project_root).resolve() @@ -547,23 +548,21 @@ class Project(ToStringMixin): source_file_path=relative_file_path, ) - def create_language_server_manager( - self, - log_level: int = logging.INFO, - ls_timeout: float | None = DEFAULT_TOOL_TIMEOUT - 5, - trace_lsp_communication: bool = False, - ls_specific_settings: dict[Language, Any] | None = None, - ) -> LanguageServerManager: + def create_language_server_manager(self) -> LanguageServerManager: """ Creates the language server manager for the project, starting one language server per configured programming language. - :param log_level: the log level for the language server - :param ls_timeout: the timeout for the language server - :param trace_lsp_communication: whether to trace LSP communication - :param ls_specific_settings: optional LS specific configuration of the language server, - see docstrings in the inits of subclasses of SolidLanguageServer to see what values may be passed. :return: the language server manager, which is also stored in the project instance """ + # determine timeout to use for LS calls + tool_timeout = self._serena_config.tool_timeout + if tool_timeout is None or tool_timeout < 0: + ls_timeout = None + else: + if tool_timeout < 10: + raise ValueError(f"Tool timeout must be at least 10 seconds, but is {tool_timeout} seconds") + ls_timeout = tool_timeout - 5 # the LS timeout is for a single call, it should be smaller than the tool timeout + # if there is an existing instance, stop its language servers first if self.language_server_manager is not None: log.info("Stopping existing language server manager ...") @@ -576,8 +575,8 @@ class Project(ToStringMixin): encoding=self.project_config.encoding, ignored_patterns=self._ignored_patterns, ls_timeout=ls_timeout, - ls_specific_settings=ls_specific_settings, - trace_lsp_communication=trace_lsp_communication, + ls_specific_settings=self._serena_config.ls_specific_settings, + trace_lsp_communication=self._serena_config.trace_lsp_communication, ) self.language_server_manager = LanguageServerManager.from_languages(self.project_config.languages, factory) return self.language_server_manager diff --git a/src/serena/project_server.py b/src/serena/project_server.py new file mode 100644 index 00000000..ef88e883 --- /dev/null +++ b/src/serena/project_server.py @@ -0,0 +1,157 @@ +import json +import logging +from typing import TYPE_CHECKING + +import requests as requests_lib +from flask import Flask, request +from pydantic import BaseModel +from sensai.util.logging import LogTime + +from serena.config.serena_config import LanguageBackend, SerenaConfig +from serena.jetbrains.jetbrains_plugin_client import JetBrainsPluginClient + +if TYPE_CHECKING: + from serena.project import Project + +log = logging.getLogger(__name__) + +# disable Werkzeug's logging to avoid cluttering the output +logging.getLogger("werkzeug").setLevel(logging.WARNING) + + +class QueryProjectRequest(BaseModel): + """ + Request model for the /query_project endpoint, matching the interface of + :class:`~serena.tools.query_project_tools.QueryProjectTool`. + """ + + project_name: str + tool_name: str + tool_params_json: str + + +class ProjectServer: + """ + A lightweight Flask server that exposes a SerenaAgent's project querying + capabilities via HTTP, using the LSP language server backend for symbolic retrieval. + + Projects are loaded on demand when a query is made for them, and cached in memory for subsequent queries. + + The server instantiates a :class:`SerenaAgent` with default options and + provides a ``/query_project`` endpoint whose interface matches + :class:`~serena.tools.query_project_tools.QueryProjectTool`. + """ + + PORT = JetBrainsPluginClient.BASE_PORT - 1 + + def __init__(self) -> None: + from serena.agent import SerenaAgent + + serena_config = SerenaConfig.from_config_file() + serena_config.gui_log_window = False + serena_config.web_dashboard = False + serena_config.language_backend = LanguageBackend.LSP + + self._agent = SerenaAgent(serena_config=serena_config) + self._loaded_projects_by_name: dict[str, "Project"] = {} + + # create the Flask application + self._app = Flask(__name__) + self._setup_routes() + + def _setup_routes(self) -> None: + @self._app.route("/heartbeat", methods=["GET"]) + def heartbeat() -> dict[str, str]: + return {"status": "alive"} + + @self._app.route("/query_project", methods=["POST"]) + def query_project() -> str: + query_request = QueryProjectRequest.model_validate(request.get_json()) + return self._query_project(query_request) + + def _get_project(self, project_name: str) -> "Project": + """Gets the project with the given name, loading it if necessary.""" + if project_name in self._loaded_projects_by_name: + return self._loaded_projects_by_name[project_name] + else: + serena_config = self._agent.serena_config + registered_project = serena_config.get_registered_project(project_name) + if registered_project is None: + raise ValueError(f"Project '{project_name}' is not registered with Serena.") + with LogTime(f"Loading project '{project_name}'"): + project = registered_project.get_project_instance(serena_config) + project.create_language_server_manager() + self._loaded_projects_by_name[project_name] = project + return project + + def _query_project(self, req: QueryProjectRequest) -> str: + """Handle a /query_project request by invoking the agent on the specified project and tool.""" + project = self._get_project(req.project_name) + with self._agent.active_project_context(project): + tool = self._agent.get_tool_by_name(req.tool_name) + params = json.loads(req.tool_params_json) + return tool.apply_ex(**params) + + def run(self, host: str = "127.0.0.1", port: int = PORT) -> int: + """Run the server on the given host and port. + + :param host: the host address to listen on. + :param port: the port to listen on. + :return: the port number the server is running on. + """ + from flask import cli + + # suppress the default Flask startup banner + cli.show_server_banner = lambda *args, **kwargs: None + + self._app.run(host=host, port=port, debug=False, use_reloader=False, threaded=True) + return port + + +class ProjectServerClient: + """Client for interacting with a running :class:`ProjectServer`. + + Upon instantiation, the client verifies that the server is reachable + by sending a heartbeat request. If the server is not running, a + :class:`ConnectionError` is raised. + """ + + def __init__(self, host: str = "127.0.0.1", port: int = ProjectServer.PORT, timeout: int = 300) -> None: + """ + :param host: the host address of the project server. + :param port: the port of the project server. + :raises ConnectionError: if the project server is not reachable. + """ + self._base_url = f"http://{host}:{port}" + self._timeout = timeout + + # verify that the server is running + try: + response = requests_lib.get(f"{self._base_url}/heartbeat", timeout=5) + response.raise_for_status() + except requests_lib.ConnectionError: + raise ConnectionError(f"ProjectServer is not reachable at {self._base_url}. Make sure the server is running.") + except requests_lib.RequestException as e: + raise ConnectionError(f"ProjectServer health check failed: {e}") + + def query_project(self, project_name: str, tool_name: str, tool_params_json: str) -> str: + """ + Query a project by executing a Serena tool in its context. + + The interface matches :meth:`QueryProjectTool.apply + `. + + :param project_name: the name of the project to query. + :param tool_name: the name of the tool to execute. The tool must be read-only. + :param tool_params_json: the parameters to pass to the tool, encoded as a JSON string. + :return: the tool's result as a string. + """ + payload = QueryProjectRequest( + project_name=project_name, + tool_name=tool_name, + tool_params_json=tool_params_json, + ).model_dump() + + response = requests_lib.post(f"{self._base_url}/query_project", json=payload, timeout=self._timeout) + response.raise_for_status() + return response.text diff --git a/src/serena/resources/config/modes/query-projects.yml b/src/serena/resources/config/modes/query-projects.yml new file mode 100644 index 00000000..ed140de9 --- /dev/null +++ b/src/serena/resources/config/modes/query-projects.yml @@ -0,0 +1,8 @@ +description: Enables tools that allow inactive projects to be queried +prompt: | + You can use the 'query_project' tool to query Serena projects without activating them. + Use this when a project is related to the active project and you need to query it for information. +excluded_tools: [] +included_optional_tools: + - list_queryable_projects + - query_project diff --git a/src/serena/tools/__init__.py b/src/serena/tools/__init__.py index e6fb4b08..08bb4338 100644 --- a/src/serena/tools/__init__.py +++ b/src/serena/tools/__init__.py @@ -7,3 +7,4 @@ from .cmd_tools import * from .config_tools import * from .workflow_tools import * from .jetbrains_tools import * +from .query_project_tools import * diff --git a/src/serena/tools/query_project_tools.py b/src/serena/tools/query_project_tools.py new file mode 100644 index 00000000..b675a0aa --- /dev/null +++ b/src/serena/tools/query_project_tools.py @@ -0,0 +1,80 @@ +import json + +from serena.config.serena_config import LanguageBackend +from serena.jetbrains.jetbrains_plugin_client import JetBrainsPluginClientManager +from serena.project_server import ProjectServerClient +from serena.tools import Tool, ToolMarkerDoesNotRequireActiveProject, ToolMarkerOptional + + +class ListQueryableProjectsTool(Tool, ToolMarkerOptional, ToolMarkerDoesNotRequireActiveProject): + """ + Tool for listing all projects that can be queried by the QueryProjectTool. + """ + + def apply(self, symbol_access: bool = True) -> str: + """ + Lists available projects that can be queried with `query_project_tool`. + + :param symbol_access: whether to return only projects for which symbol access is available. Default: true + """ + # determine relevant projects + registered_projects = self.agent.serena_config.projects + if symbol_access: + backend = self.agent.get_language_backend() + if backend.is_jetbrains(): + # projects with open IDE instances can be queried + matched_clients = JetBrainsPluginClientManager().match_clients(registered_projects) + relevant_projects = [mc.registered_project for mc in matched_clients] + else: + # all projects can be queried via ProjectServer (which instantiates projects dynamically) + relevant_projects = registered_projects + else: + relevant_projects = registered_projects + + # return project names, excluding the active project (if any) + project_names = [p.project_name for p in relevant_projects] + active_project = self.agent.get_active_project() + if active_project is not None: + project_names = [n for n in project_names if n != active_project.project_name] + return self._to_json(project_names) + + +class QueryProjectTool(Tool, ToolMarkerOptional, ToolMarkerDoesNotRequireActiveProject): + """ + Tool for querying external project information (i.e. information from projects other than the current one), + by executing a read-only tool. + """ + + def apply(self, project_name: str, tool_name: str, tool_params_json: str) -> str: + """ + Queries a project by executing a read-only Serena tool. The tool will be executed in the context of the project. + Use this to query information from projects other than the activated project. + + :param project_name: the name of the project to query + :param tool_name: the name of the tool to execute in the other project. The tool must be read-only. + :param tool_params_json: the parameters to pass to the tool, encoded as a JSON string + """ + tool = self.agent.get_tool_by_name(tool_name) + assert tool.is_active(), f"Tool {tool_name} is not active." + assert tool.is_readonly(), f"Tool {tool_name} is not read-only and cannot be executed in another project." + if self._is_project_server_required(tool): + client = ProjectServerClient() + return client.query_project(project_name, tool_name, tool_params_json) + else: + registered_project = self.agent.serena_config.get_registered_project(project_name) + assert registered_project is not None, f"Project {project_name} is not registered and cannot be queried." + project = registered_project.get_project_instance(self.agent.serena_config) + with tool.agent.active_project_context(project): + return tool.apply(**json.loads(tool_params_json)) # type: ignore + + def _is_project_server_required(self, tool: Tool) -> bool: + match self.agent.get_language_backend(): + case LanguageBackend.JETBRAINS: + return False + case LanguageBackend.LSP: + # Note: As long as only read-only tools are considered, only symbolic tools require the project server. + # But if we were to allow non-read-only tools, then tools using a CodeEditor also indirectly require language servers. + assert tool.is_readonly() + return tool.is_symbolic() + case _: + raise NotImplementedError diff --git a/src/serena/tools/tools_base.py b/src/serena/tools/tools_base.py index 76c0426b..e35c96e0 100644 --- a/src/serena/tools/tools_base.py +++ b/src/serena/tools/tools_base.py @@ -12,6 +12,7 @@ from mcp.server.fastmcp.utilities.func_metadata import FuncMetadata, func_metada from sensai.util import logging from sensai.util.string import dict_string +from serena.config.serena_config import LanguageBackend from serena.project import MemoriesManager, Project from serena.prompt_factory import PromptFactory from serena.util.class_decorators import singleton @@ -36,7 +37,7 @@ class Component(ABC): """ :return: the root directory of the active project, raises a ValueError if no active project configuration is set """ - return self.agent.get_project_root() + return self.project.project_root @property def prompt_factory(self) -> PromptFactory: @@ -49,8 +50,7 @@ class Component(ABC): def create_language_server_symbol_retriever(self) -> "LanguageServerSymbolRetriever": from serena.symbol import LanguageServerSymbolRetriever - if not self.agent.is_using_language_server(): - raise Exception("Cannot create LanguageServerSymbolRetriever; agent is not in language server mode.") + assert self.agent.get_language_backend().is_lsp(), "Language server symbol retriever can only be created for LSP language backend" language_server_manager = self.agent.get_language_server_manager_or_raise() return LanguageServerSymbolRetriever(language_server_manager, agent=self.agent) @@ -61,10 +61,13 @@ class Component(ABC): def create_code_editor(self) -> "CodeEditor": from ..code_editor import JetBrainsCodeEditor, LanguageServerCodeEditor - if self.agent.is_using_language_server(): - return LanguageServerCodeEditor(self.create_language_server_symbol_retriever(), agent=self.agent) - else: - return JetBrainsCodeEditor(project=self.project, agent=self.agent) + match self.agent.get_language_backend(): + case LanguageBackend.LSP: + return LanguageServerCodeEditor(self.create_language_server_symbol_retriever(), project_config=self.project.project_config) + case LanguageBackend.JETBRAINS: + return JetBrainsCodeEditor(project=self.project) + case _: + raise ValueError class ToolMarker: @@ -234,6 +237,12 @@ class Tool(Component): def is_active(self) -> bool: return self.agent.tool_is_active(self.get_name()) + def is_readonly(self) -> bool: + return not self.can_edit() + + def is_symbolic(self) -> bool: + return issubclass(self.__class__, ToolMarkerSymbolicRead) or issubclass(self.__class__, ToolMarkerSymbolicEdit) + def apply_ex(self, log_call: bool = True, catch_exceptions: bool = True, mcp_ctx: Context | None = None, **kwargs) -> str: # type: ignore """ Applies the tool with logging and exception handling, using the given keyword arguments @@ -393,6 +402,8 @@ class ToolRegistry: self._tool_dict[name] = RegisteredTool(tool_class=cls, is_optional=is_optional, tool_name=name) def get_tool_class_by_name(self, tool_name: str) -> type[Tool]: + if tool_name not in self._tool_dict: + raise ValueError(f"Tool named '{tool_name}' not found.") return self._tool_dict[tool_name].tool_class def get_all_tool_classes(self) -> list[type[Tool]]: diff --git a/test/conftest.py b/test/conftest.py index 565d40b7..6a93ff30 100644 --- a/test/conftest.py +++ b/test/conftest.py @@ -10,7 +10,7 @@ from typing import Any import pytest from sensai.util.logging import configure -from serena.config.serena_config import SerenaPaths +from serena.config.serena_config import SerenaConfig, SerenaPaths from serena.constants import SERENA_MANAGED_DIR_NAME from serena.project import Project from serena.util.file_system import GitignoreParser @@ -117,7 +117,8 @@ def start_default_ls_context(language: Language) -> Iterator[SolidLanguageServer def _create_default_project(language: Language) -> Project: repo_path = str(get_repo_path(language)) - return Project.load(repo_path, serena_config=None) + serena_config = SerenaConfig() + return Project.load(repo_path, serena_config=serena_config) @pytest.fixture(scope="session") diff --git a/test/serena/config/test_global_ignored_paths.py b/test/serena/config/test_global_ignored_paths.py index 23481373..b804fe8b 100644 --- a/test/serena/config/test_global_ignored_paths.py +++ b/test/serena/config/test_global_ignored_paths.py @@ -20,9 +20,7 @@ def _create_test_project( ignored_paths=project_ignored_paths or [], ignore_all_files_in_gitignore=False, ) - serena_config: SerenaConfig | None = None - if global_ignored_paths: - serena_config = SerenaConfig(gui_log_window=False, web_dashboard=False, ignored_paths=global_ignored_paths) + serena_config = SerenaConfig(gui_log_window=False, web_dashboard=False, ignored_paths=global_ignored_paths) return Project( project_root=str(project_root), project_config=config, @@ -84,21 +82,6 @@ class TestGlobalIgnoredPaths: # Non-ignored files still accessible assert not project.is_ignored_path(str(self.project_path / "node_modules" / "pkg" / "index.js")) - def test_default_global_ignored_paths_backward_compatible(self) -> None: - """Project created without global_ignored_paths parameter works as before.""" - config = ProjectConfig( - project_name="test_project", - languages=[Language.PYTHON], - ignored_paths=["build"], - ignore_all_files_in_gitignore=False, - ) - project = Project( - project_root=str(self.project_path), - project_config=config, - ) - assert project.is_ignored_path(str(self.project_path / "build" / "output.js")) - assert not project.is_ignored_path(str(self.project_path / "node_modules" / "pkg" / "index.js")) - def test_duplicate_patterns_across_global_and_project(self) -> None: """Duplicate patterns across global and project do not cause errors.""" project = _create_test_project( @@ -161,7 +144,8 @@ class TestRegisteredProjectGlobalIgnoredPaths: project_root=str(self.project_path), project_config=config, ) - project = registered.get_project_instance(serena_config=None) + serena_config = SerenaConfig(gui_log_window=False, web_dashboard=False, ignored_paths=[]) + project = registered.get_project_instance(serena_config=serena_config) assert not project.is_ignored_path(str(self.project_path / "node_modules" / "pkg.js")) def test_from_project_root_passes_global_ignored_paths(self) -> None: diff --git a/test/serena/config/test_serena_config.py b/test/serena/config/test_serena_config.py index e4515a0d..63f5b30f 100644 --- a/test/serena/config/test_serena_config.py +++ b/test/serena/config/test_serena_config.py @@ -191,6 +191,12 @@ def _make_config_with_project( global_backend: LanguageBackend = LanguageBackend.LSP, ) -> tuple[SerenaConfig, str]: """Create a SerenaConfig with a single registered project and return (config, project_name).""" + config = SerenaConfig( + gui_log_window=False, + web_dashboard=False, + log_level=logging.ERROR, + language_backend=global_backend, + ) project = Project( project_root=str(Path(__file__).parent.parent / "resources" / "repos" / "python" / "test_repo"), project_config=ProjectConfig( @@ -198,12 +204,7 @@ def _make_config_with_project( languages=[Language.PYTHON], language_backend=language_backend, ), - ) - config = SerenaConfig( - gui_log_window=False, - web_dashboard=False, - log_level=logging.ERROR, - language_backend=global_backend, + serena_config=config, ) config.projects = [RegisteredProject.from_project_instance(project)] return config, project_name @@ -217,8 +218,7 @@ class TestEffectiveLanguageBackend: config, name = _make_config_with_project("test_proj", language_backend=None, global_backend=LanguageBackend.LSP) agent = SerenaAgent(project=name, serena_config=config) try: - assert agent.get_language_backend() == LanguageBackend.LSP - assert agent.is_using_language_server() is True + assert agent.get_language_backend().is_lsp() finally: agent.shutdown(timeout=5) @@ -229,8 +229,7 @@ class TestEffectiveLanguageBackend: ) agent = SerenaAgent(project=name, serena_config=config) try: - assert agent.get_language_backend() == LanguageBackend.JETBRAINS - assert agent.is_using_language_server() is False + assert agent.get_language_backend().is_jetbrains() finally: agent.shutdown(timeout=5) @@ -261,6 +260,7 @@ class TestEffectiveLanguageBackend: languages=[Language.PYTHON], language_backend=LanguageBackend.JETBRAINS, ), + serena_config=config, ) config.projects.append(RegisteredProject.from_project_instance(jb_project)) @@ -283,6 +283,7 @@ class TestEffectiveLanguageBackend: languages=[Language.PYTHON], language_backend=LanguageBackend.LSP, ), + serena_config=config, ) config.projects.append(RegisteredProject.from_project_instance(lsp_project2)) @@ -305,6 +306,7 @@ class TestEffectiveLanguageBackend: languages=[Language.PYTHON], language_backend=None, ), + serena_config=config, ) config.projects.append(RegisteredProject.from_project_instance(proj2)) diff --git a/test/serena/test_serena_agent.py b/test/serena/test_serena_agent.py index b04dc37f..7b2d3147 100644 --- a/test/serena/test_serena_agent.py +++ b/test/serena/test_serena_agent.py @@ -21,7 +21,8 @@ from test.solidlsp import clojure as clj @pytest.fixture def serena_config(): - """Create an in-memory configuration for tests with test repositories pre-registered.""" + config = SerenaConfig(gui_log_window=False, web_dashboard=False, log_level=logging.ERROR) + # Create test projects for all supported languages test_projects = [] for language in [ @@ -47,16 +48,16 @@ def serena_config(): project_name=project_name, languages=[language], ignored_paths=[], - excluded_tools=set(), + excluded_tools=[], read_only=False, ignore_all_files_in_gitignore=True, initial_prompt="", encoding="utf-8", ), + serena_config=config, ) test_projects.append(RegisteredProject.from_project_instance(project)) - config = SerenaConfig(gui_log_window=False, web_dashboard=False, log_level=logging.ERROR) config.projects = test_projects return config