From 0f6471d2a851c8b37e29ca34f44d47e3182fb13b Mon Sep 17 00:00:00 2001 From: Dominik Jain Date: Sat, 28 Feb 2026 13:38:38 +0100 Subject: [PATCH] Introduce ProjectServer to support QueryProjectTool for the LSP backend The server allows arbitrary tools to be called remotely, spawning language servers dynamically for each requested project. This implements #1086 for LSP users. --- src/serena/agent.py | 17 ++- src/serena/cli.py | 55 +++++++++ src/serena/project_server.py | 157 ++++++++++++++++++++++++ src/serena/tools/query_project_tools.py | 36 ++++-- src/serena/tools/tools_base.py | 33 +---- 5 files changed, 261 insertions(+), 37 deletions(-) create mode 100644 src/serena/project_server.py diff --git a/src/serena/agent.py b/src/serena/agent.py index 36ac3fc9..5b779ed4 100644 --- a/src/serena/agent.py +++ b/src/serena/agent.py @@ -6,7 +6,8 @@ import os import platform import subprocess import sys -from collections.abc import Callable, Sequence +from collections.abc import Callable, Iterator, Sequence +from contextlib import contextmanager from logging import Logger from typing import TYPE_CHECKING, Optional, TypeVar @@ -853,3 +854,17 @@ class SerenaAgent: if ls_manager is None: return [] return ls_manager.get_active_languages() + + @contextmanager + def active_project_context(self, project: Project) -> Iterator[None]: + """ + Context manager for temporarily setting/overriding the active project + + :param project: the project to be active + """ + original_project = self._active_project + self._active_project = project + try: + yield + finally: + self._active_project = original_project diff --git a/src/serena/cli.py b/src/serena/cli.py index 0c77310b..9fdfc728 100644 --- a/src/serena/cli.py +++ b/src/serena/cli.py @@ -350,6 +350,61 @@ class TopLevelCommands(AutoRegisteringGroup): else: print(f"{prefix}\n{instr}\n{postfix}") + @staticmethod + @click.command( + "start-project-server", + help="Starts the Serena project server, which exposes project querying capabilities via HTTP.", + context_settings={"max_content_width": _MAX_CONTENT_WIDTH}, + ) + @click.option( + "--host", + type=str, + default="127.0.0.1", + show_default=True, + help="Listen address for the project server.", + ) + @click.option( + "--port", + type=int, + default=None, + help="Listen port for the project server (default: ProjectServer.PORT).", + ) + @click.option( + "--log-level", + type=click.Choice(["DEBUG", "INFO", "WARNING", "ERROR", "CRITICAL"]), + default=None, + help="Override log level in config.", + ) + def start_project_server( + host: str, + port: int | None, + log_level: Literal["DEBUG", "INFO", "WARNING", "ERROR", "CRITICAL"] | None, + ) -> None: + from serena.project_server import ProjectServer + + # initialize logging + Logger.root.setLevel(logging.INFO) + formatter = logging.Formatter(SERENA_LOG_FORMAT) + stderr_handler = logging.StreamHandler(stream=sys.stderr) + stderr_handler.formatter = formatter + Logger.root.addHandler(stderr_handler) + log_path = SerenaPaths().get_next_log_file_path("project-server") + file_handler = logging.FileHandler(log_path, mode="w") + file_handler.formatter = formatter + Logger.root.addHandler(file_handler) + + if log_level is not None: + Logger.root.setLevel(logging.getLevelNamesMapping()[log_level]) + + log.info("Starting Serena project server") + log.info("Storing logs in %s", log_path) + + server = ProjectServer() + run_kwargs: dict[str, Any] = {"host": host} + if port is not None: + run_kwargs["port"] = port + server.run(**run_kwargs) + class ModeCommands(AutoRegisteringGroup): """Group for 'mode' subcommands.""" diff --git a/src/serena/project_server.py b/src/serena/project_server.py new file mode 100644 index 00000000..ef88e883 --- /dev/null +++ b/src/serena/project_server.py @@ -0,0 +1,157 @@ +import json +import logging +from typing import TYPE_CHECKING + +import requests as requests_lib +from flask import Flask, request +from pydantic import BaseModel +from sensai.util.logging import LogTime + +from serena.config.serena_config import LanguageBackend, SerenaConfig +from serena.jetbrains.jetbrains_plugin_client import JetBrainsPluginClient + +if TYPE_CHECKING: + from serena.project import Project + +log = logging.getLogger(__name__) + +# disable Werkzeug's logging to avoid cluttering the output +logging.getLogger("werkzeug").setLevel(logging.WARNING) + + +class QueryProjectRequest(BaseModel): + """ + Request model for the /query_project endpoint, matching the interface of + :class:`~serena.tools.query_project_tools.QueryProjectTool`. + """ + + project_name: str + tool_name: str + tool_params_json: str + + +class ProjectServer: + """ + A lightweight Flask server that exposes a SerenaAgent's project querying + capabilities via HTTP, using the LSP language server backend for symbolic retrieval. + + Projects are loaded on demand when a query is made for them, and cached in memory for subsequent queries. + + The server instantiates a :class:`SerenaAgent` with default options and + provides a ``/query_project`` endpoint whose interface matches + :class:`~serena.tools.query_project_tools.QueryProjectTool`. + """ + + PORT = JetBrainsPluginClient.BASE_PORT - 1 + + def __init__(self) -> None: + from serena.agent import SerenaAgent + + serena_config = SerenaConfig.from_config_file() + serena_config.gui_log_window = False + serena_config.web_dashboard = False + serena_config.language_backend = LanguageBackend.LSP + + self._agent = SerenaAgent(serena_config=serena_config) + self._loaded_projects_by_name: dict[str, "Project"] = {} + + # create the Flask application + self._app = Flask(__name__) + self._setup_routes() + + def _setup_routes(self) -> None: + @self._app.route("/heartbeat", methods=["GET"]) + def heartbeat() -> dict[str, str]: + return {"status": "alive"} + + @self._app.route("/query_project", methods=["POST"]) + def query_project() -> str: + query_request = QueryProjectRequest.model_validate(request.get_json()) + return self._query_project(query_request) + + def _get_project(self, project_name: str) -> "Project": + """Gets the project with the given name, loading it if necessary.""" + if project_name in self._loaded_projects_by_name: + return self._loaded_projects_by_name[project_name] + else: + serena_config = self._agent.serena_config + registered_project = serena_config.get_registered_project(project_name) + if registered_project is None: + raise ValueError(f"Project '{project_name}' is not registered with Serena.") + with LogTime(f"Loading project '{project_name}'"): + project = registered_project.get_project_instance(serena_config) + project.create_language_server_manager() + self._loaded_projects_by_name[project_name] = project + return project + + def _query_project(self, req: QueryProjectRequest) -> str: + """Handle a /query_project request by invoking the agent on the specified project and tool.""" + project = self._get_project(req.project_name) + with self._agent.active_project_context(project): + tool = self._agent.get_tool_by_name(req.tool_name) + params = json.loads(req.tool_params_json) + return tool.apply_ex(**params) + + def run(self, host: str = "127.0.0.1", port: int = PORT) -> int: + """Run the server on the given host and port. + + :param host: the host address to listen on. + :param port: the port to listen on. + :return: the port number the server is running on. + """ + from flask import cli + + # suppress the default Flask startup banner + cli.show_server_banner = lambda *args, **kwargs: None + + self._app.run(host=host, port=port, debug=False, use_reloader=False, threaded=True) + return port + + +class ProjectServerClient: + """Client for interacting with a running :class:`ProjectServer`. + + Upon instantiation, the client verifies that the server is reachable + by sending a heartbeat request. If the server is not running, a + :class:`ConnectionError` is raised. + """ + + def __init__(self, host: str = "127.0.0.1", port: int = ProjectServer.PORT, timeout: int = 300) -> None: + """ + :param host: the host address of the project server. + :param port: the port of the project server. + :raises ConnectionError: if the project server is not reachable. + """ + self._base_url = f"http://{host}:{port}" + self._timeout = timeout + + # verify that the server is running + try: + response = requests_lib.get(f"{self._base_url}/heartbeat", timeout=5) + response.raise_for_status() + except requests_lib.ConnectionError: + raise ConnectionError(f"ProjectServer is not reachable at {self._base_url}. Make sure the server is running.") + except requests_lib.RequestException as e: + raise ConnectionError(f"ProjectServer health check failed: {e}") + + def query_project(self, project_name: str, tool_name: str, tool_params_json: str) -> str: + """ + Query a project by executing a Serena tool in its context. + + The interface matches :meth:`QueryProjectTool.apply + `. + + :param project_name: the name of the project to query. + :param tool_name: the name of the tool to execute. The tool must be read-only. + :param tool_params_json: the parameters to pass to the tool, encoded as a JSON string. + :return: the tool's result as a string. + """ + payload = QueryProjectRequest( + project_name=project_name, + tool_name=tool_name, + tool_params_json=tool_params_json, + ).model_dump() + + response = requests_lib.post(f"{self._base_url}/query_project", json=payload, timeout=self._timeout) + response.raise_for_status() + return response.text diff --git a/src/serena/tools/query_project_tools.py b/src/serena/tools/query_project_tools.py index c59cd420..b675a0aa 100644 --- a/src/serena/tools/query_project_tools.py +++ b/src/serena/tools/query_project_tools.py @@ -1,10 +1,12 @@ import json +from serena.config.serena_config import LanguageBackend from serena.jetbrains.jetbrains_plugin_client import JetBrainsPluginClientManager +from serena.project_server import ProjectServerClient from serena.tools import Tool, ToolMarkerDoesNotRequireActiveProject, ToolMarkerOptional -class ListQueryableProjectsTool(Tool, ToolMarkerDoesNotRequireActiveProject, ToolMarkerOptional): +class ListQueryableProjectsTool(Tool, ToolMarkerOptional, ToolMarkerDoesNotRequireActiveProject): """ Tool for listing all projects that can be queried by the QueryProjectTool. """ @@ -20,10 +22,12 @@ class ListQueryableProjectsTool(Tool, ToolMarkerDoesNotRequireActiveProject, Too if symbol_access: backend = self.agent.get_language_backend() if backend.is_jetbrains(): + # projects with open IDE instances can be queried matched_clients = JetBrainsPluginClientManager().match_clients(registered_projects) relevant_projects = [mc.registered_project for mc in matched_clients] else: - raise NotImplementedError(f"External symbol access is generally unavailable for language backend {backend.value}") + # all projects can be queried via ProjectServer (which instantiates projects dynamically) + relevant_projects = registered_projects else: relevant_projects = registered_projects @@ -35,7 +39,7 @@ class ListQueryableProjectsTool(Tool, ToolMarkerDoesNotRequireActiveProject, Too return self._to_json(project_names) -class QueryProjectTool(Tool, ToolMarkerDoesNotRequireActiveProject, ToolMarkerOptional): +class QueryProjectTool(Tool, ToolMarkerOptional, ToolMarkerDoesNotRequireActiveProject): """ Tool for querying external project information (i.e. information from projects other than the current one), by executing a read-only tool. @@ -53,8 +57,24 @@ class QueryProjectTool(Tool, ToolMarkerDoesNotRequireActiveProject, ToolMarkerOp tool = self.agent.get_tool_by_name(tool_name) assert tool.is_active(), f"Tool {tool_name} is not active." assert tool.is_readonly(), f"Tool {tool_name} is not read-only and cannot be executed in another project." - registered_project = self.agent.serena_config.get_registered_project(project_name) - assert registered_project is not None, f"Project {project_name} is not registered and cannot be queried." - project = registered_project.get_project_instance(self.agent.serena_config) - with tool.project_override_context(project): - return tool.apply(**json.loads(tool_params_json)) # type: ignore + if self._is_project_server_required(tool): + client = ProjectServerClient() + return client.query_project(project_name, tool_name, tool_params_json) + else: + registered_project = self.agent.serena_config.get_registered_project(project_name) + assert registered_project is not None, f"Project {project_name} is not registered and cannot be queried." + project = registered_project.get_project_instance(self.agent.serena_config) + with tool.agent.active_project_context(project): + return tool.apply(**json.loads(tool_params_json)) # type: ignore + + def _is_project_server_required(self, tool: Tool) -> bool: + match self.agent.get_language_backend(): + case LanguageBackend.JETBRAINS: + return False + case LanguageBackend.LSP: + # Note: As long as only read-only tools are considered, only symbolic tools require the project server. + # But if we were to allow non-read-only tools, then tools using a CodeEditor also indirectly require language servers. + assert tool.is_readonly() + return tool.is_symbolic() + case _: + raise NotImplementedError diff --git a/src/serena/tools/tools_base.py b/src/serena/tools/tools_base.py index 426529aa..556321d9 100644 --- a/src/serena/tools/tools_base.py +++ b/src/serena/tools/tools_base.py @@ -1,8 +1,7 @@ import inspect import json from abc import ABC -from collections.abc import Iterable, Iterator -from contextlib import contextmanager +from collections.abc import Iterable from dataclasses import dataclass from types import TracebackType from typing import TYPE_CHECKING, Any, Protocol, Self, TypeVar, cast @@ -33,27 +32,6 @@ SUCCESS_RESULT = "OK" class Component(ABC): def __init__(self, agent: "SerenaAgent"): self.agent = agent - self._project_override: Project | None = None - - @contextmanager - def project_override_context(self, project: Project) -> Iterator[None]: - """ - Context manager for temporarily overriding the active project in a component, - allowing tool executions to use the overridden project instead of the active project. - - :param project: the project to override with - """ - if self._project_override is not None: - raise RuntimeError("Nested project overrides are not supported.") - self._project_override = project - try: - yield - finally: - self._project_override = None - - def _assert_is_not_project_override(self) -> None: - if self._project_override is not None: - raise RuntimeError("This method is only supported for the active project.") def get_project_root(self) -> str: """ @@ -72,17 +50,13 @@ class Component(ABC): def create_language_server_symbol_retriever(self) -> "LanguageServerSymbolRetriever": from serena.symbol import LanguageServerSymbolRetriever - self._assert_is_not_project_override() assert self.agent.get_language_backend().is_lsp(), "Language server symbol retriever can only be created for LSP language backend" language_server_manager = self.agent.get_language_server_manager_or_raise() return LanguageServerSymbolRetriever(language_server_manager, agent=self.agent) @property def project(self) -> Project: - if self._project_override is not None: - return self._project_override - else: - return self.agent.get_active_project_or_raise() + return self.agent.get_active_project_or_raise() def create_code_editor(self) -> "CodeEditor": from ..code_editor import JetBrainsCodeEditor, LanguageServerCodeEditor @@ -266,6 +240,9 @@ class Tool(Component): def is_readonly(self) -> bool: return not self.can_edit() + def is_symbolic(self) -> bool: + return issubclass(self.__class__, ToolMarkerSymbolicRead) or issubclass(self.__class__, ToolMarkerSymbolicEdit) + def apply_ex(self, log_call: bool = True, catch_exceptions: bool = True, mcp_ctx: Context | None = None, **kwargs) -> str: # type: ignore """ Applies the tool with logging and exception handling, using the given keyword arguments