Introduce ProjectServer to support QueryProjectTool for the LSP backend

The server allows arbitrary tools to be called remotely,
spawning language servers dynamically for each requested project.

This implements #1086 for LSP users.

src/serena/agent.py

@@ -6,7 +6,8 @@ import os
 import platform
 import subprocess
 import sys
-from collections.abc import Callable, Sequence
+from collections.abc import Callable, Iterator, Sequence
+from contextlib import contextmanager
 from logging import Logger
 from typing import TYPE_CHECKING, Optional, TypeVar
 
@@ -853,3 +854,17 @@ class SerenaAgent:
         if ls_manager is None:
             return []
         return ls_manager.get_active_languages()
+
+    @contextmanager
+    def active_project_context(self, project: Project) -> Iterator[None]:
+        """
+        Context manager for temporarily setting/overriding the active project
+
+        :param project: the project to be active
+        """
+        original_project = self._active_project
+        self._active_project = project
+        try:
+            yield
+        finally:
+            self._active_project = original_project

src/serena/cli.py

@@ -350,6 +350,61 @@ class TopLevelCommands(AutoRegisteringGroup):
         else:
             print(f"{prefix}\n{instr}\n{postfix}")
 
+    @staticmethod
+    @click.command(
+        "start-project-server",
+        help="Starts the Serena project server, which exposes project querying capabilities via HTTP.",
+        context_settings={"max_content_width": _MAX_CONTENT_WIDTH},
+    )
+    @click.option(
+        "--host",
+        type=str,
+        default="127.0.0.1",
+        show_default=True,
+        help="Listen address for the project server.",
+    )
+    @click.option(
+        "--port",
+        type=int,
+        default=None,
+        help="Listen port for the project server (default: ProjectServer.PORT).",
+    )
+    @click.option(
+        "--log-level",
+        type=click.Choice(["DEBUG", "INFO", "WARNING", "ERROR", "CRITICAL"]),
+        default=None,
+        help="Override log level in config.",
+    )
+    def start_project_server(
+        host: str,
+        port: int | None,
+        log_level: Literal["DEBUG", "INFO", "WARNING", "ERROR", "CRITICAL"] | None,
+    ) -> None:
+        from serena.project_server import ProjectServer
+
+        # initialize logging
+        Logger.root.setLevel(logging.INFO)
+        formatter = logging.Formatter(SERENA_LOG_FORMAT)
+        stderr_handler = logging.StreamHandler(stream=sys.stderr)
+        stderr_handler.formatter = formatter
+        Logger.root.addHandler(stderr_handler)
+        log_path = SerenaPaths().get_next_log_file_path("project-server")
+        file_handler = logging.FileHandler(log_path, mode="w")
+        file_handler.formatter = formatter
+        Logger.root.addHandler(file_handler)
+
+        if log_level is not None:
+            Logger.root.setLevel(logging.getLevelNamesMapping()[log_level])
+
+        log.info("Starting Serena project server")
+        log.info("Storing logs in %s", log_path)
+
+        server = ProjectServer()
+        run_kwargs: dict[str, Any] = {"host": host}
+        if port is not None:
+            run_kwargs["port"] = port
+        server.run(**run_kwargs)
+
 
 class ModeCommands(AutoRegisteringGroup):
     """Group for 'mode' subcommands."""

src/serena/project_server.py

@@ -0,0 +1,157 @@
+import json
+import logging
+from typing import TYPE_CHECKING
+
+import requests as requests_lib
+from flask import Flask, request
+from pydantic import BaseModel
+from sensai.util.logging import LogTime
+
+from serena.config.serena_config import LanguageBackend, SerenaConfig
+from serena.jetbrains.jetbrains_plugin_client import JetBrainsPluginClient
+
+if TYPE_CHECKING:
+    from serena.project import Project
+
+log = logging.getLogger(__name__)
+
+# disable Werkzeug's logging to avoid cluttering the output
+logging.getLogger("werkzeug").setLevel(logging.WARNING)
+
+
+class QueryProjectRequest(BaseModel):
+    """
+    Request model for the /query_project endpoint, matching the interface of
+    :class:`~serena.tools.query_project_tools.QueryProjectTool`.
+    """
+
+    project_name: str
+    tool_name: str
+    tool_params_json: str
+
+
+class ProjectServer:
+    """
+    A lightweight Flask server that exposes a SerenaAgent's project querying
+    capabilities via HTTP, using the LSP language server backend for symbolic retrieval.
+
+    Projects are loaded on demand when a query is made for them, and cached in memory for subsequent queries.
+
+    The server instantiates a :class:`SerenaAgent` with default options and
+    provides a ``/query_project`` endpoint whose interface matches
+    :class:`~serena.tools.query_project_tools.QueryProjectTool`.
+    """
+
+    PORT = JetBrainsPluginClient.BASE_PORT - 1
+
+    def __init__(self) -> None:
+        from serena.agent import SerenaAgent
+
+        serena_config = SerenaConfig.from_config_file()
+        serena_config.gui_log_window = False
+        serena_config.web_dashboard = False
+        serena_config.language_backend = LanguageBackend.LSP
+
+        self._agent = SerenaAgent(serena_config=serena_config)
+        self._loaded_projects_by_name: dict[str, "Project"] = {}
+
+        # create the Flask application
+        self._app = Flask(__name__)
+        self._setup_routes()
+
+    def _setup_routes(self) -> None:
+        @self._app.route("/heartbeat", methods=["GET"])
+        def heartbeat() -> dict[str, str]:
+            return {"status": "alive"}
+
+        @self._app.route("/query_project", methods=["POST"])
+        def query_project() -> str:
+            query_request = QueryProjectRequest.model_validate(request.get_json())
+            return self._query_project(query_request)
+
+    def _get_project(self, project_name: str) -> "Project":
+        """Gets the project with the given name, loading it if necessary."""
+        if project_name in self._loaded_projects_by_name:
+            return self._loaded_projects_by_name[project_name]
+        else:
+            serena_config = self._agent.serena_config
+            registered_project = serena_config.get_registered_project(project_name)
+            if registered_project is None:
+                raise ValueError(f"Project '{project_name}' is not registered with Serena.")
+            with LogTime(f"Loading project '{project_name}'"):
+                project = registered_project.get_project_instance(serena_config)
+                project.create_language_server_manager()
+            self._loaded_projects_by_name[project_name] = project
+            return project
+
+    def _query_project(self, req: QueryProjectRequest) -> str:
+        """Handle a /query_project request by invoking the agent on the specified project and tool."""
+        project = self._get_project(req.project_name)
+        with self._agent.active_project_context(project):
+            tool = self._agent.get_tool_by_name(req.tool_name)
+            params = json.loads(req.tool_params_json)
+            return tool.apply_ex(**params)
+
+    def run(self, host: str = "127.0.0.1", port: int = PORT) -> int:
+        """Run the server on the given host and port.
+
+        :param host: the host address to listen on.
+        :param port: the port to listen on.
+        :return: the port number the server is running on.
+        """
+        from flask import cli
+
+        # suppress the default Flask startup banner
+        cli.show_server_banner = lambda *args, **kwargs: None
+
+        self._app.run(host=host, port=port, debug=False, use_reloader=False, threaded=True)
+        return port
+
+
+class ProjectServerClient:
+    """Client for interacting with a running :class:`ProjectServer`.
+
+    Upon instantiation, the client verifies that the server is reachable
+    by sending a heartbeat request. If the server is not running, a
+    :class:`ConnectionError` is raised.
+    """
+
+    def __init__(self, host: str = "127.0.0.1", port: int = ProjectServer.PORT, timeout: int = 300) -> None:
+        """
+        :param host: the host address of the project server.
+        :param port: the port of the project server.
+        :raises ConnectionError: if the project server is not reachable.
+        """
+        self._base_url = f"http://{host}:{port}"
+        self._timeout = timeout
+
+        # verify that the server is running
+        try:
+            response = requests_lib.get(f"{self._base_url}/heartbeat", timeout=5)
+            response.raise_for_status()
+        except requests_lib.ConnectionError:
+            raise ConnectionError(f"ProjectServer is not reachable at {self._base_url}. Make sure the server is running.")
+        except requests_lib.RequestException as e:
+            raise ConnectionError(f"ProjectServer health check failed: {e}")
+
+    def query_project(self, project_name: str, tool_name: str, tool_params_json: str) -> str:
+        """
+        Query a project by executing a Serena tool in its context.
+
+        The interface matches :meth:`QueryProjectTool.apply
+        <serena.tools.query_project_tools.QueryProjectTool.apply>`.
+
+        :param project_name: the name of the project to query.
+        :param tool_name: the name of the tool to execute. The tool must be read-only.
+        :param tool_params_json: the parameters to pass to the tool, encoded as a JSON string.
+        :return: the tool's result as a string.
+        """
+        payload = QueryProjectRequest(
+            project_name=project_name,
+            tool_name=tool_name,
+            tool_params_json=tool_params_json,
+        ).model_dump()
+
+        response = requests_lib.post(f"{self._base_url}/query_project", json=payload, timeout=self._timeout)
+        response.raise_for_status()
+        return response.text

src/serena/tools/query_project_tools.py

@@ -1,10 +1,12 @@
 import json
 
+from serena.config.serena_config import LanguageBackend
 from serena.jetbrains.jetbrains_plugin_client import JetBrainsPluginClientManager
+from serena.project_server import ProjectServerClient
 from serena.tools import Tool, ToolMarkerDoesNotRequireActiveProject, ToolMarkerOptional
 
 
-class ListQueryableProjectsTool(Tool, ToolMarkerDoesNotRequireActiveProject, ToolMarkerOptional):
+class ListQueryableProjectsTool(Tool, ToolMarkerOptional, ToolMarkerDoesNotRequireActiveProject):
     """
     Tool for listing all projects that can be queried by the QueryProjectTool.
     """
@@ -20,10 +22,12 @@ class ListQueryableProjectsTool(Tool, ToolMarkerDoesNotRequireActiveProject, Too
         if symbol_access:
             backend = self.agent.get_language_backend()
             if backend.is_jetbrains():
+                # projects with open IDE instances can be queried
                 matched_clients = JetBrainsPluginClientManager().match_clients(registered_projects)
                 relevant_projects = [mc.registered_project for mc in matched_clients]
             else:
-                raise NotImplementedError(f"External symbol access is generally unavailable for language backend {backend.value}")
+                # all projects can be queried via ProjectServer (which instantiates projects dynamically)
+                relevant_projects = registered_projects
         else:
             relevant_projects = registered_projects
 
@@ -35,7 +39,7 @@ class ListQueryableProjectsTool(Tool, ToolMarkerDoesNotRequireActiveProject, Too
         return self._to_json(project_names)
 
 
-class QueryProjectTool(Tool, ToolMarkerDoesNotRequireActiveProject, ToolMarkerOptional):
+class QueryProjectTool(Tool, ToolMarkerOptional, ToolMarkerDoesNotRequireActiveProject):
     """
     Tool for querying external project information (i.e. information from projects other than the current one),
     by executing a read-only tool.
@@ -53,8 +57,24 @@ class QueryProjectTool(Tool, ToolMarkerDoesNotRequireActiveProject, ToolMarkerOp
         tool = self.agent.get_tool_by_name(tool_name)
         assert tool.is_active(), f"Tool {tool_name} is not active."
         assert tool.is_readonly(), f"Tool {tool_name} is not read-only and cannot be executed in another project."
-        registered_project = self.agent.serena_config.get_registered_project(project_name)
-        assert registered_project is not None, f"Project {project_name} is not registered and cannot be queried."
-        project = registered_project.get_project_instance(self.agent.serena_config)
-        with tool.project_override_context(project):
-            return tool.apply(**json.loads(tool_params_json))  # type: ignore
+        if self._is_project_server_required(tool):
+            client = ProjectServerClient()
+            return client.query_project(project_name, tool_name, tool_params_json)
+        else:
+            registered_project = self.agent.serena_config.get_registered_project(project_name)
+            assert registered_project is not None, f"Project {project_name} is not registered and cannot be queried."
+            project = registered_project.get_project_instance(self.agent.serena_config)
+            with tool.agent.active_project_context(project):
+                return tool.apply(**json.loads(tool_params_json))  # type: ignore
+
+    def _is_project_server_required(self, tool: Tool) -> bool:
+        match self.agent.get_language_backend():
+            case LanguageBackend.JETBRAINS:
+                return False
+            case LanguageBackend.LSP:
+                # Note: As long as only read-only tools are considered, only symbolic tools require the project server.
+                #   But if we were to allow non-read-only tools, then tools using a CodeEditor also indirectly require language servers.
+                assert tool.is_readonly()
+                return tool.is_symbolic()
+            case _:
+                raise NotImplementedError

src/serena/tools/tools_base.py

@@ -1,8 +1,7 @@
 import inspect
 import json
 from abc import ABC
-from collections.abc import Iterable, Iterator
-from contextlib import contextmanager
+from collections.abc import Iterable
 from dataclasses import dataclass
 from types import TracebackType
 from typing import TYPE_CHECKING, Any, Protocol, Self, TypeVar, cast
@@ -33,27 +32,6 @@ SUCCESS_RESULT = "OK"
 class Component(ABC):
     def __init__(self, agent: "SerenaAgent"):
         self.agent = agent
-        self._project_override: Project | None = None
-
-    @contextmanager
-    def project_override_context(self, project: Project) -> Iterator[None]:
-        """
-        Context manager for temporarily overriding the active project in a component,
-        allowing tool executions to use the overridden project instead of the active project.
-
-        :param project: the project to override with
-        """
-        if self._project_override is not None:
-            raise RuntimeError("Nested project overrides are not supported.")
-        self._project_override = project
-        try:
-            yield
-        finally:
-            self._project_override = None
-
-    def _assert_is_not_project_override(self) -> None:
-        if self._project_override is not None:
-            raise RuntimeError("This method is only supported for the active project.")
 
     def get_project_root(self) -> str:
         """
@@ -72,17 +50,13 @@ class Component(ABC):
     def create_language_server_symbol_retriever(self) -> "LanguageServerSymbolRetriever":
         from serena.symbol import LanguageServerSymbolRetriever
 
-        self._assert_is_not_project_override()
         assert self.agent.get_language_backend().is_lsp(), "Language server symbol retriever can only be created for LSP language backend"
         language_server_manager = self.agent.get_language_server_manager_or_raise()
         return LanguageServerSymbolRetriever(language_server_manager, agent=self.agent)
 
     @property
     def project(self) -> Project:
-        if self._project_override is not None:
-            return self._project_override
-        else:
-            return self.agent.get_active_project_or_raise()
+        return self.agent.get_active_project_or_raise()
 
     def create_code_editor(self) -> "CodeEditor":
         from ..code_editor import JetBrainsCodeEditor, LanguageServerCodeEditor
@@ -266,6 +240,9 @@ class Tool(Component):
     def is_readonly(self) -> bool:
         return not self.can_edit()
 
+    def is_symbolic(self) -> bool:
+        return issubclass(self.__class__, ToolMarkerSymbolicRead) or issubclass(self.__class__, ToolMarkerSymbolicEdit)
+
     def apply_ex(self, log_call: bool = True, catch_exceptions: bool = True, mcp_ctx: Context | None = None, **kwargs) -> str:  # type: ignore
         """
         Applies the tool with logging and exception handling, using the given keyword arguments