Add documentation on the REPL interface

* Tools section: Add REPL interface description, advantages and full list of functions
* Config section: Add information on REPL configuration

docs/02-usage/050_configuration.md

@@ -26,7 +26,7 @@ Some of the configurable settings include:
   * the language backend to use by default (i.e., the JetBrains plugin or language servers);
     this can also be [overridden per project](per-project-language-backend)
   * UI settings affecting the [Serena Dashboard and GUI tool](060_dashboard.md)
-  * the set of tools to enable/disable by default
+  * the set of tools or REPL API functions to enable/disable by default
   * the set of [modes](modes) to use by default
   * tool execution parameters (timeout, max. answer length)
   * global ignore rules
@@ -55,6 +55,37 @@ You can access it
     ```shell
     serena config edit
     ```
+    
+(agent-interfaces)=
+### Agent Interfaces
+
+Serena provides its functionality to the agent (LLM) through one of two interfaces
+(see [Tools and APIs](../01-about/035_tools) for the operations they offer):
+
+* **tools**: every operation is a separate tool of the MCP server.
+* **REPL** (new in Serena v2): a single tool executes Python code, through which the agent accesses the operations
+  programmatically, being able to combine several of them in one call.
+
+The interface is selected via the `agent_interface` setting in the global configuration.
+It can be overridden in the project configuration or via the `--agent-interface` command-line option,
+and it is fixed for the duration of a session.
+
+The two interfaces are configured differently:
+
+* With the **tool interface**, the set of tools results from the tool inclusion/exclusion settings
+  (`excluded_tools`, `included_optional_tools`, `fixed_tools`) of the global configuration, the context,
+  the modes and the project configuration.
+* With the **REPL interface**, the set of tools is fixed (the REPL tool and the tools which have no
+  counterpart within the REPL, e.g. for project activation); the tool settings above consequently do not apply.
+  The operations available *within* the REPL are configured via `included_apis`/`excluded_apis` instead,
+  which are supported in the same configuration layers and reference either a group of operations
+  (e.g. `lsp`) or an individual operation (e.g. `lsp.find_symbol`).
+
+```{note}
+Restricting the operations available in the REPL is a means of steering the agent, not a security mechanism:
+the Python code that is executed can, in principle, do anything the Serena process can do.
+See [Security](070_security) for isolation options.
+```
 
 ## Modes and Contexts
 

docs/autogen_docs.py

@@ -150,30 +150,110 @@ def autogen_tool_list(target_filename = "01-about/035_tools.md"):
     from serena.tools import ToolRegistry
 
     target_file = Path(__file__).parent / target_filename
-    with open(target_file, "w") as f:
+    with open(target_file, "w", encoding="utf-8") as f:
         f.write("<!-- This file is auto-generated by docs/autogen_docs.py. Do not edit it manually. -->\n\n")
-        f.write("# Tools\n\n")
-        f.write("Find the full list of Serena's tools below.\n\n")
-        f.write("Note that in most configurations, only a subset of these tools will be enabled simultaneously.\n")
-        f.write("Tools marked as *optional* are disabled by default.\n\n")
-        f.write("Tools marked as [BETA] were recently introduced and may not be fully robust yet.\n\n")
-        tools_by_module = ToolRegistry().get_registered_tools_by_module()
-        priority_modules = {"serena.tools.symbol_tools": 1, "serena.tools.jetbrains_tools": 2}
+        f.write("# Tools and APIs\n\n")
+        f.write(
+            "Serena provides an agent (LLM) with its functionality through one of two interfaces (configured in Serena's [global configuration](global-config)):\n\n"
+            "* **Tools** (the classic interface): every operation is a separate tool of the MCP server.\n"
+            "* **REPL** (new in Serena v2): a single tool executes Python code, through which the agent accesses the operations\n"
+            "  programmatically. The agent can thus combine several operations in one call, process the results\n"
+            "  in code and return only the information it actually needs.\n\n"
+            "While both interfaces offer the same general functionality for the most part, "
+            "the REPL interface addresses several limitations inherent in the tool-based approach (see [advantages](repl-advantages) below).\n\n"
+        )
 
-        text = TextBuilder()
-        sorted_modules = sorted(tools_by_module.keys(), key=lambda m: (priority_modules.get(m, 3), m))
-        for module in sorted_modules:
-            tools = tools_by_module[module]
-            module = module.replace("serena.tools.", "")
-            text.with_line(f"* **{module}**")
-            for tool in tools:
-                info = ""
-                if tool.is_optional:
-                    info += " *(optional)*"
-                if tool.is_beta:
-                    info += " [BETA]"
-                text.with_line(f"* `{tool.tool_name}`{info}: {tool.class_docstring}", indent=2)
-        f.write(text.build())
+        def tools_section():
+            f.write("## Tools (Classic Interface)\n\n")
+            f.write("Find the full list of Serena's tools below.\n\n")
+            f.write("Note that in most configurations, only a subset of these tools will be enabled simultaneously.\n")
+            f.write("Tools marked as *optional* are disabled by default.\n\n")
+            f.write("Tools marked as [BETA] were recently introduced and may not be fully robust yet.\n\n")
+            tools_by_module = ToolRegistry().get_registered_tools_by_module()
+            priority_modules = {"serena.tools.symbol_tools": 1, "serena.tools.jetbrains_tools": 2}
+
+            text = TextBuilder()
+            sorted_modules = sorted(tools_by_module.keys(), key=lambda m: (priority_modules.get(m, 3), m))
+            for module in sorted_modules:
+                tools = tools_by_module[module]
+                module = module.replace("serena.tools.", "")
+                text.with_line(f"* **{module}**")
+                for tool in tools:
+                    info = ""
+                    if tool.is_optional:
+                        info += " *(optional)*"
+                    if tool.is_beta:
+                        info += " [BETA]"
+                    text.with_line(f"* `{tool.tool_name}`{info}: {tool.class_docstring}", indent=2)
+            f.write(text.build())
+
+        def facades_section():
+            from serena.repl.facade import ApiScope
+            from serena.agent import SerenaAgent, SerenaConfig
+            from serena.config.serena_config import LanguageBackend
+
+            f.write("\n\n## Serena's REPL (Code Execution-Based Interface)\n\n")
+            f.write(
+                "With the REPL interface, the agent uses a single tool, which executes Python code. The code accesses\n"
+                "Serena's functionality through the entrypoint object `s`, whose attributes are *facades*, each of which\n"
+                "groups the operations of one domain. For instance, this code finds a class and returns the names of its\n"
+                "members in a single call:\n\n"
+                "```python\n"
+                'result = s.lsp.find_symbol("SerenaAgent", depth=1)\n'
+                "[member.name for member in result.symbols[0].iter_children()]\n"
+                "```\n\n"
+                "The agent retrieves the details it requires (signatures, documentation, result types) at runtime via\n"
+                "`s.info(...)` (principle of *progressive disclosure*).\n\n"
+                "As with tools, only a subset of the operations is available in a given configuration:\n\n"
+                "* The facades `lsp` and `jb` are mutually exclusive, being tied to the respective language backend.\n"
+                "* Operations marked as *optional*, as well as all operations of facades marked as *optional*,\n"
+                "  are disabled by default and must be enabled explicitly.\n\n"
+                "(repl-advantages)=\n\n"
+                "### Advantages\n\n"
+                "* **Composition**: The agent can combine several operations in a single call, using control flow,\n"
+                "  filtering and aggregation. Multi-step retrievals that would otherwise require a series of\n"
+                "  round-trips (find a symbol, inspect its members, find their references) become one call.\n"
+                "* **Context economy**: Only the result the agent actually needs enters the conversation.\n"
+                "  Intermediate results remain in the Python runtime instead of consuming the context window\n"
+                "  — which, for large result sets, is the difference between a summary and thousands of tokens.\n"
+                "* **Results as objects**: Operations return objects which can be processed programmatically\n"
+                "  rather than plain text.\n"
+                "* **Progressive disclosure**: Instead of the schemas of dozens of tools, the agent is given a list of\n"
+                "  the available operations, retrieving the details it requires (signatures, documentation, result\n"
+                "  types) on demand. This eliminates the need for a client-specific tool search/dynamic tool discovery mechanism.\n"
+                "* **Reusability within a session**: Variables and helper functions defined by the agent persist\n"
+                "  across calls, allowing intermediate results to be revisited and recurring logic to be applied\n"
+                "  repeatedly.\n"
+                "* **Dynamic adaptation**: The set of available operations can change at runtime.\n"
+                "  For MCP tools, changes to the toolset are not widely supported by clients and are therefore not\n"
+                "  applied; within the REPL, the operations can follow the current situation.\n"
+                "  This is what allows the language backend to be switched during a session, e.g. when a project\n"
+                "  whose configuration demands a different backend is activated..\n\n"
+                "### List of Facades\n\n"
+            )
+            api_scope = ApiScope()
+            agent = SerenaAgent(serena_config=SerenaConfig().with_headless_mode_overrides())
+            facades = []
+            for backend in LanguageBackend:
+                facades.extend(backend.create_facades(agent, api_scope))
+            facades.extend(agent.create_default_facade_list(api_scope))
+
+            text = TextBuilder()
+            for facade in facades:
+                facade_info = " *(optional)*" if facade.is_optional() else ""
+                text.with_line(f"* **{facade.name}**{facade_info}: {facade.description}")
+                for method in facade.get_methods():
+                    method_info = ""
+                    if method.info.optional:
+                        method_info += " *(optional)*"
+                    if method.info.beta:
+                        method_info += " [BETA]"
+                    summary = method.get_summary().replace("`", "")
+                    text.with_line(f"* `{method.qualified_name}`{method_info}: {summary}", indent=2)
+            f.write(text.build())
+
+        tools_section()
+        facades_section()
 
 
 def autogen_about_intro_features():

src/serena/agent.py

@@ -53,8 +53,6 @@ from serena.repl.api.cfg_api import ConfigApi
 from serena.repl.api.edit_api import EditApi
 from serena.repl.api.ext_api import ExternalProjectsApi
 from serena.repl.api.fs_api import FsApi
-from serena.repl.api.jb_api import JetBrainsApi
-from serena.repl.api.lsp_api import LspApi
 from serena.repl.api.mem_api import MemoryApi
 from serena.repl.api.shell_api import ShellApi
 from serena.repl.facade import ApiScope, Facade
@@ -1319,6 +1317,19 @@ class SerenaAgent:
                 "Consider adjusting your configuration to include these tools if you want to use them."
             )
 
+    def create_default_facade_list(self, api_scope: ApiScope) -> list[Facade]:
+        """
+        :return: the default list of facades provided by Serena itself, not including any language backend-specific facades
+        """
+        return [
+            Facade.from_api(ConfigApi(self), api_scope),
+            Facade.from_api(FsApi(self), api_scope),
+            Facade.from_api(EditApi(self), api_scope),
+            Facade.from_api(MemoryApi(self), api_scope),
+            Facade.from_api(ShellApi(self), api_scope),
+            Facade.from_api(ExternalProjectsApi(self), api_scope, is_optional=True),
+        ]
+
     def get_repl(self) -> SerenaRepl:
         """
         :return: the REPL instance for this agent, creating it if necessary
@@ -1337,20 +1348,7 @@ class SerenaAgent:
             if not self._is_dashboard_openable(self.serena_config):
                 api_scope.process(NamedApiInclusionDefinition(name="Dashboard", excluded_apis=["cfg.open_dashboard"]))
 
-            # gather facades
-            facades = [
-                Facade.from_api(ConfigApi(self), api_scope),
-                Facade.from_api(FsApi(self), api_scope),
-                Facade.from_api(EditApi(self), api_scope),
-                Facade.from_api(MemoryApi(self), api_scope),
-                Facade.from_api(ShellApi(self), api_scope),
-                Facade.from_api(ExternalProjectsApi(self), api_scope, is_optional=True),
-            ]
-            if self._language_backend.is_lsp():
-                facades.append(Facade.from_api(LspApi(self), api_scope))
-            elif self._language_backend.is_jetbrains():
-                facades.append(Facade.from_api(JetBrainsApi(self), api_scope))
-
+            facades = self.create_default_facade_list(api_scope) + self._language_backend.create_facades(self, api_scope)
             self._repl = SerenaRepl(facades, api_scope)
         return self._repl
 

src/serena/config/serena_config.py

@@ -44,7 +44,9 @@ from ..util.cli_util import ask_yes_no
 from ..util.dataclass import get_dataclass_default
 
 if TYPE_CHECKING:
+    from ..agent import SerenaAgent
     from ..project import Project
+    from ..repl.facade import ApiScope, Facade
     from ..tools.tools_base import Tool
 
 log = logging.getLogger(__name__)
@@ -294,6 +296,19 @@ class LanguageBackend(Enum):
             case _:
                 raise NotImplementedError()
 
+    def create_facades(self, agent: "SerenaAgent", api_scope: "ApiScope") -> list["Facade"]:
+        from ..repl.facade import Facade
+
+        if self.is_lsp():
+            from ..repl.api.lsp_api import LspApi
+
+            return [Facade.from_api(LspApi(agent), api_scope)]
+        elif self.is_jetbrains():
+            from ..repl.api.jb_api import JetBrainsApi
+
+            return [Facade.from_api(JetBrainsApi(agent), api_scope)]
+        return []
+
 
 class LineEnding(Enum):
     """Line ending convention for file writes."""

src/serena/repl/facade.py

@@ -570,8 +570,9 @@ class Facade:
     A named group of related operations which an LLM can invoke from REPL code.
     """
 
-    def __init__(self, name: str, description: str, types: Sequence[ReferencedType] = ()) -> None:
+    def __init__(self, name: str, description: str, is_optional: bool = False, types: Sequence[ReferencedType] = ()) -> None:
         # NOTE: attributes are set via object.__setattr__ because __getattr__ is overridden
+        object.__setattr__(self, "_is_optional", is_optional)
         object.__setattr__(self, "_name", name)
         object.__setattr__(self, "_description", description)
         object.__setattr__(self, "_methods", {})
@@ -601,7 +602,7 @@ class Facade:
         :param is_optional: whether the facade is optional (disabled by default and must be enabled explicitly)
         :return: the facade
         """
-        facade = Facade(api.get_name_(), api.get_description_(), api.get_referenced_types_())
+        facade = Facade(api.get_name_(), api.get_description_(), is_optional, api.get_referenced_types_())
         for name, member in inspect.getmembers(api, predicate=inspect.ismethod):
             method_info = get_facade_method_info(member)
             if method_info is None:
@@ -617,6 +618,12 @@ class Facade:
         """
         return len(self.get_enabled_methods()) > 0
 
+    def is_optional(self) -> bool:
+        """
+        :return: whether the facade is optional, i.e. disabled unless it is explicitly included
+        """
+        return self._is_optional
+
     def _discover_referenced_types(self) -> None:
         """
         Adds referenced types for all classes reachable (transitively) through the annotations of the facade's methods