From 6da423c70a67adf81161feeab212b8b9e1a1438a Mon Sep 17 00:00:00 2001 From: Dominik Jain Date: Tue, 15 Sep 2026 10:26:48 +0200 Subject: [PATCH 01/53] Add `dev` command to bump_version and make its interface explicit bump_version could previously only create releases. It now provides two subcommands, and the mutually exclusive --major/--minor/--patch flags (as well as the explicit --version option) are replaced by a positional argument, which cannot be misused: bump_version.py release bump_version.py dev `dev` bumps the version to a new .dev0 version and commits it as "Set version to vX" without creating a tag or touching the changelog. This allows work on main to target a new minor/major version independently of a release. Because a .dev0 version no longer necessarily reserves the next patch version, `release` now takes the target "current", which releases the version reserved by the current .dev version (the usual case), whereas major/minor/patch bump beyond it. The former special case of not incrementing the patch version is thereby removed; `release patch` now increments the patch version as its name suggests. `release current` fails if the current version is not a development version, in which case there is no reserved version to release. --- README-dev.md | 33 ++++++-- scripts/bump_version.py | 180 ++++++++++++++++++++++------------------ 2 files changed, 124 insertions(+), 89 deletions(-) diff --git a/README-dev.md b/README-dev.md index 6207719a..d4c23de1 100644 --- a/README-dev.md +++ b/README-dev.md @@ -8,12 +8,18 @@ and tools for formatting and type checking. ## Release Process 1. Ensure clean git status. -2. Set the version for release, e.g. - - python scripts/bump_version.py --patch - python scripts/bump_version.py --minor +2. Set the version for release. Normally, the version to be released is the one already reserved by the + current `.dev0` version (e.g. `1.8.0` when the repository is at `1.8.0.dev0`): - This also creates the git tag. + python scripts/bump_version.py release current + + To release a version beyond the reserved one, name the part to bump instead, e.g. + + python scripts/bump_version.py release patch + python scripts/bump_version.py release minor + + This updates `CHANGELOG.md`, commits the release version, creates the git tag, and then + commits the subsequent `.dev0` version for the next iteration. 3. Push to GitHub: git push @@ -26,4 +32,19 @@ and tools for formatting and type checking. [GitHub Releases page](https://github.com/oraios/serena/releases). When ready, publish it (click *Publish release*). This triggers the `publish` workflow, which builds and publishes the - package to PyPI. \ No newline at end of file + package to PyPI. + +### Bumping the Development Version + +Independently of a release, the development version can be bumped, e.g. when work on `main` +begins to target a new minor or major version: + + python scripts/bump_version.py dev minor + python scripts/bump_version.py dev major + +This sets the version to the respective new `.dev0` version (e.g. `1.8.0.dev0`) and commits it as +"Set version to vX"; it creates no tag and does not modify `CHANGELOG.md`. + +The subsequent release of that version is then performed with `release current`. + +Both commands require a clean git status and support `--dry-run` to preview the changes. \ No newline at end of file diff --git a/scripts/bump_version.py b/scripts/bump_version.py index 08af6b33..ac58973f 100644 --- a/scripts/bump_version.py +++ b/scripts/bump_version.py @@ -17,98 +17,110 @@ from serena.util.git import get_git_status log = logging.getLogger(__name__) VersionPart = Literal["major", "minor", "patch"] +#: a version part to bump or, in the case of "current", the version already reserved by the current .dev version +VersionTarget = Literal["major", "minor", "patch", "current"] _VERSION_PATTERN = re.compile(r"^(?P\d+)\.(?P\d+)\.(?P\d+)(\.\w+)?$") _INIT_VERSION_PATTERN = re.compile(r'^(?P__version__\s*=\s*")(?P\d+\.\d+\.\d+(?:\.\w+)?)(?P"\s*)$', re.MULTILINE) _PYPROJECT_VERSION_PATTERN = re.compile( r'(?m)^(?P\[project\]\n(?:.*\n)*?^version\s*=\s*")(?P\d+\.\d+\.\d+(?:\.\w+)?)(?P"\s*)$' ) +_VERSION_SUFFIX_PATTERN = re.compile(r"^\d+\.\d+\.\d+\.(?P\w+)$") _UNRELEASED_HEADER = "# Unreleased (main)\n" -@click.command() -@click.option("--major", "major", is_flag=True, help="Bump the major version and reset minor and patch to 0.") -@click.option("--minor", "minor", is_flag=True, help="Bump the minor version and reset patch to 0.") -@click.option("--patch", "patch", is_flag=True, help="Bump the patch version.") -@click.option("--version", "-v", "target_version", metavar="X.Y.Z", help="Set an explicit version instead of bumping.") -@click.option("--dry-run", is_flag=True, help="Show what would change without writing any files.") -def bump_version(major: bool, minor: bool, patch: bool, target_version: str | None, dry_run: bool) -> None: - git_status = get_git_status() - if not git_status.is_clean: - raise click.ClickException("Working directory is not clean. Please commit or stash your changes first.") +_version_target_argument = click.argument("version_target", type=click.Choice(["current", "major", "minor", "patch"])) +_version_part_argument = click.argument("version_part", type=click.Choice(["major", "minor", "patch"])) +_dry_run_option = click.option("--dry-run", is_flag=True, help="Show what would change without writing any files.") - log.info("bump_version called: major=%s, minor=%s, patch=%s, target_version=%s", major, minor, patch, target_version) - # determine part to bump - version_part = resolve_version_selection(major=major, minor=minor, patch=patch, target_version=target_version) - log.info("Resolved version_part=%s", version_part) +@click.group() +def cli() -> None: + """Manages the Serena version.""" + + +@cli.command() +@_version_target_argument +@_dry_run_option +def release(version_target: VersionTarget, dry_run: bool) -> None: + """Bumps the version for a release and starts the next dev iteration. + + Bumps the version, updates the changelog, commits and tags the release, and then commits + the subsequent .dev0 version. + + VERSION_TARGET is either "current", releasing the version already reserved by the current .dev version + (the usual case), or the part of the version to bump beyond it (major, minor or patch). + """ + require_clean_working_directory() + log.info("release called: version_target=%s", version_target) - # bump it (never incrementing patch because it was already updated with the last .dev version) repo_root = find_repo_root() log.info("Repo root: %s", repo_root) - new_version = bump_repo_version( - repo_root, version_part=version_part, target_version=target_version, dry_run=dry_run, increment_patch=False - ) - log.info("New version: %s", new_version) - # commit and tag for new version + # bump to the release version + new_version = bump_repo_version(repo_root, version_target=version_target, dry_run=dry_run) + log.info("New version: %s", new_version) if dry_run: click.echo(f"Dry run complete. Version would be bumped to {new_version}") return - else: - os.system("uv lock") - click.echo(f"Bumped version to {new_version}") - os.system("git add -u") - os.system(f'git commit -m "Release v{new_version}"') - os.system(f"git tag v{new_version}") - # bump patch and add suffix for next dev iteration - new_snapshot_version = bump_repo_version( - repo_root, - version_part="patch", - target_version=None, - dry_run=dry_run, - target_version_suffix=".dev0", - increment_patch=True, - ) + # commit and tag the release version + commit_version_change(new_version, message=f"Release v{new_version}") + os.system(f"git tag v{new_version}") + + # bump patch and add the suffix for the next dev iteration + new_snapshot_version = bump_repo_version(repo_root, version_target="patch", dry_run=dry_run, target_version_suffix=".dev0") log.info("New snapshot version: %s", new_snapshot_version) + commit_version_change(new_snapshot_version, message=f"Set version to v{new_snapshot_version}") - # commit the new snapshot version + +@cli.command() +@_version_part_argument +@_dry_run_option +def dev(version_part: VersionPart, dry_run: bool) -> None: + """Bumps the development version without creating a release. + + Sets the version to a new .dev0 version and commits it; no tag is created and the changelog + is not modified. + + VERSION_PART is the part of the version to bump (major, minor or patch). + """ + require_clean_working_directory() + log.info("dev called: version_part=%s", version_part) + + repo_root = find_repo_root() + log.info("Repo root: %s", repo_root) + + new_version = bump_repo_version(repo_root, version_target=version_part, dry_run=dry_run, target_version_suffix=".dev0") + log.info("New version: %s", new_version) + if dry_run: + click.echo(f"Dry run complete. Version would be bumped to {new_version}") + return + + commit_version_change(new_version, message=f"Set version to v{new_version}") + + +def require_clean_working_directory() -> None: + if not get_git_status().is_clean: + raise click.ClickException("Working directory is not clean. Please commit or stash your changes first.") + + +def commit_version_change(new_version: str, *, message: str) -> None: os.system("uv lock") - click.echo(f"Bumped version to {new_snapshot_version}") + click.echo(f"Bumped version to {new_version}") os.system("git add -u") - os.system(f'git commit -m "Set version to v{new_snapshot_version}"') + os.system(f'git commit -m "{message}"') def find_repo_root() -> Path: return Path(REPO_ROOT) -def resolve_version_selection(*, major: bool, minor: bool, patch: bool, target_version: str | None) -> VersionPart | None: - bump_flags_selected = sum([major, minor, patch]) - if target_version is not None and bump_flags_selected > 0: - raise click.ClickException("Use either --version or one of --major/--minor/--patch, not both.") - if bump_flags_selected > 1: - raise click.ClickException("Use only one of --major, --minor, or --patch.") - if target_version is not None: - validate_version_string(target_version) - return None - if major: - return "major" - if minor: - return "minor" - if patch: - return "patch" - raise click.ClickException("No version bump selected. Use --major, --minor, --patch or --version.") - - def bump_repo_version( repo_root: Path, *, - version_part: VersionPart | None, - target_version: str | None, + version_target: VersionTarget, dry_run: bool = False, target_version_suffix: str | None = None, - increment_patch: bool = True, ) -> str: pyproject_path = repo_root / "pyproject.toml" init_path = repo_root / "src" / "serena" / "__init__.py" @@ -130,12 +142,12 @@ def bump_repo_version( f"Version mismatch between pyproject.toml and src/serena/__init__.py: {current_version} != {init_version}" ) - if target_version is not None: - new_version = validate_version_string(target_version) - else: - if version_part is None: - raise click.ClickException("No version target specified.") - new_version = increment_version(current_version, version_part, increment_patch=increment_patch) + if version_target == "current" and _VERSION_SUFFIX_PATTERN.search(current_version) is None: + raise click.ClickException( + f"The current version {current_version} is not a development version, so there is no reserved version to release. " + f"Use major, minor or patch to bump the version instead." + ) + new_version = increment_version(current_version, version_target) if target_version_suffix is not None: new_version += target_version_suffix log.info("New version will be: %s", new_version) @@ -199,7 +211,14 @@ def replace_version(text: str, pattern: re.Pattern[str], new_version: str, file_ return f"{text[: match.start('version')]}{new_version}{text[match.end('version') :]}" -def increment_version(version: str, version_part: VersionPart, increment_patch: bool) -> str: +def increment_version(version: str, version_target: VersionTarget) -> str: + """ + Computes the new version, dropping any development suffix of the given version. + + :param version: the current version + :param version_target: the part of the version to bump or "current" to keep the version as is + :return: the new version + """ match = _VERSION_PATTERN.fullmatch(version) if match is None: raise click.ClickException(f"Unsupported version format: {version}") @@ -208,22 +227,17 @@ def increment_version(version: str, version_part: VersionPart, increment_patch: minor = int(match.group("minor")) patch = int(match.group("patch")) - if version_part == "major": - return f"{major + 1}.0.0" - if version_part == "minor": - return f"{major}.{minor + 1}.0" - elif version_part == "patch": - if increment_patch: - patch += 1 - return f"{major}.{minor}.{patch}" - else: - raise ValueError(version_part) - - -def validate_version_string(version: str) -> str: - if _VERSION_PATTERN.fullmatch(version) is None: - raise click.ClickException(f"Unsupported version format: {version}") - return version + match version_target: + case "major": + return f"{major + 1}.0.0" + case "minor": + return f"{major}.{minor + 1}.0" + case "patch": + return f"{major}.{minor}.{patch + 1}" + case "current": + return f"{major}.{minor}.{patch}" + case _: + raise ValueError(version_target) def update_changelog(changelog_text: str, new_version: str) -> str: @@ -278,4 +292,4 @@ def split_unreleased_body(unreleased_body: str) -> tuple[str, str]: if __name__ == "__main__": logging.basicConfig(level=logging.DEBUG, format="%(levelname)s %(name)s: %(message)s") log.info("Script starting") - bump_version() + cli() From 8fc68d4c6e7f84643b40b6db0998b7fb3ab4abc1 Mon Sep 17 00:00:00 2001 From: Dominik Jain Date: Tue, 15 Sep 2026 10:27:01 +0200 Subject: [PATCH 02/53] Set version to v2.0.0.dev0 --- pyproject.toml | 2 +- src/serena/__init__.py | 2 +- uv.lock | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 74a7cc1f..104de42f 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ requires = ["hatchling"] [project] name = "serena-agent" -version = "1.7.1.dev0" +version = "2.0.0.dev0" description = "A powerful MCP toolkit for coding, providing semantic retrieval and editing capabilities - the IDE for your agent" authors = [{ name = "Oraios AI", email = "info@oraios-ai.de" }] readme = "README.md" diff --git a/src/serena/__init__.py b/src/serena/__init__.py index 3b043bab..0d2c577e 100644 --- a/src/serena/__init__.py +++ b/src/serena/__init__.py @@ -1,6 +1,6 @@ # SPDX-License-Identifier: GPL-3.0-or-later -__version__ = "1.7.1.dev0" +__version__ = "2.0.0.dev0" import logging diff --git a/uv.lock b/uv.lock index cb77e62c..49a20470 100644 --- a/uv.lock +++ b/uv.lock @@ -2773,7 +2773,7 @@ wheels = [ [[package]] name = "serena-agent" -version = "1.7.1.dev0" +version = "2.0.0.dev0" source = { editable = "." } dependencies = [ { name = "anthropic" }, From 2c9640c3adf7a60452770649cb87dd1dd1a286a0 Mon Sep 17 00:00:00 2001 From: Dominik Jain Date: Sun, 6 Sep 2026 19:23:19 +0200 Subject: [PATCH 03/53] Add REPL-based interaction paradigm with a first facade (lsp.find_symbol) Introduce an alternative to individual tool calls: a single tool (serena_repl) executes Python code against an entrypoint object `s`, which exposes Serena's functionality through facades. This lets the LLM compose operations, filter results in code and return only what it needs, keeping intermediate data out of the context window. Facades (serena.facades): * FacadeApi: base class for implementations. Member naming determines LLM visibility: regular names are LLM-facing, a trailing underscore marks members that are public within Serena but never exposed to the LLM, a leading underscore is private. * Facade: indirection over a FacadeApi instance (Facade.from_api), holding one FacadeMethod per LLM-facing method, each of which can be enabled or disabled independently; only enabled methods are accessible from REPL code and included in the documentation. * SerenaRepl/SerenaReplEntrypoint: execute code as the body of a function (`return` defines the result; a single expression is evaluated directly), render the result via Representable and report errors with the line within the submitted code. Progressive disclosure via s.info(): the tool description and s.info() list the facades with their method names only; s.info("") and s.info(".") provide signatures together with docstrings, never signatures alone. * Representable/Renderer: result objects carry their rendering policy. Output parameters are passed at retrieval time so that they are inherited by derived results. First facade: LspApi with find_symbol, returning an LspSymbolCollection which renders as the familiar JSON (with grouping and progressive shortening) while exposing the underlying symbols to code. FindSymbolTool now delegates to LspApi, so both surfaces share one implementation. Length limiting and JSON output were moved from Tool into TextOutputUtils so that facades can use them. The agent creates the REPL lazily (get_repl) and resets it on project activation. The tool is marked beta. Also fixes pre-existing type errors (get_tool return type, invariant list annotations in symbol_tools, test stubs) found on the way. --- src/serena/agent.py | 22 ++- src/serena/facades/api/lsp.py | 210 ++++++++++++++++++++++++++++ src/serena/facades/facade.py | 170 ++++++++++++++++++++++ src/serena/facades/repl.py | 161 +++++++++++++++++++++ src/serena/facades/representable.py | 76 ++++++++++ src/serena/tools/__init__.py | 1 + src/serena/tools/repl_tools.py | 33 +++++ src/serena/tools/symbol_tools.py | 75 +++------- src/serena/tools/tools_base.py | 21 +-- src/serena/util/text_utils.py | 37 +++++ test/serena/test_file_tools.py | 2 +- test/serena/test_mcp.py | 2 + test/serena/test_repl_tool.py | 128 +++++++++++++++++ 13 files changed, 864 insertions(+), 74 deletions(-) create mode 100644 src/serena/facades/api/lsp.py create mode 100644 src/serena/facades/facade.py create mode 100644 src/serena/facades/repl.py create mode 100644 src/serena/facades/representable.py create mode 100644 src/serena/tools/repl_tools.py create mode 100644 test/serena/test_repl_tool.py diff --git a/src/serena/agent.py b/src/serena/agent.py index c43d7198..2d87f326 100644 --- a/src/serena/agent.py +++ b/src/serena/agent.py @@ -17,7 +17,7 @@ from dataclasses import dataclass from datetime import datetime from enum import Enum from logging import Logger -from typing import TYPE_CHECKING, Optional, TypeVar +from typing import TYPE_CHECKING, Optional, TypeVar, cast import requests import webview @@ -42,6 +42,9 @@ from serena.config.serena_config import ( ToolInclusionDefinition, ) from serena.dashboard import SerenaDashboardAPI, SerenaDashboardTrayManager, SerenaDashboardViewer, open_url_in_browser +from serena.facades.api.lsp import LspApi +from serena.facades.facade import Facade +from serena.facades.repl import SerenaRepl from serena.jetbrains import launch_coordinator as jetbrains_launch_coordinator from serena.ls_manager import LanguageServerManager from serena.memories.memory_manager import MemoryManager @@ -570,6 +573,7 @@ class SerenaAgent: self._session_mode_selection_definition = modes self.version = serena_version() self._config_changed_callbacks: list[Callable[[], None]] = [] + self._repl: SerenaRepl | None = None # obtain serena configuration using the decoupled factory function self.serena_config = serena_config or SerenaConfig.from_config_file() @@ -1158,6 +1162,17 @@ class SerenaAgent: "Consider adjusting your configuration to include these tools if you want to use them." ) + def get_repl(self) -> SerenaRepl: + """ + :return: the REPL instance for this agent, creating it if necessary + """ + if self._repl is None: + facades = [] + if self._language_backend.is_lsp(): + facades.append(Facade.from_api(LspApi(self))) + self._repl = SerenaRepl(facades) + return self._repl + def issue_task( self, task: Callable[[], T], name: str | None = None, logged: bool = True, timeout: float | None = None ) -> TaskExecutor.Task[T]: @@ -1252,6 +1267,9 @@ class SerenaAgent: self._project_prompt_status = ProjectPromptProvisionStatus(newly_activated_mode_names=newly_activated_mode_names) + # reset the REPL to ensure that the new project's configuration is considered + self._repl = None + if update_active_tools: self._update_active_tools() @@ -1457,7 +1475,7 @@ class SerenaAgent: self.issue_task(lambda: self.get_active_project_or_raise().remove_language_server(ls_id), name=f"RemoveLanguage:{ls_id.get_key()}") def get_tool(self, tool_class: type[TTool]) -> TTool: - return self._all_tools[tool_class] + return cast(TTool, self._all_tools[tool_class]) def print_tool_overview(self) -> None: ToolRegistry().print_tool_overview(self._active_tools.tools) diff --git a/src/serena/facades/api/lsp.py b/src/serena/facades/api/lsp.py new file mode 100644 index 00000000..33feb8a7 --- /dev/null +++ b/src/serena/facades/api/lsp.py @@ -0,0 +1,210 @@ +# SPDX-License-Identifier: GPL-3.0-or-later + +from collections import defaultdict +from collections.abc import Sequence +from dataclasses import dataclass +from typing import TYPE_CHECKING + +from serena.symbol import LanguageServerSymbol, LanguageServerSymbolDictGrouper, LanguageServerSymbolRetriever, SymbolDictGrouper +from solidlsp.lsp_protocol_handler.lsp_types import SymbolKind + +from ...util.text_utils import TextOutputUtils +from ..facade import FacadeApi +from ..representable import Renderer, RepresentableViaRenderer + +if TYPE_CHECKING: + from serena.agent import SerenaAgent + + +class LspSymbolCollection(RepresentableViaRenderer): + def __init__(self, symbols: list[LanguageServerSymbol], renderer: "LspSymbolCollectionRenderer"): + """ + :param symbols: the list of symbols + :param renderer: the renderer to use for representing the collection + """ + super().__init__(renderer) + self.symbols = symbols + + def __len__(self): + return len(self.symbols) + + def relative_path_to_name_paths_(self) -> dict[str, list[str]]: + result: defaultdict[str, list[str]] = defaultdict(list) + for s in self.symbols: + result[s.location.relative_path or "unknown"].append(s.get_name_path()) + return result + + +@dataclass(kw_only=True) +class SymbolOutputParams: + name_path: bool = True + name: bool = False + kind: bool = False + location: bool = False + depth: int = 0 + body_location: bool = False + children_body: bool = False + children_name: bool | None = None + children_name_path: bool | None = None + relative_path: bool = False + include_body: bool = False + include_info: bool = False + + +class LspSymbolCollectionRenderer(Renderer[LspSymbolCollection]): + def __init__( + self, + agent: "SerenaAgent", + max_answer_chars: int, + symbol_retriever: LanguageServerSymbolRetriever, + output_params: SymbolOutputParams, + grouper: SymbolDictGrouper | None = None, + ): + super().__init__(agent, max_answer_chars) + self._symbol_retriever = symbol_retriever + self._output_params = output_params + self._grouper = grouper + + def set_grouper(self, grouper: SymbolDictGrouper) -> None: + self._grouper = grouper + + def render(self, obj: LspSymbolCollection) -> str: + symbols = obj.symbols + symbol_dicts = [ + s.to_dict( + kind=self._output_params.kind, + name_path=self._output_params.name_path, + name=self._output_params.name, + relative_path=self._output_params.relative_path, + body_location=self._output_params.body_location, + depth=self._output_params.depth, + body=self._output_params.include_body, + children_name=self._output_params.children_name, + children_name_path=self._output_params.children_name_path, + ) + for s in symbols + ] + if not self._output_params.include_body and self._output_params.include_info: + info_by_symbol = self._symbol_retriever.request_info_for_symbol_batch(symbols) + for s, s_dict in zip(symbols, symbol_dicts, strict=True): + if symbol_info := info_by_symbol.get(s): + # In python 3.15 we could specify extra_items=True in the TypedDict definition, + # https://peps.python.org/pep-0728/ + # If we ever upgrade to 3.15, we can remove the type: ignore[typeddict-unknown-key] + s_dict["info"] = symbol_info + + def create_short_result_relative_path_to_name_paths() -> str: + relative_path_to_name_paths = obj.relative_path_to_name_paths_() + return f"Shortened result:\n{TextOutputUtils.to_json(relative_path_to_name_paths)}" + + if self._grouper is not None: + objects = self._grouper.group(symbol_dicts) + else: + objects = symbol_dicts + result = self._to_json(objects) + return self._limit_length(result, shortened_result_factories=[create_short_result_relative_path_to_name_paths]) + + +class LspApi(FacadeApi): + def __init__(self, agent: "SerenaAgent") -> None: + super().__init__(agent, name="lsp", description="LSP-backed operations on the codebase (finding symbols, etc.)") + + def _create_symbol_retriever(self) -> LanguageServerSymbolRetriever: + assert self._agent.get_language_backend().is_lsp(), "Symbolic read operations require the language server backend" + return LanguageServerSymbolRetriever(self._get_project()) + + # group children by kind, keeping just the name (the parent's name_path makes it unambiguous); + # we don't group the top-level result list because many tests rely on it being a flat list of symbol dicts + find_symbol_dict_grouper_ = LanguageServerSymbolDictGrouper([], ["kind"], collapse_singleton=True) + + def find_symbol( + self, + name_path_pattern: str, + depth: int = 0, + relative_path: str = "", + include_body: bool = False, + include_info: bool = False, + include_kinds: Sequence[int] = (), + exclude_kinds: Sequence[int] = (), + substring_matching: bool = False, + max_matches: int = -1, + max_answer_chars: int = -1, + ) -> LspSymbolCollection: + """ + Finds symbols and code entities (classes, methods, etc.) based on the given name path pattern. + The returned symbol information can be used for edits or further queries. + Specify `depth > 0` to also retrieve children/descendants (e.g., methods of a class). + + A name path is a path in the symbol tree *within a source file*. + For example, the method `my_method` defined in class `MyClass` would have the name path `MyClass/my_method`. + If a symbol is overloaded (e.g., in Java), a 0-based index is appended (e.g. "MyClass/my_method[0]") to + uniquely identify it. + + To search for a symbol, you provide a name path pattern that is used to match against name paths. + It can be + * a simple name (e.g. "method"), which will match any symbol with that name + * a relative path like "class/method", which will match any symbol with that name path suffix + * an absolute name path "/class/method" (absolute name path), which requires an exact match of the full name path within the source file. + Append an index `[i]` to match a specific overload only, e.g. "MyClass/my_method[1]". + + :param name_path_pattern: the name path matching pattern (see above) + :param depth: depth up to which descendants shall be retrieved (e.g. use 1 to also retrieve immediate children; + for the case where the symbol is a class, this will return its methods). + Ignored if `include_body=True`. Default 0. + :param relative_path: (optional) restrict search to this file or directory. If None, searches entire codebase. + If a directory is passed, the search will be restricted to the files in that directory. + If a file is passed, the search will be restricted to that file. + :param include_body: whether to include the symbol's source code. Use judiciously. + :param include_info: whether to include additional info (hover-like, typically including docstring and signature), + about the symbol (ignored if include_body is True). Info is never included for child symbols. + Note: Depending on the language, this can be slow (e.g., C/C++). + :param include_kinds: (optional) limits results to the given LSP symbol kinds (integers) + :param exclude_kinds: (optional) list of LSP symbol kinds (integers) to exclude. + :param substring_matching: If True, use substring matching for the last element of the pattern, such that + "Foo/get" would match "Foo/getValue" and "Foo/getData". + :param max_matches: maximum number of permitted matches. If exceeded, a shortened result is returned + which allows refining the search. -1 (default) means no limit. Set to 1 to search for a unique symbol. + :param max_answer_chars: max result length; -1 for default + :return: collection of matching symbols + """ + # Note: file system sync not required; the symbol finder opens all relevant source files explicitly in the case of changes + + if include_body: + depth = 0 # ignore user-specified depth if include_body is True + assert max_matches != 0, "max_matches must be > 0 or equal to -1." + parsed_include_kinds: Sequence[SymbolKind] | None = [SymbolKind(k) for k in include_kinds] if include_kinds else None + parsed_exclude_kinds: Sequence[SymbolKind] | None = [SymbolKind(k) for k in exclude_kinds] if exclude_kinds else None + symbol_retriever = self._create_symbol_retriever() + symbols = symbol_retriever.find( + name_path_pattern, + include_kinds=parsed_include_kinds, + exclude_kinds=parsed_exclude_kinds, + substring_matching=substring_matching, + within_relative_path=relative_path, + ) + + output_params = SymbolOutputParams( + kind=True, + name_path=True, + name=False, + relative_path=True, + body_location=True, + depth=depth, + include_body=include_body, + children_name=True, + children_name_path=False, + include_info=include_info, + ) + renderer = LspSymbolCollectionRenderer( + self._agent, max_answer_chars, symbol_retriever, output_params, grouper=self.find_symbol_dict_grouper_ + ) + symbol_collection = LspSymbolCollection(symbols, renderer) + + # check for max_matches limit exceeded + n_matches = len(symbols) + if 0 < max_matches < n_matches: + raise ValueError( + f"Matched {n_matches}>{max_matches=} symbols.\n" + TextOutputUtils.to_json(symbol_collection.relative_path_to_name_paths_()) + ) + + return symbol_collection diff --git a/src/serena/facades/facade.py b/src/serena/facades/facade.py new file mode 100644 index 00000000..bd8f6b54 --- /dev/null +++ b/src/serena/facades/facade.py @@ -0,0 +1,170 @@ +""" +The facade, i.e. the object through which REPL code accesses a group of related operations. +""" + +# SPDX-License-Identifier: GPL-3.0-or-later + +import inspect +from abc import ABC +from collections.abc import Callable, Iterable +from typing import TYPE_CHECKING, Any + +from serena.project import Project + +if TYPE_CHECKING: + from serena.agent import SerenaAgent + + +class FacadeApi(ABC): + """ + The implementation of a facade's functionality. + + API design principle: a member's name determines its visibility to the LLM. + + * Names without a leading underscore and without a trailing underscore (e.g. `find_symbol`) constitute the + LLM-facing interface. Every such method of a concrete implementation is a candidate for exposure through + a `Facade`; which of them are actually exposed is decided by the facade. + * Names with a trailing underscore (e.g. `symbols_`, `to_dict_`) are public within Serena (e.g. for use by + classic tools or other facade implementations) but are never exposed to the LLM. Use this for functionality + which is not meant to be called from REPL code, in particular on the objects returned by API methods. + * Names with a leading underscore are private, as usual. + """ + + def __init__(self, agent: "SerenaAgent", name: str, description: str) -> None: + """ + :param agent: the agent providing access to the project and its resources + :param name: the attribute name under which the facade is accessible from the REPL entrypoint + :param description: a one-line description of the functionality offered by the facade + """ + self._agent = agent + self._name = name + self._description = description + + def get_name_(self) -> str: + return self._name + + def get_description_(self) -> str: + return self._description + + def _get_project(self) -> Project: + return self._agent.get_active_project_or_raise() + + +class FacadeMethod: + """ + A method of a facade, which delegates to a method of the underlying implementation and which can be + enabled or disabled; only enabled methods are accessible from REPL code. + """ + + def __init__(self, name: str, implementation: Callable[..., Any], enabled: bool = True) -> None: + self.name = name + self._implementation = implementation + self.enabled = enabled + + def __call__(self, *args: Any, **kwargs: Any) -> Any: + return self._implementation(*args, **kwargs) + + def describe(self, facade_name: str) -> str: + """ + :param facade_name: the name of the facade the method belongs to + :return: the method's signature and documentation + """ + signature = inspect.signature(self._implementation) + doc = inspect.getdoc(self._implementation) or "(no documentation)" + return f"{facade_name}.{self.name}{signature}\n{doc}\n" + + +class Facade: + """ + A named group of related operations which an LLM can invoke from REPL code. + """ + + def __init__(self, name: str, description: str, methods: Iterable[FacadeMethod]) -> None: + # NOTE: attributes are set via object.__setattr__ because __getattr__ is overridden + object.__setattr__(self, "_name", name) + object.__setattr__(self, "_description", description) + object.__setattr__(self, "_methods", {m.name: m for m in methods}) + + @staticmethod + def _is_exposable_member_name(name: str) -> bool: + """ + :param name: the name of a member of a facade implementation + :return: whether the member may be exposed through a facade, i.e. whether its name has neither a leading + nor a trailing underscore (see `FacadeApi` for the naming principle) + """ + return not name.startswith("_") and not name.endswith("_") + + @staticmethod + def from_api(api: FacadeApi, enabled_methods: Iterable[str] | None = None) -> "Facade": + """ + Creates a facade wrapping the given implementation. + + :param api: the implementation; each of its LLM-facing methods (see `_is_exposable_member_name`) becomes a facade method + :param enabled_methods: the names of the methods to enable; if None, all methods are enabled + :return: the facade + """ + enabled = None if enabled_methods is None else set(enabled_methods) + methods = [ + FacadeMethod(name, member, enabled=enabled is None or name in enabled) + for name, member in inspect.getmembers(api, predicate=inspect.ismethod) + if Facade._is_exposable_member_name(name) + ] + return Facade(api.get_name_(), api.get_description_(), methods) + + @property + def name(self) -> str: + return self._name + + @property + def description(self) -> str: + return self._description + + @property + def enabled_method_names(self) -> list[str]: + return [m.name for m in self._methods.values() if m.enabled] + + def get_method(self, method_name: str) -> FacadeMethod: + """ + :param method_name: the name of the method + :return: the method, regardless of whether it is enabled (e.g. for changing its enabled state) + """ + if method_name not in self._methods: + raise ValueError(f"Facade '{self._name}' has no method '{method_name}'") + return self._methods[method_name] + + def _get_enabled_method(self, name: str) -> FacadeMethod | None: + method = self._methods.get(name) + return method if method is not None and method.enabled else None + + def _no_such_method_message(self, name: str) -> str: + return f"Facade '{self._name}' has no method '{name}'. Available methods: {self.enabled_method_names}" + + def __getattr__(self, name: str) -> Any: + # delegate attribute access to enabled methods only (called only if regular attribute lookup fails) + method = self._get_enabled_method(name) + if method is None: + raise AttributeError(self._no_such_method_message(name)) + return method + + def __setattr__(self, name: str, value: Any) -> None: + raise AttributeError(f"Facade '{self._name}' is read-only") + + def describe(self) -> str: + """ + :return: a description of the facade listing all of its enabled methods with their signatures and documentation + """ + parts = [f"Facade '{self._name}': {self._description}", ""] + for method in self._methods.values(): + if method.enabled: + parts.append(method.describe(self._name)) + return "\n".join(parts) + + def describe_method(self, method_name: str) -> str: + """ + :param method_name: the name of one of the facade's enabled methods + :return: the method's signature and documentation + """ + method = self._get_enabled_method(method_name) + if method is None: + raise ValueError(self._no_such_method_message(method_name)) + return method.describe(self._name) diff --git a/src/serena/facades/repl.py b/src/serena/facades/repl.py new file mode 100644 index 00000000..ecbce608 --- /dev/null +++ b/src/serena/facades/repl.py @@ -0,0 +1,161 @@ +""" +The REPL through which an LLM executes Python code against Serena's facades. +""" + +# SPDX-License-Identifier: GPL-3.0-or-later + +import textwrap +import traceback +from collections.abc import Iterable +from typing import Any + +from .facade import Facade +from .representable import Representable + + +class SerenaReplEntrypoint: + """ + Represents the entrypoint object for the REPL. It holds the configured facades as attributes + and offers progressive disclosure of their interfaces via `info`. + """ + + def __init__(self, facades: Iterable[Facade]) -> None: + self._facades: dict[str, Facade] = {} + for facade in facades: + self._register(facade) + + def _register(self, facade: Facade) -> None: + if facade.name in self._facades: + raise ValueError(f"Duplicate facade name: {facade.name}") + self._facades[facade.name] = facade + setattr(self, facade.name, facade) + + def _get_facade(self, name: str) -> Facade: + if name not in self._facades: + raise ValueError(f"Unknown facade '{name}'. Available facades: {list(self._facades)}") + return self._facades[name] + + def overview(self) -> str: + """ + :return: the list of available facades, each with a one-line description and the names of its methods + """ + return "\n".join( + f"s.{facade.name}: {facade.description}\n methods: {', '.join(facade.enabled_method_names)}" + for facade in self._facades.values() + ) + + def info(self, path: str = "") -> str: + """ + Provides documentation on the available functionality. + + :param path: the empty string for an overview of all facades, a facade name (e.g. "lsp") for the + documentation of all of the facade's methods, or a dotted method path (e.g. "lsp.find_symbol") + for the documentation of a single method + :return: the requested documentation + """ + if path == "": + return self.overview() + facade_name, _, method_name = path.partition(".") + facade = self._get_facade(facade_name) + if method_name == "": + return facade.describe() + return facade.describe_method(method_name) + + +class SerenaRepl: + """ + Executes Python code submitted by an LLM, binding the configured facades to the entrypoint object `s` + and rendering the result of the execution as a string for the LLM. + + The code is executed as the body of a function, such that the `return` statement defines the result; + code consisting of a single expression is evaluated and its value is the result. + """ + + SOURCE_NAME = "" + ENTRYPOINT_NAME = "s" + _FUNCTION_NAME = "__serena_repl_fn__" + + def __init__(self, facades: Iterable[Facade]) -> None: + """ + :param facades: the facades to make available through the entrypoint + """ + self._entrypoint = SerenaReplEntrypoint(facades) + + @property + def entrypoint(self) -> SerenaReplEntrypoint: + return self._entrypoint + + @classmethod + def _represent(cls, obj: Any) -> str: + """ + Renders an arbitrary object as a string for the LLM. Representables render themselves, + lists and tuples are rendered element-wise (one element per line), everything else via `str`. + + :param obj: the object to render + :return: the textual representation + """ + if isinstance(obj, Representable): + return obj.represent() + if isinstance(obj, list | tuple): + if len(obj) == 0: + return "[]" + return "\n".join(cls._represent(item) for item in obj) + return str(obj) + + def execute(self, code: str) -> str: + """ + Executes the given code and renders its result. + + :param code: the Python code to execute + :return: the representation of the code's result, or a description of the error if execution failed + """ + try: + result = self._run(code) + except Exception as e: + return self._format_error(e, code) + return self._represent(result) + + def _run(self, code: str) -> Any: + """ + Runs the given code with the entrypoint bound, either as a single expression + or as the body of a function whose return value is the result. + """ + namespace: dict[str, Any] = {self.ENTRYPOINT_NAME: self._entrypoint} + + # try to evaluate the code as a single expression + try: + compiled = compile(code, self.SOURCE_NAME, "eval") + except SyntaxError: + compiled = None + if compiled is not None: + return eval(compiled, namespace) + + # otherwise execute the code as the body of a function + source = f"def {self._FUNCTION_NAME}({self.ENTRYPOINT_NAME}):\n" + textwrap.indent(code, " ") + exec(compile(source, self.SOURCE_NAME, "exec"), namespace) + return namespace[self._FUNCTION_NAME](self._entrypoint) + + def _format_error(self, e: Exception, code: str) -> str: + """ + :param e: the exception raised during execution + :param code: the code that was executed + :return: an error message which locates the failure within the executed code + """ + code_lines = code.splitlines() + + def location_line(line_number: int) -> str: + line_text = code_lines[line_number - 1].strip() if 0 < line_number <= len(code_lines) else "" + return f" line {line_number}: {line_text}" + + # report syntax errors in the executed code (which carry no traceback frames of their own) + if isinstance(e, SyntaxError) and e.filename == self.SOURCE_NAME and e.lineno is not None: + return f"SyntaxError: {e.msg}\n" + location_line(e.lineno - 1) # undo the function header offset + + # report runtime errors, locating them within the executed code + location_lines = [] + for frame in traceback.extract_tb(e.__traceback__): + if frame.filename != self.SOURCE_NAME or frame.lineno is None: + continue + line_number = frame.lineno - 1 if frame.name == self._FUNCTION_NAME else frame.lineno # undo the function header offset + location_lines.append(location_line(line_number)) + return "\n".join([f"{type(e).__name__}: {e}", *location_lines]) diff --git a/src/serena/facades/representable.py b/src/serena/facades/representable.py new file mode 100644 index 00000000..7b9f7f60 --- /dev/null +++ b/src/serena/facades/representable.py @@ -0,0 +1,76 @@ +""" +The representation protocol through which objects returned from REPL code are rendered for the LLM. +""" + +# SPDX-License-Identifier: GPL-3.0-or-later + +from abc import ABC, abstractmethod +from collections.abc import Callable +from typing import TYPE_CHECKING, Any, Generic, TypeVar + +from serena.util.text_utils import TextOutputUtils + +if TYPE_CHECKING: + from serena.agent import SerenaAgent + + +T = TypeVar("T") + + +class Renderer(Generic[T], ABC): + def __init__(self, agent: "SerenaAgent", max_answer_chars: int = -1): + self._agent = agent + self._max_answer_chars = max_answer_chars + + def _limit_length( + self, + result: str, + shortened_result_factories: list[Callable[[], str]] | None = None, + ) -> str: + """Limit the length of the result string, optionally trying progressively shorter versions. + + :param result: the full result string + :param max_answer_chars: maximum allowed characters. -1 means use the default from config. + :param shortened_result_factories: optional list of closures, each producing a progressively shorter + version of the result. They are tried in order until one fits within ``max_answer_chars``. + :return: the result string, potentially replaced by a shortened version + """ + max_answer_chars = self._max_answer_chars + if max_answer_chars == -1: + max_answer_chars = self._agent.serena_config.default_max_tool_answer_chars + return TextOutputUtils.limit_length( + result=result, max_answer_chars=max_answer_chars, shortened_result_factories=shortened_result_factories + ) + + def _to_json(self, x: Any) -> str: + return TextOutputUtils.to_json(x) + + @abstractmethod + def render(self, obj: T) -> str: + """ + :return: a textual representation of this object for the LLM + """ + + +class Representable(ABC): + """ + An object which can render itself as a string suitable for consumption by an LLM. + """ + + @abstractmethod + def represent(self) -> str: + """ + :return: a textual representation of this object for the LLM + """ + + +class RepresentableViaRenderer(Representable): + """ + A representable object which uses a renderer to render itself. + """ + + def __init__(self, renderer: Renderer): + self._renderer = renderer + + def represent(self) -> str: + return self._renderer.render(self) diff --git a/src/serena/tools/__init__.py b/src/serena/tools/__init__.py index 2475dfca..0ed2e6bf 100644 --- a/src/serena/tools/__init__.py +++ b/src/serena/tools/__init__.py @@ -10,3 +10,4 @@ from .config_tools import * from .workflow_tools import * from .jetbrains_tools import * from .query_project_tools import * +from .repl_tools import * diff --git a/src/serena/tools/repl_tools.py b/src/serena/tools/repl_tools.py new file mode 100644 index 00000000..ba7b4980 --- /dev/null +++ b/src/serena/tools/repl_tools.py @@ -0,0 +1,33 @@ +""" +Tools which provide access to Serena's functionality through Python code execution +""" + +# SPDX-License-Identifier: GPL-3.0-or-later + +from serena.tools.tools_base import Tool, ToolMarkerBeta + + +class SerenaReplTool(Tool, ToolMarkerBeta): + """ + Executes Python code which accesses Serena's functionality programmatically. + """ + + def get_apply_docstring(self) -> str: + return self.get_apply_docstring_from_cls() + "\n\nAvailable facades:\n" + self.agent.get_repl().entrypoint.overview() + + def apply(self, code: str) -> str: + """ + Executes the given Python code, which has access to Serena's functionality through the object `s`. + The functionality is organised in facades, which are attributes of `s` (e.g. `s.myfacade`). + Use `s.info()` to list the facades, `s.info("")` to see a facade's methods and + `s.info(".")` for the documentation of a single method. + + The code is executed as the body of a function, so use `return` to define the result; + a single expression is evaluated and its value returned directly. + Returned objects are rendered in a form suitable for you; lists are rendered element-wise. + Returned strings are passed through unchanged. + + :param code: the Python code to execute + :return: the representation of the returned value, or the error if execution failed + """ + return self.agent.get_repl().execute(code) diff --git a/src/serena/tools/symbol_tools.py b/src/serena/tools/symbol_tools.py index 1ab40e0a..24dac396 100644 --- a/src/serena/tools/symbol_tools.py +++ b/src/serena/tools/symbol_tools.py @@ -6,9 +6,10 @@ Language server-related tools import copy import os from collections import Counter, defaultdict -from collections.abc import Sequence +from collections.abc import Callable, Sequence from typing import Any +from serena.facades.api.lsp import LspApi from serena.symbol import LanguageServerSymbol, LanguageServerSymbolDictGrouper from serena.tools import ( SUCCESS_RESULT, @@ -79,6 +80,7 @@ class GetSymbolsOverviewTool(Tool, ToolMarkerSymbolicRead): def make_kind_counts() -> str: return f"Symbol counts by kind:\n{self._to_json(Counter(kind_names))}" + shortened_results: list[Callable[[], str]] if depth == 0: shortened_results = [make_kind_counts] else: @@ -137,9 +139,11 @@ class FindSymbolTool(Tool, ToolMarkerSymbolicRead): Performs a global (or local) search using the language server backend. """ - # group children by kind, keeping just the name (the parent's name_path makes it unambiguous); - # we don't group the top-level result list because many tests rely on it being a flat list of symbol dicts - symbol_dict_grouper = LanguageServerSymbolDictGrouper([], ["kind"], collapse_singleton=True) + symbol_dict_grouper = LspApi.find_symbol_dict_grouper_ + """ + Reference to the grouper that is indirectly used by this tool. + Made explicit such that grouping behaviour for this tool can be modified dynamically. + """ # noinspection PyDefaultArgument def apply( @@ -188,62 +192,23 @@ class FindSymbolTool(Tool, ToolMarkerSymbolicRead): :param substring_matching: If True, use substring matching for the last element of the pattern, such that "Foo/get" would match "Foo/getValue" and "Foo/getData". :param max_matches: maximum number of permitted matches. If exceeded, a shortened result is returned - which allows refining the search. -1 (default) means no limit. Set to 1 if you search for a single symbol. + which allows refining the search. -1 (default) means no limit. Set to 1 to search for a unique symbol. :param max_answer_chars: max result length; -1 for default :return: symbols (with locations) matching the name. """ - # Note: file system sync not required; the symbol finder opens all relevant source files explicitly in the case of changes - - if include_body: - depth = 0 # ignore user-specified depth if include_body is True - assert max_matches != 0, "max_matches must be > 0 or equal to -1." - parsed_include_kinds: Sequence[SymbolKind] | None = [SymbolKind(k) for k in include_kinds] if include_kinds else None - parsed_exclude_kinds: Sequence[SymbolKind] | None = [SymbolKind(k) for k in exclude_kinds] if exclude_kinds else None - symbol_retriever = self.create_language_server_symbol_retriever() - symbols = symbol_retriever.find( + collection = LspApi(self.agent).find_symbol( name_path_pattern, - include_kinds=parsed_include_kinds, - exclude_kinds=parsed_exclude_kinds, + depth=depth, + relative_path=relative_path, + include_body=include_body, + include_info=include_info, + include_kinds=include_kinds, + exclude_kinds=exclude_kinds, substring_matching=substring_matching, - within_relative_path=relative_path, + max_matches=max_matches, + max_answer_chars=max_answer_chars, ) - n_matches = len(symbols) - - def create_short_result_relative_path_to_name_paths() -> str: - relative_path_to_name_paths: defaultdict[str, list[str]] = defaultdict(list) - for s in symbols: - relative_path_to_name_paths[s.location.relative_path or "unknown"].append(s.get_name_path()) - return f"Shortened result:\n{self._to_json(relative_path_to_name_paths)}" - - if 0 < max_matches < n_matches: - return f"Matched {n_matches}>{max_matches=} symbols.\n" + create_short_result_relative_path_to_name_paths() - - symbol_dicts = [ - s.to_dict( - kind=True, - name_path=True, - name=False, - relative_path=True, - body_location=True, - depth=depth, - body=include_body, - children_name=True, - children_name_path=False, - ) - for s in symbols - ] - if not include_body and include_info: - info_by_symbol = symbol_retriever.request_info_for_symbol_batch(symbols) - for s, s_dict in zip(symbols, symbol_dicts, strict=True): - if symbol_info := info_by_symbol.get(s): - # In python 3.15 we could specify extra_items=True in the TypedDict definition, - # https://peps.python.org/pep-0728/ - # If we ever upgrade to 3.15, we can remove the type: ignore[typeddict-unknown-key] - s_dict["info"] = symbol_info - - grouped_symbol_dicts = self.symbol_dict_grouper.group(symbol_dicts) - result = self._to_json(grouped_symbol_dicts) - return self._limit_length(result, max_answer_chars, shortened_result_factories=[create_short_result_relative_path_to_name_paths]) + return collection.represent() @classmethod def get_param_aliases(cls) -> dict[str, str]: @@ -334,7 +299,7 @@ class FindReferencingSymbolsTool(Tool, ToolMarkerSymbolicRead): def make_summary() -> str: return f"Found {len(ref_summaries)} references." - shortened_results = [make_refs_without_context, make_per_file_counts, make_summary] + shortened_results: list[Callable[[], str]] = [make_refs_without_context, make_per_file_counts, make_summary] result_json = self._to_json(result) return self._limit_length(result_json, max_answer_chars, shortened_result_factories=shortened_results) diff --git a/src/serena/tools/tools_base.py b/src/serena/tools/tools_base.py index 60e81ad9..2009727b 100644 --- a/src/serena/tools/tools_base.py +++ b/src/serena/tools/tools_base.py @@ -22,6 +22,7 @@ from serena.prompt_factory import PromptFactory from serena.util.class_decorators import singleton from serena.util.inspection import iter_subclasses from serena.util.ls_diagnostics import DiagnosticsDiff, EditedFilePath, PublishedDiagnosticsSnapshot +from serena.util.text_utils import TextOutputUtils from solidlsp.ls_exceptions import SolidLSPException if TYPE_CHECKING: @@ -296,21 +297,9 @@ class Tool(Component): """ if max_answer_chars == -1: max_answer_chars = self.agent.serena_config.default_max_tool_answer_chars - if max_answer_chars <= 0: - raise ValueError(f"Must be positive or the default (-1), got: {max_answer_chars=}") - if (n_chars := len(result)) > max_answer_chars: - too_long_msg = ( - f"The answer is too long ({n_chars} characters). " + "You can adjust your query or raise the max_answer_chars parameter." - ) - if shortened_result_factories is not None: - # try each shortening closure in order; - for make_shorter in shortened_result_factories: - shortened = make_shorter() - candidate = f"{too_long_msg}\n{shortened}" - if len(candidate) <= max_answer_chars: - return candidate - result = too_long_msg - return result + return TextOutputUtils.limit_length( + result=result, max_answer_chars=max_answer_chars, shortened_result_factories=shortened_result_factories + ) def is_active(self) -> bool: return self.agent.tool_is_active(self.get_name()) @@ -444,7 +433,7 @@ class Tool(Component): @staticmethod def _to_json(x: Any) -> str: - return json.dumps(x, ensure_ascii=False) + return TextOutputUtils.to_json(x) def _wrapped_tool_response(self, response: Any, message: str) -> str: """ diff --git a/src/serena/util/text_utils.py b/src/serena/util/text_utils.py index e3ee0d1e..b73c5755 100644 --- a/src/serena/util/text_utils.py +++ b/src/serena/util/text_utils.py @@ -1,6 +1,7 @@ # SPDX-License-Identifier: GPL-3.0-or-later import hashlib +import json import logging import re from collections.abc import Callable @@ -676,3 +677,39 @@ def find_text_coordinates(content: str, regex: str, require_unique: bool = False index_in_content = match.start(1) line, col = TextUtils.get_line_col_from_index(content, index_in_content) return TextCoords(line, col) + + +class TextOutputUtils: + @staticmethod + def to_json(x: Any) -> str: + return json.dumps(x, ensure_ascii=False) + + @staticmethod + def limit_length( + result: str, + max_answer_chars: int, + shortened_result_factories: list[Callable[[], str]] | None = None, + ) -> str: + """Limit the length of the result string, optionally trying progressively shorter versions. + + :param result: the full result string + :param max_answer_chars: maximum allowed characters; if exceeded, attempt to use shortened versions + :param shortened_result_factories: optional list of closures, each producing a progressively shorter + version of the result. They are tried in order until one fits within ``max_answer_chars``. + :return: the result string, potentially replaced by a shortened version + """ + if max_answer_chars <= 0: + raise ValueError(f"max_answer_chars must be positive; got: {max_answer_chars=}") + if (n_chars := len(result)) > max_answer_chars: + too_long_msg = ( + f"The answer is too long ({n_chars} characters). " + "You can adjust your query or raise the max_answer_chars parameter." + ) + if shortened_result_factories is not None: + # try each shortening closure in order; + for make_shorter in shortened_result_factories: + shortened = make_shorter() + candidate = f"{too_long_msg}\n{shortened}" + if len(candidate) <= max_answer_chars: + return candidate + result = too_long_msg + return result diff --git a/test/serena/test_file_tools.py b/test/serena/test_file_tools.py index 24c1649d..8095624e 100644 --- a/test/serena/test_file_tools.py +++ b/test/serena/test_file_tools.py @@ -17,7 +17,7 @@ def read_file_tool(tmp_path: Path) -> ReadFileTool: agent.get_active_project_or_raise.return_value = project tool = ReadFileTool(agent) # bypass the length limit, which would otherwise depend on the agent configuration - tool._limit_length = lambda result, max_answer_chars: result + tool._limit_length = lambda result, max_answer_chars, shortened_result_factories=None: result # type: ignore[method-assign] return tool diff --git a/test/serena/test_mcp.py b/test/serena/test_mcp.py index db1923fd..d75d0500 100644 --- a/test/serena/test_mcp.py +++ b/test/serena/test_mcp.py @@ -1,6 +1,7 @@ """Tests for the mcp.py module in serena.""" import pytest +from mcp.server.fastmcp import Context from mcp.server.fastmcp.tools.base import Tool as MCPTool from serena import __version__ @@ -46,6 +47,7 @@ class BasicTool(BaseMockTool): self, log_call: bool = True, catch_exceptions: bool = True, + mcp_ctx: Context | None = None, **kwargs, ) -> str: """Mock implementation of apply_ex.""" diff --git a/test/serena/test_repl_tool.py b/test/serena/test_repl_tool.py new file mode 100644 index 00000000..4efc2aeb --- /dev/null +++ b/test/serena/test_repl_tool.py @@ -0,0 +1,128 @@ +""" +Tests for the REPL tool, which executes Python code against the facade entrypoint `s`. +""" + +import os +from unittest.mock import MagicMock + +import pytest + +from serena.facades.api.lsp import LspApi +from serena.facades.facade import Facade, FacadeApi +from serena.facades.repl import SerenaRepl +from serena.tools import SerenaReplTool +from solidlsp.ls_config import LanguageServerId +from test.conftest import agent_for_project_context + + +class TestReplExecution: + """Tests the code execution mechanics of the REPL, which do not require a project.""" + + @pytest.fixture + def repl(self) -> SerenaRepl: + return SerenaRepl([Facade.from_api(LspApi(MagicMock()))]) + + def test_return_statement_defines_result(self, repl: SerenaRepl) -> None: + assert repl.execute("x = 20\ny = 22\nreturn x + y") == "42" + + def test_single_expression_is_evaluated(self, repl: SerenaRepl) -> None: + assert repl.execute("1 + 2") == "3" + + def test_list_is_rendered_element_wise(self, repl: SerenaRepl) -> None: + assert repl.execute('return ["a", "b"]') == "a\nb" + + def test_error_reports_type_message_and_line(self, repl: SerenaRepl) -> None: + result = repl.execute("x = 1\nraise ValueError('boom')") + assert result.startswith("ValueError: boom") + assert "line 2" in result + + def test_syntax_error_reports_line(self, repl: SerenaRepl) -> None: + result = repl.execute("x = 1\ny = (2") + assert result.startswith("SyntaxError") + assert "line 2" in result + + def test_facade_discovery(self, repl: SerenaRepl) -> None: + overview = repl.execute("s.info()") + assert "s.lsp" in overview + assert "find_symbol" in overview # method names are listed, but not signatures + assert "name_path_pattern" not in overview + facade_info = repl.execute('s.info("lsp")') + assert "find_symbol(" in facade_info + method_info = repl.execute('s.info("lsp.find_symbol")') + assert "name_path_pattern" in method_info + + +class TestFacade: + """Tests the indirection between facades and their implementations.""" + + class DummyApi(FacadeApi): + def __init__(self, agent: MagicMock) -> None: + super().__init__(agent, name="dummy", description="a dummy facade") + + def add(self, a: int, b: int) -> int: + """Adds two numbers.""" + return a + b + + def secret(self) -> str: + return "hidden" + + def serena_internal_(self) -> str: + """Public within Serena, but not LLM-facing.""" + return "internal" + + def _internal(self) -> None: + pass + + def test_enabled_methods_delegate_to_implementation(self) -> None: + facade = Facade.from_api(self.DummyApi(MagicMock())) + assert facade.add(1, 2) == 3 + assert "dummy.add(a: int, b: int) -> int" in facade.describe() + assert "Adds two numbers." in facade.describe_method("add") + + def test_disabled_methods_are_inaccessible_and_undocumented(self) -> None: + facade = Facade.from_api(self.DummyApi(MagicMock()), enabled_methods=["add"]) + assert facade.add(1, 2) == 3 + with pytest.raises(AttributeError): + facade.secret() + with pytest.raises(ValueError): + facade.describe_method("secret") + assert "secret" not in facade.describe() + assert "_internal" not in facade.describe() + + def test_trailing_underscore_members_are_not_llm_facing(self) -> None: + facade = Facade.from_api(self.DummyApi(MagicMock())) + assert self.DummyApi(MagicMock()).serena_internal_() == "internal" # usable from within Serena + with pytest.raises(AttributeError): + facade.serena_internal_() + with pytest.raises(ValueError): + facade.get_method("serena_internal_") + assert "serena_internal_" not in facade.describe() + assert "serena_internal_" not in facade.enabled_method_names + + def test_enablement_can_be_changed(self) -> None: + facade = Facade.from_api(self.DummyApi(MagicMock())) + facade.get_method("secret").enabled = False + with pytest.raises(AttributeError): + facade.secret() + facade.get_method("secret").enabled = True + assert facade.secret() == "hidden" + + +@pytest.mark.python +class TestLspFacade: + _SERVICES_FILE = os.path.join("test_repo", "services.py") + + def test_find_symbol_via_repl(self) -> None: + with agent_for_project_context(LanguageServerId.PYTHON) as agent: + tool = agent.get_tool(SerenaReplTool) + + # a returned collection is rendered, identifying the symbol and its file + rendered = tool.apply('return s.lsp.find_symbol("create_user")') + assert "create_user" in rendered + assert "services.py" in rendered + + # the underlying symbols are accessible from code, e.g. to retrieve a body without rendering the collection + body = tool.apply( + f'result = s.lsp.find_symbol("create_user", relative_path={self._SERVICES_FILE!r})\nreturn result.symbols[0].body' + ) + assert body.startswith("def create_user") From eb53a4cee44de933fefbaae1007c72270350e99d Mon Sep 17 00:00:00 2001 From: Dominik Jain Date: Sun, 6 Sep 2026 22:31:10 +0200 Subject: [PATCH 04/53] Add JetBrains facade (jb) covering all JetBrains tools Introduce JetBrainsApi as the second facade implementation, exposing all JetBrains IDE-backed operations to the REPL: find_symbol, find_referencing_symbols, get_symbols_overview, get_type_hierarchy, find_declaration, find_implementations, rename, move, safe_delete, inline_symbol, run_inspections, list_inspections, debug_eval and debug_eval_info (which provides the debug REPL usage information otherwise obtained via the serena_info tool). Result objects carry their rendering policy, following the LSP facade: * JetBrainsSymbolCollection with renderers for symbol searches (grouped JSON, falling back to identifiers) and references (falling back to per-file counts and the total count) * JetBrainsSymbolsOverview with the compact overview format and its shortening ladder * JsonObject, a new general-purpose representable for plain JSON results with length limiting The JetBrains tools now delegate to the API via a common JetBrainsTool base, retaining only transport concerns (input sanitisation, the wildcard-to-overview convenience of the find symbol tool), so that both surfaces share one implementation. The agent adds the facade to the REPL when the JetBrains backend is active. serena_config imported JetBrainsPluginClient through a transitive re-export from jetbrains_tools; it now imports it from its module. --- src/serena/agent.py | 3 + src/serena/config/serena_config.py | 2 +- src/serena/facades/api/jb.py | 611 ++++++++++++++++++++++++++++ src/serena/facades/representable.py | 19 + src/serena/tools/jetbrains_tools.py | 374 +++-------------- test/serena/test_jetbrains_api.py | 79 ++++ test/serena/test_mcp.py | 5 + 7 files changed, 775 insertions(+), 318 deletions(-) create mode 100644 src/serena/facades/api/jb.py create mode 100644 test/serena/test_jetbrains_api.py diff --git a/src/serena/agent.py b/src/serena/agent.py index 2d87f326..8e60fad2 100644 --- a/src/serena/agent.py +++ b/src/serena/agent.py @@ -42,6 +42,7 @@ from serena.config.serena_config import ( ToolInclusionDefinition, ) from serena.dashboard import SerenaDashboardAPI, SerenaDashboardTrayManager, SerenaDashboardViewer, open_url_in_browser +from serena.facades.api.jb import JetBrainsApi from serena.facades.api.lsp import LspApi from serena.facades.facade import Facade from serena.facades.repl import SerenaRepl @@ -1170,6 +1171,8 @@ class SerenaAgent: facades = [] if self._language_backend.is_lsp(): facades.append(Facade.from_api(LspApi(self))) + elif self._language_backend.is_jetbrains(): + facades.append(Facade.from_api(JetBrainsApi(self))) self._repl = SerenaRepl(facades) return self._repl diff --git a/src/serena/config/serena_config.py b/src/serena/config/serena_config.py index bc975d73..befe648e 100644 --- a/src/serena/config/serena_config.py +++ b/src/serena/config/serena_config.py @@ -1442,7 +1442,7 @@ class SerenaConfig(SharedConfig, ModeSelectionDefinitionWithBaseModes): """ Propagate settings from this configuration to individual components that are statically configured """ - from serena.tools import JetBrainsPluginClient + from serena.jetbrains.jetbrains_plugin_client import JetBrainsPluginClient JetBrainsPluginClient.set_server_address(self.jetbrains_plugin_server_address) diff --git a/src/serena/facades/api/jb.py b/src/serena/facades/api/jb.py new file mode 100644 index 00000000..6d6b16bf --- /dev/null +++ b/src/serena/facades/api/jb.py @@ -0,0 +1,611 @@ +# SPDX-License-Identifier: GPL-3.0-or-later +""" +The implementation of JetBrains IDE-backed operations. +""" + +from collections import Counter, defaultdict +from collections.abc import Iterator +from contextlib import contextmanager +from typing import TYPE_CHECKING, Any, Literal + +import serena.jetbrains.jetbrains_types as jb +from serena.code_editor import JetBrainsCodeEditor +from serena.jetbrains.jetbrains_plugin_client import JetBrainsPluginClient +from serena.jetbrains.jetbrains_types import SymbolDTO, SymbolDTOUtil +from serena.symbol import JetBrainsSymbolDictGrouper +from serena.util.text_utils import find_text_coordinates + +from ..facade import FacadeApi +from ..representable import JsonObject, JsonObjectRenderer, Renderer, RepresentableViaRenderer + +if TYPE_CHECKING: + from serena.agent import SerenaAgent + + +class JetBrainsSymbolCollection(RepresentableViaRenderer): + """ + A collection of symbols retrieved via the JetBrains backend. + Each symbol is a dict with keys such as `name_path`, `relative_path` and `type`, and optionally + `children`, `body`, `quick_info`, `documentation` and (for references) `context`. + """ + + def __init__(self, symbols: list[SymbolDTO], renderer: "JetBrainsSymbolCollectionRenderer"): + """ + :param symbols: the symbols + :param renderer: the renderer to use for representing the collection + """ + super().__init__(renderer) + self.symbols = symbols + + def __len__(self) -> int: + return len(self.symbols) + + def relative_paths_(self) -> list[str]: + return [s.get("relative_path", "unknown") for s in self.symbols] + + def identifiers_(self) -> list[SymbolDTO]: + """ + :return: dicts containing only the identifying information (name_path, type, relative_path) of the symbols + """ + return [{"name_path": s["name_path"], "type": s["type"], "relative_path": s["relative_path"]} for s in self.symbols] + + +class JetBrainsSymbolCollectionRenderer(Renderer[JetBrainsSymbolCollection]): + """ + Renders a symbol collection as (optionally grouped) JSON, falling back to a listing of symbol identifiers + if the length limit is exceeded. + """ + + def __init__(self, agent: "SerenaAgent", max_answer_chars: int, grouper: JetBrainsSymbolDictGrouper | None = None): + super().__init__(agent, max_answer_chars) + self._grouper = grouper + + def _group(self, symbols: list[SymbolDTO]) -> Any: + return self._grouper.group(symbols) if self._grouper is not None else symbols + + def render_identifiers(self, obj: JetBrainsSymbolCollection) -> str: + """ + :return: a shortened representation containing symbol types and identifiers (path + name_path) only, without children + """ + return f"Names with paths:\n{self._to_json(self._group(obj.identifiers_()))}" + + def render(self, obj: JetBrainsSymbolCollection) -> str: + result = self._to_json(self._group(obj.symbols)) + return self._limit_length(result, shortened_result_factories=[lambda: self.render_identifiers(obj)]) + + +class JetBrainsReferencesRenderer(JetBrainsSymbolCollectionRenderer): + """ + Renders a collection of referencing symbols, falling back to per-file counts and finally the total count + if the length limit is exceeded. + """ + + def render(self, obj: JetBrainsSymbolCollection) -> str: + ref_paths = obj.relative_paths_() + result = self._to_json(self._group(obj.symbols)) + return self._limit_length( + result, + shortened_result_factories=[ + lambda: f"Reference counts per file:\n{self._to_json(Counter(ref_paths))}", + lambda: f"Found {len(ref_paths)} references.", + ], + ) + + +class JetBrainsSymbolsOverview(RepresentableViaRenderer): + """ + The overview of the symbols defined in a file, i.e. the top-level symbols (each a dict with keys such as + `name_path` and `type`, optionally with `children`) and, if requested, the file's documentation. + """ + + def __init__(self, symbols: list[SymbolDTO], documentation: str | None, renderer: "JetBrainsSymbolsOverviewRenderer"): + """ + :param symbols: the top-level symbols + :param documentation: the file's documentation, if requested and present + :param renderer: the renderer to use for representing the overview + """ + super().__init__(renderer) + self.symbols = symbols + self.documentation = documentation + + +class JetBrainsSymbolsOverviewRenderer(Renderer[JetBrainsSymbolsOverview]): + """ + Renders an overview in the compact grouped format, dropping (in order) the documentation, the children + and finally everything but symbol counts by type if the length limit is exceeded. + """ + + def __init__(self, agent: "SerenaAgent", max_answer_chars: int, grouper: JetBrainsSymbolDictGrouper, depth: int): + super().__init__(agent, max_answer_chars) + self._grouper = grouper + self._depth = depth + + def render(self, obj: JetBrainsSymbolsOverview) -> str: + grouped_symbols = self._grouper.group(obj.symbols) + shortened_result_factories = [] + + # create the full result + result: dict[str, Any] = {"symbols": grouped_symbols} + if obj.documentation: + result["docstring"] = obj.documentation + shortened_result_factories.append(lambda: self._to_json(grouped_symbols)) # shortened result without docstring + json_result = self._to_json(result) + + # create shortened results + if self._depth > 0: + + def create_short_result_depth_0() -> str: + depth_0_symbols = [d.copy() for d in obj.symbols] + for d in depth_0_symbols: + d.pop("children", None) + return "Depth 0 overview:\n" + self._to_json(self._grouper.group(depth_0_symbols)) + + shortened_result_factories.append(create_short_result_depth_0) + + def create_short_result_type_counts() -> str: + type_names = [d.get("type", "unknown") for d in obj.symbols] + return f"Symbol counts by type:\n{self._to_json(Counter(type_names))}" + + shortened_result_factories.append(create_short_result_type_counts) + + return self._limit_length(json_result, shortened_result_factories=shortened_result_factories) + + +class JetBrainsApi(FacadeApi): + # groupers for the various symbol collections; top-level symbols are grouped by the first key list, + # children by the second + find_symbol_grouper_ = JetBrainsSymbolDictGrouper( + ["relative_path", "type"], ["type"], collapse_singleton=True, map_name_path_to_name=True + ) + references_grouper_ = JetBrainsSymbolDictGrouper(["relative_path", "type"], ["type"], collapse_singleton=True) + overview_grouper_ = JetBrainsSymbolDictGrouper(["type"], ["type"], collapse_singleton=True, map_name_path_to_name=True) + + def __init__(self, agent: "SerenaAgent") -> None: + super().__init__( + agent, + name="jb", + description="JetBrains IDE-backed operations (symbol search, references, type hierarchies, refactorings, " + "inspections, debugging)", + ) + + @contextmanager + def _client(self) -> Iterator[JetBrainsPluginClient]: + with JetBrainsPluginClient.from_project(self._get_project()) as client: + yield client + + def _json_object(self, data: Any, max_answer_chars: int = -1) -> JsonObject: + return JsonObject(data, JsonObjectRenderer(self._agent, max_answer_chars)) + + # read operations + + def find_symbol( + self, + name_path_pattern: str, + depth: int = 0, + relative_path: str | None = None, + include_body: bool = False, + include_info: bool = False, + search_deps: bool = False, + max_matches: int = -1, + max_answer_chars: int = -1, + ) -> JetBrainsSymbolCollection: + """ + Finds symbols and code entities (classes, methods, etc.) based on the given name path pattern. + The returned symbol information can be used for edits or further queries. + Specify `depth > 0` to retrieve children (e.g., methods of a class). + Important: through `search_deps=True` dependencies can be searched, which + should be preferred to web search or other less sophisticated approaches to analyzing dependencies. + You will always receive at least quick info for returned symbols (even if `include_info=False`). + + A name path is a path in the symbol tree *within a source file*. + For example, the method `my_method` defined in class `MyClass` would have the name path `MyClass/my_method`. + If a symbol is overloaded (e.g., in Java), a 0-based index is appended (e.g. "MyClass/my_method[0]") to + uniquely identify it. + + To search for a symbol, you provide a name path pattern that is used to match against name paths. + It can be + * a simple name (e.g. "method"), which will match any symbol with that name + * a relative path like "class/method", which will match any symbol with that name path suffix + * an absolute name path "/class/method" (absolute name path), which requires an exact match of the full name path within the source file. + Append an index `[i]` to match a specific overload only, e.g. "MyClass/my_method[1]". + In any path component, using `*` will match any sequence of characters (excluding /), e.g. "Class/*substring*" matches a member substring. + A pattern must not contain only wildcards (e.g. "*" or "/*"); use `get_symbols_overview` to list a file's symbols. + + :param name_path_pattern: the name path matching pattern (see above) + :param depth: depth up to which descendants shall be retrieved (e.g. use 1 to also retrieve immediate children; + for the case where the symbol is a class, this will return its methods). + Ignored if `include_body=True`. Default 0. + :param relative_path: Optional. Restrict search to this file or directory. If not specified, searches entire codebase. + Note: for external dependencies, this must be an identifier starting with `{max_matches=} symbols.\n" + renderer.render_identifiers(collection)) + return collection + + def find_referencing_symbols(self, name_path: str, relative_path: str, max_answer_chars: int = -1) -> JetBrainsSymbolCollection: + """ + Finds all symbols that reference the given symbol — its callers / usages / dependents, i.e. the + symbols whose own definition (e.g. a method body) contains a reference to it. For each, returns its + name path, file, and the surrounding line of code. + + :param name_path: name path of the symbol for which to find references + :param relative_path: the relative path to the file containing the symbol (must be a file, not a directory) + Note: for external dependencies, this must be an identifier starting with `= 0: + content_around_ref = project.retrieve_content_around_line( + relative_file_path=symbol_dict["relative_path"], line=ref_line, context_lines_before=1, context_lines_after=1 + ) + symbol_dict["context"] = content_around_ref.to_display_string() + del symbol_dict["reference_line_no"] + + renderer = JetBrainsReferencesRenderer(self._agent, max_answer_chars, grouper=self.references_grouper_) + return JetBrainsSymbolCollection(symbol_dicts, renderer) + + def get_symbols_overview( + self, relative_path: str, depth: int = -1, max_answer_chars: int = -1, include_file_documentation: bool = False + ) -> JetBrainsSymbolsOverview: + """ + Gets an overview of the top-level symbols defined in the given file (classes, methods, fields) — its + STRUCTURE, without their bodies. This is the cheap, structure-first way to learn what a file + contains: it costs far less context than reading the whole file. + + :param relative_path: the relative path to the file to get the overview of + :param depth: depth up to which descendants shall be retrieved. + Default (-1) results in a language specific choice: 1 for java and kotlin and 0 for other languages + :param max_answer_chars: max characters for the result (-1 for default). If exceeded, no content/a shortened result is returned. + :param include_file_documentation: whether to include the file's docstring. Default False. + :return: the overview + """ + if depth == -1: + depth = 1 if relative_path.endswith((".java", ".kt")) else 0 + + with self._client() as client: + response = client.get_symbols_overview( + relative_path=relative_path, depth=depth, include_file_documentation=include_file_documentation + ) + renderer = JetBrainsSymbolsOverviewRenderer(self._agent, max_answer_chars, grouper=self.overview_grouper_, depth=depth) + return JetBrainsSymbolsOverview(response["symbols"], response.get("documentation"), renderer) + + @staticmethod + def _transform_hierarchy_nodes(nodes: list[jb.TypeHierarchyNodeDTO] | None) -> dict[str, list]: + """ + Transforms a list of hierarchy nodes into a file-grouped compact format. + + :return: a dict where keys are relative paths and values are lists of either a name path (for a leaf node) + or a dict mapping the name path to the (recursively transformed) children + """ + result: defaultdict[str, list] = defaultdict(list) + for node in nodes or []: + symbol = node["symbol"] + name_path = symbol["name_path"] + rel_path = symbol["relative_path"] + children = node.get("children", []) + if children: + result[rel_path].append({name_path: JetBrainsApi._transform_hierarchy_nodes(children)}) + else: + result[rel_path].append(name_path) + return dict(result) + + def get_type_hierarchy( + self, + name_path: str, + relative_path: str, + hierarchy_type: Literal["super", "sub", "both"] = "both", + depth: int | None = 1, + max_answer_chars: int = -1, + ) -> JsonObject: + """ + Gets the type hierarchy of a symbol (supertypes, subtypes, or both). + + :param name_path: name path of the symbol for which to get the type hierarchy. + :param relative_path: the relative path to the file containing the symbol. + :param hierarchy_type: which hierarchy to retrieve: "super" for parent classes/interfaces, + "sub" for subclasses/implementations, or "both" for both directions. Default is "both". + :param depth: depth limit for hierarchy traversal (None or 0 for unlimited). Default is 1. + :param max_answer_chars: max characters for the JSON result. If exceeded, no content is returned. + -1 means the default value from the config will be used. + :return: the file-grouped hierarchy, with keys "supertypes" and/or "subtypes" (and "levels_not_included" + if the depth limit truncated the hierarchy) + """ + result: dict[str, dict | list] = {} + levels_not_included = {} + with self._client() as client: + if hierarchy_type in ("super", "both"): + response = client.get_supertypes(name_path=name_path, relative_path=relative_path, depth=depth) + if "num_levels_not_included" in response: + levels_not_included["supertypes"] = response["num_levels_not_included"] + result["supertypes"] = self._transform_hierarchy_nodes(response.get("hierarchy")) + if hierarchy_type in ("sub", "both"): + response = client.get_subtypes(name_path=name_path, relative_path=relative_path, depth=depth) + if "num_levels_not_included" in response: + levels_not_included["subtypes"] = response["num_levels_not_included"] + result["subtypes"] = self._transform_hierarchy_nodes(response.get("hierarchy")) + if levels_not_included: + result["levels_not_included"] = levels_not_included + return self._json_object(result, max_answer_chars) + + def find_declaration(self, relative_path: str, regex: str, include_body: bool = False) -> JetBrainsSymbolCollection: + r""" + Finds the declaration of a symbol. + + :param relative_path: the relative path to the source file containing the symbol for which to find the declaration. + :param regex: a regular expression with one group, where the group matches the symbol for which to perform the lookup. + For example, to find the declaration of the `process` method in a call like `obj.process()`, + pass an expression like "obj\.(process)\(process_input_arg=37\)". + Prefer regexes with sufficiently large context around the group to render the match unambiguous. + Uses Python syntax with MULTILINE and DOTALL flags enabled. + :param include_body: whether to include the symbol's body in the result. Default False. + :return: the declaring symbol(s) + """ + content = self._get_project().read_file(relative_path) + coords = find_text_coordinates(content, regex, require_unique=True) + assert coords is not None + with self._client() as client: + response = client.find_declaration( + relative_path=relative_path, line=coords.line, col=coords.col, include_quick_info=False, include_body=include_body + ) + return JetBrainsSymbolCollection(response["symbols"], JetBrainsSymbolCollectionRenderer(self._agent, -1)) + + def find_implementations(self, relative_path: str, name_path: str) -> JetBrainsSymbolCollection: + """ + Finds the implementations of a symbol. + + :param relative_path: the relative path to the source file containing the symbol for which to find implementations. + :param name_path: name path of the symbol for which to find implementations + :return: the implementing symbols + """ + with self._client() as client: + response = client.find_implementations(relative_path=relative_path, name_path=name_path, include_quick_info=False) + return JetBrainsSymbolCollection(response["symbols"], JetBrainsSymbolCollectionRenderer(self._agent, -1)) + + # edit operations + + def rename( + self, + relative_path: str, + new_name: str, + name_path: str | None = None, + rename_in_comments: bool = False, + rename_in_text_occurrences: bool = False, + ) -> JsonObject: + """ + Renames a symbol, file or directory throughout the codebase. + Note: renaming in comments/text is on a best-effort basis by the IDE; if the symbol name is non-unique, further + verification is recommended. + + :param relative_path: if `name_path` is passed, the relative path of the file containing the symbol. + Otherwise, the path to the directory or file to rename. + :param new_name: the new name + :param name_path: the name path of the symbol to rename or None if renaming a file or directory. + :param rename_in_comments: whether to also rename occurrences in comments. Default False. + :param rename_in_text_occurrences: whether to also rename occurrences in text. Default False. + :return: the result of the operation + """ + result = JetBrainsCodeEditor(self._get_project()).rename_symbol( + name_path=name_path, + relative_path=relative_path, + new_name=new_name, + rename_in_comments=rename_in_comments, + rename_in_text_occurrences=rename_in_text_occurrences, + ) + return self._json_object(result) + + def move( + self, + relative_path: str, + name_path: str | None = None, + target_relative_path: str | None = None, + target_parent_name_path: str | None = None, + ) -> JsonObject: + """ + Moves a symbol, file or directory to a different location and automatically updates all references to affected symbols. + **Important**: this should always be preferred to naive moving (e.g. via file system operations or edits) + as it is much more reliable and efficient. It is always safe to use. For some symbols, moving may not be applicable, + and will result in no edits and a suitable error message. + The target location is the new parent of the symbol, + i.e. the moved entity is never renamed by the operation, only moved. + + Valid moves: + - Symbol: + * (relative_path, name_path) -> new parent symbol (target_relative_path, target_parent_name_path) + * (relative_path, name_path) -> top level of target file or directory (target_relative_path) + Always consider the concrete language-specific semantics! + - target is a file: valid for languages like Python, where files are modules + - target is a directory: valid for languages like Java, where directories are packages and can contain classes + - File or directory: + * relative_path -> new parent directory (target_relative_path) + + :param relative_path: the relative path to the file containing the symbol to move. + :param name_path: the name path of the symbol to move (empty for moving file or dir). + :param target_relative_path: the relative path of the target directory or file. + :param target_parent_name_path: the name path of the target parent symbol. + :return: the result of the operation + """ + with self._client() as client: + result = client.move( + name_path=name_path or None, + relative_path=relative_path, + target_parent_name_path=target_parent_name_path or None, + target_relative_path=target_relative_path or None, + ) + return self._json_object(result) + + def safe_delete( + self, relative_path: str, name_path: str | None = None, delete_even_if_used: bool = False, propagate: bool = False + ) -> JsonObject: + """ + Safely deletes a symbol, file, or directory, checking for usages first and propagating deletion, if desired. + Propagation means it is possible to request deleting of usages and cleaning up of unused code. + Propagation is powerful for cleaning up code but should be used with care. + **Important**: this should always be preferred to naive deleting (e.g. via file system operations or edits). + When using it, you don't have to search for usages first, as the operation will do it for you. + + :param relative_path: the relative path to the file containing the symbol to delete. + :param name_path: the name path of the symbol to delete. + A name path identifies a symbol within a source file, e.g. "MyClass/my_method". + Omit for deleting a file or directory. + :param delete_even_if_used: whether to force deletion even if the symbol still has usages. + Default is False (safe mode: will report usages instead of deleting). + :param propagate: whether to propagate the deletion to usages of the symbol and also + remove symbols that become unused after the deletion. Default is False. + :return: the result of the operation + """ + with self._client() as client: + result = client.safe_delete( + name_path=name_path or None, relative_path=relative_path, delete_even_if_used=delete_even_if_used, propagate=propagate + ) + return self._json_object(result) + + def inline_symbol(self, name_path: str, relative_path: str, keep_definition: bool = False) -> JsonObject: + """ + Inlines a symbol (usually a method/function, but also classes may be amenable to inlining, + which turns invocation into anonymous class creation), + replacing all call sites with the symbol's body. + **Important**: this should always be preferred to naive inlining (e.g. via searching for references and + editing them). + + :param name_path: the name path of the symbol to inline. + :param relative_path: the relative path to the file containing the symbol to inline. + :param keep_definition: whether to keep the original method definition after inlining all call sites. + May be ignored in some cases (e.g. when inlining a class). + :return: the result of the operation + """ + with self._client() as client: + result = client.inline_symbol(name_path=name_path, relative_path=relative_path, keep_definition=keep_definition) + return self._json_object(result) + + # inspections + + def run_inspections( + self, + relative_path: str, + min_severity: str | None = None, + inspection_names: list[str] | None = None, + start_line: int | None = None, + end_line: int | None = None, + max_answer_chars: int = -1, + ) -> JsonObject: + """ + Runs IDE inspections (code analysis) on the given file and returns the problems found. + This leverages the full power of JetBrains' static analysis engine, including language-specific + inspections, type checking, potential bugs, code style issues, and more. + + :param relative_path: the relative path to the file to inspect. + :param min_severity: minimum severity level to include in results (e.g. "ERROR", "WARNING", "WEAK_WARNING", "INFO"). + If not specified, all severities are returned. + :param inspection_names: optional list of specific inspection names to run (e.g. ["UnusedImport", "TypeMismatch"]). + If not specified, all applicable inspections are run. + :param start_line: optional 1-based start line to restrict the inspection range. + :param end_line: optional 1-based end line to restrict the inspection range. + :param max_answer_chars: max characters for the JSON result. If exceeded, no content is returned. + -1 means the default value from the config will be used. + :return: the inspection results including severity, message, and location. + """ + with self._client() as client: + result = client.run_inspections( + relative_path=relative_path, + min_severity=min_severity, + inspection_names=inspection_names, + start_line=start_line, + end_line=end_line, + ) + return self._json_object(result, max_answer_chars) + + def list_inspections( + self, language: str | None = None, group_path_contains: str | None = None, max_answer_chars: int = -1 + ) -> JsonObject: + """ + Lists the available IDE inspections. Use this to discover which inspections can be passed + to `run_inspections` via `inspection_names`. + + :param language: optional language to filter by (e.g. "Java", "Python", "Kotlin"). + :param group_path_contains: optional substring to match against the inspection group path + (e.g. "probable bugs", "code style"). + :param max_answer_chars: max characters for the JSON result. If exceeded, no content is returned. + -1 means the default value from the config will be used. + :return: the list of available inspections including name, group path, and language. + """ + with self._client() as client: + result = client.list_inspections(language=language, group_path_contains=group_path_contains) + return self._json_object(result, max_answer_chars) + + # debugging + + def debug_eval_info(self) -> str: + """ + Provides usage information for the debug REPL (method `debug_eval`) + + :return: the usage information + """ + return self._agent.prompt_factory.create_info_jet_brains_debug_repl() + + def debug_eval(self, expression: str, repl_key: str = "default") -> str: + """ + Provides debugging functionality (run configs, breakpoints, stepping, inspection, and evaluation) + via a persistent debug REPL connected to the JetBrains IDE. + + Call `debug_eval_info()` first for usage information. + + :param expression: a Groovy/Java expression/statement to evaluate in the REPL. + If empty, closes the REPL with the given key. + :param repl_key: identifier for the REPL instance. State persists across calls with the same key. + :return: the string representation of the result + """ + with self._client() as client: + if expression: + response = client.debug_eval(repl_key=repl_key, expression=expression) + else: + response = client.debug_close(repl_key=repl_key) + return response.get("result", str(response)) diff --git a/src/serena/facades/representable.py b/src/serena/facades/representable.py index 7b9f7f60..8b79e0c8 100644 --- a/src/serena/facades/representable.py +++ b/src/serena/facades/representable.py @@ -74,3 +74,22 @@ class RepresentableViaRenderer(Representable): def represent(self) -> str: return self._renderer.render(self) + + +class JsonObject(RepresentableViaRenderer): + """ + A JSON-serializable result (dict, list, etc.) which is rendered as JSON, subject to length limitation. + """ + + def __init__(self, data: Any, renderer: "JsonObjectRenderer"): + """ + :param data: the JSON-serializable data + :param renderer: the renderer to use for representing the data + """ + super().__init__(renderer) + self.data = data + + +class JsonObjectRenderer(Renderer[JsonObject]): + def render(self, obj: JsonObject) -> str: + return self._limit_length(self._to_json(obj.data)) diff --git a/src/serena/tools/jetbrains_tools.py b/src/serena/tools/jetbrains_tools.py index 6ed276de..fd5344bb 100644 --- a/src/serena/tools/jetbrains_tools.py +++ b/src/serena/tools/jetbrains_tools.py @@ -1,30 +1,28 @@ # SPDX-License-Identifier: GPL-3.0-or-later import logging -from collections import Counter -from typing import Any, Literal +from typing import Literal -import serena.jetbrains.jetbrains_types as jb -from serena.code_editor import JetBrainsCodeEditor -from serena.jetbrains.jetbrains_plugin_client import JetBrainsPluginClient -from serena.jetbrains.jetbrains_types import SymbolDTO, SymbolDTOUtil -from serena.symbol import JetBrainsSymbolDictGrouper +from serena.facades.api.jb import JetBrainsApi from serena.tools import Tool, ToolMarkerBeta, ToolMarkerOptional, ToolMarkerSymbolicEdit, ToolMarkerSymbolicRead -from serena.util.text_utils import find_text_coordinates log = logging.getLogger(__name__) -class JetBrainsFindSymbolTool(Tool, ToolMarkerSymbolicRead, ToolMarkerOptional): +class JetBrainsTool(Tool): + """ + Base class for tools which delegate to the JetBrains API + """ + + def _api(self) -> JetBrainsApi: + return JetBrainsApi(self.agent) + + +class JetBrainsFindSymbolTool(JetBrainsTool, ToolMarkerSymbolicRead, ToolMarkerOptional): """ Performs a global (or local) search for symbols using the JetBrains backend """ - # groups top-level symbols only; children are grouped separately by _group_children_by_type - symbol_dict_grouper = JetBrainsSymbolDictGrouper( - ["relative_path", "type"], ["type"], collapse_singleton=True, map_name_path_to_name=True - ) - def apply( self, name_path_pattern: str, @@ -75,76 +73,40 @@ class JetBrainsFindSymbolTool(Tool, ToolMarkerSymbolicRead, ToolMarkerOptional): :param max_answer_chars: max characters for the result (-1 for default). If exceeded, no content/a shortened result is returned. :return: symbols matching the name. """ - # check input - # - pattern with only wildcards is invalid, but in some cases we delegate to the overview tool - if name_path_pattern.replace("*", "").replace("/", "") == "": - if relative_path: - if self.project.relative_path_exists(relative_path, require_file=True): - overview_tool = self.agent.get_tool(JetBrainsGetSymbolsOverviewTool) - overview_response = overview_tool.apply(relative_path, depth=depth) - return self._wrapped_tool_response( - overview_response, f"Wildcard-only pattern not admitted; used {overview_tool.get_name()} instead" - ) - raise ValueError("name_path_pattern must not be empty or contain only wildcards; consider using the overview tool") - - if include_body: - depth = 0 # ignore user-specified depth if body is requested - name_path_pattern = self._sanitize_input_param(name_path_pattern) - if relative_path: relative_path = self._sanitize_input_param(relative_path) - if relative_path == ".": - relative_path = None - if relative_path is not None and relative_path.startswith(jb.JB_EXTERNAL_FILE_PREFIX): - search_deps = True + # for a wildcard-only pattern restricted to a file, delegate to the overview tool + if name_path_pattern.replace("*", "").replace("/", "") == "" and relative_path: + if self.project.relative_path_exists(relative_path, require_file=True): + overview_tool = self.agent.get_tool(JetBrainsGetSymbolsOverviewTool) + overview_response = overview_tool.apply(relative_path, depth=depth) + return self._wrapped_tool_response( + overview_response, f"Wildcard-only pattern not admitted; used {overview_tool.get_name()} instead" + ) - with JetBrainsPluginClient.from_project(self.project) as client: - if include_body: - include_quick_info = False - include_documentation = False - else: - if include_info: - include_documentation = True - include_quick_info = False - else: - # If no additional information is requested, we still include the quick info (type signature) - include_documentation = False - include_quick_info = True - symbol_collection_response = client.find_symbol( - name_path=name_path_pattern, - relative_path=relative_path, + return ( + self._api() + .find_symbol( + name_path_pattern, depth=depth, + relative_path=relative_path, include_body=include_body, - include_documentation=include_documentation, - include_quick_info=include_quick_info, + include_info=include_info, search_deps=search_deps, + max_matches=max_matches, + max_answer_chars=max_answer_chars, ) - symbols = symbol_collection_response["symbols"] - - def create_shortened_result() -> str: - """Shortened results containing symbol types and identifiers (path + name_path) only, without children""" - dicts: list[SymbolDTO] = [ - {"name_path": s["name_path"], "type": s["type"], "relative_path": s["relative_path"]} for s in symbols - ] - grouped = self.symbol_dict_grouper.group(dicts) - return f"Names with paths:\n{self._to_json(grouped)}" - - n_matches = len(symbols) - if 0 < max_matches < n_matches: - return f"Matched {n_matches}>{max_matches=} symbols.\n" + create_shortened_result() - - grouped_symbols = self.symbol_dict_grouper.group(symbols) - result = self._to_json(grouped_symbols) - return self._limit_length(result, max_answer_chars, shortened_result_factories=[create_shortened_result]) + .represent() + ) @classmethod def get_param_aliases(cls) -> dict[str, str]: return {"name_path": "name_path_pattern"} -class JetBrainsMoveTool(Tool, ToolMarkerSymbolicEdit, ToolMarkerOptional, ToolMarkerBeta): +class JetBrainsMoveTool(JetBrainsTool, ToolMarkerSymbolicEdit, ToolMarkerOptional, ToolMarkerBeta): """ Moves a symbol, file or directory to a new location using the JetBrains backend, updating all references """ @@ -180,21 +142,11 @@ class JetBrainsMoveTool(Tool, ToolMarkerSymbolicEdit, ToolMarkerOptional, ToolMa :param target_relative_path: the relative path of the target directory or file. :param target_parent_name_path: the name path of the target parent symbol. """ - name_path = name_path or None - target_relative_path = target_relative_path or None - target_parent_name_path = target_parent_name_path or None relative_path = self._sanitize_input_param(relative_path) - with JetBrainsPluginClient.from_project(self.project) as client: - response_dict = client.move( - name_path=name_path, - relative_path=relative_path, - target_parent_name_path=target_parent_name_path, - target_relative_path=target_relative_path, - ) - return self._to_json(response_dict) + return self._api().move(relative_path, name_path, target_relative_path, target_parent_name_path).represent() -class JetBrainsSafeDeleteTool(Tool, ToolMarkerSymbolicEdit, ToolMarkerOptional, ToolMarkerBeta): +class JetBrainsSafeDeleteTool(JetBrainsTool, ToolMarkerSymbolicEdit, ToolMarkerOptional, ToolMarkerBeta): """ Safely deletes a symbol using the JetBrains backend, checking for remaining usages first """ @@ -223,18 +175,10 @@ class JetBrainsSafeDeleteTool(Tool, ToolMarkerSymbolicEdit, ToolMarkerOptional, remove symbols that become unused after the deletion. Default is False. """ relative_path = self._sanitize_input_param(relative_path) - name_path = name_path or None - with JetBrainsPluginClient.from_project(self.project) as client: - response_dict = client.safe_delete( - name_path=name_path, - relative_path=relative_path, - delete_even_if_used=delete_even_if_used, - propagate=propagate, - ) - return self._to_json(response_dict) + return self._api().safe_delete(relative_path, name_path, delete_even_if_used, propagate).represent() -class JetBrainsInlineSymbol(Tool, ToolMarkerSymbolicEdit, ToolMarkerOptional, ToolMarkerBeta): +class JetBrainsInlineSymbol(JetBrainsTool, ToolMarkerSymbolicEdit, ToolMarkerOptional, ToolMarkerBeta): """ Inlines a symbol using the JetBrains backend, replacing all call sites with the symbol's body """ @@ -258,22 +202,14 @@ class JetBrainsInlineSymbol(Tool, ToolMarkerSymbolicEdit, ToolMarkerOptional, To May be ignored in some cases (e.g. when inlining a class). """ relative_path = self._sanitize_input_param(relative_path) - with JetBrainsPluginClient.from_project(self.project) as client: - response_dict = client.inline_symbol( - name_path=name_path, - relative_path=relative_path, - keep_definition=keep_definition, - ) - return self._to_json(response_dict) + return self._api().inline_symbol(name_path, relative_path, keep_definition).represent() -class JetBrainsFindReferencingSymbolsTool(Tool, ToolMarkerSymbolicRead, ToolMarkerOptional): +class JetBrainsFindReferencingSymbolsTool(JetBrainsTool, ToolMarkerSymbolicRead, ToolMarkerOptional): """ Finds symbols that reference the given symbol using the JetBrains backend """ - symbol_dict_grouper = JetBrainsSymbolDictGrouper(["relative_path", "type"], ["type"], collapse_singleton=True) - def apply( self, name_path: str, @@ -292,53 +228,14 @@ class JetBrainsFindReferencingSymbolsTool(Tool, ToolMarkerSymbolicRead, ToolMark :param max_answer_chars: max characters for the result (-1 for default). If exceeded, no content/a shortened result is returned. """ relative_path = self._sanitize_input_param(relative_path) - with JetBrainsPluginClient.from_project(self.project) as client: - response_dict = client.find_references( - name_path=name_path, - relative_path=relative_path, - include_quick_info=False, - ) - symbol_dicts = response_dict["symbols"] - - # replace reference line number (if present) by actual line/context - for symbol_dict in symbol_dicts: - if "reference_line_no" in symbol_dict: - ref_line = symbol_dict["reference_line_no"] - ref_relative_path = symbol_dict["relative_path"] - if not SymbolDTOUtil.is_external_symbol(symbol_dict) and ref_line is not None and ref_line >= 0: - content_around_ref = self.project.retrieve_content_around_line( - relative_file_path=ref_relative_path, line=ref_line, context_lines_before=1, context_lines_after=1 - ) - symbol_dict["context"] = content_around_ref.to_display_string() - del symbol_dict["reference_line_no"] - - # capture file paths before grouping - ref_paths = [s.get("relative_path", "unknown") for s in symbol_dicts] - - result = self.symbol_dict_grouper.group(symbol_dicts) - - def create_shortened_result_counts_per_file() -> str: - return f"Reference counts per file:\n{self._to_json(Counter(ref_paths))}" - - def create_shortened_result_num_results() -> str: - return f"Found {len(ref_paths)} references." - - result_json = self._to_json(result) - return self._limit_length( - result_json, - max_answer_chars, - shortened_result_factories=[create_shortened_result_counts_per_file, create_shortened_result_num_results], - ) + return self._api().find_referencing_symbols(name_path, relative_path, max_answer_chars).represent() -class JetBrainsGetSymbolsOverviewTool(Tool, ToolMarkerSymbolicRead, ToolMarkerOptional): +class JetBrainsGetSymbolsOverviewTool(JetBrainsTool, ToolMarkerSymbolicRead, ToolMarkerOptional): """ Retrieves an overview of the top-level symbols within a specified file using the JetBrains backend """ - USE_COMPACT_FORMAT = True - symbol_dict_grouper = JetBrainsSymbolDictGrouper(["type"], ["type"], collapse_singleton=True, map_name_path_to_name=True) - def apply( self, relative_path: str, @@ -357,95 +254,15 @@ class JetBrainsGetSymbolsOverviewTool(Tool, ToolMarkerSymbolicRead, ToolMarkerOp :param max_answer_chars: max characters for the result (-1 for default). If exceeded, no content/a shortened result is returned. :param include_file_documentation: whether to include the file's docstring. Default False. """ - if depth == -1: - if relative_path.endswith((".java", ".kt")): - depth = 1 - else: - depth = 0 - relative_path = self._sanitize_input_param(relative_path) - with JetBrainsPluginClient.from_project(self.project) as client: - symbol_overview = client.get_symbols_overview( - relative_path=relative_path, depth=depth, include_file_documentation=include_file_documentation - ) - - if self.USE_COMPACT_FORMAT: - symbols = symbol_overview["symbols"] - - grouped_symbols = self.symbol_dict_grouper.group(symbols) - - shortened_result_factories = [] - - # create full result - result: dict[str, Any] = {"symbols": grouped_symbols} - documentation = symbol_overview.pop("documentation", None) - if documentation: - result["docstring"] = documentation - shortened_result_factories.append(lambda: self._to_json(grouped_symbols)) # shortened result without docstring - json_result = self._to_json(result) - - if depth > 0: - - def create_short_result_depth_0() -> str: - depth_0_symbols = [d.copy() for d in symbols] - for d in depth_0_symbols: - d.pop("children", None) - compact_depth_0_result = self.symbol_dict_grouper.group(depth_0_symbols) - return "Depth 0 overview:\n" + self._to_json(compact_depth_0_result) - - shortened_result_factories.append(create_short_result_depth_0) - - def create_short_result_type_counts() -> str: - type_names = [d.get("type", "unknown") for d in symbols] - return f"Symbol counts by type:\n{self._to_json(Counter(type_names))}" - - shortened_result_factories.append(create_short_result_type_counts) - else: - # this path is currently abandoned, consider introducing shortened results if ever needed - shortened_result_factories = None - json_result = self._to_json(symbol_overview) - - return self._limit_length(json_result, max_answer_chars, shortened_result_factories=shortened_result_factories) + return self._api().get_symbols_overview(relative_path, depth, max_answer_chars, include_file_documentation).represent() -class JetBrainsTypeHierarchyTool(Tool, ToolMarkerSymbolicRead, ToolMarkerOptional): +class JetBrainsTypeHierarchyTool(JetBrainsTool, ToolMarkerSymbolicRead, ToolMarkerOptional): """ Retrieves the type hierarchy (supertypes and/or subtypes) of a symbol using the JetBrains backend """ - @staticmethod - def _transform_hierarchy_nodes(nodes: list[jb.TypeHierarchyNodeDTO] | None) -> dict[str, list]: - """ - Transform a list of TypeHierarchyNode into a file-grouped compact format. - - Returns a dict where keys are relative_paths and values are lists of either: - - "SymbolNamePath" (leaf node) - - {"SymbolNamePath": {nested_file_grouped_children}} (node with children) - """ - if not nodes: - return {} - - result: dict[str, list] = {} - - for node in nodes: - symbol = node["symbol"] - name_path = symbol["name_path"] - rel_path = symbol["relative_path"] - children = node.get("children", []) - - if rel_path not in result: - result[rel_path] = [] - - if children: - # Node with children - recurse - nested = JetBrainsTypeHierarchyTool._transform_hierarchy_nodes(children) - result[rel_path].append({name_path: nested}) - else: - # Leaf node - result[rel_path].append(name_path) - - return result - def apply( self, name_path: str, @@ -460,51 +277,17 @@ class JetBrainsTypeHierarchyTool(Tool, ToolMarkerSymbolicRead, ToolMarkerOptiona :param name_path: name path of the symbol for which to get the type hierarchy. :param relative_path: the relative path to the file containing the symbol. :param hierarchy_type: which hierarchy to retrieve: "super" for parent classes/interfaces, - "sub" for subclasses/implementations, or "both" for both directions. Default is "sub". + "sub" for subclasses/implementations, or "both" for both directions. Default is "both". :param depth: depth limit for hierarchy traversal (None or 0 for unlimited). Default is 1. :param max_answer_chars: max characters for the JSON result. If exceeded, no content is returned. -1 means the default value from the config will be used. :return: Compact JSON with file-grouped hierarchy. Error string if not applicable. """ relative_path = self._sanitize_input_param(relative_path) - with JetBrainsPluginClient.from_project(self.project) as client: - subtypes = None - supertypes = None - levels_not_included = {} - - if hierarchy_type in ("super", "both"): - supertypes_response = client.get_supertypes( - name_path=name_path, - relative_path=relative_path, - depth=depth, - ) - if "num_levels_not_included" in supertypes_response: - levels_not_included["supertypes"] = supertypes_response["num_levels_not_included"] - supertypes = self._transform_hierarchy_nodes(supertypes_response.get("hierarchy")) - - if hierarchy_type in ("sub", "both"): - subtypes_response = client.get_subtypes( - name_path=name_path, - relative_path=relative_path, - depth=depth, - ) - if "num_levels_not_included" in subtypes_response: - levels_not_included["subtypes"] = subtypes_response["num_levels_not_included"] - subtypes = self._transform_hierarchy_nodes(subtypes_response.get("hierarchy")) - - result_dict: dict[str, dict | list] = {} - if supertypes is not None: - result_dict["supertypes"] = supertypes - if subtypes is not None: - result_dict["subtypes"] = subtypes - if levels_not_included: - result_dict["levels_not_included"] = levels_not_included - - result = self._to_json(result_dict) - return self._limit_length(result, max_answer_chars) + return self._api().get_type_hierarchy(name_path, relative_path, hierarchy_type, depth, max_answer_chars).represent() -class JetBrainsFindDeclarationTool(Tool, ToolMarkerSymbolicRead, ToolMarkerOptional): +class JetBrainsFindDeclarationTool(JetBrainsTool, ToolMarkerSymbolicRead, ToolMarkerOptional): """ Finds the declaration of a symbol using the JetBrains backend """ @@ -523,20 +306,10 @@ class JetBrainsFindDeclarationTool(Tool, ToolMarkerSymbolicRead, ToolMarkerOptio """ relative_path = self._sanitize_input_param(relative_path) regex = self._sanitize_input_param(regex) - - editor = self.create_code_editor() - content = editor.read_file(relative_path) - coords = find_text_coordinates(content, regex, require_unique=True) - assert coords is not None - with JetBrainsPluginClient.from_project(self.project) as client: - symbol_collection = client.find_declaration( - relative_path=relative_path, line=coords.line, col=coords.col, include_quick_info=False, include_body=include_body - ) - result = self._to_json(symbol_collection) - return result + return self._api().find_declaration(relative_path, regex, include_body).represent() -class JetBrainsFindImplementationsTool(Tool, ToolMarkerSymbolicRead, ToolMarkerOptional): +class JetBrainsFindImplementationsTool(JetBrainsTool, ToolMarkerSymbolicRead, ToolMarkerOptional): """ Finds the implementations of a symbol using the JetBrains backend """ @@ -548,17 +321,10 @@ class JetBrainsFindImplementationsTool(Tool, ToolMarkerSymbolicRead, ToolMarkerO :param relative_path: the relative path to the source file containing the symbol for which to find implementations. :param name_path: name path of the symbol for which to find implementations """ - with JetBrainsPluginClient.from_project(self.project) as client: - symbol_collection = client.find_implementations( - relative_path=relative_path, - name_path=name_path, - include_quick_info=False, - ) - result = self._to_json(symbol_collection) - return result + return self._api().find_implementations(relative_path, name_path).represent() -class JetBrainsRenameTool(Tool, ToolMarkerSymbolicEdit, ToolMarkerOptional): +class JetBrainsRenameTool(JetBrainsTool, ToolMarkerSymbolicEdit, ToolMarkerOptional): """ Renames a symbol, file or directory throughout the codebase using the JetBrains backend. """ @@ -584,18 +350,10 @@ class JetBrainsRenameTool(Tool, ToolMarkerSymbolicEdit, ToolMarkerOptional): :param rename_in_text_occurrences: whether to also rename occurrences in text. Default True. :return: a status message """ - code_editor = JetBrainsCodeEditor(self.project) - result = code_editor.rename_symbol( - name_path=name_path, - relative_path=relative_path, - new_name=new_name, - rename_in_comments=rename_in_comments, - rename_in_text_occurrences=rename_in_text_occurrences, - ) - return self._to_json(result) + return self._api().rename(relative_path, new_name, name_path, rename_in_comments, rename_in_text_occurrences).represent() -class JetBrainsDebugTool(Tool, ToolMarkerOptional, ToolMarkerBeta): +class JetBrainsDebugTool(JetBrainsTool, ToolMarkerOptional, ToolMarkerBeta): """ Provides debugging functionality (run configs, breakpoints, stepping, inspection, and evaluation) via a persistent debug REPL connected to the JetBrains IDE. @@ -617,15 +375,10 @@ class JetBrainsDebugTool(Tool, ToolMarkerOptional, ToolMarkerBeta): :param repl_key: identifier for the REPL instance. State persists across calls with the same key. :return: string representation of the result """ - with JetBrainsPluginClient.from_project(self.project) as client: - if expression: - response = client.debug_eval(repl_key=repl_key, expression=expression) - else: - response = client.debug_close(repl_key=repl_key) - return response.get("result", str(response)) + return self._api().debug_eval(expression, repl_key) -class JetBrainsRunInspectionsTool(Tool, ToolMarkerSymbolicRead, ToolMarkerOptional): +class JetBrainsRunInspectionsTool(JetBrainsTool, ToolMarkerSymbolicRead, ToolMarkerOptional): """ Runs JetBrains IDE inspections on a file and returns the results. """ @@ -655,19 +408,12 @@ class JetBrainsRunInspectionsTool(Tool, ToolMarkerSymbolicRead, ToolMarkerOption -1 means the default value from the config will be used. :return: JSON string with inspection results including severity, message, and location. """ - with JetBrainsPluginClient.from_project(self.project) as client: - response_dict = client.run_inspections( - relative_path=relative_path, - min_severity=min_severity, - inspection_names=inspection_names, - start_line=start_line, - end_line=end_line, - ) - result = self._to_json(response_dict) - return self._limit_length(result, max_answer_chars) + return ( + self._api().run_inspections(relative_path, min_severity, inspection_names, start_line, end_line, max_answer_chars).represent() + ) -class JetBrainsListInspectionsTool(Tool, ToolMarkerSymbolicRead, ToolMarkerOptional): +class JetBrainsListInspectionsTool(JetBrainsTool, ToolMarkerSymbolicRead, ToolMarkerOptional): """ Lists available JetBrains IDE inspections, optionally filtered by language or group. """ @@ -689,10 +435,4 @@ class JetBrainsListInspectionsTool(Tool, ToolMarkerSymbolicRead, ToolMarkerOptio -1 means the default value from the config will be used. :return: JSON string with the list of available inspections including name, group path, and language. """ - with JetBrainsPluginClient.from_project(self.project) as client: - response_dict = client.list_inspections( - language=language, - group_path_contains=group_path_contains, - ) - result = self._to_json(response_dict) - return self._limit_length(result, max_answer_chars) + return self._api().list_inspections(language, group_path_contains, max_answer_chars).represent() diff --git a/test/serena/test_jetbrains_api.py b/test/serena/test_jetbrains_api.py new file mode 100644 index 00000000..4643e470 --- /dev/null +++ b/test/serena/test_jetbrains_api.py @@ -0,0 +1,79 @@ +""" +Tests for the JetBrains facade API, using a mocked plugin client (no IDE required). +""" + +from unittest.mock import MagicMock, patch + +import pytest + +from serena.facades.api.jb import JetBrainsApi +from serena.facades.facade import Facade + + +@pytest.fixture +def agent() -> MagicMock: + agent = MagicMock() + agent.serena_config.default_max_tool_answer_chars = 10000 + return agent + + +@pytest.fixture +def client() -> MagicMock: + client = MagicMock() + with patch("serena.facades.api.jb.JetBrainsPluginClient.from_project") as from_project: + from_project.return_value.__enter__.return_value = client + yield client + + +def test_facade_exposes_all_jetbrains_operations(agent: MagicMock) -> None: + facade = Facade.from_api(JetBrainsApi(agent)) + assert facade.name == "jb" + assert set(facade.enabled_method_names) == { + "find_symbol", + "find_referencing_symbols", + "get_symbols_overview", + "get_type_hierarchy", + "find_declaration", + "find_implementations", + "rename", + "move", + "safe_delete", + "inline_symbol", + "run_inspections", + "list_inspections", + "debug_eval", + "debug_eval_info", + } + + +def test_find_symbol_renders_grouped_and_exposes_symbols(agent: MagicMock, client: MagicMock) -> None: + symbols = [ + {"name_path": "Foo", "type": "class", "relative_path": "a.py", "quick_info": "class Foo"}, + {"name_path": "Bar/foo", "type": "method", "relative_path": "b.py", "quick_info": "def foo()"}, + ] + client.find_symbol.return_value = {"symbols": symbols} + + result = JetBrainsApi(agent).find_symbol("foo") + + # the underlying symbols are accessible from code + assert [s["name_path"] for s in result.symbols] == ["Foo", "Bar/foo"] + # and the rendering contains them, grouped by file + rendered = result.represent() + assert '"a.py"' in rendered and '"b.py"' in rendered + assert "class Foo" in rendered + + +def test_find_symbol_rejects_too_many_matches_with_identifiers(agent: MagicMock, client: MagicMock) -> None: + client.find_symbol.return_value = { + "symbols": [{"name_path": f"foo{i}", "type": "function", "relative_path": "a.py", "body": "..."} for i in range(3)] + } + with pytest.raises(ValueError, match="Matched 3>max_matches=1") as exc_info: + JetBrainsApi(agent).find_symbol("foo*", max_matches=1) + assert "foo2" in str(exc_info.value) + assert '"body"' not in str(exc_info.value) # only identifiers, no content + + +def test_find_symbol_rejects_wildcard_only_pattern(agent: MagicMock, client: MagicMock) -> None: + with pytest.raises(ValueError, match="get_symbols_overview"): + JetBrainsApi(agent).find_symbol("*") + client.find_symbol.assert_not_called() diff --git a/test/serena/test_mcp.py b/test/serena/test_mcp.py index d75d0500..ee4a1688 100644 --- a/test/serena/test_mcp.py +++ b/test/serena/test_mcp.py @@ -8,6 +8,7 @@ from serena import __version__ from serena.agent import Tool, ToolRegistry from serena.config.context_mode import SerenaAgentContext from serena.config.serena_config import SerenaConfig +from serena.facades.repl import SerenaRepl from serena.mcp import SerenaMCPFactory make_tool = SerenaMCPFactory.make_mcp_tool @@ -23,6 +24,10 @@ class MockAgent: def get_context() -> SerenaAgentContext: return SerenaAgentContext.load_default() + @staticmethod + def get_repl() -> SerenaRepl: + return SerenaRepl([]) + class BaseMockTool(Tool): """A mock Tool class for testing.""" From 91c2ec8dc22c783a082e01b9e411e4f44c3658fd Mon Sep 17 00:00:00 2001 From: Dominik Jain Date: Mon, 7 Sep 2026 00:00:32 +0200 Subject: [PATCH 05/53] Add remaining LSP operations to the lsp facade; tools delegate to LspApi LspApi now covers all language server-backed operations: restart_language_server, get_symbols_overview, find_symbol, find_referencing_symbols, find_implementations, find_declaration, get_diagnostics_for_file, get_diagnostics_for_symbol, replace_symbol_body, insert_after_symbol, insert_before_symbol, rename_symbol and safe_delete_symbol. Result objects carry their rendering policy: * LspSymbolCollectionRenderer was generalised (symbol_dicts_, child_inclusion_predicate) and is reused by find_implementations * LspSymbolsOverviewRenderer renders a file's overview with the depth-0/kind-count shortening ladder * LspSymbol/LspSymbolRenderer represent a single symbol (find_declaration), preserving the dict output shape * LspReferenceCollection with its renderer (context lines, per-file counts, total count) * LspDiagnostics wrapping GroupedDiagnostics The symbol tools are now thin adapters which delegate to the API via the LspApiMixin (the JetBrains tools use JetBrainsApiMixin analogously, replacing the intermediate tool base class). Editing tools retain the DiagnosticsContext wrapper, which the API does not use; DiagnosticsContext moved to serena.lsp.lsp_diagnostics and is created via EditingToolWithDiagnostics.diagnostics_context. SUCCESS_RESULT moved to serena.facades.facade (the API cannot import from serena.tools without an import cycle); serena.tools re-exports it. The project health check in the CLI uses LspApi directly instead of tool internals. iter_subclasses now yields each class once. --- src/serena/cli.py | 58 +- src/serena/facades/api/lsp.py | 636 ++++++++++++++++-- src/serena/facades/facade.py | 3 + .../lsp_diagnostics.py} | 56 +- src/serena/tools/file_tools.py | 12 +- src/serena/tools/jetbrains_tools.py | 37 +- src/serena/tools/symbol_tools.py | 470 +++---------- src/serena/tools/tools_base.py | 55 +- src/serena/util/inspection.py | 20 +- test/serena/test_serena_agent.py | 6 +- 10 files changed, 837 insertions(+), 516 deletions(-) rename src/serena/{util/ls_diagnostics.py => lsp/lsp_diagnostics.py} (76%) diff --git a/src/serena/cli.py b/src/serena/cli.py index ab41b61a..bd7c4776 100644 --- a/src/serena/cli.py +++ b/src/serena/cli.py @@ -931,8 +931,8 @@ class ProjectCommands(AutoRegisteringGroup): """ # NOTE: completely written by Claude Code, only functionality was reviewed, not implementation from serena.agent import SerenaAgent + from serena.facades.api.lsp import LspApi from serena.project import Project - from serena.tools import FindReferencingSymbolsTool, FindSymbolTool, GetSymbolsOverviewTool logging.configure(level=logging.INFO) project_path = os.path.abspath(project) @@ -977,61 +977,55 @@ class ProjectCommands(AutoRegisteringGroup): if not target_file: raise ProjectCommands._HealthCheckFailure("No analyzable files found") - # Get tools from agent - overview_tool = agent.get_tool(GetSymbolsOverviewTool) - find_symbol_tool = agent.get_tool(FindSymbolTool) - find_refs_tool = agent.get_tool(FindReferencingSymbolsTool) + api = LspApi(agent) - # Test 1: Get symbols overview - log.info("Testing GetSymbolsOverviewTool on file: %s", target_file) - overview_data = agent.execute_task(lambda: overview_tool.get_symbol_overview(target_file)) - log.info(f"GetSymbolsOverviewTool returned: {overview_data}") + # Test 1: symbols overview + log.info("Testing get_symbols_overview on file: %s", target_file) + overview = agent.execute_task(lambda: api.get_symbols_overview(target_file)) + log.info(f"get_symbols_overview returned: {overview.represent()}") - if not overview_data: + if len(overview) == 0: raise ProjectCommands._HealthCheckFailure(f"No symbols found in target file {target_file}") # Extract suitable symbol (prefer class or function over variables) - preferred_kinds = {SymbolKind.Class.name, SymbolKind.Function.name, SymbolKind.Method.name, SymbolKind.Constructor.name} - selected_symbol = None - for symbol in overview_data: - if symbol.get("kind") in preferred_kinds: - selected_symbol = symbol - break + preferred_kinds = {SymbolKind.Class, SymbolKind.Function, SymbolKind.Method, SymbolKind.Constructor} + selected_symbol = next((s for s in overview.symbols if s.symbol_kind in preferred_kinds), None) # If no preferred symbol found, use first available - if not selected_symbol: - selected_symbol = overview_data[0] + if selected_symbol is None: + selected_symbol = overview.symbols[0] log.info("No class or function found, using first available symbol") - symbol_name = selected_symbol["name"] - symbol_kind = selected_symbol["kind"] - log.info("Using symbol for testing: %s (kind: %s)", symbol_name, symbol_kind) + symbol_name = selected_symbol.name + log.info("Using symbol for testing: %s (kind: %s)", symbol_name, selected_symbol.symbol_kind_name) - # Test 2: FindSymbolTool - log.info("Testing FindSymbolTool for symbol: %s", symbol_name) - with find_symbol_tool.symbol_dict_grouper.disabled_context(): + # Test 2: find_symbol + log.info("Testing find_symbol for symbol: %s", symbol_name) + with LspApi.find_symbol_dict_grouper_.disabled_context(): find_symbol_result = agent.execute_task( - lambda: find_symbol_tool.apply(symbol_name, relative_path=target_file, include_body=True) + lambda: api.find_symbol(symbol_name, relative_path=target_file, include_body=True).represent() ) find_symbol_data = json.loads(find_symbol_result) - log.info("FindSymbolTool found %d matches for symbol %s", len(find_symbol_data), symbol_name) + log.info("find_symbol found %d matches for symbol %s", len(find_symbol_data), symbol_name) if not find_symbol_data: raise ProjectCommands._HealthCheckFailure("FindSymbolTool returned no results") - # Test 3: FindReferencingSymbolsTool - log.info("Testing FindReferencingSymbolsTool for symbol: %s", symbol_name) + # Test 3: find_referencing_symbols + log.info("Testing find_referencing_symbols for symbol: %s", symbol_name) try: - with find_refs_tool.symbol_dict_grouper.disabled_context(): - find_refs_result = agent.execute_task(lambda: find_refs_tool.apply(symbol_name, relative_path=target_file)) + with LspApi.references_grouper_.disabled_context(): + find_refs_result = agent.execute_task( + lambda: api.find_referencing_symbols(symbol_name, relative_path=target_file).represent() + ) find_refs_data = json.loads(find_refs_result) - log.info("FindReferencingSymbolsTool found %d references for symbol %s", len(find_refs_data), symbol_name) + log.info("find_referencing_symbols found %d references for symbol %s", len(find_refs_data), symbol_name) except Exception as e: # A symbol with no references at all is a legitimate result, so the number of # references is not asserted - but a *failure* of the reference search means the # language server is not functional, which is the single thing this command is # asked to determine. Logging it as a warning let the command print # "All tools working correctly" and exit 0 after the search had already failed. - raise ProjectCommands._HealthCheckFailure(f"FindReferencingSymbolsTool failed for symbol {symbol_name}: {e}") from e + raise ProjectCommands._HealthCheckFailure(f"find_referencing_symbols failed for symbol {symbol_name}: {e}") from e log.info("Health check completed successfully") diff --git a/src/serena/facades/api/lsp.py b/src/serena/facades/api/lsp.py index 33feb8a7..2eebcad5 100644 --- a/src/serena/facades/api/lsp.py +++ b/src/serena/facades/api/lsp.py @@ -1,15 +1,27 @@ # SPDX-License-Identifier: GPL-3.0-or-later +""" +The implementation of language server (LSP)-backed operations. +""" -from collections import defaultdict -from collections.abc import Sequence +import os +from collections import Counter, defaultdict +from collections.abc import Callable, Sequence from dataclasses import dataclass -from typing import TYPE_CHECKING +from typing import TYPE_CHECKING, Any -from serena.symbol import LanguageServerSymbol, LanguageServerSymbolDictGrouper, LanguageServerSymbolRetriever, SymbolDictGrouper +from serena.code_editor import LanguageServerCodeEditor +from serena.lsp.lsp_diagnostics import GroupedDiagnostics +from serena.symbol import ( + LanguageServerSymbol, + LanguageServerSymbolDictGrouper, + LanguageServerSymbolRetriever, + ReferenceInLanguageServerSymbol, + SymbolDictGrouper, +) +from serena.util.text_utils import TextOutputUtils, find_text_coordinates from solidlsp.lsp_protocol_handler.lsp_types import SymbolKind -from ...util.text_utils import TextOutputUtils -from ..facade import FacadeApi +from ..facade import SUCCESS_RESULT, FacadeApi from ..representable import Renderer, RepresentableViaRenderer if TYPE_CHECKING: @@ -17,6 +29,12 @@ if TYPE_CHECKING: class LspSymbolCollection(RepresentableViaRenderer): + """ + A collection of symbols retrieved via the language server. + Each symbol (`LanguageServerSymbol`) offers e.g. `get_name_path()`, `relative_path`, `symbol_kind_name`, + `body`, `get_body_line_numbers()`, `iter_children()`. + """ + def __init__(self, symbols: list[LanguageServerSymbol], renderer: "LspSymbolCollectionRenderer"): """ :param symbols: the list of symbols @@ -25,7 +43,7 @@ class LspSymbolCollection(RepresentableViaRenderer): super().__init__(renderer) self.symbols = symbols - def __len__(self): + def __len__(self) -> int: return len(self.symbols) def relative_path_to_name_paths_(self) -> dict[str, list[str]]: @@ -35,6 +53,20 @@ class LspSymbolCollection(RepresentableViaRenderer): return result +class LspSymbol(RepresentableViaRenderer): + """ + A single symbol retrieved via the language server (see `LspSymbolCollection` for the symbol's interface). + """ + + def __init__(self, symbol: LanguageServerSymbol, renderer: "LspSymbolRenderer"): + """ + :param symbol: the symbol + :param renderer: the renderer to use for representing the symbol + """ + super().__init__(renderer) + self.symbol = symbol + + @dataclass(kw_only=True) class SymbolOutputParams: name_path: bool = True @@ -49,9 +81,15 @@ class SymbolOutputParams: relative_path: bool = False include_body: bool = False include_info: bool = False + child_inclusion_predicate: Callable[[LanguageServerSymbol], bool] | None = None class LspSymbolCollectionRenderer(Renderer[LspSymbolCollection]): + """ + Renders a symbol collection as (optionally grouped) JSON according to the output parameters, falling back + to a mapping from files to name paths if the length limit is exceeded. + """ + def __init__( self, agent: "SerenaAgent", @@ -65,26 +103,30 @@ class LspSymbolCollectionRenderer(Renderer[LspSymbolCollection]): self._output_params = output_params self._grouper = grouper - def set_grouper(self, grouper: SymbolDictGrouper) -> None: - self._grouper = grouper - - def render(self, obj: LspSymbolCollection) -> str: - symbols = obj.symbols + def symbol_dicts_(self, symbols: list[LanguageServerSymbol]) -> list[LanguageServerSymbol.OutputDict]: + """ + :param symbols: the symbols to convert + :return: the dict representations of the symbols according to the output parameters (including info, if requested) + """ + p = self._output_params symbol_dicts = [ s.to_dict( - kind=self._output_params.kind, - name_path=self._output_params.name_path, - name=self._output_params.name, - relative_path=self._output_params.relative_path, - body_location=self._output_params.body_location, - depth=self._output_params.depth, - body=self._output_params.include_body, - children_name=self._output_params.children_name, - children_name_path=self._output_params.children_name_path, + kind=p.kind, + name_path=p.name_path, + name=p.name, + location=p.location, + relative_path=p.relative_path, + body_location=p.body_location, + depth=p.depth, + body=p.include_body, + children_body=p.children_body, + children_name=p.children_name, + children_name_path=p.children_name_path, + child_inclusion_predicate=p.child_inclusion_predicate, ) for s in symbols ] - if not self._output_params.include_body and self._output_params.include_info: + if not p.include_body and p.include_info: info_by_symbol = self._symbol_retriever.request_info_for_symbol_batch(symbols) for s, s_dict in zip(symbols, symbol_dicts, strict=True): if symbol_info := info_by_symbol.get(s): @@ -92,30 +134,239 @@ class LspSymbolCollectionRenderer(Renderer[LspSymbolCollection]): # https://peps.python.org/pep-0728/ # If we ever upgrade to 3.15, we can remove the type: ignore[typeddict-unknown-key] s_dict["info"] = symbol_info + return symbol_dicts + def _group(self, symbol_dicts: list[LanguageServerSymbol.OutputDict]) -> Any: + return self._grouper.group(symbol_dicts) if self._grouper is not None else symbol_dicts + + def render(self, obj: LspSymbolCollection) -> str: def create_short_result_relative_path_to_name_paths() -> str: - relative_path_to_name_paths = obj.relative_path_to_name_paths_() - return f"Shortened result:\n{TextOutputUtils.to_json(relative_path_to_name_paths)}" + return f"Shortened result:\n{TextOutputUtils.to_json(obj.relative_path_to_name_paths_())}" - if self._grouper is not None: - objects = self._grouper.group(symbol_dicts) - else: - objects = symbol_dicts - result = self._to_json(objects) + result = self._to_json(self._group(self.symbol_dicts_(obj.symbols))) return self._limit_length(result, shortened_result_factories=[create_short_result_relative_path_to_name_paths]) +class LspSymbolRenderer(Renderer[LspSymbol]): + """ + Renders a single symbol as JSON, using a collection renderer for the conversion. + """ + + def __init__(self, agent: "SerenaAgent", max_answer_chars: int, collection_renderer: LspSymbolCollectionRenderer): + super().__init__(agent, max_answer_chars) + self._collection_renderer = collection_renderer + + def render(self, obj: LspSymbol) -> str: + symbol_dict = self._collection_renderer.symbol_dicts_([obj.symbol])[0] + return self._limit_length(self._to_json(symbol_dict)) + + +class LspSymbolsOverviewRenderer(LspSymbolCollectionRenderer): + """ + Renders a file's symbol overview, falling back to a depth-0 overview and finally symbol counts by kind + if the length limit is exceeded. + """ + + def render(self, obj: LspSymbolCollection) -> str: + symbol_dicts = self.symbol_dicts_(obj.symbols) + result = self._to_json(self._group(symbol_dicts)) + + def make_kind_counts() -> str: + kind_names = [d.get("kind", "unknown") for d in symbol_dicts] + return f"Symbol counts by kind:\n{self._to_json(Counter(kind_names))}" + + shortened_results: list[Callable[[], str]] = [make_kind_counts] + if self._output_params.depth > 0: + + def make_depth_0_result() -> str: + depth_0_dicts = [d.copy() for d in symbol_dicts] + for d in depth_0_dicts: + d.pop("children", None) + return "Depth 0 overview:\n" + self._to_json(self._group(depth_0_dicts)) + + shortened_results.insert(0, make_depth_0_result) + + return self._limit_length(result, shortened_result_factories=shortened_results) + + +class LspReferenceCollection(RepresentableViaRenderer): + """ + The references to a symbol, each a `ReferenceInLanguageServerSymbol` with the referencing `symbol` + (a `LanguageServerSymbol`) and the `line` of the reference. + """ + + def __init__(self, references: list[ReferenceInLanguageServerSymbol], renderer: "LspReferenceCollectionRenderer"): + """ + :param references: the references + :param renderer: the renderer to use for representing the collection + """ + super().__init__(renderer) + self.references = references + + def __len__(self) -> int: + return len(self.references) + + +class LspReferenceCollectionRenderer(Renderer[LspReferenceCollection]): + """ + Renders references as grouped JSON including the code around each reference, falling back to + references without code, per-file counts and finally the total count if the length limit is exceeded. + """ + + def __init__(self, agent: "SerenaAgent", max_answer_chars: int, grouper: SymbolDictGrouper): + super().__init__(agent, max_answer_chars) + self._grouper = grouper + + def render(self, obj: LspReferenceCollection) -> str: + project = self._agent.get_active_project_or_raise() + + reference_dicts = [] + ref_summaries = [] + for ref in obj.references: + ref_dict = dict(ref.symbol.to_dict(kind=True, relative_path=True, depth=0, body=False, body_location=True)) + ref_relative_path = ref.symbol.location.relative_path + assert ref_relative_path is not None, f"Referencing symbol {ref.symbol.name} has no relative path, this is likely a bug." + content_around_ref = project.retrieve_content_around_line( + relative_file_path=ref_relative_path, line=ref.line, context_lines_before=1, context_lines_after=1 + ) + ref_dict["content_around_reference"] = content_around_ref.to_display_string() + reference_dicts.append(ref_dict) + ref_summaries.append( + { + "name_path": ref_dict.get("name_path"), + "kind": ref_dict.get("kind"), + "relative_path": ref_dict.get("relative_path"), + "reference_line": ref.line, + } + ) + + result = self._to_json(self._grouper.group(reference_dicts)) + + # shortened result closures, from least to most aggressive shortening + def make_refs_without_context() -> str: + return f"References without surrounding lines:\n{self._to_json(self._grouper.group([dict(s) for s in ref_summaries]))}" + + def make_per_file_counts() -> str: + counts = Counter(str(r["relative_path"]) for r in ref_summaries) + return f"Reference counts per file:\n{self._to_json(counts)}" + + def make_summary() -> str: + return f"Found {len(ref_summaries)} references." + + return self._limit_length(result, shortened_result_factories=[make_refs_without_context, make_per_file_counts, make_summary]) + + +class LspDiagnostics(RepresentableViaRenderer): + """ + Diagnostics grouped as `relative_path -> severity -> name_path -> diagnostics`; see `grouped.get_dict()`. + """ + + def __init__(self, grouped: GroupedDiagnostics, renderer: "LspDiagnosticsRenderer"): + """ + :param grouped: the grouped diagnostics + :param renderer: the renderer to use for representing the diagnostics + """ + super().__init__(renderer) + self.grouped = grouped + + +class LspDiagnosticsRenderer(Renderer[LspDiagnostics]): + def render(self, obj: LspDiagnostics) -> str: + return self._limit_length(self._to_json(obj.grouped.get_dict())) + + class LspApi(FacadeApi): - def __init__(self, agent: "SerenaAgent") -> None: - super().__init__(agent, name="lsp", description="LSP-backed operations on the codebase (finding symbols, etc.)") + FILE_LEVEL_DIAGNOSTIC_BUCKET = "" + """the name path under which diagnostics that cannot be mapped to a symbol are grouped""" - def _create_symbol_retriever(self) -> LanguageServerSymbolRetriever: - assert self._agent.get_language_backend().is_lsp(), "Symbolic read operations require the language server backend" - return LanguageServerSymbolRetriever(self._get_project()) - - # group children by kind, keeping just the name (the parent's name_path makes it unambiguous); + # groupers for the various symbol collections; top-level symbols are grouped by the first key list, + # children by the second. + # For find_symbol, we group children by kind, keeping just the name (the parent's name_path makes it unambiguous); # we don't group the top-level result list because many tests rely on it being a flat list of symbol dicts find_symbol_dict_grouper_ = LanguageServerSymbolDictGrouper([], ["kind"], collapse_singleton=True) + references_grouper_ = LanguageServerSymbolDictGrouper(["relative_path", "kind"], ["kind"], collapse_singleton=True) + overview_grouper_ = LanguageServerSymbolDictGrouper(["kind"], ["kind"], collapse_singleton=True) + + def __init__(self, agent: "SerenaAgent") -> None: + super().__init__( + agent, + name="lsp", + description="language server-backed operations on the codebase (finding symbols, references, implementations, " + "declarations and diagnostics; editing and renaming symbols)", + ) + + def _create_symbol_retriever(self) -> LanguageServerSymbolRetriever: + assert self._agent.get_language_backend().is_lsp(), "Language server operations require the language server backend" + return LanguageServerSymbolRetriever(self._get_project()) + + def _create_code_editor(self, symbol_retriever: LanguageServerSymbolRetriever | None = None) -> LanguageServerCodeEditor: + return LanguageServerCodeEditor(symbol_retriever or self._create_symbol_retriever()) + + @staticmethod + def _parse_kinds(kinds: Sequence[int]) -> Sequence[SymbolKind] | None: + return [SymbolKind(k) for k in kinds] if kinds else None + + def _create_diagnostics(self, grouped: GroupedDiagnostics, max_answer_chars: int) -> LspDiagnostics: + return LspDiagnostics(grouped, LspDiagnosticsRenderer(self._agent, max_answer_chars)) + + # language server management + + def restart_language_server(self) -> str: + """ + Restarts the language server(s). Use this only on explicit user request or after confirmation; + it may be necessary if a language server hangs. + + :return: a success message + """ + self._agent.reset_language_server_manager() + return SUCCESS_RESULT + + # read operations + + def get_symbols_overview(self, relative_path: str, depth: int = -1, max_answer_chars: int = -1) -> LspSymbolCollection: + """ + Gets an overview of the top-level symbols defined in the given file (classes, methods, fields) — its + STRUCTURE, without their bodies. This is the cheap, structure-first way to learn what a file + contains: it costs far less context than reading the whole file. + + :param relative_path: the relative path to the file to get the overview of + :param depth: depth up to which descendants shall be retrieved. + Default (-1) results in a language specific choice: 1 for java and kotlin and 0 for other languages + :param max_answer_chars: max result length; -1 for default. If exceeded, a shortened result is returned. + :return: the top-level symbols of the file + """ + # Note: file system sync not required (relevant file is opened in the language server explicitly) + if depth == -1: + depth = 1 if relative_path.endswith((".java", ".kt")) else 0 + + symbol_retriever = self._create_symbol_retriever() + + # the symbol overview is capable of working with both files and directories, but we require a file + file_path = os.path.join(self._get_project().project_root, relative_path) + if not os.path.exists(file_path): + raise FileNotFoundError(f"File or directory {relative_path} does not exist in the project.") + if os.path.isdir(file_path): + raise ValueError(f"Expected a file path, but got a directory path: {relative_path}. ") + if not symbol_retriever.can_analyze_file(relative_path): + raise ValueError( + f"Cannot extract symbols from file {relative_path}. " + f"Active language servers: {[l.get_key() for l in self._agent.get_active_language_server_ids()]}" + ) + + symbols = symbol_retriever.get_symbol_overview(relative_path)[relative_path] + output_params = SymbolOutputParams( + name_path=False, + name=True, + depth=depth, + kind=True, + relative_path=False, + location=False, + child_inclusion_predicate=lambda s: not s.is_low_level(), + ) + renderer = LspSymbolsOverviewRenderer( + self._agent, max_answer_chars, symbol_retriever, output_params, grouper=self.overview_grouper_ + ) + return LspSymbolCollection(symbols, renderer) def find_symbol( self, @@ -154,31 +405,31 @@ class LspApi(FacadeApi): :param relative_path: (optional) restrict search to this file or directory. If None, searches entire codebase. If a directory is passed, the search will be restricted to the files in that directory. If a file is passed, the search will be restricted to that file. - :param include_body: whether to include the symbol's source code. Use judiciously. + If you have some knowledge about the codebase, you should use this parameter, as it will significantly + speed up the search as well as reduce the number of results. + :param include_body: If True, include the symbol's source code. Use judiciously. :param include_info: whether to include additional info (hover-like, typically including docstring and signature), about the symbol (ignored if include_body is True). Info is never included for child symbols. Note: Depending on the language, this can be slow (e.g., C/C++). :param include_kinds: (optional) limits results to the given LSP symbol kinds (integers) :param exclude_kinds: (optional) list of LSP symbol kinds (integers) to exclude. - :param substring_matching: If True, use substring matching for the last element of the pattern, such that - "Foo/get" would match "Foo/getValue" and "Foo/getData". - :param max_matches: maximum number of permitted matches. If exceeded, a shortened result is returned + :param substring_matching: If True, use substring matching for the last segment of `name_path_pattern` + (i.e. the name of the symbol, e.g. "foo" in "Class/foo" or "my_method" in "my_method"). + :param max_matches: Maximum number of permitted matches. If exceeded, an error containing a shortened result is raised, which allows refining the search. -1 (default) means no limit. Set to 1 to search for a unique symbol. :param max_answer_chars: max result length; -1 for default - :return: collection of matching symbols + :return: the symbols (with locations) matching the name path pattern """ # Note: file system sync not required; the symbol finder opens all relevant source files explicitly in the case of changes if include_body: depth = 0 # ignore user-specified depth if include_body is True assert max_matches != 0, "max_matches must be > 0 or equal to -1." - parsed_include_kinds: Sequence[SymbolKind] | None = [SymbolKind(k) for k in include_kinds] if include_kinds else None - parsed_exclude_kinds: Sequence[SymbolKind] | None = [SymbolKind(k) for k in exclude_kinds] if exclude_kinds else None symbol_retriever = self._create_symbol_retriever() symbols = symbol_retriever.find( name_path_pattern, - include_kinds=parsed_include_kinds, - exclude_kinds=parsed_exclude_kinds, + include_kinds=self._parse_kinds(include_kinds), + exclude_kinds=self._parse_kinds(exclude_kinds), substring_matching=substring_matching, within_relative_path=relative_path, ) @@ -208,3 +459,298 @@ class LspApi(FacadeApi): ) return symbol_collection + + def find_referencing_symbols( + self, + name_path: str, + relative_path: str, + include_kinds: Sequence[int] = (), + exclude_kinds: Sequence[int] = (), + max_answer_chars: int = -1, + ) -> LspReferenceCollection: + """ + Finds references to the symbol at the given `name_path`. The result will contain metadata about the referencing symbols + as well as a short code snippet around the reference. + + :param name_path: name path of the symbol + :param relative_path: the relative path to the file containing the symbol for which to find references. + :param include_kinds: (optional) limits results to the given LSP symbol kinds (integers) + :param exclude_kinds: optional list of LSP symbol kinds (integers) to exclude. + :param max_answer_chars: max result length; -1 for default + :return: the references to the symbol + """ + # file system sync needed for case where symbol finder does not perform a global search, updating everything + if relative_path: + self._get_project().ls_sync_file_system_changes() + + symbol_retriever = self._create_symbol_retriever() + references = symbol_retriever.find_referencing_symbols( + name_path, + relative_file_path=relative_path, + include_body=False, # it is probably never a good idea to include the body of the referencing symbols + include_kinds=self._parse_kinds(include_kinds), + exclude_kinds=self._parse_kinds(exclude_kinds), + ) + return LspReferenceCollection(references, LspReferenceCollectionRenderer(self._agent, max_answer_chars, self.references_grouper_)) + + def find_implementations( + self, + name_path: str, + relative_path: str, + include_info: bool = False, + include_kinds: Sequence[int] = (), + exclude_kinds: Sequence[int] = (), + max_answer_chars: int = -1, + ) -> LspSymbolCollection: + """ + Finds implementations of the symbol at the given `name_path`. + + :param name_path: the symbol's name path + :param relative_path: the relative path to the file containing the symbol for which to find implementations. + Note that here you can't pass a directory but must pass a file. + :param include_info: whether to include additional info (hover-like, typically including docstring and signature), + about the implementing symbols. + :param include_kinds: (optional) limits results to the given LSP symbol kinds (integers) + :param exclude_kinds: (optional) list of LSP symbol kinds (integers) to exclude. + :param max_answer_chars: max result length; -1 for default + :return: the symbols implementing the given symbol + """ + self._get_project().ls_sync_file_system_changes() + + symbol_retriever = self._create_symbol_retriever() + symbols = symbol_retriever.find_implementing_symbols( + name_path, + relative_file_path=relative_path, + include_body=False, + include_kinds=self._parse_kinds(include_kinds), + exclude_kinds=self._parse_kinds(exclude_kinds), + ) + output_params = SymbolOutputParams(kind=True, relative_path=True, body_location=True, include_info=include_info) + return LspSymbolCollection(symbols, LspSymbolCollectionRenderer(self._agent, max_answer_chars, symbol_retriever, output_params)) + + def find_declaration( + self, + relative_path: str, + regex: str, + containing_symbol_name_path: str | None = None, + include_body: bool = False, + include_info: bool = False, + ) -> LspSymbol: + r""" + Finds the declaration of a symbol. + + :param relative_path: the relative path to the source file containing the symbol for which to find the declaration. + :param regex: a regular expression with one group, where the group matches the symbol for which to perform the lookup. + For example, to find the declaration of the `process` method in a call like `obj.process()`, + pass an expression like "obj\.(process)\(process_input_arg=37\)". + Prefer regexes with sufficiently large context around the group to render the match unambiguous. + Uses Python syntax with MULTILINE and DOTALL flags enabled. + :param containing_symbol_name_path: optional name path of a containing symbol whose body shall be searched instead of the full file. + :param include_body: whether to include the symbol's body in the result. Default False. + :param include_info: whether to include additional info (hover-like). Default False. + :return: the declaring symbol + """ + self._get_project().ls_sync_file_system_changes() + symbol_retriever = self._create_symbol_retriever() + + # find relevant location for lookup + editor = self._create_code_editor(symbol_retriever) + if not containing_symbol_name_path: + content = editor.read_file(relative_path) + coords = find_text_coordinates(content, regex, require_unique=True) + assert coords is not None + else: + symbol = symbol_retriever.find_unique(name_path_pattern=containing_symbol_name_path, within_relative_path=relative_path) + body_line_numbers = symbol.get_body_line_numbers_or_raise() + content = editor.read_file(relative_path, lines=body_line_numbers) + coords = find_text_coordinates(content, regex, require_unique=True) + assert coords is not None + coords.line += body_line_numbers[0] + + # retrieve declaration + defining_symbol = symbol_retriever.find_declaration( + relative_file_path=relative_path, line=coords.line, column=coords.col, include_body=include_body + ) + if defining_symbol is None: + raise ValueError( + f"No symbol declaration found at the location of the regex match. Location: {relative_path}:{coords.line}:{coords.col}." + ) + + output_params = SymbolOutputParams( + kind=True, relative_path=True, body_location=True, include_body=include_body, include_info=include_info + ) + collection_renderer = LspSymbolCollectionRenderer(self._agent, -1, symbol_retriever, output_params) + return LspSymbol(defining_symbol, LspSymbolRenderer(self._agent, -1, collection_renderer)) + + def get_diagnostics_for_file( + self, relative_path: str, start_line: int = 0, end_line: int = -1, min_severity: int = 4, max_answer_chars: int = -1 + ) -> LspDiagnostics: + """ + Gets diagnostics for a file. Diagnostics are grouped as `relative_path -> severity -> name_path -> diagnostics_results`. + If a diagnostic cannot be mapped to a symbol, it is grouped under the special name path ``. + + :param relative_path: the relative path to the file to inspect. + :param start_line: the first 0-based line to include. Defaults to 0. + :param end_line: the last 0-based line to include. Defaults to -1, which means until the end of the file. + :param min_severity: minimum LSP severity to include, where 1=Error, 2=Warning, 3=Information, 4=Hint. + Diagnostics with lower-or-equal numeric severity are returned. + :param max_answer_chars: max result length; -1 for default + :return: the grouped diagnostics for the requested file. + """ + self._get_project().ls_sync_file_system_changes() + + symbol_retriever = self._create_symbol_retriever() + diagnostics = symbol_retriever.get_file_diagnostics( + relative_file_path=relative_path, start_line=start_line, end_line=end_line, min_severity=min_severity + ) + + grouped_diagnostics = GroupedDiagnostics() + for diagnostic in diagnostics: + diag_start = diagnostic["range"]["start"] + owner_symbol = symbol_retriever.find_diagnostic_owner_symbol( + relative_file_path=relative_path, line=diag_start["line"], column=diag_start["character"] + ) + name_path = owner_symbol.get_name_path() if owner_symbol is not None else self.FILE_LEVEL_DIAGNOSTIC_BUCKET + grouped_diagnostics.add(relative_path, name_path, diagnostic) + + return self._create_diagnostics(grouped_diagnostics, max_answer_chars) + + def get_diagnostics_for_symbol( + self, + name_path: str, + reference_file: str = "", + check_symbol_references: bool = False, + min_severity: int = 4, + max_answer_chars: int = -1, + ) -> LspDiagnostics: + """ + Gets diagnostics for the specified symbol. When `check_symbol_references` is true, diagnostics for all + referencing symbols are also included. The result is grouped as + `relative_path -> severity -> name_path -> diagnostics_results`. + + :param name_path: the name path of the symbol to inspect. + :param reference_file: optional file path used to disambiguate the symbol search. + :param check_symbol_references: whether to additionally collect diagnostics for symbols that reference the symbol. + :param min_severity: minimum LSP severity to include, where 1=Error, 2=Warning, 3=Information, 4=Hint. + Diagnostics with lower-or-equal numeric severity are returned. + :param max_answer_chars: max result length; -1 for default + :return: the grouped diagnostics for the requested symbol and, optionally, its referencing symbols. + """ + self._get_project().ls_sync_file_system_changes() + + symbol_retriever = self._create_symbol_retriever() + diagnostics_by_symbol = symbol_retriever.get_symbol_diagnostics( + name_path=name_path, + reference_file=reference_file or None, + check_symbol_references=check_symbol_references, + min_severity=min_severity, + ) + + grouped_diagnostics = GroupedDiagnostics() + for symbol, diagnostics in diagnostics_by_symbol.items(): + relative_path = symbol.relative_path + if relative_path is None: + continue + for diagnostic in diagnostics: + grouped_diagnostics.add(relative_path, symbol.get_name_path(), diagnostic) + + return self._create_diagnostics(grouped_diagnostics, max_answer_chars) + + # edit operations + + def replace_symbol_body(self, name_path: str, relative_path: str, body: str) -> str: + """ + Replaces the body of the given symbol. + + IMPORTANT: Only replace symbol bodies if you have previously made a retrieval with include_body=True and thus know what + constitutes the body! + + :param name_path: name path of the symbol whose body to replace + :param relative_path: the relative path to the file containing the symbol + :param body: the new symbol body. The symbol body is the definition of a symbol + in the programming language, including e.g. the signature line for functions. + Depending on the language, it may or may not include a preceding docstring or other preceding annotations. + :return: a success message + """ + self._create_code_editor().replace_body(name_path, relative_file_path=relative_path, body=body) + return SUCCESS_RESULT + + def insert_after_symbol(self, name_path: str, relative_path: str, body: str) -> str: + """ + Inserts code after a class/method/function definition. + Don't use this to insert after assignments (constants, fields). + + :param name_path: name path of the symbol after which to insert content + :param relative_path: the relative path to the file containing the symbol + :param body: the body/content to be inserted. The inserted code shall begin with the next line after + the symbol. + :return: a success message + """ + self._create_code_editor().insert_after_symbol(name_path, relative_file_path=relative_path, body=body) + return SUCCESS_RESULT + + def insert_before_symbol(self, name_path: str, relative_path: str, body: str) -> str: + """ + Inserts the given content before the beginning of the definition of the given symbol (via the symbol's location). + A typical use case is to insert a new class, function, method, field or variable assignment; or + a new import statement before the first symbol in the file. + + :param name_path: name path of the symbol before which to insert content + :param relative_path: the relative path to the file containing the symbol + :param body: the body/content to be inserted before the line in which the referenced symbol is defined + :return: a success message + """ + self._create_code_editor().insert_before_symbol(name_path, relative_file_path=relative_path, body=body) + return SUCCESS_RESULT + + def rename_symbol(self, name_path: str, relative_path: str, new_name: str) -> str: + """ + Renames the symbol with the given `name_path` to `new_name` throughout the entire codebase. + Note: for languages with method overloading, like Java, name_path may have to include a method's + signature to uniquely identify a method. + + :param name_path: name path of the symbol to rename + :param relative_path: the relative path to the file containing the symbol to rename + :param new_name: the new name for the symbol + :return: a result summary indicating success or failure + """ + self._get_project().ls_sync_file_system_changes() + return self._create_code_editor().rename_symbol(name_path, relative_path=relative_path, new_name=new_name) + + def safe_delete_symbol(self, name_path_pattern: str, relative_path: str) -> str: + """ + Deletes the symbol if it is safe to do so (i.e., if there are no references to it) + or returns a list of references to it. + + :param name_path_pattern: name path of the symbol to delete + :param relative_path: the relative path to the file containing the symbol to delete + :return: a success message, or a message listing the references preventing deletion + """ + self._get_project().ls_sync_file_system_changes() + + symbol_retriever = self._create_symbol_retriever() + symbol = symbol_retriever.find_unique(name_path_pattern, substring_matching=False, within_relative_path=relative_path) + symbol_rel_path = symbol.relative_path + assert symbol_rel_path is not None, f"Symbol {name_path_pattern} has no relative path, this is likely a bug." + assert symbol_rel_path == relative_path, f"Symbol {name_path_pattern} is not in the expected relative path {relative_path}." + symbol_name_path = symbol.get_name_path() + + # check for references + symbol_line = symbol.line + symbol_col = symbol.column + assert symbol_line is not None and symbol_col is not None, ( + f"Symbol {name_path_pattern} has no identifier position, this is likely a bug." + ) + lang_server = symbol_retriever.get_language_server(symbol_rel_path) + references_locations = lang_server.request_references(symbol_rel_path, symbol_line, symbol_col) + file_to_lines: dict[str, list[int]] = defaultdict(list) + for ref_loc in references_locations or []: + ref_relative_path = ref_loc.get("relativePath") + if ref_relative_path is None: + continue + file_to_lines[ref_relative_path].append(ref_loc["range"]["start"]["line"]) + if file_to_lines: + return f"Cannot delete, the symbol {symbol_name_path} is referenced in: {TextOutputUtils.to_json(file_to_lines)}" + + self._create_code_editor(symbol_retriever).delete_symbol(symbol_name_path, relative_file_path=symbol_rel_path) + return SUCCESS_RESULT diff --git a/src/serena/facades/facade.py b/src/serena/facades/facade.py index bd8f6b54..4a30598a 100644 --- a/src/serena/facades/facade.py +++ b/src/serena/facades/facade.py @@ -14,6 +14,9 @@ from serena.project import Project if TYPE_CHECKING: from serena.agent import SerenaAgent +SUCCESS_RESULT = "OK" +"""the result returned by operations which have no result other than their success""" + class FacadeApi(ABC): """ diff --git a/src/serena/util/ls_diagnostics.py b/src/serena/lsp/lsp_diagnostics.py similarity index 76% rename from src/serena/util/ls_diagnostics.py rename to src/serena/lsp/lsp_diagnostics.py index c9f80d3f..661c3332 100644 --- a/src/serena/util/ls_diagnostics.py +++ b/src/serena/lsp/lsp_diagnostics.py @@ -3,12 +3,14 @@ import json from collections.abc import Iterable from dataclasses import dataclass -from typing import TYPE_CHECKING, Any +from typing import TYPE_CHECKING, Any, Optional, Self +from serena.util.text_utils import TextOutputUtils from solidlsp import ls_types from solidlsp.lsp_protocol_handler.lsp_types import DiagnosticSeverity if TYPE_CHECKING: + from serena.agent import SerenaAgent from serena.symbol import LanguageServerSymbolRetriever @@ -203,3 +205,55 @@ class DiagnosticsDiff: def get_grouped_diagnostics(self) -> GroupedDiagnostics: return self._grouped_diagnostics + + +class DiagnosticsContext: + ENABLE_DIAGNOSTICS_DEFAULT: bool = False + """ + Global flag to enable/disable diagnostics for LSP-based editing tools derived from this class. + The feature is currently disabled, because per-edit diagnostics are a questionable feature, since individual + edits often intentionally introduce diagnostics (e.g. function signature mismatches or even syntax errors) that + are then resolved in subsequent edits. + """ + + DIAGNOSTICS_KEY = "diagnostics[warning-or-higher]" + + def __init__(self, agent: "SerenaAgent", *edited_relative_paths: str, enable: bool = ENABLE_DIAGNOSTICS_DEFAULT) -> None: + self._is_diagnostics_enabled = enable and agent.is_using_language_server() + self._edited_files = [EditedFilePath(path, path) for path in edited_relative_paths] + self._before_edit_diagnostics_snapshot: PublishedDiagnosticsSnapshot | None = None + self._symbol_retriever: Optional["LanguageServerSymbolRetriever"] | None = None + if self._is_diagnostics_enabled: + from serena.symbol import LanguageServerSymbolRetriever # local import to avoid a circular dependency + + self._symbol_retriever = LanguageServerSymbolRetriever(agent.get_active_project_or_raise()) + self._before_edit_diagnostics_snapshot = PublishedDiagnosticsSnapshot(self._edited_files, self._symbol_retriever) + + def __enter__(self) -> Self: + return self + + def __exit__(self, exc_type, exc_val, exc_tb): + pass + + def format_result( + self, + base_result: str, + ) -> str: + if not self._is_diagnostics_enabled: + return base_result + + if self._before_edit_diagnostics_snapshot is None: + return base_result + + assert self._symbol_retriever is not None + diagnostics_diff = DiagnosticsDiff(self._before_edit_diagnostics_snapshot, self._edited_files, self._symbol_retriever) + grouped_diagnostics = diagnostics_diff.get_grouped_diagnostics().get_dict() + + if not grouped_diagnostics: + return base_result + else: + result_dict = { + "result": base_result, + self.DIAGNOSTICS_KEY: grouped_diagnostics, + } + return TextOutputUtils.to_json(result_dict) diff --git a/src/serena/tools/file_tools.py b/src/serena/tools/file_tools.py index 67c09fe5..70b2fe3e 100644 --- a/src/serena/tools/file_tools.py +++ b/src/serena/tools/file_tools.py @@ -69,7 +69,7 @@ class CreateTextFileTool(EditingToolWithDiagnostics): :param content: the (appropriately encoded) content to write to the file :return: a message indicating success or failure """ - with self.DiagnosticsContext(self, relative_path) as diagnostics_context: + with self.diagnostics_context(relative_path) as diagnostics_context: # validating the destination path project_root = self.get_project_root() abs_path = (Path(project_root) / relative_path).resolve() @@ -206,7 +206,7 @@ class ReplaceContentTool(EditingToolWithDiagnostics): :param allow_multiple_occurrences: whether to allow matching and replacing multiple occurrences. If false and multiple occurrences are found, an error will be returned """ - with self.DiagnosticsContext(self, relative_path) as diagnostics_context: + with self.diagnostics_context(relative_path) as diagnostics_context: self.project.validate_relative_path(relative_path) with EditedFileContext(relative_path, self.create_code_editor()) as context: original_content = context.get_original_content() @@ -427,7 +427,7 @@ class ReplaceInFilesTool(EditingToolWithDiagnostics): occurrences_by_file: dict[str, list[ReplacementOccurrence]] = {} for occ in occurrences: occurrences_by_file.setdefault(occ.relative_path, []).append(occ) - with self.DiagnosticsContext(self, *occurrences_by_file.keys()) as diagnostics_context: + with self.diagnostics_context() as diagnostics_context: code_editor = self.create_code_editor() for path, file_occurrences in occurrences_by_file.items(): with EditedFileContext(path, code_editor) as context: @@ -469,7 +469,7 @@ class DeleteLinesTool(EditingToolWithDiagnostics, ToolMarkerOptional): :param start_line: the 0-based index of the first line to be deleted :param end_line: the 0-based index of the last line to be deleted """ - with self.DiagnosticsContext(self, relative_path) as diagnostics_context: + with self.diagnostics_context(relative_path) as diagnostics_context: code_editor = self.create_code_editor() code_editor.delete_lines(relative_path, start_line, end_line) return diagnostics_context.format_result(SUCCESS_RESULT) @@ -501,7 +501,7 @@ class ReplaceLinesTool(EditingToolWithDiagnostics, ToolMarkerOptional): if not content.endswith("\n"): content += "\n" - with self.DiagnosticsContext(self, relative_path) as diagnostics_context: + with self.diagnostics_context(relative_path) as diagnostics_context: code_editor = self.create_code_editor() code_editor.delete_lines(relative_path, start_line, end_line) code_editor.insert_at_line(relative_path, start_line, content) @@ -534,7 +534,7 @@ class InsertAtLineTool(EditingToolWithDiagnostics, ToolMarkerOptional): if not content.endswith("\n"): content += "\n" - with self.DiagnosticsContext(self, relative_path) as diagnostics_context: + with self.diagnostics_context(relative_path) as diagnostics_context: code_editor = self.create_code_editor() code_editor.insert_at_line(relative_path, line, content) diff --git a/src/serena/tools/jetbrains_tools.py b/src/serena/tools/jetbrains_tools.py index fd5344bb..6615206e 100644 --- a/src/serena/tools/jetbrains_tools.py +++ b/src/serena/tools/jetbrains_tools.py @@ -1,24 +1,29 @@ # SPDX-License-Identifier: GPL-3.0-or-later import logging -from typing import Literal +from typing import TYPE_CHECKING, Literal from serena.facades.api.jb import JetBrainsApi from serena.tools import Tool, ToolMarkerBeta, ToolMarkerOptional, ToolMarkerSymbolicEdit, ToolMarkerSymbolicRead +if TYPE_CHECKING: + from serena.agent import SerenaAgent + log = logging.getLogger(__name__) -class JetBrainsTool(Tool): +class JetBrainsApiMixin: """ - Base class for tools which delegate to the JetBrains API + Mixin for tools which delegate to the JetBrains API """ + agent: "SerenaAgent" + def _api(self) -> JetBrainsApi: return JetBrainsApi(self.agent) -class JetBrainsFindSymbolTool(JetBrainsTool, ToolMarkerSymbolicRead, ToolMarkerOptional): +class JetBrainsFindSymbolTool(Tool, ToolMarkerSymbolicRead, ToolMarkerOptional, JetBrainsApiMixin): """ Performs a global (or local) search for symbols using the JetBrains backend """ @@ -106,7 +111,7 @@ class JetBrainsFindSymbolTool(JetBrainsTool, ToolMarkerSymbolicRead, ToolMarkerO return {"name_path": "name_path_pattern"} -class JetBrainsMoveTool(JetBrainsTool, ToolMarkerSymbolicEdit, ToolMarkerOptional, ToolMarkerBeta): +class JetBrainsMoveTool(Tool, ToolMarkerSymbolicEdit, ToolMarkerOptional, ToolMarkerBeta, JetBrainsApiMixin): """ Moves a symbol, file or directory to a new location using the JetBrains backend, updating all references """ @@ -146,7 +151,7 @@ class JetBrainsMoveTool(JetBrainsTool, ToolMarkerSymbolicEdit, ToolMarkerOptiona return self._api().move(relative_path, name_path, target_relative_path, target_parent_name_path).represent() -class JetBrainsSafeDeleteTool(JetBrainsTool, ToolMarkerSymbolicEdit, ToolMarkerOptional, ToolMarkerBeta): +class JetBrainsSafeDeleteTool(Tool, ToolMarkerSymbolicEdit, ToolMarkerOptional, ToolMarkerBeta, JetBrainsApiMixin): """ Safely deletes a symbol using the JetBrains backend, checking for remaining usages first """ @@ -178,7 +183,7 @@ class JetBrainsSafeDeleteTool(JetBrainsTool, ToolMarkerSymbolicEdit, ToolMarkerO return self._api().safe_delete(relative_path, name_path, delete_even_if_used, propagate).represent() -class JetBrainsInlineSymbol(JetBrainsTool, ToolMarkerSymbolicEdit, ToolMarkerOptional, ToolMarkerBeta): +class JetBrainsInlineSymbol(Tool, ToolMarkerSymbolicEdit, ToolMarkerOptional, ToolMarkerBeta, JetBrainsApiMixin): """ Inlines a symbol using the JetBrains backend, replacing all call sites with the symbol's body """ @@ -205,7 +210,7 @@ class JetBrainsInlineSymbol(JetBrainsTool, ToolMarkerSymbolicEdit, ToolMarkerOpt return self._api().inline_symbol(name_path, relative_path, keep_definition).represent() -class JetBrainsFindReferencingSymbolsTool(JetBrainsTool, ToolMarkerSymbolicRead, ToolMarkerOptional): +class JetBrainsFindReferencingSymbolsTool(Tool, ToolMarkerSymbolicRead, ToolMarkerOptional, JetBrainsApiMixin): """ Finds symbols that reference the given symbol using the JetBrains backend """ @@ -231,7 +236,7 @@ class JetBrainsFindReferencingSymbolsTool(JetBrainsTool, ToolMarkerSymbolicRead, return self._api().find_referencing_symbols(name_path, relative_path, max_answer_chars).represent() -class JetBrainsGetSymbolsOverviewTool(JetBrainsTool, ToolMarkerSymbolicRead, ToolMarkerOptional): +class JetBrainsGetSymbolsOverviewTool(Tool, ToolMarkerSymbolicRead, ToolMarkerOptional, JetBrainsApiMixin): """ Retrieves an overview of the top-level symbols within a specified file using the JetBrains backend """ @@ -258,7 +263,7 @@ class JetBrainsGetSymbolsOverviewTool(JetBrainsTool, ToolMarkerSymbolicRead, Too return self._api().get_symbols_overview(relative_path, depth, max_answer_chars, include_file_documentation).represent() -class JetBrainsTypeHierarchyTool(JetBrainsTool, ToolMarkerSymbolicRead, ToolMarkerOptional): +class JetBrainsTypeHierarchyTool(Tool, ToolMarkerSymbolicRead, ToolMarkerOptional, JetBrainsApiMixin): """ Retrieves the type hierarchy (supertypes and/or subtypes) of a symbol using the JetBrains backend """ @@ -287,7 +292,7 @@ class JetBrainsTypeHierarchyTool(JetBrainsTool, ToolMarkerSymbolicRead, ToolMark return self._api().get_type_hierarchy(name_path, relative_path, hierarchy_type, depth, max_answer_chars).represent() -class JetBrainsFindDeclarationTool(JetBrainsTool, ToolMarkerSymbolicRead, ToolMarkerOptional): +class JetBrainsFindDeclarationTool(Tool, ToolMarkerSymbolicRead, ToolMarkerOptional, JetBrainsApiMixin): """ Finds the declaration of a symbol using the JetBrains backend """ @@ -309,7 +314,7 @@ class JetBrainsFindDeclarationTool(JetBrainsTool, ToolMarkerSymbolicRead, ToolMa return self._api().find_declaration(relative_path, regex, include_body).represent() -class JetBrainsFindImplementationsTool(JetBrainsTool, ToolMarkerSymbolicRead, ToolMarkerOptional): +class JetBrainsFindImplementationsTool(Tool, ToolMarkerSymbolicRead, ToolMarkerOptional, JetBrainsApiMixin): """ Finds the implementations of a symbol using the JetBrains backend """ @@ -324,7 +329,7 @@ class JetBrainsFindImplementationsTool(JetBrainsTool, ToolMarkerSymbolicRead, To return self._api().find_implementations(relative_path, name_path).represent() -class JetBrainsRenameTool(JetBrainsTool, ToolMarkerSymbolicEdit, ToolMarkerOptional): +class JetBrainsRenameTool(Tool, ToolMarkerSymbolicEdit, ToolMarkerOptional, JetBrainsApiMixin): """ Renames a symbol, file or directory throughout the codebase using the JetBrains backend. """ @@ -353,7 +358,7 @@ class JetBrainsRenameTool(JetBrainsTool, ToolMarkerSymbolicEdit, ToolMarkerOptio return self._api().rename(relative_path, new_name, name_path, rename_in_comments, rename_in_text_occurrences).represent() -class JetBrainsDebugTool(JetBrainsTool, ToolMarkerOptional, ToolMarkerBeta): +class JetBrainsDebugTool(Tool, ToolMarkerOptional, ToolMarkerBeta, JetBrainsApiMixin): """ Provides debugging functionality (run configs, breakpoints, stepping, inspection, and evaluation) via a persistent debug REPL connected to the JetBrains IDE. @@ -378,7 +383,7 @@ class JetBrainsDebugTool(JetBrainsTool, ToolMarkerOptional, ToolMarkerBeta): return self._api().debug_eval(expression, repl_key) -class JetBrainsRunInspectionsTool(JetBrainsTool, ToolMarkerSymbolicRead, ToolMarkerOptional): +class JetBrainsRunInspectionsTool(Tool, ToolMarkerSymbolicRead, ToolMarkerOptional, JetBrainsApiMixin): """ Runs JetBrains IDE inspections on a file and returns the results. """ @@ -413,7 +418,7 @@ class JetBrainsRunInspectionsTool(JetBrainsTool, ToolMarkerSymbolicRead, ToolMar ) -class JetBrainsListInspectionsTool(JetBrainsTool, ToolMarkerSymbolicRead, ToolMarkerOptional): +class JetBrainsListInspectionsTool(Tool, ToolMarkerSymbolicRead, ToolMarkerOptional, JetBrainsApiMixin): """ Lists available JetBrains IDE inspections, optionally filtered by language or group. """ diff --git a/src/serena/tools/symbol_tools.py b/src/serena/tools/symbol_tools.py index 24dac396..63115f9f 100644 --- a/src/serena/tools/symbol_tools.py +++ b/src/serena/tools/symbol_tools.py @@ -3,45 +3,46 @@ Language server-related tools """ # SPDX-License-Identifier: GPL-3.0-or-later -import copy -import os -from collections import Counter, defaultdict -from collections.abc import Callable, Sequence -from typing import Any +from typing import TYPE_CHECKING, cast from serena.facades.api.lsp import LspApi -from serena.symbol import LanguageServerSymbol, LanguageServerSymbolDictGrouper from serena.tools import ( - SUCCESS_RESULT, EditingToolWithDiagnostics, Tool, ToolMarkerSymbolicEdit, ToolMarkerSymbolicRead, ) from serena.tools.tools_base import ToolMarkerOptional -from serena.util.ls_diagnostics import GroupedDiagnostics -from serena.util.text_utils import find_text_coordinates -from solidlsp.ls_types import SymbolKind + +if TYPE_CHECKING: + pass -class RestartLanguageServerTool(Tool, ToolMarkerOptional): +class LspApiMixin: + """ + Mixin for tools which delegate to the language server API + """ + + def _api(self) -> LspApi: + tool = cast(Tool, cast(object, self)) + return LspApi(tool.agent) + + +class RestartLanguageServerTool(Tool, ToolMarkerOptional, LspApiMixin): """Restarts the language server(s).""" def apply(self) -> str: """Use this tool only on explicit user request or after confirmation. It may be necessary to restart the language server if it hangs. """ - self.agent.reset_language_server_manager() - return SUCCESS_RESULT + return self._api().restart_language_server() -class GetSymbolsOverviewTool(Tool, ToolMarkerSymbolicRead): +class GetSymbolsOverviewTool(Tool, ToolMarkerSymbolicRead, LspApiMixin): """ Gets an overview of the top-level symbols defined in a given file. """ - symbol_dict_grouper = LanguageServerSymbolDictGrouper(["kind"], ["kind"], collapse_singleton=True) - def apply(self, relative_path: str, depth: int = -1, max_answer_chars: int = -1) -> str: """ Use this tool to get a high-level understanding of the code symbols in a file. @@ -56,96 +57,14 @@ class GetSymbolsOverviewTool(Tool, ToolMarkerSymbolicRead): Don't adjust unless there is really no other way to get the content required for the task. :return: a JSON object containing symbols grouped by kind in a compact format. """ - # Note: file system sync not required (relevant file is opened in the language server explicitly) - - if depth == -1: - if relative_path.endswith((".java", ".kt")): - depth = 1 - else: - depth = 0 - - result = self.get_symbol_overview(relative_path, depth=depth) - - # capture kind names and depth-0 snapshots before grouping, which mutates the dicts - kind_names = [d.get("kind", "unknown") for d in result] - if depth > 0: - depth_0_result = [d.copy() for d in result] - for d in depth_0_result: - d.pop("children", None) - - compact_result = self.symbol_dict_grouper.group(result) - result_json_str = self._to_json(compact_result) - - # shortened result closures - def make_kind_counts() -> str: - return f"Symbol counts by kind:\n{self._to_json(Counter(kind_names))}" - - shortened_results: list[Callable[[], str]] - if depth == 0: - shortened_results = [make_kind_counts] - else: - - def make_depth_0_result() -> str: - compact_depth_0_result = self.symbol_dict_grouper.group(depth_0_result) - return "Depth 0 overview:\n" + self._to_json(compact_depth_0_result) - - shortened_results = [make_depth_0_result, make_kind_counts] - - return self._limit_length(result_json_str, max_answer_chars, shortened_result_factories=shortened_results) - - def get_symbol_overview(self, relative_path: str, depth: int = 0) -> list[LanguageServerSymbol.OutputDict]: - """ - :param relative_path: relative path to a source file - :param depth: the depth up to which descendants shall be retrieved - :return: a list of symbol dictionaries representing the symbol overview of the file - """ - symbol_retriever = self.create_language_server_symbol_retriever() - - # The symbol overview is capable of working with both files and directories, - # but we want to ensure that the user provides a file path. - file_path = os.path.join(self.project.project_root, relative_path) - if not os.path.exists(file_path): - raise FileNotFoundError(f"File or directory {relative_path} does not exist in the project.") - if os.path.isdir(file_path): - raise ValueError(f"Expected a file path, but got a directory path: {relative_path}. ") - if not symbol_retriever.can_analyze_file(relative_path): - raise ValueError( - f"Cannot extract symbols from file {relative_path}. Active language servers: {[l.value for l in self.agent.get_active_language_server_ids()]}" - ) - - symbols = symbol_retriever.get_symbol_overview(relative_path)[relative_path] - - def child_inclusion_predicate(s: LanguageServerSymbol) -> bool: - return not s.is_low_level() - - symbol_dicts = [] - for symbol in symbols: - symbol_dicts.append( - symbol.to_dict( - name_path=False, - name=True, - depth=depth, - kind=True, - relative_path=False, - location=False, - child_inclusion_predicate=child_inclusion_predicate, - ) - ) - return symbol_dicts + return self._api().get_symbols_overview(relative_path, depth=depth, max_answer_chars=max_answer_chars).represent() -class FindSymbolTool(Tool, ToolMarkerSymbolicRead): +class FindSymbolTool(Tool, ToolMarkerSymbolicRead, LspApiMixin): """ Performs a global (or local) search using the language server backend. """ - symbol_dict_grouper = LspApi.find_symbol_dict_grouper_ - """ - Reference to the grouper that is indirectly used by this tool. - Made explicit such that grouping behaviour for this tool can be modified dynamically. - """ - - # noinspection PyDefaultArgument def apply( self, name_path_pattern: str, @@ -183,46 +102,48 @@ class FindSymbolTool(Tool, ToolMarkerSymbolicRead): :param relative_path: (optional) restrict search to this file or directory. If None, searches entire codebase. If a directory is passed, the search will be restricted to the files in that directory. If a file is passed, the search will be restricted to that file. - :param include_body: whether to include the symbol's source code. Use judiciously. + If you have some knowledge about the codebase, you should use this parameter, as it will significantly + speed up the search as well as reduce the number of results. + :param include_body: If True, include the symbol's source code. Use judiciously. :param include_info: whether to include additional info (hover-like, typically including docstring and signature), about the symbol (ignored if include_body is True). Info is never included for child symbols. Note: Depending on the language, this can be slow (e.g., C/C++). :param include_kinds: (optional) limits results to the given LSP symbol kinds (integers) :param exclude_kinds: (optional) list of LSP symbol kinds (integers) to exclude. - :param substring_matching: If True, use substring matching for the last element of the pattern, such that - "Foo/get" would match "Foo/getValue" and "Foo/getData". - :param max_matches: maximum number of permitted matches. If exceeded, a shortened result is returned + :param substring_matching: If True, use substring matching for the last segment of `name_path_pattern` + (i.e. the name of the symbol, e.g. "foo" in "Class/foo" or "my_method" in "my_method"). + :param max_matches: Maximum number of permitted matches. If exceeded, a shortened result is returned which allows refining the search. -1 (default) means no limit. Set to 1 to search for a unique symbol. :param max_answer_chars: max result length; -1 for default :return: symbols (with locations) matching the name. """ - collection = LspApi(self.agent).find_symbol( - name_path_pattern, - depth=depth, - relative_path=relative_path, - include_body=include_body, - include_info=include_info, - include_kinds=include_kinds, - exclude_kinds=exclude_kinds, - substring_matching=substring_matching, - max_matches=max_matches, - max_answer_chars=max_answer_chars, + return ( + self._api() + .find_symbol( + name_path_pattern, + depth=depth, + relative_path=relative_path, + include_body=include_body, + include_info=include_info, + include_kinds=include_kinds, + exclude_kinds=exclude_kinds, + substring_matching=substring_matching, + max_matches=max_matches, + max_answer_chars=max_answer_chars, + ) + .represent() ) - return collection.represent() @classmethod def get_param_aliases(cls) -> dict[str, str]: return {"name_path": "name_path_pattern"} -class FindReferencingSymbolsTool(Tool, ToolMarkerSymbolicRead): +class FindReferencingSymbolsTool(Tool, ToolMarkerSymbolicRead, LspApiMixin): """ - Finds symbols that reference the given symbol using the language server backend + Finds symbols that reference the given symbol """ - symbol_dict_grouper = LanguageServerSymbolDictGrouper(["relative_path", "kind"], ["kind"], collapse_singleton=True) - - # noinspection PyDefaultArgument def apply( self, name_path: str, @@ -242,75 +163,20 @@ class FindReferencingSymbolsTool(Tool, ToolMarkerSymbolicRead): :param max_answer_chars: max result length; -1 for default :return: a list of JSON objects with the symbols referencing the requested symbol """ - # file system sync needed for case where symbol finder does not perform a global search, updating everything - if relative_path: - self.project.ls_sync_file_system_changes() - - include_body = False # It is probably never a good idea to include the body of the referencing symbols - parsed_include_kinds: Sequence[SymbolKind] | None = [SymbolKind(k) for k in include_kinds] if include_kinds else None - parsed_exclude_kinds: Sequence[SymbolKind] | None = [SymbolKind(k) for k in exclude_kinds] if exclude_kinds else None - - symbol_retriever = self.create_language_server_symbol_retriever() - references_in_symbols = symbol_retriever.find_referencing_symbols( - name_path, - relative_file_path=relative_path, - include_body=include_body, - include_kinds=parsed_include_kinds, - exclude_kinds=parsed_exclude_kinds, + return ( + self._api() + .find_referencing_symbols( + name_path, relative_path, include_kinds=include_kinds, exclude_kinds=exclude_kinds, max_answer_chars=max_answer_chars + ) + .represent() ) - reference_dicts = [] - for ref in references_in_symbols: - ref_dict_orig = ref.symbol.to_dict(kind=True, relative_path=True, depth=0, body=include_body, body_location=True) - ref_dict = dict(ref_dict_orig) - if not include_body: - ref_relative_path = ref.symbol.location.relative_path - assert ref_relative_path is not None, f"Referencing symbol {ref.symbol.name} has no relative path, this is likely a bug." - content_around_ref = self.project.retrieve_content_around_line( - relative_file_path=ref_relative_path, line=ref.line, context_lines_before=1, context_lines_after=1 - ) - ref_dict["content_around_reference"] = content_around_ref.to_display_string() - reference_dicts.append(ref_dict) - # capture lightweight reference data before grouping - ref_summaries = [] - for ref, d in zip(references_in_symbols, reference_dicts, strict=True): - ref_summaries.append( - { - "name_path": d.get("name_path"), - "kind": d.get("kind"), - "relative_path": d.get("relative_path"), - "reference_line": ref.line, - } - ) - - result = self.symbol_dict_grouper.group(reference_dicts) - - # shortened result closures, from least to most aggressive shortening - def make_refs_without_context() -> str: - """References with name_path and reference line, without surrounding code lines""" - grouped = self.symbol_dict_grouper.group(copy.deepcopy(ref_summaries)) - return f"References without surrounding lines:\n{self._to_json(grouped)}" - - def make_per_file_counts() -> str: - counts = Counter(str(r["relative_path"]) for r in ref_summaries) - return f"Reference counts per file:\n{self._to_json(counts)}" - - def make_summary() -> str: - return f"Found {len(ref_summaries)} references." - - shortened_results: list[Callable[[], str]] = [make_refs_without_context, make_per_file_counts, make_summary] - - result_json = self._to_json(result) - return self._limit_length(result_json, max_answer_chars, shortened_result_factories=shortened_results) - - -class FindImplementationsTool(Tool, ToolMarkerSymbolicRead): +class FindImplementationsTool(Tool, ToolMarkerSymbolicRead, LspApiMixin): """ - Finds symbols that implement the given symbol using the language server backend. + Finds the implementations of a symbol """ - # noinspection PyDefaultArgument def apply( self, name_path: str, @@ -333,36 +199,21 @@ class FindImplementationsTool(Tool, ToolMarkerSymbolicRead): :param max_answer_chars: max result length; -1 for default :return: a list of JSON objects with the symbols implementing the requested symbol """ - self.project.ls_sync_file_system_changes() - - include_body = False - parsed_include_kinds: Sequence[SymbolKind] | None = [SymbolKind(k) for k in include_kinds] if include_kinds else None - parsed_exclude_kinds: Sequence[SymbolKind] | None = [SymbolKind(k) for k in exclude_kinds] if exclude_kinds else None - symbol_retriever = self.create_language_server_symbol_retriever() - - implementing_symbols = symbol_retriever.find_implementing_symbols( - name_path, - relative_file_path=relative_path, - include_body=include_body, - include_kinds=parsed_include_kinds, - exclude_kinds=parsed_exclude_kinds, + return ( + self._api() + .find_implementations( + name_path, + relative_path, + include_info=include_info, + include_kinds=include_kinds, + exclude_kinds=exclude_kinds, + max_answer_chars=max_answer_chars, + ) + .represent() ) - symbol_dicts = [ - dict(s.to_dict(kind=True, relative_path=True, depth=0, body=include_body, body_location=True)) for s in implementing_symbols - ] - if include_info: - info_by_symbol = symbol_retriever.request_info_for_symbol_batch(implementing_symbols) - for s, s_dict in zip(implementing_symbols, symbol_dicts, strict=True): - if symbol_info := info_by_symbol.get(s): - s_dict["info"] = symbol_info - s_dict.pop("name", None) # name is included in the info - result = self._to_json(symbol_dicts) - return self._limit_length(result, max_answer_chars) - - -class FindDeclarationTool(Tool, ToolMarkerSymbolicRead): +class FindDeclarationTool(Tool, ToolMarkerSymbolicRead, LspApiMixin): """ Finds the declaration/definition of a symbol """ @@ -388,70 +239,26 @@ class FindDeclarationTool(Tool, ToolMarkerSymbolicRead): :param include_body: whether to include the symbol's body in the result. Default False. :param include_info: whether to include additional info (hover-like). Default False. """ - self.project.ls_sync_file_system_changes() - - symbol_retriever = self.create_language_server_symbol_retriever() relative_path = self._sanitize_input_param(relative_path) regex = self._sanitize_input_param(regex) - - # find relevant location for lookup - editor = self.create_code_editor() - if not containing_symbol_name_path: - content = editor.read_file(relative_path) - coords = find_text_coordinates(content, regex, require_unique=True) - assert coords is not None - else: - symbol = symbol_retriever.find_unique(name_path_pattern=containing_symbol_name_path, within_relative_path=relative_path) - body_line_numers = symbol.get_body_line_numbers_or_raise() - content = editor.read_file(relative_path, lines=body_line_numers) - coords = find_text_coordinates(content, regex, require_unique=True) - assert coords is not None - coords.line += body_line_numers[0] - - # retrieve declaration - defining_symbol = symbol_retriever.find_declaration( - relative_file_path=relative_path, - line=coords.line, - column=coords.col, - include_body=include_body, - ) - if defining_symbol is None: - raise ValueError( - f"No symbol declaration found at the location of the regex match. Location: {relative_path}:{coords.line}:{coords.col}." + return ( + self._api() + .find_declaration( + relative_path, + regex, + containing_symbol_name_path=containing_symbol_name_path, + include_body=include_body, + include_info=include_info, ) - - # create output - symbol_dict = self._defining_symbol_to_result_dict( - symbol_retriever, - defining_symbol, - include_body, - include_info, + .represent() ) - result = self._to_json(symbol_dict) - return result - - @staticmethod - def _defining_symbol_to_result_dict( - symbol_retriever: Any, - defining_symbol: LanguageServerSymbol, - include_body: bool, - include_info: bool, - ) -> dict[str, Any]: - symbol_dict = dict(defining_symbol.to_dict(kind=True, relative_path=True, depth=0, body=include_body, body_location=True)) - if not include_body and include_info: - if symbol_info := symbol_retriever.request_info_for_symbol(defining_symbol): - symbol_dict["info"] = symbol_info - symbol_dict.pop("name", None) - return symbol_dict -class GetDiagnosticsForFileTool(Tool, ToolMarkerSymbolicRead): +class GetDiagnosticsForFileTool(Tool, ToolMarkerSymbolicRead, LspApiMixin): """ - Gets diagnostics for a file, optionally restricted to a line range, grouped by file, severity, and containing symbol. + Gets diagnostics for a file, grouped by symbol. """ - FILE_LEVEL_DIAGNOSTIC_BUCKET = "" - def apply( self, relative_path: str, @@ -472,34 +279,16 @@ class GetDiagnosticsForFileTool(Tool, ToolMarkerSymbolicRead): :param max_answer_chars: max result length; -1 for default :return: grouped diagnostics for the requested file. """ - self.project.ls_sync_file_system_changes() - - symbol_retriever = self.create_language_server_symbol_retriever() - diagnostics = symbol_retriever.get_file_diagnostics( - relative_file_path=relative_path, - start_line=start_line, - end_line=end_line, - min_severity=min_severity, + return ( + self._api() + .get_diagnostics_for_file( + relative_path, start_line=start_line, end_line=end_line, min_severity=min_severity, max_answer_chars=max_answer_chars + ) + .represent() ) - grouped_diagnostics = GroupedDiagnostics() - for diagnostic in diagnostics: - diag_range = diagnostic["range"]["start"] - name_path = self.FILE_LEVEL_DIAGNOSTIC_BUCKET - owner_symbol = symbol_retriever.find_diagnostic_owner_symbol( - relative_file_path=relative_path, - line=diag_range["line"], - column=diag_range["character"], - ) - if owner_symbol is not None: - name_path = owner_symbol.get_name_path() - grouped_diagnostics.add(relative_path, name_path, diagnostic) - result = self._to_json(grouped_diagnostics.get_dict()) - return self._limit_length(result, max_answer_chars) - - -class GetDiagnosticsForSymbolTool(Tool, ToolMarkerSymbolicRead, ToolMarkerOptional): +class GetDiagnosticsForSymbolTool(Tool, ToolMarkerSymbolicRead, ToolMarkerOptional, LspApiMixin): """ Gets diagnostics for a symbol and, optionally, for symbols that reference it. """ @@ -525,30 +314,20 @@ class GetDiagnosticsForSymbolTool(Tool, ToolMarkerSymbolicRead, ToolMarkerOption :param max_answer_chars: max result length; -1 for default :return: grouped diagnostics for the requested symbol and, optionally, its referencing symbols. """ - self.project.ls_sync_file_system_changes() - - symbol_retriever = self.create_language_server_symbol_retriever() - diagnostics_by_symbol = symbol_retriever.get_symbol_diagnostics( - name_path=name_path, - reference_file=reference_file or None, - check_symbol_references=check_symbol_references, - min_severity=min_severity, + return ( + self._api() + .get_diagnostics_for_symbol( + name_path, + reference_file=reference_file, + check_symbol_references=check_symbol_references, + min_severity=min_severity, + max_answer_chars=max_answer_chars, + ) + .represent() ) - grouped_diagnostics = GroupedDiagnostics() - for symbol, diagnostics in diagnostics_by_symbol.items(): - relative_path = symbol.relative_path - if relative_path is None: - continue - symbol_name_path = symbol.get_name_path() - for diagnostic in diagnostics: - grouped_diagnostics.add(relative_path, symbol_name_path, diagnostic) - result = self._to_json(grouped_diagnostics.get_dict()) - return self._limit_length(result, max_answer_chars) - - -class ReplaceSymbolBodyTool(EditingToolWithDiagnostics): +class ReplaceSymbolBodyTool(EditingToolWithDiagnostics, LspApiMixin): """ Replaces the full definition of a symbol using the language server backend. """ @@ -571,17 +350,12 @@ class ReplaceSymbolBodyTool(EditingToolWithDiagnostics): in the programming language, including e.g. the signature line for functions. Depending on the language, it may or may not include a preceding docstring or other preceding annotations. """ - with self.DiagnosticsContext(self, relative_path) as diagnostics_context: - code_editor = self.create_code_editor() - code_editor.replace_body( - name_path, - relative_file_path=relative_path, - body=body, - ) - return diagnostics_context.format_result(SUCCESS_RESULT) + with self.diagnostics_context(relative_path) as diagnostics_context: + result = self._api().replace_symbol_body(name_path, relative_path, body) + return diagnostics_context.format_result(result) -class InsertAfterSymbolTool(EditingToolWithDiagnostics): +class InsertAfterSymbolTool(EditingToolWithDiagnostics, LspApiMixin): """ Inserts content after the end of the definition of a given symbol. """ @@ -601,13 +375,12 @@ class InsertAfterSymbolTool(EditingToolWithDiagnostics): :param body: the body/content to be inserted. The inserted code shall begin with the next line after the symbol. """ - with self.DiagnosticsContext(self, relative_path) as diagnostics_context: - code_editor = self.create_code_editor() - code_editor.insert_after_symbol(name_path, relative_file_path=relative_path, body=body) - return diagnostics_context.format_result(SUCCESS_RESULT) + with self.diagnostics_context(relative_path) as diagnostics_context: + result = self._api().insert_after_symbol(name_path, relative_path, body) + return diagnostics_context.format_result(result) -class InsertBeforeSymbolTool(EditingToolWithDiagnostics): +class InsertBeforeSymbolTool(EditingToolWithDiagnostics, LspApiMixin): """ Inserts content before the beginning of the definition of a given symbol. """ @@ -627,13 +400,12 @@ class InsertBeforeSymbolTool(EditingToolWithDiagnostics): :param relative_path: the relative path to the file containing the symbol :param body: the body/content to be inserted before the line in which the referenced symbol is defined """ - with self.DiagnosticsContext(self, relative_path) as diagnostics_context: - code_editor = self.create_code_editor() - code_editor.insert_before_symbol(name_path, relative_file_path=relative_path, body=body) - return diagnostics_context.format_result(SUCCESS_RESULT) + with self.diagnostics_context(relative_path) as diagnostics_context: + result = self._api().insert_before_symbol(name_path, relative_path, body) + return diagnostics_context.format_result(result) -class RenameSymbolTool(Tool, ToolMarkerSymbolicEdit): +class RenameSymbolTool(Tool, ToolMarkerSymbolicEdit, LspApiMixin): """ Renames a symbol throughout the codebase using language server refactoring capabilities. For JB, we use a separate tool. @@ -655,13 +427,10 @@ class RenameSymbolTool(Tool, ToolMarkerSymbolicEdit): :param new_name: the new name for the symbol :return: result summary indicating success or failure """ - self.project.ls_sync_file_system_changes() - code_editor = self.create_ls_code_editor() - status_message = code_editor.rename_symbol(name_path, relative_path=relative_path, new_name=new_name) - return status_message + return self._api().rename_symbol(name_path, relative_path, new_name) -class SafeDeleteSymbol(Tool, ToolMarkerSymbolicEdit): +class SafeDeleteSymbol(Tool, ToolMarkerSymbolicEdit, LspApiMixin): def apply( self, name_path_pattern: str, @@ -674,31 +443,4 @@ class SafeDeleteSymbol(Tool, ToolMarkerSymbolicEdit): :param name_path_pattern: name path of the symbol to delete :param relative_path: the relative path to the file containing the symbol to delete """ - self.project.ls_sync_file_system_changes() - - ls_symbol_retriever = self.create_language_server_symbol_retriever() - symbol = ls_symbol_retriever.find_unique(name_path_pattern, substring_matching=False, within_relative_path=relative_path) - symbol_rel_path = symbol.relative_path - assert symbol_rel_path is not None, f"Symbol {name_path_pattern} has no relative path, this is likely a bug." - assert symbol_rel_path == relative_path, f"Symbol {name_path_pattern} is not in the expected relative path {relative_path}." - symbol_name_path = symbol.get_name_path() - - symbol_line = symbol.line - symbol_col = symbol.column - assert symbol_line is not None and symbol_col is not None, ( - f"Symbol {name_path_pattern} has no identifier position, this is likely a bug." - ) - lang_server = ls_symbol_retriever.get_language_server(symbol_rel_path) - references_locations = lang_server.request_references(symbol_rel_path, symbol_line, symbol_col) - file_to_lines: dict[str, list[int]] = defaultdict(list) - if references_locations: - for ref_loc in references_locations: - ref_relative_path = ref_loc.get("relativePath") - if ref_relative_path is None: - continue - file_to_lines[ref_relative_path].append(ref_loc["range"]["start"]["line"]) - if file_to_lines: - return f"Cannot delete, the symbol {symbol_name_path} is referenced in: {self._to_json(file_to_lines)}" - code_editor = self.create_ls_code_editor() - code_editor.delete_symbol(symbol_name_path, relative_file_path=symbol_rel_path) - return SUCCESS_RESULT + return self._api().safe_delete_symbol(name_path_pattern, relative_path) diff --git a/src/serena/tools/tools_base.py b/src/serena/tools/tools_base.py index 2009727b..bcdbf1bb 100644 --- a/src/serena/tools/tools_base.py +++ b/src/serena/tools/tools_base.py @@ -7,7 +7,7 @@ from collections.abc import Callable, Iterable from dataclasses import dataclass from functools import cached_property from types import TracebackType -from typing import TYPE_CHECKING, Any, Optional, Protocol, Self, TypeVar, cast +from typing import TYPE_CHECKING, Any, Protocol, Self, TypeVar, cast from mcp import Implementation from mcp.server.fastmcp import Context @@ -16,12 +16,13 @@ from sensai.util import logging from sensai.util.string import dict_string from serena.config.serena_config import LanguageBackend +from serena.facades.facade import SUCCESS_RESULT # noqa: F401 (re-exported for tools) +from serena.lsp.lsp_diagnostics import DiagnosticsContext from serena.memories.memory_manager import MemoryManager from serena.project import Project from serena.prompt_factory import PromptFactory from serena.util.class_decorators import singleton from serena.util.inspection import iter_subclasses -from serena.util.ls_diagnostics import DiagnosticsDiff, EditedFilePath, PublishedDiagnosticsSnapshot from serena.util.text_utils import TextOutputUtils from solidlsp.ls_exceptions import SolidLSPException @@ -32,7 +33,6 @@ if TYPE_CHECKING: log = logging.getLogger(__name__) T = TypeVar("T") -SUCCESS_RESULT = "OK" class Component(ABC): @@ -467,47 +467,16 @@ class EditingToolWithDiagnostics(Tool, ToolMarkerCanEdit): are then resolved in subsequent edits. """ - DIAGNOSTICS_KEY = "diagnostics[warning-or-higher]" + def diagnostics_context(self, *edited_relative_paths: str) -> DiagnosticsContext: + """ + Creates a context for use with the `with` statement, which captures the diagnostics before the edit, + such that changes can be reported - class DiagnosticsContext: - def __init__(self, tool: "EditingToolWithDiagnostics", *edited_relative_paths: str) -> None: - self._tool = tool - self._is_diagnostics_enabled = tool.ENABLE_DIAGNOSTICS and tool.agent.is_using_language_server() - self._edited_files = [EditedFilePath(path, path) for path in edited_relative_paths] - self._before_edit_diagnostics_snapshot: PublishedDiagnosticsSnapshot | None = None - self._symbol_retriever: Optional["LanguageServerSymbolRetriever"] | None = None - if self._is_diagnostics_enabled: - self._symbol_retriever = tool.create_language_server_symbol_retriever() - self._before_edit_diagnostics_snapshot = PublishedDiagnosticsSnapshot(self._edited_files, self._symbol_retriever) - - def __enter__(self) -> Self: - return self - - def __exit__(self, exc_type, exc_val, exc_tb): - pass - - def format_result( - self, - base_result: str, - ) -> str: - if not self._is_diagnostics_enabled: - return base_result - - if self._before_edit_diagnostics_snapshot is None: - return base_result - - assert self._symbol_retriever is not None - diagnostics_diff = DiagnosticsDiff(self._before_edit_diagnostics_snapshot, self._edited_files, self._symbol_retriever) - grouped_diagnostics = diagnostics_diff.get_grouped_diagnostics().get_dict() - - if not grouped_diagnostics: - return base_result - else: - result_dict = { - "result": base_result, - EditingToolWithDiagnostics.DIAGNOSTICS_KEY: grouped_diagnostics, - } - return self._tool._to_json(result_dict) + :param edited_relative_paths: the relative paths of the files that are to be edited within the context + :return: a context which captures the diagnostics before the edit, such that changes can be reported + via `format_result` + """ + return DiagnosticsContext(self.agent, *edited_relative_paths, enable=self.ENABLE_DIAGNOSTICS) class EditedFileContext: diff --git a/src/serena/util/inspection.py b/src/serena/util/inspection.py index 5ab17aa7..8bc8a328 100644 --- a/src/serena/util/inspection.py +++ b/src/serena/util/inspection.py @@ -16,17 +16,25 @@ log = logging.getLogger(__name__) def iter_subclasses( cls: type[T], recursive: bool = True, inclusion_predicate: Callable[[type[T]], bool] = lambda t: True ) -> Iterator[type[T]]: - """Iterate over all subclasses of a class. + """Iterate over all subclasses of a class, yielding each subclass once (even if it is reachable via multiple base classes). :param cls: The class whose subclasses to iterate over. :param recursive: If True, also iterate over all subclasses of all subclasses. :param inclusion_predicate: a predicate function to decide whether to include a subclass in the result """ - for subclass in cls.__subclasses__(): - if inclusion_predicate(subclass): - yield subclass - if recursive: - yield from iter_subclasses(subclass, recursive, inclusion_predicate) + seen: set[type] = set() + + def iterate(c: type[T]) -> Iterator[type[T]]: + for subclass in c.__subclasses__(): + if subclass in seen: + continue + seen.add(subclass) + if inclusion_predicate(subclass): + yield subclass + if recursive: + yield from iterate(subclass) + + yield from iterate(cls) def compute_language_server_support_composition( diff --git a/test/serena/test_serena_agent.py b/test/serena/test_serena_agent.py index cd7cbb6d..8fb5002b 100644 --- a/test/serena/test_serena_agent.py +++ b/test/serena/test_serena_agent.py @@ -15,11 +15,11 @@ from _pytest.mark import Mark, MarkDecorator, ParameterSet from serena.agent import SerenaAgent from serena.config.context_mode import SerenaAgentContext from serena.config.serena_config import ProjectConfig, RegisteredProject, SerenaConfig +from serena.lsp.lsp_diagnostics import DiagnosticsContext from serena.project import Project from serena.tools import ( SUCCESS_RESULT, ActivateProjectTool, - EditingToolWithDiagnostics, FindDeclarationTool, FindImplementationsTool, FindReferencingSymbolsTool, @@ -824,9 +824,9 @@ def read_project_file(project: Project, relative_path: str) -> str: def parse_edit_diagnostics_result(result: str) -> dict: """Utility function to parse the diagnostic payload returned by edit tools.""" - assert EditingToolWithDiagnostics.DIAGNOSTICS_KEY in result + assert DiagnosticsContext.DIAGNOSTICS_KEY in result d = json.loads(result) - return d[EditingToolWithDiagnostics.DIAGNOSTICS_KEY] + return d[DiagnosticsContext.DIAGNOSTICS_KEY] @contextmanager From eabc7e60d62d75c05a3bfb10f335ce7506cafdc7 Mon Sep 17 00:00:00 2001 From: Dominik Jain Date: Mon, 7 Sep 2026 01:28:09 +0200 Subject: [PATCH 06/53] Add facade method metadata and API scope configuration for the REPL Facade methods are now declared explicitly via the facade_method decorator, which attaches FacadeMethodInfo (optional, beta, can_edit) mirroring the tool markers; undecorated methods are never exposed. All LspApi and JetBrainsApi methods are decorated accordingly (JetBrains methods are non-optional, as the facade only exists with the JetBrains backend). Which facades/methods are enabled is determined by an ApiScope, which is built by applying ApiInclusionDefinitions (included_apis, excluded_apis, referencing facades or facade methods such as "lsp" or "lsp.find_symbol") from the global configuration, the context, the active modes and the project configuration, in that order. Optional methods and all methods of an excluded facade must be included explicitly; other methods are enabled unless excluded. For read-only projects, editing methods are excluded. The entrypoint omits facades which are not enabled. The REPL is re-created whenever the active tools are updated (mode switch, project activation), as it depends on the same configuration. The new settings are read from project.yml and documented in all configuration templates. --- src/serena/agent.py | 27 ++- src/serena/config/context_mode.py | 6 +- src/serena/config/serena_config.py | 18 +- src/serena/facades/api/jb.py | 16 +- src/serena/facades/api/lsp.py | 15 +- src/serena/facades/facade.py | 201 +++++++++++++++--- src/serena/facades/repl.py | 25 ++- .../config/contexts/context.template.yml | 7 + .../resources/config/modes/mode.template.yml | 9 +- src/serena/resources/project.template.yml | 9 + .../resources/serena_config.template.yml | 7 + test/serena/test_jetbrains_api.py | 4 +- test/serena/test_mcp.py | 3 +- test/serena/test_repl_tool.py | 100 +++++++-- 14 files changed, 382 insertions(+), 65 deletions(-) diff --git a/src/serena/agent.py b/src/serena/agent.py index 8e60fad2..cab5d93d 100644 --- a/src/serena/agent.py +++ b/src/serena/agent.py @@ -44,7 +44,7 @@ from serena.config.serena_config import ( from serena.dashboard import SerenaDashboardAPI, SerenaDashboardTrayManager, SerenaDashboardViewer, open_url_in_browser from serena.facades.api.jb import JetBrainsApi from serena.facades.api.lsp import LspApi -from serena.facades.facade import Facade +from serena.facades.facade import ApiScope, Facade from serena.facades.repl import SerenaRepl from serena.jetbrains import launch_coordinator as jetbrains_launch_coordinator from serena.ls_manager import LanguageServerManager @@ -1154,6 +1154,9 @@ class SerenaAgent: self._active_tools = tool_set.to_available_tools(self._all_tools) log.info(f"Active tools ({len(self._active_tools)}): {', '.join(self._active_tools.tool_names)}") + # reset the REPL, which depends on the same configuration (it is re-created on demand) + self._repl = None + # check if a tool was activated that is not in the exposed tool set and issue a warning if so active_tools_not_exposed = set(self._active_tools.tool_names) - set(self._exposed_tools.tool_names) if active_tools_not_exposed: @@ -1168,12 +1171,25 @@ class SerenaAgent: :return: the REPL instance for this agent, creating it if necessary """ if self._repl is None: + # determine API scope + api_scope = ApiScope() + api_scope.process(self.serena_config) + api_scope.process(self._context) + for mode in self._active_modes.get_modes(): + api_scope.process(mode) + if self._active_project: + api_scope.process(self._active_project.project_config) + if self._active_project.project_config.read_only: + api_scope.exclude_editing() + + # gather facades facades = [] if self._language_backend.is_lsp(): - facades.append(Facade.from_api(LspApi(self))) + facades.append(Facade.from_api(LspApi(self), api_scope)) elif self._language_backend.is_jetbrains(): - facades.append(Facade.from_api(JetBrainsApi(self))) - self._repl = SerenaRepl(facades) + facades.append(Facade.from_api(JetBrainsApi(self), api_scope)) + + self._repl = SerenaRepl(facades, api_scope) return self._repl def issue_task( @@ -1270,9 +1286,6 @@ class SerenaAgent: self._project_prompt_status = ProjectPromptProvisionStatus(newly_activated_mode_names=newly_activated_mode_names) - # reset the REPL to ensure that the new project's configuration is considered - self._repl = None - if update_active_tools: self._update_active_tools() diff --git a/src/serena/config/context_mode.py b/src/serena/config/context_mode.py index 7e08afb8..eae373ed 100644 --- a/src/serena/config/context_mode.py +++ b/src/serena/config/context_mode.py @@ -12,7 +12,7 @@ import yaml from sensai.util import logging from sensai.util.string import ToStringMixin -from serena.config.serena_config import SerenaPaths, ToolInclusionDefinition +from serena.config.serena_config import ApiInclusionDefinition, SerenaPaths, ToolInclusionDefinition from serena.constants import ( DEFAULT_CONTEXT, INTERNAL_MODE_YAMLS_DIR, @@ -32,7 +32,7 @@ def looks_like_yaml_path(s: str) -> bool: @dataclass(kw_only=True) -class SerenaAgentMode(ToolInclusionDefinition, ToStringMixin): +class SerenaAgentMode(ToolInclusionDefinition, ApiInclusionDefinition, ToStringMixin): """Represents a mode of operation for the agent, typically read off a YAML file. An agent can be in multiple modes simultaneously as long as they are not mutually exclusive. The modes can be adjusted after the agent is running, for example for switching from planning to editing. @@ -148,7 +148,7 @@ class SerenaAgentMode(ToolInclusionDefinition, ToStringMixin): @dataclass(kw_only=True) -class SerenaAgentContext(ToolInclusionDefinition, ToStringMixin): +class SerenaAgentContext(ToolInclusionDefinition, ApiInclusionDefinition, ToStringMixin): """Represents a context where the agent is operating (an IDE, a chat, etc.), typically read off a YAML file. An agent can only be in a single context at a time. The contexts cannot be changed after the agent is running. diff --git a/src/serena/config/serena_config.py b/src/serena/config/serena_config.py index befe648e..71043c46 100644 --- a/src/serena/config/serena_config.py +++ b/src/serena/config/serena_config.py @@ -178,6 +178,18 @@ class NamedToolInclusionDefinition(ToolInclusionDefinition): return f"ToolInclusionDefinition[{self.name}]" +@dataclass +class ApiInclusionDefinition: + """ + Defines which APIs to include/exclude in Serena's operation. + A single API inclusion/exclusion can either be a full facade (facade name, which encompasses all of its methods, e.g. "lsp") + or a method of a facade (facade name + method name, e.g. "lsp.find_symbol"). + """ + + included_apis: Sequence[str] = () + excluded_apis: Sequence[str] = () + + @dataclass class ModeSelectionDefinition: default_modes: Sequence[str] | None = None @@ -274,7 +286,7 @@ class LineEnding(Enum): @dataclass -class SharedConfig(ToolInclusionDefinition, ToStringMixin): +class SharedConfig(ToolInclusionDefinition, ApiInclusionDefinition, ToStringMixin): """Shared between SerenaConfig and ProjectConfig, the latter used to override values in the form (same as in ModeSelectionDefinition). The defaults here shall be none and should be set to the global default values in SerenaConfig. @@ -621,6 +633,8 @@ class ProjectConfig(SharedConfig, ModeSelectionDefinitionWithAddedModes): fixed_tools = data["fixed_tools"] or [] excluded_tools = data["excluded_tools"] or [] included_optional_tools = data["included_optional_tools"] or [] + excluded_apis = data.get("excluded_apis") or [] + included_apis = data.get("included_apis") or [] additional_workspace_folders = data.get("ls_additional_workspace_folders") or [] if "base_modes" in data and data["base_modes"] is not None: @@ -635,6 +649,8 @@ class ProjectConfig(SharedConfig, ModeSelectionDefinitionWithAddedModes): excluded_tools=excluded_tools, fixed_tools=fixed_tools, included_optional_tools=included_optional_tools, + excluded_apis=excluded_apis, + included_apis=included_apis, read_only=data["read_only"], read_only_memory_patterns=data.get("read_only_memory_patterns", []), ignored_memory_patterns=data.get("ignored_memory_patterns", []), diff --git a/src/serena/facades/api/jb.py b/src/serena/facades/api/jb.py index 6d6b16bf..21337b2f 100644 --- a/src/serena/facades/api/jb.py +++ b/src/serena/facades/api/jb.py @@ -15,7 +15,7 @@ from serena.jetbrains.jetbrains_types import SymbolDTO, SymbolDTOUtil from serena.symbol import JetBrainsSymbolDictGrouper from serena.util.text_utils import find_text_coordinates -from ..facade import FacadeApi +from ..facade import FacadeApi, facade_method from ..representable import JsonObject, JsonObjectRenderer, Renderer, RepresentableViaRenderer if TYPE_CHECKING: @@ -178,6 +178,7 @@ class JetBrainsApi(FacadeApi): # read operations + @facade_method() def find_symbol( self, name_path_pattern: str, @@ -261,6 +262,7 @@ class JetBrainsApi(FacadeApi): raise ValueError(f"Matched {n_matches}>{max_matches=} symbols.\n" + renderer.render_identifiers(collection)) return collection + @facade_method() def find_referencing_symbols(self, name_path: str, relative_path: str, max_answer_chars: int = -1) -> JetBrainsSymbolCollection: """ Finds all symbols that reference the given symbol — its callers / usages / dependents, i.e. the @@ -293,6 +295,7 @@ class JetBrainsApi(FacadeApi): renderer = JetBrainsReferencesRenderer(self._agent, max_answer_chars, grouper=self.references_grouper_) return JetBrainsSymbolCollection(symbol_dicts, renderer) + @facade_method() def get_symbols_overview( self, relative_path: str, depth: int = -1, max_answer_chars: int = -1, include_file_documentation: bool = False ) -> JetBrainsSymbolsOverview: @@ -338,6 +341,7 @@ class JetBrainsApi(FacadeApi): result[rel_path].append(name_path) return dict(result) + @facade_method() def get_type_hierarchy( self, name_path: str, @@ -376,6 +380,7 @@ class JetBrainsApi(FacadeApi): result["levels_not_included"] = levels_not_included return self._json_object(result, max_answer_chars) + @facade_method() def find_declaration(self, relative_path: str, regex: str, include_body: bool = False) -> JetBrainsSymbolCollection: r""" Finds the declaration of a symbol. @@ -398,6 +403,7 @@ class JetBrainsApi(FacadeApi): ) return JetBrainsSymbolCollection(response["symbols"], JetBrainsSymbolCollectionRenderer(self._agent, -1)) + @facade_method() def find_implementations(self, relative_path: str, name_path: str) -> JetBrainsSymbolCollection: """ Finds the implementations of a symbol. @@ -412,6 +418,7 @@ class JetBrainsApi(FacadeApi): # edit operations + @facade_method(can_edit=True) def rename( self, relative_path: str, @@ -442,6 +449,7 @@ class JetBrainsApi(FacadeApi): ) return self._json_object(result) + @facade_method(beta=True, can_edit=True) def move( self, relative_path: str, @@ -482,6 +490,7 @@ class JetBrainsApi(FacadeApi): ) return self._json_object(result) + @facade_method(beta=True, can_edit=True) def safe_delete( self, relative_path: str, name_path: str | None = None, delete_even_if_used: bool = False, propagate: bool = False ) -> JsonObject: @@ -508,6 +517,7 @@ class JetBrainsApi(FacadeApi): ) return self._json_object(result) + @facade_method(beta=True, can_edit=True) def inline_symbol(self, name_path: str, relative_path: str, keep_definition: bool = False) -> JsonObject: """ Inlines a symbol (usually a method/function, but also classes may be amenable to inlining, @@ -528,6 +538,7 @@ class JetBrainsApi(FacadeApi): # inspections + @facade_method() def run_inspections( self, relative_path: str, @@ -563,6 +574,7 @@ class JetBrainsApi(FacadeApi): ) return self._json_object(result, max_answer_chars) + @facade_method() def list_inspections( self, language: str | None = None, group_path_contains: str | None = None, max_answer_chars: int = -1 ) -> JsonObject: @@ -583,6 +595,7 @@ class JetBrainsApi(FacadeApi): # debugging + @facade_method(beta=True) def debug_eval_info(self) -> str: """ Provides usage information for the debug REPL (method `debug_eval`) @@ -591,6 +604,7 @@ class JetBrainsApi(FacadeApi): """ return self._agent.prompt_factory.create_info_jet_brains_debug_repl() + @facade_method(beta=True) def debug_eval(self, expression: str, repl_key: str = "default") -> str: """ Provides debugging functionality (run configs, breakpoints, stepping, inspection, and evaluation) diff --git a/src/serena/facades/api/lsp.py b/src/serena/facades/api/lsp.py index 2eebcad5..46cdc882 100644 --- a/src/serena/facades/api/lsp.py +++ b/src/serena/facades/api/lsp.py @@ -21,7 +21,7 @@ from serena.symbol import ( from serena.util.text_utils import TextOutputUtils, find_text_coordinates from solidlsp.lsp_protocol_handler.lsp_types import SymbolKind -from ..facade import SUCCESS_RESULT, FacadeApi +from ..facade import SUCCESS_RESULT, FacadeApi, facade_method from ..representable import Renderer, RepresentableViaRenderer if TYPE_CHECKING: @@ -311,6 +311,7 @@ class LspApi(FacadeApi): # language server management + @facade_method(optional=True) def restart_language_server(self) -> str: """ Restarts the language server(s). Use this only on explicit user request or after confirmation; @@ -323,6 +324,7 @@ class LspApi(FacadeApi): # read operations + @facade_method() def get_symbols_overview(self, relative_path: str, depth: int = -1, max_answer_chars: int = -1) -> LspSymbolCollection: """ Gets an overview of the top-level symbols defined in the given file (classes, methods, fields) — its @@ -368,6 +370,7 @@ class LspApi(FacadeApi): ) return LspSymbolCollection(symbols, renderer) + @facade_method() def find_symbol( self, name_path_pattern: str, @@ -460,6 +463,7 @@ class LspApi(FacadeApi): return symbol_collection + @facade_method() def find_referencing_symbols( self, name_path: str, @@ -493,6 +497,7 @@ class LspApi(FacadeApi): ) return LspReferenceCollection(references, LspReferenceCollectionRenderer(self._agent, max_answer_chars, self.references_grouper_)) + @facade_method() def find_implementations( self, name_path: str, @@ -528,6 +533,7 @@ class LspApi(FacadeApi): output_params = SymbolOutputParams(kind=True, relative_path=True, body_location=True, include_info=include_info) return LspSymbolCollection(symbols, LspSymbolCollectionRenderer(self._agent, max_answer_chars, symbol_retriever, output_params)) + @facade_method() def find_declaration( self, relative_path: str, @@ -582,6 +588,7 @@ class LspApi(FacadeApi): collection_renderer = LspSymbolCollectionRenderer(self._agent, -1, symbol_retriever, output_params) return LspSymbol(defining_symbol, LspSymbolRenderer(self._agent, -1, collection_renderer)) + @facade_method() def get_diagnostics_for_file( self, relative_path: str, start_line: int = 0, end_line: int = -1, min_severity: int = 4, max_answer_chars: int = -1 ) -> LspDiagnostics: @@ -615,6 +622,7 @@ class LspApi(FacadeApi): return self._create_diagnostics(grouped_diagnostics, max_answer_chars) + @facade_method(optional=True) def get_diagnostics_for_symbol( self, name_path: str, @@ -658,6 +666,7 @@ class LspApi(FacadeApi): # edit operations + @facade_method(can_edit=True) def replace_symbol_body(self, name_path: str, relative_path: str, body: str) -> str: """ Replaces the body of the given symbol. @@ -675,6 +684,7 @@ class LspApi(FacadeApi): self._create_code_editor().replace_body(name_path, relative_file_path=relative_path, body=body) return SUCCESS_RESULT + @facade_method(can_edit=True) def insert_after_symbol(self, name_path: str, relative_path: str, body: str) -> str: """ Inserts code after a class/method/function definition. @@ -689,6 +699,7 @@ class LspApi(FacadeApi): self._create_code_editor().insert_after_symbol(name_path, relative_file_path=relative_path, body=body) return SUCCESS_RESULT + @facade_method(can_edit=True) def insert_before_symbol(self, name_path: str, relative_path: str, body: str) -> str: """ Inserts the given content before the beginning of the definition of the given symbol (via the symbol's location). @@ -703,6 +714,7 @@ class LspApi(FacadeApi): self._create_code_editor().insert_before_symbol(name_path, relative_file_path=relative_path, body=body) return SUCCESS_RESULT + @facade_method(can_edit=True) def rename_symbol(self, name_path: str, relative_path: str, new_name: str) -> str: """ Renames the symbol with the given `name_path` to `new_name` throughout the entire codebase. @@ -717,6 +729,7 @@ class LspApi(FacadeApi): self._get_project().ls_sync_file_system_changes() return self._create_code_editor().rename_symbol(name_path, relative_path=relative_path, new_name=new_name) + @facade_method(can_edit=True) def safe_delete_symbol(self, name_path_pattern: str, relative_path: str) -> str: """ Deletes the symbol if it is safe to do so (i.e., if there are no references to it) diff --git a/src/serena/facades/facade.py b/src/serena/facades/facade.py index 4a30598a..dc7395cb 100644 --- a/src/serena/facades/facade.py +++ b/src/serena/facades/facade.py @@ -5,31 +5,83 @@ The facade, i.e. the object through which REPL code accesses a group of related # SPDX-License-Identifier: GPL-3.0-or-later import inspect +import logging from abc import ABC from collections.abc import Callable, Iterable -from typing import TYPE_CHECKING, Any +from dataclasses import dataclass +from typing import TYPE_CHECKING, Any, TypeVar +from serena.config.serena_config import ApiInclusionDefinition from serena.project import Project if TYPE_CHECKING: from serena.agent import SerenaAgent +log = logging.getLogger(__name__) +TCallable = TypeVar("TCallable", bound=Callable[..., Any]) + SUCCESS_RESULT = "OK" """the result returned by operations which have no result other than their success""" +@dataclass(kw_only=True, frozen=True) +class FacadeMethodInfo: + """ + The metadata of a method exposed through a facade (see `facade_method`), mirroring the tool markers. + """ + + name: str + """the name of the method""" + optional: bool = False + """whether the method is disabled by default and must be enabled explicitly""" + beta: bool = False + """whether the method is in beta (not yet fully stable)""" + can_edit: bool = False + """whether the method can modify the codebase (relevant for read-only contexts)""" + + +_FACADE_METHOD_INFO_ATTR = "__facade_method_info__" + + +def facade_method(*, optional: bool = False, beta: bool = False, can_edit: bool = False) -> Callable[[TCallable], TCallable]: + """ + Marks a method of a `FacadeApi` as exposed through the facade, attaching the given metadata. + The decorator only annotates the method (it does not wrap it), such that signature and docstring remain intact. + + :param optional: whether the method is disabled by default and must be enabled explicitly + :param beta: whether the method is in beta + :param can_edit: whether the method can modify the codebase + :return: the decorator + """ + + def decorator(method: TCallable) -> TCallable: + setattr(method, _FACADE_METHOD_INFO_ATTR, FacadeMethodInfo(name=method.__name__, optional=optional, beta=beta, can_edit=can_edit)) + return method + + return decorator + + +def get_facade_method_info(method: Callable[..., Any]) -> FacadeMethodInfo | None: + """ + :param method: a (bound or unbound) method + :return: the metadata attached via `facade_method`, or None if the method is not exposed + """ + return getattr(method, _FACADE_METHOD_INFO_ATTR, None) + + class FacadeApi(ABC): """ The implementation of a facade's functionality. - API design principle: a member's name determines its visibility to the LLM. + API design principles: - * Names without a leading underscore and without a trailing underscore (e.g. `find_symbol`) constitute the - LLM-facing interface. Every such method of a concrete implementation is a candidate for exposure through - a `Facade`; which of them are actually exposed is decided by the facade. - * Names with a trailing underscore (e.g. `symbols_`, `to_dict_`) are public within Serena (e.g. for use by - classic tools or other facade implementations) but are never exposed to the LLM. Use this for functionality - which is not meant to be called from REPL code, in particular on the objects returned by API methods. + * A method is exposed to the LLM if and only if it is decorated with `facade_method`, which also carries + the method's metadata (optional, beta, can_edit). Undecorated methods are never exposed, regardless of their name. + * On the objects returned by API methods (which are not decorated), the name determines visibility: + names with a trailing underscore (e.g. `symbols_`, `to_dict_`) are public within Serena (e.g. for use by + classic tools or other facade implementations) but are not meant to be called from REPL code, whereas + names without leading or trailing underscore constitute the LLM-facing interface. + The same convention applies to non-exposed helper methods of API classes. * Names with a leading underscore are private, as usual. """ @@ -59,9 +111,16 @@ class FacadeMethod: enabled or disabled; only enabled methods are accessible from REPL code. """ - def __init__(self, name: str, implementation: Callable[..., Any], enabled: bool = True) -> None: + def __init__(self, name: str, implementation: Callable[..., Any], info: FacadeMethodInfo, enabled: bool) -> None: + """ + :param name: the method's name + :param implementation: the implementation to delegate to + :param info: the method's metadata + :param enabled: whether the method is initially enabled + """ self.name = name self._implementation = implementation + self.info = info self.enabled = enabled def __call__(self, *args: Any, **kwargs: Any) -> Any: @@ -77,6 +136,101 @@ class FacadeMethod: return f"{facade_name}.{self.name}{signature}\n{doc}\n" +class ApiScope: + """ + The scope of APIs available to the LLM, i.e. which facade methods are enabled, as determined by + applying a sequence of inclusion/exclusion definitions (from the global configuration, the context, + the active modes and the project configuration) to the methods' default enablement. + """ + + class FacadeScope: + """ + The scope of a single facade: whether the facade as a whole is included, and which of its methods + were explicitly included/excluded (a method is never in both sets). + If the facade is not included, it is opt-in, i.e. only explicitly included methods are enabled. + """ + + def __init__(self) -> None: + self.is_included = True + self.method_inclusions: set[str] = set() + self.method_exclusions: set[str] = set() + + def exclude_facade(self) -> None: + self.is_included = False + self.method_inclusions = set() + self.method_exclusions = set() + + def include_facade(self) -> None: + self.is_included = True + + def exclude_method(self, method_name: str) -> None: + self.method_inclusions.discard(method_name) + self.method_exclusions.add(method_name) + + def include_method(self, method_name: str) -> None: + self.method_exclusions.discard(method_name) + self.method_inclusions.add(method_name) + + def __init__(self) -> None: + self._facade_scopes: dict[str, ApiScope.FacadeScope] = {} + self._editing_excluded = False + + def _get_facade_scope(self, facade_name: str) -> "ApiScope.FacadeScope": + if facade_name not in self._facade_scopes: + self._facade_scopes[facade_name] = ApiScope.FacadeScope() + return self._facade_scopes[facade_name] + + def process(self, definition: ApiInclusionDefinition) -> None: + """ + Applies the given definition, exclusions first, then inclusions (such that inclusions take precedence + within a definition; across definitions, later definitions take precedence). + + :param definition: the definition to apply + """ + + def apply(api_ref: str, *, excluded: bool) -> None: + components = api_ref.split(".") + if len(components) > 2: + log.warning("Ignoring invalid API reference '%s' in %s (expected 'facade' or 'facade.method')", api_ref, definition) + return + facade_scope = self._get_facade_scope(components[0]) + if len(components) == 1: + facade_scope.exclude_facade() if excluded else facade_scope.include_facade() + else: + facade_scope.exclude_method(components[1]) if excluded else facade_scope.include_method(components[1]) + + for api_exclusion in definition.excluded_apis: + apply(api_exclusion, excluded=True) + for api_inclusion in definition.included_apis: + apply(api_inclusion, excluded=False) + + def exclude_editing(self) -> None: + """ + Excludes all methods which can edit the codebase (read-only operation), regardless of other inclusions. + """ + self._editing_excluded = True + + def is_facade_enabled(self, facade_name: str) -> bool: + facade_scope = self._get_facade_scope(facade_name) + return facade_scope.is_included or len(facade_scope.method_inclusions) > 0 + + def is_method_enabled(self, facade_name: str, method_info: FacadeMethodInfo) -> bool: + """ + :param facade_name: the name of the facade + :param method_info: the method's metadata + :return: whether the method is enabled: optional methods (and all methods of an excluded facade) must be + explicitly included, other methods are enabled unless explicitly excluded; if editing is excluded, + editing methods are always disabled + """ + if self._editing_excluded and method_info.can_edit: + return False + facade_scope = self._get_facade_scope(facade_name) + if method_info.optional or not facade_scope.is_included: + return method_info.name in facade_scope.method_inclusions + else: + return method_info.name not in facade_scope.method_exclusions + + class Facade: """ A named group of related operations which an LLM can invoke from REPL code. @@ -89,29 +243,22 @@ class Facade: object.__setattr__(self, "_methods", {m.name: m for m in methods}) @staticmethod - def _is_exposable_member_name(name: str) -> bool: - """ - :param name: the name of a member of a facade implementation - :return: whether the member may be exposed through a facade, i.e. whether its name has neither a leading - nor a trailing underscore (see `FacadeApi` for the naming principle) - """ - return not name.startswith("_") and not name.endswith("_") - - @staticmethod - def from_api(api: FacadeApi, enabled_methods: Iterable[str] | None = None) -> "Facade": + def from_api(api: FacadeApi, api_scope: ApiScope) -> "Facade": """ Creates a facade wrapping the given implementation. - :param api: the implementation; each of its LLM-facing methods (see `_is_exposable_member_name`) becomes a facade method - :param enabled_methods: the names of the methods to enable; if None, all methods are enabled + :param api: the implementation; each of its methods decorated with `facade_method` becomes a facade method + :param api_scope: API scope definition determining which methods are enabled :return: the facade """ - enabled = None if enabled_methods is None else set(enabled_methods) - methods = [ - FacadeMethod(name, member, enabled=enabled is None or name in enabled) - for name, member in inspect.getmembers(api, predicate=inspect.ismethod) - if Facade._is_exposable_member_name(name) - ] + facade_name = api.get_name_() + methods = [] + for name, member in inspect.getmembers(api, predicate=inspect.ismethod): + method_info = get_facade_method_info(member) + if method_info is None: + continue + is_enabled = api_scope.is_method_enabled(facade_name, method_info) + methods.append(FacadeMethod(name, member, method_info, enabled=is_enabled)) return Facade(api.get_name_(), api.get_description_(), methods) @property diff --git a/src/serena/facades/repl.py b/src/serena/facades/repl.py index ecbce608..a8cd8d34 100644 --- a/src/serena/facades/repl.py +++ b/src/serena/facades/repl.py @@ -4,14 +4,16 @@ The REPL through which an LLM executes Python code against Serena's facades. # SPDX-License-Identifier: GPL-3.0-or-later +import logging import textwrap import traceback -from collections.abc import Iterable from typing import Any -from .facade import Facade +from .facade import ApiScope, Facade from .representable import Representable +log = logging.getLogger(__name__) + class SerenaReplEntrypoint: """ @@ -19,10 +21,18 @@ class SerenaReplEntrypoint: and offers progressive disclosure of their interfaces via `info`. """ - def __init__(self, facades: Iterable[Facade]) -> None: + def __init__(self, facades: list[Facade], api_scope: ApiScope) -> None: + """ + :param facades: the candidate facades + :param api_scope: the API scope, which determines which of the facades are made available + """ self._facades: dict[str, Facade] = {} + registered_facade_names = [] for facade in facades: - self._register(facade) + if api_scope.is_facade_enabled(facade.name): + self._register(facade) + registered_facade_names.append(facade.name) + log.info("Registered %d/%d facades: %s", len(registered_facade_names), len(facades), registered_facade_names) def _register(self, facade: Facade) -> None: if facade.name in self._facades: @@ -75,11 +85,12 @@ class SerenaRepl: ENTRYPOINT_NAME = "s" _FUNCTION_NAME = "__serena_repl_fn__" - def __init__(self, facades: Iterable[Facade]) -> None: + def __init__(self, facades: list[Facade], api_scope: ApiScope) -> None: """ - :param facades: the facades to make available through the entrypoint + :param facades: the candidate facades + :param api_scope: the API scope, which determines which of the facades are made available """ - self._entrypoint = SerenaReplEntrypoint(facades) + self._entrypoint = SerenaReplEntrypoint(facades, api_scope) @property def entrypoint(self) -> SerenaReplEntrypoint: diff --git a/src/serena/resources/config/contexts/context.template.yml b/src/serena/resources/config/contexts/context.template.yml index 1f8e37ab..71eefa51 100644 --- a/src/serena/resources/config/contexts/context.template.yml +++ b/src/serena/resources/config/contexts/context.template.yml @@ -23,6 +23,13 @@ included_optional_tools: [] # Find the list of tools here: https://oraios.github.io/serena/01-about/035_tools.html fixed_tools: [] +# APIs (facades or facade methods, e.g. "lsp" or "lsp.find_symbol") to be excluded from the REPL in this context. +excluded_apis: [] + +# included APIs that would otherwise be excluded (particularly optional facade methods, which are disabled by default), +# e.g. "lsp.get_diagnostics_for_symbol". +included_apis: [] + # mapping of tool names to an override of their descriptions (the default description is the docstring of the Tool's apply method). # Sometimes, tool descriptions are too long (e.g., for ChatGPT), or users may want to override them for another reason. tool_description_overrides: {} diff --git a/src/serena/resources/config/modes/mode.template.yml b/src/serena/resources/config/modes/mode.template.yml index c62573d7..7efb6f9b 100644 --- a/src/serena/resources/config/modes/mode.template.yml +++ b/src/serena/resources/config/modes/mode.template.yml @@ -22,4 +22,11 @@ included_optional_tools: [] # fixed set of tools to use as the base tool set (if non-empty), replacing Serena's default set of tools. # This cannot be combined with non-empty excluded_tools or included_optional_tools. # Find the list of tools here: https://oraios.github.io/serena/01-about/035_tools.html -fixed_tools: [] \ No newline at end of file +fixed_tools: [] + +# APIs (facades or facade methods, e.g. "lsp" or "lsp.find_symbol") to be excluded from the REPL in this mode. +excluded_apis: [] + +# included APIs that would otherwise be excluded (particularly optional facade methods, which are disabled by default), +# e.g. "lsp.get_diagnostics_for_symbol". +included_apis: [] diff --git a/src/serena/resources/project.template.yml b/src/serena/resources/project.template.yml index a4f2a0fb..b45df235 100644 --- a/src/serena/resources/project.template.yml +++ b/src/serena/resources/project.template.yml @@ -131,6 +131,15 @@ included_optional_tools: [] # Find the list of tools here: https://oraios.github.io/serena/01-about/035_tools.html fixed_tools: [] +# list of APIs (facades or facade methods, e.g. "lsp" or "lsp.find_symbol") to exclude from the REPL. +# This extends the existing exclusions (e.g. from the global configuration). +excluded_apis: [] + +# list of APIs (facades or facade methods, e.g. "lsp" or "lsp.get_diagnostics_for_symbol") to include in the REPL +# that would otherwise be disabled (particularly optional methods, which are disabled by default). +# This extends the existing inclusions (e.g. from the global configuration). +included_apis: [] + # list of mode names that are to be activated by default, overriding the setting in the global configuration. # The full set of modes to be activated is base_modes (from global config) + default_modes + added_modes. # If the setting is undefined/empty, the default_modes from the global configuration (serena_config.yml) apply. diff --git a/src/serena/resources/serena_config.template.yml b/src/serena/resources/serena_config.template.yml index ffdf1431..96d9cb58 100644 --- a/src/serena/resources/serena_config.template.yml +++ b/src/serena/resources/serena_config.template.yml @@ -151,6 +151,13 @@ included_optional_tools: [] # This cannot be combined with non-empty excluded_tools or included_optional_tools. fixed_tools: [] +# list of APIs (facades or facade methods, e.g. "lsp" or "lsp.find_symbol") to be globally excluded from the REPL +excluded_apis: [] + +# list of APIs (facades or facade methods, e.g. "lsp.get_diagnostics_for_symbol") to be included in the REPL +# (particularly optional methods, which are disabled by default) +included_apis: [] + # list of mode names to that are always to be included in the set of active modes. # The full set of modes to be activated is base_modes + default_modes + added_modes, # where added_modes can be defined by projects/CLI parameters. diff --git a/test/serena/test_jetbrains_api.py b/test/serena/test_jetbrains_api.py index 4643e470..a7ab80fb 100644 --- a/test/serena/test_jetbrains_api.py +++ b/test/serena/test_jetbrains_api.py @@ -7,7 +7,7 @@ from unittest.mock import MagicMock, patch import pytest from serena.facades.api.jb import JetBrainsApi -from serena.facades.facade import Facade +from serena.facades.facade import ApiScope, Facade @pytest.fixture @@ -26,7 +26,7 @@ def client() -> MagicMock: def test_facade_exposes_all_jetbrains_operations(agent: MagicMock) -> None: - facade = Facade.from_api(JetBrainsApi(agent)) + facade = Facade.from_api(JetBrainsApi(agent), ApiScope()) assert facade.name == "jb" assert set(facade.enabled_method_names) == { "find_symbol", diff --git a/test/serena/test_mcp.py b/test/serena/test_mcp.py index ee4a1688..224e5d54 100644 --- a/test/serena/test_mcp.py +++ b/test/serena/test_mcp.py @@ -8,6 +8,7 @@ from serena import __version__ from serena.agent import Tool, ToolRegistry from serena.config.context_mode import SerenaAgentContext from serena.config.serena_config import SerenaConfig +from serena.facades.facade import ApiScope from serena.facades.repl import SerenaRepl from serena.mcp import SerenaMCPFactory @@ -26,7 +27,7 @@ class MockAgent: @staticmethod def get_repl() -> SerenaRepl: - return SerenaRepl([]) + return SerenaRepl([], ApiScope()) class BaseMockTool(Tool): diff --git a/test/serena/test_repl_tool.py b/test/serena/test_repl_tool.py index 4efc2aeb..157b5a51 100644 --- a/test/serena/test_repl_tool.py +++ b/test/serena/test_repl_tool.py @@ -7,8 +7,9 @@ from unittest.mock import MagicMock import pytest +from serena.config.serena_config import ApiInclusionDefinition from serena.facades.api.lsp import LspApi -from serena.facades.facade import Facade, FacadeApi +from serena.facades.facade import ApiScope, Facade, FacadeApi, FacadeMethodInfo, facade_method from serena.facades.repl import SerenaRepl from serena.tools import SerenaReplTool from solidlsp.ls_config import LanguageServerId @@ -20,7 +21,7 @@ class TestReplExecution: @pytest.fixture def repl(self) -> SerenaRepl: - return SerenaRepl([Facade.from_api(LspApi(MagicMock()))]) + return SerenaRepl([Facade.from_api(LspApi(MagicMock()), ApiScope())], ApiScope()) def test_return_statement_defines_result(self, repl: SerenaRepl) -> None: assert repl.execute("x = 20\ny = 22\nreturn x + y") == "42" @@ -59,28 +60,81 @@ class TestFacade: def __init__(self, agent: MagicMock) -> None: super().__init__(agent, name="dummy", description="a dummy facade") + @facade_method() def add(self, a: int, b: int) -> int: """Adds two numbers.""" return a + b + @facade_method(can_edit=True) def secret(self) -> str: return "hidden" - def serena_internal_(self) -> str: - """Public within Serena, but not LLM-facing.""" + @facade_method(optional=True, beta=True) + def extra(self) -> str: + return "extra" + + def undecorated(self) -> str: + """Public within Serena, but not exposed, since it is not decorated.""" return "internal" def _internal(self) -> None: pass + @staticmethod + def _scope(*definitions: ApiInclusionDefinition, **kwargs: list[str]) -> ApiScope: + """ + :param definitions: definitions to apply in order + :param kwargs: an additional definition (`included_apis`/`excluded_apis`) to apply last + """ + scope = ApiScope() + for definition in definitions: + scope.process(definition) + if kwargs: + scope.process(ApiInclusionDefinition(**kwargs)) + return scope + + def test_api_scope_facade_exclusion_and_method_inclusion(self) -> None: + # excluding the facade disables everything + facade = Facade.from_api(self.DummyApi(MagicMock()), self._scope(excluded_apis=["dummy"])) + assert facade.enabled_method_names == [] + + # an excluded facade is opt-in: a method inclusion enables exactly that method + facade = Facade.from_api(self.DummyApi(MagicMock()), self._scope(excluded_apis=["dummy"], included_apis=["dummy.add"])) + assert facade.enabled_method_names == ["add"] + + def test_entrypoint_omits_excluded_facades(self) -> None: + def create_repl(scope: ApiScope) -> SerenaRepl: + return SerenaRepl([Facade.from_api(self.DummyApi(MagicMock()), scope)], scope) + + assert "s.dummy" in create_repl(ApiScope()).execute("s.info()") + assert "s.dummy" not in create_repl(self._scope(excluded_apis=["dummy"])).execute("s.info()") + # a method inclusion keeps the facade available (with just that method) + overview = create_repl(self._scope(excluded_apis=["dummy"], included_apis=["dummy.add"])).execute("s.info()") + assert "s.dummy" in overview and "methods: add" in overview + + def test_api_scope_later_definitions_take_precedence(self) -> None: + scope = self._scope( + ApiInclusionDefinition(included_apis=["dummy.extra"]), + ApiInclusionDefinition(excluded_apis=["dummy.extra", "dummy.add"]), + ApiInclusionDefinition(included_apis=["dummy.add"]), + ) + facade = Facade.from_api(self.DummyApi(MagicMock()), scope) + assert set(facade.enabled_method_names) == {"add", "secret"} + + def test_api_scope_read_only_excludes_editing_methods(self) -> None: + scope = self._scope(included_apis=["dummy.secret"]) + scope.exclude_editing() + facade = Facade.from_api(self.DummyApi(MagicMock()), scope) + assert facade.enabled_method_names == ["add"] + def test_enabled_methods_delegate_to_implementation(self) -> None: - facade = Facade.from_api(self.DummyApi(MagicMock())) + facade = Facade.from_api(self.DummyApi(MagicMock()), ApiScope()) assert facade.add(1, 2) == 3 assert "dummy.add(a: int, b: int) -> int" in facade.describe() assert "Adds two numbers." in facade.describe_method("add") def test_disabled_methods_are_inaccessible_and_undocumented(self) -> None: - facade = Facade.from_api(self.DummyApi(MagicMock()), enabled_methods=["add"]) + facade = Facade.from_api(self.DummyApi(MagicMock()), self._scope(excluded_apis=["dummy.secret"])) assert facade.add(1, 2) == 3 with pytest.raises(AttributeError): facade.secret() @@ -89,18 +143,36 @@ class TestFacade: assert "secret" not in facade.describe() assert "_internal" not in facade.describe() - def test_trailing_underscore_members_are_not_llm_facing(self) -> None: - facade = Facade.from_api(self.DummyApi(MagicMock())) - assert self.DummyApi(MagicMock()).serena_internal_() == "internal" # usable from within Serena + def test_undecorated_methods_are_not_exposed(self) -> None: + facade = Facade.from_api(self.DummyApi(MagicMock()), ApiScope()) + assert self.DummyApi(MagicMock()).undecorated() == "internal" # usable from within Serena with pytest.raises(AttributeError): - facade.serena_internal_() + facade.undecorated() with pytest.raises(ValueError): - facade.get_method("serena_internal_") - assert "serena_internal_" not in facade.describe() - assert "serena_internal_" not in facade.enabled_method_names + facade.get_method("undecorated") + assert "undecorated" not in facade.describe() + assert "undecorated" not in facade.enabled_method_names + + def test_optional_methods_are_disabled_by_default(self) -> None: + facade = Facade.from_api(self.DummyApi(MagicMock()), ApiScope()) + assert "extra" not in facade.enabled_method_names + with pytest.raises(AttributeError): + facade.extra() + facade.get_method("extra").enabled = True + assert facade.extra() == "extra" + + # an explicit inclusion enables it + facade = Facade.from_api(self.DummyApi(MagicMock()), self._scope(included_apis=["dummy.extra"])) + assert "extra" in facade.enabled_method_names + + def test_method_info_mirrors_decorator(self) -> None: + facade = Facade.from_api(self.DummyApi(MagicMock()), ApiScope()) + assert facade.get_method("add").info == FacadeMethodInfo(name="add") + assert facade.get_method("secret").info.can_edit + assert facade.get_method("extra").info == FacadeMethodInfo(name="extra", optional=True, beta=True) def test_enablement_can_be_changed(self) -> None: - facade = Facade.from_api(self.DummyApi(MagicMock())) + facade = Facade.from_api(self.DummyApi(MagicMock()), ApiScope()) facade.get_method("secret").enabled = False with pytest.raises(AttributeError): facade.secret() From 9b0f6cf97adeb91ea729a661590c14e10534c3cf Mon Sep 17 00:00:00 2001 From: Dominik Jain Date: Mon, 7 Sep 2026 01:38:07 +0200 Subject: [PATCH 07/53] Restore symbol_dict_grouper attributes on the symbol tools The attribute is the tools' public handle for disabling grouping (disabled_context()), used by the health check and tests. --- src/serena/tools/jetbrains_tools.py | 6 ++++++ src/serena/tools/symbol_tools.py | 6 ++++++ 2 files changed, 12 insertions(+) diff --git a/src/serena/tools/jetbrains_tools.py b/src/serena/tools/jetbrains_tools.py index 6615206e..9f49efe5 100644 --- a/src/serena/tools/jetbrains_tools.py +++ b/src/serena/tools/jetbrains_tools.py @@ -28,6 +28,8 @@ class JetBrainsFindSymbolTool(Tool, ToolMarkerSymbolicRead, ToolMarkerOptional, Performs a global (or local) search for symbols using the JetBrains backend """ + symbol_dict_grouper = JetBrainsApi.find_symbol_grouper_ + def apply( self, name_path_pattern: str, @@ -215,6 +217,8 @@ class JetBrainsFindReferencingSymbolsTool(Tool, ToolMarkerSymbolicRead, ToolMark Finds symbols that reference the given symbol using the JetBrains backend """ + symbol_dict_grouper = JetBrainsApi.references_grouper_ + def apply( self, name_path: str, @@ -241,6 +245,8 @@ class JetBrainsGetSymbolsOverviewTool(Tool, ToolMarkerSymbolicRead, ToolMarkerOp Retrieves an overview of the top-level symbols within a specified file using the JetBrains backend """ + symbol_dict_grouper = JetBrainsApi.overview_grouper_ + def apply( self, relative_path: str, diff --git a/src/serena/tools/symbol_tools.py b/src/serena/tools/symbol_tools.py index 63115f9f..b6df9c2c 100644 --- a/src/serena/tools/symbol_tools.py +++ b/src/serena/tools/symbol_tools.py @@ -43,6 +43,8 @@ class GetSymbolsOverviewTool(Tool, ToolMarkerSymbolicRead, LspApiMixin): Gets an overview of the top-level symbols defined in a given file. """ + symbol_dict_grouper = LspApi.overview_grouper_ + def apply(self, relative_path: str, depth: int = -1, max_answer_chars: int = -1) -> str: """ Use this tool to get a high-level understanding of the code symbols in a file. @@ -65,6 +67,8 @@ class FindSymbolTool(Tool, ToolMarkerSymbolicRead, LspApiMixin): Performs a global (or local) search using the language server backend. """ + symbol_dict_grouper = LspApi.find_symbol_dict_grouper_ + def apply( self, name_path_pattern: str, @@ -144,6 +148,8 @@ class FindReferencingSymbolsTool(Tool, ToolMarkerSymbolicRead, LspApiMixin): Finds symbols that reference the given symbol """ + symbol_dict_grouper = LspApi.references_grouper_ + def apply( self, name_path: str, From 8ace53c3a108ab19c0f17fa6537806042da86593 Mon Sep 17 00:00:00 2001 From: Dominik Jain Date: Mon, 7 Sep 2026 01:48:12 +0200 Subject: [PATCH 08/53] Extract multi-file replacement logic from ReplaceInFilesTool Introduce MultiFileReplacement (in serena.util.text_utils, alongside MultiFileContentReplacer), which owns the logic previously spread over the tool's private methods: resolving the files in scope (via the project's file collection and glob filtering, which the tool had re-implemented), finding occurrences, resolving occurrence ids with diagnostics, the safety checks for blind application (no matches, expected count, ambiguous matches), rendering the listing of prospective changes and applying the selected occurrences. Rejections are signalled via ReplacementRejectedError, which indicates whether the listing should accompany the message, leaving presentation (listing, length limit, diagnostics context) to the tool. EditedFileContext moves from tools_base to code_editor, where it belongs (re-exported from serena.tools); Project.create_file_collection is now public; Tool gains _resolve_max_answer_chars. --- src/serena/code_editor.py | 45 +++++- src/serena/project.py | 6 +- src/serena/tools/file_tools.py | 203 ++++---------------------- src/serena/tools/tools_base.py | 56 ++------ src/serena/util/text_utils.py | 253 ++++++++++++++++++++++++++++++++- 5 files changed, 334 insertions(+), 229 deletions(-) diff --git a/src/serena/code_editor.py b/src/serena/code_editor.py index b9091e72..fe7f52fb 100644 --- a/src/serena/code_editor.py +++ b/src/serena/code_editor.py @@ -6,7 +6,8 @@ import os from abc import ABC, abstractmethod from collections.abc import Iterable, Iterator, Reversible from contextlib import contextmanager -from typing import Generic, TypeVar, cast +from types import TracebackType +from typing import Any, Generic, Self, TypeVar, cast from serena.jetbrains.jetbrains_plugin_client import JetBrainsPluginClient from serena.symbol import JetBrainsSymbol, LanguageServerSymbol, LanguageServerSymbolRetriever, PositionInFile, Symbol @@ -493,3 +494,45 @@ class JetBrainsCodeEditor(CodeEditor[JetBrainsSymbol]): rename_in_text_occurrences=rename_in_text_occurrences, ) return "Success" + + +class EditedFileContext: + """ + Context manager for file editing. + + Create the context, then use `set_updated_content` to set the new content, the original content + being provided in `original_content`. + When exiting the context without an exception, the updated content will be written back to the file. + """ + + def __init__(self, relative_path: str, code_editor: CodeEditor): + self._relative_path = relative_path + self._code_editor = code_editor + self._edited_file: CodeEditor.EditedFile | None = None + self._edited_file_context: Any = None + + def __enter__(self) -> Self: + self._edited_file_context = self._code_editor.edited_file_context(self._relative_path) + self._edited_file = self._edited_file_context.__enter__() + return self + + def get_original_content(self) -> str: + """ + :return: the original content of the file before any modifications. + """ + assert self._edited_file is not None + return self._edited_file.get_contents() + + def set_updated_content(self, content: str) -> None: + """ + Sets the updated content of the file, which will be written back to the file + when the context is exited without an exception. + + :param content: the updated content of the file + """ + assert self._edited_file is not None + self._edited_file.set_contents(content) + + def __exit__(self, exc_type: type[BaseException] | None, exc_value: BaseException | None, traceback: TracebackType | None) -> None: + assert self._edited_file_context is not None + self._edited_file_context.__exit__(exc_type, exc_value, traceback) diff --git a/src/serena/project.py b/src/serena/project.py index 7341d891..a6dd1f04 100644 --- a/src/serena/project.py +++ b/src/serena/project.py @@ -383,7 +383,7 @@ class Project(ToStringMixin): ) return rel_file_paths - def _create_file_collection(self, relative_path: str, *, code_files_only: bool, skip_ignored_files: bool) -> FileCollection: + def create_file_collection(self, relative_path: str, *, code_files_only: bool, skip_ignored_files: bool) -> FileCollection: """ Creates the file collection for the given relative path. @@ -446,9 +446,7 @@ class Project(ToStringMixin): :param skip_ignored_files: whether to skip ignored files; has no effect if `code_files_only` is True :return: list of matches """ - file_collection = self._create_file_collection( - relative_path, code_files_only=code_files_only, skip_ignored_files=skip_ignored_files - ) + file_collection = self.create_file_collection(relative_path, code_files_only=code_files_only, skip_ignored_files=skip_ignored_files) return search_files( file_collection, pattern, diff --git a/src/serena/tools/file_tools.py b/src/serena/tools/file_tools.py index 70b2fe3e..7874f999 100644 --- a/src/serena/tools/file_tools.py +++ b/src/serena/tools/file_tools.py @@ -15,12 +15,7 @@ from typing import Literal from serena.tools import SUCCESS_RESULT, EditedFileContext, EditingToolWithDiagnostics, Tool, ToolMarkerOptional from serena.util.file_system import scan_directory -from serena.util.text_utils import ( - ContentReplacer, - GlobMatcher, - MultiFileContentReplacer, - ReplacementOccurrence, -) +from serena.util.text_utils import ContentReplacer, MultiFileReplacement, ReplacementRejectedError from solidlsp.ls_utils import TextUtils @@ -272,181 +267,35 @@ class ReplaceInFilesTool(EditingToolWithDiagnostics): returned. -1 uses the configured default. :return: in a dry run, the prospective changes; otherwise a summary of the applied replacements """ - replacer = MultiFileContentReplacer(mode=mode) - files = self._collect_files(relative_path, paths_include_glob, paths_exclude_glob) - occurrences = replacer.find_occurrences(files, needle, repl) - contents = dict(files) - - if dry_run: - return self._render_listing(replacer, occurrences, contents, max_answer_chars, dry_run=True) - - if occurrence_ids is not None: - selected, problems = self._resolve_occurrence_ids(occurrence_ids, occurrences) - if problems: - problem_lines = "\n".join(f" {p}" for p in problems) - raise ValueError( - f"{len(problems)} of the given occurrence_ids could not be resolved - NO changes were applied:\n" - f"{problem_lines}\n" - "Re-run with dry_run=True to obtain current occurrence ids." - ) - if not selected: - raise ValueError("occurrence_ids is empty - pass at least one id from a dry run, or omit the parameter to replace all.") - return self._apply_occurrences(replacer, selected, contents, needle, repl) - - # blind apply (no ids) - if not occurrences: - raise ValueError( - "No occurrences of the pattern were found - NO changes were applied. " - "Check the mode (a literal needle containing regex metacharacters must use mode 'literal'; " - "wildcards require mode 'regex') and the path/glob restrictions, " - "or locate the content with search_for_pattern first." - ) - if expected_count >= 0 and len(occurrences) != expected_count: - listing = self._render_listing(replacer, occurrences, contents, max_answer_chars, dry_run=False) - raise ValueError( - f"expected_count={expected_count}, but the pattern matches {len(occurrences)} occurrence(s) - " - f"NO changes were applied. Review the prospective changes below; re-issue with the corrected " - f"expectation, a refined pattern, or occurrence_ids selecting the intended subset.\n{listing}" - ) - ambiguous = [o for o in occurrences if o.is_ambiguous] - if ambiguous: - listing = self._render_listing(replacer, occurrences, contents, max_answer_chars, dry_run=False) - raise ValueError( - f"{len(ambiguous)} occurrence(s) are ambiguous (the pattern matches again inside the matched text, " - f"indicating possible over-matching) - NO changes were applied. Review the prospective changes below " - f"and either refine the pattern or explicitly select occurrences via occurrence_ids.\n{listing}" - ) - return self._apply_occurrences(replacer, occurrences, contents, needle, repl) - - def _collect_files(self, relative_path: str, paths_include_glob: str, paths_exclude_glob: str) -> list[tuple[str, str]]: - """Collects (relative_path, content) pairs of the non-ignored files in scope, in sorted path order.""" - relative_path = relative_path.strip() - if relative_path: - self.project.validate_relative_path(relative_path, require_not_ignored=True) - abs_path = os.path.join(self.get_project_root(), relative_path) - if not os.path.exists(abs_path): - raise FileNotFoundError(f"Relative path {relative_path} does not exist.") - if os.path.isfile(abs_path): - rel_paths = [relative_path] - else: - _dirs, rel_paths = scan_directory( - path=abs_path, - recursive=True, - is_ignored_dir=self.project.is_ignored_path, - is_ignored_file=self.project.is_ignored_path, - relative_to=self.get_project_root(), - ) - include_glob_matcher = GlobMatcher(paths_include_glob.strip()) if paths_include_glob.strip() else None - exclude_glob_matcher = GlobMatcher(paths_exclude_glob.strip()) if paths_exclude_glob.strip() else None - files: list[tuple[str, str]] = [] - for path in sorted(rel_paths): - if include_glob_matcher and not include_glob_matcher.matches(path): - continue - if exclude_glob_matcher and exclude_glob_matcher.matches(path): - continue - try: - files.append((path, self.project.read_file(path))) - except Exception: - continue # skip unreadable (e.g. binary) files - return files - - def _render_listing( - self, - replacer: MultiFileContentReplacer, - occurrences: list[ReplacementOccurrence], - contents: dict[str, str], - max_answer_chars: int, - dry_run: bool, - ) -> str: - affected_files = sorted({o.relative_path for o in occurrences}) - header = f"Found {len(occurrences)} occurrence(s) in {len(affected_files)} file(s)." - if dry_run: - header += ( - " DRY RUN - no changes were applied.\n" - "Re-issue with dry_run=False to replace all of them, or additionally pass occurrence_ids " - "with the ids of the occurrences to replace." - ) - parts = [header] - for path in affected_files: - file_occurrences = [o for o in occurrences if o.relative_path == path] - parts.append(f"\n{path} ({len(file_occurrences)} occurrence(s)):") - for occ in file_occurrences: - parts.append(replacer.render_occurrence_diff(occ, contents[path])) - result = "\n".join(parts) - - def make_locations_only() -> str: - lines = [header] + [f" [{o.occurrence_id}] line {o.start_line}" for o in occurrences] - return "\n".join(lines) - - def make_per_file_counts() -> str: - counts = {path: sum(1 for o in occurrences if o.relative_path == path) for path in affected_files} - return f"{header}\nOccurrence counts per file:\n{self._to_json(counts)}" - - def make_summary() -> str: - return header - - return self._limit_length( - result, max_answer_chars, shortened_result_factories=[make_locations_only, make_per_file_counts, make_summary] + max_answer_chars = self._resolve_max_answer_chars(max_answer_chars) + replacement = MultiFileReplacement( + self.project, + needle, + repl, + mode, + relative_path=relative_path, + paths_include_glob=paths_include_glob, + paths_exclude_glob=paths_exclude_glob, ) + if dry_run: + return replacement.render_listing(max_answer_chars, dry_run=True) - @staticmethod - def _resolve_occurrence_ids( - occurrence_ids: list[str], occurrences: list[ReplacementOccurrence] - ) -> tuple[list[ReplacementOccurrence], list[str]]: - """Resolves the requested ids against the current occurrences, diagnosing each failure.""" - occurrences_by_id = {o.occurrence_id: o for o in occurrences} - indices_by_path: dict[str, set[int]] = {} - for o in occurrences: - indices_by_path.setdefault(o.relative_path, set()).add(o.index_in_file) - selected: dict[str, ReplacementOccurrence] = {} - problems: list[str] = [] - for oid in occurrence_ids: - occurrence = occurrences_by_id.get(oid) - if occurrence is not None: - selected[oid] = occurrence - continue - id_match = MultiFileContentReplacer.OCCURRENCE_ID_REGEX.match(oid) - if id_match is None: - problems.append(f"{oid}: malformed id (expected ':@' as returned by a dry run)") - elif id_match.group("path") not in indices_by_path: - problems.append(f"{oid}: the pattern currently has no matches in this file") - elif int(id_match.group("index")) not in indices_by_path[id_match.group("path")]: - problems.append(f"{oid}: the file now has fewer matches than at dry-run time (content changed)") + # select the occurrences to replace + try: + if occurrence_ids is not None: + occurrences = replacement.select(occurrence_ids) else: - problems.append(f"{oid}: the matched text changed since the dry run (content changed)") - return list(selected.values()), problems + occurrences = replacement.select_all_guarded(expected_count) + except ReplacementRejectedError as e: + message = str(e) + if e.show_prospective_changes: + message += "\n" + replacement.render_listing(max_answer_chars, dry_run=False) + raise ValueError(message) from e - def _apply_occurrences( - self, - replacer: MultiFileContentReplacer, - occurrences: list[ReplacementOccurrence], - contents: dict[str, str], - needle: str, - repl: str, - ) -> str: - occurrences_by_file: dict[str, list[ReplacementOccurrence]] = {} - for occ in occurrences: - occurrences_by_file.setdefault(occ.relative_path, []).append(occ) - with self.diagnostics_context() as diagnostics_context: - code_editor = self.create_code_editor() - for path, file_occurrences in occurrences_by_file.items(): - with EditedFileContext(path, code_editor) as context: - original_content = context.get_original_content() - if original_content != contents[path]: - # the editor's view differs from what was scanned (e.g. line-ending normalization); - # re-derive the occurrences from the authoritative content and re-validate by id - fresh_by_id = {o.occurrence_id: o for o in replacer.find_occurrences([(path, original_content)], needle, repl)} - try: - file_occurrences = [fresh_by_id[o.occurrence_id] for o in file_occurrences] - except KeyError as e: - raise ValueError( - f"The content of {path} changed while replacing (occurrence {e} no longer resolves); " - f"the file was NOT modified. Re-run with dry_run=True for current ids." - ) from e - context.set_updated_content(replacer.apply_to_content(original_content, file_occurrences)) - per_file = "\n".join(f" {path}: {len(occs)}" for path, occs in occurrences_by_file.items()) - summary = f"Replaced {len(occurrences)} occurrence(s) in {len(occurrences_by_file)} file(s):\n{per_file}" - return diagnostics_context.format_result(summary) + # apply the replacement + with self.diagnostics_context(*replacement.affected_files) as diagnostics_context: + result = replacement.apply(self.create_code_editor(), occurrences) + return diagnostics_context.format_result(result.to_display_string()) class DeleteLinesTool(EditingToolWithDiagnostics, ToolMarkerOptional): diff --git a/src/serena/tools/tools_base.py b/src/serena/tools/tools_base.py index bcdbf1bb..97fde923 100644 --- a/src/serena/tools/tools_base.py +++ b/src/serena/tools/tools_base.py @@ -6,8 +6,7 @@ from abc import ABC from collections.abc import Callable, Iterable from dataclasses import dataclass from functools import cached_property -from types import TracebackType -from typing import TYPE_CHECKING, Any, Protocol, Self, TypeVar, cast +from typing import TYPE_CHECKING, Any, Protocol, TypeVar, cast from mcp import Implementation from mcp.server.fastmcp import Context @@ -15,6 +14,7 @@ from mcp.server.fastmcp.utilities.func_metadata import FuncMetadata, func_metada from sensai.util import logging from sensai.util.string import dict_string +from serena.code_editor import EditedFileContext # noqa: F401 (re-exported for tools) from serena.config.serena_config import LanguageBackend from serena.facades.facade import SUCCESS_RESULT # noqa: F401 (re-exported for tools) from serena.lsp.lsp_diagnostics import DiagnosticsContext @@ -281,6 +281,13 @@ class Tool(Component): params[param] = value log.info(f"{self.get_name_from_cls()}: {dict_string(params)}; session_id: {session_id}") + def _resolve_max_answer_chars(self, max_answer_chars: int) -> int: + """ + :param max_answer_chars: the maximum number of answer characters as passed to the tool; -1 for the configured default + :return: the effective maximum + """ + return self.agent.serena_config.default_max_tool_answer_chars if max_answer_chars == -1 else max_answer_chars + def _limit_length( self, result: str, @@ -295,8 +302,7 @@ class Tool(Component): version of the result. They are tried in order until one fits within ``max_answer_chars``. :return: the result string, potentially replaced by a shortened version """ - if max_answer_chars == -1: - max_answer_chars = self.agent.serena_config.default_max_tool_answer_chars + max_answer_chars = self._resolve_max_answer_chars(max_answer_chars) return TextOutputUtils.limit_length( result=result, max_answer_chars=max_answer_chars, shortened_result_factories=shortened_result_factories ) @@ -479,48 +485,6 @@ class EditingToolWithDiagnostics(Tool, ToolMarkerCanEdit): return DiagnosticsContext(self.agent, *edited_relative_paths, enable=self.ENABLE_DIAGNOSTICS) -class EditedFileContext: - """ - Context manager for file editing. - - Create the context, then use `set_updated_content` to set the new content, the original content - being provided in `original_content`. - When exiting the context without an exception, the updated content will be written back to the file. - """ - - def __init__(self, relative_path: str, code_editor: "CodeEditor"): - self._relative_path = relative_path - self._code_editor = code_editor - self._edited_file: CodeEditor.EditedFile | None = None - self._edited_file_context: Any = None - - def __enter__(self) -> Self: - self._edited_file_context = self._code_editor.edited_file_context(self._relative_path) - self._edited_file = self._edited_file_context.__enter__() - return self - - def get_original_content(self) -> str: - """ - :return: the original content of the file before any modifications. - """ - assert self._edited_file is not None - return self._edited_file.get_contents() - - def set_updated_content(self, content: str) -> None: - """ - Sets the updated content of the file, which will be written back to the file - when the context is exited without an exception. - - :param content: the updated content of the file - """ - assert self._edited_file is not None - self._edited_file.set_contents(content) - - def __exit__(self, exc_type: type[BaseException] | None, exc_value: BaseException | None, traceback: TracebackType | None) -> None: - assert self._edited_file_context is not None - self._edited_file_context.__exit__(exc_type, exc_value, traceback) - - @dataclass(kw_only=True) class RegisteredTool: tool_class: type[Tool] diff --git a/src/serena/util/text_utils.py b/src/serena/util/text_utils.py index b73c5755..986eed85 100644 --- a/src/serena/util/text_utils.py +++ b/src/serena/util/text_utils.py @@ -7,7 +7,7 @@ import re from collections.abc import Callable from dataclasses import dataclass, field from enum import StrEnum -from typing import Any, Literal, Self +from typing import TYPE_CHECKING, Any, Literal, Self from bs4 import BeautifulSoup from joblib import Parallel, delayed @@ -16,6 +16,10 @@ from sensai.util.string import ToStringMixin from serena.util.file_proxy import FileCollection, FileProxy from solidlsp.ls_utils import TextUtils +if TYPE_CHECKING: + from serena.code_editor import CodeEditor + from serena.project import Project + log = logging.getLogger(__name__) @@ -639,6 +643,253 @@ class MultiFileContentReplacer: return "\n".join(diff_lines) +class ReplacementRejectedError(ValueError): + """ + Raised when a replacement is not applied because a safety check failed; no changes have been made. + """ + + def __init__(self, message: str, show_prospective_changes: bool) -> None: + """ + :param message: the reason for the rejection + :param show_prospective_changes: whether the listing of the prospective changes should be presented along + with the message (the message refers to it) + """ + super().__init__(message) + self.show_prospective_changes = show_prospective_changes + + +@dataclass +class MultiFileReplacementResult: + """ + The result of an applied replacement. + """ + + num_occurrences_by_file: dict[str, int] + """the number of replaced occurrences per file (relative path), in application order""" + + @property + def num_occurrences(self) -> int: + return sum(self.num_occurrences_by_file.values()) + + def to_display_string(self) -> str: + per_file = "\n".join(f" {path}: {n}" for path, n in self.num_occurrences_by_file.items()) + return f"Replaced {self.num_occurrences} occurrence(s) in {len(self.num_occurrences_by_file)} file(s):\n{per_file}" + + +class MultiFileReplacement: + """ + A prospective replacement of a pattern across the files of a project within a given scope: holds the + occurrences found, supports selecting occurrences by their ids, guards blind application against + unintended replacements, renders a preview listing and applies the replacement via a code editor. + """ + + def __init__( + self, + project: "Project", + needle: str, + repl: str, + mode: Literal["literal", "regex"], + relative_path: str = "", + paths_include_glob: str = "", + paths_exclude_glob: str = "", + ) -> None: + """ + :param project: the project whose files are to be searched + :param needle: the string (mode "literal") or regular expression (mode "regex") to search for + :param repl: the replacement string (may contain $!N backreferences in regex mode) + :param mode: how `needle` is to be interpreted + :param relative_path: only consider this file or directory (default: the whole project) + :param paths_include_glob: optional glob restricting which files are considered + :param paths_exclude_glob: optional glob of files to exclude; takes precedence over the include glob + """ + self._replacer = MultiFileContentReplacer(mode=mode) + self._needle = needle + self._repl = repl + files = self._collect_files(project, relative_path.strip(), paths_include_glob.strip(), paths_exclude_glob.strip()) + self._contents = dict(files) + self.occurrences: list[ReplacementOccurrence] = self._replacer.find_occurrences(files, needle, repl) + + @staticmethod + def _collect_files(project: "Project", relative_path: str, paths_include_glob: str, paths_exclude_glob: str) -> list[tuple[str, str]]: + """ + :return: the (relative_path, content) pairs of the readable, non-ignored files in scope, in sorted path order + """ + if relative_path: + project.validate_relative_path(relative_path, require_not_ignored=True) + file_collection = project.create_file_collection(relative_path, code_files_only=False, skip_ignored_files=True).filter_glob( + paths_include_glob or None, paths_exclude_glob or None + ) + files: list[tuple[str, str]] = [] + for file_proxy in sorted(file_collection, key=lambda f: f.get_relative_path()): + try: + files.append((file_proxy.get_relative_path(), file_proxy.get_contents())) + except Exception: + continue # skip unreadable (e.g. binary) files + return files + + @property + def affected_files(self) -> list[str]: + """ + :return: the relative paths of the files containing occurrences, sorted + """ + return sorted({o.relative_path for o in self.occurrences}) + + def select(self, occurrence_ids: list[str]) -> list[ReplacementOccurrence]: + """ + Resolves the given occurrence ids (as obtained from a previous listing). + + :param occurrence_ids: the ids of the occurrences to select + :return: the selected occurrences + :raises ReplacementRejectedError: if the selection is empty or any id cannot be resolved + """ + occurrences_by_id = {o.occurrence_id: o for o in self.occurrences} + indices_by_path: dict[str, set[int]] = {} + for o in self.occurrences: + indices_by_path.setdefault(o.relative_path, set()).add(o.index_in_file) + + # resolve each id, diagnosing failures + selected: dict[str, ReplacementOccurrence] = {} + problems: list[str] = [] + for oid in occurrence_ids: + occurrence = occurrences_by_id.get(oid) + if occurrence is not None: + selected[oid] = occurrence + continue + id_match = MultiFileContentReplacer.OCCURRENCE_ID_REGEX.match(oid) + if id_match is None: + problems.append(f"{oid}: malformed id (expected ':@' as returned by a dry run)") + elif id_match.group("path") not in indices_by_path: + problems.append(f"{oid}: the pattern currently has no matches in this file") + elif int(id_match.group("index")) not in indices_by_path[id_match.group("path")]: + problems.append(f"{oid}: the file now has fewer matches than at dry-run time (content changed)") + else: + problems.append(f"{oid}: the matched text changed since the dry run (content changed)") + + if problems: + problem_lines = "\n".join(f" {p}" for p in problems) + raise ReplacementRejectedError( + f"{len(problems)} of the given occurrence_ids could not be resolved - NO changes were applied:\n" + f"{problem_lines}\n" + "Re-run with dry_run=True to obtain current occurrence ids.", + show_prospective_changes=False, + ) + if not selected: + raise ReplacementRejectedError( + "occurrence_ids is empty - pass at least one id from a dry run, or omit the parameter to replace all.", + show_prospective_changes=False, + ) + return list(selected.values()) + + def select_all_guarded(self, expected_count: int = -1) -> list[ReplacementOccurrence]: + """ + Selects all occurrences for a blind application (without explicit selection), applying safety checks. + + :param expected_count: the number of occurrences expected; -1 disables the check + :return: all occurrences + :raises ReplacementRejectedError: if there are no occurrences, the count differs from the expectation + or any occurrence is ambiguous + """ + if not self.occurrences: + raise ReplacementRejectedError( + "No occurrences of the pattern were found - NO changes were applied. " + "Check the mode (a literal needle containing regex metacharacters must use mode 'literal'; " + "wildcards require mode 'regex') and the path/glob restrictions, " + "or locate the content with search_for_pattern first.", + show_prospective_changes=False, + ) + if expected_count >= 0 and len(self.occurrences) != expected_count: + raise ReplacementRejectedError( + f"expected_count={expected_count}, but the pattern matches {len(self.occurrences)} occurrence(s) - " + "NO changes were applied. Review the prospective changes below; re-issue with the corrected " + "expectation, a refined pattern, or occurrence_ids selecting the intended subset.", + show_prospective_changes=True, + ) + num_ambiguous = sum(1 for o in self.occurrences if o.is_ambiguous) + if num_ambiguous: + raise ReplacementRejectedError( + f"{num_ambiguous} occurrence(s) are ambiguous (the pattern matches again inside the matched text, " + "indicating possible over-matching) - NO changes were applied. Review the prospective changes below " + "and either refine the pattern or explicitly select occurrences via occurrence_ids.", + show_prospective_changes=True, + ) + return list(self.occurrences) + + def render_listing(self, max_answer_chars: int, dry_run: bool) -> str: + """ + Renders the prospective changes as a list of minimal line diffs with occurrence ids, subject to the given length limit + (falling back to locations only, per-file counts and finally a summary). + + :param max_answer_chars: the maximum number of characters (must be positive) + :param dry_run: whether the listing is the result of a dry run (adding instructions on how to proceed) + :return: the listing + """ + affected_files = self.affected_files + header = f"Found {len(self.occurrences)} occurrence(s) in {len(affected_files)} file(s)." + if dry_run: + header += ( + " DRY RUN - no changes were applied.\n" + "Re-issue with dry_run=False to replace all of them, or additionally pass occurrence_ids " + "with the ids of the occurrences to replace." + ) + parts = [header] + for path in affected_files: + file_occurrences = [o for o in self.occurrences if o.relative_path == path] + parts.append(f"\n{path} ({len(file_occurrences)} occurrence(s)):") + for occ in file_occurrences: + parts.append(self._replacer.render_occurrence_diff(occ, self._contents[path])) + result = "\n".join(parts) + + # shortened result closures, from least to most aggressive shortening + def make_locations_only() -> str: + return "\n".join([header] + [f" [{o.occurrence_id}] line {o.start_line}" for o in self.occurrences]) + + def make_per_file_counts() -> str: + counts = {path: sum(1 for o in self.occurrences if o.relative_path == path) for path in affected_files} + return f"{header}\nOccurrence counts per file:\n{TextOutputUtils.to_json(counts)}" + + def make_summary() -> str: + return header + + shortened_result_factories: list[Callable[[], str]] = [make_locations_only, make_per_file_counts, make_summary] + return TextOutputUtils.limit_length(result, max_answer_chars, shortened_result_factories) + + def apply(self, code_editor: "CodeEditor", occurrences: list[ReplacementOccurrence]) -> MultiFileReplacementResult: + """ + Applies the given (selected) occurrences. + + :param code_editor: the code editor through which to modify the files + :param occurrences: the occurrences to replace (obtained from `select` or `select_all_guarded`) + :return: the result + :raises ValueError: if a file's content changed such that a selected occurrence no longer resolves + (the file is then not modified) + """ + from serena.code_editor import EditedFileContext + + occurrences_by_file: dict[str, list[ReplacementOccurrence]] = {} + for occ in occurrences: + occurrences_by_file.setdefault(occ.relative_path, []).append(occ) + + for path, file_occurrences in occurrences_by_file.items(): + with EditedFileContext(path, code_editor) as context: + original_content = context.get_original_content() + if original_content != self._contents[path]: + # the editor's view differs from what was scanned (e.g. line-ending normalization); + # re-derive the occurrences from the authoritative content and re-validate by id + fresh_by_id = { + o.occurrence_id: o for o in self._replacer.find_occurrences([(path, original_content)], self._needle, self._repl) + } + try: + file_occurrences = [fresh_by_id[o.occurrence_id] for o in file_occurrences] + except KeyError as e: + raise ValueError( + f"The content of {path} changed while replacing (occurrence {e} no longer resolves); " + f"the file was NOT modified. Re-run with dry_run=True for current ids." + ) from e + context.set_updated_content(self._replacer.apply_to_content(original_content, file_occurrences)) + + return MultiFileReplacementResult({path: len(occs) for path, occs in occurrences_by_file.items()}) + + @dataclass class TextCoords: line: int From 66ed46bef44430ed2d1a568b146c3e0aca4ebaf4 Mon Sep 17 00:00:00 2001 From: Dominik Jain Date: Mon, 7 Sep 2026 12:02:03 +0200 Subject: [PATCH 09/53] Add editing facade (edit) covering the backend-independent edit operations Introduce EditApi as the facade for editing operations which depend only on the code editor abstraction and thus work with any language backend: create_text_file, replace_content, replace_in_files, the optional line-level operations delete_lines, replace_lines and insert_at_line, and the symbol-level operations replace_symbol_body, insert_after_symbol and insert_before_symbol (moved from LspApi, which retains the LSP-specific rename_symbol and safe_delete_symbol). replace_in_files builds on MultiFileReplacement: a dry run returns a ReplacementPreview, which exposes the occurrences to code and renders the listing of prospective changes; rejections raise a ValueError carrying the listing where applicable. FacadeApi gains a backend-agnostic _create_code_editor; the LSP API's retriever-based factory is renamed to _create_ls_code_editor. The editing tools (file and symbol level) delegate to EditApi via EditApiMixin, retaining only the diagnostics context. The edit facade is always part of the REPL. --- src/serena/agent.py | 3 +- src/serena/facades/api/edit.py | 318 ++++++++++++++++++++++++++++ src/serena/facades/api/lsp.py | 56 +---- src/serena/facades/facade.py | 17 ++ src/serena/facades/representable.py | 11 +- src/serena/tools/file_tools.py | 129 ++++------- src/serena/tools/symbol_tools.py | 12 +- test/serena/test_edit_api.py | 81 +++++++ 8 files changed, 478 insertions(+), 149 deletions(-) create mode 100644 src/serena/facades/api/edit.py create mode 100644 test/serena/test_edit_api.py diff --git a/src/serena/agent.py b/src/serena/agent.py index cab5d93d..dbab9d56 100644 --- a/src/serena/agent.py +++ b/src/serena/agent.py @@ -42,6 +42,7 @@ from serena.config.serena_config import ( ToolInclusionDefinition, ) from serena.dashboard import SerenaDashboardAPI, SerenaDashboardTrayManager, SerenaDashboardViewer, open_url_in_browser +from serena.facades.api.edit import EditApi from serena.facades.api.jb import JetBrainsApi from serena.facades.api.lsp import LspApi from serena.facades.facade import ApiScope, Facade @@ -1183,7 +1184,7 @@ class SerenaAgent: api_scope.exclude_editing() # gather facades - facades = [] + facades = [Facade.from_api(EditApi(self), api_scope)] if self._language_backend.is_lsp(): facades.append(Facade.from_api(LspApi(self), api_scope)) elif self._language_backend.is_jetbrains(): diff --git a/src/serena/facades/api/edit.py b/src/serena/facades/api/edit.py new file mode 100644 index 00000000..815a8da0 --- /dev/null +++ b/src/serena/facades/api/edit.py @@ -0,0 +1,318 @@ +# SPDX-License-Identifier: GPL-3.0-or-later +""" +The implementation of editing operations, which are independent of the language backend. +""" + +from pathlib import Path +from typing import TYPE_CHECKING, Literal + +from serena.code_editor import EditedFileContext +from serena.util.text_utils import ContentReplacer, MultiFileReplacement, ReplacementOccurrence, ReplacementRejectedError + +from ..facade import SUCCESS_RESULT, FacadeApi, facade_method +from ..representable import Renderer, RepresentableViaRenderer + +if TYPE_CHECKING: + from serena.agent import SerenaAgent + + +class ReplacementPreview(RepresentableViaRenderer): + """ + The prospective changes of a multi-file replacement (nothing has been modified). + Each entry of `occurrences` (`ReplacementOccurrence`) has an `occurrence_id` (to be passed to `replace_in_files` + in order to apply exactly that occurrence), `relative_path`, `start_line`, `end_line`, `matched_text`, `replacement` + and `is_ambiguous`. + """ + + def __init__(self, replacement: MultiFileReplacement, renderer: "ReplacementPreviewRenderer"): + """ + :param replacement: the replacement + :param renderer: the renderer to use for representing the preview + """ + super().__init__(renderer) + self.replacement_ = replacement + + @property + def occurrences(self) -> list[ReplacementOccurrence]: + return self.replacement_.occurrences + + @property + def affected_files(self) -> list[str]: + return self.replacement_.affected_files + + +class ReplacementPreviewRenderer(Renderer[ReplacementPreview]): + """ + Renders the listing of prospective changes (minimal line diffs with occurrence ids), subject to the length limit. + """ + + def __init__(self, agent: "SerenaAgent", max_answer_chars: int, dry_run: bool): + """ + :param agent: the agent + :param max_answer_chars: the maximum number of characters; -1 for the configured default + :param dry_run: whether the listing is the result of a dry run (adding instructions on how to proceed) + """ + super().__init__(agent, max_answer_chars) + self._dry_run = dry_run + + def render(self, obj: ReplacementPreview) -> str: + return obj.replacement_.render_listing(self._get_max_answer_chars(), dry_run=self._dry_run) + + +class EditApi(FacadeApi): + def __init__(self, agent: "SerenaAgent") -> None: + super().__init__( + agent, + name="edit", + description="editing operations independent of the language backend (creating files, replacing content in one or " + "many files, editing at the symbol or line level)", + ) + + # file-level operations + + @facade_method(can_edit=True) + def create_text_file(self, relative_path: str, content: str) -> str: + """ + Writes a new file or overwrites an existing file with the given content. + + :param relative_path: the relative path to the file to create + :param content: the (appropriately encoded) content to write to the file + :return: a message indicating success + """ + project = self._get_project() + project_root = Path(project.project_root) + abs_path = (project_root / relative_path).resolve() + will_overwrite_existing = abs_path.exists() + + # validate the destination path + if will_overwrite_existing: + project.validate_relative_path(relative_path) + else: + assert abs_path.is_relative_to(project_root), f"Cannot create file outside of the project directory, got {relative_path=}" + + # write the file + abs_path.parent.mkdir(parents=True, exist_ok=True) + abs_path.write_text(content, encoding=project.project_config.encoding, newline=project.line_ending.newline_str) + answer = f"File created: {relative_path}." + if will_overwrite_existing: + answer += " Overwrote existing file." + return answer + + @facade_method(can_edit=True) + def replace_content( + self, + relative_path: str, + needle: str, + repl: str, + mode: Literal["literal", "regex"], + allow_multiple_occurrences: bool = False, + ) -> str: + r""" + Replaces one or more occurrences of a given pattern in a file with new content. + + VERY IMPORTANT: The "regex" mode allows very large sections of code to be replaced WITHOUT + quoting them fully: use a needle of the form "beginning.*?end-of-text-to-be-replaced" with + wildcards instead of pasting the exact original text — shorter, cheaper, and you cannot make + mistakes, because an ambiguous match returns an error you can refine, so wildcards are safe. + Prefer regex mode with suitable wildcards for long multi-line replacements; use the + symbol-level editors when replacing a whole method/class. + + :param relative_path: the relative path to the file + :param needle: the string or regex pattern to search for. + If `mode` is "literal", this string will be matched exactly. + If `mode` is "regex", this string will be treated as a regular expression (syntax of Python's `re` module, + with flags DOTALL and MULTILINE enabled). + :param repl: the replacement string (verbatim). + If mode is "regex", the string can contain backreferences to matched groups in the needle regex, + specified using the syntax $!1, $!2, etc. for groups 1, 2, etc. + :param mode: either "literal" or "regex", specifying how the `needle` parameter is to be interpreted. + :param allow_multiple_occurrences: whether to allow matching and replacing multiple occurrences. + If false and multiple occurrences are found, an error will be raised + :return: a success message + """ + self._get_project().validate_relative_path(relative_path) + with EditedFileContext(relative_path, self._create_code_editor()) as context: + replacer = ContentReplacer(mode=mode, allow_multiple_occurrences=allow_multiple_occurrences) + context.set_updated_content(replacer.replace(context.get_original_content(), needle, repl)) + return SUCCESS_RESULT + + @facade_method(can_edit=True) + def replace_in_files( + self, + needle: str, + repl: str, + mode: Literal["literal", "regex"], + relative_path: str = "", + paths_include_glob: str = "", + paths_exclude_glob: str = "", + dry_run: bool = False, + occurrence_ids: list[str] | None = None, + expected_count: int = -1, + max_answer_chars: int = -1, + ) -> ReplacementPreview | str: + r""" + Replaces occurrences of a pattern across multiple files in ONE call. + + This is the preferred operation for repeated small edits (renames, import swaps, annotation changes, + path prefixes) spanning several files or many places in one file: one call with a SHORT pattern + replaces many single-file replacements with long disambiguating needles. + + Recommended protocol whenever there is ANY risk of unintended replacements: + 1. Call with dry_run=True: every prospective change is returned as a minimal line diff with an + occurrence id; nothing is modified. + 2. Call again with dry_run=False, passing the ids you want in occurrence_ids (omit it to apply + all). You pick the desired replacements from the list - no counting, no needle-crafting. + + For clearly unambiguous bulk replacements you may skip the dry run; pass expected_count as a + guard. If the actual number of matches differs, NOTHING is changed and an error containing the + prospective changes is raised, so a failed guard costs one call and gives you the dry-run output to select from. + + :param needle: the string (mode "literal") or regular expression (mode "regex"; Python `re` + syntax with DOTALL and MULTILINE) to search for + :param repl: the replacement string. In regex mode, backreferences to matched groups can be + specified as $!1, $!2, etc. + :param mode: either "literal" or "regex", specifying how `needle` is to be interpreted + :param relative_path: only consider this file or directory (default: the whole project) + :param paths_include_glob: optional glob (relative to the project root, e.g. "src/**/*.java") + restricting which files are considered + :param paths_exclude_glob: optional glob of files to exclude; takes precedence over the include glob + :param dry_run: if True, do not modify anything; return the prospective changes with occurrence ids + :param occurrence_ids: optional list of occurrence ids (obtained from a dry run) to which the + replacement is restricted; if any id is unknown or stale, NOTHING is changed. If omitted, + all occurrences are replaced. + :param expected_count: optional guard for calls without occurrence_ids: the number of + occurrences you expect to be replaced. If the actual count differs, nothing is changed and + an error containing the prospective changes is raised. -1 disables the guard. + :param max_answer_chars: if the rendered output exceeds this many characters, a shortened version is + used. -1 uses the configured default. + :return: in a dry run, the prospective changes (`ReplacementPreview`); otherwise a summary of the applied replacements + """ + replacement = MultiFileReplacement( + self._get_project(), + needle, + repl, + mode, + relative_path=relative_path, + paths_include_glob=paths_include_glob, + paths_exclude_glob=paths_exclude_glob, + ) + if dry_run: + return ReplacementPreview(replacement, ReplacementPreviewRenderer(self._agent, max_answer_chars, dry_run=True)) + + # select the occurrences to replace + try: + if occurrence_ids is not None: + occurrences = replacement.select(occurrence_ids) + else: + occurrences = replacement.select_all_guarded(expected_count) + except ReplacementRejectedError as e: + message = str(e) + if e.show_prospective_changes: + preview = ReplacementPreview(replacement, ReplacementPreviewRenderer(self._agent, max_answer_chars, dry_run=False)) + message += "\n" + preview.represent() + raise ValueError(message) from e + + return replacement.apply(self._create_code_editor(), occurrences).to_display_string() + + # line-level operations + + @facade_method(optional=True, can_edit=True) + def delete_lines(self, relative_path: str, start_line: int, end_line: int) -> str: + """ + Deletes the given lines in the file. + Requires that the same range of lines was previously read to verify correctness of the operation. + + :param relative_path: the relative path to the file + :param start_line: the 0-based index of the first line to be deleted + :param end_line: the 0-based index of the last line to be deleted + :return: a success message + """ + self._create_code_editor().delete_lines(relative_path, start_line, end_line) + return SUCCESS_RESULT + + @facade_method(optional=True, can_edit=True) + def replace_lines(self, relative_path: str, start_line: int, end_line: int, content: str) -> str: + """ + Replaces the given range of lines in the given file. + Requires that the same range of lines was previously read to verify correctness of the operation. + + :param relative_path: the relative path to the file + :param start_line: the 0-based index of the first line to be replaced + :param end_line: the 0-based index of the last line to be replaced + :param content: the content to insert + :return: a success message + """ + code_editor = self._create_code_editor() + code_editor.delete_lines(relative_path, start_line, end_line) + code_editor.insert_at_line(relative_path, start_line, self._normalize_inserted_content(content)) + return SUCCESS_RESULT + + @facade_method(optional=True, can_edit=True) + def insert_at_line(self, relative_path: str, line: int, content: str) -> str: + """ + Inserts the given content at the given line in the file, pushing existing content of the line down. + In general, symbolic insert operations like insert_after_symbol or insert_before_symbol should be preferred if you know which + symbol you are looking for. + However, this can also be useful for small targeted edits of the body of a longer symbol (without replacing the entire body). + + :param relative_path: the relative path to the file + :param line: the 0-based index of the line to insert content at + :param content: the content to be inserted + :return: a success message + """ + self._create_code_editor().insert_at_line(relative_path, line, self._normalize_inserted_content(content)) + return SUCCESS_RESULT + + @staticmethod + def _normalize_inserted_content(content: str) -> str: + return content if content.endswith("\n") else content + "\n" + + # symbol-level operations + + @facade_method(can_edit=True) + def replace_symbol_body(self, name_path: str, relative_path: str, body: str) -> str: + """ + Replaces the body of the given symbol. + + IMPORTANT: Only replace symbol bodies if you have previously made a retrieval with include_body=True and thus know what + constitutes the body! + + :param name_path: name path of the symbol whose body to replace + :param relative_path: the relative path to the file containing the symbol + :param body: the new symbol body. The symbol body is the definition of a symbol + in the programming language, including e.g. the signature line for functions. + Depending on the language, it may or may not include a preceding docstring or other preceding annotations. + :return: a success message + """ + self._create_code_editor().replace_body(name_path, relative_file_path=relative_path, body=body) + return SUCCESS_RESULT + + @facade_method(can_edit=True) + def insert_after_symbol(self, name_path: str, relative_path: str, body: str) -> str: + """ + Inserts code after a class/method/function definition. + Don't use this to insert after assignments (constants, fields). + + :param name_path: name path of the symbol after which to insert content + :param relative_path: the relative path to the file containing the symbol + :param body: the body/content to be inserted. The inserted code shall begin with the next line after + the symbol. + :return: a success message + """ + self._create_code_editor().insert_after_symbol(name_path, relative_file_path=relative_path, body=body) + return SUCCESS_RESULT + + @facade_method(can_edit=True) + def insert_before_symbol(self, name_path: str, relative_path: str, body: str) -> str: + """ + Inserts the given content before the beginning of the definition of the given symbol (via the symbol's location). + A typical use case is to insert a new class, function, method, field or variable assignment; or + a new import statement before the first symbol in the file. + + :param name_path: name path of the symbol before which to insert content + :param relative_path: the relative path to the file containing the symbol + :param body: the body/content to be inserted before the line in which the referenced symbol is defined + :return: a success message + """ + self._create_code_editor().insert_before_symbol(name_path, relative_file_path=relative_path, body=body) + return SUCCESS_RESULT diff --git a/src/serena/facades/api/lsp.py b/src/serena/facades/api/lsp.py index 46cdc882..4b2fe62d 100644 --- a/src/serena/facades/api/lsp.py +++ b/src/serena/facades/api/lsp.py @@ -299,7 +299,7 @@ class LspApi(FacadeApi): assert self._agent.get_language_backend().is_lsp(), "Language server operations require the language server backend" return LanguageServerSymbolRetriever(self._get_project()) - def _create_code_editor(self, symbol_retriever: LanguageServerSymbolRetriever | None = None) -> LanguageServerCodeEditor: + def _create_ls_code_editor(self, symbol_retriever: LanguageServerSymbolRetriever | None = None) -> LanguageServerCodeEditor: return LanguageServerCodeEditor(symbol_retriever or self._create_symbol_retriever()) @staticmethod @@ -560,7 +560,7 @@ class LspApi(FacadeApi): symbol_retriever = self._create_symbol_retriever() # find relevant location for lookup - editor = self._create_code_editor(symbol_retriever) + editor = self._create_ls_code_editor(symbol_retriever) if not containing_symbol_name_path: content = editor.read_file(relative_path) coords = find_text_coordinates(content, regex, require_unique=True) @@ -666,54 +666,6 @@ class LspApi(FacadeApi): # edit operations - @facade_method(can_edit=True) - def replace_symbol_body(self, name_path: str, relative_path: str, body: str) -> str: - """ - Replaces the body of the given symbol. - - IMPORTANT: Only replace symbol bodies if you have previously made a retrieval with include_body=True and thus know what - constitutes the body! - - :param name_path: name path of the symbol whose body to replace - :param relative_path: the relative path to the file containing the symbol - :param body: the new symbol body. The symbol body is the definition of a symbol - in the programming language, including e.g. the signature line for functions. - Depending on the language, it may or may not include a preceding docstring or other preceding annotations. - :return: a success message - """ - self._create_code_editor().replace_body(name_path, relative_file_path=relative_path, body=body) - return SUCCESS_RESULT - - @facade_method(can_edit=True) - def insert_after_symbol(self, name_path: str, relative_path: str, body: str) -> str: - """ - Inserts code after a class/method/function definition. - Don't use this to insert after assignments (constants, fields). - - :param name_path: name path of the symbol after which to insert content - :param relative_path: the relative path to the file containing the symbol - :param body: the body/content to be inserted. The inserted code shall begin with the next line after - the symbol. - :return: a success message - """ - self._create_code_editor().insert_after_symbol(name_path, relative_file_path=relative_path, body=body) - return SUCCESS_RESULT - - @facade_method(can_edit=True) - def insert_before_symbol(self, name_path: str, relative_path: str, body: str) -> str: - """ - Inserts the given content before the beginning of the definition of the given symbol (via the symbol's location). - A typical use case is to insert a new class, function, method, field or variable assignment; or - a new import statement before the first symbol in the file. - - :param name_path: name path of the symbol before which to insert content - :param relative_path: the relative path to the file containing the symbol - :param body: the body/content to be inserted before the line in which the referenced symbol is defined - :return: a success message - """ - self._create_code_editor().insert_before_symbol(name_path, relative_file_path=relative_path, body=body) - return SUCCESS_RESULT - @facade_method(can_edit=True) def rename_symbol(self, name_path: str, relative_path: str, new_name: str) -> str: """ @@ -727,7 +679,7 @@ class LspApi(FacadeApi): :return: a result summary indicating success or failure """ self._get_project().ls_sync_file_system_changes() - return self._create_code_editor().rename_symbol(name_path, relative_path=relative_path, new_name=new_name) + return self._create_ls_code_editor().rename_symbol(name_path, relative_path=relative_path, new_name=new_name) @facade_method(can_edit=True) def safe_delete_symbol(self, name_path_pattern: str, relative_path: str) -> str: @@ -765,5 +717,5 @@ class LspApi(FacadeApi): if file_to_lines: return f"Cannot delete, the symbol {symbol_name_path} is referenced in: {TextOutputUtils.to_json(file_to_lines)}" - self._create_code_editor(symbol_retriever).delete_symbol(symbol_name_path, relative_file_path=symbol_rel_path) + self._create_ls_code_editor(symbol_retriever).delete_symbol(symbol_name_path, relative_file_path=symbol_rel_path) return SUCCESS_RESULT diff --git a/src/serena/facades/facade.py b/src/serena/facades/facade.py index dc7395cb..a20080d7 100644 --- a/src/serena/facades/facade.py +++ b/src/serena/facades/facade.py @@ -16,6 +16,7 @@ from serena.project import Project if TYPE_CHECKING: from serena.agent import SerenaAgent + from serena.code_editor import CodeEditor log = logging.getLogger(__name__) TCallable = TypeVar("TCallable", bound=Callable[..., Any]) @@ -104,6 +105,22 @@ class FacadeApi(ABC): def _get_project(self) -> Project: return self._agent.get_active_project_or_raise() + def _create_code_editor(self) -> "CodeEditor": + """ + :return: a code editor for the active project, using the active language backend + """ + from serena.code_editor import JetBrainsCodeEditor, LanguageServerCodeEditor + from serena.symbol import LanguageServerSymbolRetriever + + project = self._get_project() + backend = self._agent.get_language_backend() + if backend.is_lsp(): + return LanguageServerCodeEditor(LanguageServerSymbolRetriever(project)) + elif backend.is_jetbrains(): + return JetBrainsCodeEditor(project) + else: + raise ValueError(f"Unsupported language backend: {backend}") + class FacadeMethod: """ diff --git a/src/serena/facades/representable.py b/src/serena/facades/representable.py index 8b79e0c8..1b89a40e 100644 --- a/src/serena/facades/representable.py +++ b/src/serena/facades/representable.py @@ -35,13 +35,16 @@ class Renderer(Generic[T], ABC): version of the result. They are tried in order until one fits within ``max_answer_chars``. :return: the result string, potentially replaced by a shortened version """ - max_answer_chars = self._max_answer_chars - if max_answer_chars == -1: - max_answer_chars = self._agent.serena_config.default_max_tool_answer_chars return TextOutputUtils.limit_length( - result=result, max_answer_chars=max_answer_chars, shortened_result_factories=shortened_result_factories + result=result, max_answer_chars=self._get_max_answer_chars(), shortened_result_factories=shortened_result_factories ) + def _get_max_answer_chars(self) -> int: + """ + :return: the effective maximum number of characters, resolving the default from the configuration + """ + return self._agent.serena_config.default_max_tool_answer_chars if self._max_answer_chars == -1 else self._max_answer_chars + def _to_json(self, x: Any) -> str: return TextOutputUtils.to_json(x) diff --git a/src/serena/tools/file_tools.py b/src/serena/tools/file_tools.py index 7874f999..dcd8a7cc 100644 --- a/src/serena/tools/file_tools.py +++ b/src/serena/tools/file_tools.py @@ -10,15 +10,24 @@ File and file system-related tools, specifically for import os from collections import defaultdict from fnmatch import fnmatch -from pathlib import Path -from typing import Literal +from typing import Literal, cast -from serena.tools import SUCCESS_RESULT, EditedFileContext, EditingToolWithDiagnostics, Tool, ToolMarkerOptional +from serena.facades.api.edit import EditApi +from serena.tools import EditingToolWithDiagnostics, Tool, ToolMarkerOptional from serena.util.file_system import scan_directory -from serena.util.text_utils import ContentReplacer, MultiFileReplacement, ReplacementRejectedError from solidlsp.ls_utils import TextUtils +class EditApiMixin: + """ + Mixin for tools which delegate to the editing API + """ + + def _api(self) -> EditApi: + tool = cast(Tool, cast(object, self)) + return EditApi(tool.agent) + + class ReadFileTool(Tool): """ Reads a file within the project directory. @@ -51,7 +60,7 @@ class ReadFileTool(Tool): return self._limit_length(result, max_answer_chars) -class CreateTextFileTool(EditingToolWithDiagnostics): +class CreateTextFileTool(EditingToolWithDiagnostics, EditApiMixin): """ Creates/overwrites a file in the project directory. """ @@ -65,26 +74,7 @@ class CreateTextFileTool(EditingToolWithDiagnostics): :return: a message indicating success or failure """ with self.diagnostics_context(relative_path) as diagnostics_context: - # validating the destination path - project_root = self.get_project_root() - abs_path = (Path(project_root) / relative_path).resolve() - will_overwrite_existing = abs_path.exists() - - if will_overwrite_existing: - self.project.validate_relative_path(relative_path) - else: - assert abs_path.is_relative_to(self.get_project_root()), ( - f"Cannot create file outside of the project directory, got {relative_path=}" - ) - - # writing the file - abs_path.parent.mkdir(parents=True, exist_ok=True) - abs_path.write_text(content, encoding=self.project.project_config.encoding, newline=self.project.line_ending.newline_str) - answer = f"File created: {relative_path}." - if will_overwrite_existing: - answer += " Overwrote existing file." - - return diagnostics_context.format_result(answer) + return diagnostics_context.format_result(self._api().create_text_file(relative_path, content)) class ListDirTool(Tool): @@ -166,7 +156,7 @@ class FindFileTool(Tool): return result -class ReplaceContentTool(EditingToolWithDiagnostics): +class ReplaceContentTool(EditingToolWithDiagnostics, EditApiMixin): """ Replaces content in a file (optionally using regular expressions). """ @@ -202,16 +192,12 @@ class ReplaceContentTool(EditingToolWithDiagnostics): If false and multiple occurrences are found, an error will be returned """ with self.diagnostics_context(relative_path) as diagnostics_context: - self.project.validate_relative_path(relative_path) - with EditedFileContext(relative_path, self.create_code_editor()) as context: - original_content = context.get_original_content() - replacer = ContentReplacer(mode=mode, allow_multiple_occurrences=allow_multiple_occurrences) - updated_content = replacer.replace(original_content, needle, repl) - context.set_updated_content(updated_content) - return diagnostics_context.format_result(SUCCESS_RESULT) + return diagnostics_context.format_result( + self._api().replace_content(relative_path, needle, repl, mode, allow_multiple_occurrences=allow_multiple_occurrences) + ) -class ReplaceInFilesTool(EditingToolWithDiagnostics): +class ReplaceInFilesTool(EditingToolWithDiagnostics, EditApiMixin): """ Replaces occurrences of a pattern across multiple files, with dry-run preview and per-occurrence selection. """ @@ -267,38 +253,28 @@ class ReplaceInFilesTool(EditingToolWithDiagnostics): returned. -1 uses the configured default. :return: in a dry run, the prospective changes; otherwise a summary of the applied replacements """ - max_answer_chars = self._resolve_max_answer_chars(max_answer_chars) - replacement = MultiFileReplacement( - self.project, - needle, - repl, - mode, - relative_path=relative_path, - paths_include_glob=paths_include_glob, - paths_exclude_glob=paths_exclude_glob, - ) + api = self._api() if dry_run: - return replacement.render_listing(max_answer_chars, dry_run=True) - - # select the occurrences to replace - try: - if occurrence_ids is not None: - occurrences = replacement.select(occurrence_ids) - else: - occurrences = replacement.select_all_guarded(expected_count) - except ReplacementRejectedError as e: - message = str(e) - if e.show_prospective_changes: - message += "\n" + replacement.render_listing(max_answer_chars, dry_run=False) - raise ValueError(message) from e - - # apply the replacement - with self.diagnostics_context(*replacement.affected_files) as diagnostics_context: - result = replacement.apply(self.create_code_editor(), occurrences) - return diagnostics_context.format_result(result.to_display_string()) + return api.replace_in_files( + needle, repl, mode, relative_path, paths_include_glob, paths_exclude_glob, dry_run=True, max_answer_chars=max_answer_chars + ).represent() + with self.diagnostics_context() as diagnostics_context: + result = api.replace_in_files( + needle, + repl, + mode, + relative_path, + paths_include_glob, + paths_exclude_glob, + occurrence_ids=occurrence_ids, + expected_count=expected_count, + max_answer_chars=max_answer_chars, + ) + assert isinstance(result, str) + return diagnostics_context.format_result(result) -class DeleteLinesTool(EditingToolWithDiagnostics, ToolMarkerOptional): +class DeleteLinesTool(EditingToolWithDiagnostics, ToolMarkerOptional, EditApiMixin): """ Deletes a range of lines within a file. """ @@ -319,12 +295,10 @@ class DeleteLinesTool(EditingToolWithDiagnostics, ToolMarkerOptional): :param end_line: the 0-based index of the last line to be deleted """ with self.diagnostics_context(relative_path) as diagnostics_context: - code_editor = self.create_code_editor() - code_editor.delete_lines(relative_path, start_line, end_line) - return diagnostics_context.format_result(SUCCESS_RESULT) + return diagnostics_context.format_result(self._api().delete_lines(relative_path, start_line, end_line)) -class ReplaceLinesTool(EditingToolWithDiagnostics, ToolMarkerOptional): +class ReplaceLinesTool(EditingToolWithDiagnostics, ToolMarkerOptional, EditApiMixin): """ Replaces a range of lines within a file with new content. """ @@ -346,19 +320,11 @@ class ReplaceLinesTool(EditingToolWithDiagnostics, ToolMarkerOptional): :param end_line: the 0-based index of the last line to be deleted :param content: the content to insert """ - # normalizing the replacement content - if not content.endswith("\n"): - content += "\n" - with self.diagnostics_context(relative_path) as diagnostics_context: - code_editor = self.create_code_editor() - code_editor.delete_lines(relative_path, start_line, end_line) - code_editor.insert_at_line(relative_path, start_line, content) - - return diagnostics_context.format_result(SUCCESS_RESULT) + return diagnostics_context.format_result(self._api().replace_lines(relative_path, start_line, end_line, content)) -class InsertAtLineTool(EditingToolWithDiagnostics, ToolMarkerOptional): +class InsertAtLineTool(EditingToolWithDiagnostics, ToolMarkerOptional, EditApiMixin): """ Inserts content at a given line in a file. """ @@ -379,15 +345,8 @@ class InsertAtLineTool(EditingToolWithDiagnostics, ToolMarkerOptional): :param line: the 0-based index of the line to insert content at :param content: the content to be inserted """ - # normalizing the inserted content - if not content.endswith("\n"): - content += "\n" - with self.diagnostics_context(relative_path) as diagnostics_context: - code_editor = self.create_code_editor() - code_editor.insert_at_line(relative_path, line, content) - - return diagnostics_context.format_result(SUCCESS_RESULT) + return diagnostics_context.format_result(self._api().insert_at_line(relative_path, line, content)) class SearchForPatternTool(Tool): diff --git a/src/serena/tools/symbol_tools.py b/src/serena/tools/symbol_tools.py index b6df9c2c..a505325d 100644 --- a/src/serena/tools/symbol_tools.py +++ b/src/serena/tools/symbol_tools.py @@ -3,7 +3,7 @@ Language server-related tools """ # SPDX-License-Identifier: GPL-3.0-or-later -from typing import TYPE_CHECKING, cast +from typing import cast from serena.facades.api.lsp import LspApi from serena.tools import ( @@ -12,11 +12,9 @@ from serena.tools import ( ToolMarkerSymbolicEdit, ToolMarkerSymbolicRead, ) +from serena.tools.file_tools import EditApiMixin from serena.tools.tools_base import ToolMarkerOptional -if TYPE_CHECKING: - pass - class LspApiMixin: """ @@ -333,7 +331,7 @@ class GetDiagnosticsForSymbolTool(Tool, ToolMarkerSymbolicRead, ToolMarkerOption ) -class ReplaceSymbolBodyTool(EditingToolWithDiagnostics, LspApiMixin): +class ReplaceSymbolBodyTool(EditingToolWithDiagnostics, EditApiMixin): """ Replaces the full definition of a symbol using the language server backend. """ @@ -361,7 +359,7 @@ class ReplaceSymbolBodyTool(EditingToolWithDiagnostics, LspApiMixin): return diagnostics_context.format_result(result) -class InsertAfterSymbolTool(EditingToolWithDiagnostics, LspApiMixin): +class InsertAfterSymbolTool(EditingToolWithDiagnostics, EditApiMixin): """ Inserts content after the end of the definition of a given symbol. """ @@ -386,7 +384,7 @@ class InsertAfterSymbolTool(EditingToolWithDiagnostics, LspApiMixin): return diagnostics_context.format_result(result) -class InsertBeforeSymbolTool(EditingToolWithDiagnostics, LspApiMixin): +class InsertBeforeSymbolTool(EditingToolWithDiagnostics, EditApiMixin): """ Inserts content before the beginning of the definition of a given symbol. """ diff --git a/test/serena/test_edit_api.py b/test/serena/test_edit_api.py new file mode 100644 index 00000000..fe158be7 --- /dev/null +++ b/test/serena/test_edit_api.py @@ -0,0 +1,81 @@ +""" +Tests for the editing facade API (backend-independent parts, without a language server). +""" + +from pathlib import Path +from unittest.mock import MagicMock + +import pytest + +from serena.config.serena_config import SerenaConfig +from serena.facades.api.edit import EditApi, ReplacementPreview +from serena.facades.facade import ApiScope, Facade +from serena.project import Project + + +@pytest.fixture +def project(tmp_path: Path) -> Project: + (tmp_path / "a.py").write_text("x = foo(1)\ny = foo(2)\n", encoding="utf-8") + (tmp_path / "b.py").write_text("z = foo(3)\n", encoding="utf-8") + return Project.load(str(tmp_path), serena_config=SerenaConfig(gui_log_window=False, web_dashboard=False)) + + +@pytest.fixture +def api(project: Project) -> EditApi: + agent = MagicMock() + agent.get_active_project_or_raise.return_value = project + agent.serena_config.default_max_tool_answer_chars = 10000 + return EditApi(agent) + + +def test_facade_exposes_editing_operations(api: EditApi) -> None: + default_methods = { + "create_text_file", + "replace_content", + "replace_in_files", + "replace_symbol_body", + "insert_after_symbol", + "insert_before_symbol", + } + optional_methods = {"delete_lines", "replace_lines", "insert_at_line"} # the line-level operations are optional (as are the tools) + + facade = Facade.from_api(api, ApiScope()) + assert facade.name == "edit" + assert set(facade.enabled_method_names) == default_methods + for name in optional_methods: + assert facade.get_method(name).info.optional + assert all(facade.get_method(name).info.can_edit for name in default_methods | optional_methods) + + +def test_create_text_file(api: EditApi, project: Project) -> None: + result = api.create_text_file("sub/new.txt", "hello\n") + assert "new.txt" in result + assert (Path(project.project_root) / "sub" / "new.txt").read_text(encoding="utf-8") == "hello\n" + + result = api.create_text_file("sub/new.txt", "changed\n") + assert "Overwrote" in result + + with pytest.raises(AssertionError): + api.create_text_file("../outside.txt", "nope") + + +def test_replace_in_files_dry_run_returns_inspectable_preview(api: EditApi, project: Project) -> None: + preview = api.replace_in_files("foo", "bar", mode="literal", dry_run=True) + assert isinstance(preview, ReplacementPreview) + + # the occurrences are accessible from code + assert [o.relative_path for o in preview.occurrences] == ["a.py", "a.py", "b.py"] + assert preview.affected_files == ["a.py", "b.py"] + assert all(o.replacement == "bar" for o in preview.occurrences) + + # the rendering lists the occurrence ids and diffs; nothing was modified + rendered = preview.represent() + assert "DRY RUN" in rendered + assert all(o.occurrence_id in rendered for o in preview.occurrences) + assert (Path(project.project_root) / "a.py").read_text(encoding="utf-8") == "x = foo(1)\ny = foo(2)\n" + + +def test_replace_in_files_guard_failure_includes_preview(api: EditApi) -> None: + with pytest.raises(ValueError, match="expected_count=1") as exc_info: + api.replace_in_files("foo", "bar", mode="literal", expected_count=1) + assert "b.py" in str(exc_info.value) # the listing of prospective changes is included From fee1aef934628618fb8329433b65e823d4b7e21b Mon Sep 17 00:00:00 2001 From: Dominik Jain Date: Mon, 7 Sep 2026 13:33:20 +0200 Subject: [PATCH 10/53] Rename package facades to repl and API modules to *_api --- src/serena/agent.py | 10 +++++----- src/serena/cli.py | 2 +- .../{facades/api/edit.py => repl/api/edit_api.py} | 0 src/serena/{facades/api/jb.py => repl/api/jb_api.py} | 0 src/serena/{facades/api/lsp.py => repl/api/lsp_api.py} | 0 src/serena/{facades => repl}/facade.py | 0 src/serena/{facades => repl}/repl.py | 0 src/serena/{facades => repl}/representable.py | 0 src/serena/tools/file_tools.py | 2 +- src/serena/tools/jetbrains_tools.py | 2 +- src/serena/tools/symbol_tools.py | 2 +- src/serena/tools/tools_base.py | 2 +- test/serena/test_edit_api.py | 4 ++-- test/serena/test_jetbrains_api.py | 6 +++--- test/serena/test_mcp.py | 4 ++-- test/serena/test_repl_tool.py | 6 +++--- 16 files changed, 20 insertions(+), 20 deletions(-) rename src/serena/{facades/api/edit.py => repl/api/edit_api.py} (100%) rename src/serena/{facades/api/jb.py => repl/api/jb_api.py} (100%) rename src/serena/{facades/api/lsp.py => repl/api/lsp_api.py} (100%) rename src/serena/{facades => repl}/facade.py (100%) rename src/serena/{facades => repl}/repl.py (100%) rename src/serena/{facades => repl}/representable.py (100%) diff --git a/src/serena/agent.py b/src/serena/agent.py index dbab9d56..afd1ec97 100644 --- a/src/serena/agent.py +++ b/src/serena/agent.py @@ -42,16 +42,16 @@ from serena.config.serena_config import ( ToolInclusionDefinition, ) from serena.dashboard import SerenaDashboardAPI, SerenaDashboardTrayManager, SerenaDashboardViewer, open_url_in_browser -from serena.facades.api.edit import EditApi -from serena.facades.api.jb import JetBrainsApi -from serena.facades.api.lsp import LspApi -from serena.facades.facade import ApiScope, Facade -from serena.facades.repl import SerenaRepl from serena.jetbrains import launch_coordinator as jetbrains_launch_coordinator from serena.ls_manager import LanguageServerManager from serena.memories.memory_manager import MemoryManager from serena.project import Project from serena.prompt_factory import SerenaPromptFactory +from serena.repl.api.edit_api import EditApi +from serena.repl.api.jb_api import JetBrainsApi +from serena.repl.api.lsp_api import LspApi +from serena.repl.facade import ApiScope, Facade +from serena.repl.repl import SerenaRepl from serena.task_executor import TaskExecutor from serena.tools import ( ActivateProjectTool, diff --git a/src/serena/cli.py b/src/serena/cli.py index bd7c4776..b8ae292c 100644 --- a/src/serena/cli.py +++ b/src/serena/cli.py @@ -931,8 +931,8 @@ class ProjectCommands(AutoRegisteringGroup): """ # NOTE: completely written by Claude Code, only functionality was reviewed, not implementation from serena.agent import SerenaAgent - from serena.facades.api.lsp import LspApi from serena.project import Project + from serena.repl.api.lsp_api import LspApi logging.configure(level=logging.INFO) project_path = os.path.abspath(project) diff --git a/src/serena/facades/api/edit.py b/src/serena/repl/api/edit_api.py similarity index 100% rename from src/serena/facades/api/edit.py rename to src/serena/repl/api/edit_api.py diff --git a/src/serena/facades/api/jb.py b/src/serena/repl/api/jb_api.py similarity index 100% rename from src/serena/facades/api/jb.py rename to src/serena/repl/api/jb_api.py diff --git a/src/serena/facades/api/lsp.py b/src/serena/repl/api/lsp_api.py similarity index 100% rename from src/serena/facades/api/lsp.py rename to src/serena/repl/api/lsp_api.py diff --git a/src/serena/facades/facade.py b/src/serena/repl/facade.py similarity index 100% rename from src/serena/facades/facade.py rename to src/serena/repl/facade.py diff --git a/src/serena/facades/repl.py b/src/serena/repl/repl.py similarity index 100% rename from src/serena/facades/repl.py rename to src/serena/repl/repl.py diff --git a/src/serena/facades/representable.py b/src/serena/repl/representable.py similarity index 100% rename from src/serena/facades/representable.py rename to src/serena/repl/representable.py diff --git a/src/serena/tools/file_tools.py b/src/serena/tools/file_tools.py index dcd8a7cc..044c19a0 100644 --- a/src/serena/tools/file_tools.py +++ b/src/serena/tools/file_tools.py @@ -12,7 +12,7 @@ from collections import defaultdict from fnmatch import fnmatch from typing import Literal, cast -from serena.facades.api.edit import EditApi +from serena.repl.api.edit_api import EditApi from serena.tools import EditingToolWithDiagnostics, Tool, ToolMarkerOptional from serena.util.file_system import scan_directory from solidlsp.ls_utils import TextUtils diff --git a/src/serena/tools/jetbrains_tools.py b/src/serena/tools/jetbrains_tools.py index 9f49efe5..0e877d5a 100644 --- a/src/serena/tools/jetbrains_tools.py +++ b/src/serena/tools/jetbrains_tools.py @@ -3,7 +3,7 @@ import logging from typing import TYPE_CHECKING, Literal -from serena.facades.api.jb import JetBrainsApi +from serena.repl.api.jb_api import JetBrainsApi from serena.tools import Tool, ToolMarkerBeta, ToolMarkerOptional, ToolMarkerSymbolicEdit, ToolMarkerSymbolicRead if TYPE_CHECKING: diff --git a/src/serena/tools/symbol_tools.py b/src/serena/tools/symbol_tools.py index a505325d..76022951 100644 --- a/src/serena/tools/symbol_tools.py +++ b/src/serena/tools/symbol_tools.py @@ -5,7 +5,7 @@ Language server-related tools from typing import cast -from serena.facades.api.lsp import LspApi +from serena.repl.api.lsp_api import LspApi from serena.tools import ( EditingToolWithDiagnostics, Tool, diff --git a/src/serena/tools/tools_base.py b/src/serena/tools/tools_base.py index 97fde923..0d494537 100644 --- a/src/serena/tools/tools_base.py +++ b/src/serena/tools/tools_base.py @@ -16,11 +16,11 @@ from sensai.util.string import dict_string from serena.code_editor import EditedFileContext # noqa: F401 (re-exported for tools) from serena.config.serena_config import LanguageBackend -from serena.facades.facade import SUCCESS_RESULT # noqa: F401 (re-exported for tools) from serena.lsp.lsp_diagnostics import DiagnosticsContext from serena.memories.memory_manager import MemoryManager from serena.project import Project from serena.prompt_factory import PromptFactory +from serena.repl.facade import SUCCESS_RESULT # noqa: F401 (re-exported for tools) from serena.util.class_decorators import singleton from serena.util.inspection import iter_subclasses from serena.util.text_utils import TextOutputUtils diff --git a/test/serena/test_edit_api.py b/test/serena/test_edit_api.py index fe158be7..a0520bfe 100644 --- a/test/serena/test_edit_api.py +++ b/test/serena/test_edit_api.py @@ -8,9 +8,9 @@ from unittest.mock import MagicMock import pytest from serena.config.serena_config import SerenaConfig -from serena.facades.api.edit import EditApi, ReplacementPreview -from serena.facades.facade import ApiScope, Facade from serena.project import Project +from serena.repl.api.edit_api import EditApi, ReplacementPreview +from serena.repl.facade import ApiScope, Facade @pytest.fixture diff --git a/test/serena/test_jetbrains_api.py b/test/serena/test_jetbrains_api.py index a7ab80fb..884c0d45 100644 --- a/test/serena/test_jetbrains_api.py +++ b/test/serena/test_jetbrains_api.py @@ -6,8 +6,8 @@ from unittest.mock import MagicMock, patch import pytest -from serena.facades.api.jb import JetBrainsApi -from serena.facades.facade import ApiScope, Facade +from serena.repl.api.jb_api import JetBrainsApi +from serena.repl.facade import ApiScope, Facade @pytest.fixture @@ -20,7 +20,7 @@ def agent() -> MagicMock: @pytest.fixture def client() -> MagicMock: client = MagicMock() - with patch("serena.facades.api.jb.JetBrainsPluginClient.from_project") as from_project: + with patch("serena.repl.api.jb_api.JetBrainsPluginClient.from_project") as from_project: from_project.return_value.__enter__.return_value = client yield client diff --git a/test/serena/test_mcp.py b/test/serena/test_mcp.py index 224e5d54..66119e8a 100644 --- a/test/serena/test_mcp.py +++ b/test/serena/test_mcp.py @@ -8,9 +8,9 @@ from serena import __version__ from serena.agent import Tool, ToolRegistry from serena.config.context_mode import SerenaAgentContext from serena.config.serena_config import SerenaConfig -from serena.facades.facade import ApiScope -from serena.facades.repl import SerenaRepl from serena.mcp import SerenaMCPFactory +from serena.repl.facade import ApiScope +from serena.repl.repl import SerenaRepl make_tool = SerenaMCPFactory.make_mcp_tool diff --git a/test/serena/test_repl_tool.py b/test/serena/test_repl_tool.py index 157b5a51..a2923915 100644 --- a/test/serena/test_repl_tool.py +++ b/test/serena/test_repl_tool.py @@ -8,9 +8,9 @@ from unittest.mock import MagicMock import pytest from serena.config.serena_config import ApiInclusionDefinition -from serena.facades.api.lsp import LspApi -from serena.facades.facade import ApiScope, Facade, FacadeApi, FacadeMethodInfo, facade_method -from serena.facades.repl import SerenaRepl +from serena.repl.api.lsp_api import LspApi +from serena.repl.facade import ApiScope, Facade, FacadeApi, FacadeMethodInfo, facade_method +from serena.repl.repl import SerenaRepl from serena.tools import SerenaReplTool from solidlsp.ls_config import LanguageServerId from test.conftest import agent_for_project_context From f38f12915b4e99bf4e5a45519932934d2697e2f3 Mon Sep 17 00:00:00 2001 From: Dominik Jain Date: Mon, 7 Sep 2026 13:41:15 +0200 Subject: [PATCH 11/53] Enable ty's unresolved-import rule Broken first-party imports previously went undetected, since the rule was globally disabled to accommodate optional extras and platform-specific modules. These are now handled narrowly: per-file overrides for the agno integration (optional extra, not installed in CI) and for the pywebview integration (several macOS-only imports), and an inline suppression for the single macOS-only import in the dashboard. --- pyproject.toml | 15 +++++++++++---- src/serena/dashboard.py | 2 +- 2 files changed, 12 insertions(+), 5 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 104de42f..05cfe4cf 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -114,10 +114,8 @@ python-version = "3.11" python-platform = "all" [tool.ty.rules] -# Mirror mypy's ignore_missing_imports=true: optional extras (e.g. agno, google-genai) and -# platform-specific modules (e.g. AppKit on macOS) are not installed in the default dev environment, -# so we do not want unresolvable imports to fail the type check. -unresolved-import = "ignore" +# Unresolvable imports do fail the type check (catching broken first-party imports); the exceptions for +# optional extras and platform-specific modules are handled per file/line below. possibly-missing-submodule = "ignore" [tool.ty.src] @@ -127,6 +125,15 @@ possibly-missing-submodule = "ignore" # test/resources is instead excluded from the `ty check test` CLI task via its --exclude flag. exclude = ["build/", "docs/"] +[[tool.ty.overrides]] +# Modules whose imports cannot be resolved in the default dev environment: the agno integration depends +# on the optional extra `agno`, and the pywebview integration imports macOS-only modules (AppKit, PyObjCTools) +# in several places. Elsewhere, individual platform-specific imports are suppressed inline. +include = ["src/serena/agno.py", "src/serena/util/pywebview.py"] + +[tool.ty.overrides.rules] +unresolved-import = "ignore" + [[tool.ty.overrides]] # Test code is heavily dynamic (pytest fixtures, MagicMock, intentionally loose Optionals). ty models # pytest's fail/skip helpers and MagicMock far more strictly than mypy did (mypy inferred `Any` for diff --git a/src/serena/dashboard.py b/src/serena/dashboard.py index 4d0574b8..26201a1f 100644 --- a/src/serena/dashboard.py +++ b/src/serena/dashboard.py @@ -1190,7 +1190,7 @@ class SerenaDashboardTrayManager: # set up tray icon with a dynamic menu (callable returns items on each open) kwargs: dict[str, Any] = {} if sys.platform == "darwin": - from AppKit import NSApplication, NSApplicationActivationPolicyAccessory + from AppKit import NSApplication, NSApplicationActivationPolicyAccessory # ty: ignore[unresolved-import] (macOS only) nsapp = NSApplication.sharedApplication() # run as an accessory app so that only the menu bar icon is shown (no Dock icon) From d9cda96725514ba3ef47d8ead21df1f961890559 Mon Sep 17 00:00:00 2001 From: Dominik Jain Date: Mon, 7 Sep 2026 13:47:44 +0200 Subject: [PATCH 12/53] Add instructions on commit messages --- .serena/memories/critical_info.md | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/.serena/memories/critical_info.md b/.serena/memories/critical_info.md index d7e9023e..55124efd 100644 --- a/.serena/memories/critical_info.md +++ b/.serena/memories/critical_info.md @@ -34,9 +34,11 @@ Snapshot tests use syrupy. * Each piece of information appears exactly once, at the element that owns it: callers do not explain callees' internals, and callees do not describe their callers. -# Pull requests +# Commits & pull requests -Read `mem:creating_pull_requests` when asked to participate in the creation of a pull request. +* Commit messages: wrap all lines (subject and body) at ~100 characters; write the message to a file and commit with `-F` + rather than passing long paragraphs via `-m`. +* Read `mem:creating_pull_requests` when asked to participate in the creation of a pull request. # Memories From 8e4d68cd400ee9b047b6d10bad757d9def548f72 Mon Sep 17 00:00:00 2001 From: Dominik Jain Date: Mon, 7 Sep 2026 14:12:00 +0200 Subject: [PATCH 13/53] Add memory facade (mem) including onboarding Introduce MemoryApi, exposing the memory operations list_memories, read_memory, write_memory, edit_memory, rename_memory and delete_memory (the latter four marked as editing operations) as well as onboarding, which provides the onboarding instructions. list_memories returns a MemoryList, which exposes the (writable and read-only) memory names to code and renders as JSON. The memory tools and the onboarding tool delegate to the API via MemoryApiMixin; the onboarding tool retains its check for the availability of the memory writing tool. The mem facade is always part of the REPL. The modes no-onboarding, no-memories and benchmark exclude the corresponding APIs (mem.onboarding, or the mem facade as a whole), mirroring their tool exclusions. --- src/serena/agent.py | 3 +- src/serena/repl/api/mem_api.py | 171 ++++++++++++++++++ .../resources/config/modes/benchmark.yml | 2 + .../resources/config/modes/no-memories.yml | 2 + .../resources/config/modes/no-onboarding.yml | 2 + src/serena/tools/memory_tools.py | 53 +++--- src/serena/tools/workflow_tools.py | 11 +- test/serena/test_mem_api.py | 68 +++++++ 8 files changed, 273 insertions(+), 39 deletions(-) create mode 100644 src/serena/repl/api/mem_api.py create mode 100644 test/serena/test_mem_api.py diff --git a/src/serena/agent.py b/src/serena/agent.py index afd1ec97..79bab64c 100644 --- a/src/serena/agent.py +++ b/src/serena/agent.py @@ -50,6 +50,7 @@ from serena.prompt_factory import SerenaPromptFactory from serena.repl.api.edit_api import EditApi from serena.repl.api.jb_api import JetBrainsApi from serena.repl.api.lsp_api import LspApi +from serena.repl.api.mem_api import MemoryApi from serena.repl.facade import ApiScope, Facade from serena.repl.repl import SerenaRepl from serena.task_executor import TaskExecutor @@ -1184,7 +1185,7 @@ class SerenaAgent: api_scope.exclude_editing() # gather facades - facades = [Facade.from_api(EditApi(self), api_scope)] + facades = [Facade.from_api(EditApi(self), api_scope), Facade.from_api(MemoryApi(self), api_scope)] if self._language_backend.is_lsp(): facades.append(Facade.from_api(LspApi(self), api_scope)) elif self._language_backend.is_jetbrains(): diff --git a/src/serena/repl/api/mem_api.py b/src/serena/repl/api/mem_api.py new file mode 100644 index 00000000..0497a7b3 --- /dev/null +++ b/src/serena/repl/api/mem_api.py @@ -0,0 +1,171 @@ +# SPDX-License-Identifier: GPL-3.0-or-later +""" +The implementation of memory operations (and onboarding, which creates the initial memories). +""" + +import logging +import platform +from typing import TYPE_CHECKING, Literal + +from serena.memories.memory_manager import MemoryManager + +from ..facade import FacadeApi, facade_method +from ..representable import Renderer, RepresentableViaRenderer + +if TYPE_CHECKING: + from serena.agent import SerenaAgent + +log = logging.getLogger(__name__) + + +class MemoryList(RepresentableViaRenderer): + """ + The available memories: `memories` (writable) and `read_only_memories` (e.g. global memories), each a list of names. + """ + + def __init__(self, memories_list: MemoryManager.MemoriesList, renderer: "MemoryListRenderer"): + """ + :param memories_list: the list of memories + :param renderer: the renderer to use for representing the list + """ + super().__init__(renderer) + self.memories_list_ = memories_list + + @property + def memories(self) -> list[str]: + return sorted(self.memories_list_.memories) + + @property + def read_only_memories(self) -> list[str]: + return sorted(self.memories_list_.read_only_memories) + + +class MemoryListRenderer(Renderer[MemoryList]): + def render(self, obj: MemoryList) -> str: + return self._limit_length(self._to_json(obj.memories_list_.to_dict())) + + +class MemoryApi(FacadeApi): + def __init__(self, agent: "SerenaAgent") -> None: + super().__init__( + agent, + name="mem", + description="project memories (persistent notes for future tasks): listing, reading, writing, editing; onboarding", + ) + + def _get_memory_manager(self) -> MemoryManager: + return self._get_project().memory_manager + + @facade_method() + def list_memories(self, topic: str = "") -> MemoryList: + """ + Lists the available memories, optionally filtered by topic. + + :param topic: the topic (prefix of the memory name, e.g. "frontend") to restrict the listing to; empty for all memories + :return: the memories + """ + return MemoryList(self._get_memory_manager().list_memories(topic), MemoryListRenderer(self._agent, -1)) + + @facade_method() + def read_memory(self, memory_name: str) -> str: + """ + Reads a memory that is likely to be relevant to the current task, inferring relevance e.g. from the name. + + :param memory_name: the name of the memory + :return: the memory's content + """ + return self._get_memory_manager().load_memory(memory_name) + + @facade_method(can_edit=True) + def write_memory(self, memory_name: str, content: str, max_chars: int = -1) -> str: + """ + Writes information about this project that can be useful for future tasks in md format. + The name should be meaningful and can include "/" to organize into topics. + If explicitly instructed, use the "global/" prefix for writing a memory that is shared across projects. + References to other memories should be inside backticks and prefixed with mem:, + e.g., `mem:auth`. + + :param memory_name: the memory name + :param content: the memory content (utf-8-encoded markdown) + :param max_chars: the maximum content length; -1 for the configured default + :return: a message indicating the result + """ + if max_chars == -1: + max_chars = self._agent.serena_config.default_max_tool_answer_chars + if len(content) > max_chars: + raise ValueError( + f"Content for {memory_name} is too long. Max length is {max_chars} characters. Please make the content shorter." + ) + return self._get_memory_manager().save_memory(memory_name, content, is_tool_context=True) + + @facade_method(can_edit=True) + def edit_memory( + self, + memory_name: str, + needle: str, + repl: str, + mode: Literal["literal", "regex"], + allow_multiple_occurrences: bool = False, + ) -> str: + """ + Replaces content matching a pattern in a memory. + + :param memory_name: the name of the memory + :param needle: the string or regex pattern to search for. In regex mode, be careful to not replace too much! + If `mode` is "literal", this string will be matched exactly. + If `mode` is "regex", this string will be treated as a regular expression (syntax of Python's `re` module, + with the MULTILINE and DOTALL flags enabled). + :param repl: the replacement string (verbatim). + :param mode: either "literal" or "regex", specifying how the `needle` parameter is to be interpreted. + :param allow_multiple_occurrences: whether to allow matching and replacing multiple occurrences. + If false and multiple occurrences are found, an error will be raised. + :return: a message indicating the result + """ + return self._get_memory_manager().edit_memory( + memory_name, needle, repl, mode, allow_multiple_occurrences, is_tool_context=True, regex_multiline=True + ) + + @facade_method(can_edit=True) + def rename_memory(self, old_name: str, new_name: str) -> str: + """ + Renames or moves a memory; use "/" in the name to organize into topics. + The "global" topic should only be used if explicitly instructed. + References to other memories that are marked with the `mem:` prefix will be updated accordingly. + References in read-only memories are not affected. + + :param old_name: the current name of the memory + :param new_name: the new name of the memory + :return: a message indicating the result + """ + renaming_message, n_references_updated = self._get_memory_manager().rename_memory_and_propagate_references( + old_name, new_name, is_tool_context=True + ) + if n_references_updated > 0: + log.info(f"Updated {n_references_updated} references to memory {old_name} to {new_name}") + return renaming_message + + @facade_method(can_edit=True) + def delete_memory(self, memory_name: str) -> str: + """ + Deletes a memory; only call this if instructed explicitly or permission was granted by the user. + + :param memory_name: the name of the memory + :return: a message indicating the result + """ + return self._get_memory_manager().delete_memory(memory_name, is_tool_context=True) + + @facade_method() + def onboarding(self) -> str: + """ + Provides the instructions for performing onboarding (identifying the project structure and essential tasks, + e.g. for testing or building, and recording the findings in memories). + Call this if onboarding was not performed yet, at most once per conversation. + + :return: the instructions on how to create the onboarding information + """ + # seed the project-local memory-maintenance memory (or detect a global override) so + # the prompt can point the agent at the conventions before it writes anything + memory_maintenance_name = self._get_memory_manager().ensure_memory_maintenance_memory() + return self._agent.prompt_factory.create_onboarding_prompt( + system=platform.system(), memory_maintenance_name=memory_maintenance_name + ) diff --git a/src/serena/resources/config/modes/benchmark.yml b/src/serena/resources/config/modes/benchmark.yml index 636f5f4b..04e7d8d7 100644 --- a/src/serena/resources/config/modes/benchmark.yml +++ b/src/serena/resources/config/modes/benchmark.yml @@ -18,3 +18,5 @@ excluded_tools: - delete_memory - rename_memory - onboarding +excluded_apis: + - mem diff --git a/src/serena/resources/config/modes/no-memories.yml b/src/serena/resources/config/modes/no-memories.yml index d4175eef..0d6c50a7 100644 --- a/src/serena/resources/config/modes/no-memories.yml +++ b/src/serena/resources/config/modes/no-memories.yml @@ -9,3 +9,5 @@ excluded_tools: - rename_memory - list_memories - onboarding +excluded_apis: + - mem diff --git a/src/serena/resources/config/modes/no-onboarding.yml b/src/serena/resources/config/modes/no-onboarding.yml index d441b5d3..bae5cb36 100644 --- a/src/serena/resources/config/modes/no-onboarding.yml +++ b/src/serena/resources/config/modes/no-onboarding.yml @@ -3,3 +3,5 @@ prompt: | The onboarding process is not applied. excluded_tools: - onboarding +excluded_apis: + - mem.onboarding diff --git a/src/serena/tools/memory_tools.py b/src/serena/tools/memory_tools.py index a6e78103..6c2e966a 100644 --- a/src/serena/tools/memory_tools.py +++ b/src/serena/tools/memory_tools.py @@ -1,14 +1,22 @@ # SPDX-License-Identifier: GPL-3.0-or-later -import logging -from typing import Literal +from typing import Literal, cast +from serena.repl.api.mem_api import MemoryApi from serena.tools import Tool, ToolMarkerCanEdit -log = logging.getLogger(__name__) + +class MemoryApiMixin: + """ + Mixin for tools which delegate to the memory API + """ + + def _api(self) -> MemoryApi: + tool = cast(Tool, cast(object, self)) + return MemoryApi(tool.agent) -class WriteMemoryTool(Tool, ToolMarkerCanEdit): +class WriteMemoryTool(Tool, ToolMarkerCanEdit, MemoryApiMixin): """ Write some information (utf-8-encoded) about this project that can be useful for future tasks to a memory in md format. The memory name should be meaningful. @@ -26,18 +34,10 @@ class WriteMemoryTool(Tool, ToolMarkerCanEdit): :param content: memory content, utf8-encoded :param max_chars: see other tools """ - # NOTE: utf-8 encoding is configured in the MemoriesManager - if max_chars == -1: - max_chars = self.agent.serena_config.default_max_tool_answer_chars - if len(content) > max_chars: - raise ValueError( - f"Content for {memory_name} is too long. Max length is {max_chars} characters. " + "Please make the content shorter." - ) - - return self.memory_manager.save_memory(memory_name, content, is_tool_context=True) + return self._api().write_memory(memory_name, content, max_chars) -class ReadMemoryTool(Tool): +class ReadMemoryTool(Tool, MemoryApiMixin): """ Reads the content of a memory file. """ @@ -46,10 +46,10 @@ class ReadMemoryTool(Tool): """ Use to read a memory that is likely to be relevant to the current task, inferring relevance e.g. from the name. """ - return self.memory_manager.load_memory(memory_name) + return self._api().read_memory(memory_name) -class ListMemoriesTool(Tool): +class ListMemoriesTool(Tool, MemoryApiMixin): """ Lists available memories. """ @@ -58,10 +58,10 @@ class ListMemoriesTool(Tool): """ Lists available memories, optionally filtered by topic. """ - return self._to_json(self.memory_manager.list_memories(topic).to_dict()) + return self._api().list_memories(topic).represent() -class DeleteMemoryTool(Tool, ToolMarkerCanEdit): +class DeleteMemoryTool(Tool, ToolMarkerCanEdit, MemoryApiMixin): """ Delete a memory file. """ @@ -70,10 +70,10 @@ class DeleteMemoryTool(Tool, ToolMarkerCanEdit): """ Delete a memory, only call if instructed explicitly or permission was granted by the user. """ - return self.memory_manager.delete_memory(memory_name, is_tool_context=True) + return self._api().delete_memory(memory_name) -class RenameMemoryTool(Tool, ToolMarkerCanEdit): +class RenameMemoryTool(Tool, ToolMarkerCanEdit, MemoryApiMixin): """ Renames or moves a memory, updating references that are marked with the `mem:` prefix. """ @@ -85,15 +85,10 @@ class RenameMemoryTool(Tool, ToolMarkerCanEdit): References to other memories that are marked with the `mem:` prefix will be updated accordingly. References in read-only memories are not affected. """ - renaming_message, n_references_updated = self.memory_manager.rename_memory_and_propagate_references( - old_name, new_name, is_tool_context=True - ) - if n_references_updated > 0: - log.info(f"Updated {n_references_updated} references to memory {old_name} to {new_name}") - return renaming_message + return self._api().rename_memory(old_name, new_name) -class EditMemoryTool(Tool, ToolMarkerCanEdit): +class EditMemoryTool(Tool, ToolMarkerCanEdit, MemoryApiMixin): """ Replaces content matching a regular expression in a memory. """ @@ -119,6 +114,4 @@ class EditMemoryTool(Tool, ToolMarkerCanEdit): :param allow_multiple_occurrences: whether to allow matching and replacing multiple occurrences. If false and multiple occurrences are found, an error will be returned. """ - return self.memory_manager.edit_memory( - memory_name, needle, repl, mode, allow_multiple_occurrences, is_tool_context=True, regex_multiline=True - ) + return self._api().edit_memory(memory_name, needle, repl, mode, allow_multiple_occurrences) diff --git a/src/serena/tools/workflow_tools.py b/src/serena/tools/workflow_tools.py index 8991eea1..bcf4468f 100644 --- a/src/serena/tools/workflow_tools.py +++ b/src/serena/tools/workflow_tools.py @@ -3,12 +3,11 @@ Tools supporting the general workflow of the agent """ # SPDX-License-Identifier: GPL-3.0-or-later -import platform - from serena.tools import Tool, ToolMarkerDoesNotRequireActiveProject, ToolMarkerOptional, WriteMemoryTool +from serena.tools.memory_tools import MemoryApiMixin -class OnboardingTool(Tool): +class OnboardingTool(Tool, MemoryApiMixin): """ Performs onboarding (identifying the project structure and essential tasks, e.g. for testing or building). """ @@ -23,11 +22,7 @@ class OnboardingTool(Tool): write_memory_tool_available = self.agent.tool_is_exposed(WriteMemoryTool.get_name_from_cls()) if not write_memory_tool_available: return "Memory writing tool not activated, skipping onboarding." - system = platform.system() - # seed the project-local memory-maintenance memory (or detect a global override) so - # the prompt can point the agent at the conventions before it writes anything - memory_maintenance_name = self.memory_manager.ensure_memory_maintenance_memory() - return self.prompt_factory.create_onboarding_prompt(system=system, memory_maintenance_name=memory_maintenance_name) + return self._api().onboarding() class InitialInstructionsTool(Tool, ToolMarkerDoesNotRequireActiveProject): diff --git a/test/serena/test_mem_api.py b/test/serena/test_mem_api.py new file mode 100644 index 00000000..f3f133f2 --- /dev/null +++ b/test/serena/test_mem_api.py @@ -0,0 +1,68 @@ +""" +Tests for the memory facade API. +""" + +from pathlib import Path +from unittest.mock import MagicMock + +import pytest + +from serena.config.serena_config import SerenaConfig +from serena.project import Project +from serena.repl.api.mem_api import MemoryApi +from serena.repl.facade import ApiScope, Facade + + +@pytest.fixture +def api(tmp_path: Path) -> MemoryApi: + project = Project.load(str(tmp_path), serena_config=SerenaConfig(gui_log_window=False, web_dashboard=False)) + agent = MagicMock() + agent.get_active_project_or_raise.return_value = project + agent.serena_config.default_max_tool_answer_chars = 10000 + return MemoryApi(agent) + + +def test_facade_exposes_memory_operations(api: MemoryApi) -> None: + facade = Facade.from_api(api, ApiScope()) + assert facade.name == "mem" + assert set(facade.enabled_method_names) == { + "list_memories", + "read_memory", + "write_memory", + "edit_memory", + "rename_memory", + "delete_memory", + "onboarding", + } + assert {name for name in facade.enabled_method_names if facade.get_method(name).info.can_edit} == { + "write_memory", + "edit_memory", + "rename_memory", + "delete_memory", + } + + +def test_memory_lifecycle(api: MemoryApi) -> None: + api.write_memory("topic/first", "# First\nhello") + api.write_memory("second", "see `mem:topic/first`") + + # listing exposes the names to code and renders as JSON (global memories of the machine may be present, too) + memory_list = api.list_memories() + assert {"second", "topic/first"} <= set(memory_list.memories) + assert api.list_memories("topic").memories == ["topic/first"] + assert '"memories"' in memory_list.represent() + + # reading, editing, renaming (with reference propagation) and deleting + assert api.read_memory("topic/first") == "# First\nhello" + api.edit_memory("topic/first", "hello", "world", mode="literal") + assert api.read_memory("topic/first") == "# First\nworld" + api.rename_memory("topic/first", "topic/renamed") + assert "mem:topic/renamed" in api.read_memory("second") + api.delete_memory("second") + assert "second" not in api.list_memories().memories + assert api.list_memories("topic").memories == ["topic/renamed"] + + +def test_write_memory_rejects_overlong_content(api: MemoryApi) -> None: + with pytest.raises(ValueError, match="too long"): + api.write_memory("big", "x" * 100, max_chars=10) From c0e74c7d7ea5060115a473b72bf74d3d2390f6d6 Mon Sep 17 00:00:00 2001 From: Dominik Jain Date: Mon, 7 Sep 2026 14:32:17 +0200 Subject: [PATCH 14/53] Describe facades by domain rather than by listing operations The method list accompanies every facade description, so naming operations in it was redundant. --- src/serena/repl/api/edit_api.py | 3 +-- src/serena/repl/api/jb_api.py | 3 +-- src/serena/repl/api/lsp_api.py | 3 +-- src/serena/repl/api/mem_api.py | 2 +- 4 files changed, 4 insertions(+), 7 deletions(-) diff --git a/src/serena/repl/api/edit_api.py b/src/serena/repl/api/edit_api.py index 815a8da0..12e1b09a 100644 --- a/src/serena/repl/api/edit_api.py +++ b/src/serena/repl/api/edit_api.py @@ -64,8 +64,7 @@ class EditApi(FacadeApi): super().__init__( agent, name="edit", - description="editing operations independent of the language backend (creating files, replacing content in one or " - "many files, editing at the symbol or line level)", + description="modifying content within existing files (independent of the language backend)", ) # file-level operations diff --git a/src/serena/repl/api/jb_api.py b/src/serena/repl/api/jb_api.py index 21337b2f..b3449f91 100644 --- a/src/serena/repl/api/jb_api.py +++ b/src/serena/repl/api/jb_api.py @@ -164,8 +164,7 @@ class JetBrainsApi(FacadeApi): super().__init__( agent, name="jb", - description="JetBrains IDE-backed operations (symbol search, references, type hierarchies, refactorings, " - "inspections, debugging)", + description="operations on the codebase backed by the JetBrains IDE's code intelligence", ) @contextmanager diff --git a/src/serena/repl/api/lsp_api.py b/src/serena/repl/api/lsp_api.py index 4b2fe62d..115f9772 100644 --- a/src/serena/repl/api/lsp_api.py +++ b/src/serena/repl/api/lsp_api.py @@ -291,8 +291,7 @@ class LspApi(FacadeApi): super().__init__( agent, name="lsp", - description="language server-backed operations on the codebase (finding symbols, references, implementations, " - "declarations and diagnostics; editing and renaming symbols)", + description="symbol-level operations on the codebase backed by language servers", ) def _create_symbol_retriever(self) -> LanguageServerSymbolRetriever: diff --git a/src/serena/repl/api/mem_api.py b/src/serena/repl/api/mem_api.py index 0497a7b3..60fc305c 100644 --- a/src/serena/repl/api/mem_api.py +++ b/src/serena/repl/api/mem_api.py @@ -50,7 +50,7 @@ class MemoryApi(FacadeApi): super().__init__( agent, name="mem", - description="project memories (persistent notes for future tasks): listing, reading, writing, editing; onboarding", + description="project memories, i.e. persistent notes for future tasks", ) def _get_memory_manager(self) -> MemoryManager: From 5f129628d916269d9b2196d3fbdac27b75b33a22 Mon Sep 17 00:00:00 2001 From: Dominik Jain Date: Mon, 7 Sep 2026 14:38:54 +0200 Subject: [PATCH 15/53] Add file system facade (fs); create_text_file moves there from edit Introduce FsApi, exposing operations on the project's files as units: read_file, create_text_file, list_dir, find_file and search_for_pattern. The boundary to the edit facade, which modifies content within existing files, is stated in the facade description. Results expose their data to code (FileContent, DirectoryListing, PatternMatches) and render as before; the pattern search's shortening ladder moves from the tool into PatternMatchesRenderer. The file tools delegate to the API via FsApiMixin. The fs facade is always part of the REPL. --- src/serena/agent.py | 7 +- src/serena/repl/api/edit_api.py | 29 --- src/serena/repl/api/fs_api.py | 320 ++++++++++++++++++++++++++++++++ src/serena/tools/file_tools.py | 192 ++++--------------- test/serena/test_edit_api.py | 13 -- test/serena/test_file_tools.py | 6 +- test/serena/test_fs_api.py | 81 ++++++++ 7 files changed, 446 insertions(+), 202 deletions(-) create mode 100644 src/serena/repl/api/fs_api.py create mode 100644 test/serena/test_fs_api.py diff --git a/src/serena/agent.py b/src/serena/agent.py index 79bab64c..911c360b 100644 --- a/src/serena/agent.py +++ b/src/serena/agent.py @@ -48,6 +48,7 @@ from serena.memories.memory_manager import MemoryManager from serena.project import Project from serena.prompt_factory import SerenaPromptFactory from serena.repl.api.edit_api import EditApi +from serena.repl.api.fs_api import FsApi from serena.repl.api.jb_api import JetBrainsApi from serena.repl.api.lsp_api import LspApi from serena.repl.api.mem_api import MemoryApi @@ -1185,7 +1186,11 @@ class SerenaAgent: api_scope.exclude_editing() # gather facades - facades = [Facade.from_api(EditApi(self), api_scope), Facade.from_api(MemoryApi(self), api_scope)] + facades = [ + Facade.from_api(FsApi(self), api_scope), + Facade.from_api(EditApi(self), api_scope), + Facade.from_api(MemoryApi(self), api_scope), + ] if self._language_backend.is_lsp(): facades.append(Facade.from_api(LspApi(self), api_scope)) elif self._language_backend.is_jetbrains(): diff --git a/src/serena/repl/api/edit_api.py b/src/serena/repl/api/edit_api.py index 12e1b09a..a96f5cc4 100644 --- a/src/serena/repl/api/edit_api.py +++ b/src/serena/repl/api/edit_api.py @@ -3,7 +3,6 @@ The implementation of editing operations, which are independent of the language backend. """ -from pathlib import Path from typing import TYPE_CHECKING, Literal from serena.code_editor import EditedFileContext @@ -69,34 +68,6 @@ class EditApi(FacadeApi): # file-level operations - @facade_method(can_edit=True) - def create_text_file(self, relative_path: str, content: str) -> str: - """ - Writes a new file or overwrites an existing file with the given content. - - :param relative_path: the relative path to the file to create - :param content: the (appropriately encoded) content to write to the file - :return: a message indicating success - """ - project = self._get_project() - project_root = Path(project.project_root) - abs_path = (project_root / relative_path).resolve() - will_overwrite_existing = abs_path.exists() - - # validate the destination path - if will_overwrite_existing: - project.validate_relative_path(relative_path) - else: - assert abs_path.is_relative_to(project_root), f"Cannot create file outside of the project directory, got {relative_path=}" - - # write the file - abs_path.parent.mkdir(parents=True, exist_ok=True) - abs_path.write_text(content, encoding=project.project_config.encoding, newline=project.line_ending.newline_str) - answer = f"File created: {relative_path}." - if will_overwrite_existing: - answer += " Overwrote existing file." - return answer - @facade_method(can_edit=True) def replace_content( self, diff --git a/src/serena/repl/api/fs_api.py b/src/serena/repl/api/fs_api.py new file mode 100644 index 00000000..afd4efca --- /dev/null +++ b/src/serena/repl/api/fs_api.py @@ -0,0 +1,320 @@ +# SPDX-License-Identifier: GPL-3.0-or-later +""" +The implementation of operations on the project's files. +""" + +import os +from collections import defaultdict +from fnmatch import fnmatch +from pathlib import Path +from typing import TYPE_CHECKING + +from serena.util.file_system import scan_directory +from serena.util.text_utils import MatchedConsecutiveLines +from solidlsp.ls_utils import TextUtils + +from ..facade import FacadeApi, facade_method +from ..representable import Renderer, RepresentableViaRenderer + +if TYPE_CHECKING: + from serena.agent import SerenaAgent + + +class FileContent(RepresentableViaRenderer): + """ + The content of a file (or of a range of its lines): `text` (the joined lines) and `lines`. + """ + + def __init__(self, lines: list[str], renderer: "FileContentRenderer"): + """ + :param lines: the lines (without line breaks) + :param renderer: the renderer to use for representing the content + """ + super().__init__(renderer) + self.lines = lines + + @property + def text(self) -> str: + return "\n".join(self.lines) + + +class FileContentRenderer(Renderer[FileContent]): + def render(self, obj: FileContent) -> str: + return self._limit_length(obj.text) + + +class DirectoryListing(RepresentableViaRenderer): + """ + The entries of a directory: `dirs` and `files` (relative paths). + """ + + def __init__(self, dirs: list[str], files: list[str], renderer: "DirectoryListingRenderer"): + """ + :param dirs: the relative paths of the directories + :param files: the relative paths of the files + :param renderer: the renderer to use for representing the listing + """ + super().__init__(renderer) + self.dirs = dirs + self.files = files + + +class DirectoryListingRenderer(Renderer[DirectoryListing]): + def render(self, obj: DirectoryListing) -> str: + return self._limit_length(self._to_json({"dirs": obj.dirs, "files": obj.files})) + + +class PatternMatches(RepresentableViaRenderer): + """ + The matches of a pattern search: `matches`, each a `MatchedConsecutiveLines` with `source_file_path`, + `matched_lines` (each with `line_number` and `line_content`) and `to_display_string()`. + """ + + def __init__(self, matches: list[MatchedConsecutiveLines], renderer: "PatternMatchesRenderer"): + """ + :param matches: the matches + :param renderer: the renderer to use for representing the matches + """ + super().__init__(renderer) + self.matches = matches + + def __len__(self) -> int: + return len(self.matches) + + def matches_by_file_(self) -> dict[str, list[MatchedConsecutiveLines]]: + result: defaultdict[str, list[MatchedConsecutiveLines]] = defaultdict(list) + for match in self.matches: + assert match.source_file_path is not None + result[match.source_file_path].append(match) + return result + + +class PatternMatchesRenderer(Renderer[PatternMatches]): + """ + Renders matches as a mapping from file paths to matched line blocks (with context), falling back to progressively + shorter representations (first lines, truncated first lines, line numbers, per-file counts, a summary) if the + length limit is exceeded. + """ + + _TEXT_TRUNCATE = 60 + + def render(self, obj: PatternMatches) -> str: + matches_by_file = obj.matches_by_file_() + file_to_matches = {path: [m.to_display_string() for m in matches] for path, matches in matches_by_file.items()} + + # capture lightweight match data for shortening before serialization + match_lines_by_file = { + path: [{"line": m.matched_lines[0].line_number, "text": m.matched_lines[0].line_content.strip()} for m in matches] + for path, matches in matches_by_file.items() + } + + # shortened result closures, from least to most aggressive shortening + def render_first_lines(truncate: bool) -> str: + """Render each match's first line, either in full or truncated to a fixed length.""" + + def entry_text(text: str) -> str: + if truncate and len(text) > self._TEXT_TRUNCATE: + return text[: self._TEXT_TRUNCATE] + "..." + return text + + compact = { + path: [{"line": m["line"], "text": entry_text(str(m["text"]))} for m in lines] + for path, lines in match_lines_by_file.items() + } + if truncate: + header = ( + f"Matched lines (text over {self._TEXT_TRUNCATE} chars is truncated, marked with a trailing '...'); " + "use read_file with the line numbers for full content:" + ) + else: + header = "Matched lines per file; use read_file with the line numbers for surrounding context:" + return f"{header}\n{self._to_json(compact)}" + + def make_first_lines_full() -> str: + return render_first_lines(truncate=False) + + def make_first_lines_truncated() -> str: + return render_first_lines(truncate=True) + + def make_line_numbers_only() -> str: + numbers = {path: [m["line"] for m in lines] for path, lines in match_lines_by_file.items()} + return f"Match lines per file:\n{self._to_json(numbers)}" + + def make_per_file_counts() -> str: + counts = {path: len(lines) for path, lines in match_lines_by_file.items()} + return f"Match counts per file:\n{self._to_json(counts)}" + + def make_summary() -> str: + return f"Found {len(obj)} matches in {len(match_lines_by_file)} files." + + return self._limit_length( + self._to_json(file_to_matches), + shortened_result_factories=[ + make_first_lines_full, + make_first_lines_truncated, + make_line_numbers_only, + make_per_file_counts, + make_summary, + ], + ) + + +class FsApi(FacadeApi): + def __init__(self, agent: "SerenaAgent") -> None: + super().__init__(agent, name="fs", description="the project's files as units (as opposed to their content, see `edit`)") + + @facade_method() + def read_file(self, relative_path: str, start_line: int = 0, end_line: int | None = None, max_answer_chars: int = -1) -> FileContent: + """ + Reads the given file or a range of its lines. + + :param relative_path: the relative path to the file to read + :param start_line: the 0-based index of the first line to be retrieved, negative values count from the end of the file. + :param end_line: the 0-based index of the last line to be retrieved (inclusive). If None, read until the end of the file. + :param max_answer_chars: if the rendered content is longer than this number of characters, no content is rendered; + -1 for the configured default. Don't adjust unless there is really no other way to get the content required for the task. + :return: the content + """ + project = self._get_project() + project.validate_relative_path(relative_path) + + # read lines, using the same (LSP-compliant) notion of line breaks as the line-based editing operations + lines = TextUtils.split_lines(project.read_file(relative_path)) + lines = lines[start_line:] if end_line is None else lines[start_line : end_line + 1] + return FileContent(lines, FileContentRenderer(self._agent, max_answer_chars)) + + @facade_method(can_edit=True) + def create_text_file(self, relative_path: str, content: str) -> str: + """ + Writes a new file or overwrites an existing file with the given content. + + :param relative_path: the relative path to the file to create + :param content: the (appropriately encoded) content to write to the file + :return: a message indicating success + """ + project = self._get_project() + project_root = Path(project.project_root) + abs_path = (project_root / relative_path).resolve() + will_overwrite_existing = abs_path.exists() + + # validate the destination path + if will_overwrite_existing: + project.validate_relative_path(relative_path) + else: + assert abs_path.is_relative_to(project_root), f"Cannot create file outside of the project directory, got {relative_path=}" + + # write the file + abs_path.parent.mkdir(parents=True, exist_ok=True) + abs_path.write_text(content, encoding=project.project_config.encoding, newline=project.line_ending.newline_str) + answer = f"File created: {relative_path}." + if will_overwrite_existing: + answer += " Overwrote existing file." + return answer + + @facade_method() + def list_dir( + self, relative_path: str, recursive: bool, skip_ignored_files: bool = False, max_answer_chars: int = -1 + ) -> DirectoryListing: + """ + Lists files and directories in the given directory (optionally with recursion). + + :param relative_path: the relative path to the directory to list; pass "." to scan the project root + :param recursive: whether to scan subdirectories recursively + :param skip_ignored_files: whether to skip files and directories that are ignored + :param max_answer_chars: if the rendered listing is longer than this number of characters, no content is rendered; + -1 for the configured default. Don't adjust unless there is really no other way to get the content required for the task. + :return: the listing + """ + project = self._get_project() + if not project.relative_path_exists(relative_path): + raise FileNotFoundError(f"Directory not found: {relative_path} (check if the path is correct relative to the project root)") + project.validate_relative_path(relative_path) + + is_ignored_path_fn = project.get_is_ignored_path_fn(relative_path, skip_ignored_files) + dirs, files = scan_directory( + os.path.join(project.project_root, relative_path), + relative_to=project.project_root, + recursive=recursive, + is_ignored_dir=is_ignored_path_fn, + is_ignored_file=is_ignored_path_fn, + ) + return DirectoryListing(dirs, files, DirectoryListingRenderer(self._agent, max_answer_chars)) + + @facade_method() + def find_file(self, file_mask: str, relative_path: str) -> list[str]: + """ + Finds files matching the given file mask within the given relative path. + + :param file_mask: the filename or file mask (using the wildcards * or ?) to search for + :param relative_path: the relative path to the directory to search in; pass "." to scan the project root + :return: the relative paths of the matching files + """ + project = self._get_project() + project.validate_relative_path(relative_path) + + is_ignored_path_fn = project.get_is_ignored_path_fn(relative_path, skip_ignored_paths=False) + + # find the files by ignoring everything that doesn't match + def is_ignored_file(abs_path: str) -> bool: + if is_ignored_path_fn(abs_path): + return True + return not fnmatch(os.path.basename(abs_path), file_mask) + + _dirs, files = scan_directory( + path=os.path.join(project.project_root, relative_path), + recursive=True, + is_ignored_dir=is_ignored_path_fn, + is_ignored_file=is_ignored_file, + relative_to=project.project_root, + ) + return files + + @facade_method() + def search_for_pattern( + self, + substring_pattern: str, + context_lines_before: int = 0, + context_lines_after: int = 0, + paths_include_glob: str = "", + paths_exclude_glob: str = "", + relative_path: str = "", + restrict_search_to_code_files: bool = False, + skip_ignored_files: bool = True, + multiline: bool = True, + max_answer_chars: int = -1, + ) -> PatternMatches: + """ + Searches for a regex pattern across project files, returning whole matched lines (plus optional context). + Prefer symbolic operations if you know which symbols you are looking for! + + :param substring_pattern: regular expression to search for. + :param context_lines_before: number of context lines to include before each match. + :param context_lines_after: number of context lines to include after each match. + :param paths_include_glob: optional glob (relative to project root, e.g. ``"src/**/*.ts"``) restricting which files are searched. + :param paths_exclude_glob: optional glob to exclude files; takes precedence over `paths_include_glob`. + :param relative_path: restricts the search to this file or subdirectory of the project root + :param restrict_search_to_code_files: whether to search only (non-ignored) files containing analyzable code symbols + (useful when looking for class/method definitions); otherwise also search non-code files. + :param skip_ignored_files: whether to skip ignored sub-paths (default: True) + :param multiline: whether to apply multi-line matching (default: True), enabling the flags re.DOTALL and re.MULTILINE + :param max_answer_chars: if the rendered output exceeds this many characters, a progressively shortened summary is + rendered instead. -1 for the configured default. + :return: the matches, rendered as a mapping from file paths to matched consecutive lines (0-based line numbers) + """ + project = self._get_project() + relative_path = relative_path.strip() + if relative_path: + project.validate_relative_path(relative_path) + + matches = project.search_project_files_for_pattern( + pattern=substring_pattern, + relative_path=relative_path, + context_lines_before=context_lines_before, + context_lines_after=context_lines_after, + paths_include_glob=paths_include_glob.strip(), + paths_exclude_glob=paths_exclude_glob.strip(), + multiline=multiline, + code_files_only=restrict_search_to_code_files, + skip_ignored_files=skip_ignored_files, + ) + return PatternMatches(matches, PatternMatchesRenderer(self._agent, max_answer_chars)) diff --git a/src/serena/tools/file_tools.py b/src/serena/tools/file_tools.py index 044c19a0..f9b256ba 100644 --- a/src/serena/tools/file_tools.py +++ b/src/serena/tools/file_tools.py @@ -7,15 +7,11 @@ File and file system-related tools, specifically for """ # SPDX-License-Identifier: GPL-3.0-or-later -import os -from collections import defaultdict -from fnmatch import fnmatch from typing import Literal, cast from serena.repl.api.edit_api import EditApi +from serena.repl.api.fs_api import FsApi from serena.tools import EditingToolWithDiagnostics, Tool, ToolMarkerOptional -from serena.util.file_system import scan_directory -from solidlsp.ls_utils import TextUtils class EditApiMixin: @@ -28,7 +24,17 @@ class EditApiMixin: return EditApi(tool.agent) -class ReadFileTool(Tool): +class FsApiMixin: + """ + Mixin for tools which delegate to the file system API + """ + + def _api(self) -> FsApi: + tool = cast(Tool, cast(object, self)) + return FsApi(tool.agent) + + +class ReadFileTool(Tool, FsApiMixin): """ Reads a file within the project directory. """ @@ -45,22 +51,10 @@ class ReadFileTool(Tool): required for the task. :return: the full text of the file at the given relative path """ - self.project.validate_relative_path(relative_path) - - # read lines, using the same (LSP-compliant) notion of line breaks as the line-based editing tools - result = self.project.read_file(relative_path) - result_lines = TextUtils.split_lines(result) - - if end_line is None: - result_lines = result_lines[start_line:] - else: - result_lines = result_lines[start_line : end_line + 1] - result = "\n".join(result_lines) - - return self._limit_length(result, max_answer_chars) + return self._api().read_file(relative_path, start_line, end_line, max_answer_chars).represent() -class CreateTextFileTool(EditingToolWithDiagnostics, EditApiMixin): +class CreateTextFileTool(EditingToolWithDiagnostics, FsApiMixin): """ Creates/overwrites a file in the project directory. """ @@ -77,7 +71,7 @@ class CreateTextFileTool(EditingToolWithDiagnostics, EditApiMixin): return diagnostics_context.format_result(self._api().create_text_file(relative_path, content)) -class ListDirTool(Tool): +class ListDirTool(Tool, FsApiMixin): """ Lists files and directories in the given directory (optionally with recursion). """ @@ -94,31 +88,13 @@ class ListDirTool(Tool): Don't adjust unless there is really no other way to get the content required for the task. :return: a JSON object with the names of directories and files within the given directory """ - # Check if the directory exists before validation - if not self.project.relative_path_exists(relative_path): - error_info = { - "error": f"Directory not found: {relative_path}", - "project_root": self.get_project_root(), - "hint": "Check if the path is correct relative to the project root", - } - return self._to_json(error_info) - - self.project.validate_relative_path(relative_path) - - is_ignored_path_fn = self.project.get_is_ignored_path_fn(relative_path, skip_ignored_files) - dirs, files = scan_directory( - os.path.join(self.get_project_root(), relative_path), - relative_to=self.get_project_root(), - recursive=recursive, - is_ignored_dir=is_ignored_path_fn, - is_ignored_file=is_ignored_path_fn, - ) - - result = self._to_json({"dirs": dirs, "files": files}) - return self._limit_length(result, max_answer_chars) + try: + return self._api().list_dir(relative_path, recursive, skip_ignored_files, max_answer_chars).represent() + except FileNotFoundError as e: + return self._to_json({"error": str(e), "project_root": self.get_project_root()}) -class FindFileTool(Tool): +class FindFileTool(Tool, FsApiMixin): """ Finds files in the given relative paths """ @@ -132,28 +108,7 @@ class FindFileTool(Tool): :param skip_ignored_files: whether to skip ignored files/directories :return: a JSON object with the list of matching files """ - self.project.validate_relative_path(relative_path) - - is_ignored_path_fn = self.project.get_is_ignored_path_fn(relative_path, skip_ignored_paths=False) - dir_to_scan = os.path.join(self.get_project_root(), relative_path) - - # find the files by ignoring everything that doesn't match - def is_ignored_file(abs_path: str) -> bool: - if is_ignored_path_fn(abs_path): - return True - filename = os.path.basename(abs_path) - return not fnmatch(filename, file_mask) - - _dirs, files = scan_directory( - path=dir_to_scan, - recursive=True, - is_ignored_dir=is_ignored_path_fn, - is_ignored_file=is_ignored_file, - relative_to=self.get_project_root(), - ) - - result = self._to_json({"files": files}) - return result + return self._to_json({"files": self._api().find_file(file_mask, relative_path)}) class ReplaceContentTool(EditingToolWithDiagnostics, EditApiMixin): @@ -349,7 +304,7 @@ class InsertAtLineTool(EditingToolWithDiagnostics, ToolMarkerOptional, EditApiMi return diagnostics_context.format_result(self._api().insert_at_line(relative_path, line, content)) -class SearchForPatternTool(Tool): +class SearchForPatternTool(Tool, FsApiMixin): def apply( self, substring_pattern: str, @@ -381,92 +336,19 @@ class SearchForPatternTool(Tool): ``-1`` uses the configured default. :return: A mapping from file paths to matched consecutive lines (0-based line numbers). """ - relative_path = relative_path.strip() - if relative_path: - self.project.validate_relative_path(relative_path) - - matches = self.project.search_project_files_for_pattern( - pattern=substring_pattern, - relative_path=relative_path, - context_lines_before=context_lines_before, - context_lines_after=context_lines_after, - paths_include_glob=paths_include_glob.strip(), - paths_exclude_glob=paths_exclude_glob.strip(), - multiline=multiline, - code_files_only=restrict_search_to_code_files, - skip_ignored_files=skip_ignored_files, + return ( + self._api() + .search_for_pattern( + substring_pattern, + context_lines_before=context_lines_before, + context_lines_after=context_lines_after, + paths_include_glob=paths_include_glob, + paths_exclude_glob=paths_exclude_glob, + relative_path=relative_path, + restrict_search_to_code_files=restrict_search_to_code_files, + skip_ignored_files=skip_ignored_files, + multiline=multiline, + max_answer_chars=max_answer_chars, + ) + .represent() ) - - # group matches by file - file_to_matches: dict[str, list[str]] = defaultdict(list) - for match in matches: - assert match.source_file_path is not None - file_to_matches[match.source_file_path].append(match.to_display_string()) - - # capture lightweight match data for shortening before serialization - match_lines_by_file: dict[str, list[dict[str, int | str]]] = defaultdict(list) - for match in matches: - assert match.source_file_path is not None - first = match.matched_lines[0] - match_lines_by_file[match.source_file_path].append({"line": first.line_number, "text": first.line_content.strip()}) - - # shortened result closures, from least to most aggressive shortening - _TEXT_TRUNCATE = 60 - - def render_first_lines(truncate: bool) -> str: - """Render each match's first line, either in full or truncated to a fixed length.""" - - def entry_text(text: str) -> str: - if truncate and len(text) > _TEXT_TRUNCATE: - return text[:_TEXT_TRUNCATE] + "..." - return text - - compact = { - path: [{"line": m["line"], "text": entry_text(str(m["text"]))} for m in lines] - for path, lines in match_lines_by_file.items() - } - if truncate: - header = ( - f"Matched lines (text over {_TEXT_TRUNCATE} chars is truncated, marked with a trailing '...'); " - "use read_file with the line numbers for full content:" - ) - else: - header = "Matched lines per file; use read_file with the line numbers for surrounding context:" - return f"{header}\n{self._to_json(compact)}" - - def make_first_lines_full() -> str: - """Match locations with each match's full first line.""" - return render_first_lines(truncate=False) - - def make_first_lines_truncated() -> str: - """Match locations with each match's first line truncated to a fixed length.""" - return render_first_lines(truncate=True) - - def make_line_numbers_only() -> str: - """Match locations as bare line numbers (no text).""" - numbers = {path: [m["line"] for m in lines] for path, lines in match_lines_by_file.items()} - return f"Match lines per file:\n{self._to_json(numbers)}" - - def make_per_file_counts() -> str: - counts = {path: len(lines) for path, lines in match_lines_by_file.items()} - return f"Match counts per file:\n{self._to_json(counts)}" - - def make_summary() -> str: - return f"Found {len(matches)} matches in {len(match_lines_by_file)} files." - - result = self._to_json(file_to_matches) - return self._limit_length( - result, - max_answer_chars, - shortened_result_factories=[ - make_first_lines_full, - make_first_lines_truncated, - make_line_numbers_only, - make_per_file_counts, - make_summary, - ], - ) - - """ - Performs a search for a pattern in the project. - """ diff --git a/test/serena/test_edit_api.py b/test/serena/test_edit_api.py index a0520bfe..d961857c 100644 --- a/test/serena/test_edit_api.py +++ b/test/serena/test_edit_api.py @@ -30,7 +30,6 @@ def api(project: Project) -> EditApi: def test_facade_exposes_editing_operations(api: EditApi) -> None: default_methods = { - "create_text_file", "replace_content", "replace_in_files", "replace_symbol_body", @@ -47,18 +46,6 @@ def test_facade_exposes_editing_operations(api: EditApi) -> None: assert all(facade.get_method(name).info.can_edit for name in default_methods | optional_methods) -def test_create_text_file(api: EditApi, project: Project) -> None: - result = api.create_text_file("sub/new.txt", "hello\n") - assert "new.txt" in result - assert (Path(project.project_root) / "sub" / "new.txt").read_text(encoding="utf-8") == "hello\n" - - result = api.create_text_file("sub/new.txt", "changed\n") - assert "Overwrote" in result - - with pytest.raises(AssertionError): - api.create_text_file("../outside.txt", "nope") - - def test_replace_in_files_dry_run_returns_inspectable_preview(api: EditApi, project: Project) -> None: preview = api.replace_in_files("foo", "bar", mode="literal", dry_run=True) assert isinstance(preview, ReplacementPreview) diff --git a/test/serena/test_file_tools.py b/test/serena/test_file_tools.py index 8095624e..2704c1ae 100644 --- a/test/serena/test_file_tools.py +++ b/test/serena/test_file_tools.py @@ -15,10 +15,8 @@ def read_file_tool(tmp_path: Path) -> ReadFileTool: project = Project.load(str(tmp_path), serena_config=SerenaConfig(gui_log_window=False, web_dashboard=False)) agent = MagicMock() agent.get_active_project_or_raise.return_value = project - tool = ReadFileTool(agent) - # bypass the length limit, which would otherwise depend on the agent configuration - tool._limit_length = lambda result, max_answer_chars, shortened_result_factories=None: result # type: ignore[method-assign] - return tool + agent.serena_config.default_max_tool_answer_chars = 10000 + return ReadFileTool(agent) def _deleted_by_delete_lines(content: str, line: int) -> str: diff --git a/test/serena/test_fs_api.py b/test/serena/test_fs_api.py new file mode 100644 index 00000000..28d343de --- /dev/null +++ b/test/serena/test_fs_api.py @@ -0,0 +1,81 @@ +""" +Tests for the file system facade API. +""" + +from pathlib import Path +from unittest.mock import MagicMock + +import pytest + +from serena.config.serena_config import SerenaConfig +from serena.project import Project +from serena.repl.api.fs_api import FsApi +from serena.repl.facade import ApiScope, Facade + + +@pytest.fixture +def project(tmp_path: Path) -> Project: + (tmp_path / "src").mkdir() + (tmp_path / "src" / "a.py").write_text("x = foo(1)\ny = foo(2)\nz = 3\n", encoding="utf-8") + (tmp_path / "src" / "b.txt").write_text("foo in text\n", encoding="utf-8") + (tmp_path / "README.md").write_text("# readme\n", encoding="utf-8") + return Project.load(str(tmp_path), serena_config=SerenaConfig(gui_log_window=False, web_dashboard=False)) + + +@pytest.fixture +def api(project: Project) -> FsApi: + agent = MagicMock() + agent.get_active_project_or_raise.return_value = project + agent.serena_config.default_max_tool_answer_chars = 10000 + return FsApi(agent) + + +def test_facade_exposes_file_operations(api: FsApi) -> None: + facade = Facade.from_api(api, ApiScope()) + assert facade.name == "fs" + assert set(facade.enabled_method_names) == {"read_file", "create_text_file", "list_dir", "find_file", "search_for_pattern"} + assert {name for name in facade.enabled_method_names if facade.get_method(name).info.can_edit} == {"create_text_file"} + + +def test_read_file(api: FsApi) -> None: + content = api.read_file("src/a.py") + assert content.lines == ["x = foo(1)", "y = foo(2)", "z = 3", ""] + assert content.represent() == content.text + + assert api.read_file("src/a.py", start_line=1, end_line=1).text == "y = foo(2)" + assert api.read_file("src/a.py", start_line=-2).lines == ["z = 3", ""] + + +def test_create_text_file(api: FsApi, project: Project) -> None: + result = api.create_text_file("sub/new.txt", "hello\n") + assert "new.txt" in result + assert (Path(project.project_root) / "sub" / "new.txt").read_text(encoding="utf-8") == "hello\n" + + result = api.create_text_file("sub/new.txt", "changed\n") + assert "Overwrote" in result + + with pytest.raises(AssertionError): + api.create_text_file("../outside.txt", "nope") + + +def test_list_dir_and_find_file(api: FsApi) -> None: + listing = api.list_dir(".", recursive=True) + assert "src" in listing.dirs + assert {"src/a.py", "src/b.txt", "README.md"} <= {f.replace("\\", "/") for f in listing.files} + assert '"dirs"' in listing.represent() and '"files"' in listing.represent() + + with pytest.raises(FileNotFoundError): + api.list_dir("missing", recursive=False) + + assert [f.replace("\\", "/") for f in api.find_file("*.py", ".")] == ["src/a.py"] + + +def test_search_for_pattern(api: FsApi) -> None: + matches = api.search_for_pattern("foo", relative_path="src") + assert len(matches) == 3 + assert {m.source_file_path.replace("\\", "/") for m in matches.matches} == {"src/a.py", "src/b.txt"} + + # restricting to code files excludes the text file; the rendering maps files to matched lines + code_matches = api.search_for_pattern("foo", restrict_search_to_code_files=True) + assert all(m.source_file_path.endswith("a.py") for m in code_matches.matches) + assert "foo(1)" in code_matches.represent() From a46a46577338e6957863ad74e7343b5ee96185db Mon Sep 17 00:00:00 2001 From: Dominik Jain Date: Mon, 7 Sep 2026 17:43:48 +0200 Subject: [PATCH 16/53] Add shell facade (shell) for shell command execution Introduce ShellApi with the single editing operation execute_shell_command, which returns a ShellCommandOutput exposing stdout, stderr, return code and working directory to code and rendering as JSON as before. Keeping shell command execution in its own facade makes it a natural unit for exclusion. The shell command tool delegates to the API via ShellApiMixin; the shell facade is always part of the REPL. --- src/serena/agent.py | 2 + src/serena/repl/api/shell_api.py | 89 ++++++++++++++++++++++++++++++++ src/serena/tools/cmd_tools.py | 32 +++++------- test/serena/test_shell_api.py | 45 ++++++++++++++++ 4 files changed, 150 insertions(+), 18 deletions(-) create mode 100644 src/serena/repl/api/shell_api.py create mode 100644 test/serena/test_shell_api.py diff --git a/src/serena/agent.py b/src/serena/agent.py index 911c360b..b2efef85 100644 --- a/src/serena/agent.py +++ b/src/serena/agent.py @@ -52,6 +52,7 @@ from serena.repl.api.fs_api import FsApi from serena.repl.api.jb_api import JetBrainsApi from serena.repl.api.lsp_api import LspApi from serena.repl.api.mem_api import MemoryApi +from serena.repl.api.shell_api import ShellApi from serena.repl.facade import ApiScope, Facade from serena.repl.repl import SerenaRepl from serena.task_executor import TaskExecutor @@ -1190,6 +1191,7 @@ class SerenaAgent: Facade.from_api(FsApi(self), api_scope), Facade.from_api(EditApi(self), api_scope), Facade.from_api(MemoryApi(self), api_scope), + Facade.from_api(ShellApi(self), api_scope), ] if self._language_backend.is_lsp(): facades.append(Facade.from_api(LspApi(self), api_scope)) diff --git a/src/serena/repl/api/shell_api.py b/src/serena/repl/api/shell_api.py new file mode 100644 index 00000000..54922ace --- /dev/null +++ b/src/serena/repl/api/shell_api.py @@ -0,0 +1,89 @@ +# SPDX-License-Identifier: GPL-3.0-or-later +""" +The implementation of shell command execution. +""" + +import os.path +from typing import TYPE_CHECKING + +from serena.util.shell import ShellCommandResult, execute_shell_command + +from ..facade import FacadeApi, facade_method +from ..representable import Renderer, RepresentableViaRenderer + +if TYPE_CHECKING: + from serena.agent import SerenaAgent + + +class ShellCommandOutput(RepresentableViaRenderer): + """ + The outcome of a shell command: `stdout`, `stderr` (None if not captured), `return_code` and `cwd`. + """ + + def __init__(self, result: ShellCommandResult, renderer: "ShellCommandOutputRenderer"): + """ + :param result: the result of the command execution + :param renderer: the renderer to use for representing the output + """ + super().__init__(renderer) + self.result_ = result + + @property + def stdout(self) -> str: + return self.result_.stdout + + @property + def stderr(self) -> str | None: + return self.result_.stderr + + @property + def return_code(self) -> int: + return self.result_.return_code + + @property + def cwd(self) -> str: + return self.result_.cwd + + +class ShellCommandOutputRenderer(Renderer[ShellCommandOutput]): + def render(self, obj: ShellCommandOutput) -> str: + return self._limit_length(obj.result_.model_dump_json()) + + +class ShellApi(FacadeApi): + def __init__(self, agent: "SerenaAgent") -> None: + super().__init__(agent, name="shell", description="execution of shell commands") + + @facade_method(can_edit=True) + def execute_shell_command( + self, command: str, cwd: str | None = None, capture_stderr: bool = True, max_answer_chars: int = -1 + ) -> ShellCommandOutput: + """ + Executes a shell command and returns its output. If there is a memory about suggested commands, read that first. + Never execute unsafe shell commands! + IMPORTANT: Do not use this to start + * long-running processes (e.g. servers) that are not intended to terminate quickly, + * processes that require user interaction. + + :param command: the shell command to execute + :param cwd: the working directory to execute the command in (absolute, or relative to the project root). + If None, the project root will be used. + :param capture_stderr: whether to capture and return stderr output + :param max_answer_chars: if the rendered output is longer than this number of characters, no content is rendered; + -1 for the configured default. Don't adjust unless there is no other way to get the content required for the task. + :return: the output (object with properties stdout, stderr, return_code and cwd) + """ + project_root = self._get_project().project_root + if cwd is None: + _cwd = project_root + elif os.path.isabs(cwd): + _cwd = cwd + else: + _cwd = os.path.join(project_root, cwd) + if not os.path.isdir(_cwd): + raise FileNotFoundError( + f"Specified a relative working directory ({cwd}), but the resulting path is not a directory: {_cwd}" + ) + + result = execute_shell_command(command, cwd=_cwd, capture_stderr=capture_stderr) + return ShellCommandOutput(result, ShellCommandOutputRenderer(self._agent, max_answer_chars)) diff --git a/src/serena/tools/cmd_tools.py b/src/serena/tools/cmd_tools.py index ace9fab1..6ad18914 100644 --- a/src/serena/tools/cmd_tools.py +++ b/src/serena/tools/cmd_tools.py @@ -3,13 +3,23 @@ Tools supporting the execution of (external) commands """ # SPDX-License-Identifier: GPL-3.0-or-later -import os.path +from typing import cast +from serena.repl.api.shell_api import ShellApi from serena.tools import Tool, ToolMarkerCanEdit -from serena.util.shell import execute_shell_command -class ExecuteShellCommandTool(Tool, ToolMarkerCanEdit): +class ShellApiMixin: + """ + Mixin for tools which delegate to the shell API + """ + + def _api(self) -> ShellApi: + tool = cast(Tool, cast(object, self)) + return ShellApi(tool.agent) + + +class ExecuteShellCommandTool(Tool, ToolMarkerCanEdit, ShellApiMixin): """ Executes a shell command. """ @@ -36,18 +46,4 @@ class ExecuteShellCommandTool(Tool, ToolMarkerCanEdit): required for the task. :return: a JSON object containing the command's stdout and optionally stderr output """ - if cwd is None: - _cwd = self.get_project_root() - else: - if os.path.isabs(cwd): - _cwd = cwd - else: - _cwd = os.path.join(self.get_project_root(), cwd) - if not os.path.isdir(_cwd): - raise FileNotFoundError( - f"Specified a relative working directory ({cwd}), but the resulting path is not a directory: {_cwd}" - ) - - result = execute_shell_command(command, cwd=_cwd, capture_stderr=capture_stderr) - result = result.model_dump_json() - return self._limit_length(result, max_answer_chars) + return self._api().execute_shell_command(command, cwd, capture_stderr, max_answer_chars).represent() diff --git a/test/serena/test_shell_api.py b/test/serena/test_shell_api.py new file mode 100644 index 00000000..e10ce019 --- /dev/null +++ b/test/serena/test_shell_api.py @@ -0,0 +1,45 @@ +""" +Tests for the shell facade API. +""" + +import sys +from pathlib import Path +from unittest.mock import MagicMock + +import pytest + +from serena.config.serena_config import SerenaConfig +from serena.project import Project +from serena.repl.api.shell_api import ShellApi +from serena.repl.facade import ApiScope, Facade + + +@pytest.fixture +def api(tmp_path: Path) -> ShellApi: + (tmp_path / "sub").mkdir() + project = Project.load(str(tmp_path), serena_config=SerenaConfig(gui_log_window=False, web_dashboard=False)) + agent = MagicMock() + agent.get_active_project_or_raise.return_value = project + agent.serena_config.default_max_tool_answer_chars = 10000 + return ShellApi(agent) + + +def test_facade_exposes_shell_command_as_editing_operation(api: ShellApi) -> None: + facade = Facade.from_api(api, ApiScope()) + assert facade.name == "shell" + assert facade.enabled_method_names == ["execute_shell_command"] + assert facade.get_method("execute_shell_command").info.can_edit + + +def test_execute_shell_command(api: ShellApi, tmp_path: Path) -> None: + print_cwd = "cd" if sys.platform == "win32" else "pwd" + + output = api.execute_shell_command(f"{print_cwd}") + assert output.return_code == 0 + assert Path(output.stdout.strip()).resolve() == tmp_path.resolve() + assert '"stdout"' in output.represent() and '"return_code"' in output.represent() + + # a relative working directory is resolved against the project root and must exist + assert Path(api.execute_shell_command(print_cwd, cwd="sub").stdout.strip()).resolve() == (tmp_path / "sub").resolve() + with pytest.raises(FileNotFoundError): + api.execute_shell_command(print_cwd, cwd="missing") From 07b6831c70b579918de7c355b7bddfe66402b135 Mon Sep 17 00:00:00 2001 From: Dominik Jain Date: Mon, 7 Sep 2026 18:05:26 +0200 Subject: [PATCH 17/53] Record the correspondence between API methods and tools FacadeMethodInfo gains corresponding_tool (the tool class offering the same functionality) and get_corresponding_tool_name, and every API method with a tool counterpart names it in its facade_method decorator. This enables applying tool-level exclusions to API methods and rendering prompt conditions based on tools. Since the API modules now refer to the tool classes, the tools import the APIs locally (in the mixins' _api methods), and the symbol tools' symbol_dict_grouper attributes become properties. --- src/serena/repl/api/edit_api.py | 26 ++++++++++++------ src/serena/repl/api/fs_api.py | 11 ++++---- src/serena/repl/api/jb_api.py | 41 ++++++++++++++++++++--------- src/serena/repl/api/lsp_api.py | 32 +++++++++++++++------- src/serena/repl/api/mem_api.py | 23 +++++++++++----- src/serena/repl/api/shell_api.py | 3 ++- src/serena/repl/facade.py | 19 +++++++++++-- src/serena/tools/cmd_tools.py | 13 ++++++--- src/serena/tools/file_tools.py | 22 +++++++++++----- src/serena/tools/jetbrains_tools.py | 34 +++++++++++++++++------- src/serena/tools/memory_tools.py | 13 ++++++--- src/serena/tools/symbol_tools.py | 32 +++++++++++++++++----- test/serena/test_repl_tool.py | 11 +++++++- 13 files changed, 201 insertions(+), 79 deletions(-) diff --git a/src/serena/repl/api/edit_api.py b/src/serena/repl/api/edit_api.py index a96f5cc4..fa1fa186 100644 --- a/src/serena/repl/api/edit_api.py +++ b/src/serena/repl/api/edit_api.py @@ -6,6 +6,16 @@ The implementation of editing operations, which are independent of the language from typing import TYPE_CHECKING, Literal from serena.code_editor import EditedFileContext +from serena.tools import ( + DeleteLinesTool, + InsertAfterSymbolTool, + InsertAtLineTool, + InsertBeforeSymbolTool, + ReplaceContentTool, + ReplaceInFilesTool, + ReplaceLinesTool, + ReplaceSymbolBodyTool, +) from serena.util.text_utils import ContentReplacer, MultiFileReplacement, ReplacementOccurrence, ReplacementRejectedError from ..facade import SUCCESS_RESULT, FacadeApi, facade_method @@ -68,7 +78,7 @@ class EditApi(FacadeApi): # file-level operations - @facade_method(can_edit=True) + @facade_method(can_edit=True, corresponding_tool=ReplaceContentTool) def replace_content( self, relative_path: str, @@ -106,7 +116,7 @@ class EditApi(FacadeApi): context.set_updated_content(replacer.replace(context.get_original_content(), needle, repl)) return SUCCESS_RESULT - @facade_method(can_edit=True) + @facade_method(can_edit=True, corresponding_tool=ReplaceInFilesTool) def replace_in_files( self, needle: str, @@ -186,7 +196,7 @@ class EditApi(FacadeApi): # line-level operations - @facade_method(optional=True, can_edit=True) + @facade_method(optional=True, can_edit=True, corresponding_tool=DeleteLinesTool) def delete_lines(self, relative_path: str, start_line: int, end_line: int) -> str: """ Deletes the given lines in the file. @@ -200,7 +210,7 @@ class EditApi(FacadeApi): self._create_code_editor().delete_lines(relative_path, start_line, end_line) return SUCCESS_RESULT - @facade_method(optional=True, can_edit=True) + @facade_method(optional=True, can_edit=True, corresponding_tool=ReplaceLinesTool) def replace_lines(self, relative_path: str, start_line: int, end_line: int, content: str) -> str: """ Replaces the given range of lines in the given file. @@ -217,7 +227,7 @@ class EditApi(FacadeApi): code_editor.insert_at_line(relative_path, start_line, self._normalize_inserted_content(content)) return SUCCESS_RESULT - @facade_method(optional=True, can_edit=True) + @facade_method(optional=True, can_edit=True, corresponding_tool=InsertAtLineTool) def insert_at_line(self, relative_path: str, line: int, content: str) -> str: """ Inserts the given content at the given line in the file, pushing existing content of the line down. @@ -239,7 +249,7 @@ class EditApi(FacadeApi): # symbol-level operations - @facade_method(can_edit=True) + @facade_method(can_edit=True, corresponding_tool=ReplaceSymbolBodyTool) def replace_symbol_body(self, name_path: str, relative_path: str, body: str) -> str: """ Replaces the body of the given symbol. @@ -257,7 +267,7 @@ class EditApi(FacadeApi): self._create_code_editor().replace_body(name_path, relative_file_path=relative_path, body=body) return SUCCESS_RESULT - @facade_method(can_edit=True) + @facade_method(can_edit=True, corresponding_tool=InsertAfterSymbolTool) def insert_after_symbol(self, name_path: str, relative_path: str, body: str) -> str: """ Inserts code after a class/method/function definition. @@ -272,7 +282,7 @@ class EditApi(FacadeApi): self._create_code_editor().insert_after_symbol(name_path, relative_file_path=relative_path, body=body) return SUCCESS_RESULT - @facade_method(can_edit=True) + @facade_method(can_edit=True, corresponding_tool=InsertBeforeSymbolTool) def insert_before_symbol(self, name_path: str, relative_path: str, body: str) -> str: """ Inserts the given content before the beginning of the definition of the given symbol (via the symbol's location). diff --git a/src/serena/repl/api/fs_api.py b/src/serena/repl/api/fs_api.py index afd4efca..4272a4ba 100644 --- a/src/serena/repl/api/fs_api.py +++ b/src/serena/repl/api/fs_api.py @@ -9,6 +9,7 @@ from fnmatch import fnmatch from pathlib import Path from typing import TYPE_CHECKING +from serena.tools import CreateTextFileTool, FindFileTool, ListDirTool, ReadFileTool, SearchForPatternTool from serena.util.file_system import scan_directory from serena.util.text_utils import MatchedConsecutiveLines from solidlsp.ls_utils import TextUtils @@ -163,7 +164,7 @@ class FsApi(FacadeApi): def __init__(self, agent: "SerenaAgent") -> None: super().__init__(agent, name="fs", description="the project's files as units (as opposed to their content, see `edit`)") - @facade_method() + @facade_method(corresponding_tool=ReadFileTool) def read_file(self, relative_path: str, start_line: int = 0, end_line: int | None = None, max_answer_chars: int = -1) -> FileContent: """ Reads the given file or a range of its lines. @@ -183,7 +184,7 @@ class FsApi(FacadeApi): lines = lines[start_line:] if end_line is None else lines[start_line : end_line + 1] return FileContent(lines, FileContentRenderer(self._agent, max_answer_chars)) - @facade_method(can_edit=True) + @facade_method(can_edit=True, corresponding_tool=CreateTextFileTool) def create_text_file(self, relative_path: str, content: str) -> str: """ Writes a new file or overwrites an existing file with the given content. @@ -211,7 +212,7 @@ class FsApi(FacadeApi): answer += " Overwrote existing file." return answer - @facade_method() + @facade_method(corresponding_tool=ListDirTool) def list_dir( self, relative_path: str, recursive: bool, skip_ignored_files: bool = False, max_answer_chars: int = -1 ) -> DirectoryListing: @@ -240,7 +241,7 @@ class FsApi(FacadeApi): ) return DirectoryListing(dirs, files, DirectoryListingRenderer(self._agent, max_answer_chars)) - @facade_method() + @facade_method(corresponding_tool=FindFileTool) def find_file(self, file_mask: str, relative_path: str) -> list[str]: """ Finds files matching the given file mask within the given relative path. @@ -269,7 +270,7 @@ class FsApi(FacadeApi): ) return files - @facade_method() + @facade_method(corresponding_tool=SearchForPatternTool) def search_for_pattern( self, substring_pattern: str, diff --git a/src/serena/repl/api/jb_api.py b/src/serena/repl/api/jb_api.py index b3449f91..e8ffdb62 100644 --- a/src/serena/repl/api/jb_api.py +++ b/src/serena/repl/api/jb_api.py @@ -13,6 +13,21 @@ from serena.code_editor import JetBrainsCodeEditor from serena.jetbrains.jetbrains_plugin_client import JetBrainsPluginClient from serena.jetbrains.jetbrains_types import SymbolDTO, SymbolDTOUtil from serena.symbol import JetBrainsSymbolDictGrouper +from serena.tools import ( + JetBrainsDebugTool, + JetBrainsFindDeclarationTool, + JetBrainsFindImplementationsTool, + JetBrainsFindReferencingSymbolsTool, + JetBrainsFindSymbolTool, + JetBrainsGetSymbolsOverviewTool, + JetBrainsInlineSymbol, + JetBrainsListInspectionsTool, + JetBrainsMoveTool, + JetBrainsRenameTool, + JetBrainsRunInspectionsTool, + JetBrainsSafeDeleteTool, + JetBrainsTypeHierarchyTool, +) from serena.util.text_utils import find_text_coordinates from ..facade import FacadeApi, facade_method @@ -177,7 +192,7 @@ class JetBrainsApi(FacadeApi): # read operations - @facade_method() + @facade_method(corresponding_tool=JetBrainsFindSymbolTool) def find_symbol( self, name_path_pattern: str, @@ -261,7 +276,7 @@ class JetBrainsApi(FacadeApi): raise ValueError(f"Matched {n_matches}>{max_matches=} symbols.\n" + renderer.render_identifiers(collection)) return collection - @facade_method() + @facade_method(corresponding_tool=JetBrainsFindReferencingSymbolsTool) def find_referencing_symbols(self, name_path: str, relative_path: str, max_answer_chars: int = -1) -> JetBrainsSymbolCollection: """ Finds all symbols that reference the given symbol — its callers / usages / dependents, i.e. the @@ -294,7 +309,7 @@ class JetBrainsApi(FacadeApi): renderer = JetBrainsReferencesRenderer(self._agent, max_answer_chars, grouper=self.references_grouper_) return JetBrainsSymbolCollection(symbol_dicts, renderer) - @facade_method() + @facade_method(corresponding_tool=JetBrainsGetSymbolsOverviewTool) def get_symbols_overview( self, relative_path: str, depth: int = -1, max_answer_chars: int = -1, include_file_documentation: bool = False ) -> JetBrainsSymbolsOverview: @@ -340,7 +355,7 @@ class JetBrainsApi(FacadeApi): result[rel_path].append(name_path) return dict(result) - @facade_method() + @facade_method(corresponding_tool=JetBrainsTypeHierarchyTool) def get_type_hierarchy( self, name_path: str, @@ -379,7 +394,7 @@ class JetBrainsApi(FacadeApi): result["levels_not_included"] = levels_not_included return self._json_object(result, max_answer_chars) - @facade_method() + @facade_method(corresponding_tool=JetBrainsFindDeclarationTool) def find_declaration(self, relative_path: str, regex: str, include_body: bool = False) -> JetBrainsSymbolCollection: r""" Finds the declaration of a symbol. @@ -402,7 +417,7 @@ class JetBrainsApi(FacadeApi): ) return JetBrainsSymbolCollection(response["symbols"], JetBrainsSymbolCollectionRenderer(self._agent, -1)) - @facade_method() + @facade_method(corresponding_tool=JetBrainsFindImplementationsTool) def find_implementations(self, relative_path: str, name_path: str) -> JetBrainsSymbolCollection: """ Finds the implementations of a symbol. @@ -417,7 +432,7 @@ class JetBrainsApi(FacadeApi): # edit operations - @facade_method(can_edit=True) + @facade_method(can_edit=True, corresponding_tool=JetBrainsRenameTool) def rename( self, relative_path: str, @@ -448,7 +463,7 @@ class JetBrainsApi(FacadeApi): ) return self._json_object(result) - @facade_method(beta=True, can_edit=True) + @facade_method(beta=True, can_edit=True, corresponding_tool=JetBrainsMoveTool) def move( self, relative_path: str, @@ -489,7 +504,7 @@ class JetBrainsApi(FacadeApi): ) return self._json_object(result) - @facade_method(beta=True, can_edit=True) + @facade_method(beta=True, can_edit=True, corresponding_tool=JetBrainsSafeDeleteTool) def safe_delete( self, relative_path: str, name_path: str | None = None, delete_even_if_used: bool = False, propagate: bool = False ) -> JsonObject: @@ -516,7 +531,7 @@ class JetBrainsApi(FacadeApi): ) return self._json_object(result) - @facade_method(beta=True, can_edit=True) + @facade_method(beta=True, can_edit=True, corresponding_tool=JetBrainsInlineSymbol) def inline_symbol(self, name_path: str, relative_path: str, keep_definition: bool = False) -> JsonObject: """ Inlines a symbol (usually a method/function, but also classes may be amenable to inlining, @@ -537,7 +552,7 @@ class JetBrainsApi(FacadeApi): # inspections - @facade_method() + @facade_method(corresponding_tool=JetBrainsRunInspectionsTool) def run_inspections( self, relative_path: str, @@ -573,7 +588,7 @@ class JetBrainsApi(FacadeApi): ) return self._json_object(result, max_answer_chars) - @facade_method() + @facade_method(corresponding_tool=JetBrainsListInspectionsTool) def list_inspections( self, language: str | None = None, group_path_contains: str | None = None, max_answer_chars: int = -1 ) -> JsonObject: @@ -603,7 +618,7 @@ class JetBrainsApi(FacadeApi): """ return self._agent.prompt_factory.create_info_jet_brains_debug_repl() - @facade_method(beta=True) + @facade_method(beta=True, corresponding_tool=JetBrainsDebugTool) def debug_eval(self, expression: str, repl_key: str = "default") -> str: """ Provides debugging functionality (run configs, breakpoints, stepping, inspection, and evaluation) diff --git a/src/serena/repl/api/lsp_api.py b/src/serena/repl/api/lsp_api.py index 115f9772..28015197 100644 --- a/src/serena/repl/api/lsp_api.py +++ b/src/serena/repl/api/lsp_api.py @@ -18,6 +18,18 @@ from serena.symbol import ( ReferenceInLanguageServerSymbol, SymbolDictGrouper, ) +from serena.tools import ( + FindDeclarationTool, + FindImplementationsTool, + FindReferencingSymbolsTool, + FindSymbolTool, + GetDiagnosticsForFileTool, + GetDiagnosticsForSymbolTool, + GetSymbolsOverviewTool, + RenameSymbolTool, + RestartLanguageServerTool, + SafeDeleteSymbol, +) from serena.util.text_utils import TextOutputUtils, find_text_coordinates from solidlsp.lsp_protocol_handler.lsp_types import SymbolKind @@ -310,7 +322,7 @@ class LspApi(FacadeApi): # language server management - @facade_method(optional=True) + @facade_method(optional=True, corresponding_tool=RestartLanguageServerTool) def restart_language_server(self) -> str: """ Restarts the language server(s). Use this only on explicit user request or after confirmation; @@ -323,7 +335,7 @@ class LspApi(FacadeApi): # read operations - @facade_method() + @facade_method(corresponding_tool=GetSymbolsOverviewTool) def get_symbols_overview(self, relative_path: str, depth: int = -1, max_answer_chars: int = -1) -> LspSymbolCollection: """ Gets an overview of the top-level symbols defined in the given file (classes, methods, fields) — its @@ -369,7 +381,7 @@ class LspApi(FacadeApi): ) return LspSymbolCollection(symbols, renderer) - @facade_method() + @facade_method(corresponding_tool=FindSymbolTool) def find_symbol( self, name_path_pattern: str, @@ -462,7 +474,7 @@ class LspApi(FacadeApi): return symbol_collection - @facade_method() + @facade_method(corresponding_tool=FindReferencingSymbolsTool) def find_referencing_symbols( self, name_path: str, @@ -496,7 +508,7 @@ class LspApi(FacadeApi): ) return LspReferenceCollection(references, LspReferenceCollectionRenderer(self._agent, max_answer_chars, self.references_grouper_)) - @facade_method() + @facade_method(corresponding_tool=FindImplementationsTool) def find_implementations( self, name_path: str, @@ -532,7 +544,7 @@ class LspApi(FacadeApi): output_params = SymbolOutputParams(kind=True, relative_path=True, body_location=True, include_info=include_info) return LspSymbolCollection(symbols, LspSymbolCollectionRenderer(self._agent, max_answer_chars, symbol_retriever, output_params)) - @facade_method() + @facade_method(corresponding_tool=FindDeclarationTool) def find_declaration( self, relative_path: str, @@ -587,7 +599,7 @@ class LspApi(FacadeApi): collection_renderer = LspSymbolCollectionRenderer(self._agent, -1, symbol_retriever, output_params) return LspSymbol(defining_symbol, LspSymbolRenderer(self._agent, -1, collection_renderer)) - @facade_method() + @facade_method(corresponding_tool=GetDiagnosticsForFileTool) def get_diagnostics_for_file( self, relative_path: str, start_line: int = 0, end_line: int = -1, min_severity: int = 4, max_answer_chars: int = -1 ) -> LspDiagnostics: @@ -621,7 +633,7 @@ class LspApi(FacadeApi): return self._create_diagnostics(grouped_diagnostics, max_answer_chars) - @facade_method(optional=True) + @facade_method(optional=True, corresponding_tool=GetDiagnosticsForSymbolTool) def get_diagnostics_for_symbol( self, name_path: str, @@ -665,7 +677,7 @@ class LspApi(FacadeApi): # edit operations - @facade_method(can_edit=True) + @facade_method(can_edit=True, corresponding_tool=RenameSymbolTool) def rename_symbol(self, name_path: str, relative_path: str, new_name: str) -> str: """ Renames the symbol with the given `name_path` to `new_name` throughout the entire codebase. @@ -680,7 +692,7 @@ class LspApi(FacadeApi): self._get_project().ls_sync_file_system_changes() return self._create_ls_code_editor().rename_symbol(name_path, relative_path=relative_path, new_name=new_name) - @facade_method(can_edit=True) + @facade_method(can_edit=True, corresponding_tool=SafeDeleteSymbol) def safe_delete_symbol(self, name_path_pattern: str, relative_path: str) -> str: """ Deletes the symbol if it is safe to do so (i.e., if there are no references to it) diff --git a/src/serena/repl/api/mem_api.py b/src/serena/repl/api/mem_api.py index 60fc305c..d6e41f76 100644 --- a/src/serena/repl/api/mem_api.py +++ b/src/serena/repl/api/mem_api.py @@ -8,6 +8,15 @@ import platform from typing import TYPE_CHECKING, Literal from serena.memories.memory_manager import MemoryManager +from serena.tools import ( + DeleteMemoryTool, + EditMemoryTool, + ListMemoriesTool, + OnboardingTool, + ReadMemoryTool, + RenameMemoryTool, + WriteMemoryTool, +) from ..facade import FacadeApi, facade_method from ..representable import Renderer, RepresentableViaRenderer @@ -56,7 +65,7 @@ class MemoryApi(FacadeApi): def _get_memory_manager(self) -> MemoryManager: return self._get_project().memory_manager - @facade_method() + @facade_method(corresponding_tool=ListMemoriesTool) def list_memories(self, topic: str = "") -> MemoryList: """ Lists the available memories, optionally filtered by topic. @@ -66,7 +75,7 @@ class MemoryApi(FacadeApi): """ return MemoryList(self._get_memory_manager().list_memories(topic), MemoryListRenderer(self._agent, -1)) - @facade_method() + @facade_method(corresponding_tool=ReadMemoryTool) def read_memory(self, memory_name: str) -> str: """ Reads a memory that is likely to be relevant to the current task, inferring relevance e.g. from the name. @@ -76,7 +85,7 @@ class MemoryApi(FacadeApi): """ return self._get_memory_manager().load_memory(memory_name) - @facade_method(can_edit=True) + @facade_method(can_edit=True, corresponding_tool=WriteMemoryTool) def write_memory(self, memory_name: str, content: str, max_chars: int = -1) -> str: """ Writes information about this project that can be useful for future tasks in md format. @@ -98,7 +107,7 @@ class MemoryApi(FacadeApi): ) return self._get_memory_manager().save_memory(memory_name, content, is_tool_context=True) - @facade_method(can_edit=True) + @facade_method(can_edit=True, corresponding_tool=EditMemoryTool) def edit_memory( self, memory_name: str, @@ -125,7 +134,7 @@ class MemoryApi(FacadeApi): memory_name, needle, repl, mode, allow_multiple_occurrences, is_tool_context=True, regex_multiline=True ) - @facade_method(can_edit=True) + @facade_method(can_edit=True, corresponding_tool=RenameMemoryTool) def rename_memory(self, old_name: str, new_name: str) -> str: """ Renames or moves a memory; use "/" in the name to organize into topics. @@ -144,7 +153,7 @@ class MemoryApi(FacadeApi): log.info(f"Updated {n_references_updated} references to memory {old_name} to {new_name}") return renaming_message - @facade_method(can_edit=True) + @facade_method(can_edit=True, corresponding_tool=DeleteMemoryTool) def delete_memory(self, memory_name: str) -> str: """ Deletes a memory; only call this if instructed explicitly or permission was granted by the user. @@ -154,7 +163,7 @@ class MemoryApi(FacadeApi): """ return self._get_memory_manager().delete_memory(memory_name, is_tool_context=True) - @facade_method() + @facade_method(corresponding_tool=OnboardingTool) def onboarding(self) -> str: """ Provides the instructions for performing onboarding (identifying the project structure and essential tasks, diff --git a/src/serena/repl/api/shell_api.py b/src/serena/repl/api/shell_api.py index 54922ace..56ba2679 100644 --- a/src/serena/repl/api/shell_api.py +++ b/src/serena/repl/api/shell_api.py @@ -6,6 +6,7 @@ The implementation of shell command execution. import os.path from typing import TYPE_CHECKING +from serena.tools import ExecuteShellCommandTool from serena.util.shell import ShellCommandResult, execute_shell_command from ..facade import FacadeApi, facade_method @@ -54,7 +55,7 @@ class ShellApi(FacadeApi): def __init__(self, agent: "SerenaAgent") -> None: super().__init__(agent, name="shell", description="execution of shell commands") - @facade_method(can_edit=True) + @facade_method(can_edit=True, corresponding_tool=ExecuteShellCommandTool) def execute_shell_command( self, command: str, cwd: str | None = None, capture_stderr: bool = True, max_answer_chars: int = -1 ) -> ShellCommandOutput: diff --git a/src/serena/repl/facade.py b/src/serena/repl/facade.py index a20080d7..16a7fa4c 100644 --- a/src/serena/repl/facade.py +++ b/src/serena/repl/facade.py @@ -17,6 +17,7 @@ from serena.project import Project if TYPE_CHECKING: from serena.agent import SerenaAgent from serena.code_editor import CodeEditor + from serena.tools import Tool log = logging.getLogger(__name__) TCallable = TypeVar("TCallable", bound=Callable[..., Any]) @@ -39,12 +40,22 @@ class FacadeMethodInfo: """whether the method is in beta (not yet fully stable)""" can_edit: bool = False """whether the method can modify the codebase (relevant for read-only contexts)""" + corresponding_tool: "type[Tool] | None" = None + """the classic tool offering the same functionality, if any""" + + def get_corresponding_tool_name(self) -> str | None: + """ + :return: the name of the corresponding tool, or None if there is none + """ + return self.corresponding_tool.get_name_from_cls() if self.corresponding_tool is not None else None _FACADE_METHOD_INFO_ATTR = "__facade_method_info__" -def facade_method(*, optional: bool = False, beta: bool = False, can_edit: bool = False) -> Callable[[TCallable], TCallable]: +def facade_method( + *, optional: bool = False, beta: bool = False, can_edit: bool = False, corresponding_tool: "type[Tool] | None" = None +) -> Callable[[TCallable], TCallable]: """ Marks a method of a `FacadeApi` as exposed through the facade, attaching the given metadata. The decorator only annotates the method (it does not wrap it), such that signature and docstring remain intact. @@ -52,11 +63,15 @@ def facade_method(*, optional: bool = False, beta: bool = False, can_edit: bool :param optional: whether the method is disabled by default and must be enabled explicitly :param beta: whether the method is in beta :param can_edit: whether the method can modify the codebase + :param corresponding_tool: the classic tool offering the same functionality, if any :return: the decorator """ def decorator(method: TCallable) -> TCallable: - setattr(method, _FACADE_METHOD_INFO_ATTR, FacadeMethodInfo(name=method.__name__, optional=optional, beta=beta, can_edit=can_edit)) + info = FacadeMethodInfo( + name=method.__name__, optional=optional, beta=beta, can_edit=can_edit, corresponding_tool=corresponding_tool + ) + setattr(method, _FACADE_METHOD_INFO_ATTR, info) return method return decorator diff --git a/src/serena/tools/cmd_tools.py b/src/serena/tools/cmd_tools.py index 6ad18914..51357995 100644 --- a/src/serena/tools/cmd_tools.py +++ b/src/serena/tools/cmd_tools.py @@ -3,18 +3,23 @@ Tools supporting the execution of (external) commands """ # SPDX-License-Identifier: GPL-3.0-or-later -from typing import cast +from typing import TYPE_CHECKING, cast -from serena.repl.api.shell_api import ShellApi from serena.tools import Tool, ToolMarkerCanEdit +if TYPE_CHECKING: + from serena.repl.api.shell_api import ShellApi + class ShellApiMixin: """ - Mixin for tools which delegate to the shell API + Mixin for tools which delegate to the shell API. + The API is imported locally, since the API module refers to the tools (as corresponding tools). """ - def _api(self) -> ShellApi: + def _api(self) -> "ShellApi": + from serena.repl.api.shell_api import ShellApi + tool = cast(Tool, cast(object, self)) return ShellApi(tool.agent) diff --git a/src/serena/tools/file_tools.py b/src/serena/tools/file_tools.py index f9b256ba..fd61ed54 100644 --- a/src/serena/tools/file_tools.py +++ b/src/serena/tools/file_tools.py @@ -7,29 +7,37 @@ File and file system-related tools, specifically for """ # SPDX-License-Identifier: GPL-3.0-or-later -from typing import Literal, cast +from typing import TYPE_CHECKING, Literal, cast -from serena.repl.api.edit_api import EditApi -from serena.repl.api.fs_api import FsApi from serena.tools import EditingToolWithDiagnostics, Tool, ToolMarkerOptional +if TYPE_CHECKING: + from serena.repl.api.edit_api import EditApi + from serena.repl.api.fs_api import FsApi + class EditApiMixin: """ - Mixin for tools which delegate to the editing API + Mixin for tools which delegate to the editing API. + The API is imported locally, since the API module refers to the tools (as corresponding tools). """ - def _api(self) -> EditApi: + def _api(self) -> "EditApi": + from serena.repl.api.edit_api import EditApi + tool = cast(Tool, cast(object, self)) return EditApi(tool.agent) class FsApiMixin: """ - Mixin for tools which delegate to the file system API + Mixin for tools which delegate to the file system API. + The API is imported locally, since the API module refers to the tools (as corresponding tools). """ - def _api(self) -> FsApi: + def _api(self) -> "FsApi": + from serena.repl.api.fs_api import FsApi + tool = cast(Tool, cast(object, self)) return FsApi(tool.agent) diff --git a/src/serena/tools/jetbrains_tools.py b/src/serena/tools/jetbrains_tools.py index 0e877d5a..9985957d 100644 --- a/src/serena/tools/jetbrains_tools.py +++ b/src/serena/tools/jetbrains_tools.py @@ -1,26 +1,28 @@ # SPDX-License-Identifier: GPL-3.0-or-later import logging -from typing import TYPE_CHECKING, Literal +from typing import TYPE_CHECKING, Literal, cast -from serena.repl.api.jb_api import JetBrainsApi +from serena.symbol import SymbolDictGrouper from serena.tools import Tool, ToolMarkerBeta, ToolMarkerOptional, ToolMarkerSymbolicEdit, ToolMarkerSymbolicRead if TYPE_CHECKING: - from serena.agent import SerenaAgent + from serena.repl.api.jb_api import JetBrainsApi log = logging.getLogger(__name__) class JetBrainsApiMixin: """ - Mixin for tools which delegate to the JetBrains API + Mixin for tools which delegate to the JetBrains API. + The API is imported locally, since the API module refers to the tools (as corresponding tools). """ - agent: "SerenaAgent" + def _api(self) -> "JetBrainsApi": + from serena.repl.api.jb_api import JetBrainsApi - def _api(self) -> JetBrainsApi: - return JetBrainsApi(self.agent) + tool = cast(Tool, cast(object, self)) + return JetBrainsApi(tool.agent) class JetBrainsFindSymbolTool(Tool, ToolMarkerSymbolicRead, ToolMarkerOptional, JetBrainsApiMixin): @@ -28,7 +30,11 @@ class JetBrainsFindSymbolTool(Tool, ToolMarkerSymbolicRead, ToolMarkerOptional, Performs a global (or local) search for symbols using the JetBrains backend """ - symbol_dict_grouper = JetBrainsApi.find_symbol_grouper_ + @property + def symbol_dict_grouper(self) -> SymbolDictGrouper: + from serena.repl.api.jb_api import JetBrainsApi + + return JetBrainsApi.find_symbol_grouper_ def apply( self, @@ -217,7 +223,11 @@ class JetBrainsFindReferencingSymbolsTool(Tool, ToolMarkerSymbolicRead, ToolMark Finds symbols that reference the given symbol using the JetBrains backend """ - symbol_dict_grouper = JetBrainsApi.references_grouper_ + @property + def symbol_dict_grouper(self) -> SymbolDictGrouper: + from serena.repl.api.jb_api import JetBrainsApi + + return JetBrainsApi.references_grouper_ def apply( self, @@ -245,7 +255,11 @@ class JetBrainsGetSymbolsOverviewTool(Tool, ToolMarkerSymbolicRead, ToolMarkerOp Retrieves an overview of the top-level symbols within a specified file using the JetBrains backend """ - symbol_dict_grouper = JetBrainsApi.overview_grouper_ + @property + def symbol_dict_grouper(self) -> SymbolDictGrouper: + from serena.repl.api.jb_api import JetBrainsApi + + return JetBrainsApi.overview_grouper_ def apply( self, diff --git a/src/serena/tools/memory_tools.py b/src/serena/tools/memory_tools.py index 6c2e966a..cfbbf3e4 100644 --- a/src/serena/tools/memory_tools.py +++ b/src/serena/tools/memory_tools.py @@ -1,17 +1,22 @@ # SPDX-License-Identifier: GPL-3.0-or-later -from typing import Literal, cast +from typing import TYPE_CHECKING, Literal, cast -from serena.repl.api.mem_api import MemoryApi from serena.tools import Tool, ToolMarkerCanEdit +if TYPE_CHECKING: + from serena.repl.api.mem_api import MemoryApi + class MemoryApiMixin: """ - Mixin for tools which delegate to the memory API + Mixin for tools which delegate to the memory API. + The API is imported locally, since the API module refers to the tools (as corresponding tools). """ - def _api(self) -> MemoryApi: + def _api(self) -> "MemoryApi": + from serena.repl.api.mem_api import MemoryApi + tool = cast(Tool, cast(object, self)) return MemoryApi(tool.agent) diff --git a/src/serena/tools/symbol_tools.py b/src/serena/tools/symbol_tools.py index 76022951..9712b821 100644 --- a/src/serena/tools/symbol_tools.py +++ b/src/serena/tools/symbol_tools.py @@ -3,9 +3,9 @@ Language server-related tools """ # SPDX-License-Identifier: GPL-3.0-or-later -from typing import cast +from typing import TYPE_CHECKING, cast -from serena.repl.api.lsp_api import LspApi +from serena.symbol import SymbolDictGrouper from serena.tools import ( EditingToolWithDiagnostics, Tool, @@ -15,13 +15,19 @@ from serena.tools import ( from serena.tools.file_tools import EditApiMixin from serena.tools.tools_base import ToolMarkerOptional +if TYPE_CHECKING: + from serena.repl.api.lsp_api import LspApi + class LspApiMixin: """ - Mixin for tools which delegate to the language server API + Mixin for tools which delegate to the language server API. + The API is imported locally, since the API module refers to the tools (as corresponding tools). """ - def _api(self) -> LspApi: + def _api(self) -> "LspApi": + from serena.repl.api.lsp_api import LspApi + tool = cast(Tool, cast(object, self)) return LspApi(tool.agent) @@ -41,7 +47,11 @@ class GetSymbolsOverviewTool(Tool, ToolMarkerSymbolicRead, LspApiMixin): Gets an overview of the top-level symbols defined in a given file. """ - symbol_dict_grouper = LspApi.overview_grouper_ + @property + def symbol_dict_grouper(self) -> SymbolDictGrouper: + from serena.repl.api.lsp_api import LspApi + + return LspApi.overview_grouper_ def apply(self, relative_path: str, depth: int = -1, max_answer_chars: int = -1) -> str: """ @@ -65,7 +75,11 @@ class FindSymbolTool(Tool, ToolMarkerSymbolicRead, LspApiMixin): Performs a global (or local) search using the language server backend. """ - symbol_dict_grouper = LspApi.find_symbol_dict_grouper_ + @property + def symbol_dict_grouper(self) -> SymbolDictGrouper: + from serena.repl.api.lsp_api import LspApi + + return LspApi.find_symbol_dict_grouper_ def apply( self, @@ -146,7 +160,11 @@ class FindReferencingSymbolsTool(Tool, ToolMarkerSymbolicRead, LspApiMixin): Finds symbols that reference the given symbol """ - symbol_dict_grouper = LspApi.references_grouper_ + @property + def symbol_dict_grouper(self) -> SymbolDictGrouper: + from serena.repl.api.lsp_api import LspApi + + return LspApi.references_grouper_ def apply( self, diff --git a/test/serena/test_repl_tool.py b/test/serena/test_repl_tool.py index a2923915..fc04c434 100644 --- a/test/serena/test_repl_tool.py +++ b/test/serena/test_repl_tool.py @@ -11,7 +11,7 @@ from serena.config.serena_config import ApiInclusionDefinition from serena.repl.api.lsp_api import LspApi from serena.repl.facade import ApiScope, Facade, FacadeApi, FacadeMethodInfo, facade_method from serena.repl.repl import SerenaRepl -from serena.tools import SerenaReplTool +from serena.tools import FindSymbolTool, SerenaReplTool from solidlsp.ls_config import LanguageServerId from test.conftest import agent_for_project_context @@ -165,6 +165,15 @@ class TestFacade: facade = Facade.from_api(self.DummyApi(MagicMock()), self._scope(included_apis=["dummy.extra"])) assert "extra" in facade.enabled_method_names + def test_corresponding_tool(self) -> None: + facade = Facade.from_api(self.DummyApi(MagicMock()), ApiScope()) + assert facade.get_method("add").info.get_corresponding_tool_name() is None + + lsp_facade = Facade.from_api(LspApi(MagicMock()), ApiScope()) + info = lsp_facade.get_method("find_symbol").info + assert info.corresponding_tool is FindSymbolTool + assert info.get_corresponding_tool_name() == "find_symbol" + def test_method_info_mirrors_decorator(self) -> None: facade = Facade.from_api(self.DummyApi(MagicMock()), ApiScope()) assert facade.get_method("add").info == FacadeMethodInfo(name="add") From e7aa6a74e17b8e8dfac45875eb8917c52f22fd8f Mon Sep 17 00:00:00 2001 From: Dominik Jain Date: Mon, 7 Sep 2026 18:20:15 +0200 Subject: [PATCH 18/53] Add repl memory documenting the REPL's structure, principles and availability policy --- .serena/memories/critical_info.md | 5 ++++ .serena/memories/repl.md | 50 +++++++++++++++++++++++++++++++ 2 files changed, 55 insertions(+) create mode 100644 .serena/memories/repl.md diff --git a/.serena/memories/critical_info.md b/.serena/memories/critical_info.md index 55124efd..f6d85ce9 100644 --- a/.serena/memories/critical_info.md +++ b/.serena/memories/critical_info.md @@ -34,6 +34,11 @@ Snapshot tests use syrupy. * Each piece of information appears exactly once, at the element that owns it: callers do not explain callees' internals, and callees do not describe their callers. +# REPL & facades + +Read `mem:repl` before working on `serena.repl` (the code-execution paradigm and its facade APIs) or on tools +delegating to it: structure, exposure/naming principles, configuration of the API scope and the availability policy. + # Commits & pull requests * Commit messages: wrap all lines (subject and body) at ~100 characters; write the message to a file and commit with `-F` diff --git a/.serena/memories/repl.md b/.serena/memories/repl.md new file mode 100644 index 00000000..2d8b7f59 --- /dev/null +++ b/.serena/memories/repl.md @@ -0,0 +1,50 @@ +# REPL (`serena.repl`) + +Alternative interaction paradigm: one tool (`serena_repl`) executes Python code against entrypoint `s`, +whose attributes are facades (`s.lsp`, `s.edit`, `s.fs`, `s.mem`, `s.shell`, `s.jb`). +Code runs as a function body (`return` defines the result); a single expression is evaluated directly. + +## Structure + +- `repl/api/*_api.py`: `FacadeApi` implementations = the single implementation of each operation. + The classic tools are thin adapters delegating to the APIs (via `*ApiMixin`); tools retain only + transport concerns (input sanitisation, diagnostics context, tool-level output shaping). +- `repl/facade.py`: `Facade` = indirection over an API instance; `FacadeMethod` (enabled flag + `FacadeMethodInfo`); + `ApiScope` = which facades/methods are enabled. +- `repl/repl.py`: `SerenaRepl` (execution, error formatting), `SerenaReplEntrypoint` (`s`, `info`). +- `repl/representable.py`: `Representable`/`Renderer`; result objects carry their rendering policy. + +## Design principles + +- Exposure is explicit: a method is exposed iff decorated with `@facade_method(...)`, which carries + `optional`, `beta`, `can_edit`, `corresponding_tool` (mirroring the tool markers; the tool correspondence + is recorded for optional derivation of exclusions and prompt conditions, never applied automatically). +- Naming: on result objects and non-exposed API helpers, a trailing underscore (`symbols_`, `to_dict_`) + marks members that are Serena-public but not LLM-facing. +- Facades group by *domain*, not by read vs. write; mutation is expressed via `can_edit` (read-only projects + exclude editing methods). Boundary `fs`/`edit`: files as units vs. modifying content within existing files. +- Facade descriptions describe the domain only; never list operations (the method list is always shown alongside). +- Output parameters (depth, include_body, max_answer_chars, ...) are passed at retrieval time so that the + rendering policy is fixed once and inherited by derived results. +- Results expose data to code (`.symbols`, `.occurrences`, `.lines`, ...) and render like the classic tool output. +- Progressive disclosure: a priori only facade names, descriptions and method names; `s.info("")` / + `s.info(".")` give signature + docstring together, never a signature alone. +- APIs must not import `serena.tools` at module level except for tool classes in decorators; tools import + APIs locally in `_api()` (API modules refer to tool classes). + +## Configuration + +- `included_apis`/`excluded_apis` (references `facade` or `facade.method`) in global config, context, modes, + project config; applied in that order via `ApiScope` (exclusions first, then inclusions; later definitions win). + Optional methods and all methods of an excluded facade must be included explicitly. +- The REPL is rebuilt whenever the active tools are updated (mode switch, project activation). + +## Availability policy + +- Keep as much functionality as possible in the REPL; do not mirror the contexts' tool exclusions. + Reads must stay in (composability); exclusions can only steer the model, never enforce anything. +- Python code can always modify the system; the REPL tool is inherently fully privileged, regardless of + facade scope or the project's `read_only` setting (which only makes Serena's own API refuse edits). + A "read-only REPL" is not feasible and must not be promised. +- Session/project management (activate_project, initial_instructions, dashboard, ...) stays tool-only; + facades expose operations on the active project. From 337d545c4b4c0b7e5560b99869a34517c9efdda2 Mon Sep 17 00:00:00 2001 From: Dominik Jain Date: Mon, 7 Sep 2026 19:44:47 +0200 Subject: [PATCH 19/53] Add agent_interface setting selecting the tool or REPL interface Introduce AgentInterface (tools/REPL) as a setting in the global configuration, overridable per project (resolution: project > global > default "tools") and via the CLI option --agent-interface. The interface is fixed for the session, like the language backend. In REPL mode, the set of exposed tools is fixed: the REPL tool, initial_instructions and (unless in a single-project session) activate_project. Tool inclusion/exclusion definitions do not apply in REPL mode, as they pertain to the tool interface; the REPL is configured via API inclusions/exclusions. The REPL tool is optional in tool mode. SerenaAgent.is_single_project records whether the session is a single-project session. --- .serena/memories/repl.md | 7 ++ src/serena/agent.py | 69 +++++++++++++++---- src/serena/cli.py | 10 +++ src/serena/config/serena_config.py | 69 ++++++++++++++++++- src/serena/mcp.py | 6 +- src/serena/resources/project.template.yml | 5 ++ .../resources/serena_config.template.yml | 12 ++++ src/serena/tools/repl_tools.py | 4 +- test/serena/config/test_serena_config.py | 60 ++++++++++++++++ test/serena/test_serena_agent.py | 22 +++++- 10 files changed, 244 insertions(+), 20 deletions(-) diff --git a/.serena/memories/repl.md b/.serena/memories/repl.md index 2d8b7f59..ee88a232 100644 --- a/.serena/memories/repl.md +++ b/.serena/memories/repl.md @@ -34,6 +34,13 @@ Code runs as a function body (`return` defines the result); a single expression ## Configuration +- `agent_interface: tools | REPL` (`AgentInterface`; global config, overridable per project; CLI `--agent-interface`). + `None` = Serena's default (`tools`). Fixed for the session. In REPL mode the toolset is *fixed* + (`serena_repl`, `initial_instructions`, `activate_project` unless single-project); tool inclusion/exclusion + definitions do not apply — each interface has its own configuration vocabulary (tool definitions ↔ tools, + API definitions ↔ REPL). Contexts do not influence the interface. + Idea (not implemented, considered over-engineered for now): contexts could declare *supported* interfaces + (a capability constraint, e.g. clients that handle the REPL badly), with the user's preference choosing among them. - `included_apis`/`excluded_apis` (references `facade` or `facade.method`) in global config, context, modes, project config; applied in that order via `ApiScope` (exclusions first, then inclusions; later definitions win). Optional methods and all methods of an excluded facade must be included explicitly. diff --git a/src/serena/agent.py b/src/serena/agent.py index b2efef85..4446ae06 100644 --- a/src/serena/agent.py +++ b/src/serena/agent.py @@ -31,6 +31,7 @@ from serena import serena_version from serena.analytics import RegisteredTokenCountEstimator, ToolUsageStats from serena.config.context_mode import SerenaAgentContext, SerenaAgentMode from serena.config.serena_config import ( + AgentInterface, LanguageBackend, ModeSelectionDefinition, ModeSelectionDefinitionWithAddedModes, @@ -59,10 +60,12 @@ from serena.task_executor import TaskExecutor from serena.tools import ( ActivateProjectTool, GetCurrentConfigTool, + InitialInstructionsTool, OnboardingTool, OpenDashboardTool, ReadMemoryTool, ReplaceContentTool, + SerenaReplTool, Tool, ToolMarker, ToolRegistry, @@ -660,6 +663,13 @@ class SerenaAgent: log_choice=True, ) + # determine the effective agent interface for this session (project configuration > global configuration). + # Like the language backend, it is fixed for the session, since the set of exposed tools cannot change after startup. + self._agent_interface = self.serena_config.determine_agent_interface( + project_config=registered_project_to_activate.project_config if registered_project_to_activate is not None else None, + log_choice=True, + ) + # create the tool names mapping for prompts self._prompt_tool_names_mapping = self._create_prompt_tool_names_mapping(self._language_backend) @@ -685,8 +695,14 @@ class SerenaAgent: self._project_activation_error = str(e) self._update_active_modes() + # determine whether we are operating in a single-project session, i.e. the project that was activated at startup + # (if any) is the only project that will be worked with throughout the session (no project switching) + self._is_single_project = self._context.single_project and self._active_project is not None + # determine the base toolset defining the set of exposed tools (which e.g. the MCP shall see), - self._base_toolset = self._create_base_toolset(self.serena_config, self._context, self._active_modes, self._active_project) + self._base_toolset = self._create_base_toolset( + self.serena_config, self._context, self._active_modes, self._active_project, self._agent_interface, self._is_single_project + ) self._exposed_tools = self._base_toolset.to_available_tools(self._all_tools) log.info(f"Number of exposed tools: {len(self._exposed_tools)}. Exposed tools: {self._exposed_tools.tool_names}") @@ -753,10 +769,12 @@ class SerenaAgent: context: SerenaAgentContext, modes: ActiveModes, project: Project | None, + agent_interface: AgentInterface, + is_single_project: bool, ) -> ToolSet: """ Determines the base toolset defining the set of exposed tools (which e.g. the MCP shall see). - It depends on ... + In REPL mode, the toolset is fixed. Otherwise, it depends on ... * dashboard availability/opening on launch * Serena config * the context (which is fixed for the session) @@ -764,6 +782,13 @@ class SerenaAgent: * the optional tools enabled by initial dynamic modes * single-project mode reductions (if applicable) """ + # when in REPL mode, the toolset is fixed and does not depend on the configuration, context, modes or project + if agent_interface.is_repl(): + tool_classes: list[type[Tool]] = [SerenaReplTool, InitialInstructionsTool] + if not is_single_project: + tool_classes.append(ActivateProjectTool) + return ToolSet({tool_class.get_name_from_cls() for tool_class in tool_classes}) + # determine whether to include the OpenDashboardTool based on the Serena configuration tool_inclusion_definitions: list[ToolInclusionDefinition] = [] if serena_config.web_dashboard and not serena_config.web_dashboard_open_on_launch and not serena_config.gui_log_window: @@ -775,10 +800,6 @@ class SerenaAgent: tool_inclusion_definitions.append(serena_config) tool_inclusion_definitions.append(context) - # determine whether we are operating in a single-project context - # (i.e. the project that is activated at startup is the only project that will be worked with throughout the session) - is_single_project = context.single_project and project is not None - # consider modes # * base modes: These cannot be changed, so they are fully applied for base_mode in modes.get_base_modes(include_background_base_modes=True): @@ -830,6 +851,17 @@ class SerenaAgent: def get_language_backend(self) -> LanguageBackend: return self._language_backend + def is_single_project(self) -> bool: + """ + :return: whether this is a single-project session, i.e. the project activated at startup is the only project + that will be worked with throughout the session (no project switching); requires a single-project context + and a project at startup + """ + return self._is_single_project + + def get_agent_interface(self) -> AgentInterface: + return self._agent_interface + def get_current_tasks(self) -> list[TaskExecutor.TaskInfo]: """ Gets the list of tasks currently running or queued for execution. @@ -1142,23 +1174,29 @@ class SerenaAgent: def _update_active_tools(self) -> None: """ - Updates the active tools based on the active modes and the active project. + Updates the active tools (and the REPL, which depends on the same configuration) based on the active modes + and the active project. Must be called whenever the active modes or the active project change. The base tool set already takes the Serena configuration and the context into account (as well as many other aspects, such as JetBrains mode). """ - # apply modes - tool_set = self._base_toolset.apply(*self._active_modes.get_modes()) + if self._agent_interface.is_repl(): + # the REPL toolset is fixed; tool inclusion/exclusion definitions do not apply + tool_set = self._base_toolset + else: + # apply modes + tool_set = self._base_toolset.apply(*self._active_modes.get_modes()) - # apply active project configuration (if any) - if self._active_project is not None: - tool_set = tool_set.apply(self._active_project.project_config) - if self._active_project.project_config.read_only: - tool_set = tool_set.without_editing_tools() + # apply active project configuration (if any) + if self._active_project is not None: + tool_set = tool_set.apply(self._active_project.project_config) + if self._active_project.project_config.read_only: + tool_set = tool_set.without_editing_tools() self._active_tools = tool_set.to_available_tools(self._all_tools) log.info(f"Active tools ({len(self._active_tools)}): {', '.join(self._active_tools.tool_names)}") - # reset the REPL, which depends on the same configuration (it is re-created on demand) + # reset the REPL, whose facades/API scope depend on the active modes and project (it is re-created on demand). + # NOTE: This must happen irrespective of the agent interface, since the REPL tool may be active in tool mode as well. self._repl = None # check if a tool was activated that is not in the exposed tool set and issue a warning if so @@ -1435,6 +1473,7 @@ class SerenaAgent: result_str += f"Active project: {self._active_project.project_name}\n" else: result_str += "No active project\n" + result_str += f"Agent interface: {self._agent_interface.value}\n" result_str += f"Language backend: {self._language_backend.value}" if self._active_project and self._active_project.project_config.language_backend is not None: result_str += " (project override)" diff --git a/src/serena/cli.py b/src/serena/cli.py index b8ae292c..97ca9fc9 100644 --- a/src/serena/cli.py +++ b/src/serena/cli.py @@ -23,6 +23,7 @@ from serena import serena_version from serena.config.client_setup import client_setup_handlers from serena.config.context_mode import SerenaAgentContext, SerenaAgentMode from serena.config.serena_config import ( + AgentInterface, LanguageBackend, ModeSelectionDefinition, ModeSelectionDefinitionWithAddedModes, @@ -263,6 +264,13 @@ class TopLevelCommands(AutoRegisteringGroup): default=None, help="Override the configured language backend.", ) + @click.option( + "--agent-interface", + type=click.Choice([i.value for i in AgentInterface], case_sensitive=False), + default=None, + help="Override the configured agent interface: 'tools' (one tool per operation) or " + "'REPL' (Python code execution via the serena_repl tool, with a fixed set of tools).", + ) @click.option( "--transport", type=click.Choice(["stdio", "sse", "streamable-http"]), @@ -325,6 +333,7 @@ class TopLevelCommands(AutoRegisteringGroup): default_modes: Sequence[str], added_modes: Sequence[str], language_backend: str | None, + agent_interface: str | None, transport: Literal["stdio", "sse", "streamable-http"], host: str, port: int, @@ -385,6 +394,7 @@ class TopLevelCommands(AutoRegisteringGroup): port=port, mode_selection_def=mode_selection_def, language_backend=LanguageBackend.from_str(language_backend) if language_backend else None, + agent_interface=AgentInterface.from_str(agent_interface) if agent_interface else None, enable_web_dashboard=enable_web_dashboard, open_web_dashboard=open_web_dashboard, enable_gui_log_window=enable_gui_log_window, diff --git a/src/serena/config/serena_config.py b/src/serena/config/serena_config.py index 71043c46..3f2d5aab 100644 --- a/src/serena/config/serena_config.py +++ b/src/serena/config/serena_config.py @@ -208,6 +208,37 @@ class ModeSelectionDefinitionWithAddedModes(ModeSelectionDefinition): added_modes: Sequence[str] | None = None +class AgentInterface(Enum): + """ + The interface through which the agent (LLM) accesses Serena's functionality. + """ + + TOOLS = "tools" + """ + The classic tool interface: each operation is a separate tool, and the set of tools is configurable + (via tool inclusions/exclusions in the configuration, context, modes and project). + """ + REPL = "REPL" + """ + The REPL interface: operations are accessed programmatically via the serena_repl tool, which executes Python code. + The set of tools is fixed (the REPL tool and the tools required for session management) and tool inclusions/exclusions + do not apply; the operations available in the REPL are configured via API inclusions/exclusions instead. + """ + + @staticmethod + def from_str(interface_str: str) -> "AgentInterface": + for interface in AgentInterface: + if interface.value.lower() == interface_str.lower(): + return interface + raise ValueError(f"Unknown agent interface '{interface_str}': valid values are {[i.value for i in AgentInterface]}") + + def is_tools(self) -> bool: + return self == AgentInterface.TOOLS + + def is_repl(self) -> bool: + return self == AgentInterface.REPL + + class LanguageBackend(Enum): LSP = "LSP" """ @@ -294,6 +325,7 @@ class SharedConfig(ToolInclusionDefinition, ApiInclusionDefinition, ToStringMixi symbol_info_budget: float | None = None language_backend: LanguageBackend | None = None + agent_interface: AgentInterface | None = None line_ending: LineEnding | None = None read_only_memory_patterns: list[str] = field(default_factory=list) ignored_memory_patterns: list[str] = field(default_factory=list) @@ -624,6 +656,8 @@ class ProjectConfig(SharedConfig, ModeSelectionDefinitionWithAddedModes): language_backend_value = data.get("language_backend") language_backend = LanguageBackend.from_str(language_backend_value) if language_backend_value else None + agent_interface_value = data.get("agent_interface") + agent_interface = AgentInterface.from_str(agent_interface_value) if agent_interface_value else None line_ending_value = data.get("line_ending") line_ending = LineEnding.from_str(line_ending_value) if line_ending_value else None @@ -659,6 +693,7 @@ class ProjectConfig(SharedConfig, ModeSelectionDefinitionWithAddedModes): encoding=data["encoding"], line_ending=line_ending, language_backend=language_backend, + agent_interface=agent_interface, added_modes=data["added_modes"], default_modes=data["default_modes"], symbol_info_budget=symbol_info_budget, @@ -683,6 +718,7 @@ class ProjectConfig(SharedConfig, ModeSelectionDefinitionWithAddedModes): # map fields using non-primitive types to a YAML-compatible representation d["language_servers"] = [lang.get_key() for lang in self.language_servers] d["language_backend"] = self.language_backend.value if self.language_backend is not None else None + d["agent_interface"] = self.agent_interface.value if self.agent_interface is not None else None d["line_ending"] = self.line_ending.value if self.line_ending is not None else None return d @@ -976,7 +1012,7 @@ class SerenaConfig(SharedConfig, ModeSelectionDefinitionWithBaseModes): # *** static members *** CONFIG_FILE = "serena_config.yml" - CONFIG_FIELDS_WITH_TYPE_CONVERSION = {"projects", "language_backend", "line_ending"} + CONFIG_FIELDS_WITH_TYPE_CONVERSION = {"projects", "language_backend", "agent_interface", "line_ending"} # *** methods *** @classmethod @@ -1126,6 +1162,15 @@ class SerenaConfig(SharedConfig, ModeSelectionDefinitionWithBaseModes): del loaded_commented_yaml["jetbrains"] instance.language_backend = language_backend + # determine agent interface + agent_interface_value = loaded_commented_yaml.get("agent_interface") + agent_interface: AgentInterface | None = None + if "agent_interface" in loaded_commented_yaml: + agent_interface = AgentInterface.from_str(agent_interface_value) if agent_interface_value else None + else: + num_migrations += 1 + instance.agent_interface = agent_interface + # determine line ending line_ending_value = loaded_commented_yaml.get("line_ending") if line_ending_value: @@ -1369,6 +1414,9 @@ class SerenaConfig(SharedConfig, ModeSelectionDefinitionWithBaseModes): # convert language backend to string commented_yaml["language_backend"] = self.language_backend.value + # convert agent interface to string (None if not configured) + commented_yaml["agent_interface"] = self.agent_interface.value if self.agent_interface is not None else None + # convert line ending to string commented_yaml["line_ending"] = self.line_ending.value @@ -1475,6 +1523,25 @@ class SerenaConfig(SharedConfig, ModeSelectionDefinitionWithBaseModes): return True return False + def determine_agent_interface(self, project_config: "ProjectConfig | None" = None, log_choice: bool = False) -> AgentInterface: + """ + Determines the effective agent interface: the project configuration takes precedence over the global configuration; + if neither configures an interface, the tool interface is used. + + :param project_config: the configuration of the project to be activated, if any + :param log_choice: whether to log the choice + :return: the effective agent interface + """ + if project_config is not None and project_config.agent_interface is not None: + agent_interface, source = project_config.agent_interface, "project configuration" + elif self.agent_interface is not None: + agent_interface, source = self.agent_interface, "global configuration" + else: + agent_interface, source = AgentInterface.TOOLS, "default" + if log_choice: + log.info(f"Using agent interface '{agent_interface.value}' ({source})") + return agent_interface + def determine_language_backend(self, project_config: ProjectConfig | None = None, log_choice: bool = False): language_backend = self.language_backend if project_config and project_config.language_backend is not None: diff --git a/src/serena/mcp.py b/src/serena/mcp.py index 1bd2844b..cbe38d8b 100644 --- a/src/serena/mcp.py +++ b/src/serena/mcp.py @@ -26,7 +26,7 @@ from serena.agent import ( SerenaAgent, ) from serena.config.context_mode import SerenaAgentContext -from serena.config.serena_config import LanguageBackend, ModeSelectionDefinition, SerenaConfig +from serena.config.serena_config import AgentInterface, LanguageBackend, ModeSelectionDefinition, SerenaConfig from serena.constants import DEFAULT_CONTEXT, SERENA_LOG_FORMAT from serena.tools import Tool, ToolCallError from serena.util.exception import show_fatal_exception_safe @@ -326,6 +326,7 @@ class SerenaMCPFactory: port: int = 8000, mode_selection_def: ModeSelectionDefinition | None = None, language_backend: LanguageBackend | None = None, + agent_interface: AgentInterface | None = None, enable_web_dashboard: bool | None = None, enable_gui_log_window: bool | None = None, open_web_dashboard: bool | None = None, @@ -341,6 +342,7 @@ class SerenaMCPFactory: :param port: The port to bind to :param mode_selection_def: the mode selection definition to apply :param language_backend: the language backend to use, overriding the configuration setting. + :param agent_interface: the agent interface to use, overriding the configuration setting. :param enable_web_dashboard: Whether to enable the web dashboard. If not specified, will take the value from the serena configuration. :param enable_gui_log_window: Whether to enable the GUI log window. It currently does not work on macOS, and setting this to True will be ignored then. If not specified, will take the value from the serena configuration. @@ -371,6 +373,8 @@ class SerenaMCPFactory: config.tool_timeout = tool_timeout if language_backend is not None: config.language_backend = language_backend + if agent_interface is not None: + config.agent_interface = agent_interface self.agent = self._create_serena_agent(config, modes=mode_selection_def, project_activation_error=project_activation_error) diff --git a/src/serena/resources/project.template.yml b/src/serena/resources/project.template.yml index b45df235..c72f7ad0 100644 --- a/src/serena/resources/project.template.yml +++ b/src/serena/resources/project.template.yml @@ -68,6 +68,11 @@ line_ending: # is activated post-init, an error will be returned. language_backend: +# The interface through which the agent (LLM) accesses Serena's functionality (overrides the global setting). +# Valid values: tools, REPL (see the global configuration for details); leave empty to use the global setting. +# Note: the interface is fixed at startup. If a project is activated post-init, its setting is not applied. +agent_interface: + # whether to use project's .gitignore files to ignore files ignore_all_files_in_gitignore: true diff --git a/src/serena/resources/serena_config.template.yml b/src/serena/resources/serena_config.template.yml index 96d9cb58..f7163f2a 100644 --- a/src/serena/resources/serena_config.template.yml +++ b/src/serena/resources/serena_config.template.yml @@ -7,6 +7,18 @@ # in your IDE). language_backend: LSP +# The interface through which the agent (LLM) accesses Serena's functionality: +# * tools: the classic tool interface, in which each operation is a separate tool. +# The set of tools is configurable via the tool inclusion/exclusion settings (excluded_tools etc.) +# of the configuration, the context, modes and the project. +# * REPL: operations are accessed programmatically via the serena_repl tool, which executes Python code. +# The set of tools is fixed (the REPL tool and the tools required for session management), and the tool +# inclusion/exclusion settings do not apply; instead, the operations available in the REPL are configured +# via the API inclusion/exclusion settings (excluded_apis etc.). +# Leave empty to use Serena's default (currently "tools"). +# The interface is fixed at startup and can be overridden per project. +agent_interface: + # line ending convention to use when writing source files. # Possible values: "lf" (Unix), "crlf" (Windows), "native" (platform default). # Note that Serena's own files (e.g. memories and configuration files) always use native line endings. diff --git a/src/serena/tools/repl_tools.py b/src/serena/tools/repl_tools.py index ba7b4980..611241e4 100644 --- a/src/serena/tools/repl_tools.py +++ b/src/serena/tools/repl_tools.py @@ -4,10 +4,10 @@ Tools which provide access to Serena's functionality through Python code executi # SPDX-License-Identifier: GPL-3.0-or-later -from serena.tools.tools_base import Tool, ToolMarkerBeta +from serena.tools.tools_base import Tool, ToolMarkerBeta, ToolMarkerOptional -class SerenaReplTool(Tool, ToolMarkerBeta): +class SerenaReplTool(Tool, ToolMarkerOptional, ToolMarkerBeta): """ Executes Python code which accesses Serena's functionality programmatically. """ diff --git a/test/serena/config/test_serena_config.py b/test/serena/config/test_serena_config.py index 529667f7..cdfa91c5 100644 --- a/test/serena/config/test_serena_config.py +++ b/test/serena/config/test_serena_config.py @@ -10,6 +10,7 @@ import pytest from serena.agent import SerenaAgent from serena.config.serena_config import ( DEFAULT_PROJECT_SERENA_FOLDER_LOCATION, + AgentInterface, LanguageBackend, ProjectConfig, RegisteredProject, @@ -218,6 +219,65 @@ class TestProjectConfigLanguageBackend: assert config.language_backend is None +class TestAgentInterface: + """Tests for the agent_interface setting (global and per project).""" + + @staticmethod + def _project_config(agent_interface: AgentInterface | None) -> ProjectConfig: + return ProjectConfig(project_name="test", language_servers=[LanguageServerId.PYTHON], agent_interface=agent_interface) + + def test_agent_interface_roundtrips_through_project_yaml(self): + assert self._project_config(AgentInterface.REPL)._to_yaml_dict()["agent_interface"] == "REPL" + assert self._project_config(None)._to_yaml_dict()["agent_interface"] is None + + def test_agent_interface_parsed_from_project_dict(self): + data, _ = ProjectConfig._load_yaml_dict(PROJECT_TEMPLATE_FILE) + data["project_name"] = "test" + data["languages"] = ["python"] + data["agent_interface"] = "repl" # case-insensitive + assert ProjectConfig._from_dict(data, local_override_keys=[]).agent_interface == AgentInterface.REPL + data.pop("agent_interface") + assert ProjectConfig._from_dict(data, local_override_keys=[]).agent_interface is None + + def test_determine_agent_interface_precedence(self): + # default + assert SerenaConfig().determine_agent_interface() == AgentInterface.TOOLS + assert SerenaConfig().determine_agent_interface(self._project_config(None)) == AgentInterface.TOOLS + # global configuration + assert SerenaConfig(agent_interface=AgentInterface.REPL).determine_agent_interface() == AgentInterface.REPL + # project configuration takes precedence + config = SerenaConfig(agent_interface=AgentInterface.REPL) + assert config.determine_agent_interface(self._project_config(AgentInterface.TOOLS)) == AgentInterface.TOOLS + assert config.determine_agent_interface(self._project_config(None)) == AgentInterface.REPL + + def test_repl_toolset_is_fixed_and_repl_follows_project_activation(self): + """ + In REPL mode, neither the exposed nor the active toolset is affected by tool inclusion/exclusion definitions + (here: the project's exclusions and read-only setting), whereas the REPL's API scope follows the active project. + """ + config, name = _make_config_with_project("test_proj") + config.agent_interface = AgentInterface.REPL + project_config = config.projects[0].project_config + project_config.excluded_tools = ["initial_instructions", "serena_repl"] + project_config.excluded_apis = ["mem"] + project_config.read_only = True + + agent = SerenaAgent(project=None, serena_config=config) + try: + # before activation: the fixed toolset and the full set of facades + fixed_toolset = {"serena_repl", "initial_instructions", "activate_project"} + assert {t.get_name() for t in agent.get_exposed_tool_instances()} == fixed_toolset + assert set(agent.get_active_tool_names()) == fixed_toolset + assert "s.mem" in agent.get_repl().entrypoint.overview() + + # after activation: the toolset is unchanged, the REPL reflects the project's API exclusions + agent.activate_project_from_path_or_name(name) + assert set(agent.get_active_tool_names()) == fixed_toolset + assert "s.mem" not in agent.get_repl().entrypoint.overview() + finally: + agent.on_shutdown(timeout=5) + + def _make_config_with_project( project_name: str, language_backend: LanguageBackend | None = None, diff --git a/test/serena/test_serena_agent.py b/test/serena/test_serena_agent.py index 8fb5002b..8ee9463a 100644 --- a/test/serena/test_serena_agent.py +++ b/test/serena/test_serena_agent.py @@ -14,7 +14,7 @@ from _pytest.mark import Mark, MarkDecorator, ParameterSet from serena.agent import SerenaAgent from serena.config.context_mode import SerenaAgentContext -from serena.config.serena_config import ProjectConfig, RegisteredProject, SerenaConfig +from serena.config.serena_config import AgentInterface, ProjectConfig, RegisteredProject, SerenaConfig from serena.lsp.lsp_diagnostics import DiagnosticsContext from serena.project import Project from serena.tools import ( @@ -30,6 +30,7 @@ from serena.tools import ( ReplaceInFilesTool, ReplaceSymbolBodyTool, SafeDeleteSymbol, + SerenaReplTool, Tool, ) from solidlsp.ls_config import LanguageServerId @@ -901,6 +902,25 @@ class TestSerenaAgent: finally: agent.on_shutdown(timeout=5) + @pytest.mark.python + @pytest.mark.skipif(not language_server_tests_enabled(LanguageServerId.PYTHON), reason="python tests are disabled in this environment") + @pytest.mark.parametrize("context_name", ["desktop-app", "grok"], ids=["multi_project", "single_project"]) + def test_repl_interface_exposes_fixed_toolset(self, serena_config, context_name: str): + # the toolset is fixed regardless of tool inclusions/exclusions (e.g. the context's or the configuration's); + # only the single-project property of the context matters (no project activation in that case) + serena_config.agent_interface = AgentInterface.REPL + serena_config.included_optional_tools = ["get_diagnostics_for_symbol"] + context = SerenaAgentContext.from_name(context_name) + agent = SerenaAgent(project="test_repo_python", serena_config=serena_config, context=context) + agent.execute_task(lambda: None) + try: + exposed = {tool.get_name() for tool in agent.get_exposed_tool_instances()} + expected = {"serena_repl", "initial_instructions"} | (set() if context.single_project else {"activate_project"}) + assert exposed == expected + assert "s.lsp" in agent.get_tool(SerenaReplTool).apply("s.info()") + finally: + agent.on_shutdown(timeout=5) + def _symbol_matches_expected_name(self, symbol: dict, expected_name: str) -> bool: return ( symbol.get("name") == expected_name From 6716193d71646872c65d11e670ed5663ec2ea20c Mon Sep 17 00:00:00 2001 From: Dominik Jain Date: Tue, 8 Sep 2026 02:05:50 +0200 Subject: [PATCH 20/53] Render prompts according to the agent interface Prompt parameters (available tools/markers and the tool name mapping) are now computed per agent interface (SerenaAgent.PromptParams). In REPL mode, the available tools comprise the exposed tools and the enabled facade methods, and the tool name mapping resolves tool names to the qualified names of the corresponding facade methods (e.g. `lsp.find_symbol`), such that prompts refer to operations as they are used in REPL code. The parameters are re-computed whenever the active tools change. FacadeMethod refers to its parent facade and provides facade_name and qualified_name; its name is taken from its info. Prompt templates use the tool name mapping consistently (fixing a template syntax error in the editing mode) and use interface-neutral wording. --- src/serena/agent.py | 124 +++++++++++++----- src/serena/repl/facade.py | 57 +++++--- src/serena/repl/repl.py | 8 +- src/serena/resources/config/modes/editing.yml | 14 +- .../config/prompt_templates/system_prompt.yml | 15 +-- test/serena/test_serena_agent.py | 5 + 6 files changed, 157 insertions(+), 66 deletions(-) diff --git a/src/serena/agent.py b/src/serena/agent.py index 4446ae06..3034d77d 100644 --- a/src/serena/agent.py +++ b/src/serena/agent.py @@ -582,6 +582,7 @@ class SerenaAgent: self.version = serena_version() self._config_changed_callbacks: list[Callable[[], None]] = [] self._repl: SerenaRepl | None = None + self._prompt_params: SerenaAgent.PromptParams | None = None # obtain serena configuration using the decoupled factory function self.serena_config = serena_config or SerenaConfig.from_config_file() @@ -670,9 +671,6 @@ class SerenaAgent: log_choice=True, ) - # create the tool names mapping for prompts - self._prompt_tool_names_mapping = self._create_prompt_tool_names_mapping(self._language_backend) - # create executor for starting the language server and running tools in another thread # This executor is used to achieve linear task execution self._task_executor = TaskExecutor("SerenaAgentTaskExecutor", self._task_completion_callback) @@ -980,27 +978,88 @@ class SerenaAgent: """ return self._active_modes - @staticmethod - def _create_prompt_tool_names_mapping(language_backend: LanguageBackend) -> dict[str, str]: + @dataclass + class PromptParams: """ - Creates a mapping from tool names to new tool names, which take into consideration - - * legacy tool names, where the name was changed and - * LSP tools which are functionally replaced by other tools due to the active language backend - (e.g. "find_symbol" being replaced by "jet_brains_find_symbol" in JetBrains mode). - - The mapping is intended to be used for the generation of prompts, such that prompts can - refer to tool names as `{{ tool_names["find_symbol"] }}`, and the mapping will ensure that - the correct tool name is used in the prompt based on the active language backend. - - :return: the mapping from tool names to new tool names + Holds parameters for prompt rendering """ - result = dict(ToolSet.LEGACY_TOOL_NAME_MAPPING) - class_replacements = language_backend.get_lsp_tool_class_replacements() - for tool_class in ToolRegistry().get_all_tool_classes(): - new_tool_class: type[Tool] = class_replacements.get(tool_class, tool_class) - result[tool_class.get_name_from_cls()] = new_tool_class.get_name_from_cls() - return result + + available_tools: set[str] + """ + available tool names or, in REPL mode, the names of the raw facade methods (without facade name prefix) and + the names of the corresponding tools + """ + available_markers: set[str] + """ + names of the ToolMarkers (class names) that the available tools inherit from + """ + tool_names_mapping: dict[str, str] + """ + mapping from standard tool names to currently used and replacement tool/API method names. + In particular, this maps + * legacy tool names to current tool names + * LSP tool names to their backend- and interface-specific counterparts + (e.g. "find_symbol" to "jet_brains_find_symbol" in JetBrains mode, "find_symbol" to the corresponding API method name + when using the REPL interface). + """ + + def _get_prompt_params(self) -> PromptParams: + """ + :return: parameters for prompt rendering depending on the current agent interface, language backend and active tools/methods + """ + if self._prompt_params is not None: + return self._prompt_params + + if self._agent_interface == AgentInterface.TOOLS: + # available tool names are simply the exposed tools + available_tool_names = set(self._exposed_tools.tool_names) + available_tool_marker_names = set(self._exposed_tools.tool_marker_names) + + tool_name_mapping = dict(ToolSet.LEGACY_TOOL_NAME_MAPPING) + class_replacements = self._language_backend.get_lsp_tool_class_replacements() + for tool_class in ToolRegistry().get_all_tool_classes(): + new_tool_class: type[Tool] = class_replacements.get(tool_class, tool_class) + tool_name_mapping[tool_class.get_name_from_cls()] = new_tool_class.get_name_from_cls() + + elif self._agent_interface == AgentInterface.REPL: + # available tool names include both the names of the facade methods and the names of the corresponding tools + repl = self.get_repl() + enabled_methods = repl.entrypoint.get_enabled_methods() + corresponding_tool_classes = [m.info.corresponding_tool for m in enabled_methods if m.info.corresponding_tool is not None] + available_tools = AvailableTools( + self._exposed_tools.tools + [self._all_tools[tool_class] for tool_class in corresponding_tool_classes] + ) + available_tool_names = set(available_tools.tool_names).union({m.info.name for m in enabled_methods}) + available_tool_marker_names = set(available_tools.tool_marker_names) + + tool_class_replacements = self._language_backend.get_lsp_tool_class_replacements() + methods_by_tool_class = {m.info.corresponding_tool: m for m in enabled_methods if m.info.corresponding_tool is not None} + + def get_name(tool_class: type[Tool]) -> str: + # if there is a corresponding method in the API, return its qualified name (as used in REPL code) + method = methods_by_tool_class.get(tool_class) + if method is not None: + return method.qualified_name + # if there is a corresponding method for the replacement class, return its qualified name + replacement_class = tool_class_replacements.get(tool_class) + if replacement_class is not None: + replacement_method = methods_by_tool_class.get(replacement_class) + if replacement_method is not None: + return replacement_method.qualified_name + # otherwise, keep the tool's name + return tool_class.get_name_from_cls() + + tool_name_mapping = {} + for legacy_name, new_name in ToolSet.LEGACY_TOOL_NAME_MAPPING.items(): + tool_name_mapping[legacy_name] = get_name(ToolRegistry().get_tool_class_by_name(new_name)) + for tool_class in ToolRegistry().get_all_tool_classes(): + tool_name_mapping[tool_class.get_name_from_cls()] = get_name(tool_class) + else: + raise ValueError() + + return self.PromptParams( + available_tools=available_tool_names, available_markers=available_tool_marker_names, tool_names_mapping=tool_name_mapping + ) @staticmethod def _format_prompt_tag(text: str, tag: str, tag_name_attr: str | None = None) -> str: @@ -1027,10 +1086,11 @@ class SerenaAgent: return "" template = JinjaTemplate(prompt_template) + prompt_params = self._get_prompt_params() text = template.render( - available_tools=self._exposed_tools.tool_names, - available_markers=self._exposed_tools.tool_marker_names, - tool_names=self._prompt_tool_names_mapping, + available_tools=prompt_params.available_tools, + available_markers=prompt_params.available_markers, + tool_names=prompt_params.tool_names_mapping, embed_memory=embed_memory, ) @@ -1069,11 +1129,8 @@ class SerenaAgent: :param session_id: the client session ID for the case where this is run from a tool; "global" for the connection time case :return: the prompt """ - available_tools = self._active_tools - available_markers = available_tools.tool_marker_names global_memories = self._create_global_memory_manager().list_global_memories() global_memories_str = dict_string(global_memories.to_dict()) if len(global_memories) > 0 else "" - log.info("Generating system prompt with available_tools=(see active tools), available_markers=%s", available_markers) # determine modes for which prompts must (still) be provided, excluding modes that were already provided in a # previously provided project activation message (if any) @@ -1084,13 +1141,14 @@ class SerenaAgent: relevant_modes.append(mode) self._project_prompt_status.mark_mode_prompts_as_provided(session_id) + prompt_params = self._get_prompt_params() system_prompt = self.prompt_factory.create_system_prompt( context_system_prompt=self._render_prompt(self._context.prompt, tag="context"), mode_system_prompts=[self._render_prompt(mode.prompt, tag="mode", tag_name_attr=mode.name) for mode in relevant_modes], - available_tools=available_tools.tool_names, - available_markers=available_markers, + available_tools=prompt_params.available_tools, + available_markers=prompt_params.available_markers, global_memories_list=global_memories_str, - tool_names=self._prompt_tool_names_mapping, + tool_names=prompt_params.tool_names_mapping, ) # provide the project activation message if it hasn't yet been provided @@ -1195,9 +1253,9 @@ class SerenaAgent: self._active_tools = tool_set.to_available_tools(self._all_tools) log.info(f"Active tools ({len(self._active_tools)}): {', '.join(self._active_tools.tool_names)}") - # reset the REPL, whose facades/API scope depend on the active modes and project (it is re-created on demand). - # NOTE: This must happen irrespective of the agent interface, since the REPL tool may be active in tool mode as well. + # reset members that depend on the active tools, so that they are re-created on demand with the new active tools self._repl = None + self._prompt_params = None # check if a tool was activated that is not in the exposed tool set and issue a warning if so active_tools_not_exposed = set(self._active_tools.tool_names) - set(self._exposed_tools.tool_names) diff --git a/src/serena/repl/facade.py b/src/serena/repl/facade.py index 16a7fa4c..987b7598 100644 --- a/src/serena/repl/facade.py +++ b/src/serena/repl/facade.py @@ -7,7 +7,7 @@ The facade, i.e. the object through which REPL code accesses a group of related import inspect import logging from abc import ABC -from collections.abc import Callable, Iterable +from collections.abc import Callable from dataclasses import dataclass from typing import TYPE_CHECKING, Any, TypeVar @@ -143,29 +143,43 @@ class FacadeMethod: enabled or disabled; only enabled methods are accessible from REPL code. """ - def __init__(self, name: str, implementation: Callable[..., Any], info: FacadeMethodInfo, enabled: bool) -> None: + def __init__(self, parent: "Facade", implementation: Callable[..., Any], info: FacadeMethodInfo, enabled: bool) -> None: """ - :param name: the method's name + :param parent: the facade the method belongs to :param implementation: the implementation to delegate to - :param info: the method's metadata + :param info: the method's metadata (including the method's name) :param enabled: whether the method is initially enabled """ - self.name = name + self.parent = parent self._implementation = implementation self.info = info self.enabled = enabled + @property + def name(self) -> str: + return self.info.name + + @property + def facade_name(self) -> str: + return self.parent.name + + @property + def qualified_name(self) -> str: + """ + :return: the name under which the method is accessible from REPL code (facade name and method name) + """ + return f"{self.facade_name}.{self.name}" + def __call__(self, *args: Any, **kwargs: Any) -> Any: return self._implementation(*args, **kwargs) - def describe(self, facade_name: str) -> str: + def describe(self) -> str: """ - :param facade_name: the name of the facade the method belongs to :return: the method's signature and documentation """ signature = inspect.signature(self._implementation) doc = inspect.getdoc(self._implementation) or "(no documentation)" - return f"{facade_name}.{self.name}{signature}\n{doc}\n" + return f"{self.qualified_name}{signature}\n{doc}\n" class ApiScope: @@ -268,11 +282,15 @@ class Facade: A named group of related operations which an LLM can invoke from REPL code. """ - def __init__(self, name: str, description: str, methods: Iterable[FacadeMethod]) -> None: + def __init__(self, name: str, description: str) -> None: # NOTE: attributes are set via object.__setattr__ because __getattr__ is overridden object.__setattr__(self, "_name", name) object.__setattr__(self, "_description", description) - object.__setattr__(self, "_methods", {m.name: m for m in methods}) + object.__setattr__(self, "_methods", {}) + + def _add_method(self, method: FacadeMethod) -> None: + assert method.parent is self + self._methods[method.name] = method @staticmethod def from_api(api: FacadeApi, api_scope: ApiScope) -> "Facade": @@ -283,15 +301,14 @@ class Facade: :param api_scope: API scope definition determining which methods are enabled :return: the facade """ - facade_name = api.get_name_() - methods = [] + facade = Facade(api.get_name_(), api.get_description_()) for name, member in inspect.getmembers(api, predicate=inspect.ismethod): method_info = get_facade_method_info(member) if method_info is None: continue - is_enabled = api_scope.is_method_enabled(facade_name, method_info) - methods.append(FacadeMethod(name, member, method_info, enabled=is_enabled)) - return Facade(api.get_name_(), api.get_description_(), methods) + is_enabled = api_scope.is_method_enabled(facade.name, method_info) + facade._add_method(FacadeMethod(facade, member, method_info, enabled=is_enabled)) + return facade @property def name(self) -> str: @@ -305,6 +322,12 @@ class Facade: def enabled_method_names(self) -> list[str]: return [m.name for m in self._methods.values() if m.enabled] + def get_enabled_methods(self) -> list[FacadeMethod]: + """ + :return: the list of enabled methods + """ + return [m for m in self._methods.values() if m.enabled] + def get_method(self, method_name: str) -> FacadeMethod: """ :param method_name: the name of the method @@ -338,7 +361,7 @@ class Facade: parts = [f"Facade '{self._name}': {self._description}", ""] for method in self._methods.values(): if method.enabled: - parts.append(method.describe(self._name)) + parts.append(method.describe()) return "\n".join(parts) def describe_method(self, method_name: str) -> str: @@ -349,4 +372,4 @@ class Facade: method = self._get_enabled_method(method_name) if method is None: raise ValueError(self._no_such_method_message(method_name)) - return method.describe(self._name) + return method.describe() diff --git a/src/serena/repl/repl.py b/src/serena/repl/repl.py index a8cd8d34..79139388 100644 --- a/src/serena/repl/repl.py +++ b/src/serena/repl/repl.py @@ -9,7 +9,7 @@ import textwrap import traceback from typing import Any -from .facade import ApiScope, Facade +from .facade import ApiScope, Facade, FacadeMethod from .representable import Representable log = logging.getLogger(__name__) @@ -34,6 +34,12 @@ class SerenaReplEntrypoint: registered_facade_names.append(facade.name) log.info("Registered %d/%d facades: %s", len(registered_facade_names), len(facades), registered_facade_names) + def get_enabled_methods(self) -> list[FacadeMethod]: + """ + :return: the list of all enabled methods across all facades + """ + return [method for facade in self._facades.values() for method in facade.get_enabled_methods()] + def _register(self, facade: Facade) -> None: if facade.name in self._facades: raise ValueError(f"Duplicate facade name: {facade.name}") diff --git a/src/serena/resources/config/modes/editing.yml b/src/serena/resources/config/modes/editing.yml index f877fc51..1602ca51 100644 --- a/src/serena/resources/config/modes/editing.yml +++ b/src/serena/resources/config/modes/editing.yml @@ -4,7 +4,7 @@ prompt: | **Refactoring tools** For operations on existing symbols, prefer the dedicated refactoring tools over hand-edits: - `{{ tool_names['rename_symbol'] }}` and `{{ tool_names['safe_delete_symbol'] }}`{% if 'jet_brains_move' in available_tools %}, plus `jet_brains_move` and `jet_brains_inline_symbol`{% endif %} — + `{{ tool_names['rename_symbol'] }}` and `{{ tool_names['safe_delete_symbol'] }}`{% if 'jet_brains_move' in available_tools %}, plus `{{ tool_names['jet_brains_move'] }}` and `{{ tool_names['jet_brains_inline_symbol'] }}`{% endif %} — they are reference-aware and update or check all usages atomically. When such a tool returns success, the refactoring is already complete and consistent across all declarations, references, overrides and imports — trust it: do not re-read the changed files or re-run the build / test suite just to confirm the refactor @@ -16,9 +16,9 @@ prompt: | **Symbolic editing** Use symbolic retrieval tools to identify the symbols you need to edit. - If you need to replace the definition of a symbol, use the `replace_symbol_body` tool. - If you want to add some new code at the end of the file, use the `insert_after_symbol` tool with the last top-level symbol in the file. - Similarly, you can use `insert_before_symbol` with the first top-level symbol in the file to insert code at the beginning of a file. + If you need to replace the definition of a symbol, use the `{{ tool_names['replace_symbol_body'] }}` tool. + If you want to add some new code at the end of the file, use the `{{ tool_names['insert_after_symbol'] }}` tool with the last top-level symbol in the file. + Similarly, you can use `{{ tool_names['insert_before_symbol'] }}` with the first top-level symbol in the file to insert code at the beginning of a file. You can understand relationships between symbols by using the `{{ tool_names['find_referencing_symbols'] }}` tool. If not explicitly requested otherwise by the user, you make sure that when you edit a symbol, the change is either backward-compatible or you find and update all references as needed. The `{{ tool_names['find_referencing_symbols'] }}` tool will give you code snippets around the references as well as symbolic information. @@ -26,14 +26,14 @@ prompt: | {% if 'replace_content' in available_tools %} **File-based editing** - The `replace_content` tool allows you to perform regex-based replacements within files (as well as simple string replacements). + The `{{ tool_names['replace_content'] }}` tool allows you to perform regex-based replacements within files (as well as simple string replacements). This is your primary tool for editing code whenever replacing or deleting a whole symbol would be a more expensive operation, e.g. if you need to adjust just a few lines of code within a method. In `regex` mode, wildcards like `start.*?end` let you match a span without quoting its full text; an ambiguous match returns an error you can refine, so a tight wildcard pattern is both cheaper and safe. - For several small edits within one file, prefer a batch of targeted `replace_content` calls over rewriting the whole + For several small edits within one file, prefer a batch of targeted `{{ tool_names['replace_content'] }}` calls over rewriting the whole file: rewriting has to re-emit the file's entire contents, whereas each targeted edit emits only the changed text. - {% if 'replace_in_files' in available_tools %}For the SAME edit across many files, use `replace_in_files`, which applies + {% if 'replace_in_files' in available_tools %}For the SAME edit across many files, use `{{ tool_names['replace_in_files'] }}`, which applies it everywhere in one call — equally safe and transparent: its `dry_run` mode first previews every change as a diff with a per-occurrence id, so you can then apply all of them or just a chosen subset.{% endif %} {% endif %} diff --git a/src/serena/resources/config/prompt_templates/system_prompt.yml b/src/serena/resources/config/prompt_templates/system_prompt.yml index 99604bf1..95d5bae9 100644 --- a/src/serena/resources/config/prompt_templates/system_prompt.yml +++ b/src/serena/resources/config/prompt_templates/system_prompt.yml @@ -5,29 +5,28 @@ prompts: connection_prompt: | CRITICAL: Before starting to work on a coding task, call the `initial_instructions` tool to read the 'Serena Instructions Manual'. system_prompt: | - You have semantic coding tools that you rely on heavily. Work resource-efficiently: don't read or + You have semantic coding functions/tools that you rely on heavily. Work resource-efficiently: don't read or generate content the task doesn't need. Some tasks require understanding a large part of the codebase; others need only a few symbols or a single file. Avoid reading whole files unless necessary — acquire information step by step, using - the symbolic tools to get an overview of symbols and their relations, then reading only the bodies + the symbolic functions to get an overview of symbols and their relations, then reading only the bodies you need. {% if 'ToolMarkerSymbolicRead' in available_markers %}Once you have read a full file, there is no point re-analysing it with the symbolic read tools — you already have it.{% endif %} {% if 'search_for_pattern' in available_tools %} - If you are unsure about a symbol's name or location{% if 'find_symbol' in available_tools %} (to the extent that substring_matching for the symbol name is not enough){% endif %}, you can use the `search_for_pattern` tool, which allows fast - and flexible search for patterns in the codebase.{% if 'ToolMarkerSymbolicRead' in available_markers %} In this way, you can first find candidates for symbols or files, - and then proceed with the symbolic tools.{% endif %} + For searches beyond named symbols, you can use `{{ tool_names['search_for_pattern'] }}` and flexible search for patterns in the codebase.{% if 'ToolMarkerSymbolicRead' in available_markers %} In this way, you can first find candidates for symbols or files to explore, + and then proceed with symbolic operations.{% endif %} {% endif %} {% if 'ToolMarkerSymbolicRead' in available_markers %} Symbols are identified by their `name_path` and `relative_path`. - You can get an overview of the symbols in a file by using the `{{ tool_names['get_symbols_overview'] }}` tool, or search for a specific symbol with `{{ tool_names['find_symbol'] }}`. + You can get an overview of the symbols in a file by using `{{ tool_names['get_symbols_overview'] }}`, or search for a specific symbol with `{{ tool_names['find_symbol'] }}`. You only read the bodies of symbols when you need to (e.g. if you want to fully understand or edit it). For example, if you are working with Python code and already know that you need to read the body of the constructor of the class Foo, you can directly use `{{ tool_names['find_symbol'] }}` with name path pattern `Foo/__init__` and `include_body=True`. If you don't know yet which methods in `Foo` you need to read or edit, - you can use `{{ tool_names['find_symbol'] }}` with name path pattern `Foo`, `include_body=False` and `depth=1` to get all (top-level) methods of `Foo` before proceeding + you can use `{{ tool_names['find_symbol'] }}` with name path pattern `Foo`, `include_body=False` and `depth=1` to get all (top-level) members of `Foo` before proceeding to read the desired methods with `include_body=True`. - You can understand relationships between symbols by using the `{{ tool_names['find_referencing_symbols'] }}` tool. + You can understand relationships between symbols by using `{{ tool_names['find_referencing_symbols'] }}`. {% endif %} {% if 'read_memory' in available_tools -%} diff --git a/test/serena/test_serena_agent.py b/test/serena/test_serena_agent.py index 8ee9463a..58e9e675 100644 --- a/test/serena/test_serena_agent.py +++ b/test/serena/test_serena_agent.py @@ -918,6 +918,11 @@ class TestSerenaAgent: expected = {"serena_repl", "initial_instructions"} | (set() if context.single_project else {"activate_project"}) assert exposed == expected assert "s.lsp" in agent.get_tool(SerenaReplTool).apply("s.info()") + + # prompts refer to operations by their qualified REPL names, e.g. `lsp.find_symbol` instead of the tool name + system_prompt = agent.create_system_prompt() + assert "`lsp.find_symbol`" in system_prompt + assert "`find_symbol`" not in system_prompt finally: agent.on_shutdown(timeout=5) From 05603b2706aad868a5ef5e2efbbdb79fc4aa144b Mon Sep 17 00:00:00 2001 From: Dominik Jain Date: Tue, 8 Sep 2026 02:07:22 +0200 Subject: [PATCH 21/53] Add configuration facade (cfg) with get_current_config and open_dashboard Introduce ConfigApi, exposing Serena's configuration and session state: get_current_config and open_dashboard (both non-editing). The corresponding tools delegate to the API via ConfigApiMixin. As on the tool side, opening the dashboard is only offered if the dashboard is enabled and not opened automatically; the condition is shared (_is_dashboard_openable) and applied to the API scope via a NamedApiInclusionDefinition, the API-side counterpart of NamedToolInclusionDefinition. --- .serena/memories/repl.md | 6 ++-- src/serena/agent.py | 16 +++++++++- src/serena/config/serena_config.py | 8 +++++ src/serena/repl/api/cfg_api.py | 40 ++++++++++++++++++++++++ src/serena/tools/config_tools.py | 29 ++++++++++++----- test/serena/config/test_serena_config.py | 5 ++- test/serena/test_cfg_api.py | 25 +++++++++++++++ 7 files changed, 118 insertions(+), 11 deletions(-) create mode 100644 src/serena/repl/api/cfg_api.py create mode 100644 test/serena/test_cfg_api.py diff --git a/.serena/memories/repl.md b/.serena/memories/repl.md index ee88a232..8172b170 100644 --- a/.serena/memories/repl.md +++ b/.serena/memories/repl.md @@ -53,5 +53,7 @@ Code runs as a function body (`return` defines the result); a single expression - Python code can always modify the system; the REPL tool is inherently fully privileged, regardless of facade scope or the project's `read_only` setting (which only makes Serena's own API refuse edits). A "read-only REPL" is not feasible and must not be promised. -- Session/project management (activate_project, initial_instructions, dashboard, ...) stays tool-only; - facades expose operations on the active project. +- Project activation (activate_project) and initial_instructions stay tool-only (activation rebuilds the REPL); + Serena's configuration/session state (config overview, dashboard; later e.g. modes) lives in the `cfg` facade. + Computed conditions (read-only project, dashboard not openable) are applied to the API scope in + `SerenaAgent.get_repl` via `exclude_editing()`/`NamedApiInclusionDefinition`, mirroring the tool side. diff --git a/src/serena/agent.py b/src/serena/agent.py index 3034d77d..eba6ae85 100644 --- a/src/serena/agent.py +++ b/src/serena/agent.py @@ -36,6 +36,7 @@ from serena.config.serena_config import ( ModeSelectionDefinition, ModeSelectionDefinitionWithAddedModes, ModeSelectionDefinitionWithBaseModes, + NamedApiInclusionDefinition, NamedToolInclusionDefinition, RegisteredProject, SerenaConfig, @@ -48,6 +49,7 @@ from serena.ls_manager import LanguageServerManager from serena.memories.memory_manager import MemoryManager from serena.project import Project from serena.prompt_factory import SerenaPromptFactory +from serena.repl.api.cfg_api import ConfigApi from serena.repl.api.edit_api import EditApi from serena.repl.api.fs_api import FsApi from serena.repl.api.jb_api import JetBrainsApi @@ -760,6 +762,15 @@ class SerenaAgent: except Exception as e: log.debug(f"Failed to send usage info: {e}") + @staticmethod + def _is_dashboard_openable(serena_config: SerenaConfig) -> bool: + """ + :param serena_config: the configuration + :return: whether the web dashboard is available and opening it is a meaningful operation + (i.e. it is enabled and not opened automatically) + """ + return serena_config.web_dashboard and not serena_config.web_dashboard_open_on_launch and not serena_config.gui_log_window + @classmethod def _create_base_toolset( cls, @@ -789,7 +800,7 @@ class SerenaAgent: # determine whether to include the OpenDashboardTool based on the Serena configuration tool_inclusion_definitions: list[ToolInclusionDefinition] = [] - if serena_config.web_dashboard and not serena_config.web_dashboard_open_on_launch and not serena_config.gui_log_window: + if cls._is_dashboard_openable(serena_config): tool_inclusion_definitions.append( NamedToolInclusionDefinition(name="OpenDashboard", included_optional_tools=[OpenDashboardTool.get_name_from_cls()]) ) @@ -1281,9 +1292,12 @@ class SerenaAgent: api_scope.process(self._active_project.project_config) if self._active_project.project_config.read_only: api_scope.exclude_editing() + if not self._is_dashboard_openable(self.serena_config): + api_scope.process(NamedApiInclusionDefinition(name="Dashboard", excluded_apis=["cfg.open_dashboard"])) # gather facades facades = [ + Facade.from_api(ConfigApi(self), api_scope), Facade.from_api(FsApi(self), api_scope), Facade.from_api(EditApi(self), api_scope), Facade.from_api(MemoryApi(self), api_scope), diff --git a/src/serena/config/serena_config.py b/src/serena/config/serena_config.py index 3f2d5aab..c7809f57 100644 --- a/src/serena/config/serena_config.py +++ b/src/serena/config/serena_config.py @@ -190,6 +190,14 @@ class ApiInclusionDefinition: excluded_apis: Sequence[str] = () +@dataclass +class NamedApiInclusionDefinition(ApiInclusionDefinition): + name: str | None = None + + def __str__(self) -> str: + return f"ApiInclusionDefinition[{self.name}]" + + @dataclass class ModeSelectionDefinition: default_modes: Sequence[str] | None = None diff --git a/src/serena/repl/api/cfg_api.py b/src/serena/repl/api/cfg_api.py new file mode 100644 index 00000000..61053727 --- /dev/null +++ b/src/serena/repl/api/cfg_api.py @@ -0,0 +1,40 @@ +# SPDX-License-Identifier: GPL-3.0-or-later +""" +The implementation of operations concerning Serena's configuration and session state. +""" + +from typing import TYPE_CHECKING + +from serena.tools import GetCurrentConfigTool, OpenDashboardTool + +from ..facade import FacadeApi, facade_method + +if TYPE_CHECKING: + from serena.agent import SerenaAgent + + +class ConfigApi(FacadeApi): + def __init__(self, agent: "SerenaAgent") -> None: + super().__init__(agent, name="cfg", description="Serena's configuration and session state (incl. the dashboard)") + + @facade_method(corresponding_tool=GetCurrentConfigTool) + def get_current_config(self) -> str: + """ + Provides the current configuration of the agent, including the active and available projects, tools, contexts, and modes. + + :return: the configuration overview + """ + return self._agent.get_current_config_overview() + + @facade_method(corresponding_tool=OpenDashboardTool) + def open_dashboard(self) -> str: + """ + Opens the Serena web dashboard in the default web browser. + The dashboard provides logs, session information, and tool usage statistics. + + :return: a message indicating whether the dashboard could be opened + """ + if self._agent.open_dashboard(): + return f"Serena web dashboard has been opened in the user's default web browser: {self._agent.get_dashboard_url()}" + else: + return f"Serena web dashboard could not be opened automatically; tell the user to open it via {self._agent.get_dashboard_url()}" diff --git a/src/serena/tools/config_tools.py b/src/serena/tools/config_tools.py index aab6b13e..5d8769bc 100644 --- a/src/serena/tools/config_tools.py +++ b/src/serena/tools/config_tools.py @@ -1,11 +1,29 @@ # SPDX-License-Identifier: GPL-3.0-or-later +from typing import TYPE_CHECKING, cast + from sensai.util.helper import mark_used from serena.tools import Tool, ToolMarkerDoesNotRequireActiveProject, ToolMarkerOptional +if TYPE_CHECKING: + from serena.repl.api.cfg_api import ConfigApi -class OpenDashboardTool(Tool, ToolMarkerOptional, ToolMarkerDoesNotRequireActiveProject): + +class ConfigApiMixin: + """ + Mixin for tools which delegate to the configuration API. + The API is imported locally, since the API module refers to the tools (as corresponding tools). + """ + + def _api(self) -> "ConfigApi": + from serena.repl.api.cfg_api import ConfigApi + + tool = cast(Tool, cast(object, self)) + return ConfigApi(tool.agent) + + +class OpenDashboardTool(Tool, ToolMarkerOptional, ToolMarkerDoesNotRequireActiveProject, ConfigApiMixin): """ Opens the Serena web dashboard in the default web browser. The dashboard provides logs, session information, and tool usage statistics. @@ -15,10 +33,7 @@ class OpenDashboardTool(Tool, ToolMarkerOptional, ToolMarkerDoesNotRequireActive """ Opens the Serena web dashboard in the default web browser. """ - if self.agent.open_dashboard(): - return f"Serena web dashboard has been opened in the user's default web browser: {self.agent.get_dashboard_url()}" - else: - return f"Serena web dashboard could not be opened automatically; tell the user to open it via {self.agent.get_dashboard_url()}" + return self._api().open_dashboard() class ActivateProjectTool(Tool, ToolMarkerDoesNotRequireActiveProject): @@ -56,7 +71,7 @@ class RemoveProjectTool(Tool, ToolMarkerDoesNotRequireActiveProject, ToolMarkerO return f"Successfully removed project '{project_name}' from configuration." -class GetCurrentConfigTool(Tool): +class GetCurrentConfigTool(Tool, ConfigApiMixin): """ Prints the current configuration of the agent, including the active and available projects, tools, contexts, and modes. """ @@ -65,4 +80,4 @@ class GetCurrentConfigTool(Tool): """ Print the current configuration of the agent, including the active and available projects, tools, contexts, and modes. """ - return self.agent.get_current_config_overview() + return self._api().get_current_config() diff --git a/test/serena/config/test_serena_config.py b/test/serena/config/test_serena_config.py index cdfa91c5..33fdb60a 100644 --- a/test/serena/config/test_serena_config.py +++ b/test/serena/config/test_serena_config.py @@ -268,7 +268,10 @@ class TestAgentInterface: fixed_toolset = {"serena_repl", "initial_instructions", "activate_project"} assert {t.get_name() for t in agent.get_exposed_tool_instances()} == fixed_toolset assert set(agent.get_active_tool_names()) == fixed_toolset - assert "s.mem" in agent.get_repl().entrypoint.overview() + overview = agent.get_repl().entrypoint.overview() + assert "s.mem" in overview + # the dashboard is disabled in the test configuration, so opening it is not offered + assert "s.cfg" in overview and "open_dashboard" not in overview # after activation: the toolset is unchanged, the REPL reflects the project's API exclusions agent.activate_project_from_path_or_name(name) diff --git a/test/serena/test_cfg_api.py b/test/serena/test_cfg_api.py new file mode 100644 index 00000000..46ea3cbd --- /dev/null +++ b/test/serena/test_cfg_api.py @@ -0,0 +1,25 @@ +""" +Tests for the configuration facade API. +""" + +from unittest.mock import MagicMock + +from serena.repl.api.cfg_api import ConfigApi +from serena.repl.facade import ApiScope, Facade + + +def test_facade_exposes_config_operations() -> None: + facade = Facade.from_api(ConfigApi(MagicMock()), ApiScope()) + assert facade.name == "cfg" + assert set(facade.enabled_method_names) == {"get_current_config", "open_dashboard"} + assert not any(facade.get_method(name).info.can_edit for name in facade.enabled_method_names) + + +def test_operations_delegate_to_agent() -> None: + agent = MagicMock() + agent.get_current_config_overview.return_value = "overview" + agent.open_dashboard.return_value = False + agent.get_dashboard_url.return_value = "http://localhost:1" + api = ConfigApi(agent) + assert api.get_current_config() == "overview" + assert "http://localhost:1" in api.open_dashboard() From 6cb3d37e094fea6b89c98d38710ee71f20fb12f6 Mon Sep 17 00:00:00 2001 From: Dominik Jain Date: Tue, 8 Sep 2026 02:32:39 +0200 Subject: [PATCH 22/53] Add referenced types to facades, making result types discoverable in the REPL Facades declare the types their methods return or contain as ReferencedTypes (with an optional whitelist of members for curation, e.g. for LanguageServerSymbol, and a flag for inclusion in the facade description). Type documentation is generated from class docstrings, attribute annotations, properties and method signatures/docstrings, and is available via s.info(".") or by bare type name; method documentation points to the documentation of a referenced return type. Annotations are rendered without module paths, such that the names in signatures are the names by which types can be looked up. s.info accepts several items at once, reporting unknown items inline. All APIs declare their result types; result classes declare their attributes at class level. --- .serena/memories/repl.md | 9 ++ src/serena/repl/api/edit_api.py | 9 +- src/serena/repl/api/fs_api.py | 23 +++- src/serena/repl/api/jb_api.py | 12 ++- src/serena/repl/api/lsp_api.py | 33 +++++- src/serena/repl/api/mem_api.py | 3 +- src/serena/repl/api/shell_api.py | 4 +- src/serena/repl/facade.py | 177 ++++++++++++++++++++++++++++--- src/serena/repl/repl.py | 33 ++++-- src/serena/repl/representable.py | 3 + test/serena/test_repl_tool.py | 27 ++++- 11 files changed, 299 insertions(+), 34 deletions(-) diff --git a/.serena/memories/repl.md b/.serena/memories/repl.md index 8172b170..8729d804 100644 --- a/.serena/memories/repl.md +++ b/.serena/memories/repl.md @@ -29,6 +29,15 @@ Code runs as a function body (`return` defines the result); a single expression - Results expose data to code (`.symbols`, `.occurrences`, `.lines`, ...) and render like the classic tool output. - Progressive disclosure: a priori only facade names, descriptions and method names; `s.info("")` / `s.info(".")` give signature + docstring together, never a signature alone. + `info(*items)` documents several items at once; unknown items are reported inline. +- Result types: a type returned by a single method is documented under `:return:`. Types that are shared, + contained (`LanguageServerSymbol`) or navigated are declared per facade as `ReferencedType`s (constructor + arg `types=`), with `provide_info_with_facade` (full description in `s.info("")`, else listed by name) + and an optional `members` whitelist (curation for foreign/large classes; listed methods are shown even if + undocumented, convention-derived ones only if documented). Types are documented via `s.info(".")` + or bare `s.info("")`; method docs point to their referenced return type. Result classes declare + attribute annotations at class level (attributes set only in `__init__` are not discoverable). + Annotations are rendered without module paths, so signature names equal lookup names. - APIs must not import `serena.tools` at module level except for tool classes in decorators; tools import APIs locally in `_api()` (API modules refer to tool classes). diff --git a/src/serena/repl/api/edit_api.py b/src/serena/repl/api/edit_api.py index fa1fa186..c7222eb9 100644 --- a/src/serena/repl/api/edit_api.py +++ b/src/serena/repl/api/edit_api.py @@ -18,7 +18,7 @@ from serena.tools import ( ) from serena.util.text_utils import ContentReplacer, MultiFileReplacement, ReplacementOccurrence, ReplacementRejectedError -from ..facade import SUCCESS_RESULT, FacadeApi, facade_method +from ..facade import SUCCESS_RESULT, FacadeApi, ReferencedType, facade_method from ..representable import Renderer, RepresentableViaRenderer if TYPE_CHECKING: @@ -74,6 +74,13 @@ class EditApi(FacadeApi): agent, name="edit", description="modifying content within existing files (independent of the language backend)", + types=[ + ReferencedType(ReplacementPreview, provide_info_with_facade=True), + ReferencedType( + ReplacementOccurrence, + members=["occurrence_id", "relative_path", "start_line", "end_line", "matched_text", "replacement", "is_ambiguous"], + ), + ], ) # file-level operations diff --git a/src/serena/repl/api/fs_api.py b/src/serena/repl/api/fs_api.py index 4272a4ba..5461d2ac 100644 --- a/src/serena/repl/api/fs_api.py +++ b/src/serena/repl/api/fs_api.py @@ -14,7 +14,7 @@ from serena.util.file_system import scan_directory from serena.util.text_utils import MatchedConsecutiveLines from solidlsp.ls_utils import TextUtils -from ..facade import FacadeApi, facade_method +from ..facade import FacadeApi, ReferencedType, facade_method from ..representable import Renderer, RepresentableViaRenderer if TYPE_CHECKING: @@ -34,6 +34,8 @@ class FileContent(RepresentableViaRenderer): super().__init__(renderer) self.lines = lines + lines: list[str] + @property def text(self) -> str: return "\n".join(self.lines) @@ -59,6 +61,9 @@ class DirectoryListing(RepresentableViaRenderer): self.dirs = dirs self.files = files + dirs: list[str] + files: list[str] + class DirectoryListingRenderer(Renderer[DirectoryListing]): def render(self, obj: DirectoryListing) -> str: @@ -79,6 +84,8 @@ class PatternMatches(RepresentableViaRenderer): super().__init__(renderer) self.matches = matches + matches: list[MatchedConsecutiveLines] + def __len__(self) -> int: return len(self.matches) @@ -162,7 +169,19 @@ class PatternMatchesRenderer(Renderer[PatternMatches]): class FsApi(FacadeApi): def __init__(self, agent: "SerenaAgent") -> None: - super().__init__(agent, name="fs", description="the project's files as units (as opposed to their content, see `edit`)") + super().__init__( + agent, + name="fs", + description="the project's files as units (as opposed to their content, see `edit`)", + types=[ + ReferencedType(FileContent), + ReferencedType(DirectoryListing), + ReferencedType(PatternMatches, provide_info_with_facade=True), + ReferencedType( + MatchedConsecutiveLines, members=["source_file_path", "matched_lines", "start_line", "end_line", "to_display_string"] + ), + ], + ) @facade_method(corresponding_tool=ReadFileTool) def read_file(self, relative_path: str, start_line: int = 0, end_line: int | None = None, max_answer_chars: int = -1) -> FileContent: diff --git a/src/serena/repl/api/jb_api.py b/src/serena/repl/api/jb_api.py index e8ffdb62..66854b61 100644 --- a/src/serena/repl/api/jb_api.py +++ b/src/serena/repl/api/jb_api.py @@ -30,7 +30,7 @@ from serena.tools import ( ) from serena.util.text_utils import find_text_coordinates -from ..facade import FacadeApi, facade_method +from ..facade import FacadeApi, ReferencedType, facade_method from ..representable import JsonObject, JsonObjectRenderer, Renderer, RepresentableViaRenderer if TYPE_CHECKING: @@ -44,6 +44,8 @@ class JetBrainsSymbolCollection(RepresentableViaRenderer): `children`, `body`, `quick_info`, `documentation` and (for references) `context`. """ + symbols: list[SymbolDTO] + def __init__(self, symbols: list[SymbolDTO], renderer: "JetBrainsSymbolCollectionRenderer"): """ :param symbols: the symbols @@ -123,6 +125,9 @@ class JetBrainsSymbolsOverview(RepresentableViaRenderer): self.symbols = symbols self.documentation = documentation + symbols: list[SymbolDTO] + documentation: str | None + class JetBrainsSymbolsOverviewRenderer(Renderer[JetBrainsSymbolsOverview]): """ @@ -180,6 +185,11 @@ class JetBrainsApi(FacadeApi): agent, name="jb", description="operations on the codebase backed by the JetBrains IDE's code intelligence", + types=[ + ReferencedType(JetBrainsSymbolCollection, provide_info_with_facade=True), + ReferencedType(JetBrainsSymbolsOverview), + ReferencedType(JsonObject), + ], ) @contextmanager diff --git a/src/serena/repl/api/lsp_api.py b/src/serena/repl/api/lsp_api.py index 28015197..449fc603 100644 --- a/src/serena/repl/api/lsp_api.py +++ b/src/serena/repl/api/lsp_api.py @@ -33,7 +33,7 @@ from serena.tools import ( from serena.util.text_utils import TextOutputUtils, find_text_coordinates from solidlsp.lsp_protocol_handler.lsp_types import SymbolKind -from ..facade import SUCCESS_RESULT, FacadeApi, facade_method +from ..facade import SUCCESS_RESULT, FacadeApi, ReferencedType, facade_method from ..representable import Renderer, RepresentableViaRenderer if TYPE_CHECKING: @@ -47,6 +47,8 @@ class LspSymbolCollection(RepresentableViaRenderer): `body`, `get_body_line_numbers()`, `iter_children()`. """ + symbols: list[LanguageServerSymbol] + def __init__(self, symbols: list[LanguageServerSymbol], renderer: "LspSymbolCollectionRenderer"): """ :param symbols: the list of symbols @@ -70,6 +72,8 @@ class LspSymbol(RepresentableViaRenderer): A single symbol retrieved via the language server (see `LspSymbolCollection` for the symbol's interface). """ + symbol: LanguageServerSymbol + def __init__(self, symbol: LanguageServerSymbol, renderer: "LspSymbolRenderer"): """ :param symbol: the symbol @@ -207,6 +211,8 @@ class LspReferenceCollection(RepresentableViaRenderer): (a `LanguageServerSymbol`) and the `line` of the reference. """ + references: list[ReferenceInLanguageServerSymbol] + def __init__(self, references: list[ReferenceInLanguageServerSymbol], renderer: "LspReferenceCollectionRenderer"): """ :param references: the references @@ -273,6 +279,8 @@ class LspDiagnostics(RepresentableViaRenderer): Diagnostics grouped as `relative_path -> severity -> name_path -> diagnostics`; see `grouped.get_dict()`. """ + grouped: GroupedDiagnostics + def __init__(self, grouped: GroupedDiagnostics, renderer: "LspDiagnosticsRenderer"): """ :param grouped: the grouped diagnostics @@ -304,6 +312,29 @@ class LspApi(FacadeApi): agent, name="lsp", description="symbol-level operations on the codebase backed by language servers", + types=[ + ReferencedType(LspSymbolCollection, provide_info_with_facade=True), + ReferencedType( + LanguageServerSymbol, + members=[ + "name", + "get_name_path", + "relative_path", + "symbol_kind_name", + "line", + "column", + "body", + "get_body_line_numbers", + "iter_children", + "iter_ancestors", + "get_parent", + ], + ), + ReferencedType(LspSymbol), + ReferencedType(LspReferenceCollection), + ReferencedType(ReferenceInLanguageServerSymbol), + ReferencedType(LspDiagnostics), + ], ) def _create_symbol_retriever(self) -> LanguageServerSymbolRetriever: diff --git a/src/serena/repl/api/mem_api.py b/src/serena/repl/api/mem_api.py index d6e41f76..36bb2fdc 100644 --- a/src/serena/repl/api/mem_api.py +++ b/src/serena/repl/api/mem_api.py @@ -18,7 +18,7 @@ from serena.tools import ( WriteMemoryTool, ) -from ..facade import FacadeApi, facade_method +from ..facade import FacadeApi, ReferencedType, facade_method from ..representable import Renderer, RepresentableViaRenderer if TYPE_CHECKING: @@ -60,6 +60,7 @@ class MemoryApi(FacadeApi): agent, name="mem", description="project memories, i.e. persistent notes for future tasks", + types=[ReferencedType(MemoryList)], ) def _get_memory_manager(self) -> MemoryManager: diff --git a/src/serena/repl/api/shell_api.py b/src/serena/repl/api/shell_api.py index 56ba2679..801b8e93 100644 --- a/src/serena/repl/api/shell_api.py +++ b/src/serena/repl/api/shell_api.py @@ -9,7 +9,7 @@ from typing import TYPE_CHECKING from serena.tools import ExecuteShellCommandTool from serena.util.shell import ShellCommandResult, execute_shell_command -from ..facade import FacadeApi, facade_method +from ..facade import FacadeApi, ReferencedType, facade_method from ..representable import Renderer, RepresentableViaRenderer if TYPE_CHECKING: @@ -53,7 +53,7 @@ class ShellCommandOutputRenderer(Renderer[ShellCommandOutput]): class ShellApi(FacadeApi): def __init__(self, agent: "SerenaAgent") -> None: - super().__init__(agent, name="shell", description="execution of shell commands") + super().__init__(agent, name="shell", description="execution of shell commands", types=[ReferencedType(ShellCommandOutput)]) @facade_method(can_edit=True, corresponding_tool=ExecuteShellCommandTool) def execute_shell_command( diff --git a/src/serena/repl/facade.py b/src/serena/repl/facade.py index 987b7598..6e7d1857 100644 --- a/src/serena/repl/facade.py +++ b/src/serena/repl/facade.py @@ -6,14 +6,18 @@ The facade, i.e. the object through which REPL code accesses a group of related import inspect import logging +import re +import typing from abc import ABC -from collections.abc import Callable +from collections.abc import Callable, Sequence from dataclasses import dataclass from typing import TYPE_CHECKING, Any, TypeVar from serena.config.serena_config import ApiInclusionDefinition from serena.project import Project +from .representable import RepresentableViaRenderer + if TYPE_CHECKING: from serena.agent import SerenaAgent from serena.code_editor import CodeEditor @@ -26,6 +30,105 @@ SUCCESS_RESULT = "OK" """the result returned by operations which have no result other than their success""" +def format_annotation(annotation: Any) -> str: + """ + :param annotation: a type annotation (or a signature/annotation string) + :return: the annotation rendered without module paths (e.g. `list[LanguageServerSymbol]`), such that type names + match the names by which the types can be looked up + """ + text = annotation if isinstance(annotation, str) else inspect.formatannotation(annotation) + return re.sub(r"\b(?:[A-Za-z_]\w*\.)+([A-Za-z_]\w*)", r"\1", text) + + +def format_signature(callable_: Callable[..., Any]) -> str: + """ + :param callable_: the callable + :return: the signature rendered without module paths in annotations + """ + return format_annotation(str(inspect.signature(callable_))) + + +@dataclass +class ReferencedType: + """ + A type that is referenced by a facade's methods (returned by them or contained in their results), whose interface + the LLM can inspect via `info`. + """ + + cls: type + """the type""" + provide_info_with_facade: bool = False + """whether the type's full description is included in the facade's description (rather than just its name)""" + members: Sequence[str] | None = None + """ + the LLM-facing members (attributes, properties, methods) to describe; if None, all members admitted by the naming + convention (no leading or trailing underscore) which are documented are described. Explicitly listed methods are + described even if undocumented, as the listing is the documentation decision. + """ + + _CAPABILITIES: typing.ClassVar[dict[str, str]] = {"__len__": "len()", "__iter__": "iteration", "__getitem__": "indexing"} + + @property + def name(self) -> str: + return self.cls.__name__ + + def _get_member_names(self) -> list[str]: + if self.members is not None: + return list(self.members) + names = set(typing.get_type_hints(self.cls)) + names.update(n for n in dir(self.cls) if not n.startswith("_")) + # exclude the representation mechanism, which is not meant to be used from REPL code + names.difference_update(dir(RepresentableViaRenderer)) + return sorted(n for n in names if not n.startswith("_") and not n.endswith("_")) + + @staticmethod + def _first_doc_line(obj: Any) -> str: + doc = inspect.getdoc(obj) or "" + first_line = doc.splitlines()[0] if doc else "" + return first_line.removeprefix(":return:").strip() + + def describe(self) -> str: + """ + :return: the type's documentation: its docstring, attributes/properties with their types and methods with their + signatures and documentation + """ + attributes: list[str] = [] + methods: list[str] = [] + type_hints = typing.get_type_hints(self.cls) + for member_name in self._get_member_names(): + member = inspect.getattr_static(self.cls, member_name, None) + if isinstance(member, property): + fget = member.fget + annotation = inspect.signature(fget).return_annotation if fget is not None else inspect.Signature.empty + type_str = f": {format_annotation(annotation)}" if annotation is not inspect.Signature.empty else "" + doc = self._first_doc_line(member) + attributes.append(f" {member_name}{type_str}" + (f" # {doc}" if doc else "")) + elif inspect.isfunction(member): + doc = inspect.getdoc(member) + if doc is None and self.members is None: + continue + signature = format_signature(member).replace("(self, ", "(", 1).replace("(self)", "()", 1) + methods.append(f" {member_name}{signature}" + (f"\n {doc.replace(chr(10), chr(10) + ' ')}" if doc else "")) + elif member_name in type_hints: + attributes.append(f" {member_name}: {format_annotation(type_hints[member_name])}") + else: + attributes.append(f" {member_name}") + + # assemble the description + parts = [f"type {self.name}"] + if self.cls.__doc__: # NOTE: the class' own docstring (inspect.getdoc would fall back to base class docstrings) + doc = inspect.cleandoc(self.cls.__doc__) + parts.append(f" {doc.replace(chr(10), chr(10) + ' ')}") + if attributes: + parts.append("attributes:\n" + "\n".join(attributes)) + if methods: + parts.append("methods:\n" + "\n".join(methods)) + capabilities = [text for dunder, text in self._CAPABILITIES.items() if dunder in dir(self.cls) and dunder not in dir(object)] + if capabilities: + parts.append("supports: " + ", ".join(capabilities)) + return "\n".join(parts) + "\n" + + @dataclass(kw_only=True, frozen=True) class FacadeMethodInfo: """ @@ -101,15 +204,18 @@ class FacadeApi(ABC): * Names with a leading underscore are private, as usual. """ - def __init__(self, agent: "SerenaAgent", name: str, description: str) -> None: + def __init__(self, agent: "SerenaAgent", name: str, description: str, types: Sequence[ReferencedType] = ()) -> None: """ :param agent: the agent providing access to the project and its resources :param name: the attribute name under which the facade is accessible from the REPL entrypoint :param description: a one-line description of the functionality offered by the facade + :param types: the types referenced by the facade's methods (returned or contained in results) whose interface + the LLM shall be able to inspect """ self._agent = agent self._name = name self._description = description + self._types = list(types) def get_name_(self) -> str: return self._name @@ -117,6 +223,9 @@ class FacadeApi(ABC): def get_description_(self) -> str: return self._description + def get_referenced_types_(self) -> list[ReferencedType]: + return self._types + def _get_project(self) -> Project: return self._agent.get_active_project_or_raise() @@ -173,13 +282,22 @@ class FacadeMethod: def __call__(self, *args: Any, **kwargs: Any) -> Any: return self._implementation(*args, **kwargs) - def describe(self) -> str: + def describe(self, include_return_type_pointer: bool = True) -> str: """ + :param include_return_type_pointer: whether to append a pointer to the documentation of the return type, + if it is a type referenced by the facade :return: the method's signature and documentation """ - signature = inspect.signature(self._implementation) + signature = format_signature(self._implementation) doc = inspect.getdoc(self._implementation) or "(no documentation)" - return f"{self.qualified_name}{signature}\n{doc}\n" + text = f"{self.qualified_name}{signature}\n{doc}\n" + if include_return_type_pointer: + return_annotation = format_annotation(inspect.signature(self._implementation).return_annotation) + referenced = [t for t in self.parent.get_types() if re.search(rf"\b{re.escape(t.name)}\b", return_annotation)] + if referenced: + pointers = ", ".join(f'`s.info("{self.facade_name}.{t.name}")`' for t in referenced) + text += f"Return type: see {pointers}\n" + return text class ApiScope: @@ -282,11 +400,12 @@ class Facade: A named group of related operations which an LLM can invoke from REPL code. """ - def __init__(self, name: str, description: str) -> None: + def __init__(self, name: str, description: str, types: Sequence[ReferencedType] = ()) -> None: # NOTE: attributes are set via object.__setattr__ because __getattr__ is overridden object.__setattr__(self, "_name", name) object.__setattr__(self, "_description", description) object.__setattr__(self, "_methods", {}) + object.__setattr__(self, "_types", {t.name: t for t in types}) def _add_method(self, method: FacadeMethod) -> None: assert method.parent is self @@ -301,7 +420,7 @@ class Facade: :param api_scope: API scope definition determining which methods are enabled :return: the facade """ - facade = Facade(api.get_name_(), api.get_description_()) + facade = Facade(api.get_name_(), api.get_description_(), api.get_referenced_types_()) for name, member in inspect.getmembers(api, predicate=inspect.ismethod): method_info = get_facade_method_info(member) if method_info is None: @@ -328,6 +447,19 @@ class Facade: """ return [m for m in self._methods.values() if m.enabled] + def get_types(self) -> list[ReferencedType]: + """ + :return: the types referenced by the facade's methods + """ + return list(self._types.values()) + + def get_type(self, type_name: str) -> ReferencedType | None: + """ + :param type_name: the name of the type + :return: the referenced type, or None if the facade does not reference a type of that name + """ + return self._types.get(type_name) + def get_method(self, method_name: str) -> FacadeMethod: """ :param method_name: the name of the method @@ -357,19 +489,36 @@ class Facade: def describe(self) -> str: """ :return: a description of the facade listing all of its enabled methods with their signatures and documentation + as well as its referenced types (in full if so declared, otherwise by name) """ parts = [f"Facade '{self._name}': {self._description}", ""] for method in self._methods.values(): if method.enabled: parts.append(method.describe()) + described_types = [t for t in self._types.values() if t.provide_info_with_facade] + listed_types = [t for t in self._types.values() if not t.provide_info_with_facade] + for referenced_type in described_types: + parts.append(referenced_type.describe()) + if listed_types: + parts.append( + "Further types: " + + ", ".join(t.name for t in listed_types) + + f' (request documentation via `s.info("{self._name}.")`)' + ) return "\n".join(parts) - def describe_method(self, method_name: str) -> str: + def describe_member(self, member_name: str) -> str: """ - :param method_name: the name of one of the facade's enabled methods - :return: the method's signature and documentation + :param member_name: the name of one of the facade's enabled methods or referenced types + :return: the member's documentation """ - method = self._get_enabled_method(method_name) - if method is None: - raise ValueError(self._no_such_method_message(method_name)) - return method.describe() + method = self._get_enabled_method(member_name) + if method is not None: + return method.describe() + referenced_type = self._types.get(member_name) + if referenced_type is not None: + return referenced_type.describe() + raise ValueError( + f"Facade '{self._name}' has no method or type '{member_name}'. " + f"Available methods: {self.enabled_method_names}; types: {list(self._types)}" + ) diff --git a/src/serena/repl/repl.py b/src/serena/repl/repl.py index 79139388..93f00834 100644 --- a/src/serena/repl/repl.py +++ b/src/serena/repl/repl.py @@ -60,22 +60,35 @@ class SerenaReplEntrypoint: for facade in self._facades.values() ) - def info(self, path: str = "") -> str: + def info(self, *items: str) -> str: """ Provides documentation on the available functionality. - :param path: the empty string for an overview of all facades, a facade name (e.g. "lsp") for the - documentation of all of the facade's methods, or a dotted method path (e.g. "lsp.find_symbol") - for the documentation of a single method + :param items: the items to document; if none are given, an overview of all facades is provided. + Each item is either a facade name (e.g. "lsp") for the documentation of all of the facade's methods and types, + a dotted path (e.g. "lsp.find_symbol" or "lsp.LspSymbolCollection") for the documentation of a single method + or type, or a bare type name (e.g. "LanguageServerSymbol"), which is looked up across all facades. + Unknown items are reported without affecting the documentation of the other items. :return: the requested documentation """ - if path == "": + if not items: return self.overview() - facade_name, _, method_name = path.partition(".") - facade = self._get_facade(facade_name) - if method_name == "": - return facade.describe() - return facade.describe_method(method_name) + return "\n\n".join(self._describe_item(item) for item in items) + + def _describe_item(self, item: str) -> str: + facade_name, _, member_name = item.partition(".") + try: + if member_name: + return self._get_facade(facade_name).describe_member(member_name) + if facade_name in self._facades: + return self._facades[facade_name].describe() + # not a facade: look up the item as a type across all facades + matches = [facade for facade in self._facades.values() if facade.get_type(item) is not None] + if not matches: + raise ValueError(f"Unknown item '{item}': neither a facade nor a type. Available facades: {list(self._facades)}") + return matches[0].describe_member(item) + except ValueError as e: + return str(e) class SerenaRepl: diff --git a/src/serena/repl/representable.py b/src/serena/repl/representable.py index 1b89a40e..9867f726 100644 --- a/src/serena/repl/representable.py +++ b/src/serena/repl/representable.py @@ -92,6 +92,9 @@ class JsonObject(RepresentableViaRenderer): super().__init__(renderer) self.data = data + data: Any + """the JSON-serializable data (dict, list, etc.)""" + class JsonObjectRenderer(Renderer[JsonObject]): def render(self, obj: JsonObject) -> str: diff --git a/test/serena/test_repl_tool.py b/test/serena/test_repl_tool.py index fc04c434..da13d65d 100644 --- a/test/serena/test_repl_tool.py +++ b/test/serena/test_repl_tool.py @@ -52,6 +52,29 @@ class TestReplExecution: method_info = repl.execute('s.info("lsp.find_symbol")') assert "name_path_pattern" in method_info + def test_type_discovery(self, repl: SerenaRepl) -> None: + # signatures render type names without module paths, and point to the documentation of referenced return types + method_info = repl.execute('s.info("lsp.find_symbol")') + assert "-> LspSymbolCollection" in method_info and "lsp_api." not in method_info + assert 's.info("lsp.LspSymbolCollection")' in method_info + + # the facade description includes types declared as important in full and lists the others by name + facade_info = repl.execute('s.info("lsp")') + assert "type LspSymbolCollection" in facade_info and "symbols: list[LanguageServerSymbol]" in facade_info + assert "type LanguageServerSymbol" not in facade_info and "LanguageServerSymbol" in facade_info + + # types can be requested via the facade or by bare name, and their curated members are documented + type_info = repl.execute('s.info("lsp.LanguageServerSymbol")') + assert type_info == repl.execute('s.info("LanguageServerSymbol")') + assert "get_name_path() -> str" in type_info and "iter_children()" in type_info + assert "to_dict" not in type_info # not among the curated members + assert "represent" not in repl.execute('s.info("LspSymbolCollection")') # the representation mechanism is not exposed + + def test_info_documents_several_items(self, repl: SerenaRepl) -> None: + info = repl.execute('s.info("lsp.find_symbol", "nope", "lsp.LspSymbolCollection")') + assert "lsp.find_symbol(" in info and "type LspSymbolCollection" in info + assert "Unknown item 'nope'" in info # an unknown item does not prevent the documentation of the others + class TestFacade: """Tests the indirection between facades and their implementations.""" @@ -131,7 +154,7 @@ class TestFacade: facade = Facade.from_api(self.DummyApi(MagicMock()), ApiScope()) assert facade.add(1, 2) == 3 assert "dummy.add(a: int, b: int) -> int" in facade.describe() - assert "Adds two numbers." in facade.describe_method("add") + assert "Adds two numbers." in facade.describe_member("add") def test_disabled_methods_are_inaccessible_and_undocumented(self) -> None: facade = Facade.from_api(self.DummyApi(MagicMock()), self._scope(excluded_apis=["dummy.secret"])) @@ -139,7 +162,7 @@ class TestFacade: with pytest.raises(AttributeError): facade.secret() with pytest.raises(ValueError): - facade.describe_method("secret") + facade.describe_member("secret") assert "secret" not in facade.describe() assert "_internal" not in facade.describe() From b483bf0159c42335b86ecd2e95712570b4f0ef49 Mon Sep 17 00:00:00 2001 From: Dominik Jain Date: Tue, 8 Sep 2026 13:32:56 +0200 Subject: [PATCH 23/53] Structure REPL documentation into tiers; keep facade descriptions compact The REPL overview (tier 0, part of the tool description) now names the result type of methods returning objects that can be processed in code (e.g. `find_symbol -> LspSymbolCollection`), such that type documentation can be requested alongside a facade in one turn. The facade description (tier 1) documents the common operations in full, summarises methods marked as niche (new flag of facade_method; not yet set anywhere) with a pointer to their documentation, and lists result types by name only. Type documentation (tier 2) is provided on request only; provide_info_with_facade remains available but is no longer set for any type. The REPL tool description states this protocol, in particular that facade descriptions do not include result types and that type documentation should only be requested for processing results in code. --- .serena/memories/repl.md | 16 ++++--- src/serena/repl/api/edit_api.py | 2 +- src/serena/repl/api/fs_api.py | 2 +- src/serena/repl/api/jb_api.py | 2 +- src/serena/repl/api/lsp_api.py | 2 +- src/serena/repl/facade.py | 81 ++++++++++++++++++++++++--------- src/serena/repl/repl.py | 8 +++- src/serena/tools/repl_tools.py | 13 ++++-- test/serena/test_repl_tool.py | 30 +++++++++--- 9 files changed, 115 insertions(+), 41 deletions(-) diff --git a/.serena/memories/repl.md b/.serena/memories/repl.md index 8729d804..84f374b1 100644 --- a/.serena/memories/repl.md +++ b/.serena/memories/repl.md @@ -30,14 +30,18 @@ Code runs as a function body (`return` defines the result); a single expression - Progressive disclosure: a priori only facade names, descriptions and method names; `s.info("")` / `s.info(".")` give signature + docstring together, never a signature alone. `info(*items)` documents several items at once; unknown items are reported inline. +- Disclosure tiers: tier 0 (tool description) = facades, descriptions, method names with navigable return types + (`find_symbol -> LspSymbolCollection`); tier 1 (`s.info("")`) = all common methods in full, + `niche` methods (`@facade_method(niche=True)`, rarely needed + long docs) only as summary + pointer, result + types by name only; tier 2 = types on request. Types are never pushed (`provide_info_with_facade` exists but + is set nowhere); the tool description tells the model to request type docs only when processing results in code. - Result types: a type returned by a single method is documented under `:return:`. Types that are shared, contained (`LanguageServerSymbol`) or navigated are declared per facade as `ReferencedType`s (constructor - arg `types=`), with `provide_info_with_facade` (full description in `s.info("")`, else listed by name) - and an optional `members` whitelist (curation for foreign/large classes; listed methods are shown even if - undocumented, convention-derived ones only if documented). Types are documented via `s.info(".")` - or bare `s.info("")`; method docs point to their referenced return type. Result classes declare - attribute annotations at class level (attributes set only in `__init__` are not discoverable). - Annotations are rendered without module paths, so signature names equal lookup names. + arg `types=`), with an optional `members` whitelist (curation for foreign/large classes; listed methods are + shown even if undocumented, convention-derived ones only if documented). Types are documented via + `s.info(".")` or bare `s.info("")`; method docs point to their referenced return type. + Result classes declare attribute annotations at class level (attributes set only in `__init__` are not + discoverable). Annotations are rendered without module paths, so signature names equal lookup names. - APIs must not import `serena.tools` at module level except for tool classes in decorators; tools import APIs locally in `_api()` (API modules refer to tool classes). diff --git a/src/serena/repl/api/edit_api.py b/src/serena/repl/api/edit_api.py index c7222eb9..adac2218 100644 --- a/src/serena/repl/api/edit_api.py +++ b/src/serena/repl/api/edit_api.py @@ -75,7 +75,7 @@ class EditApi(FacadeApi): name="edit", description="modifying content within existing files (independent of the language backend)", types=[ - ReferencedType(ReplacementPreview, provide_info_with_facade=True), + ReferencedType(ReplacementPreview), ReferencedType( ReplacementOccurrence, members=["occurrence_id", "relative_path", "start_line", "end_line", "matched_text", "replacement", "is_ambiguous"], diff --git a/src/serena/repl/api/fs_api.py b/src/serena/repl/api/fs_api.py index 5461d2ac..19b681f4 100644 --- a/src/serena/repl/api/fs_api.py +++ b/src/serena/repl/api/fs_api.py @@ -176,7 +176,7 @@ class FsApi(FacadeApi): types=[ ReferencedType(FileContent), ReferencedType(DirectoryListing), - ReferencedType(PatternMatches, provide_info_with_facade=True), + ReferencedType(PatternMatches), ReferencedType( MatchedConsecutiveLines, members=["source_file_path", "matched_lines", "start_line", "end_line", "to_display_string"] ), diff --git a/src/serena/repl/api/jb_api.py b/src/serena/repl/api/jb_api.py index 66854b61..84e8cb35 100644 --- a/src/serena/repl/api/jb_api.py +++ b/src/serena/repl/api/jb_api.py @@ -186,7 +186,7 @@ class JetBrainsApi(FacadeApi): name="jb", description="operations on the codebase backed by the JetBrains IDE's code intelligence", types=[ - ReferencedType(JetBrainsSymbolCollection, provide_info_with_facade=True), + ReferencedType(JetBrainsSymbolCollection), ReferencedType(JetBrainsSymbolsOverview), ReferencedType(JsonObject), ], diff --git a/src/serena/repl/api/lsp_api.py b/src/serena/repl/api/lsp_api.py index 449fc603..fbecf02e 100644 --- a/src/serena/repl/api/lsp_api.py +++ b/src/serena/repl/api/lsp_api.py @@ -313,7 +313,7 @@ class LspApi(FacadeApi): name="lsp", description="symbol-level operations on the codebase backed by language servers", types=[ - ReferencedType(LspSymbolCollection, provide_info_with_facade=True), + ReferencedType(LspSymbolCollection), ReferencedType( LanguageServerSymbol, members=[ diff --git a/src/serena/repl/facade.py b/src/serena/repl/facade.py index 6e7d1857..caec69fb 100644 --- a/src/serena/repl/facade.py +++ b/src/serena/repl/facade.py @@ -143,6 +143,11 @@ class FacadeMethodInfo: """whether the method is in beta (not yet fully stable)""" can_edit: bool = False """whether the method can modify the codebase (relevant for read-only contexts)""" + niche: bool = False + """ + whether the method is rarely needed, such that the facade's description only summarises it (first line of its + documentation and a pointer to its full documentation) in order to keep the description compact + """ corresponding_tool: "type[Tool] | None" = None """the classic tool offering the same functionality, if any""" @@ -157,7 +162,12 @@ _FACADE_METHOD_INFO_ATTR = "__facade_method_info__" def facade_method( - *, optional: bool = False, beta: bool = False, can_edit: bool = False, corresponding_tool: "type[Tool] | None" = None + *, + optional: bool = False, + beta: bool = False, + can_edit: bool = False, + niche: bool = False, + corresponding_tool: "type[Tool] | None" = None, ) -> Callable[[TCallable], TCallable]: """ Marks a method of a `FacadeApi` as exposed through the facade, attaching the given metadata. @@ -166,13 +176,19 @@ def facade_method( :param optional: whether the method is disabled by default and must be enabled explicitly :param beta: whether the method is in beta :param can_edit: whether the method can modify the codebase + :param niche: whether the method is rarely needed (its documentation is then only summarised in the facade's description) :param corresponding_tool: the classic tool offering the same functionality, if any :return: the decorator """ def decorator(method: TCallable) -> TCallable: info = FacadeMethodInfo( - name=method.__name__, optional=optional, beta=beta, can_edit=can_edit, corresponding_tool=corresponding_tool + name=method.__name__, + optional=optional, + beta=beta, + can_edit=can_edit, + niche=niche, + corresponding_tool=corresponding_tool, ) setattr(method, _FACADE_METHOD_INFO_ATTR, info) return method @@ -282,23 +298,40 @@ class FacadeMethod: def __call__(self, *args: Any, **kwargs: Any) -> Any: return self._implementation(*args, **kwargs) - def describe(self, include_return_type_pointer: bool = True) -> str: + def get_referenced_return_types(self) -> list[ReferencedType]: """ - :param include_return_type_pointer: whether to append a pointer to the documentation of the return type, + :return: the types referenced by the facade which appear in the method's return type annotation + """ + return_annotation = format_annotation(inspect.signature(self._implementation).return_annotation) + return [t for t in self.parent.get_types() if re.search(rf"\b{re.escape(t.name)}\b", return_annotation)] + + def get_summary(self) -> str: + """ + :return: the first line of the method's documentation + """ + doc = inspect.getdoc(self._implementation) + return doc.splitlines()[0] if doc else "(no documentation)" + + def describe(self) -> str: + """ + :return: the method's signature and documentation, with a pointer to the documentation of its return type if it is a type referenced by the facade - :return: the method's signature and documentation """ signature = format_signature(self._implementation) doc = inspect.getdoc(self._implementation) or "(no documentation)" text = f"{self.qualified_name}{signature}\n{doc}\n" - if include_return_type_pointer: - return_annotation = format_annotation(inspect.signature(self._implementation).return_annotation) - referenced = [t for t in self.parent.get_types() if re.search(rf"\b{re.escape(t.name)}\b", return_annotation)] - if referenced: - pointers = ", ".join(f'`s.info("{self.facade_name}.{t.name}")`' for t in referenced) - text += f"Return type: see {pointers}\n" + referenced = self.get_referenced_return_types() + if referenced: + pointers = ", ".join(f'`s.info("{t.name}")`' for t in referenced) + text += f"Return type documentation: {pointers}\n" return text + def describe_summary(self) -> str: + """ + :return: the method's name and the first line of its documentation, with a pointer to its full documentation + """ + return f'{self.qualified_name}: {self.get_summary()} [full documentation: `s.info("{self.qualified_name}")`]\n' + class ApiScope: """ @@ -492,18 +525,24 @@ class Facade: as well as its referenced types (in full if so declared, otherwise by name) """ parts = [f"Facade '{self._name}': {self._description}", ""] - for method in self._methods.values(): - if method.enabled: + enabled_methods = self.get_enabled_methods() + for method in enabled_methods: + if not method.info.niche: parts.append(method.describe()) - described_types = [t for t in self._types.values() if t.provide_info_with_facade] - listed_types = [t for t in self._types.values() if not t.provide_info_with_facade] - for referenced_type in described_types: - parts.append(referenced_type.describe()) - if listed_types: + niche_methods = [m for m in enabled_methods if m.info.niche] + if niche_methods: + parts.append("Rarely needed methods (documented on request):\n" + "".join(m.describe_summary() for m in niche_methods)) + for referenced_type in self._types.values(): + if referenced_type.provide_info_with_facade: + parts.append(referenced_type.describe()) + result_type_names = sorted( + {t.name for m in enabled_methods for t in m.get_referenced_return_types() if not t.provide_info_with_facade} + ) + if result_type_names: parts.append( - "Further types: " - + ", ".join(t.name for t in listed_types) - + f' (request documentation via `s.info("{self._name}.")`)' + "Result types: " + + ", ".join(result_type_names) + + ' (request documentation via `s.info("")` only if you intend to process results in code)' ) return "\n".join(parts) diff --git a/src/serena/repl/repl.py b/src/serena/repl/repl.py index 93f00834..883ab4c7 100644 --- a/src/serena/repl/repl.py +++ b/src/serena/repl/repl.py @@ -54,9 +54,15 @@ class SerenaReplEntrypoint: def overview(self) -> str: """ :return: the list of available facades, each with a one-line description and the names of its methods + (with the result type of methods returning objects that can be processed in code) """ + + def method_entry(method: FacadeMethod) -> str: + return_types = method.get_referenced_return_types() + return method.name + (f" -> {'|'.join(t.name for t in return_types)}" if return_types else "") + return "\n".join( - f"s.{facade.name}: {facade.description}\n methods: {', '.join(facade.enabled_method_names)}" + f"s.{facade.name}: {facade.description}\n methods: {', '.join(method_entry(m) for m in facade.get_enabled_methods())}" for facade in self._facades.values() ) diff --git a/src/serena/tools/repl_tools.py b/src/serena/tools/repl_tools.py index 611241e4..c8e9c2e1 100644 --- a/src/serena/tools/repl_tools.py +++ b/src/serena/tools/repl_tools.py @@ -18,9 +18,16 @@ class SerenaReplTool(Tool, ToolMarkerOptional, ToolMarkerBeta): def apply(self, code: str) -> str: """ Executes the given Python code, which has access to Serena's functionality through the object `s`. - The functionality is organised in facades, which are attributes of `s` (e.g. `s.myfacade`). - Use `s.info()` to list the facades, `s.info("")` to see a facade's methods and - `s.info(".")` for the documentation of a single method. + The functionality is organised in facades, which are attributes of `s` (e.g. `s.myfacade`); the available + facades and their methods are listed below. + + Documentation: Use `s.info("")` when you will use a facade's functionality (it documents all common + operations at once) and `s.info(".")` for a single or a rarely needed operation. Several items + can be requested in one call, e.g. `s.info("lsp", "edit.replace_content")`. + `s.info("")` documents the facade's operations only, not their result types. Result types are given + in the method listing below (`method -> Type`); request their documentation via `s.info("")`, which + includes the types they contain, ONLY if you intend to process results in code (filter, aggregate, chain + calls). If you simply want the result, return it directly: returned objects are rendered for you. The code is executed as the body of a function, so use `return` to define the result; a single expression is evaluated and its value returned directly. diff --git a/test/serena/test_repl_tool.py b/test/serena/test_repl_tool.py index da13d65d..9afe043e 100644 --- a/test/serena/test_repl_tool.py +++ b/test/serena/test_repl_tool.py @@ -53,15 +53,18 @@ class TestReplExecution: assert "name_path_pattern" in method_info def test_type_discovery(self, repl: SerenaRepl) -> None: + # the overview names the result types of methods returning objects that can be processed in code + assert "find_symbol -> LspSymbolCollection" in repl.execute("s.info()") + # signatures render type names without module paths, and point to the documentation of referenced return types method_info = repl.execute('s.info("lsp.find_symbol")') assert "-> LspSymbolCollection" in method_info and "lsp_api." not in method_info - assert 's.info("lsp.LspSymbolCollection")' in method_info + assert 's.info("LspSymbolCollection")' in method_info - # the facade description includes types declared as important in full and lists the others by name + # the facade description documents the operations only and lists the result types by name facade_info = repl.execute('s.info("lsp")') - assert "type LspSymbolCollection" in facade_info and "symbols: list[LanguageServerSymbol]" in facade_info - assert "type LanguageServerSymbol" not in facade_info and "LanguageServerSymbol" in facade_info + assert "type LspSymbolCollection" not in facade_info + assert "Result types: " in facade_info and "LspSymbolCollection" in facade_info # types can be requested via the facade or by bare name, and their curated members are documented type_info = repl.execute('s.info("lsp.LanguageServerSymbol")') @@ -96,6 +99,14 @@ class TestFacade: def extra(self) -> str: return "extra" + @facade_method(niche=True) + def rarely(self, x: int) -> str: + """Rarely needed operation. + + :param x: some parameter + """ + return str(x) + def undecorated(self) -> str: """Public within Serena, but not exposed, since it is not decorated.""" return "internal" @@ -142,13 +153,13 @@ class TestFacade: ApiInclusionDefinition(included_apis=["dummy.add"]), ) facade = Facade.from_api(self.DummyApi(MagicMock()), scope) - assert set(facade.enabled_method_names) == {"add", "secret"} + assert set(facade.enabled_method_names) == {"add", "secret", "rarely"} def test_api_scope_read_only_excludes_editing_methods(self) -> None: scope = self._scope(included_apis=["dummy.secret"]) scope.exclude_editing() facade = Facade.from_api(self.DummyApi(MagicMock()), scope) - assert facade.enabled_method_names == ["add"] + assert set(facade.enabled_method_names) == {"add", "rarely"} def test_enabled_methods_delegate_to_implementation(self) -> None: facade = Facade.from_api(self.DummyApi(MagicMock()), ApiScope()) @@ -176,6 +187,13 @@ class TestFacade: assert "undecorated" not in facade.describe() assert "undecorated" not in facade.enabled_method_names + def test_niche_methods_are_summarised_in_facade_description(self) -> None: + facade = Facade.from_api(self.DummyApi(MagicMock()), ApiScope()) + description = facade.describe() + assert "dummy.rarely: Rarely needed operation." in description + assert ":param x:" not in description # only the summary, no signature or full documentation + assert ":param x:" in facade.describe_member("rarely") # full documentation on request + def test_optional_methods_are_disabled_by_default(self) -> None: facade = Facade.from_api(self.DummyApi(MagicMock()), ApiScope()) assert "extra" not in facade.enabled_method_names From b067f5f71d95b4282589ac3c2e7adf2877353212 Mon Sep 17 00:00:00 2001 From: Dominik Jain Date: Mon, 7 Sep 2026 12:03:57 +0200 Subject: [PATCH 24/53] Update Serena project --- .serena/project.yml | 56 ++++++++++++++++++++++++++++----------------- 1 file changed, 35 insertions(+), 21 deletions(-) diff --git a/.serena/project.yml b/.serena/project.yml index 7c5257c1..8745f0bc 100644 --- a/.serena/project.yml +++ b/.serena/project.yml @@ -1,24 +1,24 @@ # the name by which the project can be referenced within Serena/when chatting with the LLM. project_name: "serena" - # list of language servers to start when using the LSP backend; choose from: -# ada al angular ansible bash -# bsl clojure cpp cpp_ccls crystal -# csharp csharp_omnisharp cue dart elixir -# elm erlang fortran fsharp gdscript -# go groovy haskell haxe hlsl -# html java json julia kotlin -# latex lean4 lua luau markdown -# matlab msl nix ocaml pascal -# perl php php_phpactor php_phpantom powershell -# python python_jedi python_pyrefly python_ty r -# rego ruby ruby_solargraph rust scala -# scss solidity svelte swift systemverilog -# terraform toml typescript typescript_vts vue -# yaml zig +# ada al angular ansible bash +# bsl clojure cpp cpp_ccls crystal +# csharp csharp_omnisharp cue dart deno +# elixir elm erlang fortran fsharp +# gdscript gleam go groovy haskell +# haxe hlsl html java json +# julia julia_fatou kotlin latex lean4 +# lua luau markdown matlab msl +# nextflow nix ocaml pascal perl +# php php_phpactor php_phpantom powershell python +# python_basedpyright python_jedi python_pyrefly python_ty qml +# r rego ruby ruby_solargraph rust +# scala scss solidity svelte swift +# systemverilog terraform toml typescript typescript_vts +# vue wolfram yaml zig # (This list may be outdated; generated with scripts/print_language_list.py; -# For the current list, see values of Language enum here: +# For the current list, see values of the LanguageServerId enum here: # https://github.com/oraios/serena/blob/main/src/solidlsp/ls_config.py) # For some languages, there are several alternative language servers, e.g. csharp_omnisharp, ruby_solargraph.) # Note: @@ -26,6 +26,7 @@ project_name: "serena" # - For JavaScript, use typescript # - For Angular projects, use angular (subsumes typescript+html; requires `npm install` in the project root) # - For Svelte projects, use svelte (subsumes typescript/javascript for .svelte projects; requires npm) +# - For Deno projects, use deno (serves the same .ts/.js files as typescript; requires the deno CLI on PATH) # - For SCSS / Sass / plain CSS, use scss (some-sass-language-server handles all three) # - For Free Pascal/Lazarus, use pascal # Special requirements: @@ -35,8 +36,8 @@ project_name: "serena" # The first language server is the default language and the respective language server will be used as a fallback. # Note that when using the JetBrains backend, language servers are not used and this list is correspondingly ignored. language_servers: - - python - - typescript +- python +- typescript # whether to use project's .gitignore files to ignore files ignore_all_files_in_gitignore: true @@ -68,9 +69,8 @@ excluded_tools: [] # Find the list of tools here: https://oraios.github.io/serena/01-about/035_tools.html included_optional_tools: [] -# initial prompt for the project, which will be provided to the LLM upon project activation -# (or within Serena's initial instructions if the project is activated at startup). -## See: https://oraios.github.io/serena/02-usage/050_configuration.html#prompt-templates +# initial prompt for the project. It will always be given to the LLM upon activating the project +# (contrary to the memories, which are loaded on demand). initial_prompt: | {{ embed_memory("critical_info") }} @@ -177,3 +177,17 @@ ls_workspace_folders: # - ../sibling-package # - ../shared-lib ls_additional_workspace_folders: [] + +# list of APIs (facades or facade methods, e.g. "lsp" or "lsp.get_diagnostics_for_symbol") to include in the REPL +# that would otherwise be disabled (particularly optional methods, which are disabled by default). +# This extends the existing inclusions (e.g. from the global configuration). +included_apis: [] + +# list of APIs (facades or facade methods, e.g. "lsp" or "lsp.find_symbol") to exclude from the REPL. +# This extends the existing exclusions (e.g. from the global configuration). +excluded_apis: [] + +# The interface through which the agent (LLM) accesses Serena's functionality (overrides the global setting). +# Valid values: tools, REPL (see the global configuration for details); leave empty to use the global setting. +# Note: the interface is fixed at startup. If a project is activated post-init, its setting is not applied. +agent_interface: From 24b41dcb88eb733d88743f1d776a1e2bfc85b044 Mon Sep 17 00:00:00 2001 From: Dominik Jain Date: Tue, 8 Sep 2026 14:09:57 +0200 Subject: [PATCH 25/53] Add sessions to the REPL; document contained types once per session Type documentation now transitively includes the declared types a type's members reference, such that e.g. LspSymbolCollection's documentation includes LanguageServerSymbol. To avoid repeating widely shared types, the REPL tracks per session which types have been documented: a contained type documented earlier in the session is only pointed to (an explicit request always yields it in full). Since MCP provides no reliable session identification and clients keep a stdio server across conversations, the REPL's session identity is supplied by the LLM: the instructions (system prompt) establish a SerenaSession and state its id, and the REPL tool takes a required session_id parameter. Sessions are managed by a SessionRegistry (creation on demand, LRU eviction). Tools which may be called before the instructions have been read (activate_project, initial_instructions) are unchanged. --- .serena/memories/repl.md | 14 ++++++ src/serena/agent.py | 24 ++++++++++ src/serena/repl/api/fs_api.py | 3 +- src/serena/repl/api/lsp_api.py | 7 +-- src/serena/repl/facade.py | 22 +++++++++ src/serena/repl/repl.py | 79 ++++++++++++++++++++++++++++---- src/serena/session.py | 62 +++++++++++++++++++++++++ src/serena/tools/repl_tools.py | 5 +- test/serena/test_repl_tool.py | 26 +++++++++-- test/serena/test_serena_agent.py | 8 +++- 10 files changed, 229 insertions(+), 21 deletions(-) create mode 100644 src/serena/session.py diff --git a/.serena/memories/repl.md b/.serena/memories/repl.md index 84f374b1..2dcebf42 100644 --- a/.serena/memories/repl.md +++ b/.serena/memories/repl.md @@ -42,6 +42,20 @@ Code runs as a function body (`return` defines the result); a single expression `s.info(".")` or bare `s.info("")`; method docs point to their referenced return type. Result classes declare attribute annotations at class level (attributes set only in `__init__` are not discoverable). Annotations are rendered without module paths, so signature names equal lookup names. + A type's documentation transitively includes the declared types its members reference; within a session + (`SerenaSession.described_type_names`), a contained type is documented once and afterwards only pointed to + (explicit requests always yield full documentation). + +## Sessions (`serena.session`) + +- MCP provides no reliable session identification (newer protocol versions drop it), and clients keep a stdio + server across conversations. Hence the REPL's session identity is LLM-supplied: `create_system_prompt` creates + a `SerenaSession` and states its id; `serena_repl` takes a required `session_id`. `SessionRegistry` creates + unknown ids on demand (benign: at worst docs are repeated) and evicts LRU. Sessions survive REPL rebuilds. + Later: the persistent REPL namespace lives on the session. +- Only tools whose use presupposes having read the instructions may require the id. `activate_project` and + `initial_instructions` may be called first and keep the existing MCP-context-derived session handling (prompt + provision status); migrating that to LLM-supplied ids is a separate, future change. - APIs must not import `serena.tools` at module level except for tool classes in decorators; tools import APIs locally in `_api()` (API modules refer to tool classes). diff --git a/src/serena/agent.py b/src/serena/agent.py index eba6ae85..4e8d16ad 100644 --- a/src/serena/agent.py +++ b/src/serena/agent.py @@ -58,6 +58,7 @@ from serena.repl.api.mem_api import MemoryApi from serena.repl.api.shell_api import ShellApi from serena.repl.facade import ApiScope, Facade from serena.repl.repl import SerenaRepl +from serena.session import SerenaSession, SessionRegistry from serena.task_executor import TaskExecutor from serena.tools import ( ActivateProjectTool, @@ -580,6 +581,7 @@ class SerenaAgent: self._gui_log_viewer: Optional["GuiLogViewer"] = None self._dashboard_manager: DashboardManager | None = None self._project_prompt_status = ProjectPromptProvisionStatus() + self._session_registry = SessionRegistry() self._session_mode_selection_definition = modes self.version = serena_version() self._config_changed_callbacks: list[Callable[[], None]] = [] @@ -1133,9 +1135,24 @@ class SerenaAgent: else: return self._create_global_memory_manager() + def create_session(self) -> SerenaSession: + """ + :return: a new client session (with a random id) + """ + return self._session_registry.create_session() + + def get_session(self, session_id: str) -> SerenaSession: + """ + :param session_id: the session id (as supplied by the LLM) + :return: the session, which is created if it is unknown + """ + return self._session_registry.get_session(session_id) + def create_system_prompt(self, session_id: str = "global") -> str: """ Returns the 'Serena Instructions Manual', i.e. Serena's system prompt. + The prompt also establishes a new Serena session (see `SerenaSession`), stating its id for use with tools + which require it (e.g. the REPL tool). :param session_id: the client session ID for the case where this is run from a tool; "global" for the connection time case :return: the prompt @@ -1168,6 +1185,13 @@ class SerenaAgent: elif self._project_activation_error: system_prompt += f"\n\nNo project is active ({self._project_activation_error})." + # establish a Serena session and state its id, which the LLM must pass to tools which require it + serena_session = self.create_session() + system_prompt += "\n\n" + self._format_prompt_tag( + f"Your Serena session id is `{serena_session.session_id}`. Pass it as the `session` parameter to tools which require it.", + tag="session", + ) + return self._format_prompt_tag(system_prompt, tag="serena") def get_project_activation_message(self, session_id: str) -> str: diff --git a/src/serena/repl/api/fs_api.py b/src/serena/repl/api/fs_api.py index 19b681f4..608c7c9a 100644 --- a/src/serena/repl/api/fs_api.py +++ b/src/serena/repl/api/fs_api.py @@ -72,8 +72,7 @@ class DirectoryListingRenderer(Renderer[DirectoryListing]): class PatternMatches(RepresentableViaRenderer): """ - The matches of a pattern search: `matches`, each a `MatchedConsecutiveLines` with `source_file_path`, - `matched_lines` (each with `line_number` and `line_content`) and `to_display_string()`. + The matches of a pattern search (`MatchedConsecutiveLines`). """ def __init__(self, matches: list[MatchedConsecutiveLines], renderer: "PatternMatchesRenderer"): diff --git a/src/serena/repl/api/lsp_api.py b/src/serena/repl/api/lsp_api.py index fbecf02e..15161492 100644 --- a/src/serena/repl/api/lsp_api.py +++ b/src/serena/repl/api/lsp_api.py @@ -42,9 +42,7 @@ if TYPE_CHECKING: class LspSymbolCollection(RepresentableViaRenderer): """ - A collection of symbols retrieved via the language server. - Each symbol (`LanguageServerSymbol`) offers e.g. `get_name_path()`, `relative_path`, `symbol_kind_name`, - `body`, `get_body_line_numbers()`, `iter_children()`. + A collection of symbols (`LanguageServerSymbol`) retrieved via the language server. """ symbols: list[LanguageServerSymbol] @@ -207,8 +205,7 @@ class LspSymbolsOverviewRenderer(LspSymbolCollectionRenderer): class LspReferenceCollection(RepresentableViaRenderer): """ - The references to a symbol, each a `ReferenceInLanguageServerSymbol` with the referencing `symbol` - (a `LanguageServerSymbol`) and the `line` of the reference. + The references to a symbol (`ReferenceInLanguageServerSymbol`). """ references: list[ReferenceInLanguageServerSymbol] diff --git a/src/serena/repl/facade.py b/src/serena/repl/facade.py index caec69fb..ebcea4bd 100644 --- a/src/serena/repl/facade.py +++ b/src/serena/repl/facade.py @@ -81,6 +81,28 @@ class ReferencedType: names.difference_update(dir(RepresentableViaRenderer)) return sorted(n for n in names if not n.startswith("_") and not n.endswith("_")) + def get_referenced_type_names(self) -> list[str]: + """ + :return: the names of the types appearing in the annotations of the described members (attributes, properties, + method return types), in order of appearance (each name at most once, excluding the type itself) + """ + annotations: list[str] = [] + type_hints = typing.get_type_hints(self.cls) + for member_name in self._get_member_names(): + member = inspect.getattr_static(self.cls, member_name, None) + if isinstance(member, property) and member.fget is not None: + annotations.append(format_annotation(inspect.signature(member.fget).return_annotation)) + elif inspect.isfunction(member): + annotations.append(format_annotation(inspect.signature(member).return_annotation)) + elif member_name in type_hints: + annotations.append(format_annotation(type_hints[member_name])) + names: list[str] = [] + for annotation in annotations: + for name in re.findall(r"\b[A-Z]\w*", annotation): + if name != self.name and name not in names: + names.append(name) + return names + @staticmethod def _first_doc_line(obj: Any) -> str: doc = inspect.getdoc(obj) or "" diff --git a/src/serena/repl/repl.py b/src/serena/repl/repl.py index 883ab4c7..1e55822b 100644 --- a/src/serena/repl/repl.py +++ b/src/serena/repl/repl.py @@ -9,7 +9,8 @@ import textwrap import traceback from typing import Any -from .facade import ApiScope, Facade, FacadeMethod +from ..session import SerenaSession +from .facade import ApiScope, Facade, FacadeMethod, ReferencedType from .representable import Representable log = logging.getLogger(__name__) @@ -27,6 +28,7 @@ class SerenaReplEntrypoint: :param api_scope: the API scope, which determines which of the facades are made available """ self._facades: dict[str, Facade] = {} + self._current_session: SerenaSession | None = None registered_facade_names = [] for facade in facades: if api_scope.is_facade_enabled(facade.name): @@ -40,6 +42,12 @@ class SerenaReplEntrypoint: """ return [method for facade in self._facades.values() for method in facade.get_enabled_methods()] + def set_current_session_(self, session: SerenaSession | None) -> None: + """ + :param session: the session on whose behalf code is being executed (None if no code is being executed) + """ + self._current_session = session + def _register(self, facade: Facade) -> None: if facade.name in self._facades: raise ValueError(f"Duplicate facade name: {facade.name}") @@ -79,23 +87,72 @@ class SerenaReplEntrypoint: """ if not items: return self.overview() - return "\n\n".join(self._describe_item(item) for item in items) + described_in_call: set[str] = set() + return "\n\n".join(self._describe_item(item, described_in_call) for item in items) - def _describe_item(self, item: str) -> str: + def _describe_item(self, item: str, described_in_call: set[str]) -> str: facade_name, _, member_name = item.partition(".") try: if member_name: - return self._get_facade(facade_name).describe_member(member_name) + facade = self._get_facade(facade_name) + referenced_type = facade.get_type(member_name) + if referenced_type is not None: + return self._describe_type(referenced_type, described_in_call) + return facade.describe_member(member_name) if facade_name in self._facades: return self._facades[facade_name].describe() # not a facade: look up the item as a type across all facades - matches = [facade for facade in self._facades.values() if facade.get_type(item) is not None] - if not matches: + referenced_type = self._find_type(item) + if referenced_type is None: raise ValueError(f"Unknown item '{item}': neither a facade nor a type. Available facades: {list(self._facades)}") - return matches[0].describe_member(item) + return self._describe_type(referenced_type, described_in_call) except ValueError as e: return str(e) + def _find_type(self, type_name: str) -> ReferencedType | None: + for facade in self._facades.values(): + referenced_type = facade.get_type(type_name) + if referenced_type is not None: + return referenced_type + return None + + def _describe_type(self, referenced_type: ReferencedType, described_in_call: set[str]) -> str: + """ + Describes the given (explicitly requested) type along with the types it references (transitively), each at most + once per call. Referenced types whose documentation was already provided earlier in the session are not repeated + but pointed to (an explicit request always yields the full documentation). + + :param referenced_type: the requested type + :param described_in_call: the names of the types already described in the current `info` call (updated) + :return: the documentation + """ + session = self._current_session + parts = [] + if referenced_type.name not in described_in_call: + parts.append(referenced_type.describe()) + described_in_call.add(referenced_type.name) + if session is not None: + session.described_type_names.add(referenced_type.name) + + # append the referenced types (breadth-first), unless already described in this call or earlier in the session + pending = list(referenced_type.get_referenced_type_names()) + while pending: + type_name = pending.pop(0) + if type_name in described_in_call: + continue + contained_type = self._find_type(type_name) + if contained_type is None: + continue + described_in_call.add(type_name) + if session is not None and type_name in session.described_type_names: + parts.append(f'type {type_name}: documented earlier in this session (request `s.info("{type_name}")` to see it again)\n') + else: + parts.append(contained_type.describe()) + if session is not None: + session.described_type_names.add(type_name) + pending.extend(contained_type.get_referenced_type_names()) + return "\n".join(parts) + class SerenaRepl: """ @@ -138,17 +195,23 @@ class SerenaRepl: return "\n".join(cls._represent(item) for item in obj) return str(obj) - def execute(self, code: str) -> str: + def execute(self, code: str, session: SerenaSession | None = None) -> str: """ Executes the given code and renders its result. + Executions are expected to be serialised (the entrypoint holds the current session during execution). :param code: the Python code to execute + :param session: the client session on whose behalf the code is executed (None for session-less execution, + e.g. in tests), which determines e.g. which type documentation has already been provided :return: the representation of the code's result, or a description of the error if execution failed """ + self._entrypoint.set_current_session_(session) try: result = self._run(code) except Exception as e: return self._format_error(e, code) + finally: + self._entrypoint.set_current_session_(None) return self._represent(result) def _run(self, code: str) -> Any: diff --git a/src/serena/session.py b/src/serena/session.py new file mode 100644 index 00000000..b30ce411 --- /dev/null +++ b/src/serena/session.py @@ -0,0 +1,62 @@ +# SPDX-License-Identifier: GPL-3.0-or-later +""" +Client sessions (conversations) and their state. +""" + +import logging +import secrets +from collections import OrderedDict + +log = logging.getLogger(__name__) + + +class SerenaSession: + """ + A client session, i.e. a conversation between an LLM and Serena, and the state pertaining to it. + + Session identity is supplied by the LLM: the session id is issued as part of Serena's instructions + (system prompt/initial instructions) and passed by the LLM to tools which require it. + """ + + def __init__(self, session_id: str) -> None: + self.session_id = session_id + self.described_type_names: set[str] = set() + """the names of the REPL's result types whose documentation has already been provided in this session""" + + +class SessionRegistry: + """ + Holds the sessions of an agent, creating them on demand and evicting the least recently used ones + if their number exceeds the limit (session ids being supplied by LLMs, the set of ids is not controlled). + """ + + def __init__(self, max_sessions: int = 100) -> None: + """ + :param max_sessions: the maximum number of sessions to keep + """ + self._max_sessions = max_sessions + self._sessions: OrderedDict[str, SerenaSession] = OrderedDict() + + def create_session(self) -> SerenaSession: + """ + :return: a new session with a random id + """ + return self.get_session(secrets.token_hex(4)) + + def get_session(self, session_id: str) -> SerenaSession: + """ + :param session_id: the session id + :return: the session, which is created if it is unknown (an unknown id may e.g. stem from a session that has been + evicted or from an earlier run of the server) + """ + session = self._sessions.get(session_id) + if session is None: + session = SerenaSession(session_id) + self._sessions[session_id] = session + log.info("Created session %s (%d sessions)", session_id, len(self._sessions)) + while len(self._sessions) > self._max_sessions: + evicted_id, _ = self._sessions.popitem(last=False) + log.info("Evicted session %s", evicted_id) + else: + self._sessions.move_to_end(session_id) + return session diff --git a/src/serena/tools/repl_tools.py b/src/serena/tools/repl_tools.py index c8e9c2e1..b3af1856 100644 --- a/src/serena/tools/repl_tools.py +++ b/src/serena/tools/repl_tools.py @@ -15,7 +15,7 @@ class SerenaReplTool(Tool, ToolMarkerOptional, ToolMarkerBeta): def get_apply_docstring(self) -> str: return self.get_apply_docstring_from_cls() + "\n\nAvailable facades:\n" + self.agent.get_repl().entrypoint.overview() - def apply(self, code: str) -> str: + def apply(self, session: str, code: str) -> str: """ Executes the given Python code, which has access to Serena's functionality through the object `s`. The functionality is organised in facades, which are attributes of `s` (e.g. `s.myfacade`); the available @@ -34,7 +34,8 @@ class SerenaReplTool(Tool, ToolMarkerOptional, ToolMarkerBeta): Returned objects are rendered in a form suitable for you; lists are rendered element-wise. Returned strings are passed through unchanged. + :param session: your Serena session id, as provided in Serena's instructions (call `initial_instructions` if you do not have one) :param code: the Python code to execute :return: the representation of the returned value, or the error if execution failed """ - return self.agent.get_repl().execute(code) + return self.agent.get_repl().execute(code, self.agent.get_session(session)) diff --git a/test/serena/test_repl_tool.py b/test/serena/test_repl_tool.py index 9afe043e..82a8219b 100644 --- a/test/serena/test_repl_tool.py +++ b/test/serena/test_repl_tool.py @@ -11,6 +11,7 @@ from serena.config.serena_config import ApiInclusionDefinition from serena.repl.api.lsp_api import LspApi from serena.repl.facade import ApiScope, Facade, FacadeApi, FacadeMethodInfo, facade_method from serena.repl.repl import SerenaRepl +from serena.session import SerenaSession from serena.tools import FindSymbolTool, SerenaReplTool from solidlsp.ls_config import LanguageServerId from test.conftest import agent_for_project_context @@ -71,7 +72,24 @@ class TestReplExecution: assert type_info == repl.execute('s.info("LanguageServerSymbol")') assert "get_name_path() -> str" in type_info and "iter_children()" in type_info assert "to_dict" not in type_info # not among the curated members - assert "represent" not in repl.execute('s.info("LspSymbolCollection")') # the representation mechanism is not exposed + assert "represent()" not in repl.execute('s.info("LspSymbolCollection")') # the representation mechanism is not exposed + + def test_contained_types_are_documented_once_per_session(self, repl: SerenaRepl) -> None: + session = SerenaSession("test") + + # a type's documentation includes the types it contains (transitively) + first = repl.execute('s.info("LspReferenceCollection")', session) + assert "type LspReferenceCollection" in first + assert "type ReferenceInLanguageServerSymbol" in first and "type LanguageServerSymbol" in first + + # a contained type documented earlier in the session is only pointed to; an explicit request yields it again + second = repl.execute('s.info("LspSymbolCollection")', session) + assert "type LspSymbolCollection" in second + assert "type LanguageServerSymbol: documented earlier" in second and "get_name_path()" not in second + assert "get_name_path()" in repl.execute('s.info("LanguageServerSymbol")', session) + + # another session is unaffected + assert "get_name_path()" in repl.execute('s.info("LspSymbolCollection")', SerenaSession("other")) def test_info_documents_several_items(self, repl: SerenaRepl) -> None: info = repl.execute('s.info("lsp.find_symbol", "nope", "lsp.LspSymbolCollection")') @@ -237,14 +255,16 @@ class TestLspFacade: def test_find_symbol_via_repl(self) -> None: with agent_for_project_context(LanguageServerId.PYTHON) as agent: tool = agent.get_tool(SerenaReplTool) + session_id = agent.create_session().session_id # a returned collection is rendered, identifying the symbol and its file - rendered = tool.apply('return s.lsp.find_symbol("create_user")') + rendered = tool.apply(session_id, 'return s.lsp.find_symbol("create_user")') assert "create_user" in rendered assert "services.py" in rendered # the underlying symbols are accessible from code, e.g. to retrieve a body without rendering the collection body = tool.apply( - f'result = s.lsp.find_symbol("create_user", relative_path={self._SERVICES_FILE!r})\nreturn result.symbols[0].body' + session_id, + f'result = s.lsp.find_symbol("create_user", relative_path={self._SERVICES_FILE!r})\nreturn result.symbols[0].body', ) assert body.startswith("def create_user") diff --git a/test/serena/test_serena_agent.py b/test/serena/test_serena_agent.py index 58e9e675..dece1f1a 100644 --- a/test/serena/test_serena_agent.py +++ b/test/serena/test_serena_agent.py @@ -917,12 +917,18 @@ class TestSerenaAgent: exposed = {tool.get_name() for tool in agent.get_exposed_tool_instances()} expected = {"serena_repl", "initial_instructions"} | (set() if context.single_project else {"activate_project"}) assert exposed == expected - assert "s.lsp" in agent.get_tool(SerenaReplTool).apply("s.info()") + assert "s.lsp" in agent.get_tool(SerenaReplTool).apply(agent.create_session().session_id, "s.info()") # prompts refer to operations by their qualified REPL names, e.g. `lsp.find_symbol` instead of the tool name system_prompt = agent.create_system_prompt() assert "`lsp.find_symbol`" in system_prompt assert "`find_symbol`" not in system_prompt + + # the instructions establish a session, whose id can be used with session-aware tools + session_id_match = re.search(r"session id is `(\w+)`", system_prompt) + assert session_id_match is not None + session_id = session_id_match.group(1) + assert "s.lsp" in agent.get_tool(SerenaReplTool).apply(session_id, "s.info()") finally: agent.on_shutdown(timeout=5) From 8d1fb7424c00ff9ca399d58243629a30c6556b8a Mon Sep 17 00:00:00 2001 From: Dominik Jain Date: Tue, 8 Sep 2026 15:58:32 +0200 Subject: [PATCH 26/53] Follow parameter annotations in REPL type documentation; support enums Referenced types are now also discovered through the parameter annotations of type members and facade methods (not only through return types), such that e.g. SymbolKind (a parameter type of LanguageServerSymbol.iter_ancestors) is documented along with LanguageServerSymbol. Enums are documented with their members and values. The lsp facade declares SymbolKind as a referenced type. --- src/serena/repl/api/lsp_api.py | 13 ++++++----- src/serena/repl/facade.py | 40 ++++++++++++++++++++++++++++------ test/serena/test_repl_tool.py | 4 ++++ 3 files changed, 44 insertions(+), 13 deletions(-) diff --git a/src/serena/repl/api/lsp_api.py b/src/serena/repl/api/lsp_api.py index 15161492..7e648318 100644 --- a/src/serena/repl/api/lsp_api.py +++ b/src/serena/repl/api/lsp_api.py @@ -331,6 +331,7 @@ class LspApi(FacadeApi): ReferencedType(LspReferenceCollection), ReferencedType(ReferenceInLanguageServerSymbol), ReferencedType(LspDiagnostics), + ReferencedType(SymbolKind), ], ) @@ -453,8 +454,8 @@ class LspApi(FacadeApi): :param include_info: whether to include additional info (hover-like, typically including docstring and signature), about the symbol (ignored if include_body is True). Info is never included for child symbols. Note: Depending on the language, this can be slow (e.g., C/C++). - :param include_kinds: (optional) limits results to the given LSP symbol kinds (integers) - :param exclude_kinds: (optional) list of LSP symbol kinds (integers) to exclude. + :param include_kinds: (optional) limits results to the given LSP symbol kinds (integers, i.e. values of `SymbolKind`) + :param exclude_kinds: (optional) list of LSP symbol kinds (integers, i.e. values of `SymbolKind`) to exclude. :param substring_matching: If True, use substring matching for the last segment of `name_path_pattern` (i.e. the name of the symbol, e.g. "foo" in "Class/foo" or "my_method" in "my_method"). :param max_matches: Maximum number of permitted matches. If exceeded, an error containing a shortened result is raised, @@ -517,8 +518,8 @@ class LspApi(FacadeApi): :param name_path: name path of the symbol :param relative_path: the relative path to the file containing the symbol for which to find references. - :param include_kinds: (optional) limits results to the given LSP symbol kinds (integers) - :param exclude_kinds: optional list of LSP symbol kinds (integers) to exclude. + :param include_kinds: (optional) limits results to the given LSP symbol kinds (integers, i.e. values of `SymbolKind`) + :param exclude_kinds: optional list of LSP symbol kinds (integers, i.e. values of `SymbolKind`) to exclude. :param max_answer_chars: max result length; -1 for default :return: the references to the symbol """ @@ -554,8 +555,8 @@ class LspApi(FacadeApi): Note that here you can't pass a directory but must pass a file. :param include_info: whether to include additional info (hover-like, typically including docstring and signature), about the implementing symbols. - :param include_kinds: (optional) limits results to the given LSP symbol kinds (integers) - :param exclude_kinds: (optional) list of LSP symbol kinds (integers) to exclude. + :param include_kinds: (optional) limits results to the given LSP symbol kinds (integers, i.e. values of `SymbolKind`) + :param exclude_kinds: (optional) list of LSP symbol kinds (integers, i.e. values of `SymbolKind`) to exclude. :param max_answer_chars: max result length; -1 for default :return: the symbols implementing the given symbol """ diff --git a/src/serena/repl/facade.py b/src/serena/repl/facade.py index ebcea4bd..eb040b06 100644 --- a/src/serena/repl/facade.py +++ b/src/serena/repl/facade.py @@ -11,6 +11,7 @@ import typing from abc import ABC from collections.abc import Callable, Sequence from dataclasses import dataclass +from enum import Enum from typing import TYPE_CHECKING, Any, TypeVar from serena.config.serena_config import ApiInclusionDefinition @@ -84,8 +85,10 @@ class ReferencedType: def get_referenced_type_names(self) -> list[str]: """ :return: the names of the types appearing in the annotations of the described members (attributes, properties, - method return types), in order of appearance (each name at most once, excluding the type itself) + method parameters and return types), in order of appearance (each name at most once, excluding the type itself) """ + if self.is_enum(): + return [] annotations: list[str] = [] type_hints = typing.get_type_hints(self.cls) for member_name in self._get_member_names(): @@ -93,7 +96,7 @@ class ReferencedType: if isinstance(member, property) and member.fget is not None: annotations.append(format_annotation(inspect.signature(member.fget).return_annotation)) elif inspect.isfunction(member): - annotations.append(format_annotation(inspect.signature(member).return_annotation)) + annotations.append(format_signature(member)) elif member_name in type_hints: annotations.append(format_annotation(type_hints[member_name])) names: list[str] = [] @@ -103,6 +106,16 @@ class ReferencedType: names.append(name) return names + def is_enum(self) -> bool: + return isinstance(self.cls, type) and issubclass(self.cls, Enum) + + def _describe_enum(self) -> str: + parts = [f"enum {self.name}"] + if self.cls.__doc__: + parts.append(f" {inspect.cleandoc(self.cls.__doc__).replace(chr(10), chr(10) + ' ')}") + parts.append("members:\n" + "\n".join(f" {self.name}.{member.name} = {member.value!r}" for member in self.cls)) # type: ignore[attr-defined] + return "\n".join(parts) + "\n" + @staticmethod def _first_doc_line(obj: Any) -> str: doc = inspect.getdoc(obj) or "" @@ -112,8 +125,10 @@ class ReferencedType: def describe(self) -> str: """ :return: the type's documentation: its docstring, attributes/properties with their types and methods with their - signatures and documentation + signatures and documentation; for enums, the members with their values """ + if self.is_enum(): + return self._describe_enum() attributes: list[str] = [] methods: list[str] = [] type_hints = typing.get_type_hints(self.cls) @@ -325,7 +340,17 @@ class FacadeMethod: :return: the types referenced by the facade which appear in the method's return type annotation """ return_annotation = format_annotation(inspect.signature(self._implementation).return_annotation) - return [t for t in self.parent.get_types() if re.search(rf"\b{re.escape(t.name)}\b", return_annotation)] + return self._find_referenced_types(return_annotation) + + def get_referenced_parameter_types(self) -> list[ReferencedType]: + """ + :return: the types referenced by the facade which appear in the annotations of the method's parameters + """ + parameters = inspect.signature(self._implementation).parameters.values() + return self._find_referenced_types(" ".join(format_annotation(p.annotation) for p in parameters)) + + def _find_referenced_types(self, annotation_text: str) -> list[ReferencedType]: + return [t for t in self.parent.get_types() if re.search(rf"\b{re.escape(t.name)}\b", annotation_text)] def get_summary(self) -> str: """ @@ -336,16 +361,17 @@ class FacadeMethod: def describe(self) -> str: """ - :return: the method's signature and documentation, with a pointer to the documentation of its return type - if it is a type referenced by the facade + :return: the method's signature and documentation, with pointers to the documentation of the referenced types + appearing in its return type and parameter annotations """ signature = format_signature(self._implementation) doc = inspect.getdoc(self._implementation) or "(no documentation)" text = f"{self.qualified_name}{signature}\n{doc}\n" referenced = self.get_referenced_return_types() + referenced += [t for t in self.get_referenced_parameter_types() if t not in referenced] if referenced: pointers = ", ".join(f'`s.info("{t.name}")`' for t in referenced) - text += f"Return type documentation: {pointers}\n" + text += f"Type documentation: {pointers}\n" return text def describe_summary(self) -> str: diff --git a/test/serena/test_repl_tool.py b/test/serena/test_repl_tool.py index 82a8219b..2dae1c5b 100644 --- a/test/serena/test_repl_tool.py +++ b/test/serena/test_repl_tool.py @@ -72,6 +72,10 @@ class TestReplExecution: assert type_info == repl.execute('s.info("LanguageServerSymbol")') assert "get_name_path() -> str" in type_info and "iter_children()" in type_info assert "to_dict" not in type_info # not among the curated members + + # types referenced by parameters are documented as well; enums are documented with their members + assert "enum SymbolKind" in type_info and "SymbolKind.Class = 5" in type_info # via iter_ancestors(up_to_symbol_kind) + assert "enum SymbolKind" in repl.execute('s.info("SymbolKind")') assert "represent()" not in repl.execute('s.info("LspSymbolCollection")') # the representation mechanism is not exposed def test_contained_types_are_documented_once_per_session(self, repl: SerenaRepl) -> None: From 44a283ee9978231335db724ab24017a7df059d8b Mon Sep 17 00:00:00 2001 From: Dominik Jain Date: Tue, 8 Sep 2026 19:54:46 +0200 Subject: [PATCH 27/53] Make all types reachable through annotations documentable in the REPL Facades now discover all user-defined classes reachable through the annotations of their methods and of the members of reachable types (transitively) and make them documentable via s.info, without the types having to be declared. Explicit ReferencedType declarations serve curation only (member whitelists, flags). TypedDicts are documented with their keys (e.g. the JetBrains symbol DTOs and the LSP diagnostic structures), enums with their members. SymbolKind no longer needs declaring. --- .serena/memories/repl.md | 12 ++-- src/serena/repl/api/lsp_api.py | 1 - src/serena/repl/facade.py | 108 +++++++++++++++++++++++++++++---- src/serena/repl/repl.py | 4 +- test/serena/test_repl_tool.py | 8 ++- 5 files changed, 110 insertions(+), 23 deletions(-) diff --git a/.serena/memories/repl.md b/.serena/memories/repl.md index 2dcebf42..f0d1f389 100644 --- a/.serena/memories/repl.md +++ b/.serena/memories/repl.md @@ -35,11 +35,13 @@ Code runs as a function body (`return` defines the result); a single expression `niche` methods (`@facade_method(niche=True)`, rarely needed + long docs) only as summary + pointer, result types by name only; tier 2 = types on request. Types are never pushed (`provide_info_with_facade` exists but is set nowhere); the tool description tells the model to request type docs only when processing results in code. -- Result types: a type returned by a single method is documented under `:return:`. Types that are shared, - contained (`LanguageServerSymbol`) or navigated are declared per facade as `ReferencedType`s (constructor - arg `types=`), with an optional `members` whitelist (curation for foreign/large classes; listed methods are - shown even if undocumented, convention-derived ones only if documented). Types are documented via - `s.info(".")` or bare `s.info("")`; method docs point to their referenced return type. +- Result types: every user-defined class reachable through annotations (method parameters/returns, and the members + of reachable types, transitively) is automatically documentable (`Facade._discover_referenced_types`); builtins, + typing constructs and stdlib classes are excluded. Explicit `ReferencedType` declarations (constructor arg + `types=`) exist for curation only: an optional `members` whitelist (foreign/large classes such as + `LanguageServerSymbol`; listed methods are shown even if undocumented, convention-derived ones only if + documented) and flags. Enums render with members/values, TypedDicts with their keys. Types are documented via + `s.info(".")` or bare `s.info("")`; method docs point to their referenced types. Result classes declare attribute annotations at class level (attributes set only in `__init__` are not discoverable). Annotations are rendered without module paths, so signature names equal lookup names. A type's documentation transitively includes the declared types its members reference; within a session diff --git a/src/serena/repl/api/lsp_api.py b/src/serena/repl/api/lsp_api.py index 7e648318..fed32cee 100644 --- a/src/serena/repl/api/lsp_api.py +++ b/src/serena/repl/api/lsp_api.py @@ -331,7 +331,6 @@ class LspApi(FacadeApi): ReferencedType(LspReferenceCollection), ReferencedType(ReferenceInLanguageServerSymbol), ReferencedType(LspDiagnostics), - ReferencedType(SymbolKind), ], ) diff --git a/src/serena/repl/facade.py b/src/serena/repl/facade.py index eb040b06..05417110 100644 --- a/src/serena/repl/facade.py +++ b/src/serena/repl/facade.py @@ -14,6 +14,8 @@ from dataclasses import dataclass from enum import Enum from typing import TYPE_CHECKING, Any, TypeVar +import typing_extensions + from serena.config.serena_config import ApiInclusionDefinition from serena.project import Project @@ -49,6 +51,41 @@ def format_signature(callable_: Callable[..., Any]) -> str: return format_annotation(str(inspect.signature(callable_))) +def extract_referenced_classes(annotation: Any) -> list[type]: + """ + :param annotation: a (resolved) type annotation + :return: the user-defined classes appearing in the annotation (recursively, e.g. in `list[X] | None`), in order of + appearance; builtins, typing constructs and classes from the standard library are excluded + """ + classes: list[type] = [] + + def visit(a: Any) -> None: + if isinstance(a, type): + module = getattr(a, "__module__", "") + if module not in ("builtins", "typing", "collections.abc", "abc") and not module.startswith("_") and a not in classes: + classes.append(a) + for arg in typing.get_args(a): + visit(arg) + + visit(annotation) + return classes + + +def get_annotated_classes(callable_: Callable[..., Any]) -> list[type]: + """ + :param callable_: a function or method + :return: the user-defined classes appearing in the annotations of its parameters and return type + """ + try: + hints = typing.get_type_hints(callable_) + except Exception: # unresolvable forward references + return [] + classes: list[type] = [] + for hint in hints.values(): + classes.extend(c for c in extract_referenced_classes(hint) if c not in classes) + return classes + + @dataclass class ReferencedType: """ @@ -82,33 +119,50 @@ class ReferencedType: names.difference_update(dir(RepresentableViaRenderer)) return sorted(n for n in names if not n.startswith("_") and not n.endswith("_")) - def get_referenced_type_names(self) -> list[str]: + def get_referenced_classes(self) -> list[type]: """ - :return: the names of the types appearing in the annotations of the described members (attributes, properties, - method parameters and return types), in order of appearance (each name at most once, excluding the type itself) + :return: the user-defined classes appearing in the annotations of the described members (attributes, properties, + method parameters and return types), in order of appearance (each class at most once, excluding the type itself) """ if self.is_enum(): return [] - annotations: list[str] = [] + classes: list[type] = [] type_hints = typing.get_type_hints(self.cls) + if self.is_typed_dict(): + for hint in type_hints.values(): + classes.extend(c for c in extract_referenced_classes(hint) if c is not self.cls and c not in classes) + return classes for member_name in self._get_member_names(): member = inspect.getattr_static(self.cls, member_name, None) if isinstance(member, property) and member.fget is not None: - annotations.append(format_annotation(inspect.signature(member.fget).return_annotation)) + found = get_annotated_classes(member.fget) elif inspect.isfunction(member): - annotations.append(format_signature(member)) + found = get_annotated_classes(member) elif member_name in type_hints: - annotations.append(format_annotation(type_hints[member_name])) - names: list[str] = [] - for annotation in annotations: - for name in re.findall(r"\b[A-Z]\w*", annotation): - if name != self.name and name not in names: - names.append(name) - return names + found = extract_referenced_classes(type_hints[member_name]) + else: + found = [] + classes.extend(c for c in found if c is not self.cls and c not in classes) + return classes def is_enum(self) -> bool: return isinstance(self.cls, type) and issubclass(self.cls, Enum) + def is_typed_dict(self) -> bool: + # NOTE: TypedDicts defined via typing_extensions are not recognised by typing.is_typeddict + return typing.is_typeddict(self.cls) or typing_extensions.is_typeddict(self.cls) + + def _describe_typed_dict(self) -> str: + parts = [f"type {self.name} (a dict with the following keys)"] + if self.cls.__doc__ and not self.cls.__doc__.startswith(self.name + "("): # NOTE: the default docstring is uninformative + parts.append(f" {inspect.cleandoc(self.cls.__doc__).replace(chr(10), chr(10) + ' ')}") + type_hints = typing.get_type_hints(self.cls) + member_names = self.members if self.members is not None else list(type_hints) + parts.append( + "keys:\n" + "\n".join(f" {name}: {format_annotation(type_hints[name])}" for name in member_names if name in type_hints) + ) + return "\n".join(parts) + "\n" + def _describe_enum(self) -> str: parts = [f"enum {self.name}"] if self.cls.__doc__: @@ -129,6 +183,8 @@ class ReferencedType: """ if self.is_enum(): return self._describe_enum() + if self.is_typed_dict(): + return self._describe_typed_dict() attributes: list[str] = [] methods: list[str] = [] type_hints = typing.get_type_hints(self.cls) @@ -335,6 +391,9 @@ class FacadeMethod: def __call__(self, *args: Any, **kwargs: Any) -> Any: return self._implementation(*args, **kwargs) + def get_implementation_(self) -> Callable[..., Any]: + return self._implementation + def get_referenced_return_types(self) -> list[ReferencedType]: """ :return: the types referenced by the facade which appear in the method's return type annotation @@ -508,8 +567,31 @@ class Facade: continue is_enabled = api_scope.is_method_enabled(facade.name, method_info) facade._add_method(FacadeMethod(facade, member, method_info, enabled=is_enabled)) + facade._discover_referenced_types() return facade + def _discover_referenced_types(self) -> None: + """ + Adds referenced types for all classes reachable (transitively) through the annotations of the facade's methods + and of the referenced types' members, such that every type an LLM may encounter can be documented. + Explicitly declared types take precedence (they may curate members and carry flags). + """ + # seed the worklist with the classes referenced by the declared types and by the methods + pending: list[type] = [] + for referenced_type in self._types.values(): + pending.extend(referenced_type.get_referenced_classes()) + for method in self._methods.values(): + pending.extend(get_annotated_classes(method.get_implementation_())) + + # add undeclared classes, following their references in turn + while pending: + cls = pending.pop(0) + if cls.__name__ in self._types: + continue + referenced_type = ReferencedType(cls) + self._types[cls.__name__] = referenced_type + pending.extend(referenced_type.get_referenced_classes()) + @property def name(self) -> str: return self._name diff --git a/src/serena/repl/repl.py b/src/serena/repl/repl.py index 1e55822b..4855fa8d 100644 --- a/src/serena/repl/repl.py +++ b/src/serena/repl/repl.py @@ -135,7 +135,7 @@ class SerenaReplEntrypoint: session.described_type_names.add(referenced_type.name) # append the referenced types (breadth-first), unless already described in this call or earlier in the session - pending = list(referenced_type.get_referenced_type_names()) + pending = [cls.__name__ for cls in referenced_type.get_referenced_classes()] while pending: type_name = pending.pop(0) if type_name in described_in_call: @@ -150,7 +150,7 @@ class SerenaReplEntrypoint: parts.append(contained_type.describe()) if session is not None: session.described_type_names.add(type_name) - pending.extend(contained_type.get_referenced_type_names()) + pending.extend(cls.__name__ for cls in contained_type.get_referenced_classes()) return "\n".join(parts) diff --git a/test/serena/test_repl_tool.py b/test/serena/test_repl_tool.py index 2dae1c5b..d61b0c46 100644 --- a/test/serena/test_repl_tool.py +++ b/test/serena/test_repl_tool.py @@ -73,9 +73,13 @@ class TestReplExecution: assert "get_name_path() -> str" in type_info and "iter_children()" in type_info assert "to_dict" not in type_info # not among the curated members - # types referenced by parameters are documented as well; enums are documented with their members - assert "enum SymbolKind" in type_info and "SymbolKind.Class = 5" in type_info # via iter_ancestors(up_to_symbol_kind) + # types reachable through annotations are documented without being declared: SymbolKind (a parameter type of + # iter_ancestors) is documented as an enum with its members, both transitively and on request + assert "enum SymbolKind" in type_info and "SymbolKind.Class = 5" in type_info assert "enum SymbolKind" in repl.execute('s.info("SymbolKind")') + + # TypedDicts reachable through annotations are documented with their keys + assert "keys:" in repl.execute('s.info("Diagnostic")') and "severity" in repl.execute('s.info("Diagnostic")') assert "represent()" not in repl.execute('s.info("LspSymbolCollection")') # the representation mechanism is not exposed def test_contained_types_are_documented_once_per_session(self, repl: SerenaRepl) -> None: From e6f0d7839cdcc30af370f7ec601452f20dd2cda6 Mon Sep 17 00:00:00 2001 From: Dominik Jain Date: Tue, 8 Sep 2026 19:56:49 +0200 Subject: [PATCH 28/53] Drop ReferencedType declarations that discovery renders redundant Only the three declarations carrying member curation remain (LanguageServerSymbol, ReplacementOccurrence, MatchedConsecutiveLines); the discovered type sets are unchanged. --- src/serena/repl/api/edit_api.py | 1 - src/serena/repl/api/fs_api.py | 3 --- src/serena/repl/api/jb_api.py | 7 +------ src/serena/repl/api/lsp_api.py | 5 ----- src/serena/repl/api/mem_api.py | 3 +-- src/serena/repl/api/shell_api.py | 4 ++-- src/serena/repl/facade.py | 10 ++++++---- 7 files changed, 10 insertions(+), 23 deletions(-) diff --git a/src/serena/repl/api/edit_api.py b/src/serena/repl/api/edit_api.py index adac2218..aa150e92 100644 --- a/src/serena/repl/api/edit_api.py +++ b/src/serena/repl/api/edit_api.py @@ -75,7 +75,6 @@ class EditApi(FacadeApi): name="edit", description="modifying content within existing files (independent of the language backend)", types=[ - ReferencedType(ReplacementPreview), ReferencedType( ReplacementOccurrence, members=["occurrence_id", "relative_path", "start_line", "end_line", "matched_text", "replacement", "is_ambiguous"], diff --git a/src/serena/repl/api/fs_api.py b/src/serena/repl/api/fs_api.py index 608c7c9a..09e0a54d 100644 --- a/src/serena/repl/api/fs_api.py +++ b/src/serena/repl/api/fs_api.py @@ -173,9 +173,6 @@ class FsApi(FacadeApi): name="fs", description="the project's files as units (as opposed to their content, see `edit`)", types=[ - ReferencedType(FileContent), - ReferencedType(DirectoryListing), - ReferencedType(PatternMatches), ReferencedType( MatchedConsecutiveLines, members=["source_file_path", "matched_lines", "start_line", "end_line", "to_display_string"] ), diff --git a/src/serena/repl/api/jb_api.py b/src/serena/repl/api/jb_api.py index 84e8cb35..9c8e196c 100644 --- a/src/serena/repl/api/jb_api.py +++ b/src/serena/repl/api/jb_api.py @@ -30,7 +30,7 @@ from serena.tools import ( ) from serena.util.text_utils import find_text_coordinates -from ..facade import FacadeApi, ReferencedType, facade_method +from ..facade import FacadeApi, facade_method from ..representable import JsonObject, JsonObjectRenderer, Renderer, RepresentableViaRenderer if TYPE_CHECKING: @@ -185,11 +185,6 @@ class JetBrainsApi(FacadeApi): agent, name="jb", description="operations on the codebase backed by the JetBrains IDE's code intelligence", - types=[ - ReferencedType(JetBrainsSymbolCollection), - ReferencedType(JetBrainsSymbolsOverview), - ReferencedType(JsonObject), - ], ) @contextmanager diff --git a/src/serena/repl/api/lsp_api.py b/src/serena/repl/api/lsp_api.py index fed32cee..0459f408 100644 --- a/src/serena/repl/api/lsp_api.py +++ b/src/serena/repl/api/lsp_api.py @@ -310,7 +310,6 @@ class LspApi(FacadeApi): name="lsp", description="symbol-level operations on the codebase backed by language servers", types=[ - ReferencedType(LspSymbolCollection), ReferencedType( LanguageServerSymbol, members=[ @@ -327,10 +326,6 @@ class LspApi(FacadeApi): "get_parent", ], ), - ReferencedType(LspSymbol), - ReferencedType(LspReferenceCollection), - ReferencedType(ReferenceInLanguageServerSymbol), - ReferencedType(LspDiagnostics), ], ) diff --git a/src/serena/repl/api/mem_api.py b/src/serena/repl/api/mem_api.py index 36bb2fdc..d6e41f76 100644 --- a/src/serena/repl/api/mem_api.py +++ b/src/serena/repl/api/mem_api.py @@ -18,7 +18,7 @@ from serena.tools import ( WriteMemoryTool, ) -from ..facade import FacadeApi, ReferencedType, facade_method +from ..facade import FacadeApi, facade_method from ..representable import Renderer, RepresentableViaRenderer if TYPE_CHECKING: @@ -60,7 +60,6 @@ class MemoryApi(FacadeApi): agent, name="mem", description="project memories, i.e. persistent notes for future tasks", - types=[ReferencedType(MemoryList)], ) def _get_memory_manager(self) -> MemoryManager: diff --git a/src/serena/repl/api/shell_api.py b/src/serena/repl/api/shell_api.py index 801b8e93..56ba2679 100644 --- a/src/serena/repl/api/shell_api.py +++ b/src/serena/repl/api/shell_api.py @@ -9,7 +9,7 @@ from typing import TYPE_CHECKING from serena.tools import ExecuteShellCommandTool from serena.util.shell import ShellCommandResult, execute_shell_command -from ..facade import FacadeApi, ReferencedType, facade_method +from ..facade import FacadeApi, facade_method from ..representable import Renderer, RepresentableViaRenderer if TYPE_CHECKING: @@ -53,7 +53,7 @@ class ShellCommandOutputRenderer(Renderer[ShellCommandOutput]): class ShellApi(FacadeApi): def __init__(self, agent: "SerenaAgent") -> None: - super().__init__(agent, name="shell", description="execution of shell commands", types=[ReferencedType(ShellCommandOutput)]) + super().__init__(agent, name="shell", description="execution of shell commands") @facade_method(can_edit=True, corresponding_tool=ExecuteShellCommandTool) def execute_shell_command( diff --git a/src/serena/repl/facade.py b/src/serena/repl/facade.py index 05417110..18ed06eb 100644 --- a/src/serena/repl/facade.py +++ b/src/serena/repl/facade.py @@ -89,8 +89,10 @@ def get_annotated_classes(callable_: Callable[..., Any]) -> list[type]: @dataclass class ReferencedType: """ - A type that is referenced by a facade's methods (returned by them or contained in their results), whose interface - the LLM can inspect via `info`. + A type that is referenced by a facade's methods (returned by them, contained in their results or used as a parameter + type), whose interface the LLM can inspect via `info`. + All types reachable through annotations are discovered automatically; an explicit declaration is needed only in order + to curate the type's presentation (the members to describe, inclusion in the facade's description). """ cls: type @@ -318,8 +320,8 @@ class FacadeApi(ABC): :param agent: the agent providing access to the project and its resources :param name: the attribute name under which the facade is accessible from the REPL entrypoint :param description: a one-line description of the functionality offered by the facade - :param types: the types referenced by the facade's methods (returned or contained in results) whose interface - the LLM shall be able to inspect + :param types: declarations for referenced types whose presentation is to be curated (see `ReferencedType`); + types reachable through annotations need not be declared in order to be documentable """ self._agent = agent self._name = name From 37dda683767383e5da395581b803e0232c10c32c Mon Sep 17 00:00:00 2001 From: Dominik Jain Date: Tue, 8 Sep 2026 20:34:06 +0200 Subject: [PATCH 29/53] Persist REPL variables and functions across executions within a session Code executions of a session now share a namespace (SerenaSession.repl_namespace), which serves as the globals of the executions: variables, functions, classes and imports bound at the top level of the submitted code persist across calls, like the cells of a notebook. The code is wrapped in a function at the AST level (preserving line numbers for error reporting), with the top-level names declared global. The entrypoint `s` is re-bound before every execution, such that persisted functions always use the current entrypoint (e.g. after a project switch). `s.vars()` lists the persisted items, `s.clear()` removes them; the tool description explains the persistence. The namespace is tied to the session's lifetime; the session registry additionally evicts sessions which have been idle for too long, releasing their namespaces. --- .serena/memories/repl.md | 12 ++- src/serena/repl/repl.py | 156 +++++++++++++++++++++++++++++---- src/serena/session.py | 31 ++++++- src/serena/tools/repl_tools.py | 5 ++ test/serena/test_repl_tool.py | 32 +++++++ 5 files changed, 214 insertions(+), 22 deletions(-) diff --git a/.serena/memories/repl.md b/.serena/memories/repl.md index f0d1f389..2a27e1e1 100644 --- a/.serena/memories/repl.md +++ b/.serena/memories/repl.md @@ -53,8 +53,16 @@ Code runs as a function body (`return` defines the result); a single expression - MCP provides no reliable session identification (newer protocol versions drop it), and clients keep a stdio server across conversations. Hence the REPL's session identity is LLM-supplied: `create_system_prompt` creates a `SerenaSession` and states its id; `serena_repl` takes a required `session_id`. `SessionRegistry` creates - unknown ids on demand (benign: at worst docs are repeated) and evicts LRU. Sessions survive REPL rebuilds. - Later: the persistent REPL namespace lives on the session. + unknown ids on demand (benign: at worst docs are repeated) and evicts LRU and idle (TTL) sessions. + Sessions survive REPL rebuilds. +- Persistence (notebook semantics): `SerenaSession.repl_namespace` is the globals of the session's executions; + names bound at the top level of submitted code (assignments, def/class, imports, loop/with targets, walrus) + persist across calls (an AST pass wraps the code in a function and declares those names `global`, preserving + line numbers). `s` is re-bound in the namespace before every execution, so persisted functions always use the + current entrypoint (no closure over `s`). `s.vars()`/`s.clear()` list/remove persisted items. Data is tied to + the session's lifetime (not cleared on REPL rebuild); stored facades/project objects may go stale. + Deferred idea if memory becomes an issue: hybrid — implicit items expire after N turns, explicit store + (e.g. `s.d`) for indefinite retention. - Only tools whose use presupposes having read the instructions may require the id. `activate_project` and `initial_instructions` may be called first and keep the existing MCP-context-derived session handling (prompt provision status); migrating that to LLM-supplied ids is a separate, future change. diff --git a/src/serena/repl/repl.py b/src/serena/repl/repl.py index 4855fa8d..0b8a4f2e 100644 --- a/src/serena/repl/repl.py +++ b/src/serena/repl/repl.py @@ -4,8 +4,9 @@ The REPL through which an LLM executes Python code against Serena's facades. # SPDX-License-Identifier: GPL-3.0-or-later +import ast import logging -import textwrap +import re import traceback from typing import Any @@ -29,6 +30,7 @@ class SerenaReplEntrypoint: """ self._facades: dict[str, Facade] = {} self._current_session: SerenaSession | None = None + self._current_namespace: dict[str, Any] | None = None registered_facade_names = [] for facade in facades: if api_scope.is_facade_enabled(facade.name): @@ -42,11 +44,50 @@ class SerenaReplEntrypoint: """ return [method for facade in self._facades.values() for method in facade.get_enabled_methods()] - def set_current_session_(self, session: SerenaSession | None) -> None: + def set_current_session_(self, session: SerenaSession | None, namespace: dict[str, Any] | None) -> None: """ :param session: the session on whose behalf code is being executed (None if no code is being executed) + :param namespace: the namespace of the execution (None if no code is being executed) """ self._current_session = session + self._current_namespace = namespace + + def _get_persisted_items(self) -> dict[str, Any]: + assert self._current_namespace is not None, "No code execution in progress" + return { + name: value + for name, value in self._current_namespace.items() + if name != SerenaRepl.ENTRYPOINT_NAME and SerenaRepl.is_persisted_name(name) + } + + def vars(self) -> str: + """ + Lists the variables and functions which persist in the session's namespace across executions. + + :return: the listing (name, type and a short representation per item) + """ + items = self._get_persisted_items() + if not items: + return "No persisted variables." + lines = [] + for name, value in items.items(): + summary = value.__name__ if callable(value) and hasattr(value, "__name__") else repr(value) + if len(summary) > 80: + summary = summary[:77] + "..." + lines.append(f"{name}: {type(value).__name__} = {summary}") + return "\n".join(lines) + + def clear(self) -> str: + """ + Removes all persisted variables and functions from the session's namespace. + + :return: a message indicating the number of removed items + """ + items = self._get_persisted_items() + assert self._current_namespace is not None + for name in items: + del self._current_namespace[name] + return f"Removed {len(items)} persisted item(s)." def _register(self, facade: Facade) -> None: if facade.name in self._facades: @@ -161,11 +202,25 @@ class SerenaRepl: The code is executed as the body of a function, such that the `return` statement defines the result; code consisting of a single expression is evaluated and its value is the result. + Names assigned at the top level of the code (variables, functions, classes, imports) persist in the session's + namespace across executions (like the cells of a notebook), which serves as the globals of the executions. + The entrypoint `s` is (re)bound in the namespace before every execution, such that persisted functions always + access the current entrypoint. """ SOURCE_NAME = "" ENTRYPOINT_NAME = "s" _FUNCTION_NAME = "__serena_repl_fn__" + _PERSISTED_NAME_PATTERN = re.compile(r"^(?!__)[A-Za-z_]\w*$") + + @classmethod + def is_persisted_name(cls, name: str) -> bool: + """ + :param name: a name in a session namespace + :return: whether the name denotes a persisted item of the LLM's (as opposed to an implementation detail + such as `__builtins__`) + """ + return cls._PERSISTED_NAME_PATTERN.match(name) is not None def __init__(self, facades: list[Facade], api_scope: ApiScope) -> None: """ @@ -205,21 +260,23 @@ class SerenaRepl: e.g. in tests), which determines e.g. which type documentation has already been provided :return: the representation of the code's result, or a description of the error if execution failed """ - self._entrypoint.set_current_session_(session) + namespace = session.repl_namespace if session is not None else {} + self._entrypoint.set_current_session_(session, namespace) try: - result = self._run(code) + result = self._run(code, namespace) except Exception as e: return self._format_error(e, code) finally: - self._entrypoint.set_current_session_(None) + self._entrypoint.set_current_session_(None, None) return self._represent(result) - def _run(self, code: str) -> Any: + def _run(self, code: str, namespace: dict[str, Any]) -> Any: """ - Runs the given code with the entrypoint bound, either as a single expression - or as the body of a function whose return value is the result. + Runs the given code in the given namespace (as globals) with the entrypoint bound, either as a single expression + or as the body of a function whose return value is the result and whose top-level assignments are made global + (such that they persist in the namespace). """ - namespace: dict[str, Any] = {self.ENTRYPOINT_NAME: self._entrypoint} + namespace[self.ENTRYPOINT_NAME] = self._entrypoint # try to evaluate the code as a single expression try: @@ -229,10 +286,78 @@ class SerenaRepl: if compiled is not None: return eval(compiled, namespace) - # otherwise execute the code as the body of a function - source = f"def {self._FUNCTION_NAME}({self.ENTRYPOINT_NAME}):\n" + textwrap.indent(code, " ") - exec(compile(source, self.SOURCE_NAME, "exec"), namespace) - return namespace[self._FUNCTION_NAME](self._entrypoint) + # otherwise execute the code as the body of a function, declaring the names assigned at the top level as global. + # The function is constructed at the AST level, such that the line numbers of the code are preserved. + module = ast.parse(code, self.SOURCE_NAME) + body: list[ast.stmt] = list(module.body) + assigned_names = self._collect_top_level_assigned_names(body) + if assigned_names: + body.insert(0, ast.Global(names=sorted(assigned_names))) + function = ast.FunctionDef( + name=self._FUNCTION_NAME, + args=ast.arguments(posonlyargs=[], args=[], kwonlyargs=[], kw_defaults=[], defaults=[]), + body=body, + decorator_list=[], + returns=None, + ) + wrapper = ast.fix_missing_locations(ast.Module(body=[function], type_ignores=[])) + exec(compile(wrapper, self.SOURCE_NAME, "exec"), namespace) + try: + return namespace[self._FUNCTION_NAME]() + finally: + del namespace[self._FUNCTION_NAME] + + @classmethod + def _collect_top_level_assigned_names(cls, statements: list[ast.stmt]) -> set[str]: + """ + :param statements: the top-level statements of the code + :return: the names bound by the statements (assignment targets, function/class definitions, imports, + loop/with targets, deletions and walrus assignments outside of nested scopes) + """ + names: set[str] = set() + + def add_target(target: ast.expr) -> None: + if isinstance(target, ast.Name): + names.add(target.id) + elif isinstance(target, ast.Tuple | ast.List): + for element in target.elts: + add_target(element) + elif isinstance(target, ast.Starred): + add_target(target.value) + + def add_walrus_targets(node: ast.AST) -> None: + # walrus assignments bind in the enclosing scope, unless within a nested scope + for child in ast.iter_child_nodes(node): + if isinstance(child, ast.FunctionDef | ast.AsyncFunctionDef | ast.ClassDef | ast.Lambda): + continue + if isinstance(child, ast.NamedExpr): + add_target(child.target) + add_walrus_targets(child) + + for statement in statements: + match statement: + case ast.Assign(targets=targets): + for target in targets: + add_target(target) + case ast.AnnAssign(target=target) | ast.AugAssign(target=target): + add_target(target) + case ast.FunctionDef(name=name) | ast.AsyncFunctionDef(name=name) | ast.ClassDef(name=name): + names.add(name) + case ast.Import(names=aliases) | ast.ImportFrom(names=aliases): + for alias in aliases: + if alias.name != "*": + names.add(alias.asname or alias.name.split(".")[0]) + case ast.For(target=target) | ast.AsyncFor(target=target): + add_target(target) + case ast.With(items=items) | ast.AsyncWith(items=items): + for item in items: + if item.optional_vars is not None: + add_target(item.optional_vars) + case ast.Delete(targets=targets): + for target in targets: + add_target(target) + add_walrus_targets(statement) + return names def _format_error(self, e: Exception, code: str) -> str: """ @@ -248,13 +373,12 @@ class SerenaRepl: # report syntax errors in the executed code (which carry no traceback frames of their own) if isinstance(e, SyntaxError) and e.filename == self.SOURCE_NAME and e.lineno is not None: - return f"SyntaxError: {e.msg}\n" + location_line(e.lineno - 1) # undo the function header offset + return f"SyntaxError: {e.msg}\n" + location_line(e.lineno) # report runtime errors, locating them within the executed code location_lines = [] for frame in traceback.extract_tb(e.__traceback__): if frame.filename != self.SOURCE_NAME or frame.lineno is None: continue - line_number = frame.lineno - 1 if frame.name == self._FUNCTION_NAME else frame.lineno # undo the function header offset - location_lines.append(location_line(line_number)) + location_lines.append(location_line(frame.lineno)) return "\n".join([f"{type(e).__name__}: {e}", *location_lines]) diff --git a/src/serena/session.py b/src/serena/session.py index b30ce411..f08be716 100644 --- a/src/serena/session.py +++ b/src/serena/session.py @@ -5,7 +5,9 @@ Client sessions (conversations) and their state. import logging import secrets +import time from collections import OrderedDict +from typing import Any log = logging.getLogger(__name__) @@ -22,19 +24,28 @@ class SerenaSession: self.session_id = session_id self.described_type_names: set[str] = set() """the names of the REPL's result types whose documentation has already been provided in this session""" + self.repl_namespace: dict[str, Any] = {} + """ + the namespace (globals) of the session's REPL code executions: variables and functions defined at the top level + of executed code persist here across executions for the lifetime of the session + """ + self.last_access_time = time.time() class SessionRegistry: """ - Holds the sessions of an agent, creating them on demand and evicting the least recently used ones - if their number exceeds the limit (session ids being supplied by LLMs, the set of ids is not controlled). + Holds the sessions of an agent, creating them on demand and evicting sessions which have been idle for too long + as well as the least recently used ones if their number exceeds the limit (session ids being supplied by LLMs, + the set of ids is not controlled). """ - def __init__(self, max_sessions: int = 100) -> None: + def __init__(self, max_sessions: int = 100, idle_ttl_seconds: float = 6 * 3600) -> None: """ :param max_sessions: the maximum number of sessions to keep + :param idle_ttl_seconds: the time after which an idle session is evicted (releasing its REPL namespace) """ self._max_sessions = max_sessions + self._idle_ttl_seconds = idle_ttl_seconds self._sessions: OrderedDict[str, SerenaSession] = OrderedDict() def create_session(self) -> SerenaSession: @@ -49,6 +60,7 @@ class SessionRegistry: :return: the session, which is created if it is unknown (an unknown id may e.g. stem from a session that has been evicted or from an earlier run of the server) """ + self._evict_idle_sessions() session = self._sessions.get(session_id) if session is None: session = SerenaSession(session_id) @@ -56,7 +68,18 @@ class SessionRegistry: log.info("Created session %s (%d sessions)", session_id, len(self._sessions)) while len(self._sessions) > self._max_sessions: evicted_id, _ = self._sessions.popitem(last=False) - log.info("Evicted session %s", evicted_id) + log.info("Evicted session %s (session limit)", evicted_id) else: self._sessions.move_to_end(session_id) + session.last_access_time = time.time() return session + + def _evict_idle_sessions(self) -> None: + # sessions are ordered by last access, so the idle ones are at the front + now = time.time() + while self._sessions: + oldest_id, oldest = next(iter(self._sessions.items())) + if now - oldest.last_access_time <= self._idle_ttl_seconds: + break + del self._sessions[oldest_id] + log.info("Evicted session %s (idle)", oldest_id) diff --git a/src/serena/tools/repl_tools.py b/src/serena/tools/repl_tools.py index b3af1856..f47bf82f 100644 --- a/src/serena/tools/repl_tools.py +++ b/src/serena/tools/repl_tools.py @@ -34,6 +34,11 @@ class SerenaReplTool(Tool, ToolMarkerOptional, ToolMarkerBeta): Returned objects are rendered in a form suitable for you; lists are rendered element-wise. Returned strings are passed through unchanged. + Persistence: variables, functions and classes defined at the top level of your code persist across calls + within your session (like the cells of a notebook), so you can reuse results and define helper functions once. + `s.vars()` lists the persisted items, `s.clear()` removes them. Do not store facades (`s.`) in + variables; access them via `s` at call time. Do not keep large results longer than needed. + :param session: your Serena session id, as provided in Serena's instructions (call `initial_instructions` if you do not have one) :param code: the Python code to execute :return: the representation of the returned value, or the error if execution failed diff --git a/test/serena/test_repl_tool.py b/test/serena/test_repl_tool.py index d61b0c46..fdd46b62 100644 --- a/test/serena/test_repl_tool.py +++ b/test/serena/test_repl_tool.py @@ -8,6 +8,7 @@ from unittest.mock import MagicMock import pytest from serena.config.serena_config import ApiInclusionDefinition +from serena.repl.api.edit_api import EditApi from serena.repl.api.lsp_api import LspApi from serena.repl.facade import ApiScope, Facade, FacadeApi, FacadeMethodInfo, facade_method from serena.repl.repl import SerenaRepl @@ -43,6 +44,37 @@ class TestReplExecution: assert result.startswith("SyntaxError") assert "line 2" in result + def test_top_level_names_persist_within_session(self, repl: SerenaRepl) -> None: + session = SerenaSession("a") + repl.execute("x = 20\ndef double(v):\n return 2 * v\nfor i in range(3):\n pass\nimport os as os_module", session) + assert repl.execute("return double(x) + i", session) == "42" + assert repl.execute("os_module.sep is not None", session) == "True" + + # local variables of nested scopes do not persist; persisted items can be listed and cleared + assert "v" not in repl.execute("s.vars()", session) + listing = repl.execute("s.vars()", session) + assert "x: int = 20" in listing and "double: function" in listing + assert repl.execute("s.clear()", session) == "Removed 4 persisted item(s)." + assert repl.execute("s.vars()", session) == "No persisted variables." + assert "NameError" in repl.execute("return x", session) + + def test_sessions_have_separate_namespaces(self, repl: SerenaRepl) -> None: + session_a = SerenaSession("a") + repl.execute("x = 1", session_a) + assert "NameError" in repl.execute("return x", SerenaSession("b")) + assert repl.execute("return x", session_a) == "1" + # ... and execution without a session persists nothing + repl.execute("y = 1") + assert "NameError" in repl.execute("return y") + + def test_persisted_functions_use_the_current_entrypoint(self) -> None: + # a function defined against one REPL instance uses the entrypoint of the REPL that later calls it + session = SerenaSession("a") + SerenaRepl([Facade.from_api(LspApi(MagicMock()), ApiScope())], ApiScope()).execute("def facades():\n return s.info()", session) + rebuilt_repl = SerenaRepl([Facade.from_api(EditApi(MagicMock()), ApiScope())], ApiScope()) + overview = rebuilt_repl.execute("return facades()", session) + assert "s.edit" in overview and "s.lsp" not in overview + def test_facade_discovery(self, repl: SerenaRepl) -> None: overview = repl.execute("s.info()") assert "s.lsp" in overview From 5cae1d8ddf2317527f731d546f80bacf00e2312a Mon Sep 17 00:00:00 2001 From: Dominik Jain Date: Tue, 8 Sep 2026 20:38:29 +0200 Subject: [PATCH 30/53] Explain max_answer_chars centrally in the REPL tool description The parameter is documented once in the tool description instead of in every facade method. --- src/serena/repl/api/edit_api.py | 2 -- src/serena/repl/api/fs_api.py | 6 ------ src/serena/repl/api/jb_api.py | 9 --------- src/serena/repl/api/lsp_api.py | 6 ------ src/serena/repl/api/shell_api.py | 2 -- src/serena/tools/repl_tools.py | 5 +++++ 6 files changed, 5 insertions(+), 25 deletions(-) diff --git a/src/serena/repl/api/edit_api.py b/src/serena/repl/api/edit_api.py index aa150e92..be09770e 100644 --- a/src/serena/repl/api/edit_api.py +++ b/src/serena/repl/api/edit_api.py @@ -169,8 +169,6 @@ class EditApi(FacadeApi): :param expected_count: optional guard for calls without occurrence_ids: the number of occurrences you expect to be replaced. If the actual count differs, nothing is changed and an error containing the prospective changes is raised. -1 disables the guard. - :param max_answer_chars: if the rendered output exceeds this many characters, a shortened version is - used. -1 uses the configured default. :return: in a dry run, the prospective changes (`ReplacementPreview`); otherwise a summary of the applied replacements """ replacement = MultiFileReplacement( diff --git a/src/serena/repl/api/fs_api.py b/src/serena/repl/api/fs_api.py index 09e0a54d..b457035e 100644 --- a/src/serena/repl/api/fs_api.py +++ b/src/serena/repl/api/fs_api.py @@ -187,8 +187,6 @@ class FsApi(FacadeApi): :param relative_path: the relative path to the file to read :param start_line: the 0-based index of the first line to be retrieved, negative values count from the end of the file. :param end_line: the 0-based index of the last line to be retrieved (inclusive). If None, read until the end of the file. - :param max_answer_chars: if the rendered content is longer than this number of characters, no content is rendered; - -1 for the configured default. Don't adjust unless there is really no other way to get the content required for the task. :return: the content """ project = self._get_project() @@ -237,8 +235,6 @@ class FsApi(FacadeApi): :param relative_path: the relative path to the directory to list; pass "." to scan the project root :param recursive: whether to scan subdirectories recursively :param skip_ignored_files: whether to skip files and directories that are ignored - :param max_answer_chars: if the rendered listing is longer than this number of characters, no content is rendered; - -1 for the configured default. Don't adjust unless there is really no other way to get the content required for the task. :return: the listing """ project = self._get_project() @@ -313,8 +309,6 @@ class FsApi(FacadeApi): (useful when looking for class/method definitions); otherwise also search non-code files. :param skip_ignored_files: whether to skip ignored sub-paths (default: True) :param multiline: whether to apply multi-line matching (default: True), enabling the flags re.DOTALL and re.MULTILINE - :param max_answer_chars: if the rendered output exceeds this many characters, a progressively shortened summary is - rendered instead. -1 for the configured default. :return: the matches, rendered as a mapping from file paths to matched consecutive lines (0-based line numbers) """ project = self._get_project() diff --git a/src/serena/repl/api/jb_api.py b/src/serena/repl/api/jb_api.py index 9c8e196c..2ed024f4 100644 --- a/src/serena/repl/api/jb_api.py +++ b/src/serena/repl/api/jb_api.py @@ -245,7 +245,6 @@ class JetBrainsApi(FacadeApi): :param search_deps: If True, also search in project dependencies (e.g., libraries). :param max_matches: Maximum number of permitted matches. If exceeded, an error containing a shortened result is raised, which allows refining the search. -1 (default) means no limit. Set to 1 if you search for a single symbol. - :param max_answer_chars: max characters for the result (-1 for default). If exceeded, no content/a shortened result is returned. :return: the symbols matching the pattern """ if name_path_pattern.replace("*", "").replace("/", "") == "": @@ -292,7 +291,6 @@ class JetBrainsApi(FacadeApi): :param relative_path: the relative path to the file containing the symbol (must be a file, not a directory) Note: for external dependencies, this must be an identifier starting with ``) in From 1313bb5fc847fe19a091a287eac3ede70ac62bb6 Mon Sep 17 00:00:00 2001 From: Dominik Jain Date: Tue, 8 Sep 2026 20:57:23 +0200 Subject: [PATCH 31/53] Execute REPL code like a notebook cell; the last expression is the result Code is now executed at module level in the session's namespace (statements executed, a trailing expression evaluated as the result), replacing the function wrapper with global declarations that persistence had required. This yields notebook semantics throughout: top-level bindings persist by construction, functions resolve names from the session namespace, and `return` is no longer used (a top-level `return` yields a syntax error with a hint). Tool description and tests updated. --- .serena/memories/repl.md | 14 ++--- src/serena/repl/repl.py | 109 +++++++-------------------------- src/serena/tools/repl_tools.py | 9 ++- test/serena/test_repl_tool.py | 29 +++++---- 4 files changed, 51 insertions(+), 110 deletions(-) diff --git a/.serena/memories/repl.md b/.serena/memories/repl.md index 2a27e1e1..0b67db46 100644 --- a/.serena/memories/repl.md +++ b/.serena/memories/repl.md @@ -1,8 +1,9 @@ # REPL (`serena.repl`) Alternative interaction paradigm: one tool (`serena_repl`) executes Python code against entrypoint `s`, -whose attributes are facades (`s.lsp`, `s.edit`, `s.fs`, `s.mem`, `s.shell`, `s.jb`). -Code runs as a function body (`return` defines the result); a single expression is evaluated directly. +whose attributes are facades (`s.lsp`, `s.edit`, `s.fs`, `s.mem`, `s.shell`, `s.jb`, `s.cfg`). +Code runs like a notebook cell (module-level exec in the session namespace); the value of a trailing +expression is the result. No `return` (a top-level `return` yields a SyntaxError with a hint). ## Structure @@ -55,11 +56,10 @@ Code runs as a function body (`return` defines the result); a single expression a `SerenaSession` and states its id; `serena_repl` takes a required `session_id`. `SessionRegistry` creates unknown ids on demand (benign: at worst docs are repeated) and evicts LRU and idle (TTL) sessions. Sessions survive REPL rebuilds. -- Persistence (notebook semantics): `SerenaSession.repl_namespace` is the globals of the session's executions; - names bound at the top level of submitted code (assignments, def/class, imports, loop/with targets, walrus) - persist across calls (an AST pass wraps the code in a function and declares those names `global`, preserving - line numbers). `s` is re-bound in the namespace before every execution, so persisted functions always use the - current entrypoint (no closure over `s`). `s.vars()`/`s.clear()` list/remove persisted items. Data is tied to +- Persistence (notebook semantics): `SerenaSession.repl_namespace` is the globals of the session's executions, + which run at module level (statements exec'd, a trailing expression eval'd; line numbers preserved), so all + top-level bindings persist across calls. `s` is re-bound in the namespace before every execution, so persisted + functions always use the current entrypoint. `s.vars()`/`s.clear()` list/remove persisted items. Data is tied to the session's lifetime (not cleared on REPL rebuild); stored facades/project objects may go stale. Deferred idea if memory becomes an issue: hybrid — implicit items expire after N turns, explicit store (e.g. `s.d`) for indefinite retention. diff --git a/src/serena/repl/repl.py b/src/serena/repl/repl.py index 0b8a4f2e..b631777b 100644 --- a/src/serena/repl/repl.py +++ b/src/serena/repl/repl.py @@ -200,17 +200,15 @@ class SerenaRepl: Executes Python code submitted by an LLM, binding the configured facades to the entrypoint object `s` and rendering the result of the execution as a string for the LLM. - The code is executed as the body of a function, such that the `return` statement defines the result; - code consisting of a single expression is evaluated and its value is the result. - Names assigned at the top level of the code (variables, functions, classes, imports) persist in the session's - namespace across executions (like the cells of a notebook), which serves as the globals of the executions. + The code is executed like the cell of a notebook: it is executed at module level in the session's namespace, + such that the names it binds (variables, functions, classes, imports) persist across executions, and if its last + statement is an expression, the expression's value is the result of the execution. The entrypoint `s` is (re)bound in the namespace before every execution, such that persisted functions always access the current entrypoint. """ SOURCE_NAME = "" ENTRYPOINT_NAME = "s" - _FUNCTION_NAME = "__serena_repl_fn__" _PERSISTED_NAME_PATTERN = re.compile(r"^(?!__)[A-Za-z_]\w*$") @classmethod @@ -272,92 +270,26 @@ class SerenaRepl: def _run(self, code: str, namespace: dict[str, Any]) -> Any: """ - Runs the given code in the given namespace (as globals) with the entrypoint bound, either as a single expression - or as the body of a function whose return value is the result and whose top-level assignments are made global - (such that they persist in the namespace). + Runs the given code at module level in the given namespace (as globals) with the entrypoint bound. + + :return: the value of the code's last statement if it is an expression, None otherwise """ namespace[self.ENTRYPOINT_NAME] = self._entrypoint - - # try to evaluate the code as a single expression - try: - compiled = compile(code, self.SOURCE_NAME, "eval") - except SyntaxError: - compiled = None - if compiled is not None: - return eval(compiled, namespace) - - # otherwise execute the code as the body of a function, declaring the names assigned at the top level as global. - # The function is constructed at the AST level, such that the line numbers of the code are preserved. module = ast.parse(code, self.SOURCE_NAME) - body: list[ast.stmt] = list(module.body) - assigned_names = self._collect_top_level_assigned_names(body) - if assigned_names: - body.insert(0, ast.Global(names=sorted(assigned_names))) - function = ast.FunctionDef( - name=self._FUNCTION_NAME, - args=ast.arguments(posonlyargs=[], args=[], kwonlyargs=[], kw_defaults=[], defaults=[]), - body=body, - decorator_list=[], - returns=None, - ) - wrapper = ast.fix_missing_locations(ast.Module(body=[function], type_ignores=[])) - exec(compile(wrapper, self.SOURCE_NAME, "exec"), namespace) - try: - return namespace[self._FUNCTION_NAME]() - finally: - del namespace[self._FUNCTION_NAME] - @classmethod - def _collect_top_level_assigned_names(cls, statements: list[ast.stmt]) -> set[str]: - """ - :param statements: the top-level statements of the code - :return: the names bound by the statements (assignment targets, function/class definitions, imports, - loop/with targets, deletions and walrus assignments outside of nested scopes) - """ - names: set[str] = set() + # separate a trailing expression, whose value is the result + statements = module.body + trailing_expression: ast.expr | None = None + if statements and isinstance(statements[-1], ast.Expr): + trailing_expression = statements[-1].value + statements = statements[:-1] - def add_target(target: ast.expr) -> None: - if isinstance(target, ast.Name): - names.add(target.id) - elif isinstance(target, ast.Tuple | ast.List): - for element in target.elts: - add_target(element) - elif isinstance(target, ast.Starred): - add_target(target.value) - - def add_walrus_targets(node: ast.AST) -> None: - # walrus assignments bind in the enclosing scope, unless within a nested scope - for child in ast.iter_child_nodes(node): - if isinstance(child, ast.FunctionDef | ast.AsyncFunctionDef | ast.ClassDef | ast.Lambda): - continue - if isinstance(child, ast.NamedExpr): - add_target(child.target) - add_walrus_targets(child) - - for statement in statements: - match statement: - case ast.Assign(targets=targets): - for target in targets: - add_target(target) - case ast.AnnAssign(target=target) | ast.AugAssign(target=target): - add_target(target) - case ast.FunctionDef(name=name) | ast.AsyncFunctionDef(name=name) | ast.ClassDef(name=name): - names.add(name) - case ast.Import(names=aliases) | ast.ImportFrom(names=aliases): - for alias in aliases: - if alias.name != "*": - names.add(alias.asname or alias.name.split(".")[0]) - case ast.For(target=target) | ast.AsyncFor(target=target): - add_target(target) - case ast.With(items=items) | ast.AsyncWith(items=items): - for item in items: - if item.optional_vars is not None: - add_target(item.optional_vars) - case ast.Delete(targets=targets): - for target in targets: - add_target(target) - add_walrus_targets(statement) - return names + # execute the statements, then evaluate the trailing expression (both retain the original line numbers) + if statements: + exec(compile(ast.Module(body=statements, type_ignores=[]), self.SOURCE_NAME, "exec"), namespace) + if trailing_expression is not None: + return eval(compile(ast.Expression(body=trailing_expression), self.SOURCE_NAME, "eval"), namespace) + return None def _format_error(self, e: Exception, code: str) -> str: """ @@ -373,7 +305,10 @@ class SerenaRepl: # report syntax errors in the executed code (which carry no traceback frames of their own) if isinstance(e, SyntaxError) and e.filename == self.SOURCE_NAME and e.lineno is not None: - return f"SyntaxError: {e.msg}\n" + location_line(e.lineno) + message = f"SyntaxError: {e.msg}\n" + location_line(e.lineno) + if "return" in (e.msg or "") and "outside function" in e.msg: + message += "\nNote: the code is executed like a notebook cell; the value of the last expression is the result (do not use `return`)." + return message # report runtime errors, locating them within the executed code location_lines = [] diff --git a/src/serena/tools/repl_tools.py b/src/serena/tools/repl_tools.py index 72633ae5..04b6895d 100644 --- a/src/serena/tools/repl_tools.py +++ b/src/serena/tools/repl_tools.py @@ -27,12 +27,11 @@ class SerenaReplTool(Tool, ToolMarkerOptional, ToolMarkerBeta): `s.info("")` documents the facade's operations only, not their result types. Result types are given in the method listing below (`method -> Type`); request their documentation via `s.info("")`, which includes the types they contain, ONLY if you intend to process results in code (filter, aggregate, chain - calls). If you simply want the result, return it directly: returned objects are rendered for you. + calls). If you simply want the result, make it the last expression: results are rendered for you. - The code is executed as the body of a function, so use `return` to define the result; - a single expression is evaluated and its value returned directly. - Returned objects are rendered in a form suitable for you; lists are rendered element-wise. - Returned strings are passed through unchanged. + The code is executed like a notebook cell: if its last statement is an expression, the expression's value is + the result (do not use `return`). Results are rendered in a form suitable for you; lists are rendered + element-wise, strings are passed through unchanged. Output size: methods with a `max_answer_chars` parameter limit the size of the rendered result (-1 uses the configured default). If the limit is exceeded, a shortened result (or no content) is rendered instead; diff --git a/test/serena/test_repl_tool.py b/test/serena/test_repl_tool.py index fdd46b62..4b7b348e 100644 --- a/test/serena/test_repl_tool.py +++ b/test/serena/test_repl_tool.py @@ -25,14 +25,21 @@ class TestReplExecution: def repl(self) -> SerenaRepl: return SerenaRepl([Facade.from_api(LspApi(MagicMock()), ApiScope())], ApiScope()) - def test_return_statement_defines_result(self, repl: SerenaRepl) -> None: - assert repl.execute("x = 20\ny = 22\nreturn x + y") == "42" + def test_last_expression_defines_result(self, repl: SerenaRepl) -> None: + assert repl.execute("x = 20\ny = 22\nx + y") == "42" + assert repl.execute("x = 20\ny = 22") == "None" # no trailing expression + assert repl.execute("") == "None" + + def test_return_yields_a_hint(self, repl: SerenaRepl) -> None: + result = repl.execute("x = 1\nreturn x") + assert result.startswith("SyntaxError") and "line 2" in result + assert "last expression is the result" in result def test_single_expression_is_evaluated(self, repl: SerenaRepl) -> None: assert repl.execute("1 + 2") == "3" def test_list_is_rendered_element_wise(self, repl: SerenaRepl) -> None: - assert repl.execute('return ["a", "b"]') == "a\nb" + assert repl.execute('["a", "b"]') == "a\nb" def test_error_reports_type_message_and_line(self, repl: SerenaRepl) -> None: result = repl.execute("x = 1\nraise ValueError('boom')") @@ -47,7 +54,7 @@ class TestReplExecution: def test_top_level_names_persist_within_session(self, repl: SerenaRepl) -> None: session = SerenaSession("a") repl.execute("x = 20\ndef double(v):\n return 2 * v\nfor i in range(3):\n pass\nimport os as os_module", session) - assert repl.execute("return double(x) + i", session) == "42" + assert repl.execute("double(x) + i", session) == "42" assert repl.execute("os_module.sep is not None", session) == "True" # local variables of nested scopes do not persist; persisted items can be listed and cleared @@ -56,23 +63,23 @@ class TestReplExecution: assert "x: int = 20" in listing and "double: function" in listing assert repl.execute("s.clear()", session) == "Removed 4 persisted item(s)." assert repl.execute("s.vars()", session) == "No persisted variables." - assert "NameError" in repl.execute("return x", session) + assert "NameError" in repl.execute("x", session) def test_sessions_have_separate_namespaces(self, repl: SerenaRepl) -> None: session_a = SerenaSession("a") repl.execute("x = 1", session_a) - assert "NameError" in repl.execute("return x", SerenaSession("b")) - assert repl.execute("return x", session_a) == "1" + assert "NameError" in repl.execute("x", SerenaSession("b")) + assert repl.execute("x", session_a) == "1" # ... and execution without a session persists nothing repl.execute("y = 1") - assert "NameError" in repl.execute("return y") + assert "NameError" in repl.execute("y") def test_persisted_functions_use_the_current_entrypoint(self) -> None: # a function defined against one REPL instance uses the entrypoint of the REPL that later calls it session = SerenaSession("a") SerenaRepl([Facade.from_api(LspApi(MagicMock()), ApiScope())], ApiScope()).execute("def facades():\n return s.info()", session) rebuilt_repl = SerenaRepl([Facade.from_api(EditApi(MagicMock()), ApiScope())], ApiScope()) - overview = rebuilt_repl.execute("return facades()", session) + overview = rebuilt_repl.execute("facades()", session) assert "s.edit" in overview and "s.lsp" not in overview def test_facade_discovery(self, repl: SerenaRepl) -> None: @@ -298,13 +305,13 @@ class TestLspFacade: session_id = agent.create_session().session_id # a returned collection is rendered, identifying the symbol and its file - rendered = tool.apply(session_id, 'return s.lsp.find_symbol("create_user")') + rendered = tool.apply(session_id, 's.lsp.find_symbol("create_user")') assert "create_user" in rendered assert "services.py" in rendered # the underlying symbols are accessible from code, e.g. to retrieve a body without rendering the collection body = tool.apply( session_id, - f'result = s.lsp.find_symbol("create_user", relative_path={self._SERVICES_FILE!r})\nreturn result.symbols[0].body', + f'result = s.lsp.find_symbol("create_user", relative_path={self._SERVICES_FILE!r})\nresult.symbols[0].body', ) assert body.startswith("def create_user") From 0706e3c98eba1fb39130156f83083e67822cc9c6 Mon Sep 17 00:00:00 2001 From: Dominik Jain Date: Tue, 8 Sep 2026 21:56:58 +0200 Subject: [PATCH 32/53] Add external project access to the REPL (ext facade, remote execution of LSP operations) Introduce the ext facade with list_projects and project_context: within `with s.ext.project_context(name):`, the facades operate read-only on the given external project. The agent's active project is switched temporarily, and methods marked with `@facade_method(uses_project_server=True)` (all lsp methods) are executed in the project server when the LSP backend is active (with JetBrains, the IDE serves all projects, so they run locally). The project server gains a /call_facade_method route executing a facade method on its own REPL facades and returning the pickled result; arguments are transferred as JSON. Editing methods are refused, and contexts cannot be nested. This replaces the query_project tools in the REPL. Result objects are now self-contained (picklable): renderers no longer hold the agent (only the default length limit; the constructor interface is unchanged), LSP results carry eagerly retrieved symbol info and reference contexts instead of renderers accessing the language server or project. --- .serena/memories/repl.md | 8 ++ src/serena/agent.py | 2 + src/serena/project_server.py | 69 +++++++++++++- src/serena/repl/api/ext_api.py | 93 ++++++++++++++++++ src/serena/repl/api/lsp_api.py | 131 ++++++++++++++++---------- src/serena/repl/external_project.py | 51 ++++++++++ src/serena/repl/facade.py | 33 ++++++- src/serena/repl/repl.py | 20 ++++ src/serena/repl/representable.py | 9 +- test/serena/test_external_projects.py | 107 +++++++++++++++++++++ test/serena/test_repl_tool.py | 25 +++++ 11 files changed, 492 insertions(+), 56 deletions(-) create mode 100644 src/serena/repl/api/ext_api.py create mode 100644 src/serena/repl/external_project.py create mode 100644 test/serena/test_external_projects.py diff --git a/.serena/memories/repl.md b/.serena/memories/repl.md index 0b67db46..86c8afb4 100644 --- a/.serena/memories/repl.md +++ b/.serena/memories/repl.md @@ -90,6 +90,14 @@ expression is the result. No `return` (a top-level `return` yields a SyntaxError - Python code can always modify the system; the REPL tool is inherently fully privileged, regardless of facade scope or the project's `read_only` setting (which only makes Serena's own API refuse edits). A "read-only REPL" is not feasible and must not be promised. +- External projects (`s.ext`): `list_projects()`, `project_context(name)` (a `with`-able context; not nestable). + Within it, the agent's active project is temporarily switched (`active_project_context`) and the facades are + read-only (`can_edit` methods raise). Methods marked `@facade_method(uses_project_server=True)` (all of `lsp`) + are executed in the project server via `/call_facade_method` ({facade, method, args, kwargs} as JSON, result + pickled; the server is a trusted local process) when the LSP backend is active; with JetBrains they run locally + (the IDE serves all projects). Result objects must be self-contained/picklable: renderers hold no agent (only the + default length limit), LSP results carry eagerly retrieved info and reference contexts, no lambdas in output + params. Replaces the query_project/list_queryable_projects tools in the REPL. - Project activation (activate_project) and initial_instructions stay tool-only (activation rebuilds the REPL); Serena's configuration/session state (config overview, dashboard; later e.g. modes) lives in the `cfg` facade. Computed conditions (read-only project, dashboard not openable) are applied to the API scope in diff --git a/src/serena/agent.py b/src/serena/agent.py index 4e8d16ad..2b0de256 100644 --- a/src/serena/agent.py +++ b/src/serena/agent.py @@ -51,6 +51,7 @@ from serena.project import Project from serena.prompt_factory import SerenaPromptFactory from serena.repl.api.cfg_api import ConfigApi from serena.repl.api.edit_api import EditApi +from serena.repl.api.ext_api import ExternalProjectsApi from serena.repl.api.fs_api import FsApi from serena.repl.api.jb_api import JetBrainsApi from serena.repl.api.lsp_api import LspApi @@ -1326,6 +1327,7 @@ class SerenaAgent: Facade.from_api(EditApi(self), api_scope), Facade.from_api(MemoryApi(self), api_scope), Facade.from_api(ShellApi(self), api_scope), + Facade.from_api(ExternalProjectsApi(self), api_scope), ] if self._language_backend.is_lsp(): facades.append(Facade.from_api(LspApi(self), api_scope)) diff --git a/src/serena/project_server.py b/src/serena/project_server.py index 279f8523..4deaa7c3 100644 --- a/src/serena/project_server.py +++ b/src/serena/project_server.py @@ -2,11 +2,12 @@ import json import logging +import pickle import threading -from typing import TYPE_CHECKING +from typing import TYPE_CHECKING, Any import requests as requests_lib -from flask import Flask, request +from flask import Flask, Response, request from pydantic import BaseModel from sensai.util.logging import LogTime @@ -33,6 +34,19 @@ class QueryProjectRequest(BaseModel): tool_params_json: str +class CallFacadeMethodRequest(BaseModel): + """ + Request model for the /call_facade_method endpoint: the execution of a (read-only) REPL facade method + in the context of a project. + """ + + project_name: str + facade_name: str + method_name: str + args: list[Any] + kwargs: dict[str, Any] + + class ProjectServer: """ A lightweight Flask server that exposes a SerenaAgent's project querying @@ -86,6 +100,18 @@ class ProjectServer: query_request = QueryProjectRequest.model_validate(request.get_json()) return self._query_project(query_request) + @self._app.route("/call_facade_method", methods=["POST"]) + def call_facade_method() -> Response: + call_request = CallFacadeMethodRequest.model_validate(request.get_json()) + try: + result = self._call_facade_method(call_request) + except Exception as e: + # report the error to the client (which raises it in the REPL) instead of a generic server error page + log.warning("Facade method call failed: %s", e) + return Response(f"{type(e).__name__}: {e}", status=400, mimetype="text/plain") + # NOTE: the result is pickled; the client (a Serena instance on the same machine) unpickles it + return Response(pickle.dumps(result), mimetype="application/octet-stream") + def _get_project(self, project_root_or_name: str) -> "Project": """Gets the project with the given name, loading it if necessary.""" serena_config = self._agent.serena_config @@ -136,6 +162,20 @@ class ProjectServer: params = json.loads(req.tool_params_json) return tool.apply_ex(**params) + def _call_facade_method(self, req: CallFacadeMethodRequest) -> Any: + """ + Handles a /call_facade_method request by executing the facade method on the agent's REPL facades in the + context of the specified project (see `_query_project` regarding the lock). + Only methods which use the project server and do not edit are admissible. + """ + project = self._get_project(req.project_name) + with self._active_project_lock, self._agent.active_project_context(project): + facade = self._agent.get_repl().entrypoint.get_facade_(req.facade_name) + method = facade.get_method(req.method_name) + if not method.enabled or method.info.can_edit or not method.info.uses_project_server: + raise ValueError(f"Method '{req.facade_name}.{req.method_name}' cannot be executed via the project server") + return self._agent.execute_task(lambda: method(*req.args, **req.kwargs)) + def run(self) -> None: """ Run the server on the given host and port. @@ -158,12 +198,14 @@ class ProjectServerClient: :class:`ConnectionError` is raised. """ - def __init__(self, host: str = "127.0.0.1", port: int = ProjectServer.PORT, timeout: int = 300) -> None: + def __init__(self, host: str = "127.0.0.1", port: int | None = None, timeout: int = 300) -> None: """ :param host: the host address of the project server. - :param port: the port of the project server. + :param port: the port of the project server; None for the default port. :raises ConnectionError: if the project server is not reachable. """ + if port is None: + port = ProjectServer.PORT self._base_url = f"http://{host}:{port}" self._timeout = timeout @@ -197,3 +239,22 @@ class ProjectServerClient: response = requests_lib.post(f"{self._base_url}/query_project", json=payload, timeout=self._timeout) response.raise_for_status() return response.text + + def call_facade_method(self, project_name: str, facade_name: str, method_name: str, args: list[Any], kwargs: dict[str, Any]) -> Any: + """ + Executes a (read-only) REPL facade method in the context of a project. + + :param project_name: the name of the project to query + :param facade_name: the facade's name + :param method_name: the method's name + :param args: the positional arguments (JSON-serialisable) + :param kwargs: the keyword arguments (JSON-serialisable) + :return: the method's result, as returned by the server (unpickled; the server is a trusted local process) + """ + payload = CallFacadeMethodRequest( + project_name=project_name, facade_name=facade_name, method_name=method_name, args=args, kwargs=kwargs + ).model_dump() + response = requests_lib.post(f"{self._base_url}/call_facade_method", json=payload, timeout=self._timeout) + if not response.ok: + raise ValueError(f"Project server error ({response.status_code}): {response.text[:2000]}") + return pickle.loads(response.content) diff --git a/src/serena/repl/api/ext_api.py b/src/serena/repl/api/ext_api.py new file mode 100644 index 00000000..c559935d --- /dev/null +++ b/src/serena/repl/api/ext_api.py @@ -0,0 +1,93 @@ +# SPDX-License-Identifier: GPL-3.0-or-later +""" +The implementation of access to external projects (projects other than the active one). +""" + +from types import TracebackType +from typing import TYPE_CHECKING + +from serena.jetbrains.jetbrains_plugin_client import JetBrainsPluginClientManager +from serena.tools import ListQueryableProjectsTool, QueryProjectTool + +from ..external_project import ExternalProjectContext +from ..facade import FacadeApi, facade_method +from ..representable import JsonObject, JsonObjectRenderer + +if TYPE_CHECKING: + from serena.agent import SerenaAgent + + +class ExternalProjectContextManager: + """ + A context manager (for use in a `with` statement) within which the facades operate on an external project + (read-only): the external project is temporarily activated, and operations requiring language servers are + executed in the project's server. Contexts cannot be nested. + """ + + def __init__(self, agent: "SerenaAgent", project_name: str) -> None: + """ + :param agent: the agent + :param project_name: the name (or root path) of the registered external project + """ + self._agent = agent + self._project_name = project_name + self._active_project_context = None + + def __enter__(self) -> None: + entrypoint = self._agent.get_repl().entrypoint + if entrypoint.get_external_project_() is not None: + raise ValueError("External project contexts cannot be nested") + + # temporarily activate the external project + registered_project = self._agent.serena_config.get_registered_project(self._project_name) + if registered_project is None: + raise ValueError(f"Project '{self._project_name}' is not registered and cannot be queried") + project = registered_project.get_project_instance(self._agent.serena_config) + self._active_project_context = self._agent.active_project_context(project) + self._active_project_context.__enter__() + + # switch the facades to the external project (remote execution of language server operations for the LSP backend) + remote_execution = self._agent.get_language_backend().is_lsp() + entrypoint.set_external_project_(ExternalProjectContext(registered_project.project_name, remote_execution)) + + def __exit__(self, exc_type: type[BaseException] | None, exc_value: BaseException | None, traceback: TracebackType | None) -> None: + self._agent.get_repl().entrypoint.set_external_project_(None) + assert self._active_project_context is not None + self._active_project_context.__exit__(exc_type, exc_value, traceback) + + +class ExternalProjectsApi(FacadeApi): + def __init__(self, agent: "SerenaAgent") -> None: + super().__init__(agent, name="ext", description="read-only access to external projects (projects other than the active one)") + + @facade_method(corresponding_tool=ListQueryableProjectsTool) + def list_projects(self, symbol_access: bool = True) -> JsonObject: + """ + Lists the registered projects which can be queried. + + :param symbol_access: whether to list only projects for which symbol-level access is available + :return: the project names mapped to their root directories + """ + registered_projects = self._agent.serena_config.projects + if symbol_access and self._agent.get_language_backend().is_jetbrains(): + # only projects with open IDE instances can be queried + matched_clients = JetBrainsPluginClientManager().match_clients(registered_projects) + relevant_projects = [mc.registered_project for mc in matched_clients] + else: + # all projects can be queried (the project server instantiates projects dynamically) + relevant_projects = registered_projects + result = {p.project_name: str(p.project_root) for p in relevant_projects} + return JsonObject(result, JsonObjectRenderer(self._agent, -1)) + + @facade_method(corresponding_tool=QueryProjectTool) + def project_context(self, project_name: str) -> ExternalProjectContextManager: + """ + Provides a context (for use in a `with` statement) within which all facades operate on the given external + project instead of the active one, with read-only access: e.g. + `with s.ext.project_context("other"): result = s.lsp.find_symbol("Foo")`. + Results obtained within the context can be used after it (they are self-contained). + + :param project_name: the name (or root path) of the project, as listed by `list_projects` + :return: the context manager + """ + return ExternalProjectContextManager(self._agent, project_name) diff --git a/src/serena/repl/api/lsp_api.py b/src/serena/repl/api/lsp_api.py index 8d827ff6..f7866f5b 100644 --- a/src/serena/repl/api/lsp_api.py +++ b/src/serena/repl/api/lsp_api.py @@ -47,13 +47,20 @@ class LspSymbolCollection(RepresentableViaRenderer): symbols: list[LanguageServerSymbol] - def __init__(self, symbols: list[LanguageServerSymbol], renderer: "LspSymbolCollectionRenderer"): + def __init__( + self, + symbols: list[LanguageServerSymbol], + renderer: "LspSymbolCollectionRenderer", + info_by_symbol: dict[LanguageServerSymbol, str] | None = None, + ): """ :param symbols: the list of symbols :param renderer: the renderer to use for representing the collection + :param info_by_symbol: additional (hover-like) info per symbol, if requested """ super().__init__(renderer) self.symbols = symbols + self.info_by_symbol_ = info_by_symbol or {} def __len__(self) -> int: return len(self.symbols) @@ -72,13 +79,15 @@ class LspSymbol(RepresentableViaRenderer): symbol: LanguageServerSymbol - def __init__(self, symbol: LanguageServerSymbol, renderer: "LspSymbolRenderer"): + def __init__(self, symbol: LanguageServerSymbol, renderer: "LspSymbolRenderer", info: str | None = None): """ :param symbol: the symbol :param renderer: the renderer to use for representing the symbol + :param info: additional (hover-like) info on the symbol, if requested """ super().__init__(renderer) self.symbol = symbol + self.info_ = info @dataclass(kw_only=True) @@ -108,19 +117,20 @@ class LspSymbolCollectionRenderer(Renderer[LspSymbolCollection]): self, agent: "SerenaAgent", max_answer_chars: int, - symbol_retriever: LanguageServerSymbolRetriever, output_params: SymbolOutputParams, grouper: SymbolDictGrouper | None = None, ): super().__init__(agent, max_answer_chars) - self._symbol_retriever = symbol_retriever self._output_params = output_params self._grouper = grouper - def symbol_dicts_(self, symbols: list[LanguageServerSymbol]) -> list[LanguageServerSymbol.OutputDict]: + def symbol_dicts_( + self, symbols: list[LanguageServerSymbol], info_by_symbol: dict[LanguageServerSymbol, str] + ) -> list[LanguageServerSymbol.OutputDict]: """ :param symbols: the symbols to convert - :return: the dict representations of the symbols according to the output parameters (including info, if requested) + :param info_by_symbol: additional info to include per symbol, if any + :return: the dict representations of the symbols according to the output parameters (including the info) """ p = self._output_params symbol_dicts = [ @@ -140,14 +150,12 @@ class LspSymbolCollectionRenderer(Renderer[LspSymbolCollection]): ) for s in symbols ] - if not p.include_body and p.include_info: - info_by_symbol = self._symbol_retriever.request_info_for_symbol_batch(symbols) - for s, s_dict in zip(symbols, symbol_dicts, strict=True): - if symbol_info := info_by_symbol.get(s): - # In python 3.15 we could specify extra_items=True in the TypedDict definition, - # https://peps.python.org/pep-0728/ - # If we ever upgrade to 3.15, we can remove the type: ignore[typeddict-unknown-key] - s_dict["info"] = symbol_info + for s, s_dict in zip(symbols, symbol_dicts, strict=True): + if symbol_info := info_by_symbol.get(s): + # In python 3.15 we could specify extra_items=True in the TypedDict definition, + # https://peps.python.org/pep-0728/ + # If we ever upgrade to 3.15, we can remove the type: ignore[typeddict-unknown-key] + s_dict["info"] = symbol_info return symbol_dicts def _group(self, symbol_dicts: list[LanguageServerSymbol.OutputDict]) -> Any: @@ -157,7 +165,7 @@ class LspSymbolCollectionRenderer(Renderer[LspSymbolCollection]): def create_short_result_relative_path_to_name_paths() -> str: return f"Shortened result:\n{TextOutputUtils.to_json(obj.relative_path_to_name_paths_())}" - result = self._to_json(self._group(self.symbol_dicts_(obj.symbols))) + result = self._to_json(self._group(self.symbol_dicts_(obj.symbols, obj.info_by_symbol_))) return self._limit_length(result, shortened_result_factories=[create_short_result_relative_path_to_name_paths]) @@ -171,7 +179,8 @@ class LspSymbolRenderer(Renderer[LspSymbol]): self._collection_renderer = collection_renderer def render(self, obj: LspSymbol) -> str: - symbol_dict = self._collection_renderer.symbol_dicts_([obj.symbol])[0] + info_by_symbol = {obj.symbol: obj.info_} if obj.info_ else {} + symbol_dict = self._collection_renderer.symbol_dicts_([obj.symbol], info_by_symbol)[0] return self._limit_length(self._to_json(symbol_dict)) @@ -182,7 +191,7 @@ class LspSymbolsOverviewRenderer(LspSymbolCollectionRenderer): """ def render(self, obj: LspSymbolCollection) -> str: - symbol_dicts = self.symbol_dicts_(obj.symbols) + symbol_dicts = self.symbol_dicts_(obj.symbols, obj.info_by_symbol_) result = self._to_json(self._group(symbol_dicts)) def make_kind_counts() -> str: @@ -210,13 +219,20 @@ class LspReferenceCollection(RepresentableViaRenderer): references: list[ReferenceInLanguageServerSymbol] - def __init__(self, references: list[ReferenceInLanguageServerSymbol], renderer: "LspReferenceCollectionRenderer"): + def __init__( + self, + references: list[ReferenceInLanguageServerSymbol], + contents_around_references: list[str], + renderer: "LspReferenceCollectionRenderer", + ): """ :param references: the references + :param contents_around_references: for each reference, the code around it (for display) :param renderer: the renderer to use for representing the collection """ super().__init__(renderer) self.references = references + self.contents_around_references_ = contents_around_references def __len__(self) -> int: return len(self.references) @@ -233,18 +249,11 @@ class LspReferenceCollectionRenderer(Renderer[LspReferenceCollection]): self._grouper = grouper def render(self, obj: LspReferenceCollection) -> str: - project = self._agent.get_active_project_or_raise() - reference_dicts = [] ref_summaries = [] - for ref in obj.references: + for ref, content_around_ref in zip(obj.references, obj.contents_around_references_, strict=True): ref_dict = dict(ref.symbol.to_dict(kind=True, relative_path=True, depth=0, body=False, body_location=True)) - ref_relative_path = ref.symbol.location.relative_path - assert ref_relative_path is not None, f"Referencing symbol {ref.symbol.name} has no relative path, this is likely a bug." - content_around_ref = project.retrieve_content_around_line( - relative_file_path=ref_relative_path, line=ref.line, context_lines_before=1, context_lines_after=1 - ) - ref_dict["content_around_reference"] = content_around_ref.to_display_string() + ref_dict["content_around_reference"] = content_around_ref reference_dicts.append(ref_dict) ref_summaries.append( { @@ -292,6 +301,10 @@ class LspDiagnosticsRenderer(Renderer[LspDiagnostics]): return self._limit_length(self._to_json(obj.grouped.get_dict())) +def _is_not_low_level(symbol: LanguageServerSymbol) -> bool: + return not symbol.is_low_level() + + class LspApi(FacadeApi): FILE_LEVEL_DIAGNOSTIC_BUCKET = "" """the name path under which diagnostics that cannot be mapped to a symbol are grouped""" @@ -336,6 +349,26 @@ class LspApi(FacadeApi): def _create_ls_code_editor(self, symbol_retriever: LanguageServerSymbolRetriever | None = None) -> LanguageServerCodeEditor: return LanguageServerCodeEditor(symbol_retriever or self._create_symbol_retriever()) + @staticmethod + def _request_info( + symbol_retriever: LanguageServerSymbolRetriever, symbols: list[LanguageServerSymbol], output_params: SymbolOutputParams + ) -> dict[LanguageServerSymbol, str]: + """ + :return: additional (hover-like) info per symbol, if the output parameters request it (and not the body, which + supersedes it); requested eagerly, such that results are self-contained + """ + if output_params.include_info and not output_params.include_body: + return {s: info for s, info in symbol_retriever.request_info_for_symbol_batch(symbols).items() if info} + return {} + + def _retrieve_content_around_reference(self, reference: ReferenceInLanguageServerSymbol) -> str: + relative_path = reference.symbol.location.relative_path + assert relative_path is not None, f"Referencing symbol {reference.symbol.name} has no relative path, this is likely a bug." + content = self._get_project().retrieve_content_around_line( + relative_file_path=relative_path, line=reference.line, context_lines_before=1, context_lines_after=1 + ) + return content.to_display_string() + @staticmethod def _parse_kinds(kinds: Sequence[int]) -> Sequence[SymbolKind] | None: return [SymbolKind(k) for k in kinds] if kinds else None @@ -345,7 +378,7 @@ class LspApi(FacadeApi): # language server management - @facade_method(optional=True, corresponding_tool=RestartLanguageServerTool) + @facade_method(uses_project_server=True, optional=True, corresponding_tool=RestartLanguageServerTool) def restart_language_server(self) -> str: """ Restarts the language server(s). Use this only on explicit user request or after confirmation; @@ -358,7 +391,7 @@ class LspApi(FacadeApi): # read operations - @facade_method(corresponding_tool=GetSymbolsOverviewTool) + @facade_method(uses_project_server=True, corresponding_tool=GetSymbolsOverviewTool) def get_symbols_overview(self, relative_path: str, depth: int = -1, max_answer_chars: int = -1) -> LspSymbolCollection: """ Gets an overview of the top-level symbols defined in the given file (classes, methods, fields) — its @@ -396,14 +429,12 @@ class LspApi(FacadeApi): kind=True, relative_path=False, location=False, - child_inclusion_predicate=lambda s: not s.is_low_level(), - ) - renderer = LspSymbolsOverviewRenderer( - self._agent, max_answer_chars, symbol_retriever, output_params, grouper=self.overview_grouper_ + child_inclusion_predicate=_is_not_low_level, ) + renderer = LspSymbolsOverviewRenderer(self._agent, max_answer_chars, output_params, grouper=self.overview_grouper_) return LspSymbolCollection(symbols, renderer) - @facade_method(corresponding_tool=FindSymbolTool) + @facade_method(uses_project_server=True, corresponding_tool=FindSymbolTool) def find_symbol( self, name_path_pattern: str, @@ -481,10 +512,8 @@ class LspApi(FacadeApi): children_name_path=False, include_info=include_info, ) - renderer = LspSymbolCollectionRenderer( - self._agent, max_answer_chars, symbol_retriever, output_params, grouper=self.find_symbol_dict_grouper_ - ) - symbol_collection = LspSymbolCollection(symbols, renderer) + renderer = LspSymbolCollectionRenderer(self._agent, max_answer_chars, output_params, grouper=self.find_symbol_dict_grouper_) + symbol_collection = LspSymbolCollection(symbols, renderer, self._request_info(symbol_retriever, symbols, output_params)) # check for max_matches limit exceeded n_matches = len(symbols) @@ -495,7 +524,7 @@ class LspApi(FacadeApi): return symbol_collection - @facade_method(corresponding_tool=FindReferencingSymbolsTool) + @facade_method(uses_project_server=True, corresponding_tool=FindReferencingSymbolsTool) def find_referencing_symbols( self, name_path: str, @@ -526,9 +555,11 @@ class LspApi(FacadeApi): include_kinds=self._parse_kinds(include_kinds), exclude_kinds=self._parse_kinds(exclude_kinds), ) - return LspReferenceCollection(references, LspReferenceCollectionRenderer(self._agent, max_answer_chars, self.references_grouper_)) + contents_around_references = [self._retrieve_content_around_reference(ref) for ref in references] + renderer = LspReferenceCollectionRenderer(self._agent, max_answer_chars, self.references_grouper_) + return LspReferenceCollection(references, contents_around_references, renderer) - @facade_method(corresponding_tool=FindImplementationsTool) + @facade_method(uses_project_server=True, corresponding_tool=FindImplementationsTool) def find_implementations( self, name_path: str, @@ -561,9 +592,10 @@ class LspApi(FacadeApi): exclude_kinds=self._parse_kinds(exclude_kinds), ) output_params = SymbolOutputParams(kind=True, relative_path=True, body_location=True, include_info=include_info) - return LspSymbolCollection(symbols, LspSymbolCollectionRenderer(self._agent, max_answer_chars, symbol_retriever, output_params)) + renderer = LspSymbolCollectionRenderer(self._agent, max_answer_chars, output_params) + return LspSymbolCollection(symbols, renderer, self._request_info(symbol_retriever, symbols, output_params)) - @facade_method(corresponding_tool=FindDeclarationTool) + @facade_method(uses_project_server=True, corresponding_tool=FindDeclarationTool) def find_declaration( self, relative_path: str, @@ -615,10 +647,11 @@ class LspApi(FacadeApi): output_params = SymbolOutputParams( kind=True, relative_path=True, body_location=True, include_body=include_body, include_info=include_info ) - collection_renderer = LspSymbolCollectionRenderer(self._agent, -1, symbol_retriever, output_params) - return LspSymbol(defining_symbol, LspSymbolRenderer(self._agent, -1, collection_renderer)) + collection_renderer = LspSymbolCollectionRenderer(self._agent, -1, output_params) + info = self._request_info(symbol_retriever, [defining_symbol], output_params).get(defining_symbol) + return LspSymbol(defining_symbol, LspSymbolRenderer(self._agent, -1, collection_renderer), info) - @facade_method(corresponding_tool=GetDiagnosticsForFileTool) + @facade_method(uses_project_server=True, corresponding_tool=GetDiagnosticsForFileTool) def get_diagnostics_for_file( self, relative_path: str, start_line: int = 0, end_line: int = -1, min_severity: int = 4, max_answer_chars: int = -1 ) -> LspDiagnostics: @@ -651,7 +684,7 @@ class LspApi(FacadeApi): return self._create_diagnostics(grouped_diagnostics, max_answer_chars) - @facade_method(optional=True, corresponding_tool=GetDiagnosticsForSymbolTool) + @facade_method(uses_project_server=True, optional=True, corresponding_tool=GetDiagnosticsForSymbolTool) def get_diagnostics_for_symbol( self, name_path: str, @@ -694,7 +727,7 @@ class LspApi(FacadeApi): # edit operations - @facade_method(can_edit=True, corresponding_tool=RenameSymbolTool) + @facade_method(uses_project_server=True, can_edit=True, corresponding_tool=RenameSymbolTool) def rename_symbol(self, name_path: str, relative_path: str, new_name: str) -> str: """ Renames the symbol with the given `name_path` to `new_name` throughout the entire codebase. @@ -709,7 +742,7 @@ class LspApi(FacadeApi): self._get_project().ls_sync_file_system_changes() return self._create_ls_code_editor().rename_symbol(name_path, relative_path=relative_path, new_name=new_name) - @facade_method(can_edit=True, corresponding_tool=SafeDeleteSymbol) + @facade_method(uses_project_server=True, can_edit=True, corresponding_tool=SafeDeleteSymbol) def safe_delete_symbol(self, name_path_pattern: str, relative_path: str) -> str: """ Deletes the symbol if it is safe to do so (i.e., if there are no references to it) diff --git a/src/serena/repl/external_project.py b/src/serena/repl/external_project.py new file mode 100644 index 00000000..e702d4ff --- /dev/null +++ b/src/serena/repl/external_project.py @@ -0,0 +1,51 @@ +# SPDX-License-Identifier: GPL-3.0-or-later +""" +Execution of facade methods in the context of an external project (i.e. a project other than the active one). +""" + +from typing import TYPE_CHECKING, Any + +if TYPE_CHECKING: + from serena.project_server import ProjectServerClient + + +class ExternalProjectContext: + """ + The context in which facade methods are executed while an external project is being queried: + methods which use the project server (see `FacadeMethodInfo.uses_project_server`) are executed remotely + in the project server (if remote execution applies to the language backend), all other methods are executed + locally against the temporarily switched project. Editing methods are not permitted. + """ + + def __init__(self, project_name: str, remote_execution: bool) -> None: + """ + :param project_name: the name of the external project + :param remote_execution: whether methods using the project server are to be executed remotely + (False for the JetBrains backend, where the IDE serves all projects) + """ + self.project_name = project_name + self._remote_execution = remote_execution + self._client: ProjectServerClient | None = None + + def executes_remotely(self, uses_project_server: bool) -> bool: + """ + :param uses_project_server: whether the method in question uses the project server + :return: whether the method is to be executed remotely + """ + return self._remote_execution and uses_project_server + + def call(self, facade_name: str, method_name: str, args: tuple[Any, ...], kwargs: dict[str, Any]) -> Any: + """ + Executes the given facade method in the external project's server. + + :param facade_name: the facade's name + :param method_name: the method's name + :param args: the positional arguments (must be JSON-serialisable) + :param kwargs: the keyword arguments (must be JSON-serialisable) + :return: the method's result (unpickled) + """ + if self._client is None: + from serena.project_server import ProjectServerClient + + self._client = ProjectServerClient() + return self._client.call_facade_method(self.project_name, facade_name, method_name, list(args), kwargs) diff --git a/src/serena/repl/facade.py b/src/serena/repl/facade.py index 18ed06eb..45a2538c 100644 --- a/src/serena/repl/facade.py +++ b/src/serena/repl/facade.py @@ -26,6 +26,8 @@ if TYPE_CHECKING: from serena.code_editor import CodeEditor from serena.tools import Tool + from .external_project import ExternalProjectContext + log = logging.getLogger(__name__) TCallable = TypeVar("TCallable", bound=Callable[..., Any]) @@ -106,7 +108,12 @@ class ReferencedType: described even if undocumented, as the listing is the documentation decision. """ - _CAPABILITIES: typing.ClassVar[dict[str, str]] = {"__len__": "len()", "__iter__": "iteration", "__getitem__": "indexing"} + _CAPABILITIES: typing.ClassVar[dict[str, str]] = { + "__len__": "len()", + "__iter__": "iteration", + "__getitem__": "indexing", + "__enter__": "use in a `with` statement", + } @property def name(self) -> str: @@ -243,6 +250,11 @@ class FacadeMethodInfo: whether the method is rarely needed, such that the facade's description only summarises it (first line of its documentation and a pointer to its full documentation) in order to keep the description compact """ + uses_project_server: bool = False + """ + whether the method requires the project's language servers and must therefore be executed in the project server + when an external project is queried (see `ExternalProjectContext`) + """ corresponding_tool: "type[Tool] | None" = None """the classic tool offering the same functionality, if any""" @@ -262,6 +274,7 @@ def facade_method( beta: bool = False, can_edit: bool = False, niche: bool = False, + uses_project_server: bool = False, corresponding_tool: "type[Tool] | None" = None, ) -> Callable[[TCallable], TCallable]: """ @@ -272,6 +285,7 @@ def facade_method( :param beta: whether the method is in beta :param can_edit: whether the method can modify the codebase :param niche: whether the method is rarely needed (its documentation is then only summarised in the facade's description) + :param uses_project_server: whether the method must be executed in the project server when an external project is queried :param corresponding_tool: the classic tool offering the same functionality, if any :return: the decorator """ @@ -283,6 +297,7 @@ def facade_method( beta=beta, can_edit=can_edit, niche=niche, + uses_project_server=uses_project_server, corresponding_tool=corresponding_tool, ) setattr(method, _FACADE_METHOD_INFO_ATTR, info) @@ -391,6 +406,12 @@ class FacadeMethod: return f"{self.facade_name}.{self.name}" def __call__(self, *args: Any, **kwargs: Any) -> Any: + external_project = self.parent.get_external_project_() + if external_project is not None: + if self.info.can_edit: + raise ValueError(f"{self.qualified_name} cannot be called while an external project is being queried (read-only access)") + if external_project.executes_remotely(self.info.uses_project_server): + return external_project.call(self.facade_name, self.name, args, kwargs) return self._implementation(*args, **kwargs) def get_implementation_(self) -> Callable[..., Any]: @@ -548,6 +569,16 @@ class Facade: object.__setattr__(self, "_description", description) object.__setattr__(self, "_methods", {}) object.__setattr__(self, "_types", {t.name: t for t in types}) + object.__setattr__(self, "_external_project", None) + + def set_external_project_(self, external_project: "ExternalProjectContext | None") -> None: + """ + :param external_project: the context of the external project being queried (None if the active project is used) + """ + object.__setattr__(self, "_external_project", external_project) + + def get_external_project_(self) -> "ExternalProjectContext | None": + return self._external_project def _add_method(self, method: FacadeMethod) -> None: assert method.parent is self diff --git a/src/serena/repl/repl.py b/src/serena/repl/repl.py index b631777b..f4e4cbec 100644 --- a/src/serena/repl/repl.py +++ b/src/serena/repl/repl.py @@ -11,6 +11,7 @@ import traceback from typing import Any from ..session import SerenaSession +from .external_project import ExternalProjectContext from .facade import ApiScope, Facade, FacadeMethod, ReferencedType from .representable import Representable @@ -44,6 +45,18 @@ class SerenaReplEntrypoint: """ return [method for facade in self._facades.values() for method in facade.get_enabled_methods()] + def set_external_project_(self, external_project: "ExternalProjectContext | None") -> None: + """ + :param external_project: the context of the external project being queried by the currently executing code + (None if the active project is used); propagated to all facades + """ + for facade in self._facades.values(): + facade.set_external_project_(external_project) + + def get_external_project_(self) -> "ExternalProjectContext | None": + external_projects = {facade.get_external_project_() for facade in self._facades.values()} + return next(iter(external_projects)) if external_projects else None + def set_current_session_(self, session: SerenaSession | None, namespace: dict[str, Any] | None) -> None: """ :param session: the session on whose behalf code is being executed (None if no code is being executed) @@ -100,6 +113,13 @@ class SerenaReplEntrypoint: raise ValueError(f"Unknown facade '{name}'. Available facades: {list(self._facades)}") return self._facades[name] + def get_facade_(self, name: str) -> Facade: + """ + :param name: the facade's name + :return: the facade + """ + return self._get_facade(name) + def overview(self) -> str: """ :return: the list of available facades, each with a one-line description and the names of its methods diff --git a/src/serena/repl/representable.py b/src/serena/repl/representable.py index 9867f726..069d712c 100644 --- a/src/serena/repl/representable.py +++ b/src/serena/repl/representable.py @@ -19,7 +19,12 @@ T = TypeVar("T") class Renderer(Generic[T], ABC): def __init__(self, agent: "SerenaAgent", max_answer_chars: int = -1): - self._agent = agent + """ + :param agent: the agent, from which the configured default length limit is taken (the agent itself is not + retained, such that results remain picklable) + :param max_answer_chars: the maximum number of characters; -1 for the configured default + """ + self._default_max_answer_chars = agent.serena_config.default_max_tool_answer_chars self._max_answer_chars = max_answer_chars def _limit_length( @@ -43,7 +48,7 @@ class Renderer(Generic[T], ABC): """ :return: the effective maximum number of characters, resolving the default from the configuration """ - return self._agent.serena_config.default_max_tool_answer_chars if self._max_answer_chars == -1 else self._max_answer_chars + return self._default_max_answer_chars if self._max_answer_chars == -1 else self._max_answer_chars def _to_json(self, x: Any) -> str: return TextOutputUtils.to_json(x) diff --git a/test/serena/test_external_projects.py b/test/serena/test_external_projects.py new file mode 100644 index 00000000..a53b5ed4 --- /dev/null +++ b/test/serena/test_external_projects.py @@ -0,0 +1,107 @@ +""" +End-to-end test of querying an external project through the REPL: a project server executes the language server +operations of the queried project, and the (pickled) results are usable in the querying agent's REPL. +""" + +import socket +import threading +from collections.abc import Iterator + +import pytest +from werkzeug.serving import make_server + +from serena.agent import SerenaAgent +from serena.config.serena_config import SerenaConfig +from serena.project_server import ProjectServer, ProjectServerClient +from serena.repl.api.lsp_api import LspSymbolCollection +from serena.tools import SerenaReplTool +from solidlsp.ls_config import LanguageServerId +from test.conftest import language_server_tests_enabled +from test.serena.test_serena_agent import serena_config # noqa: F401 (fixture) + + +def _free_port() -> int: + with socket.socket() as sock: + sock.bind(("127.0.0.1", 0)) + return sock.getsockname()[1] + + +@pytest.fixture +def project_server(serena_config: SerenaConfig) -> Iterator[tuple[ProjectServer, int]]: # noqa: F811 + """ + Runs a project server (backed by a real agent) on a free port for the duration of the test. + """ + config = serena_config + port = _free_port() + + # construct the server around an agent with the test configuration (the constructor would load the user's configuration) + server = ProjectServer.__new__(ProjectServer) + server._agent = SerenaAgent(serena_config=config) + server._loaded_projects_by_root = {} + server._project_load_locks_by_root = {} + server._active_project_lock = threading.Lock() + server._loaded_projects_lock = threading.Lock() + server._port = port + server._host = "127.0.0.1" + from flask import Flask + + server._app = Flask(__name__) + server._setup_routes() + + http_server = make_server("127.0.0.1", port, server._app, threaded=True) + thread = threading.Thread(target=http_server.serve_forever, daemon=True) + thread.start() + try: + yield server, port + finally: + http_server.shutdown() + server._agent.on_shutdown(timeout=5) + + +@pytest.mark.python +@pytest.mark.skipif(not language_server_tests_enabled(LanguageServerId.PYTHON), reason="python tests are disabled in this environment") +def test_facade_method_results_are_transferred_from_the_project_server(project_server: tuple[ProjectServer, int]) -> None: + _, port = project_server + client = ProjectServerClient(port=port) + result = client.call_facade_method("test_repo_python", "lsp", "find_symbol", ["create_user"], {"include_body": True}) + + # the result is a self-contained object which can be processed and rendered locally + assert isinstance(result, LspSymbolCollection) + assert [s.name for s in result.symbols] == ["create_user"] + assert result.symbols[0].body.startswith("def create_user") + assert "create_user" in result.represent() + + # editing methods are refused + with pytest.raises(ValueError, match="cannot be executed"): + client.call_facade_method("test_repo_python", "lsp", "rename_symbol", ["create_user", "test_repo/services.py", "x"], {}) + + +@pytest.mark.python +@pytest.mark.skipif(not language_server_tests_enabled(LanguageServerId.PYTHON), reason="python tests are disabled in this environment") +def test_external_project_context_in_repl( + project_server: tuple[ProjectServer, int], + serena_config: SerenaConfig, # noqa: F811 + monkeypatch: pytest.MonkeyPatch, +) -> None: + server, port = project_server + monkeypatch.setattr(ProjectServer, "PORT", port) # let the REPL's external project context use the test server + + # the querying agent has another project active and queries the python test project + agent = SerenaAgent(project="test_repo_typescript", serena_config=serena_config) + agent.execute_task(lambda: None) + try: + tool = agent.get_tool(SerenaReplTool) + session_id = agent.create_session().session_id + code = ( + 'with s.ext.project_context("test_repo_python"):\n' + ' result = s.lsp.find_symbol("create_user")\n' + "[s.name for s in result.symbols]" + ) + assert tool.apply(session_id, code) == "create_user" + assert server._loaded_projects_by_root, "the operation was not executed by the project server" + + # the result persists and can be used after the context; the active project is restored + assert "services.py" in tool.apply(session_id, "result.represent()") + assert "test_repo_typescript" in agent.get_current_config_overview() + finally: + agent.on_shutdown(timeout=5) diff --git a/test/serena/test_repl_tool.py b/test/serena/test_repl_tool.py index 4b7b348e..f02cbbcb 100644 --- a/test/serena/test_repl_tool.py +++ b/test/serena/test_repl_tool.py @@ -10,6 +10,7 @@ import pytest from serena.config.serena_config import ApiInclusionDefinition from serena.repl.api.edit_api import EditApi from serena.repl.api.lsp_api import LspApi +from serena.repl.external_project import ExternalProjectContext from serena.repl.facade import ApiScope, Facade, FacadeApi, FacadeMethodInfo, facade_method from serena.repl.repl import SerenaRepl from serena.session import SerenaSession @@ -82,6 +83,30 @@ class TestReplExecution: overview = rebuilt_repl.execute("facades()", session) assert "s.edit" in overview and "s.lsp" not in overview + def test_external_project_dispatch(self) -> None: + class FakeExternalProject(ExternalProjectContext): + def __init__(self) -> None: + super().__init__("other", remote_execution=True) + self.calls: list[tuple[str, str, tuple, dict]] = [] + + def call(self, facade_name: str, method_name: str, args: tuple, kwargs: dict) -> str: + self.calls.append((facade_name, method_name, args, kwargs)) + return "remote result" + + facades = [Facade.from_api(LspApi(MagicMock()), ApiScope()), Facade.from_api(EditApi(MagicMock()), ApiScope())] + repl = SerenaRepl(facades, ApiScope()) + external_project = FakeExternalProject() + repl.entrypoint.set_external_project_(external_project) + + # methods using the project server are executed remotely, editing methods are refused + assert repl.execute('s.lsp.find_symbol("Foo", depth=1)') == "remote result" + assert external_project.calls == [("lsp", "find_symbol", ("Foo",), {"depth": 1})] + assert "read-only" in repl.execute('s.edit.replace_content("a.py", "x", "y", "literal")') + + # without remote execution (JetBrains backend), methods run locally + repl.entrypoint.set_external_project_(ExternalProjectContext("other", remote_execution=False)) + assert "remote result" not in repl.execute('s.info("lsp.find_symbol")') + def test_facade_discovery(self, repl: SerenaRepl) -> None: overview = repl.execute("s.info()") assert "s.lsp" in overview From e39291979ebb9c5b512f4d9a9028e0029d3ee15a Mon Sep 17 00:00:00 2001 From: Dominik Jain Date: Wed, 9 Sep 2026 11:33:20 +0200 Subject: [PATCH 33/53] Check tool functionality availability independently of the agent interface Availability check: * SerenaAgent.is_tool_function_available(tool_class): whether a tool's functionality is available to the LLM (tool mode: tool active; REPL mode: an enabled facade method corresponds to the tool) * replaces the name-based tool_is_active/tool_is_exposed (activation message, dashboard, onboarding) Prompts: * activation message names functions via PromptParams.get_function_name (interface-specific names) --- src/serena/agent.py | 42 +++++++++++++++++++++--------- src/serena/dashboard.py | 3 ++- src/serena/repl/repl.py | 17 +++++++++++- src/serena/tools/tools_base.py | 2 +- src/serena/tools/workflow_tools.py | 2 +- 5 files changed, 50 insertions(+), 16 deletions(-) diff --git a/src/serena/agent.py b/src/serena/agent.py index 2b0de256..f5a466f4 100644 --- a/src/serena/agent.py +++ b/src/serena/agent.py @@ -1017,6 +1017,15 @@ class SerenaAgent: when using the REPL interface). """ + def get_function_name(self, tool_class: type[Tool]) -> str: + """ + :param tool_class: the tool for which to get the function name + :return: the function name to use for this tool in prompts, which may be different from the tool's standard name + (e.g. when using a different language backend or when using the REPL interface) + """ + tool_name = tool_class.get_name_from_cls() + return self.tool_names_mapping.get(tool_name, tool_name) + def _get_prompt_params(self) -> PromptParams: """ :return: parameters for prompt rendering depending on the current agent interface, language backend and active tools/methods @@ -1203,6 +1212,8 @@ class SerenaAgent: proj = self._active_project assert proj is not None, "A project must be active before calling this." + prompt_params = self._get_prompt_params() + # Note: The activation message is always returned in full, even if it was already provided in the current session, # because some clients (e.g. Claude Desktop) will use the same session across multiple chats. # So while we don't want the activation message to be additionally included in the system prompt @@ -1222,16 +1233,16 @@ class SerenaAgent: msg += f"File encoding: {proj.project_config.encoding}.\n" # add list of memories (if memories are enabled) - include_memories = self._active_tools.contains_tool_class(ReadMemoryTool) + include_memories = self.is_tool_function_available(ReadMemoryTool) if include_memories: project_memories = proj.memory_manager.list_project_memories() if project_memories: msg += ( f"{json.dumps(project_memories.to_dict())}\n" - + f"Use the `{ReadMemoryTool.get_name_from_cls()}` tool to read these memories later if they are relevant to the task.\n" + + f"Use `{prompt_params.get_function_name(ReadMemoryTool)}` to read these memories later if they are relevant to the task.\n" ) - elif self._active_tools.contains_tool_class(OnboardingTool): - msg += f"Onboarding has not been performed yet. Ask the user whether to perform onboarding via the `{OnboardingTool.get_name_from_cls()}` tool.\n" + elif self.is_tool_function_available(OnboardingTool): + msg += f"Onboarding has not been performed yet. Ask the user whether to perform onboarding and if so, call `{prompt_params.get_function_name(OnboardingTool)}`.\n" # add prompts for modes that were dynamically activated by the project modes_with_prompts = self._project_prompt_status.get_modes_with_prompts_to_be_provided_for_project_activation(session_id) @@ -1546,19 +1557,26 @@ class SerenaAgent: """ return self._active_tools.tool_names - def tool_is_active(self, tool_name: str) -> bool: + def get_active_tools(self) -> AvailableTools: """ - :param tool_class: the name of the tool to check - :return: True if the tool is active, False otherwise + :return: the set of active tools """ - return self._active_tools.contains_tool_name(tool_name) + return self._active_tools - def tool_is_exposed(self, tool_name: str) -> bool: + def is_tool_function_available(self, tool_class: type[Tool]) -> bool: """ - :param tool_name: the name of the tool to check - :return: True if the tool is in the exposed tool set, False otherwise + Checks whether the functionality offered by a tool is available - either through the tool + itself being enabled or through the corresponding function being exposed in the REPL. + + :param tool_class: the tool class + :return: whether the function is available """ - return self._exposed_tools.contains_tool_name(tool_name) + if self._agent_interface == AgentInterface.TOOLS: + return self._active_tools.contains_tool_class(tool_class) + elif self._agent_interface == AgentInterface.REPL: + return self.get_repl().entrypoint.is_tool_function_available(tool_class) + else: + raise NotImplementedError def get_current_config_overview(self) -> str: """ diff --git a/src/serena/dashboard.py b/src/serena/dashboard.py index 26201a1f..fdff3019 100644 --- a/src/serena/dashboard.py +++ b/src/serena/dashboard.py @@ -27,6 +27,7 @@ from serena.analytics import ToolUsageStats from serena.config.serena_config import SerenaConfig, SerenaPaths from serena.constants import SERENA_DASHBOARD_DIR, SerenaPorts from serena.task_executor import TaskExecutor +from serena.tools import ReadMemoryTool from serena.util.logging import MemoryLogHandler from serena.util.pypi import PyPIPackageInfo from serena.util.pywebview import WebViewWithTray @@ -604,7 +605,7 @@ class SerenaDashboardAPI: # Get available memories if ReadMemoryTool is active available_memories = None - if self._agent.tool_is_active("read_memory") and project is not None: + if self._agent.is_tool_function_available(ReadMemoryTool) and project is not None: available_memories = project.memory_manager.list_memories().get_full_list() # Get list of languages for the active project diff --git a/src/serena/repl/repl.py b/src/serena/repl/repl.py index f4e4cbec..59cdfaf1 100644 --- a/src/serena/repl/repl.py +++ b/src/serena/repl/repl.py @@ -8,13 +8,16 @@ import ast import logging import re import traceback -from typing import Any +from typing import TYPE_CHECKING, Any from ..session import SerenaSession from .external_project import ExternalProjectContext from .facade import ApiScope, Facade, FacadeMethod, ReferencedType from .representable import Representable +if TYPE_CHECKING: + from ..tools.tools_base import Tool + log = logging.getLogger(__name__) @@ -45,6 +48,18 @@ class SerenaReplEntrypoint: """ return [method for facade in self._facades.values() for method in facade.get_enabled_methods()] + def is_tool_function_available(self, tool_class: "type[Tool]") -> bool: + """ + Checks whether any of the enabled methods corresponds to the given tool class. + + :param tool_class: the tool class to check for + :return: whether any enabled method corresponds to the given tool class + """ + for method in self.get_enabled_methods(): + if method.info.corresponding_tool == tool_class: + return True + return False + def set_external_project_(self, external_project: "ExternalProjectContext | None") -> None: """ :param external_project: the context of the external project being queried by the currently executing code diff --git a/src/serena/tools/tools_base.py b/src/serena/tools/tools_base.py index 0d494537..8efbd767 100644 --- a/src/serena/tools/tools_base.py +++ b/src/serena/tools/tools_base.py @@ -308,7 +308,7 @@ class Tool(Component): ) def is_active(self) -> bool: - return self.agent.tool_is_active(self.get_name()) + return self.agent.get_active_tools().contains_tool_name(self.get_name()) def is_readonly(self) -> bool: return not self.can_edit() diff --git a/src/serena/tools/workflow_tools.py b/src/serena/tools/workflow_tools.py index bcf4468f..e13b0bac 100644 --- a/src/serena/tools/workflow_tools.py +++ b/src/serena/tools/workflow_tools.py @@ -19,7 +19,7 @@ class OnboardingTool(Tool, MemoryApiMixin): :return: instructions on how to create the onboarding information """ - write_memory_tool_available = self.agent.tool_is_exposed(WriteMemoryTool.get_name_from_cls()) + write_memory_tool_available = self.agent.is_tool_function_available(WriteMemoryTool) if not write_memory_tool_available: return "Memory writing tool not activated, skipping onboarding." return self._api().onboarding() From d76dc3d87001fc91b1a6e2d4c89f2f78dfe58a60 Mon Sep 17 00:00:00 2001 From: Dominik Jain Date: Wed, 9 Sep 2026 11:54:21 +0200 Subject: [PATCH 34/53] Refine commit message guidelines in critical_info memory * subject line must cover the entire change (abstract if necessary) * details as concise bullet items, grouped by topic where the change spans several topics --- .serena/memories/critical_info.md | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/.serena/memories/critical_info.md b/.serena/memories/critical_info.md index f6d85ce9..c40068f5 100644 --- a/.serena/memories/critical_info.md +++ b/.serena/memories/critical_info.md @@ -41,8 +41,12 @@ delegating to it: structure, exposure/naming principles, configuration of the AP # Commits & pull requests -* Commit messages: wrap all lines (subject and body) at ~100 characters; write the message to a file and commit with `-F` - rather than passing long paragraphs via `-m`. +* Commit messages: + * The subject line must cover the *entire* change; make it suitably abstract if necessary + * Details are presented in concise bullet items, one point per item; no prose paragraphs. + Where the change spans several topics, group the items by topic, each group with a short heading. + * Wrap all lines at ~100 characters. + * Write the message to a file and commit with `-F`; do not pass long text via `-m`. * Read `mem:creating_pull_requests` when asked to participate in the creation of a pull request. # Memories From 4b9ce9ad4460ee7ec83de4e7389674e2967a9585 Mon Sep 17 00:00:00 2001 From: Dominik Jain Date: Wed, 9 Sep 2026 12:17:03 +0200 Subject: [PATCH 35/53] Place the REPL's facade listing in the tool description or the activation message * single-project sessions: in the REPL tool's description (fixed at registration) * otherwise: in the project activation message (facades depend on the activated project) * REPL tool description wording adjusted (listing location, notebook semantics) Peripheral changes: * is_tool_function_available: in REPL mode, active tools without facade counterparts count as available --- src/serena/agent.py | 12 +++++++++--- src/serena/tools/repl_tools.py | 20 ++++++++++++-------- test/serena/test_mcp.py | 4 ++++ test/serena/test_serena_agent.py | 7 +++++++ 4 files changed, 32 insertions(+), 11 deletions(-) diff --git a/src/serena/agent.py b/src/serena/agent.py index f5a466f4..7e4bc490 100644 --- a/src/serena/agent.py +++ b/src/serena/agent.py @@ -1251,10 +1251,15 @@ class SerenaAgent: msg += self._render_prompt(mode.prompt, tag="mode", tag_name_attr=mode.name) + "\n" self._project_prompt_status.mark_mode_prompts_as_provided(session_id) - # add project-specific prompt + # add the project's prompt (if any) if proj.project_config.initial_prompt: msg += "\n" + self._render_prompt(proj.project_config.initial_prompt, tag="project-instructions") + # when the REPL is active, add information on available facades if the agent is not in single-project mode + # (for single-project mode where the facades can't change, they are provided in the tool's description) + if self._active_tools.contains_tool_class(SerenaReplTool) and not self.is_single_project(): + msg += f"\n\nAvailable facades for the `{SerenaReplTool.get_name_from_cls()}` tool:\n" + self.get_repl().entrypoint.overview() + self._project_prompt_status.mark_project_activation_message_as_provided(session_id) return msg @@ -1571,10 +1576,11 @@ class SerenaAgent: :param tool_class: the tool class :return: whether the function is available """ + is_active_tool = self._active_tools.contains_tool_class(tool_class) if self._agent_interface == AgentInterface.TOOLS: - return self._active_tools.contains_tool_class(tool_class) + return is_active_tool elif self._agent_interface == AgentInterface.REPL: - return self.get_repl().entrypoint.is_tool_function_available(tool_class) + return is_active_tool or self.get_repl().entrypoint.is_tool_function_available(tool_class) else: raise NotImplementedError diff --git a/src/serena/tools/repl_tools.py b/src/serena/tools/repl_tools.py index 04b6895d..8fe15c2d 100644 --- a/src/serena/tools/repl_tools.py +++ b/src/serena/tools/repl_tools.py @@ -13,25 +13,29 @@ class SerenaReplTool(Tool, ToolMarkerOptional, ToolMarkerBeta): """ def get_apply_docstring(self) -> str: - return self.get_apply_docstring_from_cls() + "\n\nAvailable facades:\n" + self.agent.get_repl().entrypoint.overview() + docs = self.get_apply_docstring_from_cls() + if self.agent.is_single_project(): + docs += "\n\nAvailable facades:\n" + self.agent.get_repl().entrypoint.overview() + else: + docs += "\n\nAvailable facades are provided at project activation" + return docs def apply(self, session: str, code: str) -> str: """ Executes the given Python code, which has access to Serena's functionality through the object `s`. - The functionality is organised in facades, which are attributes of `s` (e.g. `s.myfacade`); the available - facades and their methods are listed below. + The functionality is organised in facades, which are attributes of `s` (e.g. `s.myfacade`). Documentation: Use `s.info("")` when you will use a facade's functionality (it documents all common operations at once) and `s.info(".")` for a single or a rarely needed operation. Several items can be requested in one call, e.g. `s.info("lsp", "edit.replace_content")`. - `s.info("")` documents the facade's operations only, not their result types. Result types are given - in the method listing below (`method -> Type`); request their documentation via `s.info("")`, which + `s.info("")` documents the facade's operations only, not their result types. The facade listing + provides result types (`method -> Type`); request their documentation via `s.info("")`, which includes the types they contain, ONLY if you intend to process results in code (filter, aggregate, chain - calls). If you simply want the result, make it the last expression: results are rendered for you. + calls). The code is executed like a notebook cell: if its last statement is an expression, the expression's value is - the result (do not use `return`). Results are rendered in a form suitable for you; lists are rendered - element-wise, strings are passed through unchanged. + the result. Results are rendered in a form suitable for you; lists are rendered element-wise, + strings are passed through unchanged. Output size: methods with a `max_answer_chars` parameter limit the size of the rendered result (-1 uses the configured default). If the limit is exceeded, a shortened result (or no content) is rendered instead; diff --git a/test/serena/test_mcp.py b/test/serena/test_mcp.py index 66119e8a..1da10379 100644 --- a/test/serena/test_mcp.py +++ b/test/serena/test_mcp.py @@ -29,6 +29,10 @@ class MockAgent: def get_repl() -> SerenaRepl: return SerenaRepl([], ApiScope()) + @staticmethod + def is_single_project() -> bool: + return False + class BaseMockTool(Tool): """A mock Tool class for testing.""" diff --git a/test/serena/test_serena_agent.py b/test/serena/test_serena_agent.py index dece1f1a..625bd5a2 100644 --- a/test/serena/test_serena_agent.py +++ b/test/serena/test_serena_agent.py @@ -919,6 +919,13 @@ class TestSerenaAgent: assert exposed == expected assert "s.lsp" in agent.get_tool(SerenaReplTool).apply(agent.create_session().session_id, "s.info()") + # the facade listing is part of the (fixed) tool description in single-project sessions, + # and of the activation message otherwise (where the facades depend on the activated project) + tool_description = agent.get_tool(SerenaReplTool).get_apply_docstring() + activation_message = agent.get_project_activation_message("test_session") + assert ("s.lsp:" in tool_description) == context.single_project + assert ("s.lsp:" in activation_message) == (not context.single_project) + # prompts refer to operations by their qualified REPL names, e.g. `lsp.find_symbol` instead of the tool name system_prompt = agent.create_system_prompt() assert "`lsp.find_symbol`" in system_prompt From 323e855625fe32aa17aafae5797c034cf3d120fa Mon Sep 17 00:00:00 2001 From: Dominik Jain Date: Wed, 9 Sep 2026 12:24:28 +0200 Subject: [PATCH 36/53] Allow language backend changes upon project activation with the REPL interface * _activate_project: a project's backend override switches the backend in REPL mode (background modes, facades, prompt parameters and backend initialisation are recomputed upon activation) * the tool interface keeps rejecting the mismatch (fixed toolset); the error now mentions the REPL as a workaround --- .serena/memories/repl.md | 3 +++ src/serena/agent.py | 24 ++++++++++++++------ test/serena/config/test_serena_config.py | 29 ++++++++++++++++++++++++ 3 files changed, 49 insertions(+), 7 deletions(-) diff --git a/.serena/memories/repl.md b/.serena/memories/repl.md index 86c8afb4..83403e4d 100644 --- a/.serena/memories/repl.md +++ b/.serena/memories/repl.md @@ -76,6 +76,9 @@ expression is the result. No `return` (a top-level `return` yields a SyntaxError (`serena_repl`, `initial_instructions`, `activate_project` unless single-project); tool inclusion/exclusion definitions do not apply — each interface has its own configuration vocabulary (tool definitions ↔ tools, API definitions ↔ REPL). Contexts do not influence the interface. + In REPL mode, the language backend may change upon project activation (a project's backend override is + applied; background modes, facades, prompt params and backend initialisation are recomputed), whereas the + tool interface forbids this (the toolset depends on the backend and is fixed). Idea (not implemented, considered over-engineered for now): contexts could declare *supported* interfaces (a capability constraint, e.g. clients that handle the REPL badly), with the user's preference choosing among them. - `included_apis`/`excluded_apis` (references `facade` or `facade.method`) in global config, context, modes, diff --git a/src/serena/agent.py b/src/serena/agent.py index 7e4bc490..c0927467 100644 --- a/src/serena/agent.py +++ b/src/serena/agent.py @@ -663,7 +663,8 @@ class SerenaAgent: # determine the effective language backend for this session. # If a startup project is provided and has a per-project override, use it; otherwise use the global config. - # Since we don't want to change the toolset after startup, the language backend cannot be changed within a running Serena session + # With the tool interface, the backend cannot change within a session (the toolset depends on it and is fixed); + # with the REPL interface, it may change upon project activation (see _activate_project). self._language_backend = self.serena_config.determine_language_backend( project_config=registered_project_to_activate.project_config if registered_project_to_activate is not None else None, log_choice=True, @@ -1420,15 +1421,24 @@ class SerenaAgent: self._project_activation_error = None - # check if the project requires a different language backend than the one initialized at startup + # handle the case where the project requires a different language backend than the current one. + # With the tool interface, the backend cannot change, since the set of exposed tools depends on it and is fixed + # for the session. With the REPL interface, the backend can be switched, as all backend-dependent state + # (background modes, REPL facades, prompt parameters, the project's language backend initialisation) is + # recomputed upon activation. project_backend = project.project_config.language_backend if project_backend is not None and project_backend != self._language_backend: - raise ValueError( - f"Cannot activate project '{project.project_name}': it requires the {project_backend.value} backend, " - f"but this session was initialized with {self._language_backend.value}. " - f"Workarounds: (1) Use project activation at startup via the --project flag, " - f"(2) Configure one MCP server per backend in your client." + if self._agent_interface.is_tools(): + raise ValueError( + f"Cannot activate project '{project.project_name}': it requires the {project_backend.value} backend, " + f"but this session was initialized with {self._language_backend.value}. " + f"Workarounds: (1) Use project activation at startup via the --project flag, " + f"(2) Configure one MCP server per backend in your client, (3) use the REPL interface." + ) + log.info( + f"Switching language backend from {self._language_backend.value} to {project_backend.value} for project '{project.project_name}'" ) + self._language_backend = project_backend # shut down the previously active project to release its language server processes if self._active_project is not None: diff --git a/test/serena/config/test_serena_config.py b/test/serena/config/test_serena_config.py index 33fdb60a..64b7a852 100644 --- a/test/serena/config/test_serena_config.py +++ b/test/serena/config/test_serena_config.py @@ -363,6 +363,35 @@ class TestEffectiveLanguageBackend: finally: agent.on_shutdown(timeout=5) + def test_activate_project_switches_backend_with_repl_interface(self): + """With the REPL interface, post-init activation of a project with a different backend switches the backend.""" + config, name = _make_config_with_project("lsp_proj", language_backend=None, global_backend=LanguageBackend.LSP) + config.agent_interface = AgentInterface.REPL + jb_project = Project( + project_root=str(Path(__file__).parent.parent / "resources" / "repos" / "java" / "test_repo"), + project_config=ProjectConfig( + project_name="jb_proj", + language_servers=[LanguageServerId.JAVA], + language_backend=LanguageBackend.JETBRAINS, + ), + serena_config=config, + ) + config.projects.append(RegisteredProject.from_project_instance(jb_project)) + + agent = SerenaAgent(project=name, serena_config=config) + try: + assert agent.get_language_backend() == LanguageBackend.LSP + assert "s.lsp" in agent.get_repl().entrypoint.overview() + + # the backend and everything depending on it follow the activated project + agent.activate_project_from_path_or_name("jb_proj") + assert agent.get_language_backend() == LanguageBackend.JETBRAINS + overview = agent.get_repl().entrypoint.overview() + assert "s.jb" in overview and "s.lsp" not in overview + assert "jetbrains" in [m.name for m in agent.get_active_modes().get_modes(include_background_base_modes=True)] + finally: + agent.on_shutdown(timeout=5) + def test_activate_project_allows_matching_backend(self): """Post-init activation of a project with matching backend succeeds.""" config, name = _make_config_with_project("lsp_proj", language_backend=None, global_backend=LanguageBackend.LSP) From 988f20d5f5fa04e0af778162bb838886c5917eb6 Mon Sep 17 00:00:00 2001 From: Dominik Jain Date: Tue, 15 Sep 2026 12:41:08 +0200 Subject: [PATCH 37/53] Show interface, backend and REPL function availability in the dashboard Configuration endpoint: * ResponseConfigOverview: agent_interface, language_backend and facades (None in tool mode) * facades report each facade and its methods with their enabled state (SerenaReplEntrypoint.get_facade_availability_info) Dashboard UI: * new rows "Interface" and "Backend"; "Languages" is shown for the LSP backend only * new collapsible section "Active Functions" listing the facades and their methods, dimming disabled ones --- src/serena/dashboard.py | 30 +++++++++++ src/serena/repl/facade.py | 6 +++ src/serena/repl/repl.py | 52 +++++++++++++++++--- src/serena/resources/dashboard/dashboard.css | 17 +++++++ src/serena/resources/dashboard/dashboard.js | 48 ++++++++++++++++-- 5 files changed, 141 insertions(+), 12 deletions(-) diff --git a/src/serena/dashboard.py b/src/serena/dashboard.py index fdff3019..4e303ee9 100644 --- a/src/serena/dashboard.py +++ b/src/serena/dashboard.py @@ -61,11 +61,25 @@ class ResponseToolStats(BaseModel): stats: dict[str, dict[str, int]] +class ResponseFacadeMethod(BaseModel): + name: str + is_enabled: bool + + +class ResponseFacade(BaseModel): + name: str + is_enabled: bool + methods: list[ResponseFacadeMethod] + + class ResponseConfigOverview(BaseModel): active_project: dict[str, str | None] context: dict[str, str] modes: list[dict[str, str]] active_tools: list[str] + agent_interface: str + language_backend: str + facades: list[ResponseFacade] | None tool_stats_summary: dict[str, dict[str, int]] registered_projects: list[dict[str, str | bool]] available_tools: list[dict[str, str | bool]] @@ -608,6 +622,19 @@ class SerenaDashboardAPI: if self._agent.is_tool_function_available(ReadMemoryTool) and project is not None: available_memories = project.memory_manager.list_memories().get_full_list() + # Get the availability of the REPL's facades and their methods (REPL interface only) + facades = None + if self._agent.get_agent_interface().is_repl(): + availability_info = self._agent.get_repl().entrypoint.get_facade_availability_info() + facades = [ + ResponseFacade( + name=facade_info.name, + is_enabled=facade_info.is_enabled, + methods=[ResponseFacadeMethod(name=m.name, is_enabled=m.is_enabled) for m in facade_info.methods], + ) + for facade_info in availability_info.facades + ] + # Get list of languages for the active project ls_ids = [] if project is not None: @@ -623,6 +650,9 @@ class SerenaDashboardAPI: context=context_info, modes=modes_info, active_tools=active_tools, + agent_interface=self._agent.get_agent_interface().value, + language_backend=self._agent.get_language_backend().value, + facades=facades, tool_stats_summary=tool_stats_summary, registered_projects=registered_projects, available_tools=available_tools, diff --git a/src/serena/repl/facade.py b/src/serena/repl/facade.py index 45a2538c..cb66a012 100644 --- a/src/serena/repl/facade.py +++ b/src/serena/repl/facade.py @@ -643,6 +643,12 @@ class Facade: """ return [m for m in self._methods.values() if m.enabled] + def get_methods(self) -> list[FacadeMethod]: + """ + :return: the list of all methods, regardless of whether they are enabled (e.g. for changing their enabled state) + """ + return list(self._methods.values()) + def get_types(self) -> list[ReferencedType]: """ :return: the types referenced by the facade's methods diff --git a/src/serena/repl/repl.py b/src/serena/repl/repl.py index 59cdfaf1..1e697738 100644 --- a/src/serena/repl/repl.py +++ b/src/serena/repl/repl.py @@ -8,6 +8,7 @@ import ast import logging import re import traceback +from dataclasses import dataclass from typing import TYPE_CHECKING, Any from ..session import SerenaSession @@ -21,6 +22,35 @@ if TYPE_CHECKING: log = logging.getLogger(__name__) +class FacadeAvailabilityInfo: + """ + Represents information on the availability of facades and the methods therein + """ + + @dataclass + class FacadeInfo: + name: str + is_enabled: bool + methods: list["FacadeAvailabilityInfo.MethodInfo"] + + @dataclass + class MethodInfo: + name: str + is_enabled: bool + + def __init__(self): + self.facades: list[FacadeAvailabilityInfo.FacadeInfo] = [] + + def add_facade(self, facade: Facade, is_enabled: bool): + def is_method_enabled(m: FacadeMethod) -> bool: + return is_enabled and m.enabled + + methods_info = [ + FacadeAvailabilityInfo.MethodInfo(name=method.name, is_enabled=is_method_enabled(method)) for method in facade.get_methods() + ] + self.facades.append(FacadeAvailabilityInfo.FacadeInfo(name=facade.name, is_enabled=is_enabled, methods=methods_info)) + + class SerenaReplEntrypoint: """ Represents the entrypoint object for the REPL. It holds the configured facades as attributes @@ -35,13 +65,25 @@ class SerenaReplEntrypoint: self._facades: dict[str, Facade] = {} self._current_session: SerenaSession | None = None self._current_namespace: dict[str, Any] | None = None + self._facade_availability_info = FacadeAvailabilityInfo() registered_facade_names = [] for facade in facades: - if api_scope.is_facade_enabled(facade.name): - self._register(facade) + is_facade_enabled = api_scope.is_facade_enabled(facade.name) + self._facade_availability_info.add_facade(facade, is_facade_enabled) + if is_facade_enabled: + if facade.name in self._facades: + raise ValueError(f"Duplicate facade name: {facade.name}") + self._facades[facade.name] = facade + setattr(self, facade.name, facade) registered_facade_names.append(facade.name) log.info("Registered %d/%d facades: %s", len(registered_facade_names), len(facades), registered_facade_names) + def get_facade_availability_info(self) -> FacadeAvailabilityInfo: + """ + :return: the availability of all facades and their methods (enabled or disabled) + """ + return self._facade_availability_info + def get_enabled_methods(self) -> list[FacadeMethod]: """ :return: the list of all enabled methods across all facades @@ -117,12 +159,6 @@ class SerenaReplEntrypoint: del self._current_namespace[name] return f"Removed {len(items)} persisted item(s)." - def _register(self, facade: Facade) -> None: - if facade.name in self._facades: - raise ValueError(f"Duplicate facade name: {facade.name}") - self._facades[facade.name] = facade - setattr(self, facade.name, facade) - def _get_facade(self, name: str) -> Facade: if name not in self._facades: raise ValueError(f"Unknown facade '{name}'. Available facades: {list(self._facades)}") diff --git a/src/serena/resources/dashboard/dashboard.css b/src/serena/resources/dashboard/dashboard.css index f8e4e1af..08514e77 100644 --- a/src/serena/resources/dashboard/dashboard.css +++ b/src/serena/resources/dashboard/dashboard.css @@ -603,6 +603,23 @@ code, pre, kbd, samp, cursor: default; } +.facade-block { + margin-bottom: 12px; +} + +.facade-name { + font-size: 13px; + font-weight: 600; + color: var(--text-primary); + margin-bottom: 4px; +} + +.facade-name.disabled, +.tool-item.disabled { + color: var(--text-secondary); + opacity: 0.5; +} + /* Projects List */ .project-item { padding: 10px 12px; diff --git a/src/serena/resources/dashboard/dashboard.js b/src/serena/resources/dashboard/dashboard.js index 74308c93..45330466 100644 --- a/src/serena/resources/dashboard/dashboard.js +++ b/src/serena/resources/dashboard/dashboard.js @@ -614,6 +614,7 @@ class Dashboard { const $existingToolsContent = $('#tools-content'); const $existingMemoriesContent = $('#memories-content'); const wasToolsExpanded = $existingToolsContent.is(':visible'); + const wasFunctionsExpanded = $('#functions-content').is(':visible'); const wasMemoriesExpanded = $existingMemoriesContent.is(':visible'); let html = '
'; @@ -634,10 +635,14 @@ class Dashboard { html += '
' + (config.active_project.name || 'None') + '
'; } - html += '
Languages:
'; - if (this.jetbrainsMode) { - html += '
Using JetBrains backend
'; - } else { + html += '
Interface:
'; + html += '
' + config.agent_interface + '
'; + + html += '
Backend:
'; + html += '
' + config.language_backend + '
'; + + if (!this.jetbrainsMode) { + html += '
Languages:
'; html += '
'; if (config.languages && config.languages.length > 0) { html += '
'; @@ -705,6 +710,32 @@ class Dashboard { html += '
'; html += '
'; + // Active functions of the REPL's facades - collapsible (REPL interface only) + if (config.facades) { + const enabledMethodCount = config.facades.reduce(function (count, facade) { + return count + facade.methods.filter(function (method) { return method.is_enabled; }).length; + }, 0); + html += '
'; + html += '

'; + html += 'Active Functions (' + enabledMethodCount + ')'; + html += '▼'; + html += '

'; + html += '
'; + config.facades.forEach(function (facade) { + html += '
'; + html += '
s.' + facade.name + '
'; + html += '
'; + facade.methods.forEach(function (method) { + const title = facade.name + '.' + method.name + (method.is_enabled ? '' : ' (disabled)'); + html += '
' + method.name + '
'; + }); + html += '
'; + html += '
'; + }); + html += '
'; + html += '
'; + } + // Available memories - collapsible (show if memories exist or if project exists) if (config.active_project && config.active_project.name) { html += '
'; @@ -773,6 +804,15 @@ class Dashboard { $('#create-memory-btn').click(this.openCreateMemoryModal.bind(this)); // Re-attach collapsible handler for the newly created tools header + $('#functions-header').click(function () { + const $header = $(this); + const $content = $('#functions-content'); + const $icon = $header.find('.toggle-icon'); + + $content.slideToggle(300); + $icon.toggleClass('expanded'); + }); + $('#tools-header').click(function () { const $header = $(this); const $content = $('#tools-content'); From 42ef92b6162c7db1ddc0a408f0179cbabe951bcc Mon Sep 17 00:00:00 2001 From: Dominik Jain Date: Tue, 15 Sep 2026 13:34:37 +0200 Subject: [PATCH 38/53] Handle imports for backward compatibility using mark_used --- src/serena/tools/tools_base.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/src/serena/tools/tools_base.py b/src/serena/tools/tools_base.py index 8efbd767..b80413fd 100644 --- a/src/serena/tools/tools_base.py +++ b/src/serena/tools/tools_base.py @@ -12,15 +12,16 @@ from mcp import Implementation from mcp.server.fastmcp import Context from mcp.server.fastmcp.utilities.func_metadata import FuncMetadata, func_metadata from sensai.util import logging +from sensai.util.helper import mark_used from sensai.util.string import dict_string -from serena.code_editor import EditedFileContext # noqa: F401 (re-exported for tools) +from serena.code_editor import EditedFileContext from serena.config.serena_config import LanguageBackend from serena.lsp.lsp_diagnostics import DiagnosticsContext from serena.memories.memory_manager import MemoryManager from serena.project import Project from serena.prompt_factory import PromptFactory -from serena.repl.facade import SUCCESS_RESULT # noqa: F401 (re-exported for tools) +from serena.repl.facade import SUCCESS_RESULT from serena.util.class_decorators import singleton from serena.util.inspection import iter_subclasses from serena.util.text_utils import TextOutputUtils @@ -31,6 +32,8 @@ if TYPE_CHECKING: from serena.code_editor import CodeEditor, LanguageServerCodeEditor from serena.symbol import LanguageServerSymbolRetriever + +mark_used(SUCCESS_RESULT, EditedFileContext) # backward compatibility log = logging.getLogger(__name__) T = TypeVar("T") From 750deac10f687bd13463b75073da781a343ceef4 Mon Sep 17 00:00:00 2001 From: Dominik Jain Date: Tue, 15 Sep 2026 15:13:15 +0200 Subject: [PATCH 39/53] SerenaConfig: Configure agent_interface default for existing users To ensure backward compatibility, set the default to TOOLS when migrating existing users' configuration files. --- src/serena/config/serena_config.py | 12 +++++++++--- src/serena/resources/serena_config.template.yml | 5 ++--- 2 files changed, 11 insertions(+), 6 deletions(-) diff --git a/src/serena/config/serena_config.py b/src/serena/config/serena_config.py index c7809f57..b6c8a65c 100644 --- a/src/serena/config/serena_config.py +++ b/src/serena/config/serena_config.py @@ -992,6 +992,12 @@ class SerenaConfig(SharedConfig, ModeSelectionDefinitionWithBaseModes): # settings with overridden defaults + agent_interface: AgentInterface = AgentInterface.TOOLS + """ + the agent interface to use (unless overridden by the active project's configuration). + Defaults to TOOLS for backward compatibility (as users without this settings will get this default). + The default for new users is defined in the template file. + """ language_backend: LanguageBackend = LanguageBackend.LSP """ the language backend to use for code understanding features @@ -1171,10 +1177,10 @@ class SerenaConfig(SharedConfig, ModeSelectionDefinitionWithBaseModes): instance.language_backend = language_backend # determine agent interface - agent_interface_value = loaded_commented_yaml.get("agent_interface") - agent_interface: AgentInterface | None = None + agent_interface: AgentInterface | None = get_dataclass_default(SerenaConfig, "agent_interface") if "agent_interface" in loaded_commented_yaml: - agent_interface = AgentInterface.from_str(agent_interface_value) if agent_interface_value else None + agent_interface_value = loaded_commented_yaml["agent_interface"] + agent_interface = AgentInterface.from_str(agent_interface_value) else: num_migrations += 1 instance.agent_interface = agent_interface diff --git a/src/serena/resources/serena_config.template.yml b/src/serena/resources/serena_config.template.yml index f7163f2a..ea8e3d17 100644 --- a/src/serena/resources/serena_config.template.yml +++ b/src/serena/resources/serena_config.template.yml @@ -15,9 +15,8 @@ language_backend: LSP # The set of tools is fixed (the REPL tool and the tools required for session management), and the tool # inclusion/exclusion settings do not apply; instead, the operations available in the REPL are configured # via the API inclusion/exclusion settings (excluded_apis etc.). -# Leave empty to use Serena's default (currently "tools"). -# The interface is fixed at startup and can be overridden per project. -agent_interface: +# The interface is fixed at startup; it can be overridden by the project activated at startup. +agent_interface: tools # line ending convention to use when writing source files. # Possible values: "lf" (Unix), "crlf" (Windows), "native" (platform default). From 680a7e8fc18a14642cd505cb70fa8b53c4230fef Mon Sep 17 00:00:00 2001 From: Dominik Jain Date: Tue, 15 Sep 2026 15:21:38 +0200 Subject: [PATCH 40/53] Improve REPL availability policy in memory --- .serena/memories/repl.md | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/.serena/memories/repl.md b/.serena/memories/repl.md index 83403e4d..88db9700 100644 --- a/.serena/memories/repl.md +++ b/.serena/memories/repl.md @@ -88,8 +88,15 @@ expression is the result. No `return` (a top-level `return` yields a SyntaxError ## Availability policy -- Keep as much functionality as possible in the REPL; do not mirror the contexts' tool exclusions. - Reads must stay in (composability); exclusions can only steer the model, never enforce anything. +- Keep as much functionality as possible in the REPL; exclude nothing by default. + * Do not derive API exclusions from tool exclusions automatically (not via the `corresponding_tool` + correspondence, not via an option): contexts exclude tools mostly because the *host* provides equivalents + (`read_file`, `find_file`, `replace_content`, shell). In the REPL those reads are what makes operations + composable (read → filter → return a summary), and a host tool cannot participate in REPL code. + * The host's better-integrated edit tools (diff view, undo) are a matter of guidance in the context prompt, + not of availability: exclusions can only steer the model, never enforce anything. + * For users migrating from tool mode: prefer a startup hint listing the `excluded_apis` entries corresponding + to their own (global/project, not context/mode) tool exclusions over any automatic derivation. - Python code can always modify the system; the REPL tool is inherently fully privileged, regardless of facade scope or the project's `read_only` setting (which only makes Serena's own API refuse edits). A "read-only REPL" is not feasible and must not be promised. From 4b54fa737f0c096b14489db390c5ff5994de30e6 Mon Sep 17 00:00:00 2001 From: Dominik Jain Date: Tue, 15 Sep 2026 16:26:56 +0200 Subject: [PATCH 41/53] Introduce optional facades; make facade availability derived Facade availability: * facades can be optional (`Facade.from_api(..., is_optional=True)`), mirroring optional tools * `Facade.is_enabled()` is derived: a facade is available iff it has at least one enabled method * `ApiScope.is_facade_enabled` is thereby obsolete and removed Opt-in rule (uniform for optional facades, excluded facades and optional methods): * methods of a facade which is not included and optional methods require explicit inclusion * all other methods are enabled unless explicitly excluded Application: * the `ext` facade is optional; the query-projects mode includes it (`included_apis: [ext]`) --- .serena/memories/repl.md | 5 ++- src/serena/agent.py | 2 +- src/serena/repl/facade.py | 42 ++++++++++++------- src/serena/repl/repl.py | 16 +++---- .../resources/config/modes/query-projects.yml | 2 + test/serena/test_external_projects.py | 3 ++ test/serena/test_repl_tool.py | 16 +++++++ 7 files changed, 59 insertions(+), 27 deletions(-) diff --git a/.serena/memories/repl.md b/.serena/memories/repl.md index 88db9700..b7a6683e 100644 --- a/.serena/memories/repl.md +++ b/.serena/memories/repl.md @@ -83,7 +83,10 @@ expression is the result. No `return` (a top-level `return` yields a SyntaxError (a capability constraint, e.g. clients that handle the REPL badly), with the user's preference choosing among them. - `included_apis`/`excluded_apis` (references `facade` or `facade.method`) in global config, context, modes, project config; applied in that order via `ApiScope` (exclusions first, then inclusions; later definitions win). - Optional methods and all methods of an excluded facade must be included explicitly. +- Opt-in rule: methods of a facade that is not included (excluded, or optional without explicit inclusion) and + optional methods are enabled only if included explicitly; all other methods are enabled unless excluded. + Facades can be optional (`Facade.from_api(..., is_optional=True)`, e.g. `ext`, mirroring optional tools); + `Facade.is_enabled()` is derived: a facade is available iff it has at least one enabled method. - The REPL is rebuilt whenever the active tools are updated (mode switch, project activation). ## Availability policy diff --git a/src/serena/agent.py b/src/serena/agent.py index c0927467..1815160e 100644 --- a/src/serena/agent.py +++ b/src/serena/agent.py @@ -1344,7 +1344,7 @@ class SerenaAgent: Facade.from_api(EditApi(self), api_scope), Facade.from_api(MemoryApi(self), api_scope), Facade.from_api(ShellApi(self), api_scope), - Facade.from_api(ExternalProjectsApi(self), api_scope), + Facade.from_api(ExternalProjectsApi(self), api_scope, is_optional=True), ] if self._language_backend.is_lsp(): facades.append(Facade.from_api(LspApi(self), api_scope)) diff --git a/src/serena/repl/facade.py b/src/serena/repl/facade.py index cb66a012..7de3f172 100644 --- a/src/serena/repl/facade.py +++ b/src/serena/repl/facade.py @@ -478,17 +478,23 @@ class ApiScope: """ def __init__(self) -> None: - self.is_included = True + self._is_included: bool | None = None self.method_inclusions: set[str] = set() self.method_exclusions: set[str] = set() def exclude_facade(self) -> None: - self.is_included = False + self._is_included = False self.method_inclusions = set() self.method_exclusions = set() def include_facade(self) -> None: - self.is_included = True + self._is_included = True + + def is_facade_included(self, is_facade_optional: bool) -> bool: + if is_facade_optional: + return self._is_included is True + else: + return self._is_included is not False def exclude_method(self, method_name: str) -> None: self.method_inclusions.discard(method_name) @@ -537,23 +543,24 @@ class ApiScope: """ self._editing_excluded = True - def is_facade_enabled(self, facade_name: str) -> bool: - facade_scope = self._get_facade_scope(facade_name) - return facade_scope.is_included or len(facade_scope.method_inclusions) > 0 - - def is_method_enabled(self, facade_name: str, method_info: FacadeMethodInfo) -> bool: + def is_method_enabled(self, facade_name: str, method_info: FacadeMethodInfo, is_facade_optional: bool) -> bool: """ :param facade_name: the name of the facade :param method_info: the method's metadata - :return: whether the method is enabled: optional methods (and all methods of an excluded facade) must be - explicitly included, other methods are enabled unless explicitly excluded; if editing is excluded, - editing methods are always disabled + :param is_facade_optional: whether the facade is optional (disabled by default and must be enabled explicitly) + :return: whether the method is enabled: optional methods (and all methods of a facade which is not included, + i.e. an excluded facade or an optional facade that was not explicitly included) must be explicitly + included, other methods are enabled unless explicitly excluded; if editing is excluded, editing + methods are always disabled """ if self._editing_excluded and method_info.can_edit: return False facade_scope = self._get_facade_scope(facade_name) - if method_info.optional or not facade_scope.is_included: + # A method that would be disabled because the facade it is part of is not included + # or the method itself is optional must be explicitly included in order to be enabled. + if not facade_scope.is_facade_included(is_facade_optional) or method_info.optional: return method_info.name in facade_scope.method_inclusions + # A method that is not optional and whose facade is included is enabled unless it is explicitly excluded. else: return method_info.name not in facade_scope.method_exclusions @@ -585,12 +592,13 @@ class Facade: self._methods[method.name] = method @staticmethod - def from_api(api: FacadeApi, api_scope: ApiScope) -> "Facade": + def from_api(api: FacadeApi, api_scope: ApiScope, *, is_optional: bool = False) -> "Facade": """ Creates a facade wrapping the given implementation. :param api: the implementation; each of its methods decorated with `facade_method` becomes a facade method :param api_scope: API scope definition determining which methods are enabled + :param is_optional: whether the facade is optional (disabled by default and must be enabled explicitly) :return: the facade """ facade = Facade(api.get_name_(), api.get_description_(), api.get_referenced_types_()) @@ -598,11 +606,17 @@ class Facade: method_info = get_facade_method_info(member) if method_info is None: continue - is_enabled = api_scope.is_method_enabled(facade.name, method_info) + is_enabled = api_scope.is_method_enabled(facade.name, method_info, is_facade_optional=is_optional) facade._add_method(FacadeMethod(facade, member, method_info, enabled=is_enabled)) facade._discover_referenced_types() return facade + def is_enabled(self) -> bool: + """ + :return: whether the facade is enabled + """ + return len(self.get_enabled_methods()) > 0 + def _discover_referenced_types(self) -> None: """ Adds referenced types for all classes reachable (transitively) through the annotations of the facade's methods diff --git a/src/serena/repl/repl.py b/src/serena/repl/repl.py index 1e697738..c9c29fe1 100644 --- a/src/serena/repl/repl.py +++ b/src/serena/repl/repl.py @@ -41,14 +41,9 @@ class FacadeAvailabilityInfo: def __init__(self): self.facades: list[FacadeAvailabilityInfo.FacadeInfo] = [] - def add_facade(self, facade: Facade, is_enabled: bool): - def is_method_enabled(m: FacadeMethod) -> bool: - return is_enabled and m.enabled - - methods_info = [ - FacadeAvailabilityInfo.MethodInfo(name=method.name, is_enabled=is_method_enabled(method)) for method in facade.get_methods() - ] - self.facades.append(FacadeAvailabilityInfo.FacadeInfo(name=facade.name, is_enabled=is_enabled, methods=methods_info)) + def add_facade(self, facade: Facade): + methods_info = [FacadeAvailabilityInfo.MethodInfo(name=method.name, is_enabled=method.enabled) for method in facade.get_methods()] + self.facades.append(FacadeAvailabilityInfo.FacadeInfo(name=facade.name, is_enabled=facade.is_enabled(), methods=methods_info)) class SerenaReplEntrypoint: @@ -68,9 +63,8 @@ class SerenaReplEntrypoint: self._facade_availability_info = FacadeAvailabilityInfo() registered_facade_names = [] for facade in facades: - is_facade_enabled = api_scope.is_facade_enabled(facade.name) - self._facade_availability_info.add_facade(facade, is_facade_enabled) - if is_facade_enabled: + self._facade_availability_info.add_facade(facade) + if facade.is_enabled(): if facade.name in self._facades: raise ValueError(f"Duplicate facade name: {facade.name}") self._facades[facade.name] = facade diff --git a/src/serena/resources/config/modes/query-projects.yml b/src/serena/resources/config/modes/query-projects.yml index ed140de9..9ceea8fa 100644 --- a/src/serena/resources/config/modes/query-projects.yml +++ b/src/serena/resources/config/modes/query-projects.yml @@ -6,3 +6,5 @@ excluded_tools: [] included_optional_tools: - list_queryable_projects - query_project +included_apis: + - ext diff --git a/test/serena/test_external_projects.py b/test/serena/test_external_projects.py index a53b5ed4..bee10c4d 100644 --- a/test/serena/test_external_projects.py +++ b/test/serena/test_external_projects.py @@ -86,6 +86,9 @@ def test_external_project_context_in_repl( server, port = project_server monkeypatch.setattr(ProjectServer, "PORT", port) # let the REPL's external project context use the test server + # enable the optional "ext" facade + serena_config.included_apis = ["ext"] + # the querying agent has another project active and queries the python test project agent = SerenaAgent(project="test_repo_typescript", serena_config=serena_config) agent.execute_task(lambda: None) diff --git a/test/serena/test_repl_tool.py b/test/serena/test_repl_tool.py index f02cbbcb..3be9dc42 100644 --- a/test/serena/test_repl_tool.py +++ b/test/serena/test_repl_tool.py @@ -217,6 +217,22 @@ class TestFacade: scope.process(ApiInclusionDefinition(**kwargs)) return scope + def test_optional_facade_is_opt_in(self) -> None: + # an optional facade is disabled unless it is included explicitly + facade = Facade.from_api(self.DummyApi(MagicMock()), ApiScope(), is_optional=True) + assert not facade.is_enabled() + assert facade.enabled_method_names == [] + + # including the facade enables its non-optional methods + facade = Facade.from_api(self.DummyApi(MagicMock()), self._scope(included_apis=["dummy"]), is_optional=True) + assert facade.is_enabled() + assert set(facade.enabled_method_names) == {"add", "secret", "rarely"} + + # including a single method enables the facade with just that method + facade = Facade.from_api(self.DummyApi(MagicMock()), self._scope(included_apis=["dummy.add"]), is_optional=True) + assert facade.is_enabled() + assert facade.enabled_method_names == ["add"] + def test_api_scope_facade_exclusion_and_method_inclusion(self) -> None: # excluding the facade disables everything facade = Facade.from_api(self.DummyApi(MagicMock()), self._scope(excluded_apis=["dummy"])) From bf2ce28b0eb5f512c6d8b9a18b0c5d2bf456256b Mon Sep 17 00:00:00 2001 From: Dominik Jain Date: Tue, 15 Sep 2026 16:46:34 +0200 Subject: [PATCH 42/53] Remove beta marker from JetBrains tools --- src/serena/tools/jetbrains_tools.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/src/serena/tools/jetbrains_tools.py b/src/serena/tools/jetbrains_tools.py index 9985957d..6f856ebc 100644 --- a/src/serena/tools/jetbrains_tools.py +++ b/src/serena/tools/jetbrains_tools.py @@ -4,7 +4,7 @@ import logging from typing import TYPE_CHECKING, Literal, cast from serena.symbol import SymbolDictGrouper -from serena.tools import Tool, ToolMarkerBeta, ToolMarkerOptional, ToolMarkerSymbolicEdit, ToolMarkerSymbolicRead +from serena.tools import Tool, ToolMarkerOptional, ToolMarkerSymbolicEdit, ToolMarkerSymbolicRead if TYPE_CHECKING: from serena.repl.api.jb_api import JetBrainsApi @@ -119,7 +119,7 @@ class JetBrainsFindSymbolTool(Tool, ToolMarkerSymbolicRead, ToolMarkerOptional, return {"name_path": "name_path_pattern"} -class JetBrainsMoveTool(Tool, ToolMarkerSymbolicEdit, ToolMarkerOptional, ToolMarkerBeta, JetBrainsApiMixin): +class JetBrainsMoveTool(Tool, ToolMarkerSymbolicEdit, ToolMarkerOptional, JetBrainsApiMixin): """ Moves a symbol, file or directory to a new location using the JetBrains backend, updating all references """ @@ -159,7 +159,7 @@ class JetBrainsMoveTool(Tool, ToolMarkerSymbolicEdit, ToolMarkerOptional, ToolMa return self._api().move(relative_path, name_path, target_relative_path, target_parent_name_path).represent() -class JetBrainsSafeDeleteTool(Tool, ToolMarkerSymbolicEdit, ToolMarkerOptional, ToolMarkerBeta, JetBrainsApiMixin): +class JetBrainsSafeDeleteTool(Tool, ToolMarkerSymbolicEdit, ToolMarkerOptional, JetBrainsApiMixin): """ Safely deletes a symbol using the JetBrains backend, checking for remaining usages first """ @@ -191,7 +191,7 @@ class JetBrainsSafeDeleteTool(Tool, ToolMarkerSymbolicEdit, ToolMarkerOptional, return self._api().safe_delete(relative_path, name_path, delete_even_if_used, propagate).represent() -class JetBrainsInlineSymbol(Tool, ToolMarkerSymbolicEdit, ToolMarkerOptional, ToolMarkerBeta, JetBrainsApiMixin): +class JetBrainsInlineSymbol(Tool, ToolMarkerSymbolicEdit, ToolMarkerOptional, JetBrainsApiMixin): """ Inlines a symbol using the JetBrains backend, replacing all call sites with the symbol's body """ @@ -378,7 +378,7 @@ class JetBrainsRenameTool(Tool, ToolMarkerSymbolicEdit, ToolMarkerOptional, JetB return self._api().rename(relative_path, new_name, name_path, rename_in_comments, rename_in_text_occurrences).represent() -class JetBrainsDebugTool(Tool, ToolMarkerOptional, ToolMarkerBeta, JetBrainsApiMixin): +class JetBrainsDebugTool(Tool, ToolMarkerOptional, JetBrainsApiMixin): """ Provides debugging functionality (run configs, breakpoints, stepping, inspection, and evaluation) via a persistent debug REPL connected to the JetBrains IDE. From a0bc785e0aff9139f68cf6666c7ca667d451dbc8 Mon Sep 17 00:00:00 2001 From: Dominik Jain Date: Wed, 16 Sep 2026 16:24:34 +0200 Subject: [PATCH 43/53] Improve facade method docstrings (such that first line is meaningful stand-alone) --- src/serena/repl/api/ext_api.py | 10 +++++++--- src/serena/repl/api/jb_api.py | 33 ++++++++++++++++++++------------- src/serena/repl/api/lsp_api.py | 27 +++++++++++++++++---------- src/serena/repl/api/mem_api.py | 5 +++-- 4 files changed, 47 insertions(+), 28 deletions(-) diff --git a/src/serena/repl/api/ext_api.py b/src/serena/repl/api/ext_api.py index c559935d..6554b098 100644 --- a/src/serena/repl/api/ext_api.py +++ b/src/serena/repl/api/ext_api.py @@ -82,12 +82,16 @@ class ExternalProjectsApi(FacadeApi): @facade_method(corresponding_tool=QueryProjectTool) def project_context(self, project_name: str) -> ExternalProjectContextManager: """ - Provides a context (for use in a `with` statement) within which all facades operate on the given external - project instead of the active one, with read-only access: e.g. - `with s.ext.project_context("other"): result = s.lsp.find_symbol("Foo")`. + Provides a context (for use in a `with` statement) within which all facades operate on the given external project + instead of the active one, with read-only access. + + Example: + `with s.ext.project_context("other"): result = s.lsp.find_symbol("Foo")` + Results obtained within the context can be used after it (they are self-contained). :param project_name: the name (or root path) of the project, as listed by `list_projects` :return: the context manager + """ return ExternalProjectContextManager(self._agent, project_name) diff --git a/src/serena/repl/api/jb_api.py b/src/serena/repl/api/jb_api.py index 2ed024f4..92728bdb 100644 --- a/src/serena/repl/api/jb_api.py +++ b/src/serena/repl/api/jb_api.py @@ -211,6 +211,7 @@ class JetBrainsApi(FacadeApi): ) -> JetBrainsSymbolCollection: """ Finds symbols and code entities (classes, methods, etc.) based on the given name path pattern. + The returned symbol information can be used for edits or further queries. Specify `depth > 0` to retrieve children (e.g., methods of a class). Important: through `search_deps=True` dependencies can be searched, which @@ -283,9 +284,9 @@ class JetBrainsApi(FacadeApi): @facade_method(corresponding_tool=JetBrainsFindReferencingSymbolsTool) def find_referencing_symbols(self, name_path: str, relative_path: str, max_answer_chars: int = -1) -> JetBrainsSymbolCollection: """ - Finds all symbols that reference the given symbol — its callers / usages / dependents, i.e. the - symbols whose own definition (e.g. a method body) contains a reference to it. For each, returns its - name path, file, and the surrounding line of code. + Finds all symbols that reference the given symbol (its callers / usages / dependents) + i.e. the symbols whose own definition (e.g. a method body) contains a reference to it. + For each, returns its name path, file, and the surrounding line of code. :param name_path: name path of the symbol for which to find references :param relative_path: the relative path to the file containing the symbol (must be a file, not a directory) @@ -317,8 +318,9 @@ class JetBrainsApi(FacadeApi): self, relative_path: str, depth: int = -1, max_answer_chars: int = -1, include_file_documentation: bool = False ) -> JetBrainsSymbolsOverview: """ - Gets an overview of the top-level symbols defined in the given file (classes, methods, fields) — its - STRUCTURE, without their bodies. This is the cheap, structure-first way to learn what a file + Gets an overview of the top-level symbols defined in the given file (classes, methods, fields). + + Returns STRUCTURE only, without bodies. This is the cheap, structure-first way to learn what a file contains: it costs far less context than reading the whole file. :param relative_path: the relative path to the file to get the overview of @@ -443,6 +445,7 @@ class JetBrainsApi(FacadeApi): ) -> JsonObject: """ Renames a symbol, file or directory throughout the codebase. + Note: renaming in comments/text is on a best-effort basis by the IDE; if the symbol name is non-unique, further verification is recommended. @@ -473,6 +476,7 @@ class JetBrainsApi(FacadeApi): ) -> JsonObject: """ Moves a symbol, file or directory to a different location and automatically updates all references to affected symbols. + **Important**: this should always be preferred to naive moving (e.g. via file system operations or edits) as it is much more reliable and efficient. It is always safe to use. For some symbols, moving may not be applicable, and will result in no edits and a suitable error message. @@ -510,6 +514,7 @@ class JetBrainsApi(FacadeApi): ) -> JsonObject: """ Safely deletes a symbol, file, or directory, checking for usages first and propagating deletion, if desired. + Propagation means it is possible to request deleting of usages and cleaning up of unused code. Propagation is powerful for cleaning up code but should be used with care. **Important**: this should always be preferred to naive deleting (e.g. via file system operations or edits). @@ -534,13 +539,13 @@ class JetBrainsApi(FacadeApi): @facade_method(beta=True, can_edit=True, corresponding_tool=JetBrainsInlineSymbol) def inline_symbol(self, name_path: str, relative_path: str, keep_definition: bool = False) -> JsonObject: """ - Inlines a symbol (usually a method/function, but also classes may be amenable to inlining, - which turns invocation into anonymous class creation), - replacing all call sites with the symbol's body. + Inlines a symbol, replacing all call sites with the symbol's body. + **Important**: this should always be preferred to naive inlining (e.g. via searching for references and editing them). - :param name_path: the name path of the symbol to inline. + :param name_path: the name path of the symbol to inline (usually a method/function, but also classes may be amenable to inlining, + which turns invocation into anonymous class creation) :param relative_path: the relative path to the file containing the symbol to inline. :param keep_definition: whether to keep the original method definition after inlining all call sites. May be ignored in some cases (e.g. when inlining a class). @@ -564,6 +569,7 @@ class JetBrainsApi(FacadeApi): ) -> JsonObject: """ Runs IDE inspections (code analysis) on the given file and returns the problems found. + This leverages the full power of JetBrains' static analysis engine, including language-specific inspections, type checking, potential bugs, code style issues, and more. @@ -591,8 +597,9 @@ class JetBrainsApi(FacadeApi): self, language: str | None = None, group_path_contains: str | None = None, max_answer_chars: int = -1 ) -> JsonObject: """ - Lists the available IDE inspections. Use this to discover which inspections can be passed - to `run_inspections` via `inspection_names`. + Lists available IDE inspections. + + Use this to discover which inspections can be passed to `run_inspections` via `inspection_names`. :param language: optional language to filter by (e.g. "Java", "Python", "Kotlin"). :param group_path_contains: optional substring to match against the inspection group path @@ -605,7 +612,7 @@ class JetBrainsApi(FacadeApi): # debugging - @facade_method(beta=True) + @facade_method() def debug_eval_info(self) -> str: """ Provides usage information for the debug REPL (method `debug_eval`) @@ -614,7 +621,7 @@ class JetBrainsApi(FacadeApi): """ return self._agent.prompt_factory.create_info_jet_brains_debug_repl() - @facade_method(beta=True, corresponding_tool=JetBrainsDebugTool) + @facade_method(corresponding_tool=JetBrainsDebugTool) def debug_eval(self, expression: str, repl_key: str = "default") -> str: """ Provides debugging functionality (run configs, breakpoints, stepping, inspection, and evaluation) diff --git a/src/serena/repl/api/lsp_api.py b/src/serena/repl/api/lsp_api.py index f7866f5b..675f7826 100644 --- a/src/serena/repl/api/lsp_api.py +++ b/src/serena/repl/api/lsp_api.py @@ -381,8 +381,9 @@ class LspApi(FacadeApi): @facade_method(uses_project_server=True, optional=True, corresponding_tool=RestartLanguageServerTool) def restart_language_server(self) -> str: """ - Restarts the language server(s). Use this only on explicit user request or after confirmation; - it may be necessary if a language server hangs. + Restarts the language server(s). + + Use this only on explicit user request or after confirmation; it may be necessary if a language server hangs. :return: a success message """ @@ -394,8 +395,9 @@ class LspApi(FacadeApi): @facade_method(uses_project_server=True, corresponding_tool=GetSymbolsOverviewTool) def get_symbols_overview(self, relative_path: str, depth: int = -1, max_answer_chars: int = -1) -> LspSymbolCollection: """ - Gets an overview of the top-level symbols defined in the given file (classes, methods, fields) — its - STRUCTURE, without their bodies. This is the cheap, structure-first way to learn what a file + Gets an overview of the symbols defined in the given file (classes, methods, fields, functions, etc.) + + Returns STRUCTURE only, without bodies. This is the cheap, structure-first way to learn what a file contains: it costs far less context than reading the whole file. :param relative_path: the relative path to the file to get the overview of @@ -450,6 +452,7 @@ class LspApi(FacadeApi): ) -> LspSymbolCollection: """ Finds symbols and code entities (classes, methods, etc.) based on the given name path pattern. + The returned symbol information can be used for edits or further queries. Specify `depth > 0` to also retrieve children/descendants (e.g., methods of a class). @@ -534,8 +537,9 @@ class LspApi(FacadeApi): max_answer_chars: int = -1, ) -> LspReferenceCollection: """ - Finds references to the symbol at the given `name_path`. The result will contain metadata about the referencing symbols - as well as a short code snippet around the reference. + Finds references to the symbol at the given `name_path`. + + The result will contain metadata about the referencing symbols as well as a short code snippet around the reference. :param name_path: name path of the symbol :param relative_path: the relative path to the file containing the symbol for which to find references. @@ -656,7 +660,9 @@ class LspApi(FacadeApi): self, relative_path: str, start_line: int = 0, end_line: int = -1, min_severity: int = 4, max_answer_chars: int = -1 ) -> LspDiagnostics: """ - Gets diagnostics for a file. Diagnostics are grouped as `relative_path -> severity -> name_path -> diagnostics_results`. + Gets diagnostics for a file. + + Diagnostics are grouped as `relative_path -> severity -> name_path -> diagnostics_results`. If a diagnostic cannot be mapped to a symbol, it is grouped under the special name path ``. :param relative_path: the relative path to the file to inspect. @@ -694,9 +700,10 @@ class LspApi(FacadeApi): max_answer_chars: int = -1, ) -> LspDiagnostics: """ - Gets diagnostics for the specified symbol. When `check_symbol_references` is true, diagnostics for all - referencing symbols are also included. The result is grouped as - `relative_path -> severity -> name_path -> diagnostics_results`. + Gets diagnostics for the specified symbol. + + When `check_symbol_references` is true, diagnostics for all referencing symbols are also included. + The result is grouped as `relative_path -> severity -> name_path -> diagnostics_results`. :param name_path: the name path of the symbol to inspect. :param reference_file: optional file path used to disambiguate the symbol search. diff --git a/src/serena/repl/api/mem_api.py b/src/serena/repl/api/mem_api.py index d6e41f76..91f2803a 100644 --- a/src/serena/repl/api/mem_api.py +++ b/src/serena/repl/api/mem_api.py @@ -78,7 +78,7 @@ class MemoryApi(FacadeApi): @facade_method(corresponding_tool=ReadMemoryTool) def read_memory(self, memory_name: str) -> str: """ - Reads a memory that is likely to be relevant to the current task, inferring relevance e.g. from the name. + Reads a memory. :param memory_name: the name of the memory :return: the memory's content @@ -88,7 +88,8 @@ class MemoryApi(FacadeApi): @facade_method(can_edit=True, corresponding_tool=WriteMemoryTool) def write_memory(self, memory_name: str, content: str, max_chars: int = -1) -> str: """ - Writes information about this project that can be useful for future tasks in md format. + Writes information (about the active project) to a memory. + The name should be meaningful and can include "/" to organize into topics. If explicitly instructed, use the "global/" prefix for writing a memory that is shared across projects. References to other memories should be inside backticks and prefixed with mem:, From da1accf7e168427c154bbf3aa91bdc0334e76214 Mon Sep 17 00:00:00 2001 From: Dominik Jain Date: Wed, 16 Sep 2026 16:44:11 +0200 Subject: [PATCH 44/53] Add documentation on the REPL interface * Tools section: Add REPL interface description, advantages and full list of functions * Config section: Add information on REPL configuration --- docs/02-usage/050_configuration.md | 33 +++++++- docs/autogen_docs.py | 124 ++++++++++++++++++++++++----- src/serena/agent.py | 30 ++++--- src/serena/config/serena_config.py | 15 ++++ src/serena/repl/facade.py | 11 ++- 5 files changed, 172 insertions(+), 41 deletions(-) diff --git a/docs/02-usage/050_configuration.md b/docs/02-usage/050_configuration.md index 980cc7a8..d52ec342 100644 --- a/docs/02-usage/050_configuration.md +++ b/docs/02-usage/050_configuration.md @@ -26,7 +26,7 @@ Some of the configurable settings include: * the language backend to use by default (i.e., the JetBrains plugin or language servers); this can also be [overridden per project](per-project-language-backend) * UI settings affecting the [Serena Dashboard and GUI tool](060_dashboard.md) - * the set of tools to enable/disable by default + * the set of tools or REPL API functions to enable/disable by default * the set of [modes](modes) to use by default * tool execution parameters (timeout, max. answer length) * global ignore rules @@ -55,6 +55,37 @@ You can access it ```shell serena config edit ``` + +(agent-interfaces)= +### Agent Interfaces + +Serena provides its functionality to the agent (LLM) through one of two interfaces +(see [Tools and APIs](../01-about/035_tools) for the operations they offer): + +* **tools**: every operation is a separate tool of the MCP server. +* **REPL** (new in Serena v2): a single tool executes Python code, through which the agent accesses the operations + programmatically, being able to combine several of them in one call. + +The interface is selected via the `agent_interface` setting in the global configuration. +It can be overridden in the project configuration or via the `--agent-interface` command-line option, +and it is fixed for the duration of a session. + +The two interfaces are configured differently: + +* With the **tool interface**, the set of tools results from the tool inclusion/exclusion settings + (`excluded_tools`, `included_optional_tools`, `fixed_tools`) of the global configuration, the context, + the modes and the project configuration. +* With the **REPL interface**, the set of tools is fixed (the REPL tool and the tools which have no + counterpart within the REPL, e.g. for project activation); the tool settings above consequently do not apply. + The operations available *within* the REPL are configured via `included_apis`/`excluded_apis` instead, + which are supported in the same configuration layers and reference either a group of operations + (e.g. `lsp`) or an individual operation (e.g. `lsp.find_symbol`). + +```{note} +Restricting the operations available in the REPL is a means of steering the agent, not a security mechanism: +the Python code that is executed can, in principle, do anything the Serena process can do. +See [Security](070_security) for isolation options. +``` ## Modes and Contexts diff --git a/docs/autogen_docs.py b/docs/autogen_docs.py index 43976fd9..be8bc3ff 100644 --- a/docs/autogen_docs.py +++ b/docs/autogen_docs.py @@ -150,30 +150,110 @@ def autogen_tool_list(target_filename = "01-about/035_tools.md"): from serena.tools import ToolRegistry target_file = Path(__file__).parent / target_filename - with open(target_file, "w") as f: + with open(target_file, "w", encoding="utf-8") as f: f.write("\n\n") - f.write("# Tools\n\n") - f.write("Find the full list of Serena's tools below.\n\n") - f.write("Note that in most configurations, only a subset of these tools will be enabled simultaneously.\n") - f.write("Tools marked as *optional* are disabled by default.\n\n") - f.write("Tools marked as [BETA] were recently introduced and may not be fully robust yet.\n\n") - tools_by_module = ToolRegistry().get_registered_tools_by_module() - priority_modules = {"serena.tools.symbol_tools": 1, "serena.tools.jetbrains_tools": 2} + f.write("# Tools and APIs\n\n") + f.write( + "Serena provides an agent (LLM) with its functionality through one of two interfaces (configured in Serena's [global configuration](global-config)):\n\n" + "* **Tools** (the classic interface): every operation is a separate tool of the MCP server.\n" + "* **REPL** (new in Serena v2): a single tool executes Python code, through which the agent accesses the operations\n" + " programmatically. The agent can thus combine several operations in one call, process the results\n" + " in code and return only the information it actually needs.\n\n" + "While both interfaces offer the same general functionality for the most part, " + "the REPL interface addresses several limitations inherent in the tool-based approach (see [advantages](repl-advantages) below).\n\n" + ) - text = TextBuilder() - sorted_modules = sorted(tools_by_module.keys(), key=lambda m: (priority_modules.get(m, 3), m)) - for module in sorted_modules: - tools = tools_by_module[module] - module = module.replace("serena.tools.", "") - text.with_line(f"* **{module}**") - for tool in tools: - info = "" - if tool.is_optional: - info += " *(optional)*" - if tool.is_beta: - info += " [BETA]" - text.with_line(f"* `{tool.tool_name}`{info}: {tool.class_docstring}", indent=2) - f.write(text.build()) + def tools_section(): + f.write("## Tools (Classic Interface)\n\n") + f.write("Find the full list of Serena's tools below.\n\n") + f.write("Note that in most configurations, only a subset of these tools will be enabled simultaneously.\n") + f.write("Tools marked as *optional* are disabled by default.\n\n") + f.write("Tools marked as [BETA] were recently introduced and may not be fully robust yet.\n\n") + tools_by_module = ToolRegistry().get_registered_tools_by_module() + priority_modules = {"serena.tools.symbol_tools": 1, "serena.tools.jetbrains_tools": 2} + + text = TextBuilder() + sorted_modules = sorted(tools_by_module.keys(), key=lambda m: (priority_modules.get(m, 3), m)) + for module in sorted_modules: + tools = tools_by_module[module] + module = module.replace("serena.tools.", "") + text.with_line(f"* **{module}**") + for tool in tools: + info = "" + if tool.is_optional: + info += " *(optional)*" + if tool.is_beta: + info += " [BETA]" + text.with_line(f"* `{tool.tool_name}`{info}: {tool.class_docstring}", indent=2) + f.write(text.build()) + + def facades_section(): + from serena.repl.facade import ApiScope + from serena.agent import SerenaAgent, SerenaConfig + from serena.config.serena_config import LanguageBackend + + f.write("\n\n## Serena's REPL (Code Execution-Based Interface)\n\n") + f.write( + "With the REPL interface, the agent uses a single tool, which executes Python code. The code accesses\n" + "Serena's functionality through the entrypoint object `s`, whose attributes are *facades*, each of which\n" + "groups the operations of one domain. For instance, this code finds a class and returns the names of its\n" + "members in a single call:\n\n" + "```python\n" + 'result = s.lsp.find_symbol("SerenaAgent", depth=1)\n' + "[member.name for member in result.symbols[0].iter_children()]\n" + "```\n\n" + "The agent retrieves the details it requires (signatures, documentation, result types) at runtime via\n" + "`s.info(...)` (principle of *progressive disclosure*).\n\n" + "As with tools, only a subset of the operations is available in a given configuration:\n\n" + "* The facades `lsp` and `jb` are mutually exclusive, being tied to the respective language backend.\n" + "* Operations marked as *optional*, as well as all operations of facades marked as *optional*,\n" + " are disabled by default and must be enabled explicitly.\n\n" + "(repl-advantages)=\n\n" + "### Advantages\n\n" + "* **Composition**: The agent can combine several operations in a single call, using control flow,\n" + " filtering and aggregation. Multi-step retrievals that would otherwise require a series of\n" + " round-trips (find a symbol, inspect its members, find their references) become one call.\n" + "* **Context economy**: Only the result the agent actually needs enters the conversation.\n" + " Intermediate results remain in the Python runtime instead of consuming the context window\n" + " — which, for large result sets, is the difference between a summary and thousands of tokens.\n" + "* **Results as objects**: Operations return objects which can be processed programmatically\n" + " rather than plain text.\n" + "* **Progressive disclosure**: Instead of the schemas of dozens of tools, the agent is given a list of\n" + " the available operations, retrieving the details it requires (signatures, documentation, result\n" + " types) on demand. This eliminates the need for a client-specific tool search/dynamic tool discovery mechanism.\n" + "* **Reusability within a session**: Variables and helper functions defined by the agent persist\n" + " across calls, allowing intermediate results to be revisited and recurring logic to be applied\n" + " repeatedly.\n" + "* **Dynamic adaptation**: The set of available operations can change at runtime.\n" + " For MCP tools, changes to the toolset are not widely supported by clients and are therefore not\n" + " applied; within the REPL, the operations can follow the current situation.\n" + " This is what allows the language backend to be switched during a session, e.g. when a project\n" + " whose configuration demands a different backend is activated..\n\n" + "### List of Facades\n\n" + ) + api_scope = ApiScope() + agent = SerenaAgent(serena_config=SerenaConfig().with_headless_mode_overrides()) + facades = [] + for backend in LanguageBackend: + facades.extend(backend.create_facades(agent, api_scope)) + facades.extend(agent.create_default_facade_list(api_scope)) + + text = TextBuilder() + for facade in facades: + facade_info = " *(optional)*" if facade.is_optional() else "" + text.with_line(f"* **{facade.name}**{facade_info}: {facade.description}") + for method in facade.get_methods(): + method_info = "" + if method.info.optional: + method_info += " *(optional)*" + if method.info.beta: + method_info += " [BETA]" + summary = method.get_summary().replace("`", "") + text.with_line(f"* `{method.qualified_name}`{method_info}: {summary}", indent=2) + f.write(text.build()) + + tools_section() + facades_section() def autogen_about_intro_features(): diff --git a/src/serena/agent.py b/src/serena/agent.py index 1815160e..943f4543 100644 --- a/src/serena/agent.py +++ b/src/serena/agent.py @@ -53,8 +53,6 @@ from serena.repl.api.cfg_api import ConfigApi from serena.repl.api.edit_api import EditApi from serena.repl.api.ext_api import ExternalProjectsApi from serena.repl.api.fs_api import FsApi -from serena.repl.api.jb_api import JetBrainsApi -from serena.repl.api.lsp_api import LspApi from serena.repl.api.mem_api import MemoryApi from serena.repl.api.shell_api import ShellApi from serena.repl.facade import ApiScope, Facade @@ -1319,6 +1317,19 @@ class SerenaAgent: "Consider adjusting your configuration to include these tools if you want to use them." ) + def create_default_facade_list(self, api_scope: ApiScope) -> list[Facade]: + """ + :return: the default list of facades provided by Serena itself, not including any language backend-specific facades + """ + return [ + Facade.from_api(ConfigApi(self), api_scope), + Facade.from_api(FsApi(self), api_scope), + Facade.from_api(EditApi(self), api_scope), + Facade.from_api(MemoryApi(self), api_scope), + Facade.from_api(ShellApi(self), api_scope), + Facade.from_api(ExternalProjectsApi(self), api_scope, is_optional=True), + ] + def get_repl(self) -> SerenaRepl: """ :return: the REPL instance for this agent, creating it if necessary @@ -1337,20 +1348,7 @@ class SerenaAgent: if not self._is_dashboard_openable(self.serena_config): api_scope.process(NamedApiInclusionDefinition(name="Dashboard", excluded_apis=["cfg.open_dashboard"])) - # gather facades - facades = [ - Facade.from_api(ConfigApi(self), api_scope), - Facade.from_api(FsApi(self), api_scope), - Facade.from_api(EditApi(self), api_scope), - Facade.from_api(MemoryApi(self), api_scope), - Facade.from_api(ShellApi(self), api_scope), - Facade.from_api(ExternalProjectsApi(self), api_scope, is_optional=True), - ] - if self._language_backend.is_lsp(): - facades.append(Facade.from_api(LspApi(self), api_scope)) - elif self._language_backend.is_jetbrains(): - facades.append(Facade.from_api(JetBrainsApi(self), api_scope)) - + facades = self.create_default_facade_list(api_scope) + self._language_backend.create_facades(self, api_scope) self._repl = SerenaRepl(facades, api_scope) return self._repl diff --git a/src/serena/config/serena_config.py b/src/serena/config/serena_config.py index b6c8a65c..60cd5ec6 100644 --- a/src/serena/config/serena_config.py +++ b/src/serena/config/serena_config.py @@ -44,7 +44,9 @@ from ..util.cli_util import ask_yes_no from ..util.dataclass import get_dataclass_default if TYPE_CHECKING: + from ..agent import SerenaAgent from ..project import Project + from ..repl.facade import ApiScope, Facade from ..tools.tools_base import Tool log = logging.getLogger(__name__) @@ -294,6 +296,19 @@ class LanguageBackend(Enum): case _: raise NotImplementedError() + def create_facades(self, agent: "SerenaAgent", api_scope: "ApiScope") -> list["Facade"]: + from ..repl.facade import Facade + + if self.is_lsp(): + from ..repl.api.lsp_api import LspApi + + return [Facade.from_api(LspApi(agent), api_scope)] + elif self.is_jetbrains(): + from ..repl.api.jb_api import JetBrainsApi + + return [Facade.from_api(JetBrainsApi(agent), api_scope)] + return [] + class LineEnding(Enum): """Line ending convention for file writes.""" diff --git a/src/serena/repl/facade.py b/src/serena/repl/facade.py index 7de3f172..2f198fa7 100644 --- a/src/serena/repl/facade.py +++ b/src/serena/repl/facade.py @@ -570,8 +570,9 @@ class Facade: A named group of related operations which an LLM can invoke from REPL code. """ - def __init__(self, name: str, description: str, types: Sequence[ReferencedType] = ()) -> None: + def __init__(self, name: str, description: str, is_optional: bool = False, types: Sequence[ReferencedType] = ()) -> None: # NOTE: attributes are set via object.__setattr__ because __getattr__ is overridden + object.__setattr__(self, "_is_optional", is_optional) object.__setattr__(self, "_name", name) object.__setattr__(self, "_description", description) object.__setattr__(self, "_methods", {}) @@ -601,7 +602,7 @@ class Facade: :param is_optional: whether the facade is optional (disabled by default and must be enabled explicitly) :return: the facade """ - facade = Facade(api.get_name_(), api.get_description_(), api.get_referenced_types_()) + facade = Facade(api.get_name_(), api.get_description_(), is_optional, api.get_referenced_types_()) for name, member in inspect.getmembers(api, predicate=inspect.ismethod): method_info = get_facade_method_info(member) if method_info is None: @@ -617,6 +618,12 @@ class Facade: """ return len(self.get_enabled_methods()) > 0 + def is_optional(self) -> bool: + """ + :return: whether the facade is optional, i.e. disabled unless it is explicitly included + """ + return self._is_optional + def _discover_referenced_types(self) -> None: """ Adds referenced types for all classes reachable (transitively) through the annotations of the facade's methods From 0b2b4c3b56dd966e36fe2284beab2f87ee115167 Mon Sep 17 00:00:00 2001 From: Dominik Jain Date: Wed, 16 Sep 2026 16:58:29 +0200 Subject: [PATCH 45/53] Add REPL-specific aspects to the security documentation --- docs/02-usage/070_security.md | 30 ++++++++++++++++++++++++++++-- 1 file changed, 28 insertions(+), 2 deletions(-) diff --git a/docs/02-usage/070_security.md b/docs/02-usage/070_security.md index 249409cf..c8e6b652 100644 --- a/docs/02-usage/070_security.md +++ b/docs/02-usage/070_security.md @@ -23,18 +23,22 @@ However, reports which amount to noting that Serena's tools can execute commands describe intended functionality rather than vulnerabilities, and we will reject advisories that fail to recognise this or otherwise ignore the above assumptions. Sandboxing is the *only* way to fully protect against unintended consequences when using coding agents; -constraints on the tools themselves cannot achieve this and are therefore not an approach we pursue. +constraints on the tools or REPL APIs cannot achieve this and are therefore not an approach we pursue. ::: ## General Recommendations for Risk Reduction To reduce the risk of unintended consequences, we recommend that you: - back up your work regularly (keep the project being worked on under version control), -- restrict the set of allowed tools via the [configuration](050_configuration), +- restrict the set of allowed tools via the [configuration](050_configuration) + (note that restrictions are effective for the tool interface only, see [below](repl-security)), - do not expose [Serena's network services](network-security) to untrusted networks. If you do not fully trust the client/the LLM, we additionally recommend to monitor tool executions carefully (provided that your MCP client supports this). +Note that with the REPL interface, such monitoring is necessarily coarser: every action appears as the same tool +being called, and the client cannot tell a read-only call from a modifying one by the tool's name alone. +What needs to be reviewed is the submitted code. (sandboxing)= ## Sandboxing @@ -97,6 +101,28 @@ the introduction of this setting retain a pattern that trusts all projects, ensu not broken, whereas newly created configurations trust no project by default. The applicable value can be inspected in the dashboard. +(repl-security)= +## The REPL Interface + +With the [REPL interface](agent-interfaces), the agent does not invoke individual tools; it submits Python code, +which Serena executes. +In the default configuration, Serena is equally capable either way, as shell execution and file modification are +available in both interfaces. +The difference lies in what restrictions can achieve: + +- With the **tool interface**, excluding a tool removes the respective capability: a tool that is not exposed + cannot be invoked, so forbidding shell execution or file modification is effective. +- With the **REPL interface**, there is no such guarantee. + Restricting the available operations (`included_apis`/`excluded_apis`) steers the agent towards the intended + way of working, but the submitted code can, in principle, do anything the Serena process can do — irrespective + of the operations Serena itself provides. + Excluding shell execution, for example, does not prevent the code from achieving the same effect by other means. +- The `read_only` project setting is subject to the same limitation: Serena's own editing operations are refused, + yet code executed in the REPL is not prevented from modifying files. + +The assumptions stated above therefore apply unchanged, but if you require actual constraints rather than +guidance, [sandboxing](sandboxing) is the answer. + (network-security)= ## Network Security From 613f2ca098be1a5c42d8a01570a6d60d25506687 Mon Sep 17 00:00:00 2001 From: Dominik Jain Date: Thu, 17 Sep 2026 12:08:11 +0200 Subject: [PATCH 46/53] Support writable external project contexts - Rename the read-only context to read_project_context and add writable project_context. - Dispatch LSP-backed edits remotely and enforce read-only access in the calling context. - Allow project-server facade calls independently of the target project's API restrictions. - Update dispatch coverage for both backends and access modes; remove obsolete rejection assertions. --- src/serena/project_server.py | 5 +-- src/serena/repl/api/ext_api.py | 32 +++++++++++++++---- src/serena/repl/external_project.py | 38 +++++++++++++++------- src/serena/repl/facade.py | 24 +++++++------- src/serena/repl/repl.py | 6 ++-- test/serena/test_external_projects.py | 6 +--- test/serena/test_repl_tool.py | 46 ++++++++++++++++++++------- 7 files changed, 104 insertions(+), 53 deletions(-) diff --git a/src/serena/project_server.py b/src/serena/project_server.py index 4deaa7c3..4e8a69c8 100644 --- a/src/serena/project_server.py +++ b/src/serena/project_server.py @@ -36,7 +36,7 @@ class QueryProjectRequest(BaseModel): class CallFacadeMethodRequest(BaseModel): """ - Request model for the /call_facade_method endpoint: the execution of a (read-only) REPL facade method + Request model for the /call_facade_method endpoint: the execution of a REPL facade method in the context of a project. """ @@ -166,14 +166,11 @@ class ProjectServer: """ Handles a /call_facade_method request by executing the facade method on the agent's REPL facades in the context of the specified project (see `_query_project` regarding the lock). - Only methods which use the project server and do not edit are admissible. """ project = self._get_project(req.project_name) with self._active_project_lock, self._agent.active_project_context(project): facade = self._agent.get_repl().entrypoint.get_facade_(req.facade_name) method = facade.get_method(req.method_name) - if not method.enabled or method.info.can_edit or not method.info.uses_project_server: - raise ValueError(f"Method '{req.facade_name}.{req.method_name}' cannot be executed via the project server") return self._agent.execute_task(lambda: method(*req.args, **req.kwargs)) def run(self) -> None: diff --git a/src/serena/repl/api/ext_api.py b/src/serena/repl/api/ext_api.py index 6554b098..eed18827 100644 --- a/src/serena/repl/api/ext_api.py +++ b/src/serena/repl/api/ext_api.py @@ -9,7 +9,7 @@ from typing import TYPE_CHECKING from serena.jetbrains.jetbrains_plugin_client import JetBrainsPluginClientManager from serena.tools import ListQueryableProjectsTool, QueryProjectTool -from ..external_project import ExternalProjectContext +from ..external_project import ExternalProjectExecution from ..facade import FacadeApi, facade_method from ..representable import JsonObject, JsonObjectRenderer @@ -24,14 +24,16 @@ class ExternalProjectContextManager: executed in the project's server. Contexts cannot be nested. """ - def __init__(self, agent: "SerenaAgent", project_name: str) -> None: + def __init__(self, agent: "SerenaAgent", project_name: str, read_only: bool) -> None: """ :param agent: the agent :param project_name: the name (or root path) of the registered external project + :param read_only: whether the context is read-only """ self._agent = agent self._project_name = project_name self._active_project_context = None + self._read_only = read_only def __enter__(self) -> None: entrypoint = self._agent.get_repl().entrypoint @@ -46,9 +48,8 @@ class ExternalProjectContextManager: self._active_project_context = self._agent.active_project_context(project) self._active_project_context.__enter__() - # switch the facades to the external project (remote execution of language server operations for the LSP backend) - remote_execution = self._agent.get_language_backend().is_lsp() - entrypoint.set_external_project_(ExternalProjectContext(registered_project.project_name, remote_execution)) + # switch the facades to the external project + entrypoint.set_external_project_(ExternalProjectExecution(registered_project.project_name, self._read_only, self._agent)) def __exit__(self, exc_type: type[BaseException] | None, exc_value: BaseException | None, traceback: TracebackType | None) -> None: self._agent.get_repl().entrypoint.set_external_project_(None) @@ -80,7 +81,7 @@ class ExternalProjectsApi(FacadeApi): return JsonObject(result, JsonObjectRenderer(self._agent, -1)) @facade_method(corresponding_tool=QueryProjectTool) - def project_context(self, project_name: str) -> ExternalProjectContextManager: + def read_project_context(self, project_name: str) -> ExternalProjectContextManager: """ Provides a context (for use in a `with` statement) within which all facades operate on the given external project instead of the active one, with read-only access. @@ -94,4 +95,21 @@ class ExternalProjectsApi(FacadeApi): :return: the context manager """ - return ExternalProjectContextManager(self._agent, project_name) + return ExternalProjectContextManager(self._agent, project_name, read_only=True) + + @facade_method(optional=True) + def project_context(self, project_name: str) -> ExternalProjectContextManager: + """ + Provides a context (for use in a `with` statement) within which all facades operate on the given external project + instead of the active one (read and write operations are possible). + + Example: + `with s.ext.project_context("other"): result = s.lsp.find_symbol("Foo")` + + Results obtained within the context can be used after it (they are self-contained). + + :param project_name: the name (or root path) of the project, as listed by `list_projects` + :return: the context manager + + """ + return ExternalProjectContextManager(self._agent, project_name, read_only=False) diff --git a/src/serena/repl/external_project.py b/src/serena/repl/external_project.py index e702d4ff..1e909b39 100644 --- a/src/serena/repl/external_project.py +++ b/src/serena/repl/external_project.py @@ -8,8 +8,11 @@ from typing import TYPE_CHECKING, Any if TYPE_CHECKING: from serena.project_server import ProjectServerClient + from ..agent import SerenaAgent + from .facade import FacadeMethod -class ExternalProjectContext: + +class ExternalProjectExecution: """ The context in which facade methods are executed while an external project is being queried: methods which use the project server (see `FacadeMethodInfo.uses_project_server`) are executed remotely @@ -17,26 +20,39 @@ class ExternalProjectContext: locally against the temporarily switched project. Editing methods are not permitted. """ - def __init__(self, project_name: str, remote_execution: bool) -> None: + def __init__(self, project_name: str, read_only: bool, agent: "SerenaAgent") -> None: """ :param project_name: the name of the external project - :param remote_execution: whether methods using the project server are to be executed remotely - (False for the JetBrains backend, where the IDE serves all projects) + :param read_only: whether the external project is to be treated as read-only (editing methods are not permitted) """ self.project_name = project_name - self._remote_execution = remote_execution self._client: ProjectServerClient | None = None + self._read_only = read_only + self._agent = agent - def executes_remotely(self, uses_project_server: bool) -> bool: + def is_called_remotely(self, method: "FacadeMethod") -> bool: """ - :param uses_project_server: whether the method in question uses the project server - :return: whether the method is to be executed remotely + :param method: the method to check + :return: whether the given method must be executed remotely """ - return self._remote_execution and uses_project_server + # Any method that uses the project server must be executed remotely, + # as does any edit operation when using the LSP backend (as edit operations indirectly + # use the language server via the CodeEditor abstraction) + return method.info.uses_project_server or (self._agent.get_language_backend().is_lsp() and method.info.can_edit) - def call(self, facade_name: str, method_name: str, args: tuple[Any, ...], kwargs: dict[str, Any]) -> Any: + def check_call_permission(self, method: "FacadeMethod") -> None: """ - Executes the given facade method in the external project's server. + Checks whether the given method is permitted to be called in the context of this external project execution. + Raises an exception if the method is not permitted. + + :param method: the facade method to check + """ + if self._read_only and method.info.can_edit: + raise PermissionError(f"Editing methods are not permitted in read-only external project execution: {method.qualified_name}") + + def call_remotely(self, facade_name: str, method_name: str, args: tuple[Any, ...], kwargs: dict[str, Any]) -> Any: + """ + Executes the given facade method remotely via the project server. :param facade_name: the facade's name :param method_name: the method's name diff --git a/src/serena/repl/facade.py b/src/serena/repl/facade.py index 2f198fa7..cfe53a7c 100644 --- a/src/serena/repl/facade.py +++ b/src/serena/repl/facade.py @@ -26,7 +26,7 @@ if TYPE_CHECKING: from serena.code_editor import CodeEditor from serena.tools import Tool - from .external_project import ExternalProjectContext + from .external_project import ExternalProjectExecution log = logging.getLogger(__name__) TCallable = TypeVar("TCallable", bound=Callable[..., Any]) @@ -253,7 +253,10 @@ class FacadeMethodInfo: uses_project_server: bool = False """ whether the method requires the project's language servers and must therefore be executed in the project server - when an external project is queried (see `ExternalProjectContext`) + when an external project is queried (see `ExternalProjectContext`). + Edit operations are always executed in the project server, regardless of this flag, since they implicitly + use the CodeEditor, which requires language servers when using the LSP backend. + Polymorphic edit operations therefore must not set this flag to True. """ corresponding_tool: "type[Tool] | None" = None """the classic tool offering the same functionality, if any""" @@ -285,7 +288,7 @@ def facade_method( :param beta: whether the method is in beta :param can_edit: whether the method can modify the codebase :param niche: whether the method is rarely needed (its documentation is then only summarised in the facade's description) - :param uses_project_server: whether the method must be executed in the project server when an external project is queried + :param uses_project_server: whether the method must be executed remotely in the project server when an external project is queried :param corresponding_tool: the classic tool offering the same functionality, if any :return: the decorator """ @@ -406,12 +409,11 @@ class FacadeMethod: return f"{self.facade_name}.{self.name}" def __call__(self, *args: Any, **kwargs: Any) -> Any: - external_project = self.parent.get_external_project_() - if external_project is not None: - if self.info.can_edit: - raise ValueError(f"{self.qualified_name} cannot be called while an external project is being queried (read-only access)") - if external_project.executes_remotely(self.info.uses_project_server): - return external_project.call(self.facade_name, self.name, args, kwargs) + external_project_execution = self.parent.get_external_project_() + if external_project_execution is not None: + external_project_execution.check_call_permission(self) + if external_project_execution.is_called_remotely(self): + return external_project_execution.call_remotely(self.facade_name, self.name, args, kwargs) return self._implementation(*args, **kwargs) def get_implementation_(self) -> Callable[..., Any]: @@ -579,13 +581,13 @@ class Facade: object.__setattr__(self, "_types", {t.name: t for t in types}) object.__setattr__(self, "_external_project", None) - def set_external_project_(self, external_project: "ExternalProjectContext | None") -> None: + def set_external_project_(self, external_project: "ExternalProjectExecution | None") -> None: """ :param external_project: the context of the external project being queried (None if the active project is used) """ object.__setattr__(self, "_external_project", external_project) - def get_external_project_(self) -> "ExternalProjectContext | None": + def get_external_project_(self) -> "ExternalProjectExecution | None": return self._external_project def _add_method(self, method: FacadeMethod) -> None: diff --git a/src/serena/repl/repl.py b/src/serena/repl/repl.py index c9c29fe1..9dc1b7b1 100644 --- a/src/serena/repl/repl.py +++ b/src/serena/repl/repl.py @@ -12,7 +12,7 @@ from dataclasses import dataclass from typing import TYPE_CHECKING, Any from ..session import SerenaSession -from .external_project import ExternalProjectContext +from .external_project import ExternalProjectExecution from .facade import ApiScope, Facade, FacadeMethod, ReferencedType from .representable import Representable @@ -96,7 +96,7 @@ class SerenaReplEntrypoint: return True return False - def set_external_project_(self, external_project: "ExternalProjectContext | None") -> None: + def set_external_project_(self, external_project: "ExternalProjectExecution | None") -> None: """ :param external_project: the context of the external project being queried by the currently executing code (None if the active project is used); propagated to all facades @@ -104,7 +104,7 @@ class SerenaReplEntrypoint: for facade in self._facades.values(): facade.set_external_project_(external_project) - def get_external_project_(self) -> "ExternalProjectContext | None": + def get_external_project_(self) -> "ExternalProjectExecution | None": external_projects = {facade.get_external_project_() for facade in self._facades.values()} return next(iter(external_projects)) if external_projects else None diff --git a/test/serena/test_external_projects.py b/test/serena/test_external_projects.py index bee10c4d..a2bb73d7 100644 --- a/test/serena/test_external_projects.py +++ b/test/serena/test_external_projects.py @@ -71,10 +71,6 @@ def test_facade_method_results_are_transferred_from_the_project_server(project_s assert result.symbols[0].body.startswith("def create_user") assert "create_user" in result.represent() - # editing methods are refused - with pytest.raises(ValueError, match="cannot be executed"): - client.call_facade_method("test_repo_python", "lsp", "rename_symbol", ["create_user", "test_repo/services.py", "x"], {}) - @pytest.mark.python @pytest.mark.skipif(not language_server_tests_enabled(LanguageServerId.PYTHON), reason="python tests are disabled in this environment") @@ -96,7 +92,7 @@ def test_external_project_context_in_repl( tool = agent.get_tool(SerenaReplTool) session_id = agent.create_session().session_id code = ( - 'with s.ext.project_context("test_repo_python"):\n' + 'with s.ext.read_project_context("test_repo_python"):\n' ' result = s.lsp.find_symbol("create_user")\n' "[s.name for s in result.symbols]" ) diff --git a/test/serena/test_repl_tool.py b/test/serena/test_repl_tool.py index 3be9dc42..f3f0a5f5 100644 --- a/test/serena/test_repl_tool.py +++ b/test/serena/test_repl_tool.py @@ -7,10 +7,10 @@ from unittest.mock import MagicMock import pytest -from serena.config.serena_config import ApiInclusionDefinition +from serena.config.serena_config import ApiInclusionDefinition, LanguageBackend from serena.repl.api.edit_api import EditApi from serena.repl.api.lsp_api import LspApi -from serena.repl.external_project import ExternalProjectContext +from serena.repl.external_project import ExternalProjectExecution from serena.repl.facade import ApiScope, Facade, FacadeApi, FacadeMethodInfo, facade_method from serena.repl.repl import SerenaRepl from serena.session import SerenaSession @@ -83,29 +83,51 @@ class TestReplExecution: overview = rebuilt_repl.execute("facades()", session) assert "s.edit" in overview and "s.lsp" not in overview - def test_external_project_dispatch(self) -> None: - class FakeExternalProject(ExternalProjectContext): + @pytest.mark.parametrize("backend", [LanguageBackend.LSP, LanguageBackend.JETBRAINS]) + @pytest.mark.parametrize("read_only", [True, False]) + def test_external_project_dispatch(self, backend: LanguageBackend, read_only: bool) -> None: + agent = MagicMock() + agent.get_language_backend.return_value = backend + + class FakeExternalProject(ExternalProjectExecution): def __init__(self) -> None: - super().__init__("other", remote_execution=True) + super().__init__("other", read_only=read_only, agent=agent) self.calls: list[tuple[str, str, tuple, dict]] = [] - def call(self, facade_name: str, method_name: str, args: tuple, kwargs: dict) -> str: + def call_remotely(self, facade_name: str, method_name: str, args: tuple, kwargs: dict) -> str: self.calls.append((facade_name, method_name, args, kwargs)) return "remote result" - facades = [Facade.from_api(LspApi(MagicMock()), ApiScope()), Facade.from_api(EditApi(MagicMock()), ApiScope())] + class LocalApi(FacadeApi): + @facade_method(can_edit=True) + def write(self, content: str) -> str: + return f"local result: {content}" + + # expose a server-backed read and a backend-dependent write + facades = [Facade.from_api(LspApi(agent), ApiScope()), Facade.from_api(LocalApi(agent, "local", "local operations"), ApiScope())] repl = SerenaRepl(facades, ApiScope()) external_project = FakeExternalProject() repl.entrypoint.set_external_project_(external_project) - # methods using the project server are executed remotely, editing methods are refused + # methods explicitly requiring the project server are executed remotely assert repl.execute('s.lsp.find_symbol("Foo", depth=1)') == "remote result" assert external_project.calls == [("lsp", "find_symbol", ("Foo",), {"depth": 1})] - assert "read-only" in repl.execute('s.edit.replace_content("a.py", "x", "y", "literal")') - # without remote execution (JetBrains backend), methods run locally - repl.entrypoint.set_external_project_(ExternalProjectContext("other", remote_execution=False)) - assert "remote result" not in repl.execute('s.info("lsp.find_symbol")') + # writes obey the context's access mode and use the selected backend + result = repl.execute('s.local.write("content")') + if read_only: + assert "PermissionError" in result and "read-only" in result + assert len(external_project.calls) == 1 + elif backend.is_lsp(): + assert result == "remote result" + assert external_project.calls[-1] == ("local", "write", ("content",), {}) + else: + assert result == "local result: content" + assert len(external_project.calls) == 1 + + # leaving the external execution context restores local execution + repl.entrypoint.set_external_project_(None) + assert repl.execute('s.local.write("restored")') == "local result: restored" def test_facade_discovery(self, repl: SerenaRepl) -> None: overview = repl.execute("s.info()") From 5a2063fc3a39575cb54769451ac5a49fd20b6f79 Mon Sep 17 00:00:00 2001 From: Dominik Jain Date: Thu, 17 Sep 2026 12:48:53 +0200 Subject: [PATCH 47/53] Add a persistent authentication secret to SerenaConfig - Generate a random UUID when auth_secret is missing, null, or empty. - Persist generated secrets and preserve configured values across reloads. - Document the shared secret and test generation, persistence, and direct construction. --- CHANGELOG.md | 3 ++ src/serena/config/serena_config.py | 11 +++++++ .../resources/serena_config.template.yml | 4 +++ test/serena/config/test_serena_config.py | 31 +++++++++++++++++++ 4 files changed, 49 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 7aeb6dbc..3d3c13f0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -12,6 +12,9 @@ Status of the `main` branch. Changes prior to the next official version change w see `CONTRIBUTING.md` * General: + - Add `auth_secret` to `serena_config.yml` for authenticating communication between Serena components + and services. When missing, null, or empty, a random UUID is generated and persisted; existing values + are preserved - Fix: MCP `initialize` now reports Serena's version instead of the installed mcp SDK version (#1889) - Fix: Parallel agents auto-registering projects could overwrite each other's changes to the global project list in `serena_config.yml` diff --git a/src/serena/config/serena_config.py b/src/serena/config/serena_config.py index 60cd5ec6..de47c016 100644 --- a/src/serena/config/serena_config.py +++ b/src/serena/config/serena_config.py @@ -16,6 +16,7 @@ from enum import Enum from functools import cached_property from pathlib import Path from typing import TYPE_CHECKING, Any, Optional, Self, TypeVar +from uuid import uuid4 import yaml from ruamel.yaml.comments import CommentedMap @@ -945,6 +946,11 @@ class SerenaConfig(SharedConfig, ModeSelectionDefinitionWithBaseModes): # *** fields that are mapped directly to/from the configuration file (DO NOT RENAME) *** projects: list[RegisteredProject] = field(default_factory=list) + auth_secret: str = field(default_factory=lambda: str(uuid4()), repr=False) + """ + shared secret for authenticating communication between Serena components and services. + A random UUID is generated and persisted when the configuration setting is missing or empty. + """ gui_log_window: bool = False log_level: int = logging.INFO trace_lsp_communication: bool = False @@ -1138,6 +1144,11 @@ class SerenaConfig(SharedConfig, ModeSelectionDefinitionWithBaseModes): assert hasattr(instance, field_name) setattr(instance, field_name, get_value_or_default(field_name)) + # generate a persistent authentication secret for explicitly unset settings + if not instance.auth_secret: + instance.auth_secret = str(uuid4()) + num_migrations += 1 + # read projects if "projects" not in loaded_commented_yaml: raise SerenaConfigError("`projects` key not found in Serena configuration. Please update your `serena_config.yml` file.") diff --git a/src/serena/resources/serena_config.template.yml b/src/serena/resources/serena_config.template.yml index ea8e3d17..29485b3e 100644 --- a/src/serena/resources/serena_config.template.yml +++ b/src/serena/resources/serena_config.template.yml @@ -235,5 +235,9 @@ project_serena_folder_location: "$projectDir/.serena" # The pattern "**" matches any project path, so it can be used to trust all projects. trusted_project_path_patterns: [] +# shared secret for authenticating communication between Serena components and services. +# Keep this value private. When missing, null, or empty, a random UUID is generated and saved on load. +auth_secret: + # the list of registered project paths (updated automatically). projects: [] diff --git a/test/serena/config/test_serena_config.py b/test/serena/config/test_serena_config.py index 64b7a852..4d05f615 100644 --- a/test/serena/config/test_serena_config.py +++ b/test/serena/config/test_serena_config.py @@ -4,6 +4,7 @@ import shutil import tempfile from copy import deepcopy from pathlib import Path +from uuid import UUID import pytest @@ -635,6 +636,36 @@ class TestSerenaConfigLoadSave: config = SerenaConfig.from_config_file(generate_if_missing=False) assert config.projects == [] + @pytest.mark.parametrize("setting", ["", "auth_secret: null\n", 'auth_secret: ""\n']) + def test_unset_auth_secret_is_generated_and_persisted(self, setting: str) -> None: + # load an existing configuration without a usable secret + self.master_config_path.write_text("projects: []\n" + setting) + config = SerenaConfig.from_config_file(generate_if_missing=False) + + # subsequent loads retain the generated random UUID + assert UUID(config.auth_secret).version == 4 + assert SerenaConfig.from_config_file(generate_if_missing=False).auth_secret == config.auth_secret + + def test_configured_auth_secret_is_preserved(self) -> None: + # retain a user-provided secret across loading and migration + self.master_config_path.write_text("projects: []\nauth_secret: custom-secret\n") + assert SerenaConfig.from_config_file(generate_if_missing=False).auth_secret == "custom-secret" + assert SerenaConfig.from_config_file(generate_if_missing=False).auth_secret == "custom-secret" + + def test_new_config_has_persistent_auth_secret(self) -> None: + # generate the configuration from the template and retain its secret + config = SerenaConfig.from_config_file() + assert UUID(config.auth_secret).version == 4 + assert SerenaConfig.from_config_file().auth_secret == config.auth_secret + + def test_direct_config_instances_have_distinct_auth_secrets(self) -> None: + # directly constructed configurations receive independent secrets + first = SerenaConfig() + second = SerenaConfig() + assert UUID(first.auth_secret).version == 4 + assert UUID(second.auth_secret).version == 4 + assert first.auth_secret != second.auth_secret + def test_malformed_project_is_skipped_with_warning(self, caplog): """A malformed project.yml must not abort loading of the others.""" good_project = self._make_project_dir( From b2f8f14b2e7dfc4a82a56901bb2f994daed670e0 Mon Sep 17 00:00:00 2001 From: Dominik Jain Date: Thu, 17 Sep 2026 13:17:53 +0200 Subject: [PATCH 48/53] Restrict configuration file permissions on POSIX - Set configuration permissions to 0600 before reading and log adjustments. - Log chmod failures and continue loading; leave Windows permissions unchanged. --- src/serena/config/serena_config.py | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/src/serena/config/serena_config.py b/src/serena/config/serena_config.py index de47c016..976c3304 100644 --- a/src/serena/config/serena_config.py +++ b/src/serena/config/serena_config.py @@ -7,6 +7,7 @@ import dataclasses import os import re import shutil +import stat import threading from collections.abc import Iterator, Sequence from copy import deepcopy @@ -1122,6 +1123,17 @@ class SerenaConfig(SharedConfig, ModeSelectionDefinitionWithBaseModes): log.info(f"Serena configuration file not found at {config_file_path}, autogenerating...") cls._generate_config_file(config_file_path) + # restrict access to the owner's read/write permissions (as the config file contains secrets) + if os.name == "posix": + current_mode = stat.S_IMODE(os.stat(config_file_path).st_mode) + if current_mode != 0o600: + try: + os.chmod(config_file_path, 0o600) + except Exception as e: + log.error("Failed to restrict permissions of Serena configuration %s to 0600: %s", config_file_path, e) + else: + log.info("Changed permissions of Serena configuration %s from %04o to 0600", config_file_path, current_mode) + # load the configuration log.info(f"Loading Serena configuration from {config_file_path}") try: From 734cfa958e41a70f0ac8395863d0cdcd704dc129 Mon Sep 17 00:00:00 2001 From: Dominik Jain Date: Thu, 17 Sep 2026 12:55:04 +0200 Subject: [PATCH 49/53] Authenticate project server communication with the shared secret - Require a valid Bearer token on heartbeat and project-query requests. - Send the configured auth_secret from clients and the query-project tool. - Test accepted and rejected credentials, including real HTTP client requests. --- src/serena/project_server.py | 33 ++++++++++++---- src/serena/repl/external_project.py | 2 +- src/serena/tools/query_project_tools.py | 2 +- test/serena/test_external_projects.py | 5 ++- test/serena/test_project_server.py | 52 ++++++++++++++++++++++++- 5 files changed, 82 insertions(+), 12 deletions(-) diff --git a/src/serena/project_server.py b/src/serena/project_server.py index 4e8a69c8..e00100d3 100644 --- a/src/serena/project_server.py +++ b/src/serena/project_server.py @@ -3,11 +3,12 @@ import json import logging import pickle +import secrets import threading from typing import TYPE_CHECKING, Any import requests as requests_lib -from flask import Flask, Response, request +from flask import Flask, Response, abort, request from pydantic import BaseModel from sensai.util.logging import LogTime @@ -90,7 +91,22 @@ class ProjectServer: self._setup_routes() + def get_serena_config(self) -> SerenaConfig: + return self._agent.serena_config + + def get_auth_secret(self) -> str: + """Returns the authentication secret used by the server.""" + return self._agent.serena_config.auth_secret + def _setup_routes(self) -> None: + @self._app.before_request + def authenticate() -> None: + # authenticate every request before parsing input or accessing projects + secret = self.get_auth_secret() + provided = request.headers.get("Authorization", "") + if not secret or not secrets.compare_digest(provided.encode("utf-8"), f"Bearer {secret}".encode()): + abort(401) + @self._app.route("/heartbeat", methods=["GET"]) def heartbeat() -> dict[str, str]: return {"status": "alive"} @@ -195,20 +211,23 @@ class ProjectServerClient: :class:`ConnectionError` is raised. """ - def __init__(self, host: str = "127.0.0.1", port: int | None = None, timeout: int = 300) -> None: + def __init__(self, serena_config: SerenaConfig, host: str = "127.0.0.1", port: int | None = None) -> None: """ :param host: the host address of the project server. - :param port: the port of the project server; None for the default port. + :param port: the port of the project server; if None, use default. + :param auth_secret: the shared authentication secret; defaults to the secret in Serena's configuration. :raises ConnectionError: if the project server is not reachable. """ if port is None: port = ProjectServer.PORT self._base_url = f"http://{host}:{port}" - self._timeout = timeout + self._timeout = serena_config.tool_timeout - 1 + auth_secret = serena_config.auth_secret + self._headers = {"Authorization": f"Bearer {auth_secret}"} # verify that the server is running try: - response = requests_lib.get(f"{self._base_url}/heartbeat", timeout=5) + response = requests_lib.get(f"{self._base_url}/heartbeat", headers=self._headers, timeout=5) response.raise_for_status() except requests_lib.ConnectionError: raise ConnectionError(f"ProjectServer is not reachable at {self._base_url}. Make sure the server is running.") @@ -233,7 +252,7 @@ class ProjectServerClient: tool_params_json=tool_params_json, ).model_dump() - response = requests_lib.post(f"{self._base_url}/query_project", json=payload, timeout=self._timeout) + response = requests_lib.post(f"{self._base_url}/query_project", json=payload, headers=self._headers, timeout=self._timeout) response.raise_for_status() return response.text @@ -251,7 +270,7 @@ class ProjectServerClient: payload = CallFacadeMethodRequest( project_name=project_name, facade_name=facade_name, method_name=method_name, args=args, kwargs=kwargs ).model_dump() - response = requests_lib.post(f"{self._base_url}/call_facade_method", json=payload, timeout=self._timeout) + response = requests_lib.post(f"{self._base_url}/call_facade_method", json=payload, headers=self._headers, timeout=self._timeout) if not response.ok: raise ValueError(f"Project server error ({response.status_code}): {response.text[:2000]}") return pickle.loads(response.content) diff --git a/src/serena/repl/external_project.py b/src/serena/repl/external_project.py index 1e909b39..401c2d19 100644 --- a/src/serena/repl/external_project.py +++ b/src/serena/repl/external_project.py @@ -63,5 +63,5 @@ class ExternalProjectExecution: if self._client is None: from serena.project_server import ProjectServerClient - self._client = ProjectServerClient() + self._client = ProjectServerClient(self._agent.serena_config) return self._client.call_facade_method(self.project_name, facade_name, method_name, list(args), kwargs) diff --git a/src/serena/tools/query_project_tools.py b/src/serena/tools/query_project_tools.py index 514454b8..68e4eb3e 100644 --- a/src/serena/tools/query_project_tools.py +++ b/src/serena/tools/query_project_tools.py @@ -57,7 +57,7 @@ class QueryProjectTool(Tool, ToolMarkerOptional, ToolMarkerDoesNotRequireActiveP tool = self.agent.get_tool_by_name(tool_name) assert tool.is_readonly(), f"Tool {tool_name} is not read-only and cannot be executed in another project." if self._is_project_server_required(tool): - client = ProjectServerClient() + client = ProjectServerClient(self.agent.serena_config) return client.query_project(project_name, tool_name, tool_params_json) else: registered_project = self.agent.serena_config.get_registered_project(project_name) diff --git a/test/serena/test_external_projects.py b/test/serena/test_external_projects.py index a2bb73d7..ce74565c 100644 --- a/test/serena/test_external_projects.py +++ b/test/serena/test_external_projects.py @@ -61,8 +61,8 @@ def project_server(serena_config: SerenaConfig) -> Iterator[tuple[ProjectServer, @pytest.mark.python @pytest.mark.skipif(not language_server_tests_enabled(LanguageServerId.PYTHON), reason="python tests are disabled in this environment") def test_facade_method_results_are_transferred_from_the_project_server(project_server: tuple[ProjectServer, int]) -> None: - _, port = project_server - client = ProjectServerClient(port=port) + server, port = project_server + client = ProjectServerClient(server.get_serena_config(), port=port) result = client.call_facade_method("test_repo_python", "lsp", "find_symbol", ["create_user"], {"include_body": True}) # the result is a self-contained object which can be processed and rendered locally @@ -86,6 +86,7 @@ def test_external_project_context_in_repl( serena_config.included_apis = ["ext"] # the querying agent has another project active and queries the python test project + serena_config.auth_secret = server.get_auth_secret() agent = SerenaAgent(project="test_repo_typescript", serena_config=serena_config) agent.execute_task(lambda: None) try: diff --git a/test/serena/test_project_server.py b/test/serena/test_project_server.py index 1288e177..845603b1 100644 --- a/test/serena/test_project_server.py +++ b/test/serena/test_project_server.py @@ -8,8 +8,11 @@ from typing import Any, cast from unittest.mock import MagicMock import pytest +from flask import Flask +from werkzeug.serving import make_server -from serena.project_server import ProjectServer, QueryProjectRequest +from serena.config.serena_config import SerenaConfig +from serena.project_server import ProjectServer, ProjectServerClient, QueryProjectRequest @pytest.fixture @@ -23,6 +26,53 @@ def project_server() -> ProjectServer: return server +@pytest.fixture +def authenticated_server(project_server: ProjectServer, monkeypatch: pytest.MonkeyPatch) -> ProjectServer: + # expose the real HTTP routes with a query handler that needs no language servers + project_server._agent.serena_config.auth_secret = "test-shared-secret" + project_server._app = Flask(__name__) + monkeypatch.setattr(project_server, "_query_project", lambda req: req.project_name) + project_server._setup_routes() + return project_server + + +@pytest.mark.parametrize("authorization", [None, "Bearer wrong-secret", "test-shared-secret", "Bearer café"]) +@pytest.mark.parametrize("path", ["/heartbeat", "/query_project"]) +def test_project_server_rejects_invalid_credentials(authenticated_server: ProjectServer, authorization: str | None, path: str) -> None: + # unauthorized requests are rejected even before query payload validation + headers = {} if authorization is None else {"Authorization": authorization} + with authenticated_server._app.test_client() as client: + response = client.open(path, method="GET" if path == "/heartbeat" else "POST", headers=headers) + assert response.status_code == 401 + + +@pytest.mark.parametrize("use_wrong_password", [True, False]) +def test_project_server_client_authenticates_requests(authenticated_server: ProjectServer, use_wrong_password: bool) -> None: + # run the authenticated endpoints on an ephemeral local port + http_server = make_server("127.0.0.1", 0, authenticated_server._app) + thread = threading.Thread(target=http_server.serve_forever, daemon=True) + thread.start() + try: + + def check_client(): + serena_config = SerenaConfig() + serena_config.auth_secret = "wrong-secret" if use_wrong_password else authenticated_server.get_auth_secret() + client = ProjectServerClient(serena_config, port=http_server.server_port) + assert client.query_project("other", "find_symbol", "{}") == "other" + + # construction authenticates the heartbeat, raising a Connection error if using the wrong password + if use_wrong_password: + with pytest.raises(expected_exception=ConnectionError, match="401"): + check_client() + else: + check_client() + + finally: + http_server.shutdown() + thread.join(timeout=5) + http_server.server_close() + + def test_cached_project_lookup_is_not_blocked_by_unrelated_cold_load(project_server: ProjectServer) -> None: cached_root = Path("/cached") cold_root = Path("/cold") From 205da357d57ea45e80b6db4cf5784bc6baa95f15 Mon Sep 17 00:00:00 2001 From: Dominik Jain Date: Thu, 17 Sep 2026 20:38:58 +0200 Subject: [PATCH 50/53] Add note on PRs for beta features --- CONTRIBUTING.md | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 2d8dae3b..8dad30a6 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -11,6 +11,8 @@ The following types of contributions can be submitted directly via pull requests For other changes, please open an issue first to discuss your ideas with the maintainers. +Do not submit pull requests for beta features (unless they are trivial bug fixes); instead, provide feedback via issues or discussions. + ## Licensing and Contributor License Agreement (CLA) Serena is multi-licensed by component (see [LICENSE](LICENSE)): @@ -44,7 +46,8 @@ See the corresponding [memory](.serena/memories/adding_new_language_support_guid ## Submitting Pull Requests -Before submitting a PR, be sure to document your relevant changes (i.e. new features, fixes) in `CHANGELOG.md`. +Before submitting a PR, be sure to document your relevant changes (i.e. new features, fixes) in `CHANGELOG.md`; +documentation changes should not be included. Use a concise style and add your change to the appropriate section ("Language Servers", "Tools", "JetBrains", "CLI", "Memories", "Dashboard", "Hooks", "General", "Security"). From 006487cbb63c08e617c9f1475d5a6489d216d57c Mon Sep 17 00:00:00 2001 From: Dominik Jain Date: Thu, 17 Sep 2026 20:46:02 +0200 Subject: [PATCH 51/53] Mark REPL as a beta feature --- .serena/memories/repl.md | 3 +++ CONTRIBUTING.md | 1 + docs/autogen_docs.py | 2 ++ src/serena/resources/serena_config.template.yml | 1 + 4 files changed, 7 insertions(+) diff --git a/.serena/memories/repl.md b/.serena/memories/repl.md index b7a6683e..9be43b9e 100644 --- a/.serena/memories/repl.md +++ b/.serena/memories/repl.md @@ -5,6 +5,9 @@ whose attributes are facades (`s.lsp`, `s.edit`, `s.fs`, `s.mem`, `s.shell`, `s. Code runs like a notebook cell (module-level exec in the session namespace); the value of a trailing expression is the result. No `return` (a top-level `return` yields a SyntaxError with a hint). +IMPORTANT: The REPL interface is a BETA feature. If you encounter any issues, please report them +but do not submit PRs for it (except for trivial fixes); the implementation is still evolving. + ## Structure - `repl/api/*_api.py`: `FacadeApi` implementations = the single implementation of each operation. diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 8dad30a6..d9df20eb 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -12,6 +12,7 @@ The following types of contributions can be submitted directly via pull requests For other changes, please open an issue first to discuss your ideas with the maintainers. Do not submit pull requests for beta features (unless they are trivial bug fixes); instead, provide feedback via issues or discussions. +At present, the Serena REPL is a beta feature. ## Licensing and Contributor License Agreement (CLA) diff --git a/docs/autogen_docs.py b/docs/autogen_docs.py index be8bc3ff..20b03e39 100644 --- a/docs/autogen_docs.py +++ b/docs/autogen_docs.py @@ -162,6 +162,8 @@ def autogen_tool_list(target_filename = "01-about/035_tools.md"): "While both interfaces offer the same general functionality for the most part, " "the REPL interface addresses several limitations inherent in the tool-based approach (see [advantages](repl-advantages) below).\n\n" ) + f.write("\n\n:::{note}\nThe Serena REPL is an unreleased BETA feature. Please provide feedback; if you encounter issues, report them.\n:::\n\n") + def tools_section(): f.write("## Tools (Classic Interface)\n\n") diff --git a/src/serena/resources/serena_config.template.yml b/src/serena/resources/serena_config.template.yml index 29485b3e..9f8c9580 100644 --- a/src/serena/resources/serena_config.template.yml +++ b/src/serena/resources/serena_config.template.yml @@ -16,6 +16,7 @@ language_backend: LSP # inclusion/exclusion settings do not apply; instead, the operations available in the REPL are configured # via the API inclusion/exclusion settings (excluded_apis etc.). # The interface is fixed at startup; it can be overridden by the project activated at startup. +# IMPORTANT: The REPL interface is a BETA feature. Please provide feedback; if you encounter issues, report them. agent_interface: tools # line ending convention to use when writing source files. From 6507092288fc25f3321dea1ee6bbe3c582acac18 Mon Sep 17 00:00:00 2001 From: Dominik Jain Date: Thu, 17 Sep 2026 21:05:20 +0200 Subject: [PATCH 52/53] Mark R cross-file tests as flaky #1040 --- test/solidlsp/r/test_r_basic.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/test/solidlsp/r/test_r_basic.py b/test/solidlsp/r/test_r_basic.py index 84a9ab51..e44b1864 100644 --- a/test/solidlsp/r/test_r_basic.py +++ b/test/solidlsp/r/test_r_basic.py @@ -9,7 +9,7 @@ import pytest from solidlsp import SolidLanguageServer from solidlsp.ls_config import LanguageServerId -from test.conftest import language_server_tests_enabled +from test.conftest import is_ci, language_server_tests_enabled from test.solidlsp.conftest import format_symbol_for_assert, has_malformed_name, request_all_symbols @@ -41,6 +41,7 @@ class TestRLanguageServer: expected_functions = {"calculate_mean", "process_data", "create_data_frame"} assert expected_functions.issubset(function_names), f"Expected functions {expected_functions} but found {function_names}" + @pytest.mark.xfail(is_ci, reason="Test is flaky") # See #1040 @pytest.mark.parametrize("language_server", [LanguageServerId.R], indirect=True) def test_find_definition_across_files(self, language_server: SolidLanguageServer): """Test finding function definitions across files.""" @@ -58,6 +59,7 @@ class TestRLanguageServer: # Definition should be around line 37 (0-indexed: 36) where create_data_frame is defined assert definition_location["range"]["start"]["line"] >= 35 + @pytest.mark.xfail(is_ci, reason="Test is flaky") # See #1040 @pytest.mark.parametrize("language_server", [LanguageServerId.R], indirect=True) def test_find_references_across_files(self, language_server: SolidLanguageServer): """Test finding function references across files.""" From b79e2a55f9d4072084977dd35bd07d31146d88b9 Mon Sep 17 00:00:00 2001 From: Dominik Jain Date: Thu, 17 Sep 2026 21:16:30 +0200 Subject: [PATCH 53/53] Add changelog entry for the Serena REPL --- CHANGELOG.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 3d3c13f0..5b1b51da 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -12,6 +12,10 @@ Status of the `main` branch. Changes prior to the next official version change w see `CONTRIBUTING.md` * General: + - **Major**: Add the Serena REPL as a new agent interface, reducing the tool set to a minimum and providing + a general code execution environment for all Serena operations. + This has several significant advantages over regular tool executions. + Please refer to our [documentation](https://oraios.github.io/serena/01-about/035_tools.html) for details. - Add `auth_secret` to `serena_config.yml` for authenticating communication between Serena components and services. When missing, null, or empty, a random UUID is generated and persisted; existing values are preserved