diff --git a/src/serena/agent.py b/src/serena/agent.py index 4446ae06..3034d77d 100644 --- a/src/serena/agent.py +++ b/src/serena/agent.py @@ -582,6 +582,7 @@ class SerenaAgent: self.version = serena_version() self._config_changed_callbacks: list[Callable[[], None]] = [] self._repl: SerenaRepl | None = None + self._prompt_params: SerenaAgent.PromptParams | None = None # obtain serena configuration using the decoupled factory function self.serena_config = serena_config or SerenaConfig.from_config_file() @@ -670,9 +671,6 @@ class SerenaAgent: log_choice=True, ) - # create the tool names mapping for prompts - self._prompt_tool_names_mapping = self._create_prompt_tool_names_mapping(self._language_backend) - # create executor for starting the language server and running tools in another thread # This executor is used to achieve linear task execution self._task_executor = TaskExecutor("SerenaAgentTaskExecutor", self._task_completion_callback) @@ -980,27 +978,88 @@ class SerenaAgent: """ return self._active_modes - @staticmethod - def _create_prompt_tool_names_mapping(language_backend: LanguageBackend) -> dict[str, str]: + @dataclass + class PromptParams: """ - Creates a mapping from tool names to new tool names, which take into consideration - - * legacy tool names, where the name was changed and - * LSP tools which are functionally replaced by other tools due to the active language backend - (e.g. "find_symbol" being replaced by "jet_brains_find_symbol" in JetBrains mode). - - The mapping is intended to be used for the generation of prompts, such that prompts can - refer to tool names as `{{ tool_names["find_symbol"] }}`, and the mapping will ensure that - the correct tool name is used in the prompt based on the active language backend. - - :return: the mapping from tool names to new tool names + Holds parameters for prompt rendering """ - result = dict(ToolSet.LEGACY_TOOL_NAME_MAPPING) - class_replacements = language_backend.get_lsp_tool_class_replacements() - for tool_class in ToolRegistry().get_all_tool_classes(): - new_tool_class: type[Tool] = class_replacements.get(tool_class, tool_class) - result[tool_class.get_name_from_cls()] = new_tool_class.get_name_from_cls() - return result + + available_tools: set[str] + """ + available tool names or, in REPL mode, the names of the raw facade methods (without facade name prefix) and + the names of the corresponding tools + """ + available_markers: set[str] + """ + names of the ToolMarkers (class names) that the available tools inherit from + """ + tool_names_mapping: dict[str, str] + """ + mapping from standard tool names to currently used and replacement tool/API method names. + In particular, this maps + * legacy tool names to current tool names + * LSP tool names to their backend- and interface-specific counterparts + (e.g. "find_symbol" to "jet_brains_find_symbol" in JetBrains mode, "find_symbol" to the corresponding API method name + when using the REPL interface). + """ + + def _get_prompt_params(self) -> PromptParams: + """ + :return: parameters for prompt rendering depending on the current agent interface, language backend and active tools/methods + """ + if self._prompt_params is not None: + return self._prompt_params + + if self._agent_interface == AgentInterface.TOOLS: + # available tool names are simply the exposed tools + available_tool_names = set(self._exposed_tools.tool_names) + available_tool_marker_names = set(self._exposed_tools.tool_marker_names) + + tool_name_mapping = dict(ToolSet.LEGACY_TOOL_NAME_MAPPING) + class_replacements = self._language_backend.get_lsp_tool_class_replacements() + for tool_class in ToolRegistry().get_all_tool_classes(): + new_tool_class: type[Tool] = class_replacements.get(tool_class, tool_class) + tool_name_mapping[tool_class.get_name_from_cls()] = new_tool_class.get_name_from_cls() + + elif self._agent_interface == AgentInterface.REPL: + # available tool names include both the names of the facade methods and the names of the corresponding tools + repl = self.get_repl() + enabled_methods = repl.entrypoint.get_enabled_methods() + corresponding_tool_classes = [m.info.corresponding_tool for m in enabled_methods if m.info.corresponding_tool is not None] + available_tools = AvailableTools( + self._exposed_tools.tools + [self._all_tools[tool_class] for tool_class in corresponding_tool_classes] + ) + available_tool_names = set(available_tools.tool_names).union({m.info.name for m in enabled_methods}) + available_tool_marker_names = set(available_tools.tool_marker_names) + + tool_class_replacements = self._language_backend.get_lsp_tool_class_replacements() + methods_by_tool_class = {m.info.corresponding_tool: m for m in enabled_methods if m.info.corresponding_tool is not None} + + def get_name(tool_class: type[Tool]) -> str: + # if there is a corresponding method in the API, return its qualified name (as used in REPL code) + method = methods_by_tool_class.get(tool_class) + if method is not None: + return method.qualified_name + # if there is a corresponding method for the replacement class, return its qualified name + replacement_class = tool_class_replacements.get(tool_class) + if replacement_class is not None: + replacement_method = methods_by_tool_class.get(replacement_class) + if replacement_method is not None: + return replacement_method.qualified_name + # otherwise, keep the tool's name + return tool_class.get_name_from_cls() + + tool_name_mapping = {} + for legacy_name, new_name in ToolSet.LEGACY_TOOL_NAME_MAPPING.items(): + tool_name_mapping[legacy_name] = get_name(ToolRegistry().get_tool_class_by_name(new_name)) + for tool_class in ToolRegistry().get_all_tool_classes(): + tool_name_mapping[tool_class.get_name_from_cls()] = get_name(tool_class) + else: + raise ValueError() + + return self.PromptParams( + available_tools=available_tool_names, available_markers=available_tool_marker_names, tool_names_mapping=tool_name_mapping + ) @staticmethod def _format_prompt_tag(text: str, tag: str, tag_name_attr: str | None = None) -> str: @@ -1027,10 +1086,11 @@ class SerenaAgent: return "" template = JinjaTemplate(prompt_template) + prompt_params = self._get_prompt_params() text = template.render( - available_tools=self._exposed_tools.tool_names, - available_markers=self._exposed_tools.tool_marker_names, - tool_names=self._prompt_tool_names_mapping, + available_tools=prompt_params.available_tools, + available_markers=prompt_params.available_markers, + tool_names=prompt_params.tool_names_mapping, embed_memory=embed_memory, ) @@ -1069,11 +1129,8 @@ class SerenaAgent: :param session_id: the client session ID for the case where this is run from a tool; "global" for the connection time case :return: the prompt """ - available_tools = self._active_tools - available_markers = available_tools.tool_marker_names global_memories = self._create_global_memory_manager().list_global_memories() global_memories_str = dict_string(global_memories.to_dict()) if len(global_memories) > 0 else "" - log.info("Generating system prompt with available_tools=(see active tools), available_markers=%s", available_markers) # determine modes for which prompts must (still) be provided, excluding modes that were already provided in a # previously provided project activation message (if any) @@ -1084,13 +1141,14 @@ class SerenaAgent: relevant_modes.append(mode) self._project_prompt_status.mark_mode_prompts_as_provided(session_id) + prompt_params = self._get_prompt_params() system_prompt = self.prompt_factory.create_system_prompt( context_system_prompt=self._render_prompt(self._context.prompt, tag="context"), mode_system_prompts=[self._render_prompt(mode.prompt, tag="mode", tag_name_attr=mode.name) for mode in relevant_modes], - available_tools=available_tools.tool_names, - available_markers=available_markers, + available_tools=prompt_params.available_tools, + available_markers=prompt_params.available_markers, global_memories_list=global_memories_str, - tool_names=self._prompt_tool_names_mapping, + tool_names=prompt_params.tool_names_mapping, ) # provide the project activation message if it hasn't yet been provided @@ -1195,9 +1253,9 @@ class SerenaAgent: self._active_tools = tool_set.to_available_tools(self._all_tools) log.info(f"Active tools ({len(self._active_tools)}): {', '.join(self._active_tools.tool_names)}") - # reset the REPL, whose facades/API scope depend on the active modes and project (it is re-created on demand). - # NOTE: This must happen irrespective of the agent interface, since the REPL tool may be active in tool mode as well. + # reset members that depend on the active tools, so that they are re-created on demand with the new active tools self._repl = None + self._prompt_params = None # check if a tool was activated that is not in the exposed tool set and issue a warning if so active_tools_not_exposed = set(self._active_tools.tool_names) - set(self._exposed_tools.tool_names) diff --git a/src/serena/repl/facade.py b/src/serena/repl/facade.py index 16a7fa4c..987b7598 100644 --- a/src/serena/repl/facade.py +++ b/src/serena/repl/facade.py @@ -7,7 +7,7 @@ The facade, i.e. the object through which REPL code accesses a group of related import inspect import logging from abc import ABC -from collections.abc import Callable, Iterable +from collections.abc import Callable from dataclasses import dataclass from typing import TYPE_CHECKING, Any, TypeVar @@ -143,29 +143,43 @@ class FacadeMethod: enabled or disabled; only enabled methods are accessible from REPL code. """ - def __init__(self, name: str, implementation: Callable[..., Any], info: FacadeMethodInfo, enabled: bool) -> None: + def __init__(self, parent: "Facade", implementation: Callable[..., Any], info: FacadeMethodInfo, enabled: bool) -> None: """ - :param name: the method's name + :param parent: the facade the method belongs to :param implementation: the implementation to delegate to - :param info: the method's metadata + :param info: the method's metadata (including the method's name) :param enabled: whether the method is initially enabled """ - self.name = name + self.parent = parent self._implementation = implementation self.info = info self.enabled = enabled + @property + def name(self) -> str: + return self.info.name + + @property + def facade_name(self) -> str: + return self.parent.name + + @property + def qualified_name(self) -> str: + """ + :return: the name under which the method is accessible from REPL code (facade name and method name) + """ + return f"{self.facade_name}.{self.name}" + def __call__(self, *args: Any, **kwargs: Any) -> Any: return self._implementation(*args, **kwargs) - def describe(self, facade_name: str) -> str: + def describe(self) -> str: """ - :param facade_name: the name of the facade the method belongs to :return: the method's signature and documentation """ signature = inspect.signature(self._implementation) doc = inspect.getdoc(self._implementation) or "(no documentation)" - return f"{facade_name}.{self.name}{signature}\n{doc}\n" + return f"{self.qualified_name}{signature}\n{doc}\n" class ApiScope: @@ -268,11 +282,15 @@ class Facade: A named group of related operations which an LLM can invoke from REPL code. """ - def __init__(self, name: str, description: str, methods: Iterable[FacadeMethod]) -> None: + def __init__(self, name: str, description: str) -> None: # NOTE: attributes are set via object.__setattr__ because __getattr__ is overridden object.__setattr__(self, "_name", name) object.__setattr__(self, "_description", description) - object.__setattr__(self, "_methods", {m.name: m for m in methods}) + object.__setattr__(self, "_methods", {}) + + def _add_method(self, method: FacadeMethod) -> None: + assert method.parent is self + self._methods[method.name] = method @staticmethod def from_api(api: FacadeApi, api_scope: ApiScope) -> "Facade": @@ -283,15 +301,14 @@ class Facade: :param api_scope: API scope definition determining which methods are enabled :return: the facade """ - facade_name = api.get_name_() - methods = [] + facade = Facade(api.get_name_(), api.get_description_()) for name, member in inspect.getmembers(api, predicate=inspect.ismethod): method_info = get_facade_method_info(member) if method_info is None: continue - is_enabled = api_scope.is_method_enabled(facade_name, method_info) - methods.append(FacadeMethod(name, member, method_info, enabled=is_enabled)) - return Facade(api.get_name_(), api.get_description_(), methods) + is_enabled = api_scope.is_method_enabled(facade.name, method_info) + facade._add_method(FacadeMethod(facade, member, method_info, enabled=is_enabled)) + return facade @property def name(self) -> str: @@ -305,6 +322,12 @@ class Facade: def enabled_method_names(self) -> list[str]: return [m.name for m in self._methods.values() if m.enabled] + def get_enabled_methods(self) -> list[FacadeMethod]: + """ + :return: the list of enabled methods + """ + return [m for m in self._methods.values() if m.enabled] + def get_method(self, method_name: str) -> FacadeMethod: """ :param method_name: the name of the method @@ -338,7 +361,7 @@ class Facade: parts = [f"Facade '{self._name}': {self._description}", ""] for method in self._methods.values(): if method.enabled: - parts.append(method.describe(self._name)) + parts.append(method.describe()) return "\n".join(parts) def describe_method(self, method_name: str) -> str: @@ -349,4 +372,4 @@ class Facade: method = self._get_enabled_method(method_name) if method is None: raise ValueError(self._no_such_method_message(method_name)) - return method.describe(self._name) + return method.describe() diff --git a/src/serena/repl/repl.py b/src/serena/repl/repl.py index a8cd8d34..79139388 100644 --- a/src/serena/repl/repl.py +++ b/src/serena/repl/repl.py @@ -9,7 +9,7 @@ import textwrap import traceback from typing import Any -from .facade import ApiScope, Facade +from .facade import ApiScope, Facade, FacadeMethod from .representable import Representable log = logging.getLogger(__name__) @@ -34,6 +34,12 @@ class SerenaReplEntrypoint: registered_facade_names.append(facade.name) log.info("Registered %d/%d facades: %s", len(registered_facade_names), len(facades), registered_facade_names) + def get_enabled_methods(self) -> list[FacadeMethod]: + """ + :return: the list of all enabled methods across all facades + """ + return [method for facade in self._facades.values() for method in facade.get_enabled_methods()] + def _register(self, facade: Facade) -> None: if facade.name in self._facades: raise ValueError(f"Duplicate facade name: {facade.name}") diff --git a/src/serena/resources/config/modes/editing.yml b/src/serena/resources/config/modes/editing.yml index f877fc51..1602ca51 100644 --- a/src/serena/resources/config/modes/editing.yml +++ b/src/serena/resources/config/modes/editing.yml @@ -4,7 +4,7 @@ prompt: | **Refactoring tools** For operations on existing symbols, prefer the dedicated refactoring tools over hand-edits: - `{{ tool_names['rename_symbol'] }}` and `{{ tool_names['safe_delete_symbol'] }}`{% if 'jet_brains_move' in available_tools %}, plus `jet_brains_move` and `jet_brains_inline_symbol`{% endif %} — + `{{ tool_names['rename_symbol'] }}` and `{{ tool_names['safe_delete_symbol'] }}`{% if 'jet_brains_move' in available_tools %}, plus `{{ tool_names['jet_brains_move'] }}` and `{{ tool_names['jet_brains_inline_symbol'] }}`{% endif %} — they are reference-aware and update or check all usages atomically. When such a tool returns success, the refactoring is already complete and consistent across all declarations, references, overrides and imports — trust it: do not re-read the changed files or re-run the build / test suite just to confirm the refactor @@ -16,9 +16,9 @@ prompt: | **Symbolic editing** Use symbolic retrieval tools to identify the symbols you need to edit. - If you need to replace the definition of a symbol, use the `replace_symbol_body` tool. - If you want to add some new code at the end of the file, use the `insert_after_symbol` tool with the last top-level symbol in the file. - Similarly, you can use `insert_before_symbol` with the first top-level symbol in the file to insert code at the beginning of a file. + If you need to replace the definition of a symbol, use the `{{ tool_names['replace_symbol_body'] }}` tool. + If you want to add some new code at the end of the file, use the `{{ tool_names['insert_after_symbol'] }}` tool with the last top-level symbol in the file. + Similarly, you can use `{{ tool_names['insert_before_symbol'] }}` with the first top-level symbol in the file to insert code at the beginning of a file. You can understand relationships between symbols by using the `{{ tool_names['find_referencing_symbols'] }}` tool. If not explicitly requested otherwise by the user, you make sure that when you edit a symbol, the change is either backward-compatible or you find and update all references as needed. The `{{ tool_names['find_referencing_symbols'] }}` tool will give you code snippets around the references as well as symbolic information. @@ -26,14 +26,14 @@ prompt: | {% if 'replace_content' in available_tools %} **File-based editing** - The `replace_content` tool allows you to perform regex-based replacements within files (as well as simple string replacements). + The `{{ tool_names['replace_content'] }}` tool allows you to perform regex-based replacements within files (as well as simple string replacements). This is your primary tool for editing code whenever replacing or deleting a whole symbol would be a more expensive operation, e.g. if you need to adjust just a few lines of code within a method. In `regex` mode, wildcards like `start.*?end` let you match a span without quoting its full text; an ambiguous match returns an error you can refine, so a tight wildcard pattern is both cheaper and safe. - For several small edits within one file, prefer a batch of targeted `replace_content` calls over rewriting the whole + For several small edits within one file, prefer a batch of targeted `{{ tool_names['replace_content'] }}` calls over rewriting the whole file: rewriting has to re-emit the file's entire contents, whereas each targeted edit emits only the changed text. - {% if 'replace_in_files' in available_tools %}For the SAME edit across many files, use `replace_in_files`, which applies + {% if 'replace_in_files' in available_tools %}For the SAME edit across many files, use `{{ tool_names['replace_in_files'] }}`, which applies it everywhere in one call — equally safe and transparent: its `dry_run` mode first previews every change as a diff with a per-occurrence id, so you can then apply all of them or just a chosen subset.{% endif %} {% endif %} diff --git a/src/serena/resources/config/prompt_templates/system_prompt.yml b/src/serena/resources/config/prompt_templates/system_prompt.yml index 99604bf1..95d5bae9 100644 --- a/src/serena/resources/config/prompt_templates/system_prompt.yml +++ b/src/serena/resources/config/prompt_templates/system_prompt.yml @@ -5,29 +5,28 @@ prompts: connection_prompt: | CRITICAL: Before starting to work on a coding task, call the `initial_instructions` tool to read the 'Serena Instructions Manual'. system_prompt: | - You have semantic coding tools that you rely on heavily. Work resource-efficiently: don't read or + You have semantic coding functions/tools that you rely on heavily. Work resource-efficiently: don't read or generate content the task doesn't need. Some tasks require understanding a large part of the codebase; others need only a few symbols or a single file. Avoid reading whole files unless necessary — acquire information step by step, using - the symbolic tools to get an overview of symbols and their relations, then reading only the bodies + the symbolic functions to get an overview of symbols and their relations, then reading only the bodies you need. {% if 'ToolMarkerSymbolicRead' in available_markers %}Once you have read a full file, there is no point re-analysing it with the symbolic read tools — you already have it.{% endif %} {% if 'search_for_pattern' in available_tools %} - If you are unsure about a symbol's name or location{% if 'find_symbol' in available_tools %} (to the extent that substring_matching for the symbol name is not enough){% endif %}, you can use the `search_for_pattern` tool, which allows fast - and flexible search for patterns in the codebase.{% if 'ToolMarkerSymbolicRead' in available_markers %} In this way, you can first find candidates for symbols or files, - and then proceed with the symbolic tools.{% endif %} + For searches beyond named symbols, you can use `{{ tool_names['search_for_pattern'] }}` and flexible search for patterns in the codebase.{% if 'ToolMarkerSymbolicRead' in available_markers %} In this way, you can first find candidates for symbols or files to explore, + and then proceed with symbolic operations.{% endif %} {% endif %} {% if 'ToolMarkerSymbolicRead' in available_markers %} Symbols are identified by their `name_path` and `relative_path`. - You can get an overview of the symbols in a file by using the `{{ tool_names['get_symbols_overview'] }}` tool, or search for a specific symbol with `{{ tool_names['find_symbol'] }}`. + You can get an overview of the symbols in a file by using `{{ tool_names['get_symbols_overview'] }}`, or search for a specific symbol with `{{ tool_names['find_symbol'] }}`. You only read the bodies of symbols when you need to (e.g. if you want to fully understand or edit it). For example, if you are working with Python code and already know that you need to read the body of the constructor of the class Foo, you can directly use `{{ tool_names['find_symbol'] }}` with name path pattern `Foo/__init__` and `include_body=True`. If you don't know yet which methods in `Foo` you need to read or edit, - you can use `{{ tool_names['find_symbol'] }}` with name path pattern `Foo`, `include_body=False` and `depth=1` to get all (top-level) methods of `Foo` before proceeding + you can use `{{ tool_names['find_symbol'] }}` with name path pattern `Foo`, `include_body=False` and `depth=1` to get all (top-level) members of `Foo` before proceeding to read the desired methods with `include_body=True`. - You can understand relationships between symbols by using the `{{ tool_names['find_referencing_symbols'] }}` tool. + You can understand relationships between symbols by using `{{ tool_names['find_referencing_symbols'] }}`. {% endif %} {% if 'read_memory' in available_tools -%} diff --git a/test/serena/test_serena_agent.py b/test/serena/test_serena_agent.py index 8ee9463a..58e9e675 100644 --- a/test/serena/test_serena_agent.py +++ b/test/serena/test_serena_agent.py @@ -918,6 +918,11 @@ class TestSerenaAgent: expected = {"serena_repl", "initial_instructions"} | (set() if context.single_project else {"activate_project"}) assert exposed == expected assert "s.lsp" in agent.get_tool(SerenaReplTool).apply("s.info()") + + # prompts refer to operations by their qualified REPL names, e.g. `lsp.find_symbol` instead of the tool name + system_prompt = agent.create_system_prompt() + assert "`lsp.find_symbol`" in system_prompt + assert "`find_symbol`" not in system_prompt finally: agent.on_shutdown(timeout=5)