diff --git a/src/serena/agent.py b/src/serena/agent.py index 7e900f34..53fead1d 100644 --- a/src/serena/agent.py +++ b/src/serena/agent.py @@ -18,6 +18,7 @@ from typing import TYPE_CHECKING, Any, Optional, TypeVar from sensai.util import logging from sensai.util.logging import LogTime +from interprompt.jinja_template import JinjaTemplate from serena import serena_version from serena.analytics import RegisteredTokenCountEstimator, ToolUsageStats from serena.config.context_mode import RegisteredContext, SerenaAgentContext, SerenaAgentMode @@ -25,7 +26,8 @@ from serena.config.serena_config import SerenaConfig, ToolInclusionDefinition, T from serena.dashboard import SerenaDashboardAPI from serena.project import Project from serena.prompt_factory import SerenaPromptFactory -from serena.tools import ActivateProjectTool, Tool, ToolRegistry +from serena.tools import ActivateProjectTool, Tool, ToolMarker, ToolRegistry +from serena.util.inspection import iter_subclasses from serena.util.logging import MemoryLogHandler from solidlsp import SolidLanguageServer @@ -91,6 +93,23 @@ class MemoriesManager: return f"Memory {name} deleted." +class AvailableTools: + def __init__(self, tools: list[Tool]): + """ + :param tools: the list of available tools + """ + self.tools = tools + self.tool_names = [tool.get_name_from_cls() for tool in tools] + self.tool_marker_names = set() + for marker_class in iter_subclasses(ToolMarker): + for tool in tools: + if isinstance(tool, marker_class): + self.tool_marker_names.add(marker_class.__name__) + + def __len__(self) -> int: + return len(self.tools) + + class SerenaAgent: def __init__( self, @@ -192,7 +211,7 @@ class SerenaAgent: tool_inclusion_definitions.append(SerenaAgentMode.from_name_internal("jetbrains")) self._base_tool_set = ToolSet.default().apply(*tool_inclusion_definitions) - self._exposed_tools = {tc: t for tc, t in self._all_tools.items() if self._base_tool_set.includes_name(t.get_name())} + self._exposed_tools = AvailableTools([t for t in self._all_tools.values() if self._base_tool_set.includes_name(t.get_name())]) log.info(f"Number of exposed tools: {len(self._exposed_tools)}") # create executor for starting the language server and running tools in another thread @@ -307,7 +326,7 @@ class SerenaAgent: If a client should attempt to use a tool that is dynamically disabled (e.g. because a project is activated that disables it), it will receive an error. """ - return list(self._exposed_tools.values()) + return list(self._exposed_tools.tools) def get_active_project(self) -> Project | None: """ @@ -341,11 +360,21 @@ class SerenaAgent: """ return list(self._modes) + def _format_prompt(self, prompt_template: str) -> str: + template = JinjaTemplate(prompt_template) + return template.render(available_tools=self._exposed_tools.tool_names, available_markers=self._exposed_tools.tool_marker_names) + def create_system_prompt(self) -> str: - return self.prompt_factory.create_system_prompt( - context_system_prompt=self._context.prompt, - mode_system_prompts=[mode.prompt for mode in self._modes], + available_markers = self._exposed_tools.tool_marker_names + log.info("Generating system prompt with available_tools=(see exposed tools), available_markers=%s", available_markers) + system_prompt = self.prompt_factory.create_system_prompt( + context_system_prompt=self._format_prompt(self._context.prompt), + mode_system_prompts=[self._format_prompt(mode.prompt) for mode in self._modes], + available_tools=self._exposed_tools.tool_names, + available_markers=available_markers, ) + log.info("System prompt:\n%s", system_prompt) + return system_prompt def _update_active_tools(self) -> None: """ diff --git a/src/serena/config/context_mode.py b/src/serena/config/context_mode.py index fbe34091..44cd42fb 100644 --- a/src/serena/config/context_mode.py +++ b/src/serena/config/context_mode.py @@ -38,6 +38,10 @@ class SerenaAgentMode(ToolInclusionDefinition, ToStringMixin): name: str prompt: str + """ + a Jinja2 template for the generation of the system prompt. + It is formatted by the agent (see SerenaAgent._format_prompt()). + """ description: str = "" def _tostring_includes(self) -> list[str]: @@ -121,6 +125,10 @@ class SerenaAgentContext(ToolInclusionDefinition, ToStringMixin): name: str prompt: str + """ + a Jinja2 template for the generation of the system prompt. + It is formatted by the agent (see SerenaAgent._format_prompt()). + """ description: str = "" tool_description_overrides: dict[str, str] = field(default_factory=dict) """Maps tool names to custom descriptions, default descriptions are extracted from the tool docstrings.""" diff --git a/src/serena/generated/generated_prompt_factory.py b/src/serena/generated/generated_prompt_factory.py index 6bb7a76b..3a5e1442 100644 --- a/src/serena/generated/generated_prompt_factory.py +++ b/src/serena/generated/generated_prompt_factory.py @@ -32,5 +32,7 @@ class PromptFactory(PromptFactoryBase): def create_prepare_for_new_conversation(self) -> str: return self._render_prompt("prepare_for_new_conversation", locals()) - def create_system_prompt(self, *, context_system_prompt: Any, mode_system_prompts: Any) -> str: + def create_system_prompt( + self, *, available_markers: Any, available_tools: Any, context_system_prompt: Any, mode_system_prompts: Any + ) -> str: return self._render_prompt("system_prompt", locals()) diff --git a/src/serena/resources/config/modes/editing.yml b/src/serena/resources/config/modes/editing.yml index d53750fd..464f4537 100644 --- a/src/serena/resources/config/modes/editing.yml +++ b/src/serena/resources/config/modes/editing.yml @@ -14,7 +14,7 @@ prompt: | use the regex-based approach that is described below. Let us first discuss the symbol-based approach. - Symbols are identified by their name path and relative file path, see the description of the `find_symbols` tool for more details + Symbols are identified by their name path and relative file path, see the description of the `find_symbol` tool for more details on how the `name_path` matches symbols. You can get information about available symbols by using the `get_symbols_overview` tool for finding top-level symbols in a file or directory, or by using `find_symbol` if you already know the symbol's name path. You generally try to read as little code as possible @@ -34,7 +34,8 @@ prompt: | You will generally be able to use the info from the snippets and the regex-based approach to adjust the references as well. You can assume that all symbol editing tools are reliable, so you don't need to verify the results if the tool returns without error. - Now let us discuss the regex-based approach. + {% if 'replace_regex' in available_tools %} + Let us discuss the regex-based approach. The regex-based approach is your primary tool for editing code whenever replacing or deleting a whole symbol would be a more expensive operation. This is the case if you need to adjust just a few lines of code within a method, or a chunk that is much smaller than a whole symbol. You use other tools to find the relevant content and @@ -104,7 +105,8 @@ prompt: | Generally, I remind you that you rely on the regex tool with providing you the correct feedback, no need for more verification! IMPORTANT: REMEMBER TO USE WILDCARDS WHEN APPROPRIATE! I WILL BE VERY UNHAPPY IF YOU WRITE LONG REGEXES WITHOUT USING WILDCARDS INSTEAD! -excluded_tools: + {% endif %} +excluded_tools: - replace_lines - insert_at_line - delete_lines diff --git a/src/serena/resources/config/prompt_templates/system_prompt.yml b/src/serena/resources/config/prompt_templates/system_prompt.yml index 8f3dd504..e912f7ca 100644 --- a/src/serena/resources/config/prompt_templates/system_prompt.yml +++ b/src/serena/resources/config/prompt_templates/system_prompt.yml @@ -5,32 +5,38 @@ prompts: system_prompt: | You are a professional coding agent concerned with one particular codebase. You have access to semantic coding tools on which you rely heavily for all your work, as well as collection of memory - files containing general information about the codebase. You operate in a frugal and intelligent manner, always + files containing general information about the codebase. You operate in a resource-efficient and intelligent manner, always keeping in mind to not read or generate content that is not needed for the task at hand. When reading code in order to answer a user question or task, you should try reading only the necessary code. Some tasks may require you to understand the architecture of large parts of the codebase, while for others, it may be enough to read a small set of symbols or a single file. Generally, you should avoid reading entire files unless it is absolutely necessary, instead relying on - intelligent step-by-step acquisition of information. However, if you already read a file, it does not make + intelligent step-by-step acquisition of information. {% if 'ToolMarkerSymbolicRead' in available_markers %}However, if you already read a file, it does not make sense to further analyse it with the symbolic tools (except for the `find_referencing_symbols` tool), - as you already have the information. + as you already have the information.{% endif %} - I WILL BE SERIOUSLY UPSET IF YOU READ ENTIRE FILES WITHOUT NEED! CONSIDER INSTEAD USING THE OVERVIEW TOOL AND - SYMBOLIC TOOLS TO READ ONLY THE NECESSARY CODE FIRST! + I WILL BE SERIOUSLY UPSET IF YOU READ ENTIRE FILES WITHOUT NEED! + {% if 'ToolMarkerSymbolicRead' in available_markers %} + CONSIDER INSTEAD USING THE OVERVIEW TOOL AND SYMBOLIC TOOLS TO READ ONLY THE NECESSARY CODE FIRST! I WILL BE EVEN MORE UPSET IF AFTER HAVING READ AN ENTIRE FILE YOU KEEP READING THE SAME CONTENT WITH THE SYMBOLIC TOOLS! THE PURPOSE OF THE SYMBOLIC TOOLS IS TO HAVE TO READ LESS CODE, NOT READ THE SAME CONTENT MULTIPLE TIMES! + {% endif %} You can achieve the intelligent reading of code by using the symbolic tools for getting an overview of symbols and the relations between them, and then only reading the bodies of symbols that are necessary to answer the question - or complete the task. You can also use the standard tools like list_dir, find_file and search_for_pattern if you need to. + or complete the task. + You can use the standard tools like list_dir, find_file and search_for_pattern if you need to. When tools allow it, you pass the `relative_path` parameter to restrict the search to a specific file or directory. For some tools, `relative_path` can only be a file path, so make sure to properly read the tool descriptions. - If you are unsure about a symbol's name or location (to the extent that substring_matching for the symbol name is not enough), you can use the `search_for_pattern` tool, which allows fast - and flexible search for patterns in the codebase. This way you can first find candidates for symbols or files, - and then proceed with the symbolic tools. + {% if 'search_for_pattern' in available_tools %} + If you are unsure about a symbol's name or location{% if 'find_symbol' in available_tools %} (to the extent that substring_matching for the symbol name is not enough){% endif %}, you can use the `search_for_pattern` tool, which allows fast + and flexible search for patterns in the codebase.{% if 'ToolMarkerSymbolicRead' in available_markers %}This way you can first find candidates for symbols or files, + and then proceed with the symbolic tools.{% endif %} + {% endif %} - Symbols are identified by their `name_path and `relative_path`, see the description of the `find_symbols` tool for more details + {% if 'ToolMarkerSymbolicRead' in available_markers %} + Symbols are identified by their `name_path and `relative_path`, see the description of the `find_symbol` tool for more details on how the `name_path` matches symbols. You can get information about available symbols by using the `get_symbols_overview` tool for finding top-level symbols in a file or directory, or by using `find_symbol` if you already know the symbol's name path. You generally try to read as little code as possible @@ -40,10 +46,13 @@ prompts: you can use `find_symbol` with the name path `Foo`, `include_body=False` and `depth=1` to get all (top-level) methods of `Foo` before proceeding to read the desired methods with `include_body=True` You can understand relationships between symbols by using the `find_referencing_symbols` tool. + {% endif %} + {% if 'read_memory' in available_tools %} You generally have access to memories and it may be useful for you to read them, but also only if they help you to answer the question or complete the task. You can infer which memories are relevant to the current task by reading the memory names and descriptions. + {% endif %} The context and modes of operation are described below. From them you can infer how to interact with your user and which tasks and kinds of interactions are expected of you. diff --git a/src/serena/tools/jetbrains_tools.py b/src/serena/tools/jetbrains_tools.py index 319aa590..8b9f2d1a 100644 --- a/src/serena/tools/jetbrains_tools.py +++ b/src/serena/tools/jetbrains_tools.py @@ -1,10 +1,10 @@ import json -from serena.tools import TOOL_DEFAULT_MAX_ANSWER_LENGTH, Tool, ToolMarkerOptional +from serena.tools import TOOL_DEFAULT_MAX_ANSWER_LENGTH, Tool, ToolMarkerOptional, ToolMarkerSymbolicRead from serena.tools.jetbrains_plugin_client import JetBrainsPluginClient -class JetBrainsFindSymbolTool(Tool, ToolMarkerOptional): +class JetBrainsFindSymbolTool(Tool, ToolMarkerSymbolicRead, ToolMarkerOptional): """ Performs a global (or local) search for symbols with/containing a given name/substring (optionally filtered by type). """ @@ -70,7 +70,7 @@ class JetBrainsFindSymbolTool(Tool, ToolMarkerOptional): return self._limit_length(result, max_answer_chars) -class JetBrainsFindReferencingSymbolsTool(Tool, ToolMarkerOptional): +class JetBrainsFindReferencingSymbolsTool(Tool, ToolMarkerSymbolicRead, ToolMarkerOptional): """ Finds symbols that reference the given symbol """ @@ -100,7 +100,7 @@ class JetBrainsFindReferencingSymbolsTool(Tool, ToolMarkerOptional): return self._limit_length(result, max_answer_chars) -class JetBrainsGetSymbolsOverviewTool(Tool, ToolMarkerOptional): +class JetBrainsGetSymbolsOverviewTool(Tool, ToolMarkerSymbolicRead, ToolMarkerOptional): """ Retrieves an overview of the top-level symbols within a specified file """ diff --git a/src/serena/tools/symbol_tools.py b/src/serena/tools/symbol_tools.py index 0dfb41a3..0c9b7868 100644 --- a/src/serena/tools/symbol_tools.py +++ b/src/serena/tools/symbol_tools.py @@ -9,7 +9,13 @@ from collections.abc import Sequence from copy import copy from typing import Any -from serena.tools import SUCCESS_RESULT, TOOL_DEFAULT_MAX_ANSWER_LENGTH, Tool, ToolMarkerCanEdit +from serena.tools import ( + SUCCESS_RESULT, + TOOL_DEFAULT_MAX_ANSWER_LENGTH, + Tool, + ToolMarkerSymbolicEdit, + ToolMarkerSymbolicRead, +) from solidlsp.ls_types import SymbolKind @@ -40,7 +46,7 @@ class RestartLanguageServerTool(Tool): return SUCCESS_RESULT -class GetSymbolsOverviewTool(Tool): +class GetSymbolsOverviewTool(Tool, ToolMarkerSymbolicRead): """ Gets an overview of the top-level symbols defined in a given file or directory. """ @@ -72,7 +78,7 @@ class GetSymbolsOverviewTool(Tool): return self._limit_length(result_json_str, max_answer_chars) -class FindSymbolTool(Tool): +class FindSymbolTool(Tool, ToolMarkerSymbolicRead): """ Performs a global (or local) search for symbols with/containing a given name/substring (optionally filtered by type). """ @@ -152,7 +158,7 @@ class FindSymbolTool(Tool): return self._limit_length(result, max_answer_chars) -class FindReferencingSymbolsTool(Tool): +class FindReferencingSymbolsTool(Tool, ToolMarkerSymbolicRead): """ Finds symbols that reference the symbol at the given location (optionally filtered by type). """ @@ -204,7 +210,7 @@ class FindReferencingSymbolsTool(Tool): return self._limit_length(result, max_answer_chars) -class ReplaceSymbolBodyTool(Tool, ToolMarkerCanEdit): +class ReplaceSymbolBodyTool(Tool, ToolMarkerSymbolicEdit): """ Replaces the full definition of a symbol. """ @@ -232,7 +238,7 @@ class ReplaceSymbolBodyTool(Tool, ToolMarkerCanEdit): return SUCCESS_RESULT -class InsertAfterSymbolTool(Tool, ToolMarkerCanEdit): +class InsertAfterSymbolTool(Tool, ToolMarkerSymbolicEdit): """ Inserts content after the end of the definition of a given symbol. """ @@ -257,7 +263,7 @@ class InsertAfterSymbolTool(Tool, ToolMarkerCanEdit): return SUCCESS_RESULT -class InsertBeforeSymbolTool(Tool, ToolMarkerCanEdit): +class InsertBeforeSymbolTool(Tool, ToolMarkerSymbolicEdit): """ Inserts content before the beginning of the definition of a given symbol. """ diff --git a/src/serena/tools/tools_base.py b/src/serena/tools/tools_base.py index cd025970..1f55e604 100644 --- a/src/serena/tools/tools_base.py +++ b/src/serena/tools/tools_base.py @@ -73,22 +73,40 @@ class Component(ABC): TOOL_DEFAULT_MAX_ANSWER_LENGTH = int(2e5) -class ToolMarkerCanEdit: +class ToolMarker: + """ + Base class for tool markers. + """ + + +class ToolMarkerCanEdit(ToolMarker): """ Marker class for all tools that can perform editing operations on files. """ -class ToolMarkerDoesNotRequireActiveProject: +class ToolMarkerDoesNotRequireActiveProject(ToolMarker): pass -class ToolMarkerOptional: +class ToolMarkerOptional(ToolMarker): """ Marker class for optional tools that are disabled by default. """ +class ToolMarkerSymbolicRead(ToolMarker): + """ + Marker class for tools that perform symbol read operations. + """ + + +class ToolMarkerSymbolicEdit(ToolMarkerCanEdit): + """ + Marker class for tools that perform symbolic edit operations. + """ + + class ApplyMethodProtocol(Protocol): """Callable protocol for the apply method of a tool."""