Allow system prompt to depend on available tools and tool categories (as indicated by ToolMarkers)

* Introduce additional tool markers that represent relevant categories
* Apply some conditions to current system prompt

src/serena/agent.py

@@ -25,7 +25,8 @@ from serena.config.serena_config import SerenaConfig, ToolInclusionDefinition, T
 from serena.dashboard import SerenaDashboardAPI
 from serena.project import Project
 from serena.prompt_factory import SerenaPromptFactory
-from serena.tools import ActivateProjectTool, Tool, ToolRegistry
+from serena.tools import ActivateProjectTool, Tool, ToolMarker, ToolRegistry
+from serena.util.inspection import iter_subclasses
 from serena.util.logging import MemoryLogHandler
 from solidlsp import SolidLanguageServer
 
@@ -336,10 +337,23 @@ class SerenaAgent:
         return list(self._modes)
 
     def create_system_prompt(self) -> str:
-        return self.prompt_factory.create_system_prompt(
+        available_tools = [tool.get_name_from_cls() for tool in self.get_exposed_tool_instances()]
+
+        # determine available tool markers (i.e. tool categories)
+        available_markers = set()
+        for marker_class in iter_subclasses(ToolMarker):
+            for tool in self.get_exposed_tool_instances():
+                if isinstance(tool, marker_class):
+                    available_markers.add(marker_class.__name__)
+
+        log.info("Generating system prompt with available_tools=(see exposed tools), available_markers=%s", available_markers)
+        system_prompt = self.prompt_factory.create_system_prompt(
             context_system_prompt=self._context.prompt,
             mode_system_prompts=[mode.prompt for mode in self._modes],
+            available_tools=available_tools,
+            available_markers=available_markers,
         )
+        log.info("System prompt:\n%s", system_prompt)
 
     def _update_active_tools(self) -> None:
         """

src/serena/generated/generated_prompt_factory.py

@@ -32,5 +32,7 @@ class PromptFactory(PromptFactoryBase):
     def create_prepare_for_new_conversation(self) -> str:
         return self._render_prompt("prepare_for_new_conversation", locals())
 
-    def create_system_prompt(self, *, context_system_prompt: Any, mode_system_prompts: Any) -> str:
+    def create_system_prompt(
+        self, *, available_markers: Any, available_tools: Any, context_system_prompt: Any, mode_system_prompts: Any
+    ) -> str:
         return self._render_prompt("system_prompt", locals())

src/serena/resources/config/prompt_templates/system_prompt.yml

@@ -5,32 +5,38 @@ prompts:
   system_prompt: |
     You are a professional coding agent concerned with one particular codebase. You have 
     access to semantic coding tools on which you rely heavily for all your work, as well as collection of memory 
-    files containing general information about the codebase. You operate in a frugal and intelligent manner, always
+    files containing general information about the codebase. You operate in a resource-efficient and intelligent manner, always
     keeping in mind to not read or generate content that is not needed for the task at hand.
 
     When reading code in order to answer a user question or task, you should try reading only the necessary code. 
     Some tasks may require you to understand the architecture of large parts of the codebase, while for others,
     it may be enough to read a small set of symbols or a single file.
     Generally, you should avoid reading entire files unless it is absolutely necessary, instead relying on
-    intelligent step-by-step acquisition of information. However, if you already read a file, it does not make
+    intelligent step-by-step acquisition of information. {% if 'ToolMarkerSymbolicRead' in available_markers %}However, if you already read a file, it does not make
     sense to further analyse it with the symbolic tools (except for the `find_referencing_symbols` tool), 
-    as you already have the information.
+    as you already have the information.{% endif %}
 
-    I WILL BE SERIOUSLY UPSET IF YOU READ ENTIRE FILES WITHOUT NEED! CONSIDER INSTEAD USING THE OVERVIEW TOOL AND
-    SYMBOLIC TOOLS TO READ ONLY THE NECESSARY CODE FIRST!
+    I WILL BE SERIOUSLY UPSET IF YOU READ ENTIRE FILES WITHOUT NEED!
+    {% if 'ToolMarkerSymbolicRead' in available_markers %}
+    CONSIDER INSTEAD USING THE OVERVIEW TOOL AND SYMBOLIC TOOLS TO READ ONLY THE NECESSARY CODE FIRST!
     I WILL BE EVEN MORE UPSET IF AFTER HAVING READ AN ENTIRE FILE YOU KEEP READING THE SAME CONTENT WITH THE SYMBOLIC TOOLS!
     THE PURPOSE OF THE SYMBOLIC TOOLS IS TO HAVE TO READ LESS CODE, NOT READ THE SAME CONTENT MULTIPLE TIMES!
+    {% endif %}
 
     You can achieve the intelligent reading of code by using the symbolic tools for getting an overview of symbols and
     the relations between them, and then only reading the bodies of symbols that are necessary to answer the question 
-    or complete the task. You can also use the standard tools like list_dir, find_file and search_for_pattern if you need to.
+    or complete the task. 
+    You can use the standard tools like list_dir, find_file and search_for_pattern if you need to.
     When tools allow it, you pass the `relative_path` parameter to restrict the search to a specific file or directory.
     For some tools, `relative_path` can only be a file path, so make sure to properly read the tool descriptions.
-    If you are unsure about a symbol's name or location (to the extent that substring_matching for the symbol name is not enough), you can use the `search_for_pattern` tool, which allows fast
-    and flexible search for patterns in the codebase. This way you can first find candidates for symbols or files,
-    and then proceed with the symbolic tools.
+    {% if 'search_for_pattern' in available_tools %}
+    If you are unsure about a symbol's name or location{% if 'find_symbol' in available_tools %} (to the extent that substring_matching for the symbol name is not enough){% endif %}, you can use the `search_for_pattern` tool, which allows fast
+    and flexible search for patterns in the codebase.{% if 'ToolMarkerSymbolicRead' in available_markers %}This way you can first find candidates for symbols or files,
+    and then proceed with the symbolic tools.{% endif %}
+    {% endif %}
 
-    Symbols are identified by their `name_path and `relative_path`, see the description of the `find_symbols` tool for more details
+    {% if 'ToolMarkerSymbolicRead' in available_markers %}
+    Symbols are identified by their `name_path and `relative_path`, see the description of the `find_symbol` tool for more details
     on how the `name_path` matches symbols.
     You can get information about available symbols by using the `get_symbols_overview` tool for finding top-level symbols in a file
     or directory, or by using `find_symbol` if you already know the symbol's name path. You generally try to read as little code as possible
@@ -40,10 +46,13 @@ prompts:
     you can use `find_symbol` with the name path `Foo`, `include_body=False` and `depth=1` to get all (top-level) methods of `Foo` before proceeding
     to read the desired methods with `include_body=True`
     You can understand relationships between symbols by using the `find_referencing_symbols` tool.
+    {% endif %}
 
+    {% if 'read_memory' in available_tools %}
     You generally have access to memories and it may be useful for you to read them, but also only if they help you
     to answer the question or complete the task. You can infer which memories are relevant to the current task by reading
     the memory names and descriptions.
+    {% endif %}
 
     The context and modes of operation are described below. From them you can infer how to interact with your user
     and which tasks and kinds of interactions are expected of you.

src/serena/tools/jetbrains_tools.py

@@ -1,10 +1,10 @@
 import json
 
-from serena.tools import TOOL_DEFAULT_MAX_ANSWER_LENGTH, Tool, ToolMarkerOptional
+from serena.tools import TOOL_DEFAULT_MAX_ANSWER_LENGTH, Tool, ToolMarkerOptional, ToolMarkerSymbolicRead
 from serena.tools.jetbrains_plugin_client import JetBrainsPluginClient
 
 
-class JetBrainsFindSymbolTool(Tool, ToolMarkerOptional):
+class JetBrainsFindSymbolTool(Tool, ToolMarkerSymbolicRead, ToolMarkerOptional):
     """
     Performs a global (or local) search for symbols with/containing a given name/substring (optionally filtered by type).
     """
@@ -70,7 +70,7 @@ class JetBrainsFindSymbolTool(Tool, ToolMarkerOptional):
         return self._limit_length(result, max_answer_chars)
 
 
-class JetBrainsFindReferencingSymbolsTool(Tool, ToolMarkerOptional):
+class JetBrainsFindReferencingSymbolsTool(Tool, ToolMarkerSymbolicRead, ToolMarkerOptional):
     """
     Finds symbols that reference the given symbol
     """
@@ -100,7 +100,7 @@ class JetBrainsFindReferencingSymbolsTool(Tool, ToolMarkerOptional):
         return self._limit_length(result, max_answer_chars)
 
 
-class JetBrainsGetSymbolsOverviewTool(Tool, ToolMarkerOptional):
+class JetBrainsGetSymbolsOverviewTool(Tool, ToolMarkerSymbolicRead, ToolMarkerOptional):
     """
     Retrieves an overview of the top-level symbols within a specified file
     """

src/serena/tools/symbol_tools.py

@@ -8,7 +8,13 @@ from collections.abc import Sequence
 from copy import copy
 from typing import Any
 
-from serena.tools import SUCCESS_RESULT, TOOL_DEFAULT_MAX_ANSWER_LENGTH, Tool, ToolMarkerCanEdit
+from serena.tools import (
+    SUCCESS_RESULT,
+    TOOL_DEFAULT_MAX_ANSWER_LENGTH,
+    Tool,
+    ToolMarkerSymbolicEdit,
+    ToolMarkerSymbolicRead,
+)
 from solidlsp.ls_types import SymbolKind
 
 
@@ -42,7 +48,7 @@ class RestartLanguageServerTool(Tool):
         return SUCCESS_RESULT
 
 
-class GetSymbolsOverviewTool(Tool):
+class GetSymbolsOverviewTool(Tool, ToolMarkerSymbolicRead):
     """
     Gets an overview of the top-level symbols defined in a given file or directory.
     """
@@ -70,7 +76,7 @@ class GetSymbolsOverviewTool(Tool):
         return self._limit_length(result_json_str, max_answer_chars)
 
 
-class FindSymbolTool(Tool):
+class FindSymbolTool(Tool, ToolMarkerSymbolicRead):
     """
     Performs a global (or local) search for symbols with/containing a given name/substring (optionally filtered by type).
     """
@@ -150,7 +156,7 @@ class FindSymbolTool(Tool):
         return self._limit_length(result, max_answer_chars)
 
 
-class FindReferencingSymbolsTool(Tool):
+class FindReferencingSymbolsTool(Tool, ToolMarkerSymbolicRead):
     """
     Finds symbols that reference the symbol at the given location (optionally filtered by type).
     """
@@ -204,7 +210,7 @@ class FindReferencingSymbolsTool(Tool):
         return self._limit_length(result, max_answer_chars)
 
 
-class ReplaceSymbolBodyTool(Tool, ToolMarkerCanEdit):
+class ReplaceSymbolBodyTool(Tool, ToolMarkerSymbolicEdit):
     """
     Replaces the full definition of a symbol.
     """
@@ -232,7 +238,7 @@ class ReplaceSymbolBodyTool(Tool, ToolMarkerCanEdit):
         return SUCCESS_RESULT
 
 
-class InsertAfterSymbolTool(Tool, ToolMarkerCanEdit):
+class InsertAfterSymbolTool(Tool, ToolMarkerSymbolicEdit):
     """
     Inserts content after the end of the definition of a given symbol.
     """
@@ -257,7 +263,7 @@ class InsertAfterSymbolTool(Tool, ToolMarkerCanEdit):
         return SUCCESS_RESULT
 
 
-class InsertBeforeSymbolTool(Tool, ToolMarkerCanEdit):
+class InsertBeforeSymbolTool(Tool, ToolMarkerSymbolicEdit):
     """
     Inserts content before the beginning of the definition of a given symbol.
     """

src/serena/tools/tools_base.py

@@ -73,22 +73,40 @@ class Component(ABC):
 TOOL_DEFAULT_MAX_ANSWER_LENGTH = int(2e5)
 
 
-class ToolMarkerCanEdit:
+class ToolMarker:
+    """
+    Base class for tool markers.
+    """
+
+
+class ToolMarkerCanEdit(ToolMarker):
     """
     Marker class for all tools that can perform editing operations on files.
     """
 
 
-class ToolMarkerDoesNotRequireActiveProject:
+class ToolMarkerDoesNotRequireActiveProject(ToolMarker):
     pass
 
 
-class ToolMarkerOptional:
+class ToolMarkerOptional(ToolMarker):
     """
     Marker class for optional tools that are disabled by default.
     """
 
 
+class ToolMarkerSymbolicRead(ToolMarker):
+    """
+    Marker class for tools that perform symbol read operations.
+    """
+
+
+class ToolMarkerSymbolicEdit(ToolMarkerCanEdit):
+    """
+    Marker class for tools that perform symbolic edit operations.
+    """
+
+
 class ApplyMethodProtocol(Protocol):
     """Callable protocol for the apply method of a tool."""
 

src/solidlsp/language_servers/solargraph.py

@@ -7,7 +7,6 @@ import json
 import logging
 import os
 import pathlib
-import stat
 import subprocess
 import threading